Commit e03e1c63 authored by Daniele Kruse's avatar Daniele Kruse
Browse files

First commit of AULFile structure

parent cb1662e2
......@@ -39,6 +39,7 @@
namespace castor {
namespace tape {
namespace SCSI {
const unsigned int defaultTimeout=900000; //millisecs
/**
* Structures as defined in the SCSI specifications, and helper functions for them.
* SPC-4 (SCSI primary commands) can be found at:
......@@ -69,7 +70,7 @@ namespace SCSI {
*/
class LinuxSGIO_t: public sg_io_hdr_t {
public:
LinuxSGIO_t() { zeroStruct(this); interface_id = 'S'; timeout = 30000; }
LinuxSGIO_t() { zeroStruct(this); interface_id = 'S'; timeout = defaultTimeout; }
template <typename T>
void setCDB(T * cdb) { cmdp = (unsigned char *)cdb; cmd_len = sizeof(T); }
......
......@@ -198,7 +198,7 @@ namespace UnitTests {
to the original structure (virtual table, for example)*/
sg_io_hdr_t & sgio_hdr = *(sg_io_hdr_t *)&lsg;
/* Also make sure the constructor does its initialization job */
ASSERT_EQ(30000U, sgio_hdr.timeout);
ASSERT_EQ(900000U, sgio_hdr.timeout);
ASSERT_EQ('S', sgio_hdr.interface_id);
/* The rest is safe. It's just a struct with added functions */
}
......
......@@ -148,7 +148,7 @@ throw (Exception) {
sgh.setCDB(&cdb);
sgh.setSenseBuffer(&senseBuff);
sgh.dxfer_direction = SG_DXFER_NONE;
sgh.timeout = 180000; // TODO castor.conf LOCATE_TIMEOUT or default
//sgh.timeout = defaultTimeout; // set globally by SCSI::Structures.hpp (defaultTimeout)
/* Manage both system error and SCSI errors. */
castor::exception::Errnum::throwOnMinusOne(
......@@ -322,7 +322,7 @@ void drives::DriveGeneric::setSTBufferWrite(bool bufWrite) throw (Exception) {
struct mtop m_mtCmd;
m_mtCmd.mt_op = MTSETDRVBUFFER;
m_mtCmd.mt_count = bufWrite ? (MT_ST_SETBOOLEANS | MT_ST_BUFFER_WRITES) : (MT_ST_CLEARBOOLEANS | MT_ST_BUFFER_WRITES);
castor::exception::Errnum::throwOnMinusOne(
castor::exception::Errnum::throwOnMinusOne(
m_sysWrapper.ioctl(m_tapeFD, MTIOCTOP, &m_mtCmd),
"Failed ST ioctl (MTSETDRVBUFFER) in DriveGeneric::setSTBufferWrite");
}
......@@ -340,7 +340,7 @@ void drives::DriveGeneric::spaceToEOM(void) throw (Exception) {
struct mtop m_mtCmd;
m_mtCmd.mt_op = MTEOM;
m_mtCmd.mt_count = 1;
castor::exception::Errnum::throwOnMinusOne(
castor::exception::Errnum::throwOnMinusOne(
m_sysWrapper.ioctl(m_tapeFD, MTIOCTOP, &m_mtCmd),
"Failed ST ioctl (MTEOM) in DriveGeneric::spaceToEOM");
}
......@@ -355,7 +355,7 @@ void drives::DriveGeneric::setSTFastMTEOM(bool fastMTEOM) throw (Exception) {
struct mtop m_mtCmd;
m_mtCmd.mt_op = MTSETDRVBUFFER;
m_mtCmd.mt_count = fastMTEOM ? (MT_ST_SETBOOLEANS | MT_ST_FAST_MTEOM) : (MT_ST_CLEARBOOLEANS | MT_ST_FAST_MTEOM);
castor::exception::Errnum::throwOnMinusOne(
castor::exception::Errnum::throwOnMinusOne(
m_sysWrapper.ioctl(m_tapeFD, MTIOCTOP, &m_mtCmd),
"Failed ST ioctl (MTSETDRVBUFFER) in DriveGeneric::setSTFastMTEOM");
}
......@@ -502,7 +502,7 @@ void drives::DriveGeneric::writeImmediateFileMarks(size_t count) throw (Exceptio
* @param data pointer the the data block
* @param count size of the data block
*/
void drives::DriveGeneric::writeBlock(void * data, size_t count) throw (Exception) {
void drives::DriveGeneric::writeBlock(const void * data, size_t count) throw (Exception) {
castor::exception::Errnum::throwOnMinusOne(
m_sysWrapper.write(m_tapeFD, data, count),
"Failed ST write in DriveGeneric::writeBlock");
......
......@@ -334,7 +334,7 @@ namespace drives {
* @param data pointer the the data block
* @param count size of the data block
*/
virtual void writeBlock(void * data, size_t count) throw (Exception);
virtual void writeBlock(const void * data, size_t count) throw (Exception);
/**
* Read a data block from tape.
......@@ -359,7 +359,7 @@ namespace drives {
* @return the actual size of read data
*/
virtual void readFileMark(std::string context= "") throw (Exception);
virtual ~DriveGeneric() {
if (-1 != m_tapeFD)
m_sysWrapper.close(m_tapeFD);
......
......@@ -100,8 +100,8 @@ int main ()
<< "----------------------------------------------" << std::endl;
drive.rewind();
/* For some unexplained (TODO) reason, mhvtl does not accept blocks smaller than 4 bytes */
drive.writeBlock((unsigned char *)"X123", 4);
drive.writeBlock((unsigned char *)"Y123", 4);
drive.writeBlock((void *)"X123", 4);
drive.writeBlock((void *)"Y123", 4);
/**
* trying to do position to the block 2.
*/
......@@ -195,7 +195,7 @@ int main ()
memset(data, 'a', count-1);
std::cout << "Writing 1st block (9 a's)..." << std::endl;
drive.writeBlock(data, count); // write 9 a's + string term
drive.writeBlock((void *)data, count); // write 9 a's + string term
print_and_assert_position(drive, 1);
std::cout << "Writing 1st Synchronous filemark..." << std::endl;
......@@ -204,7 +204,7 @@ int main ()
memset(data, 'b', count-1);
std::cout << "Writing 2nd block (9 b's)..." << std::endl;
drive.writeBlock(data, count); // write 9 b's + string term
drive.writeBlock((void *)data, count); // write 9 b's + string term
print_and_assert_position(drive, 3);
std::cout << "Writing 2nd Synchronous filemark..." << std::endl;
......@@ -213,7 +213,7 @@ int main ()
memset(data, 'c', count-1);
std::cout << "Writing 3rd block (9 c's)..." << std::endl;
drive.writeBlock(data, count); // write 9 c's + string term
drive.writeBlock((void *)data, count); // write 9 c's + string term
print_and_assert_position(drive, 5);
std::cout << "Writing EOD (2 filemarks)..." << std::endl;
......@@ -226,37 +226,37 @@ int main ()
std::cout << "Reading back 1st block 9 a's)..." << std::endl;
memset(data, 0, count);
drive.readBlock(data, count); // read 9 a's + string term
drive.readBlock((void *)data, count); // read 9 a's + string term
print_and_assert_position(drive, 1);
print_and_assert_data("aaaaaaaaa", (const char *)data);
std::cout << "Skipping first file mark..." << std::endl;
memset(data, 0, count);
drive.readBlock(data, count);
drive.readBlock((void *)data, count);
print_and_assert_position(drive, 2);
std::cout << "Reading back 2nd block (9 b's)..." << std::endl;
memset(data, 0, count);
drive.readBlock(data, count); // read 9 b's + string term
drive.readBlock((void *)data, count); // read 9 b's + string term
print_and_assert_position(drive, 3);
print_and_assert_data("bbbbbbbbb", (const char *)data);
std::cout << "Skipping first file mark..." << std::endl;
memset(data, 0, count);
drive.readBlock(data, count);
drive.readBlock((void *)data, count);
print_and_assert_position(drive, 4);
std::cout << "Reading back 3rd block (9 c's)..." << std::endl;
memset(data, 0, count);
drive.readBlock(data, count); // read 9 c's + string term
drive.readBlock((void *)data, count); // read 9 c's + string term
print_and_assert_position(drive, 5);
print_and_assert_data("ccccccccc", (const char *)data);
std::cout << "Skipping the last two file marks..." << std::endl;
memset(data, 0, count);
drive.readBlock(data, count);
drive.readBlock((void *)data, count);
memset(data, 0, count);
drive.readBlock(data, count);
drive.readBlock((void *)data, count);
print_and_assert_position(drive, 7);
std::cout << "Rewinding..." << std::endl;
......@@ -297,7 +297,7 @@ int main ()
memset(data, 'd', count-1);
std::cout << "Writing 9 d's..." << std::endl;
drive.writeBlock(data, count); // write 9 d's + string term
drive.writeBlock((void *)data, count); // write 9 d's + string term
print_and_assert_position(drive, 7);
std::cout << "Writing Asynchronous filemark..." << std::endl;
......@@ -306,7 +306,7 @@ int main ()
memset(data, 'e', count-1);
std::cout << "Writing 9 e's..." << std::endl;
drive.writeBlock(data, count); // write 9 e's + string term
drive.writeBlock((void *)data, count); // write 9 e's + string term
print_and_assert_position(drive, 9);
std::cout << "Writing Asynchronous EOD..." << std::endl;
......@@ -324,7 +324,7 @@ int main ()
for(int i=0; i<9; i++) {
memset(data, '0'+i, count-1);
std::cout << "Writing 9 " << i << "'s..." << std::endl;
drive.writeBlock(data, count);
drive.writeBlock((void *)data, count);
print_and_assert_position(drive, i+1);
}
......@@ -338,7 +338,7 @@ int main ()
std::cout << "Reading..." << std::endl;
memset(data, 0, count);
drive.readBlock(data, count);
drive.readBlock((void *)data, count);
print_and_assert_position(drive, 3);
print_and_assert_data("222222222", (const char *)data);
......@@ -356,7 +356,7 @@ int main ()
std::cout << "Reading..." << std::endl;
memset(data, 0, count);
drive.readBlock(data, count);
drive.readBlock((void *)data, count);
print_and_assert_position(drive, 5);
print_and_assert_data("444444444", (const char *)data);
......
add_library(File File.cpp Structures.cpp)
add_executable(castor-tape-AULFileFormatTest FileFormatTest.cpp)
target_link_libraries(castor-tape-AULFileFormatTest File TapeDrive Exception SCSI System Utils
castorcommon castorclient ${GTEST_LIBRARY} gmock pthread)
install(TARGETS castor-tape-AULFileFormatTest
RUNTIME DESTINATION bin)
/******************************************************************************
* Drive.hpp
* File.cpp
*
* This file is part of the Castor project.
* See http://castor.web.cern.ch/castor
......@@ -24,6 +24,10 @@
#include "File.hpp"
#include "castor/exception/Errnum.hpp"
#include "castor/exception/Mismatch.hpp"
#include "castor/exception/InvalidArgument.hpp"
#include <sstream>
#include <iomanip>
using namespace castor::tape::AULFile;
/**
......@@ -32,10 +36,44 @@ using namespace castor::tape::AULFile;
* @param drive
* @param VSN
*/
ReadSession::ReadSession(drives::Drive & drive, std::string VSN) throw (Exception) {
castor::tape::drives::DriveGeneric & dg = drive;
ReadSession::ReadSession(drives::DriveGeneric & drive, std::string volId) : dg(drive), VSN(volId), current_block_size(0) throw (Exception) {
checkVOL1(); //after which we are at the end of VOL1 header (i.e. beginning of HDR1 of the first file) on success, or at BOT in case of exception
}
/**
* checks the volume label to make sure the label is valid and that we
* have the correct tape (checks VSN). Leaves the tape at the end of the
* first header block (i.e. right before the first data block) in case
* of success, or rewinds the tape in case of volume label problems.
* Might also leave the tape in unknown state in case any of the st
* operations fail.
*/
void ReadSession::checkVOL1() throw (Exception) {
dg.rewind();
if(dg.getPositionInfo().currentPosition!=0) {
std::stringstream ex_str;
ex_str << "Cannot rewind volume " << VSN;
TapeMediaError(ex_str.str());
}
VOL1 vol1;
ssize_t res = dg.readBlock((void * )&vol1, sizeof(vol1));
if(res!=sizeof(vol1)) {
dg.rewind();
std::stringstream ex_str;
ex_str << "Volume " << VSN << " label has invalid size: " << res << " instead of " << sizeof(vol1);
throw TapeFormatError(ex_str.str());
}
try {
vol1.verify();
} catch (std::exception & e) {
throw TapeFormatError(e.what());
}
if(vol1.getVSN().compare(VSN)) {
dg.rewind();
std::stringstream ex_str;
ex_str << "VSN of tape (" << vol1.getVSN() << ") is not the one requested (" << VSN << ")";
throw TapeFormatError(ex_str.str());
}
}
/**
......@@ -46,18 +84,95 @@ ReadSession::ReadSession(drives::Drive & drive, std::string VSN) throw (Exceptio
* @param fileInfo: all relevant information passed by the stager about
* the file.
*/
void ReadSession::position(Information fileInfo) throw (Exception) {
void ReadSession::position(const Information &fileInfo) throw (Exception) {
if(fileInfo.checksum==0 or fileInfo.nsFileId==0 or fileInfo.size==0 or fileInfo.fseq<1) {
throw castor::exception::InvalidArgument();
}
uint32_t destination_block = fileInfo.blockId ? fileInfo.blockId : 0; //if we want the first file on tape (fileInfo.blockId==0) we need to skip the VOL1 header
//we position using the sg locate because it is supposed to do the right thing possibly in a more optimized way (better than st's spaceBlocksForward/Backwards)
dg.positionToLogicalObject(destination_block);
//at this point we should be at the beginning of the headers of the desired file, so now let's check the headers...
HDR1 hdr1;
HDR2 hdr2;
UHL1 uhl1;
ssize_t res = dg.readBlock((void *)&hdr1, sizeof(hdr1)); //TODO throws exception
if(res!=sizeof(hdr1)) {
std::stringstream ex_str;
ex_str << "Invalid hdr1 header detected at block " << fileInfo.blockId << ". The header has invalid size: " << res << " instead of " << sizeof(hdr1);
throw TapeFormatError(ex_str.str());
}
res = dg.readBlock((void *)&hdr2, sizeof(hdr2));
if(res!=sizeof(hdr2)) {
std::stringstream ex_str;
ex_str << "Invalid hdr2 header detected at block " << fileInfo.blockId << ". The header has invalid size: " << res << " instead of " << sizeof(hdr2);
throw TapeFormatError(ex_str.str());
}
res = dg.readBlock((void *)&uhl1, sizeof(uhl1));
if(res!=sizeof(uhl1)) {
std::stringstream ex_str;
ex_str << "Invalid uhl1 header detected at block " << fileInfo.blockId << ". The header has invalid size: " << res << " instead of " << sizeof(uhl1);
throw TapeFormatError(ex_str.str());
}
char empty[4];
dg.readBlock(empty, 4); //TODO to put inside a function
res = dg.readBlock(empty, 4);//after this we should be where we want, i.e. at the beginning of the file
if(res!=0) {
std::stringstream ex_str;
ex_str << "Tape mark not found after uhl1";
throw TapeFormatError(ex_str.str());
}
//the size of the headers is fine, now let's check each header
try {
hdr1.verify();
hdr2.verify();
uhl1.verify();
} catch (std::exception & e) {
throw TapeFormatError(e.what());
}
//headers are valid here, let's see if they contain the right info, i.e. are we in the correct place?
//the fileid stored in HDR1 is in hex while the one supplied is in DEC
std::stringstream nsFileId_converter(hdr1.getFileId().c_str());
uint64_t nsFileId = 0;
nsFileId_converter >> std::hex >> nsFileId;
if(nsFileId!=fileInfo.nsFileId) {
std::stringstream ex_str;
ex_str << "Invalid fileid. Detected: " << nsFileId << ". Wanted: " << fileInfo.nsFileId << std::endl;
ex_str << "hdr1 dump: " << (char *)&hdr1 << std::endl;
throw TapeFormatError(ex_str.str());
}
//the following should never ever happen... but never say never...
if(hdr1.getVSN().compare(VSN)) {
std::stringstream ex_str;
ex_str << "Wrong volume ID info found in header. Detected: " << hdr1.getVSN() << ". Wanted: " << VSN;
throw TapeFormatError(ex_str.str());
}
//we disregard hdr2 on purpose as it contains no useful information, we now check that also uhl1 (hdr1 also contains fseq info but it is modulo 10000, therefore useless)
if((uint32_t)atol(uhl1.getfSeq().c_str())!=fileInfo.fseq) {
std::stringstream ex_str;
ex_str << "Invalid fseq in uhl1. Detected: " << atol(uhl1.getfSeq().c_str()) << ". Wanted: " << fileInfo.fseq;
throw TapeFormatError(ex_str.str());
}
//now that we are all happy with the information contained within the headers we finally get the block size for our file (provided it has a reasonable value)
current_block_size = (size_t)atol(uhl1.getBlockSize().c_str());
if(current_block_size<1) {
std::stringstream ex_str;
ex_str << "Invalid block size in uhl1 detected: " << current_block_size;
throw TapeFormatError(ex_str.str());
}
}
/**
* After positioning at the beginning of a file for readings, this function
* allows the reader to know which block sizes to provide.
* If called before the end of a file read, the file reading will be
* interrupted and positioning to the new file will occur.
* @return the block size in bytes.
*/
size_t ReadSession::getBlockSize() throw (Exception) {
return 0;
if(current_block_size<1) {
std::stringstream ex_str;
ex_str << "Invalid block size: " << current_block_size;
throw TapeFormatError(ex_str.str());
}
return current_block_size;
}
/**
......@@ -72,6 +187,6 @@ size_t ReadSession::getBlockSize() throw (Exception) {
* @param len size of the buffer
* @return The amount of data actually copied. Zero at end of file.
*/
size_t ReadSession::read(void * buff, size_t len) throw (Exception) {
return 0;
size_t ReadSession::read(void * buff, size_t len) throw (Exception) {
return dg.readBlock(buff, len);
}
......@@ -37,13 +37,13 @@ namespace castor {
* Class containing all the information related to a file being migrated to
* tape.
*/
class Information {
class Information { //no information about path and filename here as it cannot be used nor checked on tape
public:
std::string lastKnownPath;
uint32_t checksum;
uint64_t nsFileId;
uint64_t size;
uint32_t fseq;
uint32_t fseq; //this is the payload (i.e. real file) sequence number, not the tape file sequence number (which would include headers and trailers as well)
uint32_t blockId;
};
class BufferTooSmall: public Exception {
......@@ -65,6 +65,16 @@ namespace castor {
public:
NotReadingAFile(const std::string & what): Exception(what) {}
};
class TapeFormatError: public Exception {
public:
TapeFormatError(const std::string & what): Exception(what) {}
};
class TapeMediaError: public Exception {
public:
TapeMediaError(const std::string & what): Exception(what) {}
};
/**
* Class keeping track of a whole tape read session over an AUL formated
......@@ -81,7 +91,8 @@ namespace castor {
* @param drive
* @param VSN
*/
ReadSession(drives::Drive & drive, std::string VSN) throw (Exception);
ReadSession(drives::DriveGeneric & dg, std::string volId) throw (Exception);
/**
* Positions the tape for reading the file. Depending on the previous activity,
* it is the duty of this function to determine how to best move to the next
......@@ -90,15 +101,15 @@ namespace castor {
* @param fileInfo: all relevant information passed by the stager about
* the file.
*/
void position(Information fileInfo) throw (Exception);
void position(const Information &fileInfo) throw (Exception);
/**
* After positioning at the beginning of a file for readings, this function
* allows the reader to know which block sizes to provide.
* If called before the end of a file read, the file reading will be
* interrupted and positioning to the new file will occur.
* @return the block size in bytes.
*/
size_t getBlockSize() throw (Exception);
/**
* Read data from the file. The buffer should equal to or bigger than the
* block size. Will try to actually fill up the provided buffer (this
......@@ -112,6 +123,24 @@ namespace castor {
* @return The amount of data actually copied. Zero at end of file.
*/
size_t read(void * buff, size_t len) throw (Exception);
private:
/**
* checks the volume label to make sure the label is valid and that we
* have the correct tape (checks VSN). Leaves the tape at the end of the
* first header block (i.e. right before the first data block) in case
* of success, or rewinds the tape in case of volume label problems.
* Might also leave the tape in unknown state in case any of the st
* operations fail.
*/
void checkVOL1() throw (Exception);
/**
* DriveGeneric object referencing the drive used during this read session
*/
drives::DriveGeneric & dg;
std::string VSN;
size_t current_block_size;
};
/**
......
/******************************************************************************
* FileFormatTest.cpp
*
* This file is part of the Castor project.
* See http://castor.web.cern.ch/castor
*
* Copyright (C) 2003 CERN
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
*
*
* @author Castor Dev team, castor-dev@cern.ch
*****************************************************************************/
/**
* Test main program. For development use.
*/
#include "../system/Wrapper.hpp"
#include "../SCSI/Device.hpp"
#include "File.hpp"
#include "../drive/Drive.hpp"
#include <iostream>
#include <assert.h>
#include <fstream>
#include <sstream>
#include <iomanip>
int main(int argc, char* argv[])
{
if (argc != 3) {
std::cerr << "Usage: " << argv[0] << " <device(ex. /dev/nst0)> <VSN(ex. V92003)>" << std::endl;
return 1;
}
int fail = 0;
castor::tape::System::realWrapper sWrapper;
castor::tape::SCSI::DeviceVector dl(sWrapper);
for(castor::tape::SCSI::DeviceVector::iterator i = dl.begin(); i != dl.end(); i++) {
castor::tape::SCSI::DeviceInfo & dev = (*i);
std::cout << std::endl << "-- SCSI device: " << dev.sg_dev << " (" << dev.nst_dev << ")" << std::endl;
if (dev.type == castor::tape::SCSI::Types::tape) {
castor::tape::drives::Drive dContainer(dev, sWrapper);
castor::tape::drives::DriveGeneric & drive = dContainer;
castor::tape::drives::deviceInfo devInfo;
try {
devInfo = drive.getDeviceInfo();
std::cout << "-- INFO --------------------------------------" << std::endl
<< " devInfo.vendor : '" << devInfo.vendor << "'" << std::endl
<< " devInfo.product : '" << devInfo.product << "'" << std::endl
<< " devInfo.productRevisionLevel : '" << devInfo.productRevisionLevel << "'" << std::endl
<< " devInfo.serialNumber : '" << devInfo.serialNumber << "'" << std::endl
<< "----------------------------------------------" << std::endl;
} catch (std::exception & e) {
fail = 1;
std::string temp = e.what();
std::cout << "----------------------------------------------" << std::endl
<< temp
<< "----------------------------------------------" << std::endl;
}
/**
* File Format Check Test
*/
if(!strcmp(dev.nst_dev.c_str(),argv[1])) {
try {
castor::tape::AULFile::ReadSession my_sess(drive, argv[2]);
std::cout << "Read session on " << argv[2] << " (" << argv[1] << ") established." << std::endl;
int f=0;
while(!f){
std::cout << "******* Menu ********\n\n";
std::cout << "1) Read file and dump output\n";
std::cout << "2) Exit\n";
std::cout << "Enter your choice (1 or 2): ";
int choice;
std::cin >> choice;
std::cin.ignore();
if(choice==1) {
castor::tape::AULFile::Information info;
std::cout << "Please enter the blockId: ";
std::string blockId;
std::cin >> blockId;
std::cin.ignore();
std::stringstream blockId_converter(blockId.c_str());
blockId_converter >> std::hex >> info.blockId;
std::cout << "Please enter the fseq number of the file: ";
std::cin >> info.fseq;
std::cin.ignore();
std::cout << "Please enter the checksum of the file: ";
std::string checksum;
std::cin >> checksum;
std::cin.ignore();
std::stringstream checksum_converter(checksum.c_str());
checksum_converter >> std::hex >> info.checksum;
std::cout << "Please enter the nsFileId of the file: ";
std::cin >> info.nsFileId;
std::cin.ignore();
std::cout << "Please enter the size of the file: ";