diff --git a/ReleaseNotes.md b/ReleaseNotes.md index aff24ff4ee70335cf4bd4811e4484a6324d36fab..479015b53d16322fb712e867b3f06645d922182c 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -5,6 +5,8 @@ ### Features - cta/CTA#1001 Maximum file size is now defined by VO instead of globally. +- cta/CTA#1019 New command `cta-readtp` allows reading files from tape and verifying their checksum + # v4.1-1 ## Summary diff --git a/catalogue/CatalogueTest.cpp b/catalogue/CatalogueTest.cpp index d49f7acdd391ac4b0a3b3385b7700298b8850914..6a7eb664704813bec24a86ffbf5337fba0052231 100644 --- a/catalogue/CatalogueTest.cpp +++ b/catalogue/CatalogueTest.cpp @@ -8832,6 +8832,17 @@ TEST_P(cta_catalogue_CatalogueTest, getArchiveFiles_non_existance_archiveFileId) ASSERT_THROW(m_catalogue->getArchiveFilesItor(searchCriteria), exception::UserError); } +TEST_P(cta_catalogue_CatalogueTest, getArchiveFiles_fSeq_without_vid) { + using namespace cta; + + ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore()); + + catalogue::TapeFileSearchCriteria searchCriteria; + searchCriteria.fSeq = 1234; + + ASSERT_THROW(m_catalogue->getArchiveFilesItor(searchCriteria), exception::UserError); +} + TEST_P(cta_catalogue_CatalogueTest, getArchiveFiles_disk_file_id_without_instance) { using namespace cta; diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index 143a7e1a97d43f7b76ec010eade077623841faa9..3068299e22a896100c127bcde68078dc25f35dd1 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -7057,6 +7057,10 @@ void RdbmsCatalogue::checkTapeFileSearchCriteria(rdbms::Conn &conn, const TapeFi throw exception::UserError(std::string("Disk file IDs are ambiguous without disk instance name")); } + if (searchCriteria.fSeq && !searchCriteria.vid) { + throw exception::UserError(std::string("fSeq makes no sense without vid")); + } + if(searchCriteria.vid) { if(!tapeExists(conn, searchCriteria.vid.value())) { throw exception::UserError(std::string("Tape ") + searchCriteria.vid.value() + " does not exist"); @@ -7072,8 +7076,8 @@ Catalogue::ArchiveFileItor RdbmsCatalogue::getArchiveFilesItor(const TapeFileSea checkTapeFileSearchCriteria(searchCriteria); // If this is the listing of the contents of a tape - if (!searchCriteria.archiveFileId && !searchCriteria.diskInstance && !searchCriteria.diskFileIds && - searchCriteria.vid) { + if (!searchCriteria.archiveFileId && !searchCriteria.diskInstance && !searchCriteria.diskFileIds && + !searchCriteria.fSeq && searchCriteria.vid) { return getTapeContentsItor(searchCriteria.vid.value()); } diff --git a/catalogue/RdbmsCatalogueGetArchiveFilesItor.cpp b/catalogue/RdbmsCatalogueGetArchiveFilesItor.cpp index 81c9ac8544cbd56c7698c1c9f649a5a8e8171e78..c7092b6e35cb6e58ab21e27f854710c58076431a 100644 --- a/catalogue/RdbmsCatalogueGetArchiveFilesItor.cpp +++ b/catalogue/RdbmsCatalogueGetArchiveFilesItor.cpp @@ -114,7 +114,8 @@ RdbmsCatalogueGetArchiveFilesItor::RdbmsCatalogueGetArchiveFilesItor( searchCriteria.archiveFileId || searchCriteria.diskInstance || searchCriteria.vid || - searchCriteria.diskFileIds; + searchCriteria.diskFileIds || + searchCriteria.fSeq; if(thereIsAtLeastOneSearchCriteria) { sql += " WHERE "; @@ -136,6 +137,11 @@ RdbmsCatalogueGetArchiveFilesItor::RdbmsCatalogueGetArchiveFilesItor( sql += "TAPE_FILE.VID = :VID"; addedAWhereConstraint = true; } + if (searchCriteria.fSeq) { + if(addedAWhereConstraint) sql += " AND "; + sql += "TAPE_FILE.FSEQ = :FSEQ"; + addedAWhereConstraint = true; + } if(searchCriteria.diskFileIds) { if(addedAWhereConstraint) sql += " AND "; sql += "ARCHIVE_FILE.DISK_FILE_ID IN (SELECT DISK_FILE_ID FROM " + tempDiskFxidsTableName + ")"; @@ -163,6 +169,11 @@ RdbmsCatalogueGetArchiveFilesItor::RdbmsCatalogueGetArchiveFilesItor( if(searchCriteria.vid) { m_stmt.bindString(":VID", searchCriteria.vid.value()); } + + if(searchCriteria.fSeq) { + m_stmt.bindUint64(":FSEQ", searchCriteria.fSeq.value()); + } + m_rset = m_stmt.executeQuery(); { log::LogContext lc(m_log); diff --git a/catalogue/TapeFileSearchCriteria.hpp b/catalogue/TapeFileSearchCriteria.hpp index df604f20d1fff7f9f1c94e94da2ab5679cd3d339..efd52ab41e7d1f5ea195a7dffb2f79757d3d9d29 100644 --- a/catalogue/TapeFileSearchCriteria.hpp +++ b/catalogue/TapeFileSearchCriteria.hpp @@ -47,6 +47,11 @@ struct TapeFileSearchCriteria { */ optional<std::string> vid; + /** + * The fSeq of the file on tape. + */ + optional<uint64_t> fSeq; + /** * List of disk file IDs. * diff --git a/cta.spec.in b/cta.spec.in index 6199934aa8e9240b568d527d02cd0041ffe602fe..fa403efb3cab4ee499ab5b74399b8eea4e526cb7 100644 --- a/cta.spec.in +++ b/cta.spec.in @@ -441,6 +441,22 @@ The command-line tool for pre-labelling a CTA tape. %post -n cta-tape-label /usr/sbin/setcap cap_sys_rawio+ep %{_bindir}/cta-tape-label +%package -n cta-readtp +Summary: The command-line tool for reading files from a CTA tape. +Group: Application/CTA +Requires: cta-lib = %{version}-%{release} +Requires: xrootd-client-libs >= %{xrootdVersion} +Requires(post): /usr/sbin/setcap +%description -n cta-readtp +CERN Tape Archive: +The command-line tool for reading files from a CTA tape. +%files -n cta-readtp +%defattr(-,root,root) +%attr(0750,cta,tape) %{_bindir}/cta-readtp +%attr(0644,root,root) %doc /usr/share/man/man1/cta-readtp.1cta.gz +%post -n cta-readtp +/usr/sbin/setcap cap_sys_rawio+ep %{_bindir}/cta-readtp + %package -n cta-common Summary: CERN Tape Archive common items Group: Application/CTA diff --git a/tapeserver/CMakeLists.txt b/tapeserver/CMakeLists.txt index 8e94972cb38ccfde9feec7e0c0775006bba6664d..6f492b91cb296421ecb32cf20bbe48f8c123b8bf 100644 --- a/tapeserver/CMakeLists.txt +++ b/tapeserver/CMakeLists.txt @@ -23,6 +23,7 @@ add_subdirectory (session) # The tape session's threads are in a separate directory (session, but compiled # from the previous one to create a single library). add_subdirectory (tapelabel) +add_subdirectory (readtp) include_directories (${PROTOBUF3_INCLUDE_DIRS}) add_executable (cta-taped cta-taped.cpp) diff --git a/tapeserver/readtp/CMakeLists.txt b/tapeserver/readtp/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..6607d13cd832fd4f0c136ae5d0ed84bbddae74d4 --- /dev/null +++ b/tapeserver/readtp/CMakeLists.txt @@ -0,0 +1,48 @@ +# @project The CERN Tape Archive (CTA) +# @copyright Copyright(C) 2015-2021 CERN +# @license This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +cmake_minimum_required (VERSION 2.6) + +include_directories(${CMAKE_SOURCE_DIR}/tapeserver) + +find_package( ZLIB REQUIRED ) + + +add_executable(cta-readtp + ReadtpCmd.cpp + ReadtpCmdMain.cpp + ReadtpCmdLineArgs.cpp + CmdLineTool.cpp + TapeFseqRange.cpp + TapeFseqRangeSequence.cpp + TapeFseqRangeListSequence.cpp + TapeFseqSequenceParser.cpp) + +target_link_libraries (cta-readtp + ctacommon + TapeDrive + ctamediachanger + ctacatalogue + SCSI +) + +# need to be removed when drop dependencies to taped +find_package(Protobuf3 REQUIRED) +set_property (TARGET cta-readtp APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) +if (OCCI_SUPPORT) + set_property (TARGET cta-readtp APPEND PROPERTY INSTALL_RPATH ${ORACLE-INSTANTCLIENT_RPATH}) +endif (OCCI_SUPPORT) + +install (TARGETS cta-readtp DESTINATION /usr/bin) +install (FILES ${CMAKE_CURRENT_SOURCE_DIR}/cta-readtp.1cta DESTINATION /usr/share/man/man1) \ No newline at end of file diff --git a/tapeserver/readtp/CmdLineTool.cpp b/tapeserver/readtp/CmdLineTool.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bc1e38b077583de83c5c17b18f3f7b2c658e77a9 --- /dev/null +++ b/tapeserver/readtp/CmdLineTool.cpp @@ -0,0 +1,105 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright(C) 2015-2021 CERN + * @license This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "tapeserver/readtp/CmdLineTool.hpp" +#include "common/exception/CommandLineNotParsed.hpp" + +#include <unistd.h> + +namespace cta { +namespace tapeserver { +namespace readtp { + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +CmdLineTool::CmdLineTool( + std::istream &inStream, + std::ostream &outStream, + std::ostream &errStream) noexcept: + m_in(inStream), + m_out(outStream), + m_err(errStream) { +} + +//------------------------------------------------------------------------------ +// destructor +//------------------------------------------------------------------------------ +CmdLineTool::~CmdLineTool() noexcept { +} + +//------------------------------------------------------------------------------ +// getUsername +//------------------------------------------------------------------------------ +std::string CmdLineTool::getUsername() { + char buf[256]; + + if(getlogin_r(buf, sizeof(buf))) { + return "UNKNOWN"; + } else { + return buf; + } +} + +//------------------------------------------------------------------------------ +// getHostname +//------------------------------------------------------------------------------ +std::string CmdLineTool::getHostname() { + char buf[256]; + + if(gethostname(buf, sizeof(buf))) { + return "UNKNOWN"; + } else { + buf[sizeof(buf) - 1] = '\0'; + return buf; + } +} + +//------------------------------------------------------------------------------ +// main +//------------------------------------------------------------------------------ +int CmdLineTool::main(const int argc, char *const *const argv) { + bool cmdLineNotParsed = false; + std::string errorMessage; + + try { + return exceptionThrowingMain(argc, argv); + } catch(exception::CommandLineNotParsed &ue) { + errorMessage = ue.getMessage().str(); + cmdLineNotParsed = true; + } catch(exception::Exception &ex) { + errorMessage = ex.getMessage().str(); + } catch(std::exception &se) { + errorMessage = se.what(); + } catch(...) { + errorMessage = "An unknown exception was thrown"; + } + + // Reaching this point means the command has failed, an exception was throw + // and errorMessage has been set accordingly + + m_err << "Aborting: " << errorMessage << std::endl; + if(cmdLineNotParsed) { + m_err << std::endl; + printUsage(m_err); + } + return 1; +} + +} // namespace readtp +} // namespace tapeserver +} // namespace cta diff --git a/tapeserver/readtp/CmdLineTool.hpp b/tapeserver/readtp/CmdLineTool.hpp new file mode 100644 index 0000000000000000000000000000000000000000..90a236af75bbd7bd6c22a0b785cfea95ae307eb2 --- /dev/null +++ b/tapeserver/readtp/CmdLineTool.hpp @@ -0,0 +1,108 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright(C) 2015-2021 CERN + * @license This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#pragma once + +#include <istream> +#include <ostream> + +namespace cta { +namespace tapeserver { +namespace readtp { + +/** + * Abstract class implementing common code and data structures for a + * command-line tool. + */ +class CmdLineTool { +public: + /** + * Constructor. + * + * @param inStream Standard input stream. + * @param outStream Standard output stream. + * @param errStream Standard error stream. + */ + CmdLineTool(std::istream &inStream, std::ostream &outStream, std::ostream &errStream) noexcept; + + /** + * Pure-virtual destructor to guarantee this class is abstract. + */ + virtual ~CmdLineTool() noexcept = 0; + + /** + * The object's implementation of main() that should be called from the main() + * of the program. + * + * @param argc The number of command-line arguments including the program name. + * @param argv The command-line arguments. + * @return The exit value of the program. + */ + int main(const int argc, char *const *const argv); + +protected: + + /** + * An exception throwing version of main(). + * + * @param argc The number of command-line arguments including the program name. + * @param argv The command-line arguments. + * @return The exit value of the program. + */ + virtual int exceptionThrowingMain(const int argc, char *const *const argv) = 0; + + /** + * Prints the usage message of the command-line tool. + * + * @param os The output stream to which the usage message is to be printed. + */ + virtual void printUsage(std::ostream &os) = 0; + + /** + * Standard input stream. + */ + std::istream &m_in; + + /** + * Standard output stream. + */ + std::ostream &m_out; + + /** + * Standard error stream. + */ + std::ostream &m_err; + + /** + * Returns the name of the user running the command-line tool. + * + * @return The name of the user running the command-line tool. + */ + static std::string getUsername(); + + /** + * Returns the name of the host on which the command-line tool is running. + * + * @return The name of the host on which the command-line tool is running. + */ + static std::string getHostname(); + +}; // class CmdLineTool + +} // namespace readtp +} // namespace tapeserver +} // namespace cta diff --git a/tapeserver/readtp/ReadtpCmd.cpp b/tapeserver/readtp/ReadtpCmd.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2419d83949c522701fb7ad8b1fe021a474fc7726 --- /dev/null +++ b/tapeserver/readtp/ReadtpCmd.cpp @@ -0,0 +1,588 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright(C) 2015-2021 CERN + * @license This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "common/Constants.hpp" +#include "common/log/DummyLogger.hpp" +#include "tapeserver/castor/tape/Constants.hpp" +#include "tapeserver/castor/tape/tapeserver/file/File.hpp" +#include "tapeserver/castor/tape/tapeserver/file/Structures.hpp" +#include "tapeserver/readtp/ReadtpCmd.hpp" +#include "tapeserver/readtp/ReadtpCmdLineArgs.hpp" +#include "tapeserver/readtp/TapeFseqRange.hpp" +#include "tapeserver/readtp/TapeFseqRangeListSequence.hpp" +#include "tapeserver/daemon/Tpconfig.hpp" +#include "tapeserver/castor/tape/tapeserver/daemon/Payload.hpp" +#include "mediachanger/LibrarySlotParser.hpp" +#include "disk/DiskFile.hpp" +#include "disk/RadosStriperPool.hpp" +#include "catalogue/TapeSearchCriteria.hpp" + + +namespace cta { +namespace tapeserver { +namespace readtp { + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +ReadtpCmd::ReadtpCmd(std::istream &inStream, std::ostream &outStream, + std::ostream &errStream, cta::log::StdoutLogger &log, cta::log::DummyLogger &dummyLog, + cta::mediachanger::MediaChangerFacade &mc): + CmdLineTool(inStream, outStream, errStream), + m_log(log), + m_dummyLog(dummyLog), + m_mc(mc), + m_useLbp(true), + m_nbSuccessReads(0), + m_nbFailedReads(0) { +} + +//------------------------------------------------------------------------------ +// destructor +//------------------------------------------------------------------------------ +ReadtpCmd::~ReadtpCmd() noexcept { +} + +//------------------------------------------------------------------------------ +// exceptionThrowingMain +//------------------------------------------------------------------------------ +int ReadtpCmd::exceptionThrowingMain(const int argc, char *const *const argv) { + const ReadtpCmdLineArgs cmdLineArgs(argc, argv); + if (cmdLineArgs.help) { + printUsage(m_out); + return 0; + } + + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", getUsername())); + params.push_back(cta::log::Param("tapeVid", cmdLineArgs.m_vid)); + m_log(cta::log::INFO, "Started", params); + + readAndSetConfiguration(getUsername(), cmdLineArgs); + + setProcessCapabilities("cap_sys_rawio+ep"); + + std::unique_ptr<castor::tape::tapeserver::drive::DriveInterface> drivePtr = createDrive(); + castor::tape::tapeserver::drive::DriveInterface &drive = *drivePtr.get(); + + if (!isDriveSupportLbp(drive)) { + m_log(cta::log::WARNING, "Drive does not support LBP", params); + m_driveSupportLbp = false; + } else { + m_driveSupportLbp = true; + }; + + mountTape(m_vid); + waitUntilTapeLoaded(drive, TAPE_LABEL_UNITREADY_TIMEOUT); + + int returnCode = 0; + try { + readTapeFiles(drive); + + } catch(cta::exception::Exception &ne) { + params.push_back(cta::log::Param("tapeReadError", ne.getMessage().str())); + m_log(cta::log::ERR, "Failed to read the tape", params); + returnCode = 1; + } + unloadTape(m_vid, drive); + dismountTape(m_vid); + + return returnCode; +} + +//------------------------------------------------------------------------------ +// readAndSetConfiguration +//------------------------------------------------------------------------------ +void ReadtpCmd::readAndSetConfiguration(const std::string &userName, const ReadtpCmdLineArgs &cmdLineArgs) { + m_vid = cmdLineArgs.m_vid; + m_fSeqRangeList = cmdLineArgs.m_fSeqRangeList; + m_xrootPrivateKeyPath = cmdLineArgs.m_xrootPrivateKeyPath; + m_userName = userName; + m_destinationFiles = readListFromFile(cmdLineArgs.m_destinationFileListURL); + cta::tape::daemon::Tpconfig tpConfig; + tpConfig = cta::tape::daemon::Tpconfig::parseFile(castor::tape::TPCONFIGPATH); + + if (tpConfig.empty()) { + cta::exception::Exception ex; + ex.getMessage() << "Unable to obtain drive info as TPCONFIG is empty"; + throw ex; + } + const auto &tpConfigLine = tpConfig.begin()->second.value(); + m_devFilename = tpConfigLine.devFilename; + m_rawLibrarySlot = tpConfigLine.rawLibrarySlot; + m_logicalLibrary = tpConfigLine.logicalLibrary; + m_unitName = tpConfigLine.unitName; + + const cta::rdbms::Login catalogueLogin = cta::rdbms::Login::parseFile(CATALOGUE_CONFIG_PATH); + const uint64_t nbConns = 1; + const uint64_t nbArchiveFileListingConns = 1; + + auto catalogueFactory = cta::catalogue::CatalogueFactoryFactory::create(m_dummyLog, // to supress catalogue output messages + catalogueLogin, nbConns, nbArchiveFileListingConns); + m_catalogue = catalogueFactory->create(); + + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("catalogueDbType", catalogueLogin.dbTypeToString(catalogueLogin.dbType))); + params.push_back(cta::log::Param("catalogueDatabase", catalogueLogin.database)); + params.push_back(cta::log::Param("catalogueUsername", catalogueLogin.username)); + params.push_back(cta::log::Param("devFilename", m_devFilename)); + params.push_back(cta::log::Param("rawLibrarySlot", m_rawLibrarySlot)); + params.push_back(cta::log::Param("logicalLibrary", m_logicalLibrary)); + params.push_back(cta::log::Param("unitName", m_unitName)); + m_log(cta::log::INFO, "Read configuration", params); +} + +//------------------------------------------------------------------------------ +// readListFromFile +//------------------------------------------------------------------------------ +std::list<std::string> ReadtpCmd::readListFromFile(const std::string &filename) const { + std::ifstream file(filename); + std::list<std::string> str_list; + if (file.fail()) { + throw std::runtime_error("Unable to open file " + filename); + } + + std::string line; + + while(std::getline(file, line)) { + // Strip out comments + auto pos = line.find('#'); + if(pos != std::string::npos) { + line.resize(pos); + } + + // Extract the list items + std::stringstream ss(line); + while(!ss.eof()) { + std::string item; + ss >> item; + // skip blank lines or lines consisting only of whitespace + if(item.empty()) continue; + + str_list.push_back(item); + } + } + return str_list; +} + + +//------------------------------------------------------------------------------ +// setProcessCapabilities +//------------------------------------------------------------------------------ +void ReadtpCmd::setProcessCapabilities(const std::string &capabilities) { + m_capUtils.setProcText(capabilities); + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("capabilities", capabilities)); + m_log(cta::log::DEBUG, "Set process capabilities", params); +} + +//------------------------------------------------------------------------------ +// createDrive +//------------------------------------------------------------------------------ +std::unique_ptr<castor::tape::tapeserver::drive::DriveInterface> + ReadtpCmd::createDrive() { + castor::tape::SCSI::DeviceVector dv(m_sysWrapper); + castor::tape::SCSI::DeviceInfo driveInfo = dv.findBySymlink(m_devFilename); + + // Instantiate the drive object + std::unique_ptr<castor::tape::tapeserver::drive::DriveInterface> + drive(castor::tape::tapeserver::drive::createDrive(driveInfo, m_sysWrapper)); + + if(NULL == drive.get()) { + cta::exception::Exception ex; + ex.getMessage() << "Failed to instantiate drive object"; + throw ex; + } + + return drive; +} + +//------------------------------------------------------------------------------ +// isDriveSupportLbp +//------------------------------------------------------------------------------ +bool ReadtpCmd::isDriveSupportLbp( + castor::tape::tapeserver::drive::DriveInterface &drive) const { + castor::tape::tapeserver::drive::deviceInfo devInfo = drive.getDeviceInfo(); + if (devInfo.isPIsupported) { //drive supports LBP + return true; + } else { + return false; + } +} + +//------------------------------------------------------------------------------ +// setLbpMode +//------------------------------------------------------------------------------ +void ReadtpCmd::setLbpMode( + castor::tape::tapeserver::drive::DriveInterface &drive) { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", m_userName)); + params.push_back(cta::log::Param("tapeVid", m_vid)); + params.push_back(cta::log::Param("tapeDrive", m_unitName)); + params.push_back(cta::log::Param("logicalLibrary", m_logicalLibrary)); + params.push_back(cta::log::Param("useLbp",boolToStr(m_useLbp))); + params.push_back(cta::log::Param("driveSupportLbp",boolToStr(m_driveSupportLbp))); + + if(m_useLbp) { + if (m_driveSupportLbp) { + // only crc32c lbp mode is supported + drive.enableCRC32CLogicalBlockProtectionReadWrite(); + m_log(cta::log::INFO, "Enabling LBP on drive", params); + } else { + m_useLbp = false; + drive.disableLogicalBlockProtection(); + m_log(cta::log::WARNING, "Disabling LBP on not supported drive", params); + } + } else { + drive.disableLogicalBlockProtection(); + m_log(cta::log::INFO, "Disabling LBP on drive", params); + } +} + +//------------------------------------------------------------------------------ +// mountTape +//------------------------------------------------------------------------------ +void ReadtpCmd::mountTape(const std::string &vid) { + std::unique_ptr<cta::mediachanger::LibrarySlot> librarySlotPtr; + librarySlotPtr.reset( + cta::mediachanger::LibrarySlotParser::parse(m_rawLibrarySlot)); + const cta::mediachanger::LibrarySlot &librarySlot = *librarySlotPtr.get(); + + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", m_userName)); + params.push_back(cta::log::Param("tapeVid", vid)); + params.push_back(cta::log::Param("tapeDrive", m_unitName)); + params.push_back(cta::log::Param("logicalLibrary", m_logicalLibrary)); + params.push_back(cta::log::Param("librarySlot", librarySlot.str())); + params.push_back(cta::log::Param("useLbp",boolToStr(m_useLbp))); + params.push_back(cta::log::Param("driveSupportLbp",boolToStr(m_driveSupportLbp))); + + m_log(cta::log::INFO, "Mounting tape", params); + m_mc.mountTapeReadOnly(vid, librarySlot); + if(cta::mediachanger::TAPE_LIBRARY_TYPE_MANUAL == librarySlot.getLibraryType()) { + m_log(cta::log::INFO, "Did not mount the tape because the media" + " changer is manual", params); + } else { + m_log(cta::log::INFO, "Mounted tape", params); + } +} + +//------------------------------------------------------------------------------ +// waitUntilTapeLoaded +//------------------------------------------------------------------------------ +void ReadtpCmd::waitUntilTapeLoaded( + castor::tape::tapeserver::drive::DriveInterface &drive, const int timeoutSecond) { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", m_userName)); + params.push_back(cta::log::Param("tapeVid", m_vid)); + params.push_back(cta::log::Param("tapeDrive", m_unitName)); + params.push_back(cta::log::Param("logicalLibrary", m_logicalLibrary)); + params.push_back(cta::log::Param("useLbp",boolToStr(m_useLbp))); + params.push_back(cta::log::Param("driveSupportLbp",boolToStr(m_driveSupportLbp))); + + try { + m_log(cta::log::INFO, "Loading tape", params); + drive.waitUntilReady(timeoutSecond); + m_log(cta::log::INFO, "Loaded tape", params); + } catch(cta::exception::Exception &ne) { + cta::exception::Exception ex; + ex.getMessage() << "Failed to wait for tape to be loaded: " << + ne.getMessage().str(); + throw ex; + } +} + +//Basic class representing a read job +class BasicRetrieveJob: public cta::RetrieveJob { +public: + BasicRetrieveJob() : cta::RetrieveJob(nullptr, + cta::common::dataStructures::RetrieveRequest(), + cta::common::dataStructures::ArchiveFile(), 1, + cta::PositioningMethod::ByFSeq) {} +}; + + +//------------------------------------------------------------------------------ +// getNextDestinationUrl +//------------------------------------------------------------------------------ +std::string ReadtpCmd::getNextDestinationUrl() { + if (m_destinationFiles.empty()) { + return "file:///dev/null"; + } else { + std::string ret = m_destinationFiles.front(); + m_destinationFiles.pop_front(); + return ret; + } +} + +//------------------------------------------------------------------------------ +// readTapeFiles +//------------------------------------------------------------------------------ +void ReadtpCmd::readTapeFiles( + castor::tape::tapeserver::drive::DriveInterface &drive) { + cta::disk::RadosStriperPool striperPool; + cta::disk::DiskFileFactory fileFactory(m_xrootPrivateKeyPath, 0, striperPool); + + catalogue::TapeSearchCriteria searchCriteria; + searchCriteria.vid = m_vid; + + auto tapeList = m_catalogue->getTapes(searchCriteria); + if (tapeList.empty()) { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", getUsername())); + params.push_back(cta::log::Param("tapeVid", m_vid)); + params.push_back(cta::log::Param("tapeDrive", m_unitName)); + params.push_back(cta::log::Param("logicalLibrary", m_logicalLibrary)); + params.push_back(cta::log::Param("useLbp",boolToStr(m_useLbp))); + params.push_back(cta::log::Param("driveSupportLbp",boolToStr(m_driveSupportLbp))); + m_log(cta::log::ERR, "Failed to get tape from catalogue", params); + return; + } + auto tape = tapeList.front(); + + TapeFseqRangeListSequence fSeqRangeListSequence(&m_fSeqRangeList); + std::string destinationFile = getNextDestinationUrl(); + uint64_t fSeq; + while (fSeqRangeListSequence.hasMore()) { + try { + fSeq = fSeqRangeListSequence.next(); + if (fSeq > tape.lastFSeq) { + break; //reached end of tape + } + std::unique_ptr<cta::disk::WriteFile> wfptr; + wfptr.reset(fileFactory.createWriteFile(destinationFile)); + cta::disk::WriteFile &wf = *wfptr.get(); + readTapeFile(drive, fSeq, wf); + m_nbSuccessReads++; // if readTapeFile returns, file was read successfully + destinationFile = getNextDestinationUrl(); + } catch (tapeserver::readtp::NoSuchFSeqException) { + //Do nothing + } catch(exception::Exception &ne) { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", getUsername())); + params.push_back(cta::log::Param("tapeVid", m_vid)); + params.push_back(cta::log::Param("destinationFile", destinationFile)); + params.push_back(cta::log::Param("tapeDrive", m_unitName)); + params.push_back(cta::log::Param("logicalLibrary", m_logicalLibrary)); + params.push_back(cta::log::Param("useLbp",boolToStr(m_useLbp))); + params.push_back(cta::log::Param("driveSupportLbp",boolToStr(m_driveSupportLbp))); + params.push_back(cta::log::Param("fSeq", fSeq)); + params.push_back(cta::log::Param("tapeReadError", ne.getMessage().str())); + m_log(cta::log::ERR, "Failed to read file from tape", params); + m_nbFailedReads++; + } + } + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", getUsername())); + params.push_back(cta::log::Param("tapeVid", m_vid)); + params.push_back(cta::log::Param("tapeDrive", m_unitName)); + params.push_back(cta::log::Param("logicalLibrary", m_logicalLibrary)); + params.push_back(cta::log::Param("useLbp",boolToStr(m_useLbp))); + params.push_back(cta::log::Param("driveSupportLbp",boolToStr(m_driveSupportLbp))); + params.push_back(cta::log::Param("nbReads", m_nbSuccessReads + m_nbFailedReads)); + params.push_back(cta::log::Param("nbSuccessfullReads", m_nbSuccessReads)); + params.push_back(cta::log::Param("nbFailedReads", m_nbFailedReads)); + + m_log(cta::log::INFO, "Finished reading tape", params); + +} + +//------------------------------------------------------------------------------ +// readTapeFile +//------------------------------------------------------------------------------ +void ReadtpCmd::readTapeFile( + castor::tape::tapeserver::drive::DriveInterface &drive, const uint64_t &fSeq, cta::disk::WriteFile &wf) { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", m_userName)); + params.push_back(cta::log::Param("tapeVid", m_vid)); + params.push_back(cta::log::Param("fSeq", fSeq)); + params.push_back(cta::log::Param("tapeDrive", m_unitName)); + params.push_back(cta::log::Param("logicalLibrary", m_logicalLibrary)); + params.push_back(cta::log::Param("useLbp",boolToStr(m_useLbp))); + params.push_back(cta::log::Param("driveSupportLbp",boolToStr(m_driveSupportLbp))); + params.push_back(cta::log::Param("destinationURL", wf.URL())); + + castor::tape::tapeserver::daemon::VolumeInfo volInfo; + volInfo.vid=m_vid; + volInfo.nbFiles = 0; + volInfo.mountType = cta::common::dataStructures::MountType::Retrieve; + castor::tape::tapeFile::ReadSession rs(drive, volInfo, m_useLbp); + + catalogue::TapeFileSearchCriteria searchCriteria; + searchCriteria.vid = m_vid; + searchCriteria.fSeq = fSeq; + auto itor = m_catalogue->getArchiveFilesItor(searchCriteria); + + if (!itor.hasMore()) { + throw tapeserver::readtp::NoSuchFSeqException(); + } + + m_log(cta::log::INFO, "Reading file from tape", params); + + const auto archiveFile = itor.next(); + + BasicRetrieveJob fileToRecall; + fileToRecall.retrieveRequest.archiveFileID = archiveFile.archiveFileID; + fileToRecall.selectedCopyNb = 0; + fileToRecall.archiveFile.tapeFiles.push_back(cta::common::dataStructures::TapeFile()); + fileToRecall.selectedTapeFile().fSeq = fSeq; + fileToRecall.positioningMethod = cta::PositioningMethod::ByFSeq; + + castor::tape::tapeFile::ReadFile rf(&rs, fileToRecall); + auto checksum_adler32 = castor::tape::tapeserver::daemon::Payload::zeroAdler32(); + const size_t buffer_size = 1 * 1024 * 1024 * 1024; //1Gb + size_t read_data_size = 0; + auto payload = new castor::tape::tapeserver::daemon::Payload(buffer_size); //allocate one gigabyte buffer + try { + while(1) { + if (payload->remainingFreeSpace() <= rf.getBlockSize()) { + //buffer is full, flush to file and update checksum + read_data_size += payload->size(); + checksum_adler32 = payload->adler32(checksum_adler32); + payload->write(wf); + payload->reset(); + } + payload->append(rf); + } + } catch (cta::exception::EndOfFile ex) { + //File completely read + } + read_data_size += payload->size(); + checksum_adler32 = payload->adler32(checksum_adler32); + payload->write(wf); + auto cb = cta::checksum::ChecksumBlob(cta::checksum::ChecksumType::ADLER32, checksum_adler32); + + archiveFile.checksumBlob.validate(cb); //exception thrown if checksums differ + + params.push_back(cta::log::Param("checksumType", "ADLER32")); + std::stringstream sstream; + sstream << std::hex << checksum_adler32; + params.push_back(cta::log::Param("checksumValue", "0x" + sstream.str())); + params.push_back(cta::log::Param("readFileSize", read_data_size)); + m_log(cta::log::INFO, "Read file from tape successfully", params); +} + +//------------------------------------------------------------------------------ +// unloadTape +//------------------------------------------------------------------------------ +void ReadtpCmd::unloadTape( + const std::string &vid, castor::tape::tapeserver::drive::DriveInterface &drive) { + std::unique_ptr<cta::mediachanger::LibrarySlot> librarySlotPtr; + librarySlotPtr.reset( + cta::mediachanger::LibrarySlotParser::parse(m_rawLibrarySlot)); + const cta::mediachanger::LibrarySlot &librarySlot = *librarySlotPtr.get(); + + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", m_userName)); + params.push_back(cta::log::Param("tapeVid", m_vid)); + params.push_back(cta::log::Param("tapeDrive", m_unitName)); + params.push_back(cta::log::Param("logicalLibrary", m_logicalLibrary)); + params.push_back(cta::log::Param("useLbp",boolToStr(m_useLbp))); + params.push_back(cta::log::Param("driveSupportLbp",boolToStr(m_driveSupportLbp))); + + // We implement the same policy as with the tape sessions: + // if the librarySlot parameter is "manual", do nothing. + if(cta::mediachanger::TAPE_LIBRARY_TYPE_MANUAL == librarySlot.getLibraryType()) { + m_log(cta::log::INFO, "Not unloading tape because media changer is" + " manual", params); + return; + } + try { + m_log(cta::log::INFO, "Unloading tape", params); + drive.unloadTape(); + m_log(cta::log::INFO, "Unloaded tape", params); + } catch (cta::exception::Exception &ne) { + cta::exception::Exception ex; + ex.getMessage() << "Failed to unload tape: " << + ne.getMessage().str(); + throw ex; + } +} + +//------------------------------------------------------------------------------ +// dismountTape +//------------------------------------------------------------------------------ +void ReadtpCmd::dismountTape(const std::string &vid) { + std::unique_ptr<cta::mediachanger::LibrarySlot> librarySlotPtr; + librarySlotPtr.reset( + cta::mediachanger::LibrarySlotParser::parse(m_rawLibrarySlot)); + const cta::mediachanger::LibrarySlot &librarySlot = *librarySlotPtr.get(); + + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", m_userName)); + params.push_back(cta::log::Param("tapeVid", m_vid)); + params.push_back(cta::log::Param("tapeDrive", m_unitName)); + params.push_back(cta::log::Param("logicalLibrary", m_logicalLibrary)); + params.push_back(cta::log::Param("librarySlot", librarySlot.str())); + params.push_back(cta::log::Param("useLbp",boolToStr(m_useLbp))); + params.push_back(cta::log::Param("driveSupportLbp",boolToStr(m_driveSupportLbp))); + + try { + m_log(cta::log::INFO, "Dismounting tape", params); + m_mc.dismountTape(vid, librarySlot); + const bool dismountWasManual = cta::mediachanger::TAPE_LIBRARY_TYPE_MANUAL == + librarySlot.getLibraryType(); + if(dismountWasManual) { + m_log(cta::log::INFO, "Did not dismount tape because media" + " changer is manual", params); + } else { + m_log(cta::log::INFO, "Dismounted tape", params); + } + } catch(cta::exception::Exception &ne) { + cta::exception::Exception ex; + ex.getMessage() << "Failed to dismount tape: " << + ne.getMessage().str(); + throw ex; + } +} + +//------------------------------------------------------------------------------ +// rewindDrive +//------------------------------------------------------------------------------ +void ReadtpCmd::rewindDrive( + castor::tape::tapeserver::drive::DriveInterface &drive) { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", m_userName)); + params.push_back(cta::log::Param("tapeVid", m_vid)); + params.push_back(cta::log::Param("tapeDrive", m_unitName)); + params.push_back(cta::log::Param("logicalLibrary", m_logicalLibrary)); + params.push_back(cta::log::Param("useLbp",boolToStr(m_useLbp))); + params.push_back(cta::log::Param("driveSupportLbp",boolToStr(m_driveSupportLbp))); + + m_log(cta::log::INFO, "Rewinding tape", params); + drive.rewind(); + m_log(cta::log::INFO, "Successfully rewound tape", params); +} + +//------------------------------------------------------------------------------ +// printUsage +//------------------------------------------------------------------------------ +void ReadtpCmd::printUsage(std::ostream &os) { + ReadtpCmdLineArgs::printUsage(os); +} + +//------------------------------------------------------------------------------ +// boolToStr +//------------------------------------------------------------------------------ +const char *ReadtpCmd::boolToStr( + const bool value) { + return value ? "true" : "false"; +} + +} // namespace readtp +} // namespace tapeserver +} // namespace cta \ No newline at end of file diff --git a/tapeserver/readtp/ReadtpCmd.hpp b/tapeserver/readtp/ReadtpCmd.hpp new file mode 100644 index 0000000000000000000000000000000000000000..75f8d79fa07ff3a6ab93841556d682c33fc95f9c --- /dev/null +++ b/tapeserver/readtp/ReadtpCmd.hpp @@ -0,0 +1,311 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright(C) 2015-2021 CERN + * @license This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#pragma once + +#include "common/log/StdoutLogger.hpp" +#include "common/log/DummyLogger.hpp" +#include "common/log/LogContext.hpp" +#include "common/processCap/ProcessCap.hpp" +#include "tapeserver/readtp/ReadtpCmdLineArgs.hpp" +#include "tapeserver/castor/tape/tapeserver/drive/DriveInterface.hpp" +#include "tapeserver/castor/tape/tapeserver/drive/DriveGeneric.hpp" +#include "tapeserver/castor/tape/tapeserver/daemon/EncryptionControl.hpp" +#include "tapeserver/daemon/Tpconfig.hpp" +#include "tapeserver/readtp/CmdLineTool.hpp" +#include "tapeserver/readtp/TapeFseqRange.hpp" +#include "tapeserver/readtp/TapeFseqRangeListSequence.hpp" +#include "catalogue/CatalogueFactoryFactory.hpp" +#include "mediachanger/MediaChangerFacade.hpp" +#include "disk/DiskFile.hpp" + +#include <memory> + +namespace cta { +namespace tapeserver { +namespace readtp { + +/** + * Command-line tool for reading files from a CTA tape. + */ +class ReadtpCmd: public CmdLineTool { +public: + + /** + * Constructor. + * + * @param inStream Standard input stream. + * @param outStream Standard output stream. + * @param errStream Standard error stream. + * @param log The object representing the API of the CTA logging system. + * @param mc Interface to the media changer. + */ + ReadtpCmd(std::istream &inStream, std::ostream &outStream, + std::ostream &errStream, cta::log::StdoutLogger &log, + cta::log::DummyLogger &dummyLog, + cta::mediachanger::MediaChangerFacade &mc); + + /** + * Destructor. + */ + ~ReadtpCmd() noexcept; + +private: + + /** + * An exception throwing version of main(). + * + * @param argc The number of command-line arguments including the program name. + * @param argv The command-line arguments. + * @return The exit value of the program. + */ + int exceptionThrowingMain(const int argc, char *const *const argv) override; + + /** + * Prints the usage message of the command-line tool. + * + * @param os The output stream to which the usage message is to be printed. + */ + void printUsage(std::ostream &os) override; + + /** + * Sets internal configuration parameters to be used for reading. + * It reads drive and library parameters from /etc/cta/TPCONFIG and catalogue + * login parameters from /etc/cta/cta-catalogue.conf. + * + * @param username The name of the user running the command-line tool. + * @param cmdLineArgs The arguments parsed from the command line. + */ + void readAndSetConfiguration(const std::string &userName, const ReadtpCmdLineArgs &cmdLineArgs); + + /** + * Reads a file line by line, strips comments and returns a list of the file lines. + * + * @param filename The name of the file to be read. + * @return A list of line strings after stripping comments. + */ + std::list<std::string> readListFromFile(const std::string &filename) const; + + /** + * Returns the next destination file URL, or file:///dev/null if all destination files have been used. + * + * @return The URL of the next destination file. + */ + std::string getNextDestinationUrl(); + + /** + * Sets the capabilities of the process and logs the result. + * + * @param capabilities The string representation of the capabilities. + */ + void setProcessCapabilities(const std::string &capabilities); + + /** + * Returns a Drive object representing the tape drive to be used to read + * the tape. + * + * @return The drive object. + */ + std::unique_ptr<castor::tape::tapeserver::drive::DriveInterface> createDrive(); + + /** + * Detects if the drive supports the logical block protection. + * + * @param drive The tape drive. + * @return The boolean value true if the drive supports LBP or false otherwise. + */ + bool isDriveSupportLbp(castor::tape::tapeserver::drive::DriveInterface &drive) const; + + /** + * Sets the logical block protection mode on the drive + * depending on useLbp and driveSupportLbp parameters. This method needs to + * be used to avoid exceptions in setLbp if drive does not support LBP (mhvtl). + * + * @param drive The tape drive. + */ + void setLbpMode(castor::tape::tapeserver::drive::DriveInterface &drive); + + /** + * Mounts the tape to be read. + * @param vid The volume identifier of the tape to be mounted. + */ + void mountTape(const std::string &vid); + + /** + * Waits for the tape to be loaded into the tape drive. + * + * @param drive Object representing the drive hardware. + * @param timeoutSecond The number of seconds to wait for the tape to be + * loaded into the tape drive. + */ + void waitUntilTapeLoaded(castor::tape::tapeserver::drive::DriveInterface &drive, + const int timeoutSecond); + + /** + * Read the files requested from tape + * + * @param drive Object representing the drive hardware. + */ + void readTapeFiles(castor::tape::tapeserver::drive::DriveInterface &drive); + + /** + * Read a specific file from tape + * @param drive Object representing the drive hardware. + * @param fSeq The tape file fSeq. + */ + void readTapeFile(castor::tape::tapeserver::drive::DriveInterface &drive, const uint64_t &fSeq, + cta::disk::WriteFile &wf); + + + /** + * Unloads the specified tape from the specified tape drive. + * + * @param vid The volume identifier of the tape to be unloaded. Please note + * that the value of this field is only used for logging purposes. + * @param drive The tape drive. + */ + void unloadTape(const std::string &vid, castor::tape::tapeserver::drive::DriveInterface &drive); + + /** + * Dismounts the specified tape. + * + * @param vid The volume identifier of the tape to be dismounted. + */ + void dismountTape(const std::string &vid); + + /** + * Rewinds the specified tape drive. + * + * @param drive The tape drive. + */ + void rewindDrive(castor::tape::tapeserver::drive::DriveInterface &drive); + + /** + * Returns the string representation of the specified boolean value. + * + * @param value The boolean value. + * @return The string representation. + */ + const char *boolToStr(const bool value); + + /** + * The object representing the API of the CTA logging system. + */ + cta::log::StdoutLogger &m_log; + +/** + *Dummy logger for the catalogue + */ + cta::log::DummyLogger &m_dummyLog; + + /** + * Hard coded path for the catalogue login configuration. + */ + const std::string CATALOGUE_CONFIG_PATH = "/etc/cta/cta-catalogue.conf"; + + /** + * Unique pointer to the catalogue interface; + */ + std::unique_ptr<cta::catalogue::Catalogue> m_catalogue; + + /** + * Object providing utilities for working UNIX capabilities. + */ + cta::server::ProcessCap m_capUtils; + + /** + * The system wrapper used to find the device and instantiate the drive object. + */ + castor::tape::System::realWrapper m_sysWrapper; + + /** + * The filename of the device file of the tape drive. + */ + std::string m_devFilename; + + /** + * The slot in the tape library that contains the tape drive (string encoded). + */ + std::string m_rawLibrarySlot; + + /** + * The logical library of the tape drive. + */ + std::string m_logicalLibrary; + + /** + * The unit name of the tape drive. + */ + std::string m_unitName; + + /** + * The name of the user running the command-line tool. + */ + std::string m_userName; + + /** + * The tape VID to read. + */ + std::string m_vid; + + /** + * Path to the xroot private key file. + */ + std::string m_xrootPrivateKeyPath; + + /** + * The iterator of destination urls the data read is sent to + */ + std::list<std::string> m_destinationFiles; + + /** + * The file fSeq to read. + */ + TapeFseqRangeList m_fSeqRangeList; + + /** + * The object representing the media changer. + */ + cta::mediachanger::MediaChangerFacade &m_mc; + + /** + * The boolean variable which determinate logical block protection usage by + * readtp commands. Hard coded when we create the class. + */ + bool m_useLbp; + + /** + * The boolean variable to store if drive support LBP. + */ + bool m_driveSupportLbp; + + /** + * Number of files read successsfully. + */ + uint64_t m_nbSuccessReads; + + /** + * Number of failed reads. + */ + uint64_t m_nbFailedReads; +}; // class ReadtpCmd + +CTA_GENERATE_EXCEPTION_CLASS(NoSuchFSeqException); + +} // namespace readtp +} // namespace tapeserver +} // namespace cta \ No newline at end of file diff --git a/tapeserver/readtp/ReadtpCmdLineArgs.cpp b/tapeserver/readtp/ReadtpCmdLineArgs.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3f409ffc2bd637f41d0fffab3d9ad145849112e9 --- /dev/null +++ b/tapeserver/readtp/ReadtpCmdLineArgs.cpp @@ -0,0 +1,133 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright(C) 2015-2021 CERN + * @license This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "tapeserver/readtp/ReadtpCmdLineArgs.hpp" +#include "tapeserver/readtp/TapeFseqSequenceParser.hpp" +#include "common/exception/CommandLineNotParsed.hpp" +#include "common/utils/utils.hpp" +#include "common/Constants.hpp" + +#include <getopt.h> +#include <ostream> + +#include <string.h> + +namespace cta { +namespace tapeserver { +namespace readtp { + + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +ReadtpCmdLineArgs::ReadtpCmdLineArgs(const int argc, char *const *const argv): + help(false), m_vid(""), m_destinationFileListURL(""), m_xrootPrivateKeyPath("") { + if (argc < 3) { + help = true; + return; + } + if ((strlen(argv[1])) > CA_MAXVIDLEN) { + exception::CommandLineNotParsed ex; + ex.getMessage() << "The vid is too long"; + throw ex; + } + m_vid = std::string(argv[1]); + utils::toUpper(m_vid); + + m_fSeqRangeList = TapeFileSequenceParser::parse(argv[2]); + + static struct option longopts[] = { + {"destination_files", required_argument, NULL, 'f'}, + {"xroot_private_key", required_argument, NULL, 'p'}, + {"help", no_argument, NULL, 'h'}, + {NULL , 0, NULL, 0} + }; + + opterr = 0; + int opt = 0; + int opt_index = 3; + + while ((opt = getopt_long(argc, argv, ":d:f:p:h", longopts, &opt_index)) != -1) { + switch(opt) { + case 'f': + m_destinationFileListURL = std::string(optarg); + break; + case 'p': + m_xrootPrivateKeyPath = std::string(optarg); + break; + case 'h': + help = true; + break; + case ':': // Missing parameter + { + exception::CommandLineNotParsed ex; + ex.getMessage() << "The -" << (char)optopt << " option requires a parameter"; + throw ex; + } + case '?': // Unknown option + { + exception::CommandLineNotParsed ex; + if(0 == optopt) { + ex.getMessage() << "Unknown command-line option"; + } else { + ex.getMessage() << "Unknown command-line option: -" << (char)optopt; + } + throw ex; + } + default: + { + exception::CommandLineNotParsed ex; + ex.getMessage() << + "getopt_long returned the following unknown value: 0x" << + std::hex << (int)opt; + throw ex; + } + } // switch(opt) + } // while getopt_long() + + if (m_destinationFileListURL.empty()) { + m_destinationFileListURL = "/dev/null"; // Equivalent to an empty file + } +} + + +//------------------------------------------------------------------------------ +// printUsage +//------------------------------------------------------------------------------ +void ReadtpCmdLineArgs::printUsage(std::ostream &os) { + os << + "Usage:" << std::endl << + " cta-readtp <VID> <SEQUENCE> [options]" << std::endl << + "Where:" << std::endl << + " <VID> The VID of the tape to be read" << std::endl << + " <SEQUENCE> A sequence of tape file sequence numbers. The syntax to be used is:" << std::endl << + " f1-f2 Files f1 to f2 inclusive" << std::endl << + " f1- Files f1 to the last file on the tape" << std::endl << + " f1-f2,f4,f6- A series of non-consecutive ranges of files" << std::endl << + "Options:" <<std::endl << + " -h, --help Print this help message and exit." << std::endl << + " -p, --xroot_private_key <KEY PATH> Path to the xroot private key file. Necessary if" << std::endl << + " read files are to be written using xroot." << std::endl << + " -f, --destination_files <FILE URL> URL to file containing a list of destination files." << std::endl << + " If not set, all data read is written to file:///dev/null" << std::endl << + " If there are less destination files than read files, the remaining" << std::endl << + " files read will be written to file:///dev/null." << std::endl; +} + +} // namespace readtp +} // namespace tapeserver +} // namespace cta \ No newline at end of file diff --git a/tapeserver/readtp/ReadtpCmdLineArgs.hpp b/tapeserver/readtp/ReadtpCmdLineArgs.hpp new file mode 100644 index 0000000000000000000000000000000000000000..e62cea3bb819ef2ca1e3f816b61efe4ceeb42ce0 --- /dev/null +++ b/tapeserver/readtp/ReadtpCmdLineArgs.hpp @@ -0,0 +1,77 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright(C) 2015-2021 CERN + * @license This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#pragma once + +#include "tapeserver/readtp/TapeFseqRangeListSequence.hpp" + +#include <string> + +namespace cta { +namespace tapeserver { +namespace readtp { + +/** + * Structure to store the command-line arguments of the command-line tool + * named cta-readtp. + */ +struct ReadtpCmdLineArgs { + /** + * True if the usage message should be printed. + */ + bool help; + + /** + * The tape VID to read. + */ + std::string m_vid; + + /** + * Sequence of file fSeqs to read. + */ + TapeFseqRangeList m_fSeqRangeList; + + /** + * The destination file list url. + */ + std::string m_destinationFileListURL; + + /** + * Path to the xroot private key file. + */ + std::string m_xrootPrivateKeyPath; + + /** + * Constructor that parses the specified command-line arguments. + * + * @param argc The number of command-line arguments including the name of the + * executable. + * @param argv The vector of command-line arguments. + */ + ReadtpCmdLineArgs(const int argc, char *const *const argv); + + /** + * Prints the usage message of the command-line tool. + * + * @param os The output stream to which the usage message is to be printed. + */ + static void printUsage(std::ostream &os); +}; // class ReadtpCmdLineArgs + +} // namespace readtp +} // namespace tapeserver +} // namespace cta diff --git a/tapeserver/readtp/ReadtpCmdMain.cpp b/tapeserver/readtp/ReadtpCmdMain.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4ec024ccfcdcbeea016e291681647f9b8199dd8c --- /dev/null +++ b/tapeserver/readtp/ReadtpCmdMain.cpp @@ -0,0 +1,40 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright(C) 2015-2021 CERN + * @license This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + + +#include "tapeserver/readtp/ReadtpCmd.hpp" +#include <iostream> + +//------------------------------------------------------------------------------ +// main +//------------------------------------------------------------------------------ +int main(const int argc, char *const *const argv) { + char buf[256]; + std::string hostName; + if(gethostname(buf, sizeof(buf))) { + hostName = "UNKNOWN"; + } else { + buf[sizeof(buf) - 1] = '\0'; + hostName = buf; + } + cta::log::StdoutLogger log(hostName, "cta-readtp"); + cta::log::DummyLogger dummyLog("dummy", "dummy"); + cta::mediachanger::MediaChangerFacade mc(log); + + cta::tapeserver::readtp::ReadtpCmd cmd(std::cin, std::cout, std::cerr, log, dummyLog, mc); + return cmd.main(argc, argv); +} \ No newline at end of file diff --git a/tapeserver/readtp/TapeFseqRange.cpp b/tapeserver/readtp/TapeFseqRange.cpp new file mode 100644 index 0000000000000000000000000000000000000000..81cd07918bdfffe6536ab688f2201c4bc822d733 --- /dev/null +++ b/tapeserver/readtp/TapeFseqRange.cpp @@ -0,0 +1,152 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright(C) 2015-2021 CERN + * @license This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "tapeserver/readtp/TapeFseqRange.hpp" +#include "common/exception/InvalidArgument.hpp" + +#include <getopt.h> +#include <ostream> + +#include <string.h> + +namespace cta { +namespace tapeserver { +namespace readtp { + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +TapeFseqRange::TapeFseqRange() throw() { + + reset(); +} + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +TapeFseqRange::TapeFseqRange(const uint32_t lower, const uint32_t upper) { + + reset(lower, upper); +} + +//------------------------------------------------------------------------------ +// reset +//------------------------------------------------------------------------------ +void TapeFseqRange::reset() throw() { + m_isEmpty = true; + m_lower = 0; // Ignored + m_upper = 0; // Ignored +} + + +//------------------------------------------------------------------------------ +// reset +//------------------------------------------------------------------------------ +void TapeFseqRange::reset(const uint32_t lower, + const uint32_t upper) { + + if(lower == 0) { + exception::InvalidArgument ex; + + ex.getMessage() << "Lower boundary must not be zero"; + throw ex; + } + + // If the upper boundary is not 0 meaning infinity and the lower boundary is + // greater than the upper boundary + if(upper != 0 && lower > upper) { + exception::InvalidArgument ex; + + ex.getMessage() << + "Lower boundary must be less than or equal to the upper boundary" + ": lower=" << lower << " upper=" << upper; + + throw ex; + } + + m_isEmpty = false; + m_lower = lower; + m_upper = upper; +} + + +//------------------------------------------------------------------------------ +// isEmpty +//------------------------------------------------------------------------------ +bool TapeFseqRange::isEmpty() const throw() { + return m_isEmpty; +} + +//------------------------------------------------------------------------------ +// lower +//------------------------------------------------------------------------------ +uint32_t TapeFseqRange::lower() const throw() { + + return m_isEmpty ? 0 : m_lower; +} + +//------------------------------------------------------------------------------ +// upper +//------------------------------------------------------------------------------ +uint32_t TapeFseqRange::upper() const throw() { + + return m_isEmpty ? 0 : m_upper; +} + +//------------------------------------------------------------------------------ +// size +//------------------------------------------------------------------------------ +uint32_t TapeFseqRange::size() const throw() { + + return m_isEmpty || m_upper == 0 ? 0 : m_upper - m_lower + 1; +} + +} // namespace readtp +} // namespace tapeserver +} // namespace cta + +//------------------------------------------------------------------------------ +// ostream << operator for cta::tapeserver::readtp::TapeFseqRange +//------------------------------------------------------------------------------ +std::ostream &operator<<(std::ostream &os, + const cta::tapeserver::readtp::TapeFseqRange &value) { + + if(value.isEmpty()) { + os << "EMPTY"; + } else { + uint32_t lower = 0; + uint32_t upper = 0; + + try { + lower = value.lower(); + upper = value.upper(); + + os << lower << "-"; + + // An upper value of 0 means END of tape + if(upper !=0) { + os << upper; + } else { + os << "END"; + } + } catch(cta::exception::Exception &ex) { + os << "ERROR"; + } + } + + return os; +} diff --git a/tapeserver/readtp/TapeFseqRange.hpp b/tapeserver/readtp/TapeFseqRange.hpp new file mode 100644 index 0000000000000000000000000000000000000000..35d903cc7faeea222986baf5abef2f63709c9633 --- /dev/null +++ b/tapeserver/readtp/TapeFseqRange.hpp @@ -0,0 +1,129 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright(C) 2015-2021 CERN + * @license This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#pragma once + +#include <string> +#include <ostream> + +namespace cta { +namespace tapeserver { +namespace readtp { + +/** + * A range of tape file sequence numbers specified by an inclusive lower + * boundary and an inclusive upper boundary. + */ +class TapeFseqRange { + +public: + + /** + * Constructor. + * + * Constructs an empty range. + */ + TapeFseqRange() throw(); + + /** + * Constructor. + * + * Constructs a range with the specified inclusive lower and upper + * boundaries. An upper boundary of 0 means infinity. + * + * Throws an InvalidArgument exception if either the lower boundary is 0 + * or if the lower boundary is greater than the upper boundary. + * + * @param lower The inclusive lower bound of the range. + * @param upper The inclusive upper bound of the range. + */ + TapeFseqRange(const uint32_t lower, const uint32_t upper) + ; + + /** + * Resets the range to be an empty range. + */ + void reset() throw(); + + /** + * Resets the range to be a finite range with the specified inclusive lower + * and upper boundaries. + * + * Throws an InvalidArgument exception if either the lower boundary is 0 + * or if the lower boundary is greater than the upper boundary. + * + * @param lower The inclusive lower bound of the range. + * @param upper The inclusive upper bound of the range. + */ + void reset(const uint32_t lower, const uint32_t upper) + ; + + /** + * Returns true if the range is empty. + */ + bool isEmpty() const throw(); + + /** + * Returns the inclusive lower bound of the range or 0 if the range is empty. + */ + uint32_t lower() const throw(); + + /** + * Returns the inclusive upper bound of the range. If the range is finite, + * then a value greater than 0 is returned. If the range is either empty or + * infinite then 0 is returned. The method isEmpty() or a lower() return + * value of 0 can be used to distinguish between the two cases when upper() + * returns 0. + */ + uint32_t upper() const throw(); + + /** + * Returns the size of the range. An empty or infinite range returns 0. + * The method isEmpty() or a lower() return value of 0 can be used to + * distinguish between the two cases when size() returns 0. + */ + uint32_t size() const throw(); + + +private: + + /** + * True if this range is empty, else false. + */ + bool m_isEmpty; + + /** + * The inclusive lower bound of the range. + */ + uint32_t m_lower; + + /** + * The inclusive upper bound of the range. A value of 0 means infinity. + */ + uint32_t m_upper; +}; + +} // namespace readtp +} // namespace tapeserver +} // namespace cta + +/** + * ostream << operator for cta::tapeserver::readtp::TapeFseqRange + */ +std::ostream &operator<<(std::ostream &os, + const cta::tapeserver::readtp::TapeFseqRange &value); + diff --git a/tapeserver/readtp/TapeFseqRangeListSequence.cpp b/tapeserver/readtp/TapeFseqRangeListSequence.cpp new file mode 100644 index 0000000000000000000000000000000000000000..479ffd6cb259b051e73dca8b5d5ba3a9c4d7c153 --- /dev/null +++ b/tapeserver/readtp/TapeFseqRangeListSequence.cpp @@ -0,0 +1,174 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright(C) 2015-2021 CERN + * @license This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "tapeserver/readtp/TapeFseqRange.hpp" +#include "tapeserver/readtp/TapeFseqRangeListSequence.hpp" +#include "common/exception/InvalidArgument.hpp" +#include "common/exception/Exception.hpp" + +#include <getopt.h> +#include <ostream> + +#include <list> +#include <errno.h> + +namespace cta { +namespace tapeserver { +namespace readtp { + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +TapeFseqRangeListSequence::TapeFseqRangeListSequence() + { + reset(NULL); +} + + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +TapeFseqRangeListSequence::TapeFseqRangeListSequence( + const TapeFseqRangeList *const list) { + reset(list); +} + + +//------------------------------------------------------------------------------ +// reset +//------------------------------------------------------------------------------ +void TapeFseqRangeListSequence::reset( + const TapeFseqRangeList *const list) { + m_list = list; + + if(m_list == NULL) { + m_isFinite = true; + m_totalSize = 0; + } else { + m_rangeItor = list->begin(); + m_nbSequence = (*(list->begin())); + + // Determine the values of m_isFinite and m_totalSize + m_isFinite = true; // Initial guess + m_totalSize = 0; // Initial guess + for(TapeFseqRangeList::const_iterator itor=list->begin(); + itor != list->end(); itor++) { + const TapeFseqRange &range = *itor; + + // If upper bound of range is infinity + if(range.upper() == 0) { + m_isFinite = false; + m_totalSize = 0; + + // No need to continue counting + break; + + // Else the upper bound is finite + } else { + m_totalSize += range.size(); + } + } + } +} + + +//------------------------------------------------------------------------------ +// hasMore +//------------------------------------------------------------------------------ +bool TapeFseqRangeListSequence::hasMore() const throw() { + if(m_list != NULL) { + return m_nbSequence.hasMore(); + } else { + return false; + } +} + + +//------------------------------------------------------------------------------ +// next +//------------------------------------------------------------------------------ +uint32_t TapeFseqRangeListSequence::next() + { + + if(!hasMore()) { + exception::Exception ex; + + ex.getMessage() + << "Invalid operation: Sequence::next() called after end of sequence"; + + throw ex; + } + + uint32_t tmp = m_nbSequence.next(); + + // If the end of the current range sequence has been reached + if(!m_nbSequence.hasMore()) { + + // Move on to the next if there is one + m_rangeItor++; + if(m_rangeItor != m_list->end()) { + m_nbSequence = *m_rangeItor; + } + } + + return tmp; +} + + +//------------------------------------------------------------------------------ +// isFinite +//------------------------------------------------------------------------------ +bool TapeFseqRangeListSequence::isFinite() const throw() { + return m_isFinite; +} + + +//------------------------------------------------------------------------------ +// totalSize +//------------------------------------------------------------------------------ +uint32_t TapeFseqRangeListSequence::totalSize() + const throw() { + return m_totalSize; +} + +} // namespace readtp +} // namespace tapeserver +} // namespace cta + +//------------------------------------------------------------------------------ +// ostream << operator for castor::tape::tpcp::TapeFseqRangeList +//------------------------------------------------------------------------------ +std::ostream &operator<<(std::ostream &os, + const cta::tapeserver::readtp::TapeFseqRangeList &value) { + + os << '{'; + + for(cta::tapeserver::readtp::TapeFseqRangeList::const_iterator itor = + value.begin(); itor != value.end(); itor++) { + + // Write a separating comma if not the first item in the list + if(itor!=value.begin()) { + os << ","; + } + + os << *itor; + } + + os << '}'; + + return os; +} diff --git a/tapeserver/readtp/TapeFseqRangeListSequence.hpp b/tapeserver/readtp/TapeFseqRangeListSequence.hpp new file mode 100644 index 0000000000000000000000000000000000000000..7ce698f802b5ddb594b81faf193b14f227aeba77 --- /dev/null +++ b/tapeserver/readtp/TapeFseqRangeListSequence.hpp @@ -0,0 +1,135 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright(C) 2015-2021 CERN + * @license This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#pragma once + +#include "TapeFseqRange.hpp" +#include "TapeFseqRangeSequence.hpp" + +#include <list> +#include <ostream> + +namespace cta { +namespace tapeserver { +namespace readtp { + +/** + * A list of tape file sequence ranges. + */ +class TapeFseqRangeList: public std::list<TapeFseqRange> { +}; // class TapeFseqRangeList + +/** + * Generates a sequence of tape file sequence numbers from a list of tape file + * sequence ranges. + */ +class TapeFseqRangeListSequence { +public: + + /** + * Constructor. + * + * Creates an empty sequence, in other word hasMore() will always return + * false. + */ + TapeFseqRangeListSequence() ; + + /** + * Constructor. + * + * @param list The list of tape file sequence ranges from which the sequence + * of tape file sequence numbers is to be generated. + */ + TapeFseqRangeListSequence(const TapeFseqRangeList *const list) + ; + + /** + * Resets the sequence. + * + * @param list The list of tape file sequence ranges from which the sequence + * of tape file sequence numbers is to be generated. + */ + void reset(const TapeFseqRangeList *const list) + ; + + /** + * Returns true if there is another tape file sequence number in the + * sequence. + */ + bool hasMore() const throw(); + + /** + * Returns the next tape file sequence number in the sequence, or throws an + * exception if there isn't one. + */ + uint32_t next() ; + + /** + * Returns true if the sequence is finite, else false if it is infinite. + */ + bool isFinite() const throw(); + + /** + * Returns the total number of values the sequence could ever generate. The + * value returned by this method is not affected by calls to next(). This + * method returns 0 if the total number of values is 0 or infinity. The + * isFinite() method can be used to distinguish between the two cases. + */ + uint32_t totalSize() const throw(); + + +private: + + /** + * The list of tape file sequence ranges. + */ + const TapeFseqRangeList *m_list; + + /** + * Iterator pointing to the current range of tape file sequence numbers. + */ + TapeFseqRangeList::const_iterator m_rangeItor; + + /** + * True if the sequence is finite, else false if it is infinite. + */ + bool m_isFinite; + + /** + * The total number of values the sequence could ever generate. The + * value returned by this method is not affected by calls to next(). This + * method returns 0 if the total number of values is 0 or infinity. The + * isFinite() method can be used to distinguish between the two cases. + */ + uint32_t m_totalSize; + + /** + * The current sequence of the tape file sequence numbers. + */ + TapeFseqRangeSequence m_nbSequence; + +}; // class TapeFseqRangeListSequence + +} // namespace readtp +} // namespace tapeserver +} // namespace cta + +/** + * ostream << operator for cta::tapeserver::readtp::TapeFseqRangeList + */ +std::ostream &operator<<(std::ostream &os, + const cta::tapeserver::readtp::TapeFseqRangeList &value); diff --git a/tapeserver/readtp/TapeFseqRangeSequence.cpp b/tapeserver/readtp/TapeFseqRangeSequence.cpp new file mode 100644 index 0000000000000000000000000000000000000000..de7c65ed20013bab3fc956b1f18c964edd956855 --- /dev/null +++ b/tapeserver/readtp/TapeFseqRangeSequence.cpp @@ -0,0 +1,122 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright(C) 2015-2021 CERN + * @license This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "tapeserver/readtp/TapeFseqRangeSequence.hpp" +#include "tapeserver/readtp/TapeFseqRange.hpp" +#include "common/exception/Exception.hpp" + +#include <getopt.h> +#include <ostream> + +#include <list> + +namespace cta { +namespace tapeserver { +namespace readtp { + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +TapeFseqRangeSequence::TapeFseqRangeSequence() throw() { + reset(); +} + + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +TapeFseqRangeSequence::TapeFseqRangeSequence( + const TapeFseqRange &range) throw() { + reset(range); +} + + +//------------------------------------------------------------------------------ +// reset +//------------------------------------------------------------------------------ +void TapeFseqRangeSequence::reset() throw() { + + // Reset the range to be empty + m_range.reset(); + + m_next = 0; // Ignored +} + + +//------------------------------------------------------------------------------ +// reset +//------------------------------------------------------------------------------ +void TapeFseqRangeSequence::reset( + const TapeFseqRange &range) throw() { + m_range = range; + + if(range.isEmpty()) { + m_next = 0; // Ignored + } else { + m_next = m_range.lower(); + } +} + + +//------------------------------------------------------------------------------ +// hasMore +//------------------------------------------------------------------------------ +bool TapeFseqRangeSequence::hasMore() const throw() { + + if(m_range.isEmpty()) { + + return false; + + } else { + + // Infinity is represented by range.upper() == 0 + return m_range.upper() == 0 || m_next <= m_range.upper(); + } +} + + +//------------------------------------------------------------------------------ +// next +//------------------------------------------------------------------------------ +uint32_t TapeFseqRangeSequence::next() + { + + if(!hasMore()) { + exception::Exception ex; + + ex.getMessage() + << "Sequence::next() called after end of sequence"; + + throw ex; + } + + return m_next++; +} + + +//------------------------------------------------------------------------------ +// range +//------------------------------------------------------------------------------ +const TapeFseqRange + &TapeFseqRangeSequence::range() const throw() { + return m_range; +} + + +} // namespace readtp +} // namespace tapeserver +} // namespace cta \ No newline at end of file diff --git a/tapeserver/readtp/TapeFseqRangeSequence.hpp b/tapeserver/readtp/TapeFseqRangeSequence.hpp new file mode 100644 index 0000000000000000000000000000000000000000..ae068dfd0a68bc570711f93c733eb9b36331c7cd --- /dev/null +++ b/tapeserver/readtp/TapeFseqRangeSequence.hpp @@ -0,0 +1,100 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright(C) 2015-2021 CERN + * @license This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#pragma once + +#include "TapeFseqRange.hpp" + +#include <list> + +namespace cta { +namespace tapeserver { +namespace readtp { + +/** + * Generates a sequence of tape file sequence numbers from a range of tape file + * sequence numbers. + */ +class TapeFseqRangeSequence { +public: + + /** + * Constructor. + * + * Constructs an empty sequence. + */ + TapeFseqRangeSequence() throw(); + + /** + * Constructor. + * + * @param range The range from which the sequence is generated. + */ + TapeFseqRangeSequence(const TapeFseqRange &range) throw(); + + /** + * Resets the sequence to empty. + */ + void reset() throw(); + + /** + * Resets the sequence using the specified range. + * + * @param range The range from which the sequence is generated. + */ + void reset(const TapeFseqRange &range) throw(); + + /** + * Returns true if there is another tape file sequence number in the + * sequence. + */ + bool hasMore() const throw(); + + /** + * Returns the next tape file sequence number in the sequence, or throws + * NoValue exception if there isn't one. + */ + uint32_t next() ; + + /** + * Returns the number of values generated by the sequence so far. + */ + uint32_t nbGeneratedValues() const throw(); + + /** + * Returns the range used by this sequence. + */ + const TapeFseqRange &range() const throw(); + + +private: + + /** + * The range from which the sequence is generated. + */ + TapeFseqRange m_range; + + /** + * The value to be returned by a call to next(). + */ + uint32_t m_next; + +}; // class TapeFseqRangeSequence + +} // namespace readtp +} // namespace tapeserver +} // namespace cta \ No newline at end of file diff --git a/tapeserver/readtp/TapeFseqSequenceParser.cpp b/tapeserver/readtp/TapeFseqSequenceParser.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8e4d9a47c1547b30ed042a13bf1747056b453c1f --- /dev/null +++ b/tapeserver/readtp/TapeFseqSequenceParser.cpp @@ -0,0 +1,150 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright(C) 2015-2021 CERN + * @license This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "common/exception/InvalidArgument.hpp" +#include "tapeserver/readtp/TapeFseqSequenceParser.hpp" +#include "common/utils/utils.hpp" + +#include <list> +#include <string> +#include <vector> + +namespace cta { +namespace tapeserver { +namespace readtp { + +//------------------------------------------------------------------------------ +// parseTapeFileSequence +//------------------------------------------------------------------------------ +TapeFseqRangeList TapeFileSequenceParser::parse(char *const str) +{ + TapeFseqRangeList tapeFseqRanges; + std::vector<std::string> rangeStrs; + + // Range strings are separated by commas + utils::splitString(str, ',', rangeStrs); + + // For each range string + for(std::vector<std::string>::const_iterator itor=rangeStrs.begin(); + itor!=rangeStrs.end(); itor++) { + + std::vector<std::string> boundaryStrs; + + // Lower and upper boundary strings are separated by a dash ('-') + utils::splitString(*itor, '-', boundaryStrs); + + int nbBoundaries = boundaryStrs.size(); + + switch(nbBoundaries) { + case 1: // Range string = "n" + if(!utils::isValidUInt(boundaryStrs[0].c_str())) { + exception::InvalidArgument ex; + ex.getMessage() << "Invalid range string: '" << boundaryStrs[0] + << "': Expecting an unsigned integer"; + throw ex; + } + + { + const uint32_t upperLower = atoi(boundaryStrs[0].c_str()); + tapeFseqRanges.push_back(TapeFseqRange(upperLower, upperLower)); + } + break; + + case 2: // Range string = "m-n" or "-n" or "m-" or "-" + + // If "-n" or "-" then the range string is invalid + if(boundaryStrs[0] == "") { + exception::InvalidArgument ex; + ex.getMessage() << "Invalid range string: '" << *itor + << "': Strings of the form '-n' or '-' are invalid"; + throw ex; + } + + // At this point the range string must be either "m-n" or "m-" + + // Parse the "m" of "m-n" or "m-" + if(!utils::isValidUInt(boundaryStrs[0].c_str())) { + exception::InvalidArgument ex; + ex.getMessage() << "Invalid range string: '" << *itor + << "': The lower boundary should be an unsigned integer"; + throw ex; + } + + { + const uint32_t lower = atoi(boundaryStrs[0].c_str()); + + if(lower == 0) { + exception::InvalidArgument ex; + ex.getMessage() << "Invalid range string: '" << *itor + << "': The lower boundary can not be '0'"; + throw ex; + } + + // If "m-" + if(boundaryStrs[1] == "") { + + // Infinity (or until the end of tape) is represented by 0 + tapeFseqRanges.push_back(TapeFseqRange(lower, 0)); + + // Else "m-n" + } else { + + // Parse the "n" of "m-n" + if(!utils::isValidUInt(boundaryStrs[1].c_str())) { + exception::InvalidArgument ex; + ex.getMessage() << "Invalid range string: '" << *itor + << "': The upper boundary should be an unsigned integer"; + throw ex; + } + + const uint32_t upper = atoi(boundaryStrs[1].c_str()); + + if(upper == 0){ + exception::InvalidArgument ex; + ex.getMessage() << "Invalid range string: '" << *itor + << "': The upper boundary can not be '0'"; + throw ex; + } + + if(lower > upper){ + exception::InvalidArgument ex; + ex.getMessage() << "Invalid range string: '" << *itor + << "': The lower boundary cannot be greater than the upper " + "boundary"; + throw ex; + } + + tapeFseqRanges.push_back(TapeFseqRange(lower, upper)); + } // Else "m-n" + } + + + break; + + default: // Range string is invalid + exception::InvalidArgument ex; + ex.getMessage() << "Invalid range string: '" << *itor + << "': A range string can only contain one or no dashes ('-')"; + throw ex; + } + } + return tapeFseqRanges; +} + +} // namespace readtp +} // namespace tapeserver +} // namespace cta \ No newline at end of file diff --git a/tapeserver/readtp/TapeFseqSequenceParser.hpp b/tapeserver/readtp/TapeFseqSequenceParser.hpp new file mode 100644 index 0000000000000000000000000000000000000000..d413ca27963e3fe1b0949e43eaf4d8a9695bbcde --- /dev/null +++ b/tapeserver/readtp/TapeFseqSequenceParser.hpp @@ -0,0 +1,58 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright(C) 2015-2021 CERN + * @license This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#pragma once + +#include "common/exception/Exception.hpp" +#include "tapeserver/readtp/TapeFseqRangeListSequence.hpp" + +#include <list> + +namespace cta { +namespace tapeserver { +namespace readtp { + +/** + * Helper class to parse tape file sequence parameter strings. + */ +class TapeFileSequenceParser { +public: + + /** + * Parse the specified tape file sequence parameter string and store the + * resulting ranges into m_parsedCommandLine.tapeFseqRanges. + * + * The syntax rules for a tape file sequence specification are: + * <ul> + * <li> f1 File f1. + * <li> f1-f2 Files f1 to f2 included. + * <li> f1- Files from f1 to the last file of the tape. + * <li> f1-f2,f4,f6- Series of ranges "," separated. + * </ul> + * + * @param str The string received as an argument for the TapeFileSequence + * option. + * @return The resulting list of tape file sequence ranges. + */ + static TapeFseqRangeList parse(char *const str) + ; + +}; // class TapeFileSequenceParser + +} // namespace readtp +} // namespace tapeserver +} // namespace cta \ No newline at end of file diff --git a/tapeserver/readtp/cta-readtp.1cta b/tapeserver/readtp/cta-readtp.1cta new file mode 100644 index 0000000000000000000000000000000000000000..cba3f3363f2719da2b2edd1c18f8f48690955eb0 --- /dev/null +++ b/tapeserver/readtp/cta-readtp.1cta @@ -0,0 +1,53 @@ +.\" @project The CERN Tape Archive (CTA) +.\" @copyright Copyright(C) 2019-2021 CERN +.\" @license This program is free software: you can redistribute it and/or modify +.\" it under the terms of the GNU General Public License as published by +.\" the Free Software Foundation, either version 3 of the License, or +.\" (at your option) any later version. +.\" +.\" This program is distributed in the hope that it will be useful, +.\" but WITHOUT ANY WARRANTY; without even the implied warranty of +.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +.\" GNU General Public License for more details. +.\" +.\" You should have received a copy of the GNU General Public License +.\" along with this program. If not, see <http://www.gnu.org/licenses/>. + +.TH CTA-READTP 1CTA "August 2021" CTA CTA +.SH NAME +cta-readtp \- Read files from a tape +.SH SYNOPSIS +.BI "cta-readtp VID SEQUENCE [OPTIONS]" + +.SH DESCRIPTION +\fBcta-readtp\fP is a command-line tool for reading files from tape and validating their checksums. + +The tape to be read is specified by the \fBVID\fP argument. The tape files to be read are specified as a \fBSEQUENCE\fP of tape file sequence numbers. The +syntax used to specify the sequence is as follows: + + f1-f2 Files f1 to f2 inclusive. + f1- Files f1 to the last file on the tape. + f1-f2,f4,f6- A series of non-consecutive ranges of files. + +.SH OPTIONS +.TP +.TP +\fB\-h, \-\-help +Prints the usage message. +.TP +\fB\-f, \-\-destination_files +Path to a file containing a list of URLs the read files will be written to. If not specified, read files will be written to file:///dev/null. +If there are more read files than destination files, the remaining read files will be written to file:///dev/null +.TP +\fB\-p, \-\-xroot_private_key +Path to the xroot private key file. Necessary if any destination file URL is for xroot. +. + +.SH RETURN VALUE +Zero on success and non-zero on failure. +.SH EXAMPLES +.br +cta-readtp V01007 10002,10004-10006,10008- + +.SH AUTHOR +\fBCTA\fP Team