diff --git a/catalogue/CMakeLists.txt b/catalogue/CMakeLists.txt index 6b697726a1eed00e749fef7c761d4e9660d263ed..43cf8af1d246b5b707c51049a72f554bed52183c 100644 --- a/catalogue/CMakeLists.txt +++ b/catalogue/CMakeLists.txt @@ -18,7 +18,7 @@ cmake_minimum_required (VERSION 2.6) set (CATALOGUE_LIB_SRC_FILES Catalogue.cpp DummyCatalogue.cpp - SQLiteDatabase.cpp) + SqliteDatabase.cpp) add_library (ctacatalogue ${CATALOGUE_LIB_SRC_FILES}) diff --git a/catalogue/SQLiteDatabase.cpp b/catalogue/SqliteDatabase.cpp similarity index 85% rename from catalogue/SQLiteDatabase.cpp rename to catalogue/SqliteDatabase.cpp index ad85c60202829f759ab00dd6167bc8e277bddc41..d9c93f90ccd1c885d50432787ec5983b590136e5 100644 --- a/catalogue/SQLiteDatabase.cpp +++ b/catalogue/SqliteDatabase.cpp @@ -16,18 +16,18 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#include "catalogue/SQLiteDatabase.hpp" +#include "catalogue/SqliteDatabase.hpp" #include "common/exception/Exception.hpp" //------------------------------------------------------------------------------ // constructor //------------------------------------------------------------------------------ -cta::catalogue::SQLiteDatabase::SQLiteDatabase(const std::string &filename) { +cta::catalogue::SqliteDatabase::SqliteDatabase(const std::string &filename) { m_dbHandle = NULL; if(sqlite3_open(filename.c_str(), &m_dbHandle)) { sqlite3_close(m_dbHandle); exception::Exception ex; - ex.getMessage() << "Failed to construct SQLiteDatabase: sqlite3_open failed" + ex.getMessage() << "Failed to construct SqliteDatabase: sqlite3_open failed" ": " << sqlite3_errmsg(m_dbHandle); throw(ex); } @@ -36,13 +36,13 @@ cta::catalogue::SQLiteDatabase::SQLiteDatabase(const std::string &filename) { //------------------------------------------------------------------------------ // destructor //------------------------------------------------------------------------------ -cta::catalogue::SQLiteDatabase::~SQLiteDatabase() { +cta::catalogue::SqliteDatabase::~SqliteDatabase() { sqlite3_close(m_dbHandle); } //------------------------------------------------------------------------------ // getHandle //------------------------------------------------------------------------------ -sqlite3 *cta::catalogue::SQLiteDatabase::getHandle() { +sqlite3 *cta::catalogue::SqliteDatabase::getHandle() { return m_dbHandle; } diff --git a/catalogue/SQLiteDatabase.hpp b/catalogue/SqliteDatabase.hpp similarity index 91% rename from catalogue/SQLiteDatabase.hpp rename to catalogue/SqliteDatabase.hpp index a72676626febfa8c16af293ca20668e6ac094bcd..f8a016aa4b42926c7d88e3e3f51c8688121e727b 100644 --- a/catalogue/SQLiteDatabase.hpp +++ b/catalogue/SqliteDatabase.hpp @@ -27,7 +27,7 @@ namespace catalogue { /** * A C++ wrapper around an SQLite database handle. */ -class SQLiteDatabase { +class SqliteDatabase { public: /** @@ -35,12 +35,12 @@ public: * * @param filename The filename to be passed to the sqlit3_open() function. */ - SQLiteDatabase(const std::string &filename); + SqliteDatabase(const std::string &filename); /** * Destructor. */ - ~SQLiteDatabase(); + ~SqliteDatabase(); /** * Returns the underlying database handle. @@ -52,7 +52,7 @@ public: private: /** - * SQLite database handle. + * The database handle. */ sqlite3 *m_dbHandle; diff --git a/common/ArchiveRequest.cpp b/common/ArchiveRequest.cpp deleted file mode 100644 index 510fa710e0b0146a41be59a4f547d0a8cb545a96..0000000000000000000000000000000000000000 --- a/common/ArchiveRequest.cpp +++ /dev/null @@ -1,241 +0,0 @@ -/* - * The CERN Tape Archive (CTA) project - * Copyright (C) 2015 CERN - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#include "common/ArchiveRequest.hpp" -#include "exception/Exception.hpp" - -//------------------------------------------------------------------------------ -// constructor -//------------------------------------------------------------------------------ -cta::ArchiveRequest::ArchiveRequest() { - m_srcURLSet = false; - m_fileSizeSet = false; - m_checksumTypeSet = false; - m_checksumValueSet = false; - m_storageClassSet = false; - m_drInstanceSet = false; - m_drPathSet = false; - m_drOwnerSet = false; - m_drGroupSet = false; - m_drBlobSet = false; -} - -//------------------------------------------------------------------------------ -// destructor -//------------------------------------------------------------------------------ -cta::ArchiveRequest::~ArchiveRequest() throw() { -} - -//------------------------------------------------------------------------------ -// allFieldsSet -//------------------------------------------------------------------------------ -bool cta::ArchiveRequest::allFieldsSet() const { - if(m_srcURLSet==true - && m_fileSizeSet==true - && m_checksumTypeSet==true - && m_checksumValueSet==true - && m_storageClassSet==true - && m_drInstanceSet==true - && m_drPathSet==true - && m_drOwnerSet==true - && m_drGroupSet==true - && m_drBlobSet==true) { - return true; - } - return false; -} - -//------------------------------------------------------------------------------ -// setDrBlob -//------------------------------------------------------------------------------ -void cta::ArchiveRequest::setDrBlob(std::string drBlob) { - m_drBlob = drBlob; - m_drBlobSet = true; -} - -//------------------------------------------------------------------------------ -// getDrBlob -//------------------------------------------------------------------------------ -std::string cta::ArchiveRequest::getDrBlob() const { - if(!allFieldsSet()) { - throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the ArchiveRequest have been set!"); - } - return m_drBlob; -} - -//------------------------------------------------------------------------------ -// setDrGroup -//------------------------------------------------------------------------------ -void cta::ArchiveRequest::setDrGroup(std::string drGroup) { - m_drGroup = drGroup; - m_drGroupSet = true; -} - -//------------------------------------------------------------------------------ -// getDrGroup -//------------------------------------------------------------------------------ -std::string cta::ArchiveRequest::getDrGroup() const { - if(!allFieldsSet()) { - throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the ArchiveRequest have been set!"); - } - return m_drGroup; -} - -//------------------------------------------------------------------------------ -// setDrOwner -//------------------------------------------------------------------------------ -void cta::ArchiveRequest::setDrOwner(std::string drOwner) { - m_drOwner = drOwner; - m_drOwnerSet = true; -} - -//------------------------------------------------------------------------------ -// getDrOwner -//------------------------------------------------------------------------------ -std::string cta::ArchiveRequest::getDrOwner() const { - if(!allFieldsSet()) { - throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the ArchiveRequest have been set!"); - } - return m_drOwner; -} - -//------------------------------------------------------------------------------ -// setDrPath -//------------------------------------------------------------------------------ -void cta::ArchiveRequest::setDrPath(std::string drPath) { - m_drPath = drPath; - m_drPathSet = true; -} - -//------------------------------------------------------------------------------ -// getDrPath -//------------------------------------------------------------------------------ -std::string cta::ArchiveRequest::getDrPath() const { - if(!allFieldsSet()) { - throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the ArchiveRequest have been set!"); - } - return m_drPath; -} - -//------------------------------------------------------------------------------ -// setDrInstance -//------------------------------------------------------------------------------ -void cta::ArchiveRequest::setDrInstance(std::string drInstance) { - m_drInstance = drInstance; - m_drInstanceSet = true; -} - -//------------------------------------------------------------------------------ -// getDrInstance -//------------------------------------------------------------------------------ -std::string cta::ArchiveRequest::getDrInstance() const { - if(!allFieldsSet()) { - throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the ArchiveRequest have been set!"); - } - return m_drInstance; -} - -//------------------------------------------------------------------------------ -// setStorageClass -//------------------------------------------------------------------------------ -void cta::ArchiveRequest::setStorageClass(std::string storageClass) { - m_storageClass = storageClass; - m_storageClassSet = true; -} - -//------------------------------------------------------------------------------ -// getStorageClass -//------------------------------------------------------------------------------ -std::string cta::ArchiveRequest::getStorageClass() const { - if(!allFieldsSet()) { - throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the ArchiveRequest have been set!"); - } - return m_storageClass; -} - -//------------------------------------------------------------------------------ -// setChecksumValue -//------------------------------------------------------------------------------ -void cta::ArchiveRequest::setChecksumValue(std::string checksumValue) { - m_checksumValue = checksumValue; - m_checksumValueSet = true; -} - -//------------------------------------------------------------------------------ -// getChecksumValue -//------------------------------------------------------------------------------ -std::string cta::ArchiveRequest::getChecksumValue() const { - if(!allFieldsSet()) { - throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the ArchiveRequest have been set!"); - } - return m_checksumValue; -} - -//------------------------------------------------------------------------------ -// setChecksumType -//------------------------------------------------------------------------------ -void cta::ArchiveRequest::setChecksumType(std::string checksumType) { - m_checksumType = checksumType; - m_checksumTypeSet = true; -} - -//------------------------------------------------------------------------------ -// getChecksumType -//------------------------------------------------------------------------------ -std::string cta::ArchiveRequest::getChecksumType() const { - if(!allFieldsSet()) { - throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the ArchiveRequest have been set!"); - } - return m_checksumType; -} - -//------------------------------------------------------------------------------ -// setFileSize -//------------------------------------------------------------------------------ -void cta::ArchiveRequest::setFileSize(uint64_t fileSize) { - m_fileSize = fileSize; - m_fileSizeSet = true; -} - -//------------------------------------------------------------------------------ -// getFileSize -//------------------------------------------------------------------------------ -uint64_t cta::ArchiveRequest::getFileSize() const { - if(!allFieldsSet()) { - throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the ArchiveRequest have been set!"); - } - return m_fileSize; -} - -//------------------------------------------------------------------------------ -// setSrcURL -//------------------------------------------------------------------------------ -void cta::ArchiveRequest::setSrcURL(std::string srcURL) { - m_srcURL = srcURL; - m_srcURLSet = true; -} - -//------------------------------------------------------------------------------ -// getSrcURL -//------------------------------------------------------------------------------ -std::string cta::ArchiveRequest::getSrcURL() const { - if(!allFieldsSet()) { - throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the ArchiveRequest have been set!"); - } - return m_srcURL; -} diff --git a/common/ArchiveRequest.hpp b/common/ArchiveRequest.hpp deleted file mode 100644 index 31f6b0da81eb2bf256e8a78a2b2cd62f2b78a62c..0000000000000000000000000000000000000000 --- a/common/ArchiveRequest.hpp +++ /dev/null @@ -1,133 +0,0 @@ -/* - * The CERN Tape Archive (CTA) project - * Copyright (C) 2015 CERN - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#pragma once - -#include <stdint.h> -#include <string> - -namespace cta { - -/** - * Structure to store an archive request. - */ -struct ArchiveRequest { - -public: - - /** - * Constructor - */ - ArchiveRequest(); - - /** - * Destructor - */ - ~ArchiveRequest() throw(); - - void setDrBlob(std::string drBlob); - std::string getDrBlob() const; - void setDrGroup(std::string drGroup); - std::string getDrGroup() const; - void setDrOwner(std::string drOwner); - std::string getDrOwner() const; - void setDrPath(std::string drPath); - std::string getDrPath() const; - void setDrInstance(std::string drInstance); - std::string getDrInstance() const; - void setStorageClass(std::string storageClass); - std::string getStorageClass() const; - void setChecksumValue(std::string checksumValue); - std::string getChecksumValue() const; - void setChecksumType(std::string checksumType); - std::string getChecksumType() const; - void setFileSize(uint64_t fileSize); - uint64_t getFileSize() const; - void setSrcURL(std::string srcURL); - std::string getSrcURL() const; - -private: - - /** - * @return true if all fields have been set, false otherwise - */ - bool allFieldsSet() const; - - /** - * The EOS src URL. - */ - std::string m_srcURL; - bool m_srcURLSet; - - /** - * The size of the file to be archived in bytes. - */ - uint64_t m_fileSize; - bool m_fileSizeSet; - - /** - * The checksum type. - */ - std::string m_checksumType; - bool m_checksumTypeSet; - - /** - * The checksum value. - */ - std::string m_checksumValue; - bool m_checksumValueSet; - - /** - * The storage class name. - */ - std::string m_storageClass; - bool m_storageClassSet; - - /** - * The disaster recovery EOS instance. - */ - std::string m_drInstance; - bool m_drInstanceSet; - - /** - * The disaster recovery EOS path. - */ - std::string m_drPath; - bool m_drPathSet; - - /** - * The disaster recovery EOS owner. - */ - std::string m_drOwner; - bool m_drOwnerSet; - - /** - * The disaster recovery EOS group. - */ - std::string m_drGroup; - bool m_drGroupSet; - - /** - * The disaster recovery EOS key-value string containing everything above and more (no parsing by CTA). - */ - std::string m_drBlob; - bool m_drBlobSet; - -}; // struct ArchiveRequest - -} // namespace cta diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 7963b9a73d069388d49e4c9471f79d35a2fe3715..ede3830b2c7dad8c190fce1d4477db0eda5f43de 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -73,7 +73,6 @@ set (COMMON_LIB_SRC_FILES threading/Threading.cpp utils/utils.cpp utils/strerror_r_wrapper.cpp - ArchiveRequest.cpp CreationLog.cpp Configuration.cpp SecurityIdentity.cpp diff --git a/common/dataStructures/ArchiveFile.cpp b/common/dataStructures/ArchiveFile.cpp index 42b93d99e53acb336207539c28e6271ff94714cf..1772c18678b8558a1c664d53333e0743621da308 100644 --- a/common/dataStructures/ArchiveFile.cpp +++ b/common/dataStructures/ArchiveFile.cpp @@ -31,6 +31,7 @@ cta::common::dataStructures::ArchiveFile::ArchiveFile() { m_fileSizeSet = false; m_storageClassSet = false; m_tapeCopiesSet = false; + m_creationLogSet = false; } //------------------------------------------------------------------------------ @@ -46,6 +47,7 @@ bool cta::common::dataStructures::ArchiveFile::allFieldsSet() const { return m_archiveFileIDSet && m_checksumTypeSet && m_checksumValueSet + && m_creationLogSet && m_drDataSet && m_eosFileIDSet && m_fileSizeSet @@ -196,3 +198,21 @@ std::map<int,cta::common::dataStructures::TapeFileLocation> cta::common::dataStr } return m_tapeCopies; } + +//------------------------------------------------------------------------------ +// setCreationLog +//------------------------------------------------------------------------------ +void cta::common::dataStructures::ArchiveFile::setCreationLog(const cta::common::dataStructures::EntryLog &creationLog) { + m_creationLog = creationLog; + m_creationLogSet = true; +} + +//------------------------------------------------------------------------------ +// getCreationLog +//------------------------------------------------------------------------------ +cta::common::dataStructures::EntryLog cta::common::dataStructures::ArchiveFile::getCreationLog() const { + if(!allFieldsSet()) { + throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the AdminHost have been set!"); + } + return m_creationLog; +} diff --git a/common/dataStructures/ArchiveFile.hpp b/common/dataStructures/ArchiveFile.hpp index db0330be62afa863eabc8a215ef0219ab549dfc3..9ab761d437ca99d4a32a05f8a564d7de2259dcd9 100644 --- a/common/dataStructures/ArchiveFile.hpp +++ b/common/dataStructures/ArchiveFile.hpp @@ -24,6 +24,7 @@ #include <string> #include "common/dataStructures/DRData.hpp" +#include "common/dataStructures/EntryLog.hpp" #include "common/dataStructures/TapeFileLocation.hpp" namespace cta { @@ -67,6 +68,9 @@ public: void setTapeCopies(const std::map<int,cta::common::dataStructures::TapeFileLocation> &tapeCopies); std::map<int,cta::common::dataStructures::TapeFileLocation> getTapeCopies() const; + + void setCreationLog(const cta::common::dataStructures::EntryLog &creationLog); + cta::common::dataStructures::EntryLog getCreationLog() const; private: @@ -100,6 +104,9 @@ private: std::map<int,cta::common::dataStructures::TapeFileLocation> m_tapeCopies; bool m_tapeCopiesSet; + cta::common::dataStructures::EntryLog m_creationLog; + bool m_creationLogSet; + }; // class ArchiveFile } // namespace dataStructures diff --git a/common/dataStructures/ArchiveRequest.cpp b/common/dataStructures/ArchiveRequest.cpp index eb88eb20cb64e64510bb86647254c15172bed2ec..7576a0f15115b108a8d2457d07f3779ce676a86e 100644 --- a/common/dataStructures/ArchiveRequest.cpp +++ b/common/dataStructures/ArchiveRequest.cpp @@ -33,6 +33,7 @@ cta::common::dataStructures::ArchiveRequest::ArchiveRequest() { m_requesterSet = false; m_srcURLSet = false; m_storageClassSet = false; + m_creationLogSet = false; } //------------------------------------------------------------------------------ @@ -47,6 +48,7 @@ cta::common::dataStructures::ArchiveRequest::~ArchiveRequest() throw() { bool cta::common::dataStructures::ArchiveRequest::allFieldsSet() const { return m_checksumTypeSet && m_checksumValueSet + && m_creationLogSet && m_diskpoolNameSet && m_diskpoolThroughputSet && m_drDataSet @@ -236,3 +238,21 @@ std::string cta::common::dataStructures::ArchiveRequest::getStorageClass() const } return m_storageClass; } + +//------------------------------------------------------------------------------ +// setCreationLog +//------------------------------------------------------------------------------ +void cta::common::dataStructures::ArchiveRequest::setCreationLog(const cta::common::dataStructures::EntryLog &creationLog) { + m_creationLog = creationLog; + m_creationLogSet = true; +} + +//------------------------------------------------------------------------------ +// getCreationLog +//------------------------------------------------------------------------------ +cta::common::dataStructures::EntryLog cta::common::dataStructures::ArchiveRequest::getCreationLog() const { + if(!allFieldsSet()) { + throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the AdminHost have been set!"); + } + return m_creationLog; +} diff --git a/common/dataStructures/ArchiveRequest.hpp b/common/dataStructures/ArchiveRequest.hpp index c32223a71965a9ade342897a6bd5aef672b31d47..d54fd3f0479443ae84ca273be120104eff3b926f 100644 --- a/common/dataStructures/ArchiveRequest.hpp +++ b/common/dataStructures/ArchiveRequest.hpp @@ -24,6 +24,7 @@ #include <string> #include "common/dataStructures/DRData.hpp" +#include "common/dataStructures/EntryLog.hpp" #include "common/dataStructures/Requester.hpp" namespace cta { @@ -73,6 +74,9 @@ public: void setStorageClass(const std::string &storageClass); std::string getStorageClass() const; + + void setCreationLog(const cta::common::dataStructures::EntryLog &creationLog); + cta::common::dataStructures::EntryLog getCreationLog() const; private: @@ -112,6 +116,9 @@ private: std::string m_storageClass; bool m_storageClassSet; + cta::common::dataStructures::EntryLog m_creationLog; + bool m_creationLogSet; + }; // class ArchiveRequest } // namespace dataStructures diff --git a/common/dataStructures/CancelRetrieveRequest.cpp b/common/dataStructures/CancelRetrieveRequest.cpp index f2e9e00408af144d845a87b79bf8f2892536b288..9000f1c40ff37278bd7a8de465f7eea5a23d8f12 100644 --- a/common/dataStructures/CancelRetrieveRequest.cpp +++ b/common/dataStructures/CancelRetrieveRequest.cpp @@ -27,6 +27,7 @@ cta::common::dataStructures::CancelRetrieveRequest::CancelRetrieveRequest() { m_drDataSet = false; m_dstURLSet = false; m_requesterSet = false; + m_creationLogSet = false; } //------------------------------------------------------------------------------ @@ -40,6 +41,7 @@ cta::common::dataStructures::CancelRetrieveRequest::~CancelRetrieveRequest() thr //------------------------------------------------------------------------------ bool cta::common::dataStructures::CancelRetrieveRequest::allFieldsSet() const { return m_archiveFileIDSet + && m_creationLogSet && m_drDataSet && m_dstURLSet && m_requesterSet; @@ -116,3 +118,21 @@ cta::common::dataStructures::Requester cta::common::dataStructures::CancelRetrie } return m_requester; } + +//------------------------------------------------------------------------------ +// setCreationLog +//------------------------------------------------------------------------------ +void cta::common::dataStructures::CancelRetrieveRequest::setCreationLog(const cta::common::dataStructures::EntryLog &creationLog) { + m_creationLog = creationLog; + m_creationLogSet = true; +} + +//------------------------------------------------------------------------------ +// getCreationLog +//------------------------------------------------------------------------------ +cta::common::dataStructures::EntryLog cta::common::dataStructures::CancelRetrieveRequest::getCreationLog() const { + if(!allFieldsSet()) { + throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the AdminHost have been set!"); + } + return m_creationLog; +} diff --git a/common/dataStructures/CancelRetrieveRequest.hpp b/common/dataStructures/CancelRetrieveRequest.hpp index 5f221a0bd992cf2a7a7c411034756b026942b8a7..d4fd4333646a8ff3d86d3291e506bdf164434000 100644 --- a/common/dataStructures/CancelRetrieveRequest.hpp +++ b/common/dataStructures/CancelRetrieveRequest.hpp @@ -24,6 +24,7 @@ #include <string> #include "common/dataStructures/DRData.hpp" +#include "common/dataStructures/EntryLog.hpp" #include "common/dataStructures/Requester.hpp" namespace cta { @@ -55,6 +56,9 @@ public: void setRequester(const cta::common::dataStructures::Requester &requester); cta::common::dataStructures::Requester getRequester() const; + + void setCreationLog(const cta::common::dataStructures::EntryLog &creationLog); + cta::common::dataStructures::EntryLog getCreationLog() const; private: @@ -76,6 +80,9 @@ private: cta::common::dataStructures::Requester m_requester; bool m_requesterSet; + cta::common::dataStructures::EntryLog m_creationLog; + bool m_creationLogSet; + }; // class CancelRetrieveRequest } // namespace dataStructures diff --git a/common/dataStructures/DeleteArchiveRequest.cpp b/common/dataStructures/DeleteArchiveRequest.cpp index 5b8bfc687d34cc674b98964abd19ebdeed583da1..eb044ed20f95b8e48e830ffe60ea931ad683ac47 100644 --- a/common/dataStructures/DeleteArchiveRequest.cpp +++ b/common/dataStructures/DeleteArchiveRequest.cpp @@ -25,6 +25,7 @@ cta::common::dataStructures::DeleteArchiveRequest::DeleteArchiveRequest() { m_archiveFileIDSet = false; m_requesterSet = false; + m_creationLogSet = false; } //------------------------------------------------------------------------------ @@ -38,6 +39,7 @@ cta::common::dataStructures::DeleteArchiveRequest::~DeleteArchiveRequest() throw //------------------------------------------------------------------------------ bool cta::common::dataStructures::DeleteArchiveRequest::allFieldsSet() const { return m_archiveFileIDSet + && m_creationLogSet && m_requesterSet; } @@ -76,3 +78,21 @@ cta::common::dataStructures::Requester cta::common::dataStructures::DeleteArchiv } return m_requester; } + +//------------------------------------------------------------------------------ +// setCreationLog +//------------------------------------------------------------------------------ +void cta::common::dataStructures::DeleteArchiveRequest::setCreationLog(const cta::common::dataStructures::EntryLog &creationLog) { + m_creationLog = creationLog; + m_creationLogSet = true; +} + +//------------------------------------------------------------------------------ +// getCreationLog +//------------------------------------------------------------------------------ +cta::common::dataStructures::EntryLog cta::common::dataStructures::DeleteArchiveRequest::getCreationLog() const { + if(!allFieldsSet()) { + throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the AdminHost have been set!"); + } + return m_creationLog; +} diff --git a/common/dataStructures/DeleteArchiveRequest.hpp b/common/dataStructures/DeleteArchiveRequest.hpp index 19c02b5c8cf997805e33e1ce7a250c454e8a05a4..33d61e8ad5763c19155cc620a0fd728aa79d5ae5 100644 --- a/common/dataStructures/DeleteArchiveRequest.hpp +++ b/common/dataStructures/DeleteArchiveRequest.hpp @@ -23,6 +23,7 @@ #include <stdint.h> #include <string> +#include "common/dataStructures/EntryLog.hpp" #include "common/dataStructures/Requester.hpp" namespace cta { @@ -48,6 +49,9 @@ public: void setRequester(const cta::common::dataStructures::Requester &requester); cta::common::dataStructures::Requester getRequester() const; + + void setCreationLog(const cta::common::dataStructures::EntryLog &creationLog); + cta::common::dataStructures::EntryLog getCreationLog() const; private: @@ -63,6 +67,9 @@ private: cta::common::dataStructures::Requester m_requester; bool m_requesterSet; + cta::common::dataStructures::EntryLog m_creationLog; + bool m_creationLogSet; + }; // class DeleteArchiveRequest } // namespace dataStructures diff --git a/common/dataStructures/ListStorageClassRequest.cpp b/common/dataStructures/ListStorageClassRequest.cpp index 4d7309fda51e03710eaeda9a773f7f9fd4c48182..b3c2392a1fad4ed617ff5a06ec86129d900bb5bc 100644 --- a/common/dataStructures/ListStorageClassRequest.cpp +++ b/common/dataStructures/ListStorageClassRequest.cpp @@ -24,6 +24,7 @@ //------------------------------------------------------------------------------ cta::common::dataStructures::ListStorageClassRequest::ListStorageClassRequest() { m_requesterSet = false; + m_creationLogSet = false; } //------------------------------------------------------------------------------ @@ -36,7 +37,8 @@ cta::common::dataStructures::ListStorageClassRequest::~ListStorageClassRequest() // allFieldsSet //------------------------------------------------------------------------------ bool cta::common::dataStructures::ListStorageClassRequest::allFieldsSet() const { - return m_requesterSet; + return m_requesterSet + && m_creationLogSet; } //------------------------------------------------------------------------------ @@ -56,3 +58,21 @@ cta::common::dataStructures::Requester cta::common::dataStructures::ListStorageC } return m_requester; } + +//------------------------------------------------------------------------------ +// setCreationLog +//------------------------------------------------------------------------------ +void cta::common::dataStructures::ListStorageClassRequest::setCreationLog(const cta::common::dataStructures::EntryLog &creationLog) { + m_creationLog = creationLog; + m_creationLogSet = true; +} + +//------------------------------------------------------------------------------ +// getCreationLog +//------------------------------------------------------------------------------ +cta::common::dataStructures::EntryLog cta::common::dataStructures::ListStorageClassRequest::getCreationLog() const { + if(!allFieldsSet()) { + throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the AdminHost have been set!"); + } + return m_creationLog; +} diff --git a/common/dataStructures/ListStorageClassRequest.hpp b/common/dataStructures/ListStorageClassRequest.hpp index d8948ed2c8b7c591abed885442dd6546d2c35105..c1b08baf643d844be2797fea851961ae7642bbc9 100644 --- a/common/dataStructures/ListStorageClassRequest.hpp +++ b/common/dataStructures/ListStorageClassRequest.hpp @@ -23,6 +23,7 @@ #include <stdint.h> #include <string> +#include "common/dataStructures/EntryLog.hpp" #include "common/dataStructures/Requester.hpp" namespace cta { @@ -45,6 +46,9 @@ public: void setRequester(const cta::common::dataStructures::Requester &requester); cta::common::dataStructures::Requester getRequester() const; + + void setCreationLog(const cta::common::dataStructures::EntryLog &creationLog); + cta::common::dataStructures::EntryLog getCreationLog() const; private: @@ -57,6 +61,9 @@ private: cta::common::dataStructures::Requester m_requester; bool m_requesterSet; + cta::common::dataStructures::EntryLog m_creationLog; + bool m_creationLogSet; + }; // class ListStorageClassRequest } // namespace dataStructures diff --git a/common/dataStructures/RetrieveRequest.cpp b/common/dataStructures/RetrieveRequest.cpp index d55893e2d3de62f6d70f010df2cda520dd12a50d..75c25aa595c7b3e3d0a52053cc8d5b74fefce3c6 100644 --- a/common/dataStructures/RetrieveRequest.cpp +++ b/common/dataStructures/RetrieveRequest.cpp @@ -29,6 +29,7 @@ cta::common::dataStructures::RetrieveRequest::RetrieveRequest() { m_drDataSet = false; m_dstURLSet = false; m_requesterSet = false; + m_creationLogSet = false; } //------------------------------------------------------------------------------ @@ -42,6 +43,7 @@ cta::common::dataStructures::RetrieveRequest::~RetrieveRequest() throw() { //------------------------------------------------------------------------------ bool cta::common::dataStructures::RetrieveRequest::allFieldsSet() const { return m_archiveFileIDSet + && m_creationLogSet && m_diskpoolNameSet && m_diskpoolThroughputSet && m_drDataSet @@ -156,3 +158,21 @@ cta::common::dataStructures::Requester cta::common::dataStructures::RetrieveRequ } return m_requester; } + +//------------------------------------------------------------------------------ +// setCreationLog +//------------------------------------------------------------------------------ +void cta::common::dataStructures::RetrieveRequest::setCreationLog(const cta::common::dataStructures::EntryLog &creationLog) { + m_creationLog = creationLog; + m_creationLogSet = true; +} + +//------------------------------------------------------------------------------ +// getCreationLog +//------------------------------------------------------------------------------ +cta::common::dataStructures::EntryLog cta::common::dataStructures::RetrieveRequest::getCreationLog() const { + if(!allFieldsSet()) { + throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the AdminHost have been set!"); + } + return m_creationLog; +} diff --git a/common/dataStructures/RetrieveRequest.hpp b/common/dataStructures/RetrieveRequest.hpp index 3ba674ffa5b74a82a3fa837c49f69e59f3f37ff6..d025121f0aef937242d5ba9e1b330d39dfaf68b3 100644 --- a/common/dataStructures/RetrieveRequest.hpp +++ b/common/dataStructures/RetrieveRequest.hpp @@ -24,6 +24,7 @@ #include <string> #include "common/dataStructures/DRData.hpp" +#include "common/dataStructures/EntryLog.hpp" #include "common/dataStructures/Requester.hpp" namespace cta { @@ -61,6 +62,9 @@ public: void setRequester(const cta::common::dataStructures::Requester &requester); cta::common::dataStructures::Requester getRequester() const; + + void setCreationLog(const cta::common::dataStructures::EntryLog &creationLog); + cta::common::dataStructures::EntryLog getCreationLog() const; private: @@ -88,6 +92,9 @@ private: cta::common::dataStructures::Requester m_requester; bool m_requesterSet; + cta::common::dataStructures::EntryLog m_creationLog; + bool m_creationLogSet; + }; // class RetrieveRequest } // namespace dataStructures diff --git a/common/dataStructures/UpdateFileInfoRequest.cpp b/common/dataStructures/UpdateFileInfoRequest.cpp index f37171ccf8eaf8f1adb2e01de34c8514e5562e39..c12fa3f610f65e7003dc5942e921f3847290ef07 100644 --- a/common/dataStructures/UpdateFileInfoRequest.cpp +++ b/common/dataStructures/UpdateFileInfoRequest.cpp @@ -27,6 +27,7 @@ cta::common::dataStructures::UpdateFileInfoRequest::UpdateFileInfoRequest() { m_drDataSet = false; m_requesterSet = false; m_storageClassSet = false; + m_creationLogSet = false; } //------------------------------------------------------------------------------ @@ -40,6 +41,7 @@ cta::common::dataStructures::UpdateFileInfoRequest::~UpdateFileInfoRequest() thr //------------------------------------------------------------------------------ bool cta::common::dataStructures::UpdateFileInfoRequest::allFieldsSet() const { return m_archiveFileIDSet + && m_creationLogSet && m_drDataSet && m_requesterSet && m_storageClassSet; @@ -116,3 +118,21 @@ std::string cta::common::dataStructures::UpdateFileInfoRequest::getStorageClass( } return m_storageClass; } + +//------------------------------------------------------------------------------ +// setCreationLog +//------------------------------------------------------------------------------ +void cta::common::dataStructures::UpdateFileInfoRequest::setCreationLog(const cta::common::dataStructures::EntryLog &creationLog) { + m_creationLog = creationLog; + m_creationLogSet = true; +} + +//------------------------------------------------------------------------------ +// getCreationLog +//------------------------------------------------------------------------------ +cta::common::dataStructures::EntryLog cta::common::dataStructures::UpdateFileInfoRequest::getCreationLog() const { + if(!allFieldsSet()) { + throw cta::exception::Exception(std::string(__FUNCTION__)+" Error: not all fields of the AdminHost have been set!"); + } + return m_creationLog; +} diff --git a/common/dataStructures/UpdateFileInfoRequest.hpp b/common/dataStructures/UpdateFileInfoRequest.hpp index 5d0d676df91927b7b55cf29466bf29c5dd8338eb..a4adb8db3c99c1d9b3605d0ef6e89115f6a50b10 100644 --- a/common/dataStructures/UpdateFileInfoRequest.hpp +++ b/common/dataStructures/UpdateFileInfoRequest.hpp @@ -24,6 +24,7 @@ #include <string> #include "common/dataStructures/DRData.hpp" +#include "common/dataStructures/EntryLog.hpp" #include "common/dataStructures/Requester.hpp" namespace cta { @@ -55,6 +56,9 @@ public: void setStorageClass(const std::string &storageClass); std::string getStorageClass() const; + + void setCreationLog(const cta::common::dataStructures::EntryLog &creationLog); + cta::common::dataStructures::EntryLog getCreationLog() const; private: @@ -76,6 +80,9 @@ private: std::string m_storageClass; bool m_storageClassSet; + cta::common::dataStructures::EntryLog m_creationLog; + bool m_creationLogSet; + }; // class UpdateFileInfoRequest } // namespace dataStructures diff --git a/common/threading/Daemon.cpp b/common/threading/Daemon.cpp index 358746784b11e1d00cbbf6ce8e6d886a3a4989d5..2d56c740e0347cf3fcc701c75ea0ac61955548d8 100644 --- a/common/threading/Daemon.cpp +++ b/common/threading/Daemon.cpp @@ -32,10 +32,7 @@ //------------------------------------------------------------------------------ // constructor //------------------------------------------------------------------------------ -cta::server::Daemon::Daemon(std::ostream &stdOut, std::ostream &stdErr, - log::Logger &log) throw(): - m_stdOut(stdOut), - m_stdErr(stdErr), +cta::server::Daemon::Daemon(log::Logger &log) throw(): m_log(log), m_foreground(false), m_commandLineHasBeenParsed(false) { @@ -47,69 +44,6 @@ cta::server::Daemon::Daemon(std::ostream &stdOut, std::ostream &stdErr, cta::server::Daemon::~Daemon() { } -//------------------------------------------------------------------------------ -// parseCommandLine -//------------------------------------------------------------------------------ -void cta::server::Daemon::parseCommandLine(int argc, - char *argv[]) { - struct ::option longopts[4]; - - longopts[0].name = "foreground"; - longopts[0].has_arg = no_argument; - longopts[0].flag = NULL; - longopts[0].val = 'f'; - - longopts[1].name = "config"; - longopts[1].has_arg = required_argument; - longopts[1].flag = NULL; - longopts[1].val = 'c'; - - longopts[2].name = "help"; - longopts[2].has_arg = no_argument; - longopts[2].flag = NULL; - longopts[2].val = 'h'; - - longopts[3].name = 0; - - char c; - while ((c = getopt_long(argc, argv, "fc:h", longopts, NULL)) != -1) { - switch (c) { - case 'f': - m_foreground = true; - break; - case 'c': - setenv("PATH_CONFIG", optarg, 1); - m_stdOut << "Using configuration file " << optarg << std::endl; - break; - case 'h': - help(argv[0]); - exit(0); - break; - default: - break; - } - } - - m_commandLineHasBeenParsed = true; -} - -//------------------------------------------------------------------------------ -// help -//------------------------------------------------------------------------------ -void cta::server::Daemon::help(const std::string &programName) - throw() { - m_stdOut << "Usage: " << programName << " [options]\n" - "\n" - "where options can be:\n" - "\n" - "\t--foreground or -f \tRemain in the Foreground\n" - "\t--config <config-file> or -c \tConfiguration file\n" - "\t--metrics or -m \tEnable metrics collection\n" - "\t--help or -h \tPrint this help and exit\n" - "\n" - "Comments to: Castor.Support@cern.ch\n"; -} - //------------------------------------------------------------------------------ // getServerName //------------------------------------------------------------------------------ diff --git a/common/threading/Daemon.hpp b/common/threading/Daemon.hpp index 542dca8a3bd372731e1b634d5041dd479c3a7214..c2af2c05bd6d1c634ef97cd5aa46529323147058 100644 --- a/common/threading/Daemon.hpp +++ b/common/threading/Daemon.hpp @@ -46,7 +46,7 @@ public: * @param stdErr Stream representing standard error. * @param log Object representing the API of the CASTOR logging system. */ - Daemon(std::ostream &stdOut, std::ostream &stdErr, log::Logger &log) + Daemon(log::Logger &log) throw(); /** @@ -54,20 +54,6 @@ public: */ virtual ~Daemon(); - /** - * Parses a command line to set the server options. - * - * @param argc The size of the command-line vector. - * @param argv The command-line vector. - */ - virtual void parseCommandLine(int argc, char *argv[]) - ; - - /** - * Prints out the online help - */ - virtual void help(const std::string &programName) throw(); - /** * Returns this server's name as used by the CASTOR logging system. */ @@ -105,16 +91,6 @@ protected: */ void daemonizeIfNotRunInForeground(const bool runAsStagerSuperuser); - /** - * Stream representing standard out. - */ - std::ostream &m_stdOut; - - /** - * Stream representing standard in. - */ - std::ostream &m_stdErr; - /** * Object representing the API of the CASTOR logging system. */ diff --git a/common/threading/DaemonTest.cpp b/common/threading/DaemonTest.cpp index c3d2b6937096e07afcc7d80b5053c144c5c1b4a3..680a32231a925c961e8c57345d4514e74ebb35e4 100644 --- a/common/threading/DaemonTest.cpp +++ b/common/threading/DaemonTest.cpp @@ -59,43 +59,10 @@ protected: }; TEST_F(cta_threading_DaemonTest, getForegroundBeforeParseCommandLine) { - std::ostringstream dummyStdOut; - std::ostringstream dummyStdErr; cta::log::DummyLogger log(m_programName); - cta::server::Daemon daemon(dummyStdOut, dummyStdErr, log); + cta::server::Daemon daemon(log); ASSERT_THROW(daemon.getForeground(), cta::server::Daemon::CommandLineNotParsed); } -TEST_F(cta_threading_DaemonTest, parseEmptyCmdLine) { - m_argv = new char *[2]; - m_argv[0] = strdup(m_programName.c_str()); - m_argv[1] = NULL; - m_argc = 1; - - std::ostringstream dummyStdOut; - std::ostringstream dummyStdErr; - cta::log::DummyLogger log(m_programName); - cta::server::Daemon daemon(dummyStdOut, dummyStdErr, log); - - ASSERT_NO_THROW(daemon.parseCommandLine(m_argc, m_argv)); - ASSERT_FALSE(daemon.getForeground()); -} - -TEST_F(cta_threading_DaemonTest, parseFOnCmdLine) { - m_argv = new char *[3]; - m_argv[0] = strdup(m_programName.c_str()); - m_argv[1] = strdup("-f"); - m_argv[2] = NULL; - m_argc = 2; - - std::ostringstream dummyStdOut; - std::ostringstream dummyStdErr; - cta::log::DummyLogger log(m_programName); - cta::server::Daemon daemon(dummyStdOut, dummyStdErr, log); - - ASSERT_NO_THROW(daemon.parseCommandLine(m_argc, m_argv)); - ASSERT_EQ(true, daemon.getForeground()); -} - } // namespace unitTests diff --git a/cta.spec.in b/cta.spec.in index e8bbce164784b5df300c6bc3eb8fc2179515a420..7da09c2597a25a74efbe2ccef74238f2f4cbcfe8 100644 --- a/cta.spec.in +++ b/cta.spec.in @@ -135,12 +135,14 @@ The shared libraries Summary: CERN Tape Archive: unit and system tests with virtual tape drives Group: Application/CTA Requires: valgrind >= 3.8.1 +Requires: cta-taped = %{ctaVersion}-%{ctaRelease}%{mydist} %description -n cta-systemtests CERN Tape Archive: Unit tests and system tests with virtual tape drives %files -n cta-systemtests %attr(0755,root,root) %{_bindir}/cta-systemTests %attr(0755,root,root) %{_libdir}/libsystemTestHelperTests.so +%attr(0755,root,root) %{_libdir}/libcta-tapedSystemTests.so %attr(0755,root,root) %{_bindir}/cta-unitTests %attr(0755,root,root) %{_bindir}/cta-valgrindUnitTests.sh %attr(0755,root,root) %{_bindir}/cta-unitPlusSystemTests.sh @@ -162,4 +164,5 @@ Unit tests and system tests with virtual tape drives %attr(0755,root,root) %{_libdir}/libctatapeserverscsiunittests.so %attr(0755,root,root) %{_libdir}/libctatapeserverutilsunittests.so %attr(0755,root,root) %{_libdir}/libctautilsunittests.so +%attr(0755,root,root) %{_libdir}/libctadaemonunittests.so %attr(0644,root,root) %{_datadir}/%{name}-%{ctaVersion}/unittest/*.suppr \ No newline at end of file diff --git a/objectstore/ArchiveRequest.cpp b/objectstore/ArchiveRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..07a638a82f756c3e79f84f3ec6f1d6abab510290 --- /dev/null +++ b/objectstore/ArchiveRequest.cpp @@ -0,0 +1,549 @@ +/* + * The CERN Tape Archive (CTA) project + * Copyright (C) 2015 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "ArchiveRequest.hpp" +#include "GenericObject.hpp" +#include "TapePool.hpp" +#include "common/dataStructures/EntryLog.hpp" +#include <json-c/json.h> + +cta::objectstore::ArchiveRequest::ArchiveRequest(const std::string& address, Backend& os): + ObjectOps<serializers::ArchiveRequest>(os, address){ } + +cta::objectstore::ArchiveRequest::ArchiveRequest(Backend& os): + ObjectOps<serializers::ArchiveRequest>(os) { } + +cta::objectstore::ArchiveRequest::ArchiveRequest(GenericObject& go): + ObjectOps<serializers::ArchiveRequest>(go.objectStore()) { + // Here we transplant the generic object into the new object + go.transplantHeader(*this); + // And interpret the header. + getPayloadFromHeader(); +} + +void cta::objectstore::ArchiveRequest::initialize() { + // Setup underlying object + ObjectOps<serializers::ArchiveRequest>::initialize(); + // This object is good to go (to storage) + m_payloadInterpreted = true; +} + +void cta::objectstore::ArchiveRequest::addJob(uint16_t copyNumber, + const std::string& tapepool, const std::string& tapepooladdress) { + checkPayloadWritable(); + auto *j = m_payload.add_jobs(); + j->set_copynb(copyNumber); + j->set_status(serializers::ArchiveJobStatus::AJS_PendingNsCreation); + j->set_tapepool(tapepool); + j->set_owner(""); + j->set_tapepooladdress(tapepooladdress); + j->set_totalretries(0); + j->set_retrieswithinmount(0); + j->set_lastmountwithfailure(0); + // Those 2 values are set to 0 as at creation time, we do not read the + // tape pools yet. + j->set_maxretrieswithinmount(0); + j->set_maxtotalretries(0); +} + +bool cta::objectstore::ArchiveRequest::setJobSuccessful(uint16_t copyNumber) { + checkPayloadWritable(); + auto * jl = m_payload.mutable_jobs(); + for (auto j=jl->begin(); j!=jl->end(); j++) { + if (j->copynb() == copyNumber) { + j->set_status(serializers::ArchiveJobStatus::AJS_Complete); + for (auto j2=jl->begin(); j2!=jl->end(); j2++) { + if (j2->status()!= serializers::ArchiveJobStatus::AJS_Complete && + j2->status()!= serializers::ArchiveJobStatus::AJS_Failed) + return false; + } + return true; + } + } + throw NoSuchJob("In ArchiveRequest::setJobSuccessful(): job not found"); +} + + +void cta::objectstore::ArchiveRequest::setJobFailureLimits(uint16_t copyNumber, + uint16_t maxRetiesWithinMount, uint16_t maxTotalRetries) { + checkPayloadWritable(); + auto * jl = m_payload.mutable_jobs(); + for (auto j=jl->begin(); j!=jl->end(); j++) { + if (j->copynb() == copyNumber) { + j->set_maxretrieswithinmount(maxRetiesWithinMount); + j->set_maxtotalretries(maxTotalRetries); + return; + } + } + throw NoSuchJob("In ArchiveRequest::setJobFailureLimits(): job not found"); +} + +bool cta::objectstore::ArchiveRequest::addJobFailure(uint16_t copyNumber, + uint64_t mountId) { + checkPayloadWritable(); + auto * jl = m_payload.mutable_jobs(); + // Find the job and update the number of failures (and return the new count) + for (auto j=jl->begin(); j!=jl->end(); j++) { + if (j->copynb() == copyNumber) { + if (j->lastmountwithfailure() == mountId) { + j->set_retrieswithinmount(j->retrieswithinmount() + 1); + } else { + j->set_retrieswithinmount(1); + j->set_lastmountwithfailure(mountId); + } + j->set_totalretries(j->totalretries() + 1); + } + if (j->totalretries() >= j->maxtotalretries()) { + j->set_status(serializers::AJS_Failed); + finishIfNecessary(); + return true; + } else { + j->set_status(serializers::AJS_PendingMount); + return false; + } + } + throw NoSuchJob ("In ArchiveRequest::addJobFailure(): could not find job"); +} + + +void cta::objectstore::ArchiveRequest::setAllJobsLinkingToTapePool() { + checkPayloadWritable(); + auto * jl=m_payload.mutable_jobs(); + for (auto j=jl->begin(); j!=jl->end(); j++) { + j->set_status(serializers::AJS_LinkingToTapePool); + } +} + +void cta::objectstore::ArchiveRequest::setAllJobsFailed() { + checkPayloadWritable(); + auto * jl=m_payload.mutable_jobs(); + for (auto j=jl->begin(); j!=jl->end(); j++) { + j->set_status(serializers::AJS_Failed); + } +} + +void cta::objectstore::ArchiveRequest::setAllJobsPendingNSdeletion() { + checkPayloadWritable(); + auto * jl=m_payload.mutable_jobs(); + for (auto j=jl->begin(); j!=jl->end(); j++) { + j->set_status(serializers::AJS_PendingNsDeletion); + } +} + +//------------------------------------------------------------------------------ +// setChecksumType +//------------------------------------------------------------------------------ +void cta::objectstore::ArchiveRequest::setChecksumType(const std::string &checksumType) { + checkPayloadWritable(); + m_payload.set_checksumtype(checksumType); +} + +//------------------------------------------------------------------------------ +// getChecksumType +//------------------------------------------------------------------------------ +std::string cta::objectstore::ArchiveRequest::getChecksumType() { + checkPayloadReadable(); + return m_payload.checksumtype(); +} + +//------------------------------------------------------------------------------ +// setChecksumValue +//------------------------------------------------------------------------------ +void cta::objectstore::ArchiveRequest::setChecksumValue(const std::string &checksumValue) { + checkPayloadWritable(); + m_payload.set_checksumvalue(checksumValue); +} + +//------------------------------------------------------------------------------ +// getChecksumValue +//------------------------------------------------------------------------------ +std::string cta::objectstore::ArchiveRequest::getChecksumValue() { + checkPayloadReadable(); + return m_payload.checksumvalue(); +} + +//------------------------------------------------------------------------------ +// setDiskpoolName +//------------------------------------------------------------------------------ +void cta::objectstore::ArchiveRequest::setDiskpoolName(const std::string &diskpoolName) { + checkPayloadWritable(); + m_payload.set_diskpoolname(diskpoolName); +} + +//------------------------------------------------------------------------------ +// getDiskpoolName +//------------------------------------------------------------------------------ +std::string cta::objectstore::ArchiveRequest::getDiskpoolName() { + checkPayloadReadable(); + return m_payload.diskpoolname(); +} + +//------------------------------------------------------------------------------ +// setDiskpoolThroughput +//------------------------------------------------------------------------------ +void cta::objectstore::ArchiveRequest::setDiskpoolThroughput(const uint64_t diskpoolThroughput) { + checkPayloadWritable(); + m_payload.set_diskpoolthroughput(diskpoolThroughput); +} + +//------------------------------------------------------------------------------ +// getDiskpoolThroughput +//------------------------------------------------------------------------------ +uint64_t cta::objectstore::ArchiveRequest::getDiskpoolThroughput() { + checkPayloadReadable(); + return m_payload.diskpoolthroughput(); +} + +//------------------------------------------------------------------------------ +// setDrData +//------------------------------------------------------------------------------ +void cta::objectstore::ArchiveRequest::setDrData(const cta::common::dataStructures::DRData &drData) { + checkPayloadWritable(); + auto payloadDrData = m_payload.mutable_drdata(); + payloadDrData->set_drblob(drData.getDrBlob()); + payloadDrData->set_drgroup(drData.getDrGroup()); + payloadDrData->set_drinstance(drData.getDrInstance()); + payloadDrData->set_drowner(drData.getDrOwner()); + payloadDrData->set_drpath(drData.getDrPath()); +} + +//------------------------------------------------------------------------------ +// getDrData +//------------------------------------------------------------------------------ +cta::common::dataStructures::DRData cta::objectstore::ArchiveRequest::getDrData() { + checkPayloadReadable(); + cta::common::dataStructures::DRData drData; + auto payloadDrData = m_payload.drdata(); + drData.setDrBlob(payloadDrData.drblob()); + drData.setDrGroup(payloadDrData.drgroup()); + drData.setDrInstance(payloadDrData.drinstance()); + drData.setDrOwner(payloadDrData.drowner()); + drData.setDrPath(payloadDrData.drpath()); + return drData; +} + +//------------------------------------------------------------------------------ +// setEosFileID +//------------------------------------------------------------------------------ +void cta::objectstore::ArchiveRequest::setEosFileID(const std::string &eosFileID) { + checkPayloadWritable(); + m_payload.set_eosfileid(eosFileID); +} + +//------------------------------------------------------------------------------ +// getEosFileID +//------------------------------------------------------------------------------ +std::string cta::objectstore::ArchiveRequest::getEosFileID() { + checkPayloadReadable(); + return m_payload.eosfileid(); +} + +//------------------------------------------------------------------------------ +// setFileSize +//------------------------------------------------------------------------------ +void cta::objectstore::ArchiveRequest::setFileSize(const uint64_t fileSize) { + checkPayloadWritable(); + m_payload.set_filesize(fileSize); +} + +//------------------------------------------------------------------------------ +// getFileSize +//------------------------------------------------------------------------------ +uint64_t cta::objectstore::ArchiveRequest::getFileSize() { + checkPayloadReadable(); + return m_payload.filesize(); +} + +//------------------------------------------------------------------------------ +// setRequester +//------------------------------------------------------------------------------ +void cta::objectstore::ArchiveRequest::setRequester(const cta::common::dataStructures::Requester &requester) { + checkPayloadWritable(); + auto payloadRequester = m_payload.mutable_requester(); + payloadRequester->set_username(requester.getUserName()); + payloadRequester->set_groupname(requester.getGroupName()); +} + +//------------------------------------------------------------------------------ +// getRequester +//------------------------------------------------------------------------------ +cta::common::dataStructures::Requester cta::objectstore::ArchiveRequest::getRequester() { + checkPayloadReadable(); + cta::common::dataStructures::Requester requester; + auto payloadRequester = m_payload.requester(); + requester.setUserName(payloadRequester.username()); + requester.setGroupName(payloadRequester.groupname()); + return requester; +} + +//------------------------------------------------------------------------------ +// setSrcURL +//------------------------------------------------------------------------------ +void cta::objectstore::ArchiveRequest::setSrcURL(const std::string &srcURL) { + checkPayloadWritable(); + m_payload.set_srcurl(srcURL); +} + +//------------------------------------------------------------------------------ +// getSrcURL +//------------------------------------------------------------------------------ +std::string cta::objectstore::ArchiveRequest::getSrcURL() { + checkPayloadReadable(); + return m_payload.srcurl(); +} + +//------------------------------------------------------------------------------ +// setStorageClass +//------------------------------------------------------------------------------ +void cta::objectstore::ArchiveRequest::setStorageClass(const std::string &storageClass) { + checkPayloadWritable(); + m_payload.set_storageclass(storageClass); +} + +//------------------------------------------------------------------------------ +// getStorageClass +//------------------------------------------------------------------------------ +std::string cta::objectstore::ArchiveRequest::getStorageClass() { + checkPayloadReadable(); + return m_payload.storageclass(); +} + +//------------------------------------------------------------------------------ +// setCreationLog +//------------------------------------------------------------------------------ +void cta::objectstore::ArchiveRequest::setCreationLog(const cta::common::dataStructures::EntryLog &creationLog) { + checkPayloadWritable(); + auto payloadCreationLog = m_payload.mutable_creationlog(); + payloadCreationLog->set_time(creationLog.getTime()); + payloadCreationLog->set_host(creationLog.getHost()); + payloadCreationLog->set_uid(creationLog.getUser().getUid()); + payloadCreationLog->set_gid(creationLog.getUser().getGid()); +} + +//------------------------------------------------------------------------------ +// getCreationLog +//------------------------------------------------------------------------------ +cta::common::dataStructures::EntryLog cta::objectstore::ArchiveRequest::getCreationLog() { + checkPayloadReadable(); + cta::common::dataStructures::EntryLog creationLog; + cta::common::dataStructures::UserIdentity user; + auto payloadCreationLog = m_payload.creationlog(); + user.setUid(payloadCreationLog.uid()); + user.setGid(payloadCreationLog.gid()); + creationLog.setUser(user); + creationLog.setHost(payloadCreationLog.host()); + creationLog.setTime(payloadCreationLog.time()); + return creationLog; +} + +auto cta::objectstore::ArchiveRequest::dumpJobs() -> std::list<JobDump> { + checkPayloadReadable(); + std::list<JobDump> ret; + auto & jl = m_payload.jobs(); + for (auto j=jl.begin(); j!=jl.end(); j++) { + ret.push_back(JobDump()); + ret.back().copyNb = j->copynb(); + ret.back().tapePool = j->tapepool(); + ret.back().tapePoolAddress = j->tapepooladdress(); + } + return ret; +} + +void cta::objectstore::ArchiveRequest::garbageCollect(const std::string &presumedOwner) { + checkPayloadWritable(); + // The behavior here depends on which job the agent is supposed to own. + // We should first find this job (if any). This is for covering the case + // of a selected job. The Request could also still being connected to tape + // pools. In this case we will finish the connection to tape pools unconditionally. + auto * jl = m_payload.mutable_jobs(); + for (auto j=jl->begin(); j!=jl->end(); j++) { + auto owner=j->owner(); + auto status=j->status(); + if (status==serializers::AJS_LinkingToTapePool || + (status==serializers::AJS_Selected && owner==presumedOwner)) { + // If the job was being connected to the tape pool or was selected + // by the dead agent, then we have to ensure it is indeed connected to + // the tape pool and set its status to pending. + // (Re)connect the job to the tape pool and make it pending. + // If we fail to reconnect, we have to fail the job and potentially + // finish the request. + try { + TapePool tp(j->tapepooladdress(), m_objectStore); + ScopedExclusiveLock tpl(tp); + tp.fetch(); + ArchiveToFileRequest::JobDump jd; + jd.copyNb = j->copynb(); + jd.tapePool = j->tapepool(); + jd.tapePoolAddress = j->tapepooladdress(); + if (tp.addJobIfNecessary(jd, getAddressIfSet(), + m_payload.drdata().drpath(), m_payload.filesize())) + tp.commit(); + j->set_status(serializers::AJS_PendingMount); + commit(); + } catch (...) { + j->set_status(serializers::AJS_Failed); + // This could be the end of the request, with various consequences. + // This is handled here: + if (finishIfNecessary()) + return; + } + } else if (status==serializers::AJS_PendingNsCreation) { + // If the job is pending NsCreation, we have to queue it in the tape pool's + // queue for files orphaned pending ns creation. Some user process will have + // to pick them up actively (recovery involves schedulerDB + NameServerDB) + try { + TapePool tp(j->tapepooladdress(), m_objectStore); + ScopedExclusiveLock tpl(tp); + tp.fetch(); + ArchiveToFileRequest::JobDump jd; + jd.copyNb = j->copynb(); + jd.tapePool = j->tapepool(); + jd.tapePoolAddress = j->tapepooladdress(); + if (tp.addOrphanedJobPendingNsCreation(jd, getAddressIfSet(), + m_payload.drdata().drpath(), m_payload.filesize())) + tp.commit(); + } catch (...) { + j->set_status(serializers::AJS_Failed); + // This could be the end of the request, with various consequences. + // This is handled here: + if (finishIfNecessary()) + return; + } + } else if (status==serializers::AJS_PendingNsDeletion) { + // If the job is pending NsDeletion, we have to queue it in the tape pool's + // queue for files orphaned pending ns deletion. Some user process will have + // to pick them up actively (recovery involves schedulerDB + NameServerDB) + try { + TapePool tp(j->tapepooladdress(), m_objectStore); + ScopedExclusiveLock tpl(tp); + tp.fetch(); + ArchiveToFileRequest::JobDump jd; + jd.copyNb = j->copynb(); + jd.tapePool = j->tapepool(); + jd.tapePoolAddress = j->tapepooladdress(); + if (tp.addOrphanedJobPendingNsCreation(jd, getAddressIfSet(), + m_payload.drdata().drpath(), m_payload.filesize())) + tp.commit(); + j->set_status(serializers::AJS_PendingMount); + commit(); + } catch (...) { + j->set_status(serializers::AJS_Failed); + // This could be the end of the request, with various consequences. + // This is handled here: + if (finishIfNecessary()) + return; + } + } else { + return; + } + } +} + +void cta::objectstore::ArchiveRequest::setJobOwner( + uint16_t copyNumber, const std::string& owner) { + checkPayloadWritable(); + // Find the right job + auto mutJobs = m_payload.mutable_jobs(); + for (auto job=mutJobs->begin(); job!=mutJobs->end(); job++) { + if (job->copynb() == copyNumber) { + job->set_owner(owner); + return; + } + } + throw NoSuchJob("In ArchiveRequest::setJobOwner: no such job"); +} + +bool cta::objectstore::ArchiveRequest::finishIfNecessary() { + checkPayloadWritable(); + // This function is typically called after changing the status of one job + // in memory. If the job is complete, we will just remove it. + // TODO: we will have to push the result to the ArchiveToDirRequest when + // it gets implemented. + // If all the jobs are either complete or failed, we can remove the request. + auto & jl=m_payload.jobs(); + for (auto j=jl.begin(); j!=jl.end(); j++) { + if (j->status() != serializers::AJS_Complete + && j->status() != serializers::AJS_Failed) { + return false; + } + } + remove(); + return true; +} + +std::string cta::objectstore::ArchiveRequest::dump() { + checkPayloadReadable(); + std::stringstream ret; + ret << "ArchiveRequest" << std::endl; + struct json_object * jo = json_object_new_object(); + json_object_object_add(jo, "checksumtype", json_object_new_string(m_payload.checksumtype().c_str())); + json_object_object_add(jo, "checksumvalue", json_object_new_string(m_payload.checksumvalue().c_str())); + json_object_object_add(jo, "diskpoolname", json_object_new_string(m_payload.diskpoolname().c_str())); + json_object_object_add(jo, "diskpoolthroughput", json_object_new_int64(m_payload.diskpoolthroughput())); + json_object_object_add(jo, "eosfileid", json_object_new_string(m_payload.eosfileid().c_str())); + json_object_object_add(jo, "filesize", json_object_new_int64(m_payload.filesize())); + json_object_object_add(jo, "srcurl", json_object_new_string(m_payload.srcurl().c_str())); + json_object_object_add(jo, "storageclass", json_object_new_string(m_payload.storageclass().c_str())); + // Object for creation log + json_object * jaf = json_object_new_object(); + json_object_object_add(jaf, "host", json_object_new_string(m_payload.creationlog().host().c_str())); + json_object_object_add(jaf, "time", json_object_new_int64(m_payload.creationlog().time())); + json_object_object_add(jaf, "uid", json_object_new_int64(m_payload.creationlog().uid())); + json_object_object_add(jaf, "gid", json_object_new_int64(m_payload.creationlog().gid())); + json_object_object_add(jo, "creationlog", jaf); + // Array for jobs + json_object * jja = json_object_new_array(); + auto & jl = m_payload.jobs(); + for (auto j=jl.begin(); j!=jl.end(); j++) { + // Object for job + json_object * jj = json_object_new_object(); + json_object_object_add(jj, "copynb", json_object_new_int64(j->copynb())); + json_object_object_add(jj, "lastmountwithfailure", json_object_new_int64(j->lastmountwithfailure())); + json_object_object_add(jj, "maxretrieswithinmount", json_object_new_int64(j->maxretrieswithinmount())); + json_object_object_add(jj, "maxtotalretries", json_object_new_int64(j->maxtotalretries())); + json_object_object_add(jj, "owner", json_object_new_string(j->owner().c_str())); + json_object_object_add(jj, "retrieswithinmount", json_object_new_int64(j->retrieswithinmount())); + json_object_object_add(jj, "status", json_object_new_int64(j->status())); + json_object_object_add(jj, "tapepool", json_object_new_string(j->tapepool().c_str())); + json_object_object_add(jj, "tapepoolAddress", json_object_new_string(j->tapepooladdress().c_str())); + json_object_object_add(jj, "totalRetries", json_object_new_int64(j->totalretries())); + json_object_array_add(jja, jj); + } + json_object_object_add(jo, "jobs", jja); + // Object for drdata + json_object * jlog = json_object_new_object(); + json_object_object_add(jlog, "drblob", json_object_new_string(m_payload.drdata().drblob().c_str())); + json_object_object_add(jlog, "drgroup", json_object_new_string(m_payload.drdata().drgroup().c_str())); + json_object_object_add(jlog, "drinstance", json_object_new_string(m_payload.drdata().drinstance().c_str())); + json_object_object_add(jlog, "drowner", json_object_new_string(m_payload.drdata().drowner().c_str())); + json_object_object_add(jlog, "drpath", json_object_new_string(m_payload.drdata().drpath().c_str())); + json_object_object_add(jo, "drdata", jlog); + // Object for requester + json_object * jrf = json_object_new_object(); + json_object_object_add(jrf, "username", json_object_new_string(m_payload.requester().username().c_str())); + json_object_object_add(jrf, "groupname", json_object_new_string(m_payload.requester().groupname().c_str())); + json_object_object_add(jo, "requester", jrf); + ret << json_object_to_json_string_ext(jo, JSON_C_TO_STRING_PRETTY) << std::endl; + json_object_put(jo); + return ret.str(); +} + + + + + diff --git a/objectstore/ArchiveRequest.hpp b/objectstore/ArchiveRequest.hpp new file mode 100644 index 0000000000000000000000000000000000000000..aa5a7921abf151828b1a906a85c10fa5297860cb --- /dev/null +++ b/objectstore/ArchiveRequest.hpp @@ -0,0 +1,113 @@ + +/* + * The CERN Tape Archive (CTA) project + * Copyright (C) 2015 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#pragma once + +#include "common/dataStructures/DRData.hpp" +#include "common/dataStructures/EntryLog.hpp" +#include "common/dataStructures/Requester.hpp" +#include "ObjectOps.hpp" +#include "objectstore/cta.pb.h" +#include <list> + +namespace cta { namespace objectstore { + +class Backend; +class Agent; +class GenericObject; +class CreationLog; + +class ArchiveRequest: public ObjectOps<serializers::ArchiveRequest> { +public: + ArchiveRequest(const std::string & address, Backend & os); + ArchiveRequest(Backend & os); + ArchiveRequest(GenericObject & go); + void initialize(); + // Job management ============================================================ + void addJob(uint16_t copyNumber, const std::string & tapepool, + const std::string & tapepooladdress); + void setJobFailureLimits(uint16_t copyNumber, + uint16_t maxRetiesWithinMount, uint16_t maxTotalRetries); + void setJobSelected(uint16_t copyNumber, const std::string & owner); + void setJobPending(uint16_t copyNumber); + bool setJobSuccessful(uint16_t copyNumber); //< returns true if this is the last job + bool addJobFailure(uint16_t copyNumber, uint64_t sessionId); //< returns true the job failed + serializers::ArchiveJobStatus getJobStatus(uint16_t copyNumber); + // Handling of the consequences of a job status change for the entire request. + // This function returns true if the request got finished. + bool finishIfNecessary(); + // Mark all jobs as pending mount (following their linking to a tape pool) + void setAllJobsLinkingToTapePool(); + // Mark all the jobs as being deleted, in case of a cancellation + void setAllJobsFailed(); + // Mark all the jobs as pending deletion from NS. + void setAllJobsPendingNSdeletion(); + CTA_GENERATE_EXCEPTION_CLASS(NoSuchJob); + // Set a job ownership + void setJobOwner(uint16_t copyNumber, const std::string & owner); + // Request management ======================================================== + void setSuccessful(); + void setFailed(); + // =========================================================================== + void setChecksumType(const std::string &checksumType); + std::string getChecksumType(); + + void setChecksumValue(const std::string &checksumValue); + std::string getChecksumValue(); + + void setDiskpoolName(const std::string &diskpoolName); + std::string getDiskpoolName(); + + void setDiskpoolThroughput(const uint64_t diskpoolThroughput); + uint64_t getDiskpoolThroughput(); + + void setDrData(const cta::common::dataStructures::DRData &drData); + cta::common::dataStructures::DRData getDrData(); + + void setEosFileID(const std::string &eosFileID); + std::string getEosFileID(); + + void setFileSize(const uint64_t fileSize); + uint64_t getFileSize(); + + void setRequester(const cta::common::dataStructures::Requester &requester); + cta::common::dataStructures::Requester getRequester(); + + void setSrcURL(const std::string &srcURL); + std::string getSrcURL(); + + void setStorageClass(const std::string &storageClass); + std::string getStorageClass(); + + void setCreationLog(const cta::common::dataStructures::EntryLog &creationLog); + cta::common::dataStructures::EntryLog getCreationLog(); + + class JobDump { + public: + uint16_t copyNb; + std::string tapePool; + std::string tapePoolAddress; + }; + + std::list<JobDump> dumpJobs(); + void garbageCollect(const std::string &presumedOwner); + std::string dump(); +}; + +}} diff --git a/objectstore/CMakeLists.txt b/objectstore/CMakeLists.txt index bd4b3795a1cfcd6b16ace63d0916350fa0a4821d..ff397125dc9e95439f24c67184107fdceca51742 100644 --- a/objectstore/CMakeLists.txt +++ b/objectstore/CMakeLists.txt @@ -13,10 +13,12 @@ PROTOBUF_GENERATE_CPP(CTAProtoSources CTAProtoHeaders ${CTAProtoFiles}) set (CTAProtoDependants objectstore/Agent.hpp objectstore/ArchiveToFileRequest.hpp + objectstore/ArchiveRequest.hpp objectstore/CreationLog.hpp objectstore/GenericObject.hpp objectstore/ObjectOps.cpp objectstore/ObjectOps.hpp + objectstore/RetrieveRequest.hpp objectstore/RetrieveToFileRequest.cpp objectstore/RetrieveToFileRequest.hpp objectstore/RootEntry.hpp @@ -39,7 +41,9 @@ add_library (ctaobjectstore SHARED TapePool.cpp Tape.cpp ArchiveToFileRequest.cpp + ArchiveRequest.cpp RetrieveToFileRequest.cpp + RetrieveRequest.cpp DriveRegister.cpp BackendVFS.cpp BackendRados.cpp diff --git a/objectstore/ObjectOps.cpp b/objectstore/ObjectOps.cpp index 1eaab5ef8902fe5de09b34a3a5a10359f7c8445b..852c8782e35abcc264476205a44a4a3fcacfee6f 100644 --- a/objectstore/ObjectOps.cpp +++ b/objectstore/ObjectOps.cpp @@ -32,7 +32,9 @@ namespace cta { namespace objectstore { MAKE_CTA_OBJECTSTORE_OBJECTOPS_TYPEID(DriveRegister); MAKE_CTA_OBJECTSTORE_OBJECTOPS_TYPEID(Tape); MAKE_CTA_OBJECTSTORE_OBJECTOPS_TYPEID(ArchiveToFileRequest); + MAKE_CTA_OBJECTSTORE_OBJECTOPS_TYPEID(ArchiveRequest); MAKE_CTA_OBJECTSTORE_OBJECTOPS_TYPEID(RetrieveToFileRequest); + MAKE_CTA_OBJECTSTORE_OBJECTOPS_TYPEID(RetrieveRequest); MAKE_CTA_OBJECTSTORE_OBJECTOPS_TYPEID(SchedulerGlobalLock); #undef MAKE_CTA_OBJECTSTORE_OBJECTOPS_TYPEID diff --git a/objectstore/RetrieveRequest.cpp b/objectstore/RetrieveRequest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b9fbff8a1f03717264c6691e8dc8f4fe16a5b5d2 --- /dev/null +++ b/objectstore/RetrieveRequest.cpp @@ -0,0 +1,300 @@ +/* + * The CERN Tape Archive (CTA) project + * Copyright (C) 2015 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "RetrieveRequest.hpp" +#include "GenericObject.hpp" +#include "CreationLog.hpp" +#include "objectstore/cta.pb.h" +#include <json-c/json.h> + +cta::objectstore::RetrieveRequest::RetrieveRequest( + const std::string& address, Backend& os): + ObjectOps<serializers::RetrieveRequest>(os, address) { } + +cta::objectstore::RetrieveRequest::RetrieveRequest(GenericObject& go): + ObjectOps<serializers::RetrieveRequest>(go.objectStore()) { + // Here we transplant the generic object into the new object + go.transplantHeader(*this); + // And interpret the header. + getPayloadFromHeader(); +} + +void cta::objectstore::RetrieveRequest::initialize() { + // Setup underlying object + ObjectOps<serializers::RetrieveRequest>::initialize(); + // This object is good to go (to storage) + m_payloadInterpreted = true; +} + +void cta::objectstore::RetrieveRequest::addJob(const cta::TapeFileLocation & tapeFileLocation, + const std::string& tapeaddress) { + checkPayloadWritable(); + auto *j = m_payload.add_jobs(); + j->set_copynb(tapeFileLocation.copyNb); + j->set_status(serializers::RetrieveJobStatus::RJS_LinkingToTape); + j->set_tape(tapeFileLocation.vid); + j->set_tapeaddress(tapeaddress); + j->set_totalretries(0); + j->set_retrieswithinmount(0); + j->set_blockid(tapeFileLocation.blockId); + j->set_fseq(tapeFileLocation.fSeq); +} + +bool cta::objectstore::RetrieveRequest::setJobSuccessful(uint16_t copyNumber) { + checkPayloadWritable(); + auto * jl = m_payload.mutable_jobs(); + for (auto j=jl->begin(); j!=jl->end(); j++) { + if (j->copynb() == copyNumber) { + j->set_status(serializers::RetrieveJobStatus::RJS_Complete); + for (auto j2=jl->begin(); j2!=jl->end(); j2++) { + if (j2->status()!= serializers::RetrieveJobStatus::RJS_Complete && + j2->status()!= serializers::RetrieveJobStatus::RJS_Failed) + return false; + } + return true; + } + } + throw NoSuchJob("In RetrieveRequest::setJobSuccessful(): job not found"); +} + + +//------------------------------------------------------------------------------ +// setArchiveFileID +//------------------------------------------------------------------------------ +void cta::objectstore::RetrieveRequest::setArchiveFileID(const uint64_t archiveFileID) { + checkPayloadWritable(); + m_payload.set_archivefileid(archiveFileID); +} + +//------------------------------------------------------------------------------ +// getArchiveFileID +//------------------------------------------------------------------------------ +uint64_t cta::objectstore::RetrieveRequest::getArchiveFileID() { + checkPayloadReadable(); + return m_payload.archivefileid(); +} + +//------------------------------------------------------------------------------ +// setDiskpoolName +//------------------------------------------------------------------------------ +void cta::objectstore::RetrieveRequest::setDiskpoolName(const std::string &diskpoolName) { + checkPayloadWritable(); + m_payload.set_diskpoolname(diskpoolName); +} + +//------------------------------------------------------------------------------ +// getDiskpoolName +//------------------------------------------------------------------------------ +std::string cta::objectstore::RetrieveRequest::getDiskpoolName() { + checkPayloadReadable(); + return m_payload.diskpoolname(); +} + +//------------------------------------------------------------------------------ +// setDiskpoolThroughput +//------------------------------------------------------------------------------ +void cta::objectstore::RetrieveRequest::setDiskpoolThroughput(const uint64_t diskpoolThroughput) { + checkPayloadWritable(); + m_payload.set_diskpoolthroughput(diskpoolThroughput); +} + +//------------------------------------------------------------------------------ +// getDiskpoolThroughput +//------------------------------------------------------------------------------ +uint64_t cta::objectstore::RetrieveRequest::getDiskpoolThroughput() { + checkPayloadReadable(); + return m_payload.diskpoolthroughput(); +} + +//------------------------------------------------------------------------------ +// setDrData +//------------------------------------------------------------------------------ +void cta::objectstore::RetrieveRequest::setDrData(const cta::common::dataStructures::DRData &drData) { + checkPayloadWritable(); + auto payloadDrData = m_payload.mutable_drdata(); + payloadDrData->set_drblob(drData.getDrBlob()); + payloadDrData->set_drgroup(drData.getDrGroup()); + payloadDrData->set_drinstance(drData.getDrInstance()); + payloadDrData->set_drowner(drData.getDrOwner()); + payloadDrData->set_drpath(drData.getDrPath()); +} + +//------------------------------------------------------------------------------ +// getDrData +//------------------------------------------------------------------------------ +cta::common::dataStructures::DRData cta::objectstore::RetrieveRequest::getDrData() { + checkPayloadReadable(); + cta::common::dataStructures::DRData drData; + auto payloadDrData = m_payload.drdata(); + drData.setDrBlob(payloadDrData.drblob()); + drData.setDrGroup(payloadDrData.drgroup()); + drData.setDrInstance(payloadDrData.drinstance()); + drData.setDrOwner(payloadDrData.drowner()); + drData.setDrPath(payloadDrData.drpath()); + return drData; +} + +//------------------------------------------------------------------------------ +// setDstURL +//------------------------------------------------------------------------------ +void cta::objectstore::RetrieveRequest::setDstURL(const std::string &dstURL) { + checkPayloadWritable(); + m_payload.set_dsturl(dstURL); +} + +//------------------------------------------------------------------------------ +// getDstURL +//------------------------------------------------------------------------------ +std::string cta::objectstore::RetrieveRequest::getDstURL() { + checkPayloadReadable(); + return m_payload.dsturl(); +} + +//------------------------------------------------------------------------------ +// setRequester +//------------------------------------------------------------------------------ +void cta::objectstore::RetrieveRequest::setRequester(const cta::common::dataStructures::Requester &requester) { + checkPayloadWritable(); + auto payloadRequester = m_payload.mutable_requester(); + payloadRequester->set_username(requester.getUserName()); + payloadRequester->set_groupname(requester.getGroupName()); +} + +//------------------------------------------------------------------------------ +// getRequester +//------------------------------------------------------------------------------ +cta::common::dataStructures::Requester cta::objectstore::RetrieveRequest::getRequester() { + checkPayloadReadable(); + cta::common::dataStructures::Requester requester; + auto payloadRequester = m_payload.requester(); + requester.setUserName(payloadRequester.username()); + requester.setGroupName(payloadRequester.groupname()); + return requester; +} + +//------------------------------------------------------------------------------ +// setCreationLog +//------------------------------------------------------------------------------ +void cta::objectstore::RetrieveRequest::setCreationLog(const cta::common::dataStructures::EntryLog &creationLog) { + checkPayloadWritable(); + auto payloadCreationLog = m_payload.mutable_creationlog(); + payloadCreationLog->set_time(creationLog.getTime()); + payloadCreationLog->set_host(creationLog.getHost()); + payloadCreationLog->set_uid(creationLog.getUser().getUid()); + payloadCreationLog->set_gid(creationLog.getUser().getGid()); +} + +//------------------------------------------------------------------------------ +// getCreationLog +//------------------------------------------------------------------------------ +cta::common::dataStructures::EntryLog cta::objectstore::RetrieveRequest::getCreationLog() { + checkPayloadReadable(); + cta::common::dataStructures::EntryLog creationLog; + cta::common::dataStructures::UserIdentity user; + auto payloadCreationLog = m_payload.creationlog(); + user.setUid(payloadCreationLog.uid()); + user.setGid(payloadCreationLog.gid()); + creationLog.setUser(user); + creationLog.setHost(payloadCreationLog.host()); + creationLog.setTime(payloadCreationLog.time()); + return creationLog; +} + +auto cta::objectstore::RetrieveRequest::dumpJobs() -> std::list<JobDump> { + checkPayloadReadable(); + std::list<JobDump> ret; + auto & jl = m_payload.jobs(); + for (auto j=jl.begin(); j!=jl.end(); j++) { + ret.push_back(JobDump()); + ret.back().copyNb = j->copynb(); + ret.back().tape = j->tape(); + ret.back().tapeAddress = j->tapeaddress(); + } + return ret; +} + +auto cta::objectstore::RetrieveRequest::getJob(uint16_t copyNb) -> JobDump { + checkPayloadReadable(); + // find the job + auto & jl = m_payload.jobs(); + for (auto j=jl.begin(); j!=jl.end(); j++) { + if (j->copynb() == copyNb) { + JobDump ret; + ret.blockid = j->blockid(); + ret.copyNb = j->copynb(); + ret.fseq = j->fseq(); + ret.tape = j->tape(); + ret.tapeAddress = j->tapeaddress(); + return ret; + } + } + throw NoSuchJob("In objectstore::RetrieveRequest::getJob(): job not found for this copyNb"); +} + +std::string cta::objectstore::RetrieveRequest::dump() { + checkPayloadReadable(); + std::stringstream ret; + ret << "RetrieveRequest" << std::endl; + struct json_object * jo = json_object_new_object(); + json_object_object_add(jo, "archivefileid", json_object_new_int64(m_payload.archivefileid())); + json_object_object_add(jo, "dsturl", json_object_new_string(m_payload.dsturl().c_str())); + json_object_object_add(jo, "diskpoolname", json_object_new_string(m_payload.diskpoolname().c_str())); + json_object_object_add(jo, "diskpoolthroughput", json_object_new_int64(m_payload.diskpoolthroughput())); + // Object for creation log + json_object * jaf = json_object_new_object(); + json_object_object_add(jaf, "host", json_object_new_string(m_payload.creationlog().host().c_str())); + json_object_object_add(jaf, "time", json_object_new_int64(m_payload.creationlog().time())); + json_object_object_add(jaf, "uid", json_object_new_int64(m_payload.creationlog().uid())); + json_object_object_add(jaf, "gid", json_object_new_int64(m_payload.creationlog().gid())); + json_object_object_add(jo, "creationlog", jaf); + // Array for jobs + json_object * jja = json_object_new_array(); + auto & jl = m_payload.jobs(); + for (auto j=jl.begin(); j!=jl.end(); j++) { + // Object for job + json_object * jj = json_object_new_object(); + + json_object_object_add(jj, "copynb", json_object_new_int(j->copynb())); + json_object_object_add(jj, "retrieswithinmount", json_object_new_int(j->retrieswithinmount())); + json_object_object_add(jj, "totalretries", json_object_new_int(j->totalretries())); + json_object_object_add(jj, "status", json_object_new_int64(j->status())); + json_object_object_add(jj, "fseq", json_object_new_int64(j->fseq())); + json_object_object_add(jj, "blockid", json_object_new_int64(j->blockid())); + json_object_object_add(jj, "tape", json_object_new_string(j->tape().c_str())); + json_object_object_add(jj, "tapeAddress", json_object_new_string(j->tapeaddress().c_str())); + json_object_array_add(jja, jj); + } + json_object_object_add(jo, "jobs", jja); + // Object for drdata + json_object * jlog = json_object_new_object(); + json_object_object_add(jlog, "drblob", json_object_new_string(m_payload.drdata().drblob().c_str())); + json_object_object_add(jlog, "drgroup", json_object_new_string(m_payload.drdata().drgroup().c_str())); + json_object_object_add(jlog, "drinstance", json_object_new_string(m_payload.drdata().drinstance().c_str())); + json_object_object_add(jlog, "drowner", json_object_new_string(m_payload.drdata().drowner().c_str())); + json_object_object_add(jlog, "drpath", json_object_new_string(m_payload.drdata().drpath().c_str())); + json_object_object_add(jo, "drdata", jlog); + // Object for requester + json_object * jrf = json_object_new_object(); + json_object_object_add(jrf, "username", json_object_new_string(m_payload.requester().username().c_str())); + json_object_object_add(jrf, "groupname", json_object_new_string(m_payload.requester().groupname().c_str())); + json_object_object_add(jo, "requester", jrf); + ret << json_object_to_json_string_ext(jo, JSON_C_TO_STRING_PRETTY) << std::endl; + json_object_put(jo); + return ret.str(); +} + diff --git a/objectstore/RetrieveRequest.hpp b/objectstore/RetrieveRequest.hpp new file mode 100644 index 0000000000000000000000000000000000000000..7b7f241beb18dc537653ab7e15a14f7e4cf49d7e --- /dev/null +++ b/objectstore/RetrieveRequest.hpp @@ -0,0 +1,103 @@ +/* + * The CERN Tape Archive (CTA) project + * Copyright (C) 2015 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#pragma once + +#include "ObjectOps.hpp" +#include "objectstore/cta.pb.h" +#include <list> +#include "common/dataStructures/DRData.hpp" +#include "common/dataStructures/EntryLog.hpp" +#include "common/dataStructures/Requester.hpp" +#include "common/archiveNS/TapeFileLocation.hpp" + +namespace cta { namespace objectstore { + +class Backend; +class Agent; +class GenericObject; +class CreationLog; + +class RetrieveRequest: public ObjectOps<serializers::RetrieveRequest> { +public: + RetrieveRequest(const std::string & address, Backend & os); + RetrieveRequest(GenericObject & go); + void initialize(); + // Job management ============================================================ + void addJob(const cta::TapeFileLocation & tapeFileLocation, + const std::string & tapeaddress); + void setJobFailureLimits(uint16_t copyNumber, + uint16_t maxRetiesWithinMount, uint16_t maxTotalRetries); + void setJobSelected(uint16_t copyNumber, const std::string & owner); + void setJobPending(uint16_t copyNumber); + bool setJobSuccessful(uint16_t copyNumber); //< returns true if this is the last job + class JobDump { + public: + uint16_t copyNb; + std::string tape; + std::string tapeAddress; + uint64_t fseq; + uint64_t blockid; + }; + JobDump getJob(uint16_t copyNb); + struct FailuresCount { + uint16_t failuresWithinMount; + uint16_t totalFailures; + }; + FailuresCount addJobFailure(uint16_t copyNumber, uint64_t sessionId); + serializers::RetrieveJobStatus getJobStatus(uint16_t copyNumber); + // Handling of the consequences of a job status. This is simpler that archival + // as one finish is enough. + void finish(); + // Mark all jobs as pending mount (following their linking to a tape pool) + void setAllJobsLinkingToTapePool(); + // Mark all the jobs as being deleted, in case of a cancellation + void setAllJobsFailed(); + // Mark all the jobs as pending deletion from NS. + void setAllJobsPendingNSdeletion(); + CTA_GENERATE_EXCEPTION_CLASS(NoSuchJob); + // Request management ======================================================== + void setSuccessful(); + void setFailed(); + // =========================================================================== + void setArchiveFileID(const uint64_t archiveFileID); + uint64_t getArchiveFileID(); + + void setDiskpoolName(const std::string &diskpoolName); + std::string getDiskpoolName(); + + void setDiskpoolThroughput(const uint64_t diskpoolThroughput); + uint64_t getDiskpoolThroughput(); + + void setDrData(const cta::common::dataStructures::DRData &drData); + cta::common::dataStructures::DRData getDrData(); + + void setDstURL(const std::string &dstURL); + std::string getDstURL(); + + void setRequester(const cta::common::dataStructures::Requester &requester); + cta::common::dataStructures::Requester getRequester(); + + void setCreationLog(const cta::common::dataStructures::EntryLog &creationLog); + cta::common::dataStructures::EntryLog getCreationLog(); + // =========================================================================== + std::list<JobDump> dumpJobs(); + std::string dump(); +}; + +}} diff --git a/objectstore/TapePool.cpp b/objectstore/TapePool.cpp index e8529ca32f63510ad4de4eacbbcec48cd2c3887a..0548f9bc5333751ffc3ab6f6de31fa5bd4c8b927 100644 --- a/objectstore/TapePool.cpp +++ b/objectstore/TapePool.cpp @@ -345,6 +345,29 @@ void cta::objectstore::TapePool::addJob(const ArchiveToFileRequest::JobDump& job j->set_copynb(job.copyNb); } +void cta::objectstore::TapePool::addJob(const ArchiveRequest::JobDump& job, + const std::string & archiveToFileAddress, const std::string & path, + uint64_t size, uint64_t priority, time_t startTime) { + checkPayloadWritable(); + // The tape pool gets the highest priority of its jobs + if (m_payload.pendingarchivejobs_size()) { + if (priority > m_payload.priority()) + m_payload.set_priority(priority); + if ((uint64_t)startTime < m_payload.oldestjobcreationtime()) + m_payload.set_oldestjobcreationtime(startTime); + m_payload.set_archivejobstotalsize(m_payload.archivejobstotalsize() + size); + } else { + m_payload.set_priority(priority); + m_payload.set_archivejobstotalsize(size); + m_payload.set_oldestjobcreationtime(startTime); + } + auto * j = m_payload.add_pendingarchivejobs(); + j->set_address(archiveToFileAddress); + j->set_size(size); + j->set_path(path); + j->set_copynb(job.copyNb); +} + auto cta::objectstore::TapePool::getJobsSummary() -> JobsSummary { checkPayloadReadable(); JobsSummary ret; @@ -373,6 +396,24 @@ bool cta::objectstore::TapePool::addJobIfNecessary( return true; } +bool cta::objectstore::TapePool::addJobIfNecessary( + const ArchiveRequest::JobDump& job, + const std::string& archiveToFileAddress, + const std::string & path, uint64_t size) { + checkPayloadWritable(); + auto & jl=m_payload.pendingarchivejobs(); + for (auto j=jl.begin(); j!= jl.end(); j++) { + if (j->address() == archiveToFileAddress) + return false; + } + auto * j = m_payload.add_pendingarchivejobs(); + j->set_address(archiveToFileAddress); + j->set_size(size); + j->set_path(path); + j->set_copynb(job.copyNb); + return true; +} + void cta::objectstore::TapePool::removeJob(const std::string& archiveToFileAddress) { checkPayloadWritable(); auto * jl=m_payload.mutable_pendingarchivejobs(); @@ -445,6 +486,42 @@ bool cta::objectstore::TapePool::addOrphanedJobPendingNsDeletion( return true; } +bool cta::objectstore::TapePool::addOrphanedJobPendingNsCreation( + const ArchiveRequest::JobDump& job, + const std::string& archiveToFileAddress, + const std::string & path, + uint64_t size) { + checkPayloadWritable(); + auto & jl=m_payload.orphanedarchivejobsnscreation(); + for (auto j=jl.begin(); j!= jl.end(); j++) { + if (j->address() == archiveToFileAddress) + return false; + } + auto * j = m_payload.add_orphanedarchivejobsnscreation(); + j->set_address(archiveToFileAddress); + j->set_size(size); + j->set_path(path); + j->set_copynb(job.copyNb); + return true; +} + +bool cta::objectstore::TapePool::addOrphanedJobPendingNsDeletion( + const ArchiveRequest::JobDump& job, + const std::string& archiveToFileAddress, + const std::string & path, uint64_t size) { + checkPayloadWritable(); + auto & jl=m_payload.orphanedarchivejobsnsdeletion(); + for (auto j=jl.begin(); j!= jl.end(); j++) { + if (j->address() == archiveToFileAddress) + return false; + } + auto * j = m_payload.add_orphanedarchivejobsnsdeletion(); + j->set_address(archiveToFileAddress); + j->set_size(size); + j->set_path(path); + return true; +} + cta::MountCriteriaByDirection cta::objectstore::TapePool::getMountCriteriaByDirection() { MountCriteriaByDirection ret; diff --git a/objectstore/TapePool.hpp b/objectstore/TapePool.hpp index 91c45852f2deb08215fb9795b91c760aa7d45e5d..37d8508a2c262bb87663b3918685327634a5e6b3 100644 --- a/objectstore/TapePool.hpp +++ b/objectstore/TapePool.hpp @@ -25,6 +25,7 @@ #include "common/CreationLog.hpp" #include "common/MountControl.hpp" #include "ArchiveToFileRequest.hpp" +#include "ArchiveRequest.hpp" #include "CreationLog.hpp" #include "Agent.hpp" #include "common/archiveNS/Tape.hpp" @@ -96,6 +97,25 @@ public: const std::string& archiveToFileAddress, const std::string & path, uint64_t size); + void addJob(const ArchiveRequest::JobDump & job, + const std::string & archiveToFileAddress, const std::string & path, + uint64_t size, uint64_t priority, time_t startTime); + /// This version will check for existence of the job in the queue before + // returns true if a new job was actually inserted. + bool addJobIfNecessary(const ArchiveRequest::JobDump & job, + const std::string & archiveToFileAddress, + const std::string & path, uint64_t size); + /// This version will check for existence of the job in the queue before + // returns true if a new job was actually inserted. + bool addOrphanedJobPendingNsCreation(const ArchiveRequest::JobDump& job, + const std::string& archiveToFileAddress, const std::string & path, + uint64_t size); + /// This version will check for existence of the job in the queue before + // returns true if a new job was actually inserted. + bool addOrphanedJobPendingNsDeletion(const ArchiveRequest::JobDump& job, + const std::string& archiveToFileAddress, + const std::string & path, uint64_t size); + struct JobsSummary { uint64_t files; uint64_t bytes; diff --git a/objectstore/cta.proto b/objectstore/cta.proto index 3a60107bc36ebde69220ea0cc044bbb790e3ca47..f7cf62fe27ea5e0fb005f1aaacf9baaab2289ec3 100644 --- a/objectstore/cta.proto +++ b/objectstore/cta.proto @@ -12,6 +12,8 @@ enum ObjectType { ArchiveToFileRequest_t = 6; RetrieveToFileRequest_t = 7; SchedulerGlobalLock_t = 8; + ArchiveRequest_t = 9; + RetrieveRequest_t = 10; GenericObject_t = 1000; } @@ -284,8 +286,8 @@ enum ArchiveJobStatus { AJS_LinkingToTapePool = 1; AJS_PendingMount = 2; AJS_PendingNsDeletion = 99; - AJS_Selected = 2; - AJS_Complete = 3; + AJS_Selected = 3; + AJS_Complete = 4; AJS_Failed = 999; } @@ -404,4 +406,50 @@ message SchedulerGlobalLock { required uint64 nextmountid = 8000; } +// ------------- New interface ------------------------------------------------ +message Requester { + required string groupName = 8800; + required string userName = 8810; +} + +message DRData { + required string drBlob = 8900; + required string drGroup = 8910; + required string drInstance = 8920; + required string drOwner = 8930; + required string drPath = 8940; +} + +message EntryLog { + required uint64 uid = 8950; + required uint64 gid = 8955; + required string host = 8960; + required uint64 time = 8970; +} + +message ArchiveRequest { + required string checksumtype = 9000; + required string checksumValue = 9010; + required string diskpoolName = 9020; + required uint64 diskpoolThroughput = 9030; + required DRData drData = 9040; + required string eosFileID = 9050; + required uint64 fileSize = 9060; + required Requester requester = 9070; + required string srcURL = 9080; + required string storageClass = 9090; + required EntryLog creationLog = 9091; + repeated ArchiveJobEntry jobs = 9092; +} + +message RetrieveRequest { + required uint64 archiveFileID = 9100; + required string diskpoolName = 9110; + required uint64 diskpoolThroughput = 9120; + required DRData drData = 9130; + required string dstURL = 9140; + required Requester requester = 9150; + required EntryLog creationLog = 9151; + repeated RetrieveJobEntry jobs = 9152; +} diff --git a/scheduler/OStoreDB/OStoreDB.cpp b/scheduler/OStoreDB/OStoreDB.cpp index a5d8fa6cea75b2af9aac5515226cb6b3673674ef..657d7bb9c44396e6640a23c43a21f850fe170656 100644 --- a/scheduler/OStoreDB/OStoreDB.cpp +++ b/scheduler/OStoreDB/OStoreDB.cpp @@ -39,6 +39,7 @@ #include "common/archiveNS/TapeFileLocation.hpp" #include "ArchiveToTapeCopyRequest.hpp" #include "common/archiveNS/ArchiveFile.hpp" +#include "objectstore/ArchiveRequest.hpp" #include <algorithm> #include <stdlib.h> /* srand, rand */ #include <time.h> /* time */ @@ -368,6 +369,109 @@ OStoreDB::ArchiveToFileRequestCreation::~ArchiveToFileRequestCreation() { } catch (...) {} } +void OStoreDB::ArchiveRequestCreation::complete() { + // We inherited all the objects from the creation. + // Lock is still here at that point. + // First, record that we are fine for next step. + m_request.setAllJobsLinkingToTapePool(); + m_request.commit(); + objectstore::RootEntry re(m_objectStore); + // We can now plug the request onto its tape pools. + // We can discover at that point that a tape pool is actually not + // really owned by the root entry, and hence a dangling pointer + // We should then unlink the jobs from that already connected + // tape pools and abort the job creation. + // The list of done tape pools is held here for this purpose + // Reconstruct the job list + auto jl = m_request.dumpJobs(); + std::list<std::string> linkedTapePools; + try { + for (auto j=jl.begin(); j!=jl.end(); j++) { + objectstore::TapePool tp(j->tapePoolAddress, m_objectStore); + ScopedExclusiveLock tpl(tp); + tp.fetch(); + if (tp.getOwner() != re.getAddressIfSet()) + throw NoSuchTapePool("In OStoreDB::queue: non-existing tape pool found " + "(dangling pointer): cancelling request creation."); + tp.addJob(*j, m_request.getAddressIfSet(), m_request.getDrData().getDrPath(), + m_request.getFileSize(), 0, //TODO: fix priorities and mount criteria to come from usergroups + m_request.getCreationLog().getTime()); + // Now that we have the tape pool handy, get the retry limits from it and + // assign them to the job + m_request.setJobFailureLimits(j->copyNb, tp.getMaxRetriesWithinMount(), + tp.getMaxTotalRetries()); + tp.commit(); + linkedTapePools.push_back(j->tapePoolAddress); + } + } catch (NoSuchTapePool &) { + // Unlink the request from already connected tape pools + for (auto tpa=linkedTapePools.begin(); tpa!=linkedTapePools.end(); tpa++) { + objectstore::TapePool tp(*tpa, m_objectStore); + ScopedExclusiveLock tpl(tp); + tp.fetch(); + tp.removeJob(m_request.getAddressIfSet()); + tp.commit(); + m_request.remove(); + } + throw; + } + // The request is now fully set. As it's multi-owned, we do not set the owner, + // just to disown it from the agent. + m_request.setOwner(""); + m_request.commit(); + m_lock.release(); + // And remove reference from the agent + { + objectstore::ScopedExclusiveLock al(*m_agent); + m_agent->fetch(); + m_agent->removeFromOwnership(m_request.getAddressIfSet()); + m_agent->commit(); + } + m_closed=true; + return; +} + +void OStoreDB::ArchiveRequestCreation::cancel() { + // We inherited everything from the creation, and all we have to + // do here is to delete the request from storage and dereference it from + // the agent's entry + if (m_closed) { + throw ArchiveRequestAlreadyCompleteOrCanceled( + "In OStoreDB::ArchiveToFileRequestCreation::cancel: trying the close " + "the request creation twice"); + } + m_request.remove(); + { + objectstore::ScopedExclusiveLock al(*m_agent); + m_agent->fetch(); + m_agent->removeFromOwnership(m_request.getAddressIfSet()); + m_agent->commit(); + } + m_closed=true; + return; +} + +OStoreDB::ArchiveRequestCreation::~ArchiveRequestCreation() { + // We have to determine whether complete() or cancel() were called, in which + // case there is nothing to do, or not, in which case we have to garbage + // collect the archive to file request. This will queue it to the appropriate + // tape pool(s) orphanesArchiveToFileCreations. The schedule will then + // determine its fate depending on the status of the NS entry creation + // (no entry, just cancel, already created in NS, carry on). + if (m_closed) + return; + try { + m_request.garbageCollect(m_agent->getAddressIfSet()); + { + objectstore::ScopedExclusiveLock al(*m_agent); + m_agent->fetch(); + m_agent->removeFromOwnership(m_request.getAddressIfSet()); + m_agent->commit(); + } + m_closed=true; + } catch (...) {} +} + void OStoreDB::createTapePool(const std::string& name, const uint32_t nbPartialTapes, const cta::CreationLog &creationLog) { RootEntry re(m_objectStore); @@ -639,6 +743,11 @@ std::unique_ptr<cta::SchedulerDatabase::ArchiveToFileRequestCreation> return ret; } +std::unique_ptr<cta::SchedulerDatabase::ArchiveRequestCreation> + OStoreDB::queue(const cta::common::dataStructures::ArchiveRequest &request, const uint64_t archiveFileId) { + return std::unique_ptr<cta::SchedulerDatabase::ArchiveRequestCreation>(); +} + void OStoreDB::deleteArchiveRequest(const SecurityIdentity& requester, const std::string& archiveFile) { // First of, find the archive request form all the tape pools. diff --git a/scheduler/OStoreDB/OStoreDB.hpp b/scheduler/OStoreDB/OStoreDB.hpp index 7ca32049fae86c52ad1e7c3d33e15222c6ee28cf..df03194c7166009335ddb23f6830bd7fb7e494bb 100644 --- a/scheduler/OStoreDB/OStoreDB.hpp +++ b/scheduler/OStoreDB/OStoreDB.hpp @@ -21,6 +21,7 @@ #include "scheduler/SchedulerDatabase.hpp" #include "objectstore/Agent.hpp" #include "objectstore/ArchiveToFileRequest.hpp" +#include "objectstore/ArchiveRequest.hpp" #include "objectstore/DriveRegister.hpp" #include "objectstore/RetrieveToFileRequest.hpp" #include "objectstore/SchedulerGlobalLock.hpp" @@ -230,9 +231,28 @@ public: bool m_closed; friend class cta::OStoreDB; }; + + class ArchiveRequestCreation: + public cta::SchedulerDatabase::ArchiveRequestCreation { + public: + ArchiveRequestCreation(objectstore::Agent * agent, + objectstore::Backend & be): m_request(be), m_lock(), m_objectStore(be), + m_agent(agent), m_closed(false) {} + virtual void complete(); + virtual void cancel(); + virtual ~ArchiveRequestCreation(); + private: + objectstore::ArchiveRequest m_request; + objectstore::ScopedExclusiveLock m_lock; + objectstore::Backend & m_objectStore; + objectstore::Agent * m_agent; + bool m_closed; + friend class cta::OStoreDB; + }; - virtual std::unique_ptr<cta::SchedulerDatabase::ArchiveToFileRequestCreation> - queue(const ArchiveToFileRequest& rqst); + virtual std::unique_ptr<cta::SchedulerDatabase::ArchiveToFileRequestCreation> queue(const ArchiveToFileRequest& rqst); + + virtual std::unique_ptr<cta::SchedulerDatabase::ArchiveRequestCreation> queue(const cta::common::dataStructures::ArchiveRequest &request, const uint64_t archiveFileId); CTA_GENERATE_EXCEPTION_CLASS(NoSuchArchiveRequest); CTA_GENERATE_EXCEPTION_CLASS(ArchiveRequestAlreadyDeleted); diff --git a/scheduler/OStoreDB/OStoreDBFactory.hpp b/scheduler/OStoreDB/OStoreDBFactory.hpp index d9913bdcee208419f7da817e275c1326dfdfb8a6..e0e6f6be3492bb703f6ce122cad16d68767899a1 100644 --- a/scheduler/OStoreDB/OStoreDBFactory.hpp +++ b/scheduler/OStoreDB/OStoreDBFactory.hpp @@ -165,6 +165,10 @@ public: return m_OStoreDB.queue(rqst); } + virtual std::unique_ptr<ArchiveRequestCreation> queue(const cta::common::dataStructures::ArchiveRequest &request, const uint64_t archiveFileId) { + return m_OStoreDB.queue(request, archiveFileId); + } + virtual void queue(const RetrieveToFileRequest& rqst) { m_OStoreDB.queue(rqst); } diff --git a/scheduler/Scheduler.cpp b/scheduler/Scheduler.cpp index 6615847d5ad9c1d62261f4bd802a0f466af99bb5..9c640358678839a3b7b41f5a731e3b4e7a3b5199 100644 --- a/scheduler/Scheduler.cpp +++ b/scheduler/Scheduler.cpp @@ -47,7 +47,11 @@ cta::Scheduler::~Scheduler() throw() { //------------------------------------------------------------------------------ // queueArchiveRequest //------------------------------------------------------------------------------ -uint64_t cta::Scheduler::queueArchiveRequest(const cta::common::dataStructures::SecurityIdentity &requestPusher, const cta::common::dataStructures::ArchiveRequest &request) { +uint64_t cta::Scheduler::queueArchiveRequest(const cta::common::dataStructures::SecurityIdentity &requestPusher, const cta::common::dataStructures::ArchiveRequest &request) { + const uint64_t archiveFileId = m_catalogue.getNextArchiveFileId(); + std::unique_ptr<SchedulerDatabase::ArchiveRequestCreation> requestCreation(m_db.queue(request, archiveFileId)); + requestCreation->complete(); + return 0; } diff --git a/scheduler/SchedulerDatabase.hpp b/scheduler/SchedulerDatabase.hpp index 4888e26e4d445176a9ba8503612968c846459b8b..d8f8486055af7368ae08255d78b26f35fd975bca 100644 --- a/scheduler/SchedulerDatabase.hpp +++ b/scheduler/SchedulerDatabase.hpp @@ -36,6 +36,7 @@ #include "nameserver/NameServerTapeFile.hpp" #include "scheduler/MountType.hpp" #include "common/forwardDeclarations.hpp" +#include "common/dataStructures/ArchiveRequest.hpp" namespace cta { /** @@ -64,13 +65,34 @@ public: virtual void cancel() = 0; virtual ~ArchiveToFileRequestCreation() {}; }; + + /* + * Subclass allowing the tracking and automated cleanup of a + * ArchiveToFile requests on the SchdulerDB. Those 2 operations (creation+close + * or cancel) surround an NS operation. This class can keep references, locks, + * etc... handy to simplify the implementation of the completion and cancelling + * (plus the destructor in case the caller fails half way through). + */ + class ArchiveRequestCreation { + public: + virtual void complete() = 0; + virtual void cancel() = 0; + virtual ~ArchiveRequestCreation() {}; + }; /** - * Queues the specified request. + * Queues the specified request. DEPRECATED * * @param rqst The request. */ virtual std::unique_ptr<ArchiveToFileRequestCreation> queue(const ArchiveToFileRequest &rqst) = 0; + + /** + * Queues the specified request. + * + * @param rqst The request. + */ + virtual std::unique_ptr<ArchiveRequestCreation> queue(const cta::common::dataStructures::ArchiveRequest &request, const uint64_t archiveFileId) = 0; /** * Returns all of the queued archive requests. The returned requests are diff --git a/scheduler/mockDB/MockSchedulerDatabase.cpp b/scheduler/mockDB/MockSchedulerDatabase.cpp index 01c092f7e1084d00615e1d9726d3669694b8d400..82afe36df56de2c29bc943e510f3a12d1fa57d1b 100644 --- a/scheduler/mockDB/MockSchedulerDatabase.cpp +++ b/scheduler/mockDB/MockSchedulerDatabase.cpp @@ -266,7 +266,13 @@ void cta::MockSchedulerDatabase::ArchiveToFileRequestCreation::cancel() { m_parent.deleteArchiveRequest(m_requester, m_archiveFile); } - +//------------------------------------------------------------------------------ +// queue +//------------------------------------------------------------------------------ +std::unique_ptr<cta::SchedulerDatabase::ArchiveToFileRequestCreation> + cta::MockSchedulerDatabase::queue(const cta::common::dataStructures::ArchiveRequest &request, const uint64_t archiveFileId) { + return std::unique_ptr<cta::SchedulerDatabase::ArchiveToFileRequestCreation> (); +} //------------------------------------------------------------------------------ // queue diff --git a/scheduler/mockDB/MockSchedulerDatabase.hpp b/scheduler/mockDB/MockSchedulerDatabase.hpp index f4c391866b76989c740745d382fdae81557fbf5b..64dc66b3f5ff0e5f2e682cdc0417855170c6d8e1 100644 --- a/scheduler/mockDB/MockSchedulerDatabase.hpp +++ b/scheduler/mockDB/MockSchedulerDatabase.hpp @@ -87,6 +87,13 @@ public: * @param rqst The request. */ std::unique_ptr<SchedulerDatabase::ArchiveToFileRequestCreation> queue(const ArchiveToFileRequest &rqst); + + /** + * Queues the specified request. + * + * @param rqst The request. + */ + std::unique_ptr<ArchiveToFileRequestCreation> queue(const cta::common::dataStructures::ArchiveRequest &request, const uint64_t archiveFileId); /** * Returns all of the queued archive requests. The returned requests are diff --git a/tapeserver/CMakeLists.txt b/tapeserver/CMakeLists.txt index aca7a237950017bd61da6eab0b731c117dcd501b..c2298f7b3ef7522eb861be0fa030f24bd364bcfd 100644 --- a/tapeserver/CMakeLists.txt +++ b/tapeserver/CMakeLists.txt @@ -11,3 +11,13 @@ add_executable (cta-taped cta-taped.cpp) target_link_libraries(cta-taped ctatapedaemon ctacommon protobuf) install (TARGETS cta-taped DESTINATION usr/bin) + +# CTA's cta-taped system tests. +add_library(cta-tapedSystemTests SHARED + cta-tapedSystemtests.cpp) + +target_link_libraries(cta-tapedSystemTests + systemTestHelper + ctacommon) + +install(TARGETS cta-tapedSystemTests DESTINATION usr/${CMAKE_INSTALL_LIBDIR}) \ No newline at end of file diff --git a/tapeserver/castor/tape/tapeserver/daemon/TapeDaemon.cpp b/tapeserver/castor/tape/tapeserver/daemon/TapeDaemon.cpp index c6e392c5edbd35aeba6ca9976d18d1de16d10219..e6ec89087996ffa320871af817738e4ba2b813ed 100644 --- a/tapeserver/castor/tape/tapeserver/daemon/TapeDaemon.cpp +++ b/tapeserver/castor/tape/tapeserver/daemon/TapeDaemon.cpp @@ -152,24 +152,17 @@ void castor::tape::tapeserver::daemon::TapeDaemon::destroyZmqContext() throw() { //------------------------------------------------------------------------------ int castor::tape::tapeserver::daemon::TapeDaemon::main() throw() { try { - exceptionThrowingMain(m_argc, m_argv); - } catch (castor::exception::Exception &ex) { - // Write the error to standard error - m_stdErr << std::endl << "Aborting: " << ex.getMessage().str() << std::endl - << std::endl; - // Log the error std::list<log::Param> params = { log::Param("Message", ex.getMessage().str()), log::Param("Code" , ex.code())}; m_log(LOG_ERR, "Aborting", params); - return 1; + return EXIT_FAILURE; } - - return 0; + return EXIT_SUCCESS; } //------------------------------------------------------------------------------ diff --git a/tapeserver/cta-taped.cpp b/tapeserver/cta-taped.cpp index ebfc77cf861804b9c4c976a94b232241f454d06d..2beb0c88b087e839cfb4520b75da350a58cab914 100644 --- a/tapeserver/cta-taped.cpp +++ b/tapeserver/cta-taped.cpp @@ -45,7 +45,7 @@ namespace cta { namespace taped { // @param argv The command-line arguments. // @param log The logging system. //------------------------------------------------------------------------------ -static int exceptionThrowingMain(const int argc, char **const argv, +static int exceptionThrowingMain(const cta::daemon::CommandLineParams & commandLine, cta::log::Logger &log); //------------------------------------------------------------------------------ @@ -66,8 +66,8 @@ std::string gHelpString = //------------------------------------------------------------------------------ // Logs the start of the daemon. //------------------------------------------------------------------------------ -static void logStartOfDaemon(cta::log::Logger &log, const int argc, - const char *const *const argv); +void logStartOfDaemon(cta::log::Logger &log, + const daemon::CommandLineParams& commandLine); //------------------------------------------------------------------------------ // Creates a string that contains the specified command-line arguments @@ -76,7 +76,7 @@ static void logStartOfDaemon(cta::log::Logger &log, const int argc, // @param argc The number of command-line arguments. // @param argv The array of command-line arguments. //------------------------------------------------------------------------------ -static std::string argvToString(const int argc, const char *const *const argv); +//static std::string argvToString(const int argc, const char *const *const argv); ////------------------------------------------------------------------------------ //// Writes the specified TPCONFIG lines to the specified logging system. @@ -99,25 +99,23 @@ static std::string argvToString(const int argc, const char *const *const argv); //------------------------------------------------------------------------------ // exceptionThrowingMain //------------------------------------------------------------------------------ -static int exceptionThrowingMain(const int argc, char **const argv, +static int exceptionThrowingMain( + const cta::daemon::CommandLineParams & commandLine, cta::log::Logger &log) { using namespace cta::tape::daemon; - logStartOfDaemon(log, argc, argv); + logStartOfDaemon(log, commandLine); // Parse /etc/cta/cta.conf and /etc/cta/TPCONFIG for global parameters const GlobalConfiguration globalConfig = - GlobalConfiguration::createFromCtaConf(log); + GlobalConfiguration::createFromCtaConf(commandLine.configFileLocation, log); // Create the object providing utilities for working with UNIX capabilities cta::server::ProcessCap capUtils; // Create the main tapeserverd object cta::tape::daemon::TapeDaemon daemon( - argc, - argv, - std::cout, - std::cerr, + commandLine, log, globalConfig, capUtils); @@ -129,32 +127,30 @@ static int exceptionThrowingMain(const int argc, char **const argv, //------------------------------------------------------------------------------ // logStartOfDaemon //------------------------------------------------------------------------------ -static void logStartOfDaemon(cta::log::Logger &log, const int argc, - const char *const *const argv) { +void logStartOfDaemon(cta::log::Logger &log, + const cta::daemon::CommandLineParams & commandLine) { using namespace cta; - const std::string concatenatedArgs = argvToString(argc, argv); - std::list<log::Param> params = { - log::Param("version", CTA_VERSION), - log::Param("argv", concatenatedArgs)}; - log(log::INFO, "tapeserverd started", params); + std::list<log::Param> params = {log::Param("version", CTA_VERSION)}; + params.splice(params.end(), commandLine.toLogParams()); + log(log::INFO, "cta-taped started", params); } //------------------------------------------------------------------------------ // argvToString //------------------------------------------------------------------------------ -static std::string argvToString(const int argc, const char *const *const argv) { - std::string str; - - for(int i=0; i < argc; i++) { - if(i != 0) { - str += " "; - } - - str += argv[i]; - } - return str; -} +//static std::string argvToString(const int argc, const char *const *const argv) { +// std::string str; +// +// for(int i=0; i < argc; i++) { +// if(i != 0) { +// str += " "; +// } +// +// str += argv[i]; +// } +// return str; +//} ////------------------------------------------------------------------------------ //// logTpconfigLines @@ -225,7 +221,7 @@ int main(const int argc, char **const argv) { int programRc = EXIT_FAILURE; // Default return code when receiving an exception. try { - programRc = cta::taped::exceptionThrowingMain(argc, argv, log); + programRc = cta::taped::exceptionThrowingMain(*commandLine, log); } catch(exception::Exception &ex) { std::list<log::Param> params = { log::Param("message", ex.getMessage().str())}; diff --git a/tapeserver/cta-tapedSystemtests.cpp b/tapeserver/cta-tapedSystemtests.cpp index 8b137891791fe96927ad78e64b0aad7bded08bdc..6dad0973e70a9407a01779f40745fd843a726abd 100644 --- a/tapeserver/cta-tapedSystemtests.cpp +++ b/tapeserver/cta-tapedSystemtests.cpp @@ -1 +1,44 @@ +/* + * The CERN Tape Archive (CTA) project + * Copyright (C) 2015 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +#include "tests/Subprocess.hpp" + +#include <gtest/gtest.h> + +namespace systemTests { +TEST(cta_taped, InvocationTests) { + // Do we get help with -h or --help? + Subprocess spHelpShort("cta-taped", std::list<std::string>({"cta-taped", "-h"})); + spHelpShort.wait(); + ASSERT_NE(std::string::npos, spHelpShort.stdout().find("Usage")); + ASSERT_TRUE(spHelpShort.stderr().empty()); + ASSERT_EQ(EXIT_SUCCESS, spHelpShort.exitValue()); + Subprocess spHelpLong("cta-taped", std::list<std::string>({"cta-taped", "--help"})); + spHelpLong.wait(); + ASSERT_NE(std::string::npos, spHelpLong.stdout().find("Usage: cta-taped [options]")); + ASSERT_TRUE(spHelpLong.stderr().empty()); + ASSERT_EQ(EXIT_SUCCESS, spHelpLong.exitValue()); + + // Does the tape server complain about absence of drive configuration? + Subprocess spNoDrive("cta-taped", std::list<std::string>({"cta-taped", "-f", "-s"})); + spNoDrive.wait(); + ASSERT_NE(std::string::npos, spNoDrive.stdout().find("MSG=\"Aborting\" Message=\"No drive found in configuration\"")); + ASSERT_TRUE(spNoDrive.stderr().empty()); + ASSERT_EQ(EXIT_FAILURE, spNoDrive.exitValue()); +} +} \ No newline at end of file diff --git a/tapeserver/daemon/CMakeLists.txt b/tapeserver/daemon/CMakeLists.txt index e58372083b9ddf8963098530c52a8812fc625dd2..f222dff0f8e4e8921033fe2923101bee83fda09e 100644 --- a/tapeserver/daemon/CMakeLists.txt +++ b/tapeserver/daemon/CMakeLists.txt @@ -2,7 +2,17 @@ cmake_minimum_required (VERSION 2.6) add_library(ctatapedaemon CommandLineParams.cpp + ConfigurationFile.cpp GlobalConfiguration.cpp TapeDaemon.cpp TpconfigLine.cpp - TpconfigLines.cpp) \ No newline at end of file + TpconfigLines.cpp) + +add_library(ctadaemonunittests SHARED + ConfigurationFileTests.cpp) + +target_link_libraries(ctadaemonunittests + ctatapedaemon + unitTestHelper) + +install(TARGETS ctadaemonunittests DESTINATION usr/${CMAKE_INSTALL_LIBDIR}) \ No newline at end of file diff --git a/tapeserver/daemon/CommandLineParams.cpp b/tapeserver/daemon/CommandLineParams.cpp index 19bf884220a2e4b10f90d8c10c7b2369bc74791b..685e9bbc2598a3c33a362f8b40a9eb4df54b5504 100644 --- a/tapeserver/daemon/CommandLineParams.cpp +++ b/tapeserver/daemon/CommandLineParams.cpp @@ -21,7 +21,9 @@ #include <getopt.h> #include <string.h> -cta::daemon::CommandLineParams::CommandLineParams(int argc, char** argv): +namespace cta { namespace daemon { + +CommandLineParams::CommandLineParams(int argc, char** argv): foreground(false), logToStdout(false), configFileLocation("/etc/cta/cta.conf"), helpRequested(false){ @@ -77,3 +79,15 @@ cta::daemon::CommandLineParams::CommandLineParams(int argc, char** argv): throw cta::exception::Exception("In CommandLineParams::CommandLineParams(): cannot log to stdout without running in the foreground"); } } + +std::list<log::Param> CommandLineParams::toLogParams() const { + std::list<log::Param> ret; + ret.push_back(log::Param("foreground", foreground)); + ret.push_back(log::Param("logToStdout", logToStdout)); + ret.push_back(log::Param("configFileLocation", configFileLocation)); + ret.push_back(log::Param("helpRequested", helpRequested)); + return ret; +} + +}} // namespace cta::daemon + diff --git a/tapeserver/daemon/CommandLineParams.hpp b/tapeserver/daemon/CommandLineParams.hpp index d69224be6e8dcfcc2352e62239fbd28feeb04080..8db291fb79c23c9d7422c3c119f1faede19d02e2 100644 --- a/tapeserver/daemon/CommandLineParams.hpp +++ b/tapeserver/daemon/CommandLineParams.hpp @@ -19,6 +19,8 @@ #pragma once #include <string> +#include <list> +#include "common/log/Param.hpp" namespace cta { namespace daemon { /// A class parsing the command line and turning it into a struct. @@ -33,5 +35,6 @@ struct CommandLineParams{ bool logToStdout; ///< Log to stdout instead of syslog. Foreground is required. std::string configFileLocation; ///< Location of the configuration file. Defaults to /etc/cta/cta.conf bool helpRequested; ///< Help requested: will print out help and exit. + std::list<log::Param> toLogParams() const; ///< Convert the command line into set of parameters for logging. }; }} \ No newline at end of file diff --git a/tapeserver/daemon/ConfigurationFile.cpp b/tapeserver/daemon/ConfigurationFile.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6b6a90289f2ed6f290d6810cc73277eaf9227160 --- /dev/null +++ b/tapeserver/daemon/ConfigurationFile.cpp @@ -0,0 +1,64 @@ +/* + * The CERN Tape Archive (CTA) project + * Copyright (C) 2015 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "ConfigurationFile.hpp" +#include "common/exception/Exception.hpp" + +#include <fstream> +#include <algorithm> + +namespace cta { namespace tape { namespace daemon { + +ConfigurationFile::ConfigurationFile(const std::string& path) { + // Try to open the configuration file, throwing an exception if there is a + // failure + std::ifstream file(path); + if (file.fail()) { + cta::exception::Exception ex; + ex.getMessage() << __FUNCTION__ << " failed" + ": Failed to open configuration file" + ": m_fileName=" << path; + throw ex; + } + + std::string line; + size_t lineNumber=0; + while(++lineNumber, std::getline(file, line)) { + // get rid of potential tabs + std::replace(line.begin(),line.end(),'\t',' '); + // get the category + std::istringstream sline(line); + std::string category; + if (!(sline >> category)) continue; // empty line + if (category[0] == '#') continue; // comment + // get the key + std::string key; + if (!(sline >> key)) continue; // no key on line + if (key[0] == '#') continue; // key commented + // get and store value + while (sline.get() == ' '){}; sline.unget(); // skip spaces + std::string value; + std::getline(sline, value, '#'); + value.erase(value.find_last_not_of(" \n\r\t")+1); // right trim + auto & entry = entries[category][key]; + entry.value = value; + entry.line = lineNumber; + } +} + +}}} // namespace cta::tape::daemon \ No newline at end of file diff --git a/tapeserver/daemon/ConfigurationFile.hpp b/tapeserver/daemon/ConfigurationFile.hpp new file mode 100644 index 0000000000000000000000000000000000000000..4db4d83ebbe5af81ebe50001c69e73cfb798632d --- /dev/null +++ b/tapeserver/daemon/ConfigurationFile.hpp @@ -0,0 +1,34 @@ +/* + * The CERN Tape Archive (CTA) project + * Copyright (C) 2015 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#pragma once + +#include <map> +#include <string> + +namespace cta { namespace tape { namespace daemon { +struct ConfigurationFile { +public: + ConfigurationFile(const std::string & path); + struct value_t { + std::string value; + uint32_t line; + }; + std::map<std::string, std::map<std::string, value_t> > entries; +}; +}}} // namespace cta::tape::daemon \ No newline at end of file diff --git a/tapeserver/daemon/ConfigurationFileTests.cpp b/tapeserver/daemon/ConfigurationFileTests.cpp new file mode 100644 index 0000000000000000000000000000000000000000..440c513b87ebd92bd827ff1bd656338aeb52a311 --- /dev/null +++ b/tapeserver/daemon/ConfigurationFileTests.cpp @@ -0,0 +1,39 @@ +/* + * The CERN Tape Archive (CTA) project + * Copyright (C) 2015 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <gtest/gtest.h> + +#include "ConfigurationFile.hpp" +#include "tests/TempFile.hpp" + +namespace unitTests { + +TEST(cta_Daemon, ConfigurationFile) { + TempFile tf; + tf.stringFill("# My test config file\n" + "cat1 key1 val1\n" + "cat1 #key2 val2\n" + "cat1 key3 #val3\n"); + cta::tape::daemon::ConfigurationFile cf(tf.path()); + ASSERT_EQ(1, cf.entries.size()); + ASSERT_NO_THROW(cf.entries.at("cat1").at("key1")); + ASSERT_EQ("val1", cf.entries.at("cat1").at("key1").value); + ASSERT_EQ(2, cf.entries.at("cat1").at("key1").line); +} + +} // namespace unitTests \ No newline at end of file diff --git a/tapeserver/daemon/GlobalConfiguration.cpp b/tapeserver/daemon/GlobalConfiguration.cpp index 6d14b9f24ec1f4d2be0d4a081f76215994d2f61f..e9de3c79a252e231b04c2491454a9cf41cd958cc 100644 --- a/tapeserver/daemon/GlobalConfiguration.cpp +++ b/tapeserver/daemon/GlobalConfiguration.cpp @@ -17,20 +17,33 @@ */ #include "GlobalConfiguration.hpp" +#include "ConfigurationFile.hpp" namespace cta { namespace tape { namespace daemon { GlobalConfiguration GlobalConfiguration::createFromCtaConf(cta::log::Logger& log) { - return createFromCtaConf("/etc/cta/cta.conf", "/etc/cta/TPCONFIG", log); + return createFromCtaConf("/etc/cta/cta.conf", log); } GlobalConfiguration GlobalConfiguration::createFromCtaConf( - const std::string& generalConfigPath, - const std::string& tapeConfigFile, cta::log::Logger& log) { + const std::string& generalConfigPath, cta::log::Logger& log) { GlobalConfiguration ret; + // Parse config file + ConfigurationFile cf(generalConfigPath); + // Extract configuration from parsed config file + // tpConfigPath: this element is optional + try { + ConfigurationFile::value_t & v = cf.entries.at("Taped").at("tpConfigPath"); + std::stringstream src; + src << generalConfigPath << ":" << v.line; + ret.tpConfigPath.set(v.value, src.str()); + } catch (...) {} return ret; } +GlobalConfiguration::GlobalConfiguration(): + tpConfigPath("tpConfigPath", "/etc/cta/TPCONFIG", "Compile time default") {} + cta::log::DummyLogger GlobalConfiguration::gDummyLogger(""); }}} // namespace cta::tape::daemon diff --git a/tapeserver/daemon/GlobalConfiguration.hpp b/tapeserver/daemon/GlobalConfiguration.hpp index d136c0cffc92d0ab09715e6c0a5e1196f4841986..6742a407b1fc9d47a0a130d3008b9a2e29b5ca5e 100644 --- a/tapeserver/daemon/GlobalConfiguration.hpp +++ b/tapeserver/daemon/GlobalConfiguration.hpp @@ -19,8 +19,11 @@ #pragma once #include <string> #include <map> +#include <type_traits> +#include <limits> #include "DriveConfiguration.hpp" #include "common/log/DummyLogger.hpp" +#include "common/exception/Exception.hpp" namespace cta { namespace tape { @@ -34,9 +37,49 @@ struct GlobalConfiguration { cta::log::Logger &log = gDummyLogger); static GlobalConfiguration createFromCtaConf( const std::string & generalConfigPath, - const std::string & tapeConfigFile, cta::log::Logger & log = gDummyLogger); + // Default constructor. + GlobalConfiguration(); std::map<std::string, DriveConfiguration> driveConfigs; + + + /** + * A templated class allowing the tracking of parameter with their source. + * If the parameter is not set (implicitly defined as the source being + * an empty string), access to the value will be denied (exception) + */ + template<class C> + class SourcedParameter { + public: + CTA_GENERATE_EXCEPTION_CLASS(ParameterNotDefined); + SourcedParameter(const std::string & name): m_name(name) { + if (std::is_arithmetic<C>::value) { + m_value=std::numeric_limits<C>::max(); + } + } + SourcedParameter(const std::string & name, C value, const std::string & source): + m_name(name), m_value(value), m_source(source) {} + C operator() () { + if (m_source.empty()) { + throw ParameterNotDefined(std::string("In SourcedParameter::operator(): " + "value not defined for parameter \'" + m_name + "\' :")); + } + return m_value; + } + void set(const std::string & value, const std::string & source) { + m_value = value; + m_source = source; + } + const std::string & name() { return m_name; } + const std::string & source() { return m_source; } + private: + std::string m_name; + C m_value; + std::string m_source; + }; + + // The actual parameters: + SourcedParameter<std::string> tpConfigPath; private: /** A private dummy logger which will simplify the implementaion of the * functions (just unconditionally log things). */ diff --git a/tapeserver/daemon/TapeDaemon.cpp b/tapeserver/daemon/TapeDaemon.cpp index 9b193530b33556c9621fd1ebf7d60611b5df94f6..18925c544649d3853803f2c6355775c80ae46b43 100644 --- a/tapeserver/daemon/TapeDaemon.cpp +++ b/tapeserver/daemon/TapeDaemon.cpp @@ -19,17 +19,16 @@ #include "TapeDaemon.hpp" #include "common/exception/Errnum.hpp" #include "common/utils/utils.hpp" +#include "tapeserver/daemon/CommandLineParams.hpp" #include <google/protobuf/service.h> namespace cta { namespace tape { namespace daemon { -TapeDaemon::TapeDaemon(const int argc, char* * const argv, - std::ostream& stdOut, std::ostream& stdErr, +TapeDaemon::TapeDaemon(const cta::daemon::CommandLineParams & commandLine, log::Logger& log, const GlobalConfiguration& globalConfig, cta::server::ProcessCap& capUtils): - cta::server::Daemon(stdOut, stdErr, log), - m_argc(argc), m_argv(argv), + cta::server::Daemon(log), m_globalConfiguration(globalConfig), m_capUtils(capUtils), m_programName("cta-taped"), m_hostName(getHostName()) { } @@ -42,19 +41,12 @@ TapeDaemon::~TapeDaemon() { //------------------------------------------------------------------------------ int TapeDaemon::main() { try { - - exceptionThrowingMain(m_argc, m_argv); - + exceptionThrowingMain(); } catch (cta::exception::Exception &ex) { - // Write the error to standard error - m_stdErr << std::endl << "Aborting: " << ex.getMessage().str() << std::endl - << std::endl; - // Log the error std::list<log::Param> params = { log::Param("Message", ex.getMessage().str())}; m_log(log::ERR, "Aborting", params); - return 1; } @@ -74,12 +66,9 @@ std::string cta::tape::daemon::TapeDaemon::getHostName() const { //------------------------------------------------------------------------------ // exceptionThrowingMain //------------------------------------------------------------------------------ -void cta::tape::daemon::TapeDaemon::exceptionThrowingMain( - const int argc, char **const argv) { - parseCommandLine(argc, argv); - +void cta::tape::daemon::TapeDaemon::exceptionThrowingMain() { if(m_globalConfiguration.driveConfigs.empty()) - throw cta::exception::Exception("/etc/cta/TPCONFIG is empty"); + throw cta::exception::Exception("No drive found in configuration"); // Process must be able to change user now and should be permitted to perform // raw IO in the future diff --git a/tapeserver/daemon/TapeDaemon.hpp b/tapeserver/daemon/TapeDaemon.hpp index dea03694a391209f3b116777e867dfc01185b012..0a9378b0d1636474f52f1aa0c286d08ff08a6bc4 100644 --- a/tapeserver/daemon/TapeDaemon.hpp +++ b/tapeserver/daemon/TapeDaemon.hpp @@ -19,6 +19,7 @@ #pragma once #include "common/threading/Daemon.hpp" +#include "tapeserver/daemon/CommandLineParams.hpp" #include "tapeserver/daemon/GlobalConfiguration.hpp" #include "common/processCap/ProcessCap.hpp" #include <signal.h> @@ -34,18 +35,12 @@ class TapeDaemon : public cta::server::Daemon { public: /** Constructor. - * @param argc The argc of main(). - * @param argv The argv of main(). - * @param stdOut Stream representing standard out. - * @param stdErr Stream representing standard error. + * @param commandLine The parameters extracted from the command line. * @param log The object representing the API of the CTA logging system. * @param globalConfig The configuration of the tape server. * @param capUtils Object providing utilities for working UNIX capabilities. */ TapeDaemon( - const int argc, - char **const argv, - std::ostream &stdOut, - std::ostream &stdErr, + const cta::daemon::CommandLineParams & commandLine, log::Logger &log, const GlobalConfiguration &globalConfig, cta::server::ProcessCap &capUtils); @@ -77,10 +72,8 @@ protected: /** Returns the name of the host on which the daemon is running. */ std::string getHostName() const; - /** Exception throwing main() function. - * @param argc The number of command-line arguments. - * @param argv The array of command-line arguments. */ - void exceptionThrowingMain(const int argc, char **const argv); + /** Exception throwing main() function. */ + void exceptionThrowingMain(); /** Sets the dumpable attribute of the current process to true. */ void setDumpable(); @@ -232,16 +225,6 @@ protected: void logChildProcessTerminated(const pid_t pid, const int waitpidStat) throw(); - /** - * The argc of main(). - */ - const int m_argc; - - /** - * The argv of main(). - */ - char **const m_argv; - /** The tape server's configuration */ const GlobalConfiguration& m_globalConfiguration; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0f75860cd6f21b78eff24f95c8c0b7a4cb31a37e..753b49d74de0628c4bc6b1ff1298b5cc63937cb0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -43,10 +43,14 @@ target_link_libraries(cta-unitTests ctaremotensunittests ctaschedulerunittests ctaio + ctadaemonunittests ${GMOCK_LIB} gtest pthread) +add_library(unitTestHelper + TempFile.cpp) + add_library(systemTestHelper Subprocess.cpp) @@ -64,6 +68,7 @@ add_executable(cta-systemTests target_link_libraries(cta-systemTests systemTestHelper systemTestHelperTests + cta-tapedSystemTests gtest pthread) diff --git a/tests/TempFile.cpp b/tests/TempFile.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1f8b11f757d88e4df466570034e466d79decf1e9 --- /dev/null +++ b/tests/TempFile.cpp @@ -0,0 +1,68 @@ +/* + * The CERN Tape Archive (CTA) project + * Copyright (C) 2015 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "TempFile.hpp" +#include "common/exception/Errnum.hpp" + +#include <stdlib.h> +#include <unistd.h> +#include <fstream> +#include <memory> + + +namespace unitTests { + +TempFile::TempFile() { + char path[] = "/tmp/testCTA-XXXXXX"; + int fd = ::mkstemp(path); + cta::exception::Errnum::throwOnMinusOne(fd, "In TempFile::TempFile: failed to mkstemp: "); + ::close(fd); + m_path = path; +} + +TempFile::TempFile(const std::string& path) : m_path(path) { } + +TempFile::~TempFile() { + if (m_path.size()) { + ::unlink(m_path.c_str()); + } +} + +std::string TempFile::path() { + return m_path; +} + +void TempFile::randomFill(size_t size) { + std::ofstream out(m_path, std::ios::out | std::ios::binary); + std::ifstream in("/dev/urandom", std::ios::in | std::ios::binary); + std::unique_ptr<char[] > buff(new char[size]); + in.read(buff.get(), size); + out.write(buff.get(), size); +} + +void TempFile::stringFill(const std::string& string) { + std::ofstream out(m_path, std::ios::out | std::ios::binary | std::ios::trunc); + out << string; +} + +void TempFile::stringAppend(const std::string& string) { + std::ofstream out(m_path, std::ios::out | std::ios::binary | std::ios::app); + out << string; +} + +} \ No newline at end of file diff --git a/tests/TempFile.hpp b/tests/TempFile.hpp new file mode 100644 index 0000000000000000000000000000000000000000..63d2d110489988bbe7cb79feabf174392bc0c3ab --- /dev/null +++ b/tests/TempFile.hpp @@ -0,0 +1,40 @@ +/* + * The CERN Tape Archive (CTA) project + * Copyright (C) 2015 CERN + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#pragma once + +#include <string> + +namespace unitTests { +/** + * A class creating a temporary file (deleted by destructor). Various population + * operations are provided. + */ +class TempFile { +public: + TempFile(); + TempFile(const std::string& path); + std::string path(); + void randomFill(size_t size); + void stringFill(const std::string &string); + void stringAppend(const std::string &string); + ~TempFile(); +private: + std::string m_path; +}; +} \ No newline at end of file diff --git a/tests/cta-valgrindUnitTests.sh.in b/tests/cta-valgrindUnitTests.sh.in index 64f644db475dd0bcbed59ebf910cddeadd0a1a47..d3c0fbe260c589959598871f6f22d41692a4923d 100644 --- a/tests/cta-valgrindUnitTests.sh.in +++ b/tests/cta-valgrindUnitTests.sh.in @@ -1,12 +1,12 @@ #!/bin/bash # exit from the script on any error. set -e -/usr/bin/cta-unitTests --gtest_color=yes +/usr/bin/cta-unitTests valgrind --track-fds=yes --leak-check=full --demangle=yes --gen-suppressions=all --show-reachable=yes \ --error-exitcode=1 --suppressions=/usr/share/cta-@CTA_VERSION@/unittest/valgrind.suppr \ - /usr/bin/cta-unitTests --gtest_color=yes + /usr/bin/cta-unitTests valgrind --tool=helgrind -v --demangle=yes --gen-suppressions=all --conflict-cache-size=30000000 \ --error-exitcode=1 --suppressions=/usr/share/cta-@CTA_VERSION@/unittest/helgrind.suppr \ - /usr/bin/cta-unitTests --gtest_color=yes + /usr/bin/cta-unitTests diff --git a/tests/unit_tests.cpp b/tests/unit_tests.cpp index 602c1b51ec664c2e41a52303d28dc91e1a06f0c2..e9dce4f2137da286d4b30b0f9e7c910490bf8c1a 100644 --- a/tests/unit_tests.cpp +++ b/tests/unit_tests.cpp @@ -18,8 +18,15 @@ #include <gtest/gtest.h> #include <gmock/gmock.h> +#include <sqlite3.h> int main(int argc, char** argv) { + // The unit tests use SQLite it must be initialized before they are run + if(SQLITE_OK != sqlite3_initialize()) { + std::cerr << "Failed to initialize SQLite" << std::endl; + return 1; // Error + } + // The following line must be executed to initialize Google Mock // (and Google Test) before running the tests. ::testing::InitGoogleMock(&argc, argv); @@ -33,5 +40,12 @@ int main(int argc, char** argv) { close(1); close(2); + // The unit tests used SQLite and so it should shutodwn in order to release + // its resources + if(SQLITE_OK != sqlite3_shutdown()) { + std::cerr << "Failed to shutdown SQLite" << std::endl; + return 1; // Error + } + return ret; } diff --git a/version.hpp.in b/version.hpp.in index d9e67f98f4db96195fb644e5fae297e554964585..d33b955cd50b46223344c2edfd88fbba6d7eec06 100644 --- a/version.hpp.in +++ b/version.hpp.in @@ -18,5 +18,5 @@ #pragma once -#define CTA_VERSION "@CTA_VERSION@-@CTA_REPLEASE@" +#define CTA_VERSION "@CTA_VERSION@-@CTA_RELEASE@"