diff --git a/CMakeLists.txt b/CMakeLists.txt
index 87f87a6f8889a3d744f049cfa2fafd0a3214be89..969f897f4f034259bfb04ef7f43c99de8c6e98a4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -139,6 +139,8 @@ ELSE(DEFINED PackageOnly)
add_subdirectory(eos_cta)
+ add_subdirectory(eos_grpc_client)
+
add_subdirectory(migration)
add_subdirectory(cmdline)
diff --git a/catalogue/Catalogue.hpp b/catalogue/Catalogue.hpp
index 71bc1c64e3d46e8810478ff359d0e38585cbf584..e33e6129caabd002fea748e02cca88e5051eb042 100644
--- a/catalogue/Catalogue.hpp
+++ b/catalogue/Catalogue.hpp
@@ -924,11 +924,12 @@ public:
/**
- * Restores the deleted files in the Recycle log that match the criteria passed
+ * Restores the deleted file in the Recycle log that match the criteria passed
*
* @param searchCriteria The search criteria
+ * @param newFid the new Fid of the archive file (if the archive file must be restored)
*/
- virtual void restoreFilesInRecycleLog(const RecycleTapeFileSearchCriteria & searchCriteria = RecycleTapeFileSearchCriteria()) = 0;
+ virtual void restoreFileInRecycleLog(const RecycleTapeFileSearchCriteria & searchCriteria, const std::string &newFid) = 0;
/**
diff --git a/catalogue/CatalogueRetryWrapper.hpp b/catalogue/CatalogueRetryWrapper.hpp
index 012decde670aba3631367ac40adbfac949deaf31..b7cbb9735962cc374cd14251e46162a8467bb2f4 100644
--- a/catalogue/CatalogueRetryWrapper.hpp
+++ b/catalogue/CatalogueRetryWrapper.hpp
@@ -540,8 +540,8 @@ public:
return retryOnLostConnection(m_log, [&]{return m_catalogue->getFileRecycleLogItor(searchCriteria);}, m_maxTriesToConnect);
}
- void restoreFilesInRecycleLog(const RecycleTapeFileSearchCriteria & searchCriteria) override {
- return retryOnLostConnection(m_log, [&]{return m_catalogue->restoreFilesInRecycleLog(searchCriteria);}, m_maxTriesToConnect);
+ void restoreFileInRecycleLog(const RecycleTapeFileSearchCriteria & searchCriteria, const std::string &newFid) override {
+ return retryOnLostConnection(m_log, [&]{return m_catalogue->restoreFileInRecycleLog(searchCriteria, newFid);}, m_maxTriesToConnect);
}
void deleteFileFromRecycleBin(const uint64_t archiveFileId, log::LogContext &lc){
diff --git a/catalogue/CatalogueTest.cpp b/catalogue/CatalogueTest.cpp
index 4d96b5ae1a10b3aed8f2980620bf12e0b42aab73..2cf6c9bca03d1bda0f4e253a06ace8610753ceca 100644
--- a/catalogue/CatalogueTest.cpp
+++ b/catalogue/CatalogueTest.cpp
@@ -16706,7 +16706,7 @@ TEST_P(cta_catalogue_CatalogueTest, RestoreTapeFileCopy) {
searchCriteria.archiveFileId = 1;
searchCriteria.vid = tape1.vid;
- m_catalogue->restoreFilesInRecycleLog(searchCriteria);
+ m_catalogue->restoreFileInRecycleLog(searchCriteria, "0"); //new FID does not matter because archive file still exists in catalogue
auto archiveFile = m_catalogue->getArchiveFileById(1);
//assert both copies present
ASSERT_EQ(2, archiveFile.tapeFiles.size());
@@ -16871,7 +16871,7 @@ TEST_P(cta_catalogue_CatalogueTest, RestoreRewrittenTapeFileCopyFails) {
searchCriteria.archiveFileId = 1;
searchCriteria.vid = tape1.vid;
- ASSERT_THROW(m_catalogue->restoreFilesInRecycleLog(searchCriteria), catalogue::UserSpecifiedExistingDeletedFileCopy);
+ ASSERT_THROW(m_catalogue->restoreFileInRecycleLog(searchCriteria, "0"), catalogue::UserSpecifiedExistingDeletedFileCopy);
auto archiveFile = m_catalogue->getArchiveFileById(1);
//assert only two copies present
ASSERT_EQ(2, archiveFile.tapeFiles.size());
@@ -17050,17 +17050,148 @@ TEST_P(cta_catalogue_CatalogueTest, RestoreVariousDeletedTapeFileCopies) {
{
- //restore all deleted copies
+ //try to restore all deleted copies should give an error
catalogue::RecycleTapeFileSearchCriteria searchCriteria;
searchCriteria.archiveFileId = 1;
- m_catalogue->restoreFilesInRecycleLog(searchCriteria);
+ ASSERT_THROW(m_catalogue->restoreFileInRecycleLog(searchCriteria, "0"), cta::exception::UserError);
+
+ }
+}
+
+TEST_P(cta_catalogue_CatalogueTest, RestoreArchiveFileAndCopy) {
+ using namespace cta;
+
+ const bool logicalLibraryIsDisabled= false;
+ const std::string tapePoolName1 = "tape_pool_name_1";
+ const std::string tapePoolName2 = "tape_pool_name_2";
+ const uint64_t nbPartialTapes = 1;
+ const bool isEncrypted = true;
+ const cta::optional<std::string> supply("value for the supply pool mechanism");
+ const std::string diskInstance = "disk_instance";
+ const std::string tapeDrive = "tape_drive";
+ const std::string reason = "reason";
+
+ m_catalogue->createMediaType(m_admin, m_mediaType);
+ m_catalogue->createLogicalLibrary(m_admin, m_tape1.logicalLibraryName, logicalLibraryIsDisabled, "Create logical library");
+ m_catalogue->createVirtualOrganization(m_admin, m_vo);
+ m_catalogue->createTapePool(m_admin, tapePoolName1, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
+ m_catalogue->createTapePool(m_admin, tapePoolName2, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
+ m_catalogue->createStorageClass(m_admin, m_storageClassDualCopy);
+
+ auto tape1 = m_tape1;
+ auto tape2 = m_tape2;
+ tape1.tapePoolName = tapePoolName1;
+ tape2.tapePoolName = tapePoolName2;
+
+ m_catalogue->createTape(m_admin, tape1);
+ m_catalogue->createTape(m_admin, tape2);
+
+ ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore());
+ const uint64_t archiveFileSize = 2 * 1000 * 1000 * 1000;
+
+
+ // Write a file on tape
+ {
+ std::set<catalogue::TapeItemWrittenPointer> tapeFilesWrittenCopy1;
+
+ std::ostringstream diskFileId;
+ diskFileId << 12345677;
+
+ std::ostringstream diskFilePath;
+ diskFilePath << "/test/file1";
+
+ auto fileWrittenUP=cta::make_unique<cta::catalogue::TapeFileWritten>();
+ auto & fileWritten = *fileWrittenUP;
+ fileWritten.archiveFileId = 1;
+ fileWritten.diskInstance = diskInstance;
+ fileWritten.diskFileId = diskFileId.str();
+ fileWritten.diskFilePath = diskFilePath.str();
+ fileWritten.diskFileOwnerUid = PUBLIC_DISK_USER;
+ fileWritten.diskFileGid = PUBLIC_DISK_GROUP;
+ fileWritten.size = archiveFileSize;
+ fileWritten.checksumBlob.insert(checksum::ADLER32, "1357");
+ fileWritten.storageClassName = m_storageClassDualCopy.name;
+ fileWritten.vid = tape1.vid;
+ fileWritten.fSeq = 1;
+ fileWritten.blockId = 1 * 100;
+ fileWritten.copyNb = 1;
+ fileWritten.tapeDrive = tapeDrive;
+ tapeFilesWrittenCopy1.emplace(fileWrittenUP.release());
+
+ m_catalogue->filesWrittenToTape(tapeFilesWrittenCopy1);
+ }
+
+ // Write a second copy of file on tape
+ {
+ std::set<catalogue::TapeItemWrittenPointer> tapeFilesWrittenCopy1;
+
+ std::ostringstream diskFileId;
+ diskFileId << 12345677;
+
+ std::ostringstream diskFilePath;
+ diskFilePath << "/test/file1";
+
+ auto fileWrittenUP=cta::make_unique<cta::catalogue::TapeFileWritten>();
+ auto & fileWritten = *fileWrittenUP;
+ fileWritten.archiveFileId = 1;
+ fileWritten.diskInstance = diskInstance;
+ fileWritten.diskFileId = diskFileId.str();
+ fileWritten.diskFilePath = diskFilePath.str();
+ fileWritten.diskFileOwnerUid = PUBLIC_DISK_USER;
+ fileWritten.diskFileGid = PUBLIC_DISK_GROUP;
+ fileWritten.size = archiveFileSize;
+ fileWritten.checksumBlob.insert(checksum::ADLER32, "1357");
+ fileWritten.storageClassName = m_storageClassDualCopy.name;
+ fileWritten.vid = tape2.vid;
+ fileWritten.fSeq = 1;
+ fileWritten.blockId = 1 * 100;
+ fileWritten.copyNb = 2;
+ fileWritten.tapeDrive = tapeDrive;
+ tapeFilesWrittenCopy1.emplace(fileWrittenUP.release());
+
+ m_catalogue->filesWrittenToTape(tapeFilesWrittenCopy1);
+ }
+ {
+ //Assert both copies written
auto archiveFile = m_catalogue->getArchiveFileById(1);
- //assert only two copies present
- ASSERT_EQ(3, archiveFile.tapeFiles.size());
+ ASSERT_EQ(2, archiveFile.tapeFiles.size());
+ }
- //assert recycle log still contains deleted copy
+ {
+ //delete archive file
+ common::dataStructures::DeleteArchiveRequest deleteRequest;
+ deleteRequest.archiveFileID = 1;
+ deleteRequest.archiveFile = m_catalogue->getArchiveFileById(1);
+ deleteRequest.diskInstance = diskInstance;
+ deleteRequest.diskFileId = std::to_string(12345677);
+ deleteRequest.diskFilePath = "/test/file1";
+
+ log::LogContext dummyLc(m_dummyLog);
+ m_catalogue->moveArchiveFileToRecycleLog(deleteRequest, dummyLc);
+ ASSERT_THROW(m_catalogue->getArchiveFileById(1), cta::exception::Exception);
+ }
+
+
+ {
+ //restore copy of file on tape1
+ catalogue::RecycleTapeFileSearchCriteria searchCriteria;
+ searchCriteria.archiveFileId = 1;
+ searchCriteria.vid = tape1.vid;
+
+ m_catalogue->restoreFileInRecycleLog(searchCriteria, std::to_string(12345678)); //previous fid + 1
+
+ //assert archive file has been restored in the catalogue
+ auto archiveFile = m_catalogue->getArchiveFileById(1);
+ ASSERT_EQ(1, archiveFile.tapeFiles.size());
+ ASSERT_EQ(archiveFile.diskFileId, std::to_string(12345678));
+ ASSERT_EQ(archiveFile.diskInstance, diskInstance);
+ ASSERT_EQ(archiveFile.storageClass, m_storageClassDualCopy.name);
+
+ //assert recycle log has the other tape file copy
auto fileRecycleLogItor = m_catalogue->getFileRecycleLogItor();
+ ASSERT_TRUE(fileRecycleLogItor.hasMore());
+ auto fileRecycleLog = fileRecycleLogItor.next();
ASSERT_FALSE(fileRecycleLogItor.hasMore());
}
diff --git a/catalogue/DummyCatalogue.hpp b/catalogue/DummyCatalogue.hpp
index 592aa35db43c9555438244b0b12b72e48e242411..e608e333781d87d34bea3e6a869341b2c43ab3ef 100644
--- a/catalogue/DummyCatalogue.hpp
+++ b/catalogue/DummyCatalogue.hpp
@@ -85,7 +85,7 @@ public:
common::dataStructures::ArchiveFile getArchiveFileById(const uint64_t id) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
ArchiveFileItor getArchiveFilesItor(const TapeFileSearchCriteria& searchCriteria) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
FileRecycleLogItor getFileRecycleLogItor(const RecycleTapeFileSearchCriteria & searchCriteria) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
- void restoreFilesInRecycleLog(const RecycleTapeFileSearchCriteria & searchCriteria) { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
+ void restoreFileInRecycleLog(const RecycleTapeFileSearchCriteria & searchCriteria, const std::string &newFid) { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void deleteFileFromRecycleBin(const uint64_t archiveFileId, log::LogContext &lc) {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
void deleteFilesFromRecycleLog(const std::string & vid, log::LogContext & lc) {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
void createTapeDrive(const common::dataStructures::TapeDrive &tapeDrive) {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
diff --git a/catalogue/MysqlCatalogue.cpp b/catalogue/MysqlCatalogue.cpp
index f8f450b0064fdf6a635e4bc723e48e32fd8c6b1d..f2fdc87f1fae2485f68b384e9fcfcb6bfe15d1bf 100644
--- a/catalogue/MysqlCatalogue.cpp
+++ b/catalogue/MysqlCatalogue.cpp
@@ -821,50 +821,47 @@ void MysqlCatalogue::copyTapeFileToFileRecyleLogAndDelete(rdbms::Conn & conn, co
}
//------------------------------------------------------------------------------
-// restoreFileCopiesInRecycleLog
+// restoreEntryInRecycleLog
//------------------------------------------------------------------------------
-void MysqlCatalogue::restoreFileCopiesInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor, log::LogContext & lc) {
-try {
+void MysqlCatalogue::restoreEntryInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor,
+ const std::string &newFid, log::LogContext & lc) {
+ try {
utils::Timer t;
log::TimingList tl;
- //put fileRecycleLogs in std::list so we can release the underlying fileRecycleLogItor database connection
- //otherwise we are using two conns when calling getArchiveFilesItor
- std::list<common::dataStructures::FileRecycleLog> fileRecycleLogList;
- while (fileRecycleLogItor.hasMore()) {
- auto fileRecycleLog = fileRecycleLogItor.next();
- fileRecycleLogList.push_back(fileRecycleLog);
+ if (!fileRecycleLogItor.hasMore()) {
+ throw cta::exception::UserError("No file in the recycle bin matches the parameters passed");
+ }
+ auto fileRecycleLog = fileRecycleLogItor.next();
+ if (fileRecycleLogItor.hasMore()) {
+ //stop restoring more than one file at once
+ throw cta::exception::UserError("More than one recycle bin file matches the parameters passed");
}
- //We currently do all file copies restoring in a single transaction
conn.executeNonQuery("START TRANSACTION");
- for (auto &fileRecycleLog: fileRecycleLogList) {
- TapeFileSearchCriteria searchCriteria;
- searchCriteria.archiveFileId = fileRecycleLog.archiveFileId;
- searchCriteria.diskInstance = fileRecycleLog.diskInstanceName;
- searchCriteria.diskFileIds = std::vector<std::string>();
- searchCriteria.diskFileIds.value().push_back(fileRecycleLog.diskFileId);
-
- auto itor = getArchiveFilesItor(conn, searchCriteria);
- if (itor.hasMore()) {
- //only restore file copies, do nothing if file has been completely deleted in CTA
- cta::common::dataStructures::ArchiveFile archiveFile = itor.next();
- if (archiveFile.tapeFiles.find(fileRecycleLog.copyNb) != archiveFile.tapeFiles.end()) {
- //copy with same copy_nb exists, cannot restore
- UserSpecifiedExistingDeletedFileCopy ex;
- ex.getMessage() << "Cannot restore file copy with archiveFileId " << std::to_string(fileRecycleLog.archiveFileId)
- << " and copy_nb " << std::to_string(fileRecycleLog.copyNb) << " because a tapefile with same archiveFileId and copy_nb already exists";
- throw ex;
- }
- restoreFileCopyInRecycleLog(conn, fileRecycleLog, lc);
+
+ std::unique_ptr<common::dataStructures::ArchiveFile> archiveFilePtr = getArchiveFileById(conn, fileRecycleLog.archiveFileId);
+ if (!archiveFilePtr) {
+ restoreArchiveFileInRecycleLog(conn, fileRecycleLog, newFid, lc);
+ } else {
+ if (archiveFilePtr->tapeFiles.find(fileRecycleLog.copyNb) != archiveFilePtr->tapeFiles.end()) {
+ //copy with same copy_nb exists, cannot restore
+ UserSpecifiedExistingDeletedFileCopy ex;
+ ex.getMessage() << "Cannot restore file copy with archiveFileId " << std::to_string(fileRecycleLog.archiveFileId)
+ << " and copy_nb " << std::to_string(fileRecycleLog.copyNb) << " because a tapefile with same archiveFileId and copy_nb already exists";
+ throw ex;
}
}
+
+
+ restoreFileCopyInRecycleLog(conn, fileRecycleLog, lc);
+
conn.commit();
log::ScopedParamContainer spc(lc);
tl.insertAndReset("commitTime",t);
tl.addToLog(spc);
- lc.log(log::INFO,"In MysqlCatalogue::restoreFileCopiesInRecycleLog: all file copies successfully restored.");
+ lc.log(log::INFO,"In MysqlCatalogue::restoreEntryInRecycleLog: all file copies successfully restored.");
} catch(exception::UserError &) {
throw;
} catch(exception::Exception &ex) {
diff --git a/catalogue/MysqlCatalogue.hpp b/catalogue/MysqlCatalogue.hpp
index 75df118fd39fbd9d4da9adf8d5f62617771f7a5b..8be5fc4f438708ba07ad33c29710f69f94b5ba14 100644
--- a/catalogue/MysqlCatalogue.hpp
+++ b/catalogue/MysqlCatalogue.hpp
@@ -225,7 +225,7 @@ protected:
* @param fileRecycleLogItor the collection of fileRecycleLogs we want to restore
* @param lc the log context
*/
- void restoreFileCopiesInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor, log::LogContext & lc) override;
+ void restoreEntryInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor, const std::string &newFid, log::LogContext & lc) override;
/**
* Copy the fileRecycleLog to the TAPE_FILE table and deletes the corresponding FILE_RECYCLE_LOG table entry
@@ -235,7 +235,6 @@ protected:
*/
void restoreFileCopyInRecycleLog(rdbms::Conn & conn, const common::dataStructures::FileRecycleLog &fileRecycleLogItor, log::LogContext & lc);
-
private:
/**
diff --git a/catalogue/OracleCatalogue.cpp b/catalogue/OracleCatalogue.cpp
index 98717eee30bc6ff9500e5670629ec3c5d6cd9c52..012b723d8facb3156f1dbebb2f7aac062db0ba36 100644
--- a/catalogue/OracleCatalogue.cpp
+++ b/catalogue/OracleCatalogue.cpp
@@ -1150,52 +1150,47 @@ void OracleCatalogue::copyTapeFileToFileRecyleLogAndDelete(rdbms::Conn & conn, c
}
//------------------------------------------------------------------------------
-// restoreFileCopiesInRecycleLog
+// restoreEntryInRecycleLog
//------------------------------------------------------------------------------
-void OracleCatalogue::restoreFileCopiesInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor, log::LogContext & lc) {
-try {
+void OracleCatalogue::restoreEntryInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor,
+ const std::string &newFid, log::LogContext & lc) {
+ try {
utils::Timer t;
log::TimingList tl;
- //put fileRecycleLogs in std::list so we can release the underlying fileRecycleLogItor database connection
- //otherwise we are using two conns when calling getArchiveFilesItor
- std::list<common::dataStructures::FileRecycleLog> fileRecycleLogList;
- while (fileRecycleLogItor.hasMore()) {
- auto fileRecycleLog = fileRecycleLogItor.next();
- fileRecycleLogList.push_back(fileRecycleLog);
+ if (!fileRecycleLogItor.hasMore()) {
+ throw cta::exception::UserError("No file in the recycle bin matches the parameters passed");
+ }
+ auto fileRecycleLog = fileRecycleLogItor.next();
+ if (fileRecycleLogItor.hasMore()) {
+ //stop restoring more than one file at once
+ throw cta::exception::UserError("More than one recycle bin file matches the parameters passed");
}
-
- //We currently do all file copies restoring in a single transaction
conn.setAutocommitMode(rdbms::AutocommitMode::AUTOCOMMIT_OFF);
- for (auto &fileRecycleLog: fileRecycleLogList) {
- TapeFileSearchCriteria searchCriteria;
- searchCriteria.archiveFileId = fileRecycleLog.archiveFileId;
- searchCriteria.diskInstance = fileRecycleLog.diskInstanceName;
- searchCriteria.diskFileIds = std::vector<std::string>();
- searchCriteria.diskFileIds.value().push_back(fileRecycleLog.diskFileId);
-
- auto itor = getArchiveFilesItor(conn, searchCriteria);
- if (itor.hasMore()) {
- //only restore file copies, do nothing if file has been completely deleted in CTA
- cta::common::dataStructures::ArchiveFile archiveFile = itor.next();
- if (archiveFile.tapeFiles.find(fileRecycleLog.copyNb) != archiveFile.tapeFiles.end()) {
- //copy with same copy_nb exists, cannot restore
- UserSpecifiedExistingDeletedFileCopy ex;
- ex.getMessage() << "Cannot restore file copy with archiveFileId " << std::to_string(fileRecycleLog.archiveFileId)
- << " and copy_nb " << std::to_string(fileRecycleLog.copyNb) << " because a tapefile with same archiveFileId and copy_nb already exists";
- throw ex;
- }
- restoreFileCopyInRecycleLog(conn, fileRecycleLog, lc);
+
+ std::unique_ptr<common::dataStructures::ArchiveFile> archiveFilePtr = getArchiveFileById(conn, fileRecycleLog.archiveFileId);
+ if (!archiveFilePtr) {
+ restoreArchiveFileInRecycleLog(conn, fileRecycleLog, newFid, lc);
+ } else {
+ if (archiveFilePtr->tapeFiles.find(fileRecycleLog.copyNb) != archiveFilePtr->tapeFiles.end()) {
+ //copy with same copy_nb exists, cannot restore
+ UserSpecifiedExistingDeletedFileCopy ex;
+ ex.getMessage() << "Cannot restore file copy with archiveFileId " << std::to_string(fileRecycleLog.archiveFileId)
+ << " and copy_nb " << std::to_string(fileRecycleLog.copyNb) << " because a tapefile with same archiveFileId and copy_nb already exists";
+ throw ex;
}
}
+
+
+ restoreFileCopyInRecycleLog(conn, fileRecycleLog, lc);
+
conn.setAutocommitMode(rdbms::AutocommitMode::AUTOCOMMIT_ON);
conn.commit();
log::ScopedParamContainer spc(lc);
tl.insertAndReset("commitTime",t);
tl.addToLog(spc);
- lc.log(log::INFO,"In OracleCatalogue::restoreFileCopiesInRecycleLog: all file copies successfully restored.");
-
+ lc.log(log::INFO,"In OracleCatalogue::restoreEntryInRecycleLog: all file copies successfully restored.");
} catch(exception::UserError &) {
throw;
} catch(exception::Exception &ex) {
diff --git a/catalogue/OracleCatalogue.hpp b/catalogue/OracleCatalogue.hpp
index 172cb55e0fe1f8502355ec607ce271689a236162..a28e8c800d1f4e4f9818695e2c3dbeab07a5167f 100644
--- a/catalogue/OracleCatalogue.hpp
+++ b/catalogue/OracleCatalogue.hpp
@@ -268,7 +268,7 @@ private:
* @param fileRecycleLogItor the collection of fileRecycleLogs we want to restore
* @param lc the log context
*/
- void restoreFileCopiesInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor, log::LogContext & lc) override;
+ void restoreEntryInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor, const std::string &newFid, log::LogContext & lc) override;
/**
* Copy the fileRecycleLog to the TAPE_FILE table and deletes the corresponding FILE_RECYCLE_LOG table entry
diff --git a/catalogue/PostgresCatalogue.cpp b/catalogue/PostgresCatalogue.cpp
index 7b06ba81b33c23e5aa601ed3575d3dc467888885..28ef06bdbcb884cb0db3eb4e6427a4db4e90c11d 100644
--- a/catalogue/PostgresCatalogue.cpp
+++ b/catalogue/PostgresCatalogue.cpp
@@ -1110,50 +1110,45 @@ void PostgresCatalogue::copyTapeFileToFileRecyleLogAndDelete(rdbms::Conn & conn,
}
//------------------------------------------------------------------------------
-// restoreFileCopiesInRecycleLog
+// restoreEntryInRecycleLog
//------------------------------------------------------------------------------
-void PostgresCatalogue::restoreFileCopiesInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor, log::LogContext & lc) {
-try {
+void PostgresCatalogue::restoreEntryInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor,
+ const std::string &newFid, log::LogContext & lc) {
+ try {
utils::Timer t;
log::TimingList tl;
-
- //put fileRecycleLogs in std::list so we can release the underlying fileRecycleLogItor database connection
- //otherwise we are using two conns when calling getArchiveFilesItor
- std::list<common::dataStructures::FileRecycleLog> fileRecycleLogList;
- while (fileRecycleLogItor.hasMore()) {
- auto fileRecycleLog = fileRecycleLogItor.next();
- fileRecycleLogList.push_back(fileRecycleLog);
+
+ if (!fileRecycleLogItor.hasMore()) {
+ throw cta::exception::UserError("No file in the recycle bin matches the parameters passed");
+ }
+ auto fileRecycleLog = fileRecycleLogItor.next();
+ if (fileRecycleLogItor.hasMore()) {
+ //stop restoring more than one file at once
+ throw cta::exception::UserError("More than one recycle bin file matches the parameters passed");
}
//We currently do all file copies restoring in a single transaction
conn.executeNonQuery("BEGIN");
- for (auto &fileRecycleLog: fileRecycleLogList) {
- TapeFileSearchCriteria searchCriteria;
- searchCriteria.archiveFileId = fileRecycleLog.archiveFileId;
- searchCriteria.diskInstance = fileRecycleLog.diskInstanceName;
- searchCriteria.diskFileIds = std::vector<std::string>();
- searchCriteria.diskFileIds.value().push_back(fileRecycleLog.diskFileId);
-
- auto itor = getArchiveFilesItor(conn, searchCriteria);
- if (itor.hasMore()) {
- //only restore file copies, do nothing if file has been completely deleted in CTA
- cta::common::dataStructures::ArchiveFile archiveFile = itor.next();
- if (archiveFile.tapeFiles.find(fileRecycleLog.copyNb) != archiveFile.tapeFiles.end()) {
- //copy with same copy_nb exists, cannot restore
- UserSpecifiedExistingDeletedFileCopy ex;
- ex.getMessage() << "Cannot restore file copy with archiveFileId " << std::to_string(fileRecycleLog.archiveFileId)
- << " and copy_nb " << std::to_string(fileRecycleLog.copyNb) << " because a tapefile with same archiveFileId and copy_nb already exists";
- throw ex;
- }
- restoreFileCopyInRecycleLog(conn, fileRecycleLog, lc);
+ std::unique_ptr<common::dataStructures::ArchiveFile> archiveFilePtr = getArchiveFileById(conn, fileRecycleLog.archiveFileId);
+ if (!archiveFilePtr) {
+ restoreArchiveFileInRecycleLog(conn, fileRecycleLog, newFid, lc);
+ } else {
+ if (archiveFilePtr->tapeFiles.find(fileRecycleLog.copyNb) != archiveFilePtr->tapeFiles.end()) {
+ //copy with same copy_nb exists, cannot restore
+ UserSpecifiedExistingDeletedFileCopy ex;
+ ex.getMessage() << "Cannot restore file copy with archiveFileId " << std::to_string(fileRecycleLog.archiveFileId)
+ << " and copy_nb " << std::to_string(fileRecycleLog.copyNb) << " because a tapefile with same archiveFileId and copy_nb already exists";
+ throw ex;
}
}
+
+ restoreFileCopyInRecycleLog(conn, fileRecycleLog, lc);
conn.commit();
log::ScopedParamContainer spc(lc);
tl.insertAndReset("commitTime",t);
tl.addToLog(spc);
- lc.log(log::INFO,"In PostgresCatalogue::restoreFileCopiesInRecycleLog: all file copies successfully restored.");
+ lc.log(log::INFO,"In PostgresCatalogue::restoreEntryInRecycleLog: all file copies successfully restored.");
} catch(exception::UserError &) {
throw;
} catch(exception::Exception &ex) {
diff --git a/catalogue/PostgresCatalogue.hpp b/catalogue/PostgresCatalogue.hpp
index 4d50cef0d35d6fcc8062de55f08719d7b0c11ed2..976c98d1c27a9e9e91e2e13560e2983dd6afb9a8 100644
--- a/catalogue/PostgresCatalogue.hpp
+++ b/catalogue/PostgresCatalogue.hpp
@@ -302,7 +302,7 @@ private:
* @param fileRecycleLogItor the collection of fileRecycleLogs we want to restore
* @param lc the log context
*/
- void restoreFileCopiesInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor, log::LogContext & lc) override;
+ void restoreEntryInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor, const std::string &newFid, log::LogContext & lc) override;
/**
* Copy the fileRecycleLog to the TAPE_FILE table and deletes the corresponding FILE_RECYCLE_LOG table entry
diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp
index c6cbbc9ee52e5147bf92b9e46175a30f00c73a44..1703cbc806024064570e8b3568bb40372f38ab6c 100644
--- a/catalogue/RdbmsCatalogue.cpp
+++ b/catalogue/RdbmsCatalogue.cpp
@@ -7169,16 +7169,32 @@ Catalogue::FileRecycleLogItor RdbmsCatalogue::getFileRecycleLogItor(const Recycl
}
}
+//------------------------------------------------------------------------------
+// restoreArchiveFileInRecycleLog
+//------------------------------------------------------------------------------
+void RdbmsCatalogue::restoreArchiveFileInRecycleLog(rdbms::Conn &conn,
+ const cta::common::dataStructures::FileRecycleLog &fileRecycleLog, const std::string &newFid, log::LogContext & lc) {
+ cta::catalogue::ArchiveFileRowWithoutTimestamps row;
+ row.diskFileId = newFid;
+ row.archiveFileId = fileRecycleLog.archiveFileId;
+ row.checksumBlob = fileRecycleLog.checksumBlob;
+ row.diskFileOwnerUid = fileRecycleLog.diskFileUid;
+ row.diskFileGid = fileRecycleLog.diskFileGid;
+ row.diskInstance = fileRecycleLog.diskInstanceName;
+ row.size = fileRecycleLog.sizeInBytes;
+ row.storageClassName = fileRecycleLog.storageClassName;
+ insertArchiveFile(conn, row);
+}
//------------------------------------------------------------------------------
// restoreFilesInRecycleLog
//------------------------------------------------------------------------------
-void RdbmsCatalogue::restoreFilesInRecycleLog(const RecycleTapeFileSearchCriteria & searchCriteria) {
+void RdbmsCatalogue::restoreFileInRecycleLog(const RecycleTapeFileSearchCriteria & searchCriteria, const std::string &newFid) {
try {
auto fileRecycleLogitor = getFileRecycleLogItor(searchCriteria);
auto conn = m_connPool.getConn();
- log::LogContext lc(m_log);
- restoreFileCopiesInRecycleLog(conn, fileRecycleLogitor, lc);
+ log::LogContext lc(m_log);
+ restoreEntryInRecycleLog(conn, fileRecycleLogitor, newFid, lc);
} catch(exception::UserError &) {
throw;
} catch(exception::Exception &ex) {
diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp
index e72b0382a9ac21ec6ffd00640544ab6dd4f9b7df..9b55d937589c49b0946412d671295dae16d0b1cc 100644
--- a/catalogue/RdbmsCatalogue.hpp
+++ b/catalogue/RdbmsCatalogue.hpp
@@ -897,11 +897,22 @@ public:
FileRecycleLogItor getFileRecycleLogItor(const RecycleTapeFileSearchCriteria & searchCriteria) const override;
/**
- * Restores the deleted files in the Recycle log that match the criteria passed
+ * Restores the deleted file in the Recycle log that match the criteria passed
*
* @param searchCriteria The search criteria
+ * @param newFid the new Fid of the archive file (if the archive file must be restored)
*/
- void restoreFilesInRecycleLog(const RecycleTapeFileSearchCriteria & searchCriteria) override;
+ void restoreFileInRecycleLog(const RecycleTapeFileSearchCriteria & searchCriteria, const std::string &newFid) override;
+
+ /**
+ * Copy the fileRecycleLog to the ARCHIVE_FILE with a new eos fxid
+ * @param conn the database connection
+ * @param fileRecycleLog the fileRecycleLog we want to restore
+ * @param newFid the new eos file id of the archive file
+ * @param lc the log context
+ */
+ void restoreArchiveFileInRecycleLog(rdbms::Conn & conn, const common::dataStructures::FileRecycleLog &fileRecycleLogItor,
+ const std::string &newFid, log::LogContext & lc);
/**
* Returns the specified files in tape file sequence order.
@@ -1947,12 +1958,14 @@ protected:
virtual void copyArchiveFileToFileRecyleLogAndDelete(rdbms::Conn & conn,const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) = 0;
/**
- * Copy the fileRecycleLog to the TAPE_FILE table and deletes the corresponding FILE_RECYCLE_LOG table entry
+ * Copy the fileRecycleLog to the TAPE_FILE and ARCHIVE_FILE (if the archive file no longer exists)
+ * table and deletes the corresponding FILE_RECYCLE_LOG table entry
* @param conn the database connection
* @param fileRecycleLog the fileRecycleLog we want to restore
+ * @param newFid The new eos file id of the archive file to create
* @param lc the log context
*/
- virtual void restoreFileCopiesInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor, log::LogContext & lc) = 0;
+ virtual void restoreEntryInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor, const std::string &newFid, log::LogContext & lc) = 0;
/**
* Copies the ARCHIVE_FILE and TAPE_FILE entries to the recycle-bin tables
diff --git a/catalogue/SqliteCatalogue.cpp b/catalogue/SqliteCatalogue.cpp
index a3f3b98c7a30adc3823ca61e359d5b6b3729213d..fbcc428854fe82ab44c4226ca2d1b08c9cdb6351 100644
--- a/catalogue/SqliteCatalogue.cpp
+++ b/catalogue/SqliteCatalogue.cpp
@@ -686,53 +686,47 @@ void SqliteCatalogue::copyTapeFileToFileRecyleLogAndDelete(rdbms::Conn & conn, c
}
//------------------------------------------------------------------------------
-// restoreFileCopiesInRecycleLog
+// restoreEntryInRecycleLog
//------------------------------------------------------------------------------
-void SqliteCatalogue::restoreFileCopiesInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor, log::LogContext & lc) {
-try {
+void SqliteCatalogue::restoreEntryInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor, const std::string &newFid, log::LogContext & lc) {
+ try {
utils::Timer t;
log::TimingList tl;
- //put fileRecycleLogs in std::list so we can release the underlying fileRecycleLogItor database connection
- //otherwise we are using two conns when calling getArchiveFilesItor
- std::list<common::dataStructures::FileRecycleLog> fileRecycleLogList;
- while (fileRecycleLogItor.hasMore()) {
- auto fileRecycleLog = fileRecycleLogItor.next();
- fileRecycleLogList.push_back(fileRecycleLog);
+ if (!fileRecycleLogItor.hasMore()) {
+ throw cta::exception::UserError("No file in the recycle bin matches the parameters passed");
+ }
+ auto fileRecycleLog = fileRecycleLogItor.next();
+ if (fileRecycleLogItor.hasMore()) {
+ //stop restoring more than one file at once
+ throw cta::exception::UserError("More than one recycle bin file matches the parameters passed");
}
- //We currently do all file copy restoring in a single transaction
conn.executeNonQuery("BEGIN TRANSACTION");
- for (auto &fileRecycleLog: fileRecycleLogList) {
- TapeFileSearchCriteria searchCriteria;
- searchCriteria.archiveFileId = fileRecycleLog.archiveFileId;
- searchCriteria.diskInstance = fileRecycleLog.diskInstanceName;
- searchCriteria.diskFileIds = std::vector<std::string>();
- searchCriteria.diskFileIds.value().push_back(fileRecycleLog.diskFileId);
-
- auto itor = getArchiveFilesItor(conn, searchCriteria);
- if (itor.hasMore()) {
- //only restore file copies, do nothing if file has been completely deleted in CTA
- cta::common::dataStructures::ArchiveFile archiveFile = itor.next();
- if (archiveFile.tapeFiles.find(fileRecycleLog.copyNb) != archiveFile.tapeFiles.end()) {
- //copy with same copy_nb exists, cannot restore
- UserSpecifiedExistingDeletedFileCopy ex;
- ex.getMessage() << "Cannot restore file copy with archiveFileId " << std::to_string(fileRecycleLog.archiveFileId)
- << " and copy_nb " << std::to_string(fileRecycleLog.copyNb) << " because a tapefile with same archiveFileId and copy_nb already exists";
- throw ex;
- }
- restoreFileCopyInRecycleLog(conn, fileRecycleLog, lc);
+
+ std::unique_ptr<common::dataStructures::ArchiveFile> archiveFilePtr = getArchiveFileById(conn, fileRecycleLog.archiveFileId);
+ if (!archiveFilePtr) {
+ restoreArchiveFileInRecycleLog(conn, fileRecycleLog, newFid, lc);
+ } else {
+ if (archiveFilePtr->tapeFiles.find(fileRecycleLog.copyNb) != archiveFilePtr->tapeFiles.end()) {
+ //copy with same copy_nb exists, cannot restore
+ UserSpecifiedExistingDeletedFileCopy ex;
+ ex.getMessage() << "Cannot restore file copy with archiveFileId " << std::to_string(fileRecycleLog.archiveFileId)
+ << " and copy_nb " << std::to_string(fileRecycleLog.copyNb) << " because a tapefile with same archiveFileId and copy_nb already exists";
+ throw ex;
}
}
+
+ restoreFileCopyInRecycleLog(conn, fileRecycleLog, lc);
conn.commit();
log::ScopedParamContainer spc(lc);
tl.insertAndReset("commitTime",t);
tl.addToLog(spc);
- lc.log(log::INFO,"In SqliteCatalogue::restoreFileCopiesInRecycleLog: all file copies successfully restored.");
+ lc.log(log::INFO,"In SqliteCatalogue::restoreEntryInRecycleLog: all file copies successfully restored.");
} catch(exception::UserError &) {
- throw;
- } catch(exception::Exception &ex) {
+ throw;
+ } catch(exception::Exception &ex) {
ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
throw;
}
diff --git a/catalogue/SqliteCatalogue.hpp b/catalogue/SqliteCatalogue.hpp
index 70394195df73617de384388c2f98d67583983fb0..9ffca8054a1769631f4e2d6872675d7011958b8e 100644
--- a/catalogue/SqliteCatalogue.hpp
+++ b/catalogue/SqliteCatalogue.hpp
@@ -233,7 +233,7 @@ protected:
* @param fileRecycleLogItor the collection of fileRecycleLogs we want to restore
* @param lc the log context
*/
- void restoreFileCopiesInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor, log::LogContext & lc) override;
+ void restoreEntryInRecycleLog(rdbms::Conn & conn, FileRecycleLogItor &fileRecycleLogItor, const std::string &newFid, log::LogContext & lc) override;
/**
* Copy the fileRecycleLog to the TAPE_FILE table and deletes the corresponding FILE_RECYCLE_LOG table entry
@@ -244,7 +244,6 @@ protected:
void restoreFileCopyInRecycleLog(rdbms::Conn & conn, const common::dataStructures::FileRecycleLog &fileRecycleLogItor, log::LogContext & lc);
-
private:
/**
diff --git a/cmdline/CMakeLists.txt b/cmdline/CMakeLists.txt
index 0feec77374d48d8c619719c2882440ade2dedb54..9ce60b7e60c84daa3c3e4b23be2f734fa30fca92 100644
--- a/cmdline/CMakeLists.txt
+++ b/cmdline/CMakeLists.txt
@@ -15,6 +15,8 @@
cmake_minimum_required (VERSION 2.6)
+add_subdirectory (restore_files)
+
find_package(xrootdclient REQUIRED)
find_package(Protobuf3 REQUIRED)
diff --git a/cmdline/CtaAdminCmdParse.hpp b/cmdline/CtaAdminCmdParse.hpp
index a1e5f38b6384b3bd6038dfc349e1d1867b3b31ca..045ed7219da215ec4cd9c0e19bf061886363954c 100644
--- a/cmdline/CtaAdminCmdParse.hpp
+++ b/cmdline/CtaAdminCmdParse.hpp
@@ -250,7 +250,6 @@ const subcmdLookup_t subcmdLookup = {
{ "rm", AdminCmd::SUBCMD_RM },
{ "up", AdminCmd::SUBCMD_UP },
{ "down", AdminCmd::SUBCMD_DOWN },
- { "restore", AdminCmd::SUBCMD_RESTORE }
};
@@ -426,11 +425,11 @@ const std::map<AdminCmd::Cmd, CmdHelp> cmdHelp = {
}},
{ AdminCmd::CMD_VERSION, { "version", "v", { } }},
{ AdminCmd::CMD_SCHEDULINGINFOS, { "schedulinginfo", "si", { "ls" } }},
- { AdminCmd::CMD_RECYCLETAPEFILE, { "recycletf", "rtf", { "ls", "restore" },
+ { AdminCmd::CMD_RECYCLETAPEFILE, { "recycletf", "rtf", { "ls" },
" This command allows to manage files in the recycle log.\n"
" Tape files in the recycle log can be listed by VID, EOS disk file ID, EOS disk instance, ArchiveFileId or copy number.\n"
- " Disk file IDs should be provided in hexadecimal (fxid).\n"
- " Deleted files can be restored with the restore command\n\n" }},
+ " Disk file IDs should be provided in hexadecimal (fxid).\n\n"
+ }},
};
@@ -641,7 +640,7 @@ const std::map<cmd_key_t, cmd_val_t> cmdOptions = {
{{ AdminCmd::CMD_RECYCLETAPEFILE, AdminCmd::SUBCMD_LS },
{ opt_vid.optional(), opt_fid.optional(), opt_fidfile.optional(), opt_copynb.optional(), opt_archivefileid.optional(), opt_instance.optional() }},
{{ AdminCmd::CMD_RECYCLETAPEFILE, AdminCmd::SUBCMD_RESTORE },
- { opt_vid.optional(), opt_fid.optional(), opt_fidfile.optional(), opt_copynb.optional(), opt_archivefileid.optional(), opt_instance.optional() }},
+ { opt_vid.optional(), opt_fid, opt_copynb.optional(), opt_archivefileid.optional(), opt_instance.optional() }},
};
diff --git a/cmdline/restore_files/CMakeLists.txt b/cmdline/restore_files/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..5fcfc8587b1b469682484592f0aa553f1fb93740
--- /dev/null
+++ b/cmdline/restore_files/CMakeLists.txt
@@ -0,0 +1,37 @@
+# @project The CERN Tape Archive (CTA)
+# @copyright Copyright(C) 2015-2021 CERN
+# @license This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+cmake_minimum_required (VERSION 2.6)
+
+find_package(xrootdclient REQUIRED)
+find_package(Protobuf3 REQUIRED)
+
+# XRootD SSI
+include_directories(${XROOTD_INCLUDE_DIR} ${XROOTD_INCLUDE_DIR}/private )
+
+# XRootD SSI Protocol Buffer bindings
+include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include)
+
+# Compiled protocol buffers
+include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS})
+
+add_executable(cta-restore-deleted-files RestoreFilesCmdLineArgs.cpp RestoreFilesCmdMain.cpp CmdLineTool.cpp RestoreFilesCmd.cpp)
+target_link_libraries(cta-restore-deleted-files ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon XrdSsiCta EosGrpcClient)
+set_property (TARGET cta-restore-deleted-files APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
+
+
+install(TARGETS cta-restore-deleted-files DESTINATION usr/bin)
+install(FILES cta-restore-deleted-files.1cta DESTINATION usr/share/man/man1)
+
diff --git a/cmdline/restore_files/CmdLineTool.cpp b/cmdline/restore_files/CmdLineTool.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b6f17f46bf38309df27c9b1e7427972b881d1aa9
--- /dev/null
+++ b/cmdline/restore_files/CmdLineTool.cpp
@@ -0,0 +1,103 @@
+/*
+ * @project The CERN Tape Archive (CTA)
+ * @copyright Copyright(C) 2015-2021 CERN
+ * @license This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cmdline/restore_files/CmdLineTool.hpp"
+#include "common/exception/CommandLineNotParsed.hpp"
+
+#include <unistd.h>
+
+namespace cta {
+namespace admin {
+
+//------------------------------------------------------------------------------
+// constructor
+//------------------------------------------------------------------------------
+CmdLineTool::CmdLineTool(
+ std::istream &inStream,
+ std::ostream &outStream,
+ std::ostream &errStream) noexcept:
+ m_in(inStream),
+ m_out(outStream),
+ m_err(errStream) {
+}
+
+//------------------------------------------------------------------------------
+// destructor
+//------------------------------------------------------------------------------
+CmdLineTool::~CmdLineTool() noexcept {
+}
+
+//------------------------------------------------------------------------------
+// getUsername
+//------------------------------------------------------------------------------
+std::string CmdLineTool::getUsername() {
+ char buf[256];
+
+ if(getlogin_r(buf, sizeof(buf))) {
+ return "UNKNOWN";
+ } else {
+ return buf;
+ }
+}
+
+//------------------------------------------------------------------------------
+// getHostname
+//------------------------------------------------------------------------------
+std::string CmdLineTool::getHostname() {
+ char buf[256];
+
+ if(gethostname(buf, sizeof(buf))) {
+ return "UNKNOWN";
+ } else {
+ buf[sizeof(buf) - 1] = '\0';
+ return buf;
+ }
+}
+
+//------------------------------------------------------------------------------
+// main
+//------------------------------------------------------------------------------
+int CmdLineTool::main(const int argc, char *const *const argv) {
+ bool cmdLineNotParsed = false;
+ std::string errorMessage;
+
+ try {
+ return exceptionThrowingMain(argc, argv);
+ } catch(exception::CommandLineNotParsed &ue) {
+ errorMessage = ue.getMessage().str();
+ cmdLineNotParsed = true;
+ } catch(exception::Exception &ex) {
+ errorMessage = ex.getMessage().str();
+ } catch(std::exception &se) {
+ errorMessage = se.what();
+ } catch(...) {
+ errorMessage = "An unknown exception was thrown";
+ }
+
+ // Reaching this point means the command has failed, an exception was throw
+ // and errorMessage has been set accordingly
+
+ m_err << "Aborting: " << errorMessage << std::endl;
+ if(cmdLineNotParsed) {
+ m_err << std::endl;
+ printUsage(m_err);
+ }
+ return 1;
+}
+
+} // namespace admin
+} // namespace cta
diff --git a/cmdline/restore_files/CmdLineTool.hpp b/cmdline/restore_files/CmdLineTool.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..013d7ac3c5c0a03b9f2436855efc0fb55d99643c
--- /dev/null
+++ b/cmdline/restore_files/CmdLineTool.hpp
@@ -0,0 +1,106 @@
+/*
+ * @project The CERN Tape Archive (CTA)
+ * @copyright Copyright(C) 2015-2021 CERN
+ * @license This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <istream>
+#include <ostream>
+
+namespace cta {
+namespace admin {
+
+/**
+ * Abstract class implementing common code and data structures for a
+ * command-line tool.
+ */
+class CmdLineTool {
+public:
+ /**
+ * Constructor.
+ *
+ * @param inStream Standard input stream.
+ * @param outStream Standard output stream.
+ * @param errStream Standard error stream.
+ */
+ CmdLineTool(std::istream &inStream, std::ostream &outStream, std::ostream &errStream) noexcept;
+
+ /**
+ * Pure-virtual destructor to guarantee this class is abstract.
+ */
+ virtual ~CmdLineTool() noexcept = 0;
+
+ /**
+ * The object's implementation of main() that should be called from the main()
+ * of the program.
+ *
+ * @param argc The number of command-line arguments including the program name.
+ * @param argv The command-line arguments.
+ * @return The exit value of the program.
+ */
+ int main(const int argc, char *const *const argv);
+
+protected:
+
+ /**
+ * An exception throwing version of main().
+ *
+ * @param argc The number of command-line arguments including the program name.
+ * @param argv The command-line arguments.
+ * @return The exit value of the program.
+ */
+ virtual int exceptionThrowingMain(const int argc, char *const *const argv) = 0;
+
+ /**
+ * Prints the usage message of the command-line tool.
+ *
+ * @param os The output stream to which the usage message is to be printed.
+ */
+ virtual void printUsage(std::ostream &os) = 0;
+
+ /**
+ * Standard input stream.
+ */
+ std::istream &m_in;
+
+ /**
+ * Standard output stream.
+ */
+ std::ostream &m_out;
+
+ /**
+ * Standard error stream.
+ */
+ std::ostream &m_err;
+
+ /**
+ * Returns the name of the user running the command-line tool.
+ *
+ * @return The name of the user running the command-line tool.
+ */
+ static std::string getUsername();
+
+ /**
+ * Returns the name of the host on which the command-line tool is running.
+ *
+ * @return The name of the host on which the command-line tool is running.
+ */
+ static std::string getHostname();
+
+}; // class CmdLineTool
+
+} // namespace admin
+} // namespace cta
diff --git a/cmdline/restore_files/RestoreFilesCmd.cpp b/cmdline/restore_files/RestoreFilesCmd.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9225af4ba227f807d1483d0b56197150cf367d0d
--- /dev/null
+++ b/cmdline/restore_files/RestoreFilesCmd.cpp
@@ -0,0 +1,710 @@
+/*
+ * @project The CERN Tape Archive (CTA)
+ * @copyright Copyright(C) 2015-2021 CERN
+ * @license This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cmdline/restore_files/RestoreFilesCmd.hpp"
+#include "cmdline/restore_files/RestoreFilesCmdLineArgs.hpp"
+#include "common/utils/utils.hpp"
+#include "common/make_unique.hpp"
+#include "CtaFrontendApi.hpp"
+
+#include <XrdSsiPbLog.hpp>
+#include <XrdSsiPbIStreamBuffer.hpp>
+
+#include <grpc++/grpc++.h>
+#include "Rpc.grpc.pb.h"
+
+#include <sys/stat.h>
+#include <iostream>
+#include <memory>
+
+// GLOBAL VARIABLES : used to pass information between main thread and stream handler thread
+
+// global synchronisation flag
+std::atomic<bool> isHeaderSent(false);
+
+std::list<cta::admin::RecycleTapeFileLsItem> deletedTapeFiles;
+
+namespace XrdSsiPb {
+
+/*!
+ * User error exception
+ */
+class UserException : public std::runtime_error
+{
+public:
+ UserException(const std::string &err_msg) : std::runtime_error(err_msg) {}
+}; // class UserException
+
+/*!
+ * Alert callback.
+ *
+ * Defines how Alert messages should be logged
+ */
+template<>
+void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert)
+{
+ std::cout << "AlertCallback():" << std::endl;
+ Log::DumpProtobuf(Log::PROTOBUF, &alert);
+}
+
+/*!
+ * Data/Stream callback.
+ *
+ * Defines how incoming records from the stream should be handled
+ */
+template<>
+void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const
+{
+ using namespace cta::xrd;
+ using namespace cta::admin;
+
+ // Wait for primary response to be handled before allowing stream response
+ while(!isHeaderSent) { std::this_thread::yield(); }
+
+ switch(record.data_case()) {
+ case Data::kRtflsItem:
+ {
+ auto item = record.rtfls_item();
+ deletedTapeFiles.push_back(item);
+ break;
+ }
+ case Data::kTflsItem:
+ break;
+ default:
+ throw std::runtime_error("Received invalid stream data from CTA Frontend for the cta-restore-deleted-files command.");
+ }
+}
+
+} // namespace XrdSsiPb
+
+
+namespace cta{
+namespace admin {
+
+/*!
+ * RestoreFilesCmdException
+ */
+class RestoreFilesCmdException : public std::runtime_error
+{
+public:
+ RestoreFilesCmdException(const std::string &err_msg) : std::runtime_error(err_msg) {}
+}; // class UserException
+
+
+
+//------------------------------------------------------------------------------
+// constructor
+//------------------------------------------------------------------------------
+RestoreFilesCmd::RestoreFilesCmd(std::istream &inStream, std::ostream &outStream,
+ std::ostream &errStream, cta::log::StdoutLogger &log):
+ CmdLineTool(inStream, outStream, errStream),
+ m_log(log) {
+
+ // Default layout: see EOS common/LayoutId.hh for definitions of constants
+ const int kAdler = 0x2;
+ const int kReplica = (0x1 << 4);
+ const int kStripeSize = (0x0 << 8); // 1 stripe
+ const int kStripeWidth = (0x0 << 16); // 4K blocks
+ const int kBlockChecksum = (0x1 << 20);
+
+ // Default single replica layout id should be 00100012
+ m_defaultFileLayout = kReplica | kAdler | kStripeSize | kStripeWidth | kBlockChecksum;
+
+}
+
+//------------------------------------------------------------------------------
+// exceptionThrowingMain
+//------------------------------------------------------------------------------
+int RestoreFilesCmd::exceptionThrowingMain(const int argc, char *const *const argv) {
+ RestoreFilesCmdLineArgs cmdLineArgs(argc, argv);
+ if (cmdLineArgs.m_help) {
+ printUsage(m_out);
+ return 0;
+ }
+
+ readAndSetConfiguration(getUsername(), cmdLineArgs);
+ listDeletedFilesCta();
+ for (auto &file: deletedTapeFiles) {
+ /*From https://codimd.web.cern.ch/f9JQv3YzSmKJ_W_ezXN3fA#
+ When a user deletes a file, there are tree scenarios for recovery:
+ * The file has been deleted in the EOS namespace and the CTA catalogue
+ (normal case, must restore in both places)
+ * EOS namespace entry is kept and diskFileId has not changed
+ (just restore in CTA)
+ * EOS namespace entry is kept and diskFileId has changed
+ (restore the file with the new eos disk file id)
+ */
+ try {
+ if (!fileExistsEos(file.disk_instance(), file.disk_file_id())) {
+ uint64_t new_fid = restoreDeletedFileEos(file);
+ file.set_disk_file_id(std::to_string(new_fid));
+ }
+ //archive file exists in CTA, so only need to restore the file copy
+ restoreDeletedFileCopyCta(file);
+ } catch (RestoreFilesCmdException &e) {
+ m_log(cta::log::ERR,e.what());
+ }
+ }
+ return 0;
+}
+
+//------------------------------------------------------------------------------
+// readAndSetConfiguration
+//------------------------------------------------------------------------------
+void RestoreFilesCmd::readAndSetConfiguration(
+ const std::string &userName,
+ const RestoreFilesCmdLineArgs &cmdLineArgs) {
+
+ m_vid = cmdLineArgs.m_vid;
+ m_diskInstance = cmdLineArgs.m_diskInstance;
+ m_eosFxids = cmdLineArgs.m_eosFxids;
+ m_copyNumber = cmdLineArgs.m_copyNumber;
+ m_archiveFileId = cmdLineArgs.m_archiveFileId;
+
+ if (cmdLineArgs.m_debug) {
+ m_log.setLogMask("DEBUG");
+ } else {
+ m_log.setLogMask("INFO");
+ }
+
+ // Set cta frontend configuration options
+ const std::string cli_config_file = "/etc/cta/cta-cli.conf";
+ XrdSsiPb::Config cliConfig(cli_config_file, "cta");
+ cliConfig.set("resource", "/ctafrontend");
+ cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes
+ cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s
+
+ // Allow environment variables to override config file
+ cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT");
+
+ // If XRDDEBUG=1, switch on all logging
+ if(getenv("XRDDEBUG")) {
+ cliConfig.set("log", "all");
+ }
+ // If fine-grained control over log level is required, use XrdSsiPbLogLevel
+ cliConfig.getEnv("log", "XrdSsiPbLogLevel");
+
+ // Validate that endpoint was specified in the config file
+ if(!cliConfig.getOptionValueStr("endpoint").first) {
+ throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file);
+ }
+
+ // If the server is down, we want an immediate failure. Set client retry to a single attempt.
+ XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1);
+
+ m_serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig));
+
+ // Set cta frontend configuration options to connect to eos
+ const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf";
+ XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta");
+
+ // Get the endpoint for namespace queries
+ auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config");
+ if(nsConf.first) {
+ setNamespaceMap(nsConf.second);
+ } else {
+ throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file);
+ }
+}
+
+void RestoreFilesCmd::setNamespaceMap(const std::string &keytab_file) {
+ // Open the keytab file for reading
+ std::ifstream file(keytab_file);
+ if(!file) {
+ throw cta::exception::UserError("Failed to open namespace keytab configuration file " + keytab_file);
+ }
+ ::eos::client::NamespaceMap_t namespaceMap;
+ // Parse the keytab line by line
+ std::string line;
+ for(int lineno = 0; std::getline(file, line); ++lineno) {
+ // Strip out comments
+ auto pos = line.find('#');
+ if(pos != std::string::npos) {
+ line.resize(pos);
+ }
+
+ // Parse one line
+ std::stringstream ss(line);
+ std::string diskInstance;
+ std::string endpoint;
+ std::string token;
+ std::string eol;
+ ss >> diskInstance >> endpoint >> token >> eol;
+
+ // Ignore blank lines, all other lines must have exactly 3 elements
+ if(token.empty() || !eol.empty()) {
+ if(diskInstance.empty() && endpoint.empty() && token.empty()) continue;
+ throw cta::exception::UserError("Could not parse namespace keytab configuration file line " + std::to_string(lineno) + ": " + line);
+ }
+ namespaceMap.insert(std::make_pair(diskInstance, ::eos::client::Namespace(endpoint, token)));
+ }
+ m_endpointMapPtr = cta::make_unique<::eos::client::EndpointMap>(namespaceMap);
+}
+
+//------------------------------------------------------------------------------
+// listDeletedFilesCta
+//------------------------------------------------------------------------------
+void RestoreFilesCmd::listDeletedFilesCta() const {
+ std::list<cta::log::Param> params;
+ params.push_back(cta::log::Param("userName", getUsername()));
+
+ cta::xrd::Request request;
+
+ auto &admincmd = *(request.mutable_admincmd());
+
+ request.set_client_cta_version(CTA_VERSION);
+ request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION);
+ admincmd.set_cmd(AdminCmd::CMD_RECYCLETAPEFILE);
+ admincmd.set_subcmd(AdminCmd::SUBCMD_LS);
+
+ if (m_vid) {
+ params.push_back(cta::log::Param("tapeVid", m_vid.value()));
+ auto key = OptionString::VID;
+ auto new_opt = admincmd.add_option_str();
+ new_opt->set_key(key);
+ new_opt->set_value(m_vid.value());
+ }
+ if (m_diskInstance) {
+ params.push_back(cta::log::Param("diskInstance", m_diskInstance.value()));
+ auto key = OptionString::INSTANCE;
+ auto new_opt = admincmd.add_option_str();
+ new_opt->set_key(key);
+ new_opt->set_value(m_diskInstance.value());
+ }
+ if (m_archiveFileId) {
+ params.push_back(cta::log::Param("archiveFileId", m_archiveFileId.value()));
+ auto key = OptionUInt64::ARCHIVE_FILE_ID;
+ auto new_opt = admincmd.add_option_uint64();
+ new_opt->set_key(key);
+ new_opt->set_value(m_archiveFileId.value());
+ }
+ if (m_copyNumber) {
+ params.push_back(cta::log::Param("copyNb", m_copyNumber.value()));
+ auto key = OptionUInt64::COPY_NUMBER;
+ auto new_opt = admincmd.add_option_uint64();
+ new_opt->set_key(key);
+ new_opt->set_value(m_copyNumber.value());
+ }
+ if (m_eosFxids) {
+ std::stringstream ss;
+ auto key = OptionStrList::FILE_ID;
+ auto new_opt = admincmd.add_option_str_list();
+ new_opt->set_key(key);
+ for (auto &fxid: m_eosFxids.value()) {
+ new_opt->add_item(fxid);
+ ss << fxid << ",";
+ }
+ auto fids = ss.str();
+ fids.pop_back(); //remove last ","
+ params.push_back(cta::log::Param("diskFileId", fids));
+
+ }
+
+
+ m_log(cta::log::INFO, "Listing deleted file in cta catalogue", params);
+
+ // Send the Request to the Service and get a Response
+ cta::xrd::Response response;
+ auto stream_future = m_serviceProviderPtr->SendAsync(request, response);
+
+ // Handle responses
+ switch(response.type())
+ {
+ using namespace cta::xrd;
+ using namespace cta::admin;
+ case Response::RSP_SUCCESS:
+ // Print message text
+ std::cout << response.message_txt();
+ // Allow stream processing to commence
+ isHeaderSent = true;
+ break;
+ case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt());
+ case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt());
+ case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt());
+ default: throw XrdSsiPb::PbException("Invalid response type.");
+ }
+
+ // wait until the data stream has been processed before exiting
+ stream_future.wait();
+
+ params.push_back(cta::log::Param("nbFiles", deletedTapeFiles.size()));
+ m_log(cta::log::INFO, "Listed deleted file in cta catalogue", params);
+
+}
+
+//------------------------------------------------------------------------------
+// restoreDeletedFileCopyCta
+//------------------------------------------------------------------------------
+void RestoreFilesCmd::restoreDeletedFileCopyCta(const RecycleTapeFileLsItem &file) const {
+
+ std::list<cta::log::Param> params;
+ params.push_back(cta::log::Param("userName", getUsername()));
+ params.push_back(cta::log::Param("tapeVid", file.vid()));
+ params.push_back(cta::log::Param("diskInstance", file.disk_instance()));
+ params.push_back(cta::log::Param("archiveFileId", file.archive_file_id()));
+ params.push_back(cta::log::Param("copyNb", file.copy_nb()));
+ params.push_back(cta::log::Param("diskFileId", file.disk_file_id()));
+
+ m_log(cta::log::DEBUG, "Restoring file copy in cta catalogue", params);
+
+ cta::xrd::Request request;
+
+ auto &admincmd = *(request.mutable_admincmd());
+
+ request.set_client_cta_version(CTA_VERSION);
+ request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION);
+ admincmd.set_cmd(AdminCmd::CMD_RECYCLETAPEFILE);
+ admincmd.set_subcmd(AdminCmd::SUBCMD_RESTORE);
+
+ {
+ auto key = OptionString::VID;
+ auto new_opt = admincmd.add_option_str();
+ new_opt->set_key(key);
+ new_opt->set_value(file.vid());
+ }
+ {
+ auto key = OptionString::INSTANCE;
+ auto new_opt = admincmd.add_option_str();
+ new_opt->set_key(key);
+ new_opt->set_value(file.disk_instance());
+ }
+ {
+ auto key = OptionUInt64::ARCHIVE_FILE_ID;
+ auto new_opt = admincmd.add_option_uint64();
+ new_opt->set_key(key);
+ new_opt->set_value(file.archive_file_id());
+ }
+ {
+ auto key = OptionUInt64::COPY_NUMBER;
+ auto new_opt = admincmd.add_option_uint64();
+ new_opt->set_key(key);
+ new_opt->set_value(file.copy_nb());
+ }
+ {
+ auto key = OptionString::FXID;
+ auto new_opt = admincmd.add_option_str();
+ new_opt->set_key(key);
+ new_opt->set_value(file.disk_file_id());
+
+ }
+ // Send the Request to the Service and get a Response
+ cta::xrd::Response response;
+ m_serviceProviderPtr->Send(request, response);
+
+ // Handle responses
+ switch(response.type())
+ {
+ using namespace cta::xrd;
+ using namespace cta::admin;
+ case Response::RSP_SUCCESS:
+ // Print message text
+ std::cout << response.message_txt();
+ m_log(cta::log::INFO, "Restored file copy in cta catalogue", params);
+ break;
+ case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt());
+ case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt());
+ case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt());
+ default: throw XrdSsiPb::PbException("Invalid response type.");
+ }
+}
+
+//------------------------------------------------------------------------------
+// addContainerEos
+//------------------------------------------------------------------------------
+int RestoreFilesCmd::addContainerEos(const std::string &diskInstance, const std::string &path, const std::string &sc) const {
+ int c_id = containerExistsEos(diskInstance, path);
+ if (c_id) {
+ return c_id;
+ }
+ auto enclosingPath = cta::utils::getEnclosingPath(path);
+ int parent_id = containerExistsEos(diskInstance, enclosingPath);
+ if (!parent_id) {
+ //parent does not exist, need to add it as well
+ parent_id = addContainerEos(diskInstance, enclosingPath, sc);
+ }
+
+ std::list<cta::log::Param> params;
+ params.push_back(cta::log::Param("userName", getUsername()));
+ params.push_back(cta::log::Param("diskInstance", diskInstance));
+ params.push_back(cta::log::Param("path", path));
+ m_log(cta::log::DEBUG, "Inserting container in EOS namespace", params);
+
+ ::eos::rpc::ContainerMdProto dir;
+ dir.set_path(path);
+ dir.set_name(cta::utils::getEnclosedName(path));
+
+ // Filemode: filter out S_ISUID, S_ISGID and S_ISVTX because EOS does not follow POSIX semantics for these bits
+ uint64_t filemode = (S_IRWXU | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); // 0755 permissions by default
+ filemode &= ~(S_ISUID | S_ISGID | S_ISVTX);
+ dir.set_mode(filemode);
+
+ auto time = ::time(nullptr);
+ // Timestamps
+ dir.mutable_ctime()->set_sec(time);
+ dir.mutable_mtime()->set_sec(time);
+ // we don't care about dir.stime (sync time, used for CERNBox)
+
+ dir.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("sys.archive.storage_class", sc));
+
+
+ auto reply = m_endpointMapPtr->containerInsert(diskInstance, dir);
+
+ m_log(cta::log::DEBUG, "Inserted container in EOS namespace successfully, querying again for its id", params);
+
+ int cont_id = containerExistsEos(diskInstance, path);
+ if (!cont_id) {
+ throw RestoreFilesCmdException(std::string("Container ") + path + " does not exist after being inserted in EOS.");
+ }
+ return cont_id;
+}
+
+//------------------------------------------------------------------------------
+// containerExistsEos
+//------------------------------------------------------------------------------
+int RestoreFilesCmd::containerExistsEos(const std::string &diskInstance, const std::string &path) const {
+ std::list<cta::log::Param> params;
+ params.push_back(cta::log::Param("userName", getUsername()));
+ params.push_back(cta::log::Param("diskInstance", diskInstance));
+ params.push_back(cta::log::Param("path", path));
+
+ m_log(cta::log::DEBUG, "Verifying if the container exists in the EOS namespace", params);
+
+ auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::CONTAINER, 0, path, false);
+ int cid = md_response.cmd().id();
+ params.push_back(cta::log::Param("containerId", cid));
+ if (cid != 0) {
+ m_log(cta::log::DEBUG, "Container exists in the eos namespace", params);
+ } else {
+ m_log(cta::log::DEBUG, "Container does not exist in the eos namespace", params);
+ }
+ return cid;
+}
+
+bool RestoreFilesCmd::fileWasDeletedByRM(const RecycleTapeFileLsItem &file) const {
+ return file.reason_log().rfind("(Deleted using cta-admin tf rm)", 0) == 0;
+}
+
+//------------------------------------------------------------------------------
+// archiveFileExistsCTA
+//------------------------------------------------------------------------------
+bool RestoreFilesCmd::archiveFileExistsCTA(const uint64_t &archiveFileId) const {
+
+ std::list<cta::log::Param> params;
+ params.push_back(cta::log::Param("userName", getUsername()));
+ params.push_back(cta::log::Param("archiveFileId", archiveFileId));
+
+ m_log(cta::log::DEBUG, "Looking for archive file in the cta catalogue", params);
+
+ cta::xrd::Request request;
+
+ auto &admincmd = *(request.mutable_admincmd());
+
+ request.set_client_cta_version(CTA_VERSION);
+ request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION);
+ admincmd.set_cmd(AdminCmd::CMD_TAPEFILE);
+ admincmd.set_subcmd(AdminCmd::SUBCMD_LS);
+
+ auto key = OptionUInt64::ARCHIVE_FILE_ID;
+ auto new_opt = admincmd.add_option_uint64();
+ new_opt->set_key(key);
+ new_opt->set_value(archiveFileId);
+
+ // Send the Request to the Service and get a Response
+ cta::xrd::Response response;
+ auto stream_future = m_serviceProviderPtr->SendAsync(request, response);
+
+ bool ret;
+ // Handle responses
+ switch(response.type())
+ {
+ using namespace cta::xrd;
+ using namespace cta::admin;
+ case Response::RSP_SUCCESS: ret = true; break; //success sent if archive file does not exist
+ case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt());
+ case Response::RSP_ERR_USER: ret = false; break; //user error sent if archive file does not exist
+ case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt());
+ default: throw XrdSsiPb::PbException("Invalid response type.");
+ }
+
+ // wait until the data stream has been processed before exiting
+ if (ret) {
+ stream_future.wait();
+ }
+ if (ret) {
+ m_log(cta::log::DEBUG, "Archive file is present in the CTA catalogue", params);
+ } else {
+ m_log(cta::log::DEBUG, "Archive file is missing in the CTA catalogue", params);
+ }
+ return ret;
+}
+
+//------------------------------------------------------------------------------
+// fileExistsEos
+//------------------------------------------------------------------------------
+bool RestoreFilesCmd::fileExistsEos(const std::string &diskInstance, const std::string &diskFileId) const {
+ std::list<cta::log::Param> params;
+ params.push_back(cta::log::Param("userName", getUsername()));
+ params.push_back(cta::log::Param("diskInstance", diskInstance));
+ params.push_back(cta::log::Param("diskFileId", diskFileId));
+
+ m_log(cta::log::DEBUG, "Verifying if eos fxid exists in the EOS namespace", params);
+ try {
+ auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId);
+ params.push_back(cta::log::Param("diskFilePath", path));
+ m_log(cta::log::DEBUG, "eos fxid exists in the EOS namespace");
+ return true;
+ } catch(cta::exception::Exception) {
+ m_log(cta::log::DEBUG, "eos fxid does not exist in the EOS namespace");
+ return false;
+ }
+}
+
+//------------------------------------------------------------------------------
+// getFileIdEos
+//------------------------------------------------------------------------------
+int RestoreFilesCmd::getFileIdEos(const std::string &diskInstance, const std::string &path) const {
+ std::list<cta::log::Param> params;
+ params.push_back(cta::log::Param("userName", getUsername()));
+ params.push_back(cta::log::Param("diskInstance", diskInstance));
+ params.push_back(cta::log::Param("path", path));
+
+ m_log(cta::log::DEBUG, "Querying for file metadata in the eos namespace", params);
+ auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, 0, path, false);
+ int fid = md_response.fmd().id();
+ params.push_back(cta::log::Param("diskFileId", fid));
+ if (fid != 0) {
+ m_log(cta::log::DEBUG, "File path exists in the eos namespace", params);
+ } else {
+ m_log(cta::log::DEBUG, "File path does not exist in the eos namespace", params);
+ }
+ return fid;
+}
+
+//------------------------------------------------------------------------------
+// getCurrentEosIds
+//------------------------------------------------------------------------------
+void RestoreFilesCmd::getCurrentEosIds(const std::string &diskInstance) const {
+ uint64_t cid;
+ uint64_t fid;
+ m_endpointMapPtr->getCurrentIds(diskInstance, cid, fid);
+
+ std::list<cta::log::Param> params;
+ params.push_back(cta::log::Param("diskInstance", diskInstance));
+ params.push_back(cta::log::Param("ContainerId", cid));
+ params.push_back(cta::log::Param("FileId", fid));
+ m_log(cta::log::DEBUG, "Obtained current eos container and file id", params);
+}
+
+
+//------------------------------------------------------------------------------
+// restoreDeletedFileEos
+//------------------------------------------------------------------------------
+uint64_t RestoreFilesCmd::restoreDeletedFileEos(const RecycleTapeFileLsItem &rtfls_item) const {
+
+ std::list<cta::log::Param> params;
+ params.push_back(cta::log::Param("userName", getUsername()));
+ params.push_back(cta::log::Param("diskInstance", rtfls_item.disk_instance()));
+ params.push_back(cta::log::Param("archiveFileId", rtfls_item.archive_file_id()));
+ params.push_back(cta::log::Param("diskFileId", rtfls_item.disk_file_id()));
+ params.push_back(cta::log::Param("diskFilePath", rtfls_item.disk_file_path()));
+
+ m_log(cta::log::INFO, "Restoring file in the eos namespace", params);
+
+ getCurrentEosIds(rtfls_item.disk_instance());
+ uint64_t file_id = getFileIdEos(rtfls_item.disk_instance(), rtfls_item.disk_file_path());
+ if (file_id) {
+ return file_id; //eos disk file id was changed since the file was deleted, just return the new file id
+ }
+
+ ::eos::rpc::FileMdProto file;
+
+ auto fullPath = rtfls_item.disk_file_path();
+ auto cont_id = addContainerEos(rtfls_item.disk_instance(), cta::utils::getEnclosingPath(fullPath), rtfls_item.storage_class());
+
+ // We do not set the file id. Since the file was deleted the fid cannot be reused, so EOS will generate a new file id
+ file.set_cont_id(cont_id);
+ file.set_uid(rtfls_item.disk_file_uid());
+ file.set_gid(rtfls_item.disk_file_gid());
+ file.set_size(rtfls_item.size_in_bytes());
+ file.set_layout_id(m_defaultFileLayout);
+
+ // Filemode: filter out S_ISUID, S_ISGID and S_ISVTX because EOS does not follow POSIX semantics for these bits
+ uint64_t filemode = (S_IRWXU | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); // 0755
+ filemode &= ~(S_ISUID | S_ISGID | S_ISVTX);
+ file.set_flags(filemode);
+
+ // Timestamps
+ auto time = ::time(nullptr);
+ file.mutable_ctime()->set_sec(time);
+ file.mutable_mtime()->set_sec(time);
+ // we don't care about file.stime (sync time, used for CERNBox)
+ // BTIME is set as an extended attribute (see below)
+
+ // Filename and path
+ file.set_path(fullPath);
+ file.set_name(cta::utils::getEnclosedName(fullPath));
+
+ // Checksums
+ if(rtfls_item.checksum().empty()) {
+ throw RestoreFilesCmdException("File " + rtfls_item.disk_file_id() + " does not have a checksum");
+ }
+ std::string checksumType("NONE");
+ std::string checksumValue;
+ const google::protobuf::EnumDescriptor *descriptor = cta::common::ChecksumBlob::Checksum::Type_descriptor();
+ checksumType = descriptor->FindValueByNumber(rtfls_item.checksum().begin()->type())->name();
+ checksumValue = rtfls_item.checksum().begin()->value();
+ file.mutable_checksum()->set_type(checksumType); //only support adler for now
+ file.mutable_checksum()->set_value(checksumValue);
+
+ // Extended attributes:
+ //
+ // 1. Archive File ID
+ std::string archiveId(std::to_string(rtfls_item.archive_file_id()));
+ file.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("sys.archive.file_id", archiveId));
+ // 2. Storage Class
+ file.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("sys.archive.storage_class", rtfls_item.storage_class()));
+ // 3. Birth Time
+ // POSIX ATIME (Access Time) is used by CASTOR to store the file creation time. EOS calls this "birth time",
+ // but there is no place in the namespace to store it, so it is stored as an extended attribute.
+ file.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("eos.btime", std::to_string(time)));
+
+ // Indicate that there is a tape-resident replica of this file (except for zero-length files)
+ if(file.size() > 0) {
+ file.mutable_locations()->Add(65535);
+ }
+
+ auto diskInstance = rtfls_item.disk_instance();
+ auto reply = m_endpointMapPtr->fileInsert(diskInstance, file);
+
+ m_log(cta::log::INFO, "File successfully restored in the eos namespace", params);
+
+ m_log(cta::log::DEBUG, "Querying eos for the new eos file id", params);
+
+ auto new_fid = getFileIdEos(rtfls_item.disk_instance(), rtfls_item.disk_file_path());
+ return new_fid;
+
+}
+
+//------------------------------------------------------------------------------
+// printUsage
+//------------------------------------------------------------------------------
+void RestoreFilesCmd::printUsage(std::ostream &os) {
+ RestoreFilesCmdLineArgs::printUsage(os);
+}
+
+} // namespace admin
+} // namespace cta
\ No newline at end of file
diff --git a/cmdline/restore_files/RestoreFilesCmd.hpp b/cmdline/restore_files/RestoreFilesCmd.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..0150a0e8a56fc43bb21e3184db00b4143cd9bd6f
--- /dev/null
+++ b/cmdline/restore_files/RestoreFilesCmd.hpp
@@ -0,0 +1,209 @@
+/*
+ * @project The CERN Tape Archive (CTA)
+ * @copyright Copyright(C) 2015-2021 CERN
+ * @license This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "cmdline/restore_files/CmdLineTool.hpp"
+#include "cmdline/restore_files/RestoreFilesCmdLineArgs.hpp"
+#include "eos_grpc_client/GrpcEndpoint.hpp"
+#include "catalogue/Catalogue.hpp"
+#include "common/log/StdoutLogger.hpp"
+#include "common/optional.hpp"
+#include <memory>
+
+#include "CtaFrontendApi.hpp"
+
+namespace cta {
+namespace admin {
+
+class RestoreFilesCmd: public CmdLineTool {
+public:
+ /**
+ * Constructor.
+ *
+ * @param inStream Standard input stream.
+ * @param outStream Standard output stream.
+ * @param errStream Standard error stream.
+ * @param log The object representing the API of the CTA logging system.
+ */
+ RestoreFilesCmd(std::istream &inStream, std::ostream &outStream,
+ std::ostream &errStream, cta::log::StdoutLogger &log);
+
+ /**
+ * An exception throwing version of main().
+ *
+ * @param argc The number of command-line arguments including the program name.
+ * @param argv The command-line arguments.
+ * @return The exit value of the program.
+ */
+ int exceptionThrowingMain(const int argc, char *const *const argv) override;
+
+ /**
+ * Sets internal configuration parameters to be used for reading.
+ * It reads cta frontend parameters from /etc/cta/cta-cli.conf
+ *
+ * @param username The name of the user running the command-line tool.
+ * @param cmdLineArgs The arguments parsed from the command line.
+ */
+ void readAndSetConfiguration(const std::string &userName, const RestoreFilesCmdLineArgs &cmdLineArgs);
+
+ /**
+ * Populate the namespace endpoint configuration from a keytab file
+ */
+ void setNamespaceMap(const std::string &keytab_file);
+
+ /**
+ * Restores the specified deleted files in the cta catalogue
+ */
+ void listDeletedFilesCta() const;
+
+ /**
+ * Queries the eos mgm for the current eos file and container id
+ * Must be called before any other call to EOS, to initialize the grpc
+ * client cid and fid
+ * @param diskInstance the disk instance of the eos instance
+ */
+ void getCurrentEosIds(const std::string &diskInstance) const;
+
+ /**
+ * Restores the deleted file present in the cta catalogue recycle bin
+ * @param file the deleted tape file
+ */
+ void restoreDeletedFileCopyCta(const RecycleTapeFileLsItem &file) const;
+
+ /**
+ * Adds a container in the eos namespace, if it does not exist
+ * @param diskInstance eos disk instance
+ * @param path the path of the container
+ * @param sc the storage class of the container
+ * @returns the container id of the container identified by path
+ */
+ int addContainerEos(const std::string &diskInstance, const std::string &path, const std::string &sc) const;
+
+ /**
+ * Returns true (i.e. not zero) if a container exists in the eos namespace
+ * @param diskInstance eos disk instance
+ * @param path the path of the container
+ */
+ int containerExistsEos(const std::string &diskInstance, const std::string &path) const;
+
+ /**
+ * Returns true (i.e. not zero) if a file with given id exists in the eos namespace
+ * @param diskInstance eos disk instance
+ * @param diskFileId the eos file id to check
+ */
+ bool fileExistsEos(const std::string &diskInstance, const std::string &diskFileId) const;
+
+ /**
+ * Returns true (i.e. not zero) if a file was deleted using cta-admin tf rm
+ * @param diskInstance eos disk instance
+ * @param diskFileId the eos file id to check
+ */
+ bool fileWasDeletedByRM(const RecycleTapeFileLsItem &file) const;
+
+ /**
+ * Returns true (i.e. not zero) if an archive file with given id exists in the cta catalogue
+ * @param archiveFileId the archive file id to check
+ */
+ bool archiveFileExistsCTA(const uint64_t &archiveFileId) const;
+
+ /**
+ * Returns the id of a given file in eos or zero if the files does not exist
+ * @param diskInstance eos disk instance
+ * @param path the path to check
+ */
+ int getFileIdEos(const std::string &diskInstance, const std::string &path) const;
+
+ /**
+ * Restores the deleted file present in the eos namespace
+ * @param file the deleted tape file
+ */
+ uint64_t restoreDeletedFileEos(const RecycleTapeFileLsItem &file) const;
+
+ /**
+ * Prints the usage message of the command-line tool.
+ *
+ * @param os The output stream to which the usage message is to be printed.
+ */
+ void printUsage(std::ostream &os) override;
+
+private:
+ /**
+ * The object representing the API of the CTA logging system.
+ */
+ cta::log::StdoutLogger &m_log;
+
+ /**
+ * True if the command should just restore deleted tape file copies
+ */
+ bool m_restoreCopies;
+
+ /**
+ * True if the command should just restore deleted archive files
+ */
+ bool m_restoreFiles;
+
+ /**
+ * Archive file id of the files to restore
+ */
+ optional<uint64_t> m_archiveFileId;
+
+ /**
+ * Disk instance of the files to restore
+ */
+ optional<std::string> m_diskInstance;
+
+ /**
+ * Fxids of the files to restore
+ */
+ optional<std::list<std::string>> m_eosFxids;
+
+ /**
+ * Vid of the tape of the files to restore
+ */
+ optional<std::string> m_vid;
+
+ /**
+ *Copy number of the files to restore
+ */
+ optional<uint64_t> m_copyNumber;
+
+ /**
+ *Default file layout for restored files in EOS
+ */
+ int m_defaultFileLayout;
+
+ /**
+ * CTA Frontend service provider
+ */
+ std::unique_ptr<XrdSsiPbServiceType> m_serviceProviderPtr;
+
+ std::unique_ptr<::eos::client::EndpointMap> m_endpointMapPtr;
+
+ const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses
+ const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by
+ //!< XRD_REQUESTTIMEOUT environment variable.
+
+ const std::string m_ctaVersion = CTA_VERSION; //!< The version of CTA
+ const std::string m_xrootdSsiProtobufInterfaceVersion = //!< The xrootd-ssi-protobuf-interface version (=tag)
+ XROOTD_SSI_PROTOBUF_INTERFACE_VERSION;
+
+
+} ; // class RestoreFilesCmd
+
+} // namespace admin
+} // namespace cta
diff --git a/cmdline/restore_files/RestoreFilesCmdLineArgs.cpp b/cmdline/restore_files/RestoreFilesCmdLineArgs.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b552a8ab519bf35d137b17fd17fabd1d4d661ae9
--- /dev/null
+++ b/cmdline/restore_files/RestoreFilesCmdLineArgs.cpp
@@ -0,0 +1,193 @@
+/*
+ * @project The CERN Tape Archive (CTA)
+ * @copyright Copyright(C) 2015-2021 CERN
+ * @license This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "RestoreFilesCmdLineArgs.hpp"
+#include "common/exception/CommandLineNotParsed.hpp"
+
+#include <getopt.h>
+#include <ostream>
+#include <sstream>
+#include <iostream>
+#include <string>
+#include <limits.h>
+#include <fstream>
+
+
+namespace cta {
+namespace admin{
+
+//------------------------------------------------------------------------------
+// constructor
+//------------------------------------------------------------------------------
+RestoreFilesCmdLineArgs::RestoreFilesCmdLineArgs(const int argc, char *const *const argv):
+m_help(false), m_debug(false) {
+
+ static struct option longopts[] = {
+ {"id", required_argument, NULL, 'I'},
+ {"instance", required_argument, NULL, 'i'},
+ {"fxid", required_argument, NULL, 'f'},
+ {"fxidfile", required_argument, NULL, 'F'},
+ {"vid", required_argument, NULL, 'v'},
+ {"copynb", required_argument, NULL, 'c'},
+ {"help", no_argument, NULL, 'h'},
+ {"debug", no_argument, NULL, 'd'},
+ {NULL, 0, NULL, 0}
+ };
+
+ opterr = 0;
+ int opt = 0;
+ int opt_index = 3;
+
+ while ((opt = getopt_long(argc, argv, "I:i:f:F:v:c:hd", longopts, &opt_index)) != -1) {
+ switch(opt) {
+ case 'I':
+ {
+ int64_t archiveId = std::stol(std::string(optarg));
+ if(archiveId < 0) throw std::out_of_range("archive id value cannot be negative");
+ m_archiveFileId = archiveId;
+ break;
+ }
+ case 'i':
+ {
+ m_diskInstance = std::string(optarg);
+ break;
+ }
+ case 'f':
+ {
+ if (! m_eosFxids) {
+ m_eosFxids = std::list<std::string>();
+ }
+ auto fid = strtol(optarg, nullptr, 16);
+ if(fid < 1 || fid == LONG_MAX) {
+ throw std::runtime_error(std::string(optarg) + " is not a valid file ID");
+ }
+ m_eosFxids->push_back(std::to_string(fid));
+ break;
+ }
+ case 'F':
+ {
+ if (! m_eosFxids) {
+ m_eosFxids = std::list<std::string>();
+ }
+ readFidListFromFile(std::string(optarg), m_eosFxids.value());
+ break;
+ }
+ case 'v':
+ {
+ m_vid = std::string(optarg);
+ break;
+ }
+ case 'c':
+ {
+ int64_t copyNumber = std::stol(std::string(optarg));
+ if(copyNumber < 0) throw std::out_of_range("copy number value cannot be negative");
+ m_copyNumber = copyNumber;
+ break;
+ }
+ case 'h':
+ {
+ m_help = true;
+ break;
+ }
+ case 'd':
+ {
+ m_debug = true;
+ break;
+ }
+ case ':': // Missing parameter
+ {
+ exception::CommandLineNotParsed ex;
+ ex.getMessage() << "The -" << (char)optopt << " option requires a parameter";
+ throw ex;
+ }
+ case '?': // Unknown option
+ {
+ exception::CommandLineNotParsed ex;
+ if(0 == optopt) {
+ ex.getMessage() << "Unknown command-line option";
+ } else {
+ ex.getMessage() << "Unknown command-line option: -" << (char)optopt;
+ }
+ throw ex;
+ }
+ default:
+ {
+ exception::CommandLineNotParsed ex;
+ ex.getMessage() <<
+ "getopt_long returned the following unknown value: 0x" <<
+ std::hex << (int)opt;
+ throw ex;
+ }
+ }
+ }
+}
+
+//------------------------------------------------------------------------------
+// readFidListFromFile
+//------------------------------------------------------------------------------
+void RestoreFilesCmdLineArgs::readFidListFromFile(const std::string &filename, std::list<std::string> &optionList) {
+ std::ifstream file(filename);
+ if (file.fail()) {
+ throw std::runtime_error("Unable to open file " + filename);
+ }
+
+ std::string line;
+
+ while(std::getline(file, line)) {
+ // Strip out comments
+ auto pos = line.find('#');
+ if(pos != std::string::npos) {
+ line.resize(pos);
+ }
+
+ // Extract the list items
+ std::stringstream ss(line);
+ while(!ss.eof()) {
+ std::string item;
+ ss >> item;
+ // skip blank lines or lines consisting only of whitespace
+ if(item.empty()) continue;
+
+ // Special handling for file id lists. The output from "eos find --fid <fid> /path" is:
+ // path=/path fid=<fid>
+ // We discard everything except the list of fids. <fid> is a zero-padded hexadecimal number,
+ // but in the CTA catalogue we store disk IDs as a decimal string, so we need to convert it.
+ if(item.substr(0, 4) == "fid=") {
+ auto fid = strtol(item.substr(4).c_str(), nullptr, 16);
+ if(fid < 1 || fid == LONG_MAX) {
+ throw std::runtime_error(item + " is not a valid file ID");
+ }
+ optionList.push_back(std::to_string(fid));
+ } else {
+ continue;
+ }
+ }
+ }
+}
+
+//------------------------------------------------------------------------------
+// printUsage
+//------------------------------------------------------------------------------
+void RestoreFilesCmdLineArgs::printUsage(std::ostream &os) {
+ os << "Usage:" << std::endl <<
+ " cta-restore-deleted-files [--id/-I <archive_file_id>] [--instance/-i <disk_instance>]" << std::endl <<
+ " [--fxid/-f <eos_fxid>] [--fxidfile/-F <filename>]" << std::endl <<
+ " [--vid/-v <vid>] [--copynb/-c <copy_number>] [--debug/-d]" << std::endl;
+}
+
+} // namespace admin
+} // namespace cta
\ No newline at end of file
diff --git a/cmdline/restore_files/RestoreFilesCmdLineArgs.hpp b/cmdline/restore_files/RestoreFilesCmdLineArgs.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..715f38e23b513f22c9ecca282a8d90d499a97cd1
--- /dev/null
+++ b/cmdline/restore_files/RestoreFilesCmdLineArgs.hpp
@@ -0,0 +1,97 @@
+/*
+ * @project The CERN Tape Archive (CTA)
+ * @copyright Copyright(C) 2015-2021 CERN
+ * @license This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "version.h"
+#include "common/optional.hpp"
+
+#include <list>
+
+namespace cta {
+namespace admin {
+
+/**
+ * Structure to store the command-line arguments of the command-line tool
+ * named cta-restore-deleted-archive.
+ */
+struct RestoreFilesCmdLineArgs {
+ /**
+ * True if the usage message should be printed.
+ */
+ bool m_help;
+
+ /**
+ * True if debug messages should be printed
+ */
+ bool m_debug;
+
+ /**
+ * Archive file id of the files to restore
+ */
+ optional<uint64_t> m_archiveFileId;
+
+ /**
+ * Disk instance of the files to restore
+ */
+ optional<std::string> m_diskInstance;
+
+ /**
+ * Fxids of the files to restore
+ */
+ optional<std::list<std::string>> m_eosFxids;
+
+ /**
+ * Vid of the tape of the files to restore
+ */
+ optional<std::string> m_vid;
+
+ /**
+ *Copy number of the files to restore
+ */
+ optional<uint64_t> m_copyNumber;
+
+ /**
+ * Constructor that parses the specified command-line arguments.
+ *
+ * @param argc The number of command-line arguments including the name of the
+ * executable.
+ * @param argv The vector of command-line arguments.
+ */
+ RestoreFilesCmdLineArgs(const int argc, char *const *const argv);
+
+ /**
+ * Read a list of eos file ids from a file and write the options to a list
+ *
+ * @param filename The name of the file to read
+ * @param optionList The list to append the options.
+ */
+ void readFidListFromFile(const std::string &filename, std::list<std::string> &optionList);
+
+ /**
+ * Prints the usage message of the command-line tool.
+ *
+ * @param os The output stream to which the usage message is to be printed.
+ */
+ static void printUsage(std::ostream &os);
+
+
+
+}; // class RestoreFilesCmdLineArgs
+
+} // namespace admin
+} // namespace cta
diff --git a/cmdline/restore_files/RestoreFilesCmdMain.cpp b/cmdline/restore_files/RestoreFilesCmdMain.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b5acf017da1435a3f8b46717fe36d03c4be07311
--- /dev/null
+++ b/cmdline/restore_files/RestoreFilesCmdMain.cpp
@@ -0,0 +1,48 @@
+/*
+ * @project The CERN Tape Archive (CTA)
+ * @copyright Copyright(C) 2015-2021 CERN
+ * @license This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include <sstream>
+#include <iostream>
+
+#include <XrdSsiPbLog.hpp>
+#include <XrdSsiPbIStreamBuffer.hpp>
+
+#include "RestoreFilesCmd.hpp"
+#include "common/utils/utils.hpp"
+#include "common/log/StdoutLogger.hpp"
+
+//------------------------------------------------------------------------------
+// main
+//------------------------------------------------------------------------------
+int main(const int argc, char *const *const argv) {
+ char buf[256];
+ std::string hostName;
+ if(gethostname(buf, sizeof(buf))) {
+ hostName = "UNKNOWN";
+ } else {
+ buf[sizeof(buf) - 1] = '\0';
+ hostName = buf;
+ }
+ cta::log::StdoutLogger log(hostName, "cta-restore-deleted-files");
+
+ cta::admin::RestoreFilesCmd cmd(std::cin, std::cout, std::cerr, log);
+ int ret = cmd.main(argc, argv);
+ // Delete all global objects allocated by libprotobuf
+ google::protobuf::ShutdownProtobufLibrary();
+ return ret;
+}
\ No newline at end of file
diff --git a/cmdline/restore_files/cta-restore-deleted-files.1cta b/cmdline/restore_files/cta-restore-deleted-files.1cta
new file mode 100644
index 0000000000000000000000000000000000000000..e68111342b7767b5d13f196ec9a7b69812d679b1
--- /dev/null
+++ b/cmdline/restore_files/cta-restore-deleted-files.1cta
@@ -0,0 +1,80 @@
+.\" @project The CERN Tape Archive (CTA)
+.\" @copyright Copyright(C) 2019-2021 CERN
+.\" @license This program is free software: you can redistribute it and/or modify
+.\" it under the terms of the GNU General Public License as published by
+.\" the Free Software Foundation, either version 3 of the License, or
+.\" (at your option) any later version.
+.\"
+.\" This program is distributed in the hope that it will be useful,
+.\" but WITHOUT ANY WARRANTY; without even the implied warranty of
+.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+.\" GNU General Public License for more details.
+.\"
+.\" You should have received a copy of the GNU General Public License
+.\" along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+.TH CTA-RESTORE-DELETED-FILES 1CTA "NOVEMBER 2021" CTA CTA
+.SH NAME
+cta-restore-deleted-files \- Restore files deleted from cta
+.SH SYNOPSIS
+.BI "cta-restore-deleted-file [OPTIONS]"
+
+.SH DESCRIPTION
+\fBcta-restore-deleted-files\fP is a command-line tool for restoring files deleted from cta that match some criteria.
+
+The tool recovers files that have been deleted by a user, by EOS or by an operator using cta-admin tf rm.
+
+There are tree scenarios for disk file recovery:
+
+1. The file has been deleted in the EOS namespace and in the CTA catalogue
+ This happens in the case of a normal file removal from the user (eos rm).
+ The strategy is to reinject the file metadata in EOS and restore the file
+ entry in the cta catalogue (with a new diskFileId)
+
+2. The file has been deleted in the CTA catalogue but the EOS diskFileId remains the same
+ This can happen during a disk draining. The file is kept in the EOS namespace
+ but the entry is removed from the Catalogue. The strategy is to just
+ restore the CTA Catalogue entry.
+
+3. The file has been deleted in the CTA catalogue but the EOS diskFileId changed
+ This happens during the conversion of a file from a space to another. The strategy is to recover the
+ CTA Catalogue entry and update the diskFileId to the one that corresponds to its EOS entry.
+
+
+.SH OPTIONS
+.TP
+.TP
+\fB\-h, \-\-help
+Prints the usage message.
+.TP
+\fB\-d, \-\-debug
+Enable debug log messages
+.TP
+\fB\-I, \-\-id
+Archive file id of the files to restore
+.TP
+\fB\-i, \-\-instance
+Disk instance of the files to restore
+.TP
+\fB\-f, \-\-fxid
+Disk file id of the files to restore
+.TP
+\fB\-F, \-\-fxidfile
+Path to file containing a list of disk file ids to restore
+.TP
+\fB\-v, \-\-vid
+tape vid of the files to restore
+.TP
+\fB\-c, \-\-copynb
+copy number of the files to restore
+.TP
+
+
+.SH RETURN VALUE
+Zero on success and non-zero on failure.
+.SH EXAMPLES
+.br
+cta-restore-deleted-file --vid V01007
+
+.SH AUTHOR
+\fBCTA\fP Team
diff --git a/cta.spec.in b/cta.spec.in
index 4268821dbbcb5f8af8ad43cdde67e938446aaf8d..60736e88c5790057c651ca553bfc6ebb2bd8e5ec 100644
--- a/cta.spec.in
+++ b/cta.spec.in
@@ -189,6 +189,8 @@ The command line utilities
%defattr(-,root,root)
%attr(0755,root,root) %{_bindir}/cta-admin
%attr(0644,root,root) %doc /usr/share/man/man1/cta-admin.1cta.gz
+%attr(0755,root,root) %{_bindir}/cta-restore-deleted-files
+%attr(0644,root,root) %doc /usr/share/man/man1/cta-restore-deleted-files.1cta.gz
%attr(0755,root,root) %{_bindir}/cta-send-event
%attr(0755,root,root) %{_bindir}/cta-send-closew.sh
%attr(0755,root,root) %{_bindir}/cta-verify-file
diff --git a/eos_grpc_client/CMakeLists.txt b/eos_grpc_client/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b838bb000965142e2b013295f150fbeb0ffc384e
--- /dev/null
+++ b/eos_grpc_client/CMakeLists.txt
@@ -0,0 +1,35 @@
+# @project The CERN Tape Archive (CTA)
+# @copyright Copyright(C) 2019-2021 CERN
+# @license This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+cmake_minimum_required (VERSION 2.6)
+
+find_package(Protobuf3 REQUIRED)
+find_package(GRPC REQUIRED)
+#
+# XRootD SSI Protocol Buffer bindings
+#
+include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include)
+
+#
+# Compiled protocol buffers
+#
+include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS})
+
+
+add_library(EosGrpcClient STATIC GrpcClient.cpp GrpcUtils.cpp GrpcEndpoint.cpp)
+
+target_link_libraries(EosGrpcClient ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY})
+set_property (TARGET EosGrpcClient APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
+
diff --git a/eos_grpc_client/GrpcClient.cpp b/eos_grpc_client/GrpcClient.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..73292d4f6e834361b93f46946cf34a07bc6a0924
--- /dev/null
+++ b/eos_grpc_client/GrpcClient.cpp
@@ -0,0 +1,266 @@
+/*
+ * @project The CERN Tape Archive (CTA)
+ * @copyright Copyright(C) 2015-2021 CERN
+ * @license This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <sys/stat.h>
+#include <sstream>
+#include "GrpcClient.hpp"
+
+namespace eos {
+namespace client {
+
+
+std::unique_ptr<GrpcClient>
+GrpcClient::Create(std::string endpoint, std::string token)
+{
+ std::unique_ptr<eos::client::GrpcClient> p(
+ new eos::client::GrpcClient(grpc::CreateChannel(endpoint, grpc::InsecureChannelCredentials()))
+ );
+ p->set_ssl(false);
+ p->set_token(token);
+ return p;
+}
+
+
+std::string GrpcClient::ping(const std::string& payload)
+{
+ eos::rpc::PingRequest request;
+ request.set_message(payload);
+ request.set_authkey(token());
+ eos::rpc::PingReply reply;
+ grpc::ClientContext context;
+ // The producer-consumer queue we use to communicate asynchronously with the
+ // gRPC runtime.
+ grpc::CompletionQueue cq;
+ grpc::Status status;
+ // stub_->AsyncPing() performs the RPC call, returning an instance we
+ // store in "rpc". Because we are using the asynchronous API, we need to
+ // hold on to the "rpc" instance in order to get updates on the ongoing RPC.
+ std::unique_ptr<grpc::ClientAsyncResponseReader<eos::rpc::PingReply>> rpc(
+ stub_->AsyncPing(&context, request, &cq));
+ // Request that, upon completion of the RPC, "reply" be updated with the
+ // server's response; "status" with the indication of whether the operation
+ // was successful. Tag the request.
+ auto tag = nextTag();
+ rpc->Finish(&reply, &status, tag);
+ void* got_tag;
+ bool ok = false;
+ // Block until the next result is available in the completion queue "cq".
+ // The return value of Next should always be checked. This return value
+ // tells us whether there is any kind of event or the cq_ is shutting down.
+ GPR_ASSERT(cq.Next(&got_tag, &ok));
+ // Verify that the result from "cq" corresponds, by its tag, our previous
+ // request.
+ GPR_ASSERT(got_tag == tag);
+ // ... and that the request was completed successfully. Note that "ok"
+ // corresponds solely to the request for updates introduced by Finish().
+ GPR_ASSERT(ok);
+
+ // Act upon the status of the actual RPC.
+ if(status.ok()) {
+ return reply.message();
+ } else {
+ throw std::runtime_error("Ping failed with error: " + status.error_message());
+ }
+}
+
+
+int GrpcClient::FileInsert(const std::vector<eos::rpc::FileMdProto> &files, eos::rpc::InsertReply &replies)
+{
+ eos::rpc::FileInsertRequest request;
+ for(auto &file : files) {
+ if(file.id() >= m_eos_fid) {
+ std::stringstream err;
+ err << "FATAL ERROR: attempt to inject file with id=" << file.id()
+ << ", which exceeds EOS current file id=" << m_eos_fid;
+ throw std::runtime_error(err.str());
+ }
+ *(request.add_files()) = file;
+ }
+
+ request.set_authkey(token());
+ grpc::ClientContext context;
+ // The producer-consumer queue we use to communicate asynchronously with the gRPC runtime.
+ grpc::CompletionQueue cq;
+ grpc::Status status;
+
+ std::unique_ptr<grpc::ClientAsyncResponseReader<eos::rpc::InsertReply>> rpc(
+ stub_->AsyncFileInsert(&context, request, &cq));
+ // Request that, upon completion of the RPC, "replies" be updated with the
+ // server's response; "status" with the indication of whether the operation
+ // was successful. Tag the request.
+ auto tag = nextTag();
+ rpc->Finish(&replies, &status, tag);
+ void* got_tag;
+ bool ok = false;
+ // Block until the next result is available in the completion queue "cq".
+ // The return value of Next should always be checked. This return value
+ // tells us whether there is any kind of event or the cq_ is shutting down.
+ GPR_ASSERT(cq.Next(&got_tag, &ok));
+ // Verify that the result from "cq" corresponds, by its tag, our previous
+ // request.
+ GPR_ASSERT(got_tag == tag);
+ // ... and that the request was completed successfully. Note that "ok"
+ // corresponds solely to the request for updates introduced by Finish().
+ GPR_ASSERT(ok);
+
+ // Act upon the status of the actual RPC.
+ if(status.ok()) {
+ int num_errors = 0;
+ for(auto &retc : replies.retc()) {
+ if(retc != 0) ++num_errors;
+ }
+ return num_errors;
+ } else {
+ throw std::runtime_error("FileInsert failed with error: " + status.error_message());
+ }
+}
+
+
+int GrpcClient::ContainerInsert(const std::vector<eos::rpc::ContainerMdProto> &dirs, eos::rpc::InsertReply &replies)
+{
+ eos::rpc::ContainerInsertRequest request;
+
+ // Tell EOS gRPC to behave like "eos mkdir": inherit xattrs from parent dir
+ request.set_inherit_md(true);
+
+ for(auto &dir : dirs) {
+ if(dir.id() >= m_eos_cid) {
+ std::stringstream err;
+ err << "FATAL ERROR: attempt to inject container with id=" << dir.id()
+ << ", which exceeds EOS current container id=" << m_eos_cid;
+ throw std::runtime_error(err.str());
+ }
+ *(request.add_container()) = dir;
+ }
+
+ request.set_authkey(token());
+ grpc::ClientContext context;
+ // The producer-consumer queue we use to communicate asynchronously with the gRPC runtime
+ grpc::CompletionQueue cq;
+ grpc::Status status;
+
+ std::unique_ptr<grpc::ClientAsyncResponseReader<eos::rpc::InsertReply>> rpc(
+ stub_->AsyncContainerInsert(&context, request, &cq));
+ // Request that, upon completion of the RPC, "replies" be updated with the
+ // server's response; "status" with the indication of whether the operation
+ // was successful. Tag the request.
+ auto tag = nextTag();
+ rpc->Finish(&replies, &status, tag);
+ void* got_tag;
+ bool ok = false;
+ // Block until the next result is available in the completion queue "cq".
+ // The return value of Next should always be checked. This return value
+ // tells us whether there is any kind of event or the cq_ is shutting down.
+ GPR_ASSERT(cq.Next(&got_tag, &ok));
+ // Verify that the result from "cq" corresponds, by its tag, our previous
+ // request.
+ GPR_ASSERT(got_tag == tag);
+ // ... and that the request was completed successfully. Note that "ok"
+ // corresponds solely to the request for updates introduced by Finish().
+ GPR_ASSERT(ok);
+
+ // Return the status of the RPC
+ if(status.ok()) {
+ int num_errors = 0;
+ for(auto &retc : replies.retc()) {
+ if(retc != 0) ++num_errors;
+ }
+ return num_errors;
+ } else {
+ throw std::runtime_error("ContainerInsert failed with error: " + status.error_message());
+ }
+}
+
+
+void GrpcClient::GetCurrentIds(uint64_t &cid, uint64_t &fid)
+{
+ eos::rpc::NsStatRequest request;
+ request.set_authkey(token());
+
+ grpc::ClientContext context;
+ grpc::CompletionQueue cq;
+
+ std::unique_ptr<grpc::ClientAsyncResponseReader<eos::rpc::NsStatResponse>> rpc(
+ stub_->AsyncNsStat(&context, request, &cq));
+
+ eos::rpc::NsStatResponse response;
+ grpc::Status status;
+ auto tag = nextTag();
+ rpc->Finish(&response, &status, tag);
+
+ void* got_tag;
+ bool ok = false;
+ GPR_ASSERT(cq.Next(&got_tag, &ok));
+ GPR_ASSERT(got_tag == tag);
+ GPR_ASSERT(ok);
+
+ // Act upon the status of the actual RPC
+ if(status.ok()) {
+ cid = m_eos_cid = response.current_cid();
+ fid = m_eos_fid = response.current_fid();
+ } else {
+ throw std::runtime_error("EOS namespace query failed with error: " + status.error_message());
+ }
+}
+
+
+eos::rpc::MDResponse GrpcClient::GetMD(eos::rpc::TYPE type, uint64_t id, const std::string &path, bool showJson)
+{
+ eos::rpc::MDRequest request;
+
+ request.set_type(type);
+ request.mutable_id()->set_id(id);
+ request.mutable_id()->set_path(path);
+ request.set_authkey(token());
+
+ if(showJson) {
+ google::protobuf::util::JsonPrintOptions options;
+ options.add_whitespace = true;
+ options.always_print_primitive_fields = true;
+ std::string logstring;
+ google::protobuf::util::MessageToJsonString(request, &logstring, options);
+ std::cout << logstring;
+ }
+
+ grpc::ClientContext context;
+ grpc::CompletionQueue cq;
+
+ auto tag = nextTag();
+ std::unique_ptr<grpc::ClientAsyncReader<eos::rpc::MDResponse>> rpc(
+ stub_->AsyncMD(&context, request, &cq, tag));
+
+ eos::rpc::MDResponse response;
+ while(true) {
+ void *got_tag;
+ bool ok = false;
+ bool ret = cq.Next(&got_tag, &ok);
+ if(!ret || !ok || got_tag != tag) break;
+ rpc->Read(&response, tag);
+ }
+ if(showJson) {
+ google::protobuf::util::JsonPrintOptions options;
+ options.add_whitespace = true;
+ options.always_print_primitive_fields = true;
+ std::string logstring;
+ google::protobuf::util::MessageToJsonString(response, &logstring, options);
+ std::cout << logstring;
+ }
+
+ return response;
+}
+
+}} // namespace eos::client
diff --git a/eos_grpc_client/GrpcClient.hpp b/eos_grpc_client/GrpcClient.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..9e1851db4546d4c6df4b3c4078c6b9dc97843c1b
--- /dev/null
+++ b/eos_grpc_client/GrpcClient.hpp
@@ -0,0 +1,80 @@
+/*
+ * @project The CERN Tape Archive (CTA)
+ * @copyright Copyright(C) 2015-2021 CERN
+ * @license This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <memory>
+#include <grpc++/grpc++.h>
+#include "Rpc.grpc.pb.h"
+
+namespace eos {
+namespace client {
+
+class GrpcClient
+{
+public:
+ explicit GrpcClient(std::shared_ptr<grpc::Channel> channel) :
+ stub_(eos::rpc::Eos::NewStub(channel)),
+ m_tag(0),
+ m_eos_cid(0),
+ m_eos_fid(0) { }
+
+ // factory function
+ static std::unique_ptr<GrpcClient> Create(std::string endpoint, std::string token);
+
+ std::string ping(const std::string& payload);
+
+ int FileInsert(const std::vector<eos::rpc::FileMdProto> &paths, eos::rpc::InsertReply &replies);
+
+ int ContainerInsert(const std::vector<eos::rpc::ContainerMdProto> &dirs, eos::rpc::InsertReply &replies);
+
+ // Obtain current container ID and current file ID
+ void GetCurrentIds(uint64_t &cid, uint64_t &fid);
+
+ // Obtain container or file metadata
+ eos::rpc::MDResponse GetMD(eos::rpc::TYPE type, uint64_t id, const std::string &path, bool showJson = false);
+
+ void set_ssl(bool onoff) {
+ m_SSL = onoff;
+ }
+
+ bool ssl() const {
+ return m_SSL;
+ }
+
+ void set_token(const std::string &token) {
+ m_token = token;
+ }
+
+ std::string token() const {
+ return m_token;
+ }
+
+ void *nextTag() {
+ return reinterpret_cast<void*>(++m_tag);
+ }
+
+private:
+ std::unique_ptr<eos::rpc::Eos::Stub> stub_;
+ bool m_SSL;
+ std::string m_token;
+ uint64_t m_tag;
+ uint64_t m_eos_cid; //!< EOS current container ID
+ uint64_t m_eos_fid; //!< EOS current file ID
+};
+
+}} // namespace eos::client
diff --git a/eos_grpc_client/GrpcEndpoint.cpp b/eos_grpc_client/GrpcEndpoint.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6ee6d50e8c4a5f0699ba9d3219fbf5324df81039
--- /dev/null
+++ b/eos_grpc_client/GrpcEndpoint.cpp
@@ -0,0 +1,63 @@
+/*
+ * @project The CERN Tape Archive (CTA)
+ * @copyright Copyright(C) 2015-2021 CERN
+ * @license This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "GrpcEndpoint.hpp"
+
+#include "common/exception/UserError.hpp"
+#include "common/exception/Exception.hpp"
+
+
+std::string eos::client::Endpoint::getPath(const std::string &diskFileId) const {
+ // diskFileId is sent to CTA as a uint64_t, but we store it as a decimal string, cf.:
+ // XrdSsiCtaRequestMessage.cpp: request.diskFileID = std::to_string(notification.file().fid());
+ // Here we convert it back to make the namespace query:
+ uint64_t id = strtoull(diskFileId.c_str(), NULL, 0);
+ if(id == 0) throw cta::exception::UserError("Invalid disk ID");
+ auto response = m_grpcClient->GetMD(eos::rpc::FILE, id, "");
+
+ if (response.fmd().name().empty()) {
+ throw cta::exception::Exception("Bad response from nameserver");
+ } else {
+ return response.fmd().path();
+ }
+}
+
+eos::rpc::InsertReply eos::client::Endpoint::fileInsert(const eos::rpc::FileMdProto &file) const {
+ std::vector<eos::rpc::FileMdProto> paths;
+ paths.push_back(file);
+ eos::rpc::InsertReply replies;
+ m_grpcClient->FileInsert(paths, replies);
+ return replies;
+
+}
+
+eos::rpc::InsertReply eos::client::Endpoint::containerInsert(const eos::rpc::ContainerMdProto &container) const {
+ std::vector<eos::rpc::ContainerMdProto> paths;
+ paths.push_back(container);
+ eos::rpc::InsertReply replies;
+ m_grpcClient->ContainerInsert(paths, replies);
+ return replies;
+
+}
+
+void eos::client::Endpoint::getCurrentIds(uint64_t &cid, uint64_t &fid) const {
+ m_grpcClient->GetCurrentIds(cid, fid);
+}
+
+eos::rpc::MDResponse eos::client::Endpoint::getMD(eos::rpc::TYPE type, uint64_t id, const std::string &path, bool showJson) const {
+ return m_grpcClient->GetMD(type, id, path, showJson);
+}
\ No newline at end of file
diff --git a/eos_grpc_client/GrpcEndpoint.hpp b/eos_grpc_client/GrpcEndpoint.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..8d6e98ba26f209c50049d93b943b4e50a1b9d78f
--- /dev/null
+++ b/eos_grpc_client/GrpcEndpoint.hpp
@@ -0,0 +1,104 @@
+/*
+ * @project The CERN Tape Archive (CTA)
+ * @copyright Copyright(C) 2015-2021 CERN
+ * @license This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "Namespace.hpp"
+#include "GrpcClient.hpp"
+
+#include "common/exception/UserError.hpp"
+
+namespace eos {
+namespace client {
+
+class Endpoint
+{
+public:
+ Endpoint(const Namespace &endpoint) :
+ m_grpcClient(::eos::client::GrpcClient::Create(endpoint.endpoint, endpoint.token)) { }
+
+ std::string getPath(const std::string &diskFileId) const;
+ ::eos::rpc::InsertReply fileInsert(const ::eos::rpc::FileMdProto &file) const;
+ ::eos::rpc::InsertReply containerInsert(const ::eos::rpc::ContainerMdProto &container) const;
+ void getCurrentIds(uint64_t &cid, uint64_t &fid) const;
+ ::eos::rpc::MDResponse getMD(::eos::rpc::TYPE type, uint64_t id, const std::string &path, bool showJson) const;
+
+private:
+ std::unique_ptr<::eos::client::GrpcClient> m_grpcClient;
+};
+
+
+class EndpointMap
+{
+public:
+ EndpointMap(NamespaceMap_t nsMap) {
+ for(auto &ns : nsMap) {
+ m_endpointMap.insert(std::make_pair(ns.first, Endpoint(ns.second)));
+ }
+ }
+
+ std::string getPath(const std::string &diskInstance, const std::string &diskFileId) const {
+ auto ep_it = m_endpointMap.find(diskInstance);
+ if(ep_it == m_endpointMap.end()) {
+ throw cta::exception::UserError("Namespace for disk instance \"" + diskInstance + "\" is not configured in the CTA Frontend");
+ } else {
+ return ep_it->second.getPath(diskFileId);
+ }
+ }
+
+ ::eos::rpc::InsertReply fileInsert(const std::string &diskInstance, const ::eos::rpc::FileMdProto &file) const{
+ auto ep_it = m_endpointMap.find(diskInstance);
+ if(ep_it == m_endpointMap.end()) {
+ throw cta::exception::UserError("Namespace for disk instance \"" + diskInstance + "\" is not configured in the CTA Frontend");
+ } else {
+ return ep_it->second.fileInsert(file);
+ }
+ }
+
+ ::eos::rpc::InsertReply containerInsert(const std::string &diskInstance, const ::eos::rpc::ContainerMdProto &container) const{
+ auto ep_it = m_endpointMap.find(diskInstance);
+ if(ep_it == m_endpointMap.end()) {
+ throw cta::exception::UserError("Namespace for disk instance \"" + diskInstance + "\" is not configured in the CTA Frontend");
+ } else {
+ return ep_it->second.containerInsert(container);
+ }
+ }
+
+ void getCurrentIds(const std::string &diskInstance, uint64_t &cid, uint64_t &fid) const {
+ auto ep_it = m_endpointMap.find(diskInstance);
+ if(ep_it == m_endpointMap.end()) {
+ throw cta::exception::UserError("Namespace for disk instance \"" + diskInstance + "\" is not configured in the CTA Frontend");
+ } else {
+ return ep_it->second.getCurrentIds(cid, fid);
+ }
+ }
+
+ ::eos::rpc::MDResponse getMD(const std::string &diskInstance, ::eos::rpc::TYPE type,
+ uint64_t id, const std::string &path, bool showJson) const {
+ auto ep_it = m_endpointMap.find(diskInstance);
+ if(ep_it == m_endpointMap.end()) {
+ throw cta::exception::UserError("Namespace for disk instance \"" + diskInstance + "\" is not configured in the CTA Frontend");
+ } else {
+ return ep_it->second.getMD(type, id, path, showJson);
+ }
+ }
+
+private:
+ std::map<std::string, Endpoint> m_endpointMap;
+};
+
+}} // namespace eos::client
\ No newline at end of file
diff --git a/eos_grpc_client/GrpcUtils.cpp b/eos_grpc_client/GrpcUtils.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0e9936bd936930de3bda6e43a6af3f26d977ac40
--- /dev/null
+++ b/eos_grpc_client/GrpcUtils.cpp
@@ -0,0 +1,58 @@
+/*
+ * @project The CERN Tape Archive (CTA)
+ * @copyright Copyright(C) 2015-2021 CERN
+ * @license This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <string>
+#include "GrpcUtils.hpp"
+
+
+namespace eos {
+namespace client {
+
+void checkPrefix(std::string &prefix)
+{
+ if(prefix.empty()) {
+ prefix = '/';
+ } else {
+ if(prefix.at(0) != '/') prefix = '/' + prefix;
+ if(prefix.at(prefix.length()-1) != '/') prefix += '/';
+ }
+}
+
+
+Dirname manglePathname(const std::string &remove_prefix, const std::string &add_prefix, const std::string &pathname, const std::string &filename)
+{
+ Dirname dir;
+
+ // Set the pathname
+ size_t clip = (pathname.rfind(remove_prefix, 0) == std::string::npos) ? 0 : remove_prefix.length();
+ if(pathname.length() > clip && pathname.at(clip) == '/') ++clip;
+ dir.pathname = add_prefix + pathname.substr(clip);
+
+ // Set the filename
+ if(filename.empty()) {
+ clip = dir.pathname.find_last_of('/');
+ dir.basename = dir.pathname.substr(clip+1);
+ } else {
+ if(!dir.pathname.empty() && dir.pathname.at(dir.pathname.length()-1) != '/') dir.pathname += '/';
+ dir.pathname += filename;
+ dir.basename = filename;
+ }
+
+ return dir;
+}
+
+}} // namespace eos::client
diff --git a/eos_grpc_client/GrpcUtils.hpp b/eos_grpc_client/GrpcUtils.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..cb024e3b0cf50849b9c9464d6e54a25bac097b80
--- /dev/null
+++ b/eos_grpc_client/GrpcUtils.hpp
@@ -0,0 +1,43 @@
+/*
+ * @project The CERN Tape Archive (CTA)
+ * @copyright Copyright(C) 2015-2021 CERN
+ * @license This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+namespace eos {
+namespace client {
+
+/*!
+ * Basename and pathname of a path
+ */
+struct Dirname {
+ std::string basename; //!< Just the directory name
+ std::string pathname; //!< The full path including the directory name
+};
+
+/*!
+ * Enforce prefixes to begin and end with a slash
+ */
+void checkPrefix(std::string &prefix);
+
+/*!
+ * Remove the first prefix from pathname, prepend the second prefix, then split path into basename and pathname parts
+ *
+ * Note: prefixes must begin and end with a slash
+ */
+Dirname manglePathname(const std::string &remove_prefix, const std::string &add_prefix, const std::string &pathname, const std::string &filename = "");
+
+}} // namespace eos::client
diff --git a/eos_grpc_client/Namespace.hpp b/eos_grpc_client/Namespace.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..2f64a88f59618aa75dde3fd752a107d1887c7709
--- /dev/null
+++ b/eos_grpc_client/Namespace.hpp
@@ -0,0 +1,39 @@
+/*
+ * @project The CERN Tape Archive (CTA)
+ * @copyright Copyright(C) 2015-2021 CERN
+ * @license This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <map>
+#include <iostream>
+
+namespace eos {
+namespace client {
+
+struct Namespace
+{
+ Namespace(const std::string &ep, const std::string &tk) :
+ endpoint(ep), token(tk) {
+std::cerr << "Created namespace endpoint " << endpoint << " with token " << token << std::endl;
+ }
+
+ std::string endpoint;
+ std::string token;
+};
+
+typedef std::map<std::string, Namespace> NamespaceMap_t;
+
+}} // namespace eos::client
\ No newline at end of file
diff --git a/rdbms/wrapper/OcciStmt.cpp b/rdbms/wrapper/OcciStmt.cpp
index 70b9cd1626ed379a6c7150f7c09d2ea234494b6b..266892ec380d81cff9a005905689b09b6aee13ff 100644
--- a/rdbms/wrapper/OcciStmt.cpp
+++ b/rdbms/wrapper/OcciStmt.cpp
@@ -138,7 +138,11 @@ void OcciStmt::bindUint64(const std::string ¶mName, const optional<uint64_t>
// bindBlob
//------------------------------------------------------------------------------
void OcciStmt::bindBlob(const std::string ¶mName, const std::string ¶mValue) {
- throw exception::Exception("OcciStmt::bindBlob not implemented.");
+ try {
+ bindString(paramName, paramValue);
+ } catch(exception::Exception &ex) {
+ throw exception::Exception(std::string(__FUNCTION__) + " failed: " + ex.getMessage().str());
+ }
}
//------------------------------------------------------------------------------
diff --git a/rdbms/wrapper/PostgresStmt.cpp b/rdbms/wrapper/PostgresStmt.cpp
index a97499d464759d6910ae85d8ba3d3030d95eee32..473533ba4dbafd1b058ffd6d382a1299f7b7e49e 100644
--- a/rdbms/wrapper/PostgresStmt.cpp
+++ b/rdbms/wrapper/PostgresStmt.cpp
@@ -156,7 +156,17 @@ void PostgresStmt::bindUint64(const std::string ¶mName, const optional<uint6
// bindBlob
//------------------------------------------------------------------------------
void PostgresStmt::bindBlob(const std::string ¶mName, const std::string ¶mValue) {
- throw exception::Exception("PostgresStmt::bindBlob not implemented.");
+ /*Escape the bytea string according to https://www.postgresql.org/docs/12/libpq-exec.html*/
+ size_t escaped_length;
+ auto escapedByteA = PQescapeByteaConn(m_conn.get(), reinterpret_cast<const unsigned char*>(paramValue.c_str()),
+ paramValue.length(), &escaped_length);
+ std::string escapedParamValue(reinterpret_cast<const char*>(escapedByteA), escaped_length);
+ PQfreemem(escapedByteA);
+ try {
+ bindString(paramName, escapedParamValue);
+ } catch(exception::Exception &ex) {
+ throw exception::Exception(std::string(__FUNCTION__) + " failed: " + ex.getMessage().str());
+ }
}
//------------------------------------------------------------------------------
diff --git a/xroot_plugins/GrpcEndpoint.cpp b/xroot_plugins/GrpcEndpoint.cpp
index 9af339092a011b5b14248ac7bf10dc808f77a117..8b242d336d73e662bdcff5315eddbed083dd3447 100644
--- a/xroot_plugins/GrpcEndpoint.cpp
+++ b/xroot_plugins/GrpcEndpoint.cpp
@@ -17,6 +17,9 @@
#include <xroot_plugins/GrpcEndpoint.hpp>
+#include "common/exception/UserError.hpp"
+#include "common/exception/Exception.hpp"
+
std::string cta::grpc::Endpoint::getPath(const std::string &diskFileId) const {
// diskFileId is sent to CTA as a uint64_t, but we store it as a decimal string, cf.:
@@ -28,3 +31,18 @@ std::string cta::grpc::Endpoint::getPath(const std::string &diskFileId) const {
return response.fmd().name().empty() ? "Bad response from nameserver" : response.fmd().path();
}
+
+std::string cta::grpc::Endpoint::getPathExceptionThrowing(const std::string &diskFileId) const {
+ // diskFileId is sent to CTA as a uint64_t, but we store it as a decimal string, cf.:
+ // XrdSsiCtaRequestMessage.cpp: request.diskFileID = std::to_string(notification.file().fid());
+ // Here we convert it back to make the namespace query:
+ uint64_t id = strtoull(diskFileId.c_str(), NULL, 0);
+ if(id == 0) throw cta::exception::UserError("Invalid disk ID");
+ auto response = m_grpcClient->GetMD(eos::rpc::FILE, id, "");
+
+ if (response.fmd().name().empty()) {
+ throw cta::exception::Exception("Bad response from nameserver");
+ } else {
+ return response.fmd().path();
+ }
+}
diff --git a/xroot_plugins/GrpcEndpoint.hpp b/xroot_plugins/GrpcEndpoint.hpp
index c95d12eb3644a600239fbb7c068db6ecd9ddb113..cc718b8aec04b3eaade1851f7218c8187431983e 100644
--- a/xroot_plugins/GrpcEndpoint.hpp
+++ b/xroot_plugins/GrpcEndpoint.hpp
@@ -20,6 +20,9 @@
#include <xroot_plugins/Namespace.hpp>
#include <xroot_plugins/GrpcClient.hpp>
+#include "common/exception/UserError.hpp"
+
+
namespace cta { namespace grpc {
class Endpoint
@@ -29,6 +32,7 @@ public:
m_grpcClient(::eos::client::GrpcClient::Create(endpoint.endpoint, endpoint.token)) { }
std::string getPath(const std::string &diskFileId) const;
+ std::string getPathExceptionThrowing(const std::string &diskFileId) const;
private:
std::unique_ptr<::eos::client::GrpcClient> m_grpcClient;
@@ -53,6 +57,15 @@ public:
}
}
+ std::string getPathExceptionThrowing(const std::string &diskInstance, const std::string &diskFileId) const {
+ auto ep_it = m_endpointMap.find(diskInstance);
+ if(ep_it == m_endpointMap.end()) {
+ throw cta::exception::UserError("Namespace for disk instance \"" + diskInstance + "\" is not configured in the CTA Frontend");
+ } else {
+ return ep_it->second.getPathExceptionThrowing(diskFileId);
+ }
+ }
+
private:
std::map<std::string, Endpoint> m_endpointMap;
};
diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp
index 3329cba3c0a130115982ef383cb455859f95f96c..92046af66f0e92317184edc7949998a3ec416d0c 100644
--- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp
+++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp
@@ -2248,20 +2248,13 @@ void RequestMessage::processRecycleTapeFile_Restore(cta::xrd::Response& response
cta::catalogue::RecycleTapeFileSearchCriteria searchCriteria;
searchCriteria.vid = getOptional(OptionString::VID, &has_any);
- auto diskFileId = getOptional(OptionString::FXID, &has_any);
- searchCriteria.diskFileIds = getOptional(OptionStrList::FILE_ID, &has_any);
-
- if(diskFileId){
- // single option on the command line we need to do the conversion ourselves.
- if(!searchCriteria.diskFileIds) searchCriteria.diskFileIds = std::vector<std::string>();
-
- auto fid = strtol(diskFileId->c_str(), nullptr, 16);
- if(fid < 1 || fid == LONG_MAX) {
- throw cta::exception::UserError(*diskFileId + " is not a valid file ID");
- }
-
- searchCriteria.diskFileIds->push_back(std::to_string(fid));
+ auto diskFileId = getRequired(OptionString::FXID);
+
+ auto fid = strtol(diskFileId.c_str(), nullptr, 16);
+ if(fid < 1 || fid == LONG_MAX) {
+ throw cta::exception::UserError(diskFileId + " is not a valid file ID");
}
+
// Disk instance on its own does not give a valid set of search criteria (no &has_any)
searchCriteria.diskInstance = getOptional(OptionString::INSTANCE);
searchCriteria.archiveFileId = getOptional(OptionUInt64::ARCHIVE_FILE_ID, &has_any);
@@ -2271,7 +2264,7 @@ void RequestMessage::processRecycleTapeFile_Restore(cta::xrd::Response& response
if(!has_any){
throw cta::exception::UserError("Must specify at least one of the following search options: vid, fxid, fxidfile or archiveFileId");
}
- m_catalogue.restoreFilesInRecycleLog(searchCriteria);
+ m_catalogue.restoreFileInRecycleLog(searchCriteria, std::to_string(fid));
response.set_type(cta::xrd::Response::RSP_SUCCESS);
}