[catalogue] Implemented RdbmsCatalogueGetFileRecycleLogItor + unit tests

catalogue/CMakeLists.txt

@@ -53,6 +53,7 @@ set (CATALOGUE_LIB_SRC_FILES
   RdbmsCatalogueGetArchiveFilesItor.cpp
   RdbmsCatalogueGetDeletedArchiveFilesItor.cpp
   RdbmsCatalogueTapeContentsItor.cpp
+  RdbmsCatalogueGetFileRecycleLogItor.cpp
   SchemaCreatingSqliteCatalogue.cpp
   SqliteCatalogue.cpp
   SchemaVersion.cpp

catalogue/Catalogue.hpp

@@ -41,6 +41,7 @@
 #include "common/dataStructures/DeletedArchiveFile.hpp"
 #include "common/dataStructures/DiskFileInfo.hpp"
 #include "common/dataStructures/DriveState.hpp"
+#include "common/dataStructures/FileRecycleLog.hpp"
 #include "common/dataStructures/EntryLog.hpp"
 #include "common/dataStructures/ListStorageClassRequest.hpp"
 #include "common/dataStructures/LogicalLibrary.hpp"
@@ -869,6 +870,15 @@ public:
    * @return The deleted archive files ordered by archive file ID.
    */
   virtual DeletedArchiveFileItor getDeletedArchiveFilesItor() const = 0;
+  
+  typedef CatalogueItor<common::dataStructures::FileRecycleLog> FileRecycleLogItor;
+  
+  /**
+   * Returns all the currently deleted files by looking at the FILE_RECYCLE_LOG table
+   *
+   * @return The deleted archive files ordered by archive file ID.
+   */
+  virtual FileRecycleLogItor getFileRecycleLogItor() const = 0;
 
   /**
    * Returns the specified files in tape file sequence order.
@@ -1034,6 +1044,18 @@ public:
    * @param lc The log context.
    */
   virtual void deleteFileFromRecycleBin(const uint64_t archiveFileId, log::LogContext &lc) = 0;
+  
+  /**
+   * Deletes all the log entries corresponding to the vid passed in parameter.
+   * 
+   * Please note that this method is idempotent.  If there are no recycle log
+   * entries associated to the vid passed in parameter, the method will return
+   * without any error.
+   * 
+   * @param vid, the vid of the files to be deleted
+   * @param lc, the logContext
+   */
+  virtual void deleteFilesFromRecycleLog(const std::string & vid, log::LogContext & lc) = 0;
 
 }; // class Catalogue
 

catalogue/CatalogueRetryWrapper.hpp

@@ -524,9 +524,27 @@ public:
     return retryOnLostConnection(m_log, [&]{return m_catalogue->getDeletedArchiveFilesItor();}, m_maxTriesToConnect);
   }
   
+  FileRecycleLogItor getFileRecycleLogItor() const override {
+    return retryOnLostConnection(m_log, [&]{return m_catalogue->getFileRecycleLogItor();}, m_maxTriesToConnect);
+  }
+  
   void deleteFileFromRecycleBin(const uint64_t archiveFileId, log::LogContext &lc){
     return retryOnLostConnection(m_log,[&]{return m_catalogue->deleteFileFromRecycleBin(archiveFileId,lc);},m_maxTriesToConnect);
   }
+  
+  /**
+   * Deletes all the log entries corresponding to the vid passed in parameter.
+   * 
+   * Please note that this method is idempotent.  If there are no recycle log
+   * entries associated to the vid passed in parameter, the method will return
+   * without any error.
+   * 
+   * @param vid, the vid of the files to be deleted
+   * @param lc, the logContext
+   */
+  void deleteFilesFromRecycleLog(const std::string & vid, log::LogContext & lc){
+    return retryOnLostConnection(m_log,[&]{return m_catalogue->deleteFilesFromRecycleLog(vid,lc);},m_maxTriesToConnect); 
+  }
 
   std::list<common::dataStructures::ArchiveFile> getFilesForRepack(const std::string &vid, const uint64_t startFSeq, const uint64_t maxNbFiles) const override {
     return retryOnLostConnection(m_log, [&]{return m_catalogue->getFilesForRepack(vid, startFSeq, maxNbFiles);}, m_maxTriesToConnect);

catalogue/CatalogueTest.cpp

@@ -28,6 +28,7 @@
 #include "common/threading/MutexLocker.hpp"
 #include "rdbms/wrapper/ConnFactoryFactory.hpp"
 #include "rdbms/Conn.hpp"
+#include "InsertFileRecycleLog.hpp"
 #include <algorithm>
 #include <gtest/gtest.h>
@@ -162,7 +163,7 @@ cta_catalogue_CatalogueTest::cta_catalogue_CatalogueTest():
 void cta_catalogue_CatalogueTest::SetUp() {
   using namespace cta;
   using namespace cta::catalogue;
-
+  log::LogContext dummyLc(m_dummyLog);
   try {
     CatalogueFactory *const *const catalogueFactoryPtrPtr = GetParam();
@@ -213,7 +214,6 @@ void cta_catalogue_CatalogueTest::SetUp() {
       }
       for(const auto &archiveFile: archiveFiles) {
-        log::LogContext dummyLc(m_dummyLog);
         m_catalogue->DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(archiveFile.diskInstance, archiveFile.archiveFileID, dummyLc);
       }
     }
@@ -226,10 +226,17 @@ void cta_catalogue_CatalogueTest::SetUp() {
       }
       
       for(const auto &deletedArchiveFile: deletedArchiveFiles){
-        log::LogContext dummyLc(m_dummyLog);
         m_catalogue->deleteFileFromRecycleBin(deletedArchiveFile.archiveFileID,dummyLc);
       }
     }
+    {
+      //Delete all the entries from the recycle log table
+      auto itor = m_catalogue->getFileRecycleLogItor();
+      std::list<common::dataStructures::FileRecycleLog> filesRecycleLog;
+      while(itor.hasMore()){
+        m_catalogue->deleteFilesFromRecycleLog(itor.next().vid,dummyLc);
+      }
+    }
     {
       const std::list<common::dataStructures::Tape> tapes = m_catalogue->getTapes();
       for(auto &tape: tapes) {
@@ -15869,4 +15876,211 @@ TEST_P(cta_catalogue_CatalogueTest, reclaimTapeRemovesFilesFromRecycleBin) {
   }
 }
+TEST_P(cta_catalogue_CatalogueTest, emptyFileRecycleLogItorTest) {
+  using namespace cta;
+  auto itor = m_catalogue->getFileRecycleLogItor();
+  ASSERT_FALSE(itor.hasMore());
+  ASSERT_THROW(itor.next(),cta::exception::Exception);
+}
+
+TEST_P(cta_catalogue_CatalogueTest, filesArePutInTheFileRecycleLogInsteadOfBeingSuperseded) {
+  using namespace cta;
+
+  const bool logicalLibraryIsDisabled= false;
+  const std::string tapePoolName1 = "tape_pool_name_1";
+  const std::string tapePoolName2 = "tape_pool_name_2";
+  const uint64_t nbPartialTapes = 1;
+  const bool isEncrypted = true;
+  const cta::optional<std::string> supply("value for the supply pool mechanism");
+  const std::string diskInstance = "disk_instance";
+  const std::string tapeDrive = "tape_drive";
+
+  m_catalogue->createMediaType(m_admin, m_mediaType);
+  m_catalogue->createLogicalLibrary(m_admin, m_tape1.logicalLibraryName, logicalLibraryIsDisabled, "Create logical library");
+  m_catalogue->createVirtualOrganization(m_admin, m_vo);
+  m_catalogue->createTapePool(m_admin, tapePoolName1, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
+  m_catalogue->createTapePool(m_admin, tapePoolName2, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
+  m_catalogue->createStorageClass(m_admin, m_storageClassSingleCopy);
+
+  auto tape1 = m_tape1;
+  tape1.tapePoolName = tapePoolName1;
+  auto tape2 = m_tape2;
+  tape2.tapePoolName = tapePoolName2;
+
+  m_catalogue->createTape(m_admin, tape1);
+  m_catalogue->createTape(m_admin, tape2);
+
+  ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore());
+  const uint64_t nbArchiveFiles = 10; // Must be a multiple of 2 for this test
+  const uint64_t archiveFileSize = 2 * 1000 * 1000 * 1000;
+
+  std::set<catalogue::TapeItemWrittenPointer> tapeFilesWrittenCopy1;
+  for(uint64_t i = 1; i <= nbArchiveFiles; i++) {
+    std::ostringstream diskFileId;
+    diskFileId << (12345677 + i);
+    
+    std::ostringstream diskFilePath;
+    diskFilePath << "/test/file"<<i;
+
+    // Tape copy 1 written to tape
+    auto fileWrittenUP=cta::make_unique<cta::catalogue::TapeFileWritten>();
+    auto & fileWritten = *fileWrittenUP;
+    fileWritten.archiveFileId = i;
+    fileWritten.diskInstance = diskInstance;
+    fileWritten.diskFileId = diskFileId.str();
+    fileWritten.diskFilePath = diskFilePath.str();
+    fileWritten.diskFileOwnerUid = PUBLIC_DISK_USER;
+    fileWritten.diskFileGid = PUBLIC_DISK_GROUP;
+    fileWritten.size = archiveFileSize;
+    fileWritten.checksumBlob.insert(checksum::ADLER32, "1357");
+    fileWritten.storageClassName = m_storageClassSingleCopy.name;
+    fileWritten.vid = tape1.vid;
+    fileWritten.fSeq = i;
+    fileWritten.blockId = i * 100;
+    fileWritten.copyNb = 1;
+    fileWritten.tapeDrive = tapeDrive;
+    tapeFilesWrittenCopy1.emplace(fileWrittenUP.release());
+  }
+  m_catalogue->filesWrittenToTape(tapeFilesWrittenCopy1);
+  {
+    ASSERT_TRUE(m_catalogue->getArchiveFilesItor().hasMore());
+    ASSERT_FALSE(m_catalogue->getFileRecycleLogItor().hasMore());
+  }
+  log::LogContext dummyLc(m_dummyLog); 
+  //Archive the same files but in a new tape
+  for(auto & tapeItemWritten: tapeFilesWrittenCopy1){
+    cta::catalogue::TapeFileWritten * tapeItem = static_cast<cta::catalogue::TapeFileWritten *>(tapeItemWritten.get());
+    tapeItem->vid = tape2.vid;
+  }
+  m_catalogue->filesWrittenToTape(tapeFilesWrittenCopy1);
+  //Change the vid back to the first one to test the content of the file recycle log afterwards
+  for(auto & tapeItemWritten: tapeFilesWrittenCopy1){
+    cta::catalogue::TapeFileWritten * tapeItem = static_cast<cta::catalogue::TapeFileWritten *>(tapeItemWritten.get());
+    tapeItem->vid = tape1.vid;
+  }
+  //Check that the new files written exist on the catalogue
+  {
+    auto archiveFilesItor = m_catalogue->getArchiveFilesItor();
+    bool hasArchiveFilesItor = false;
+    while(archiveFilesItor.hasMore()){
+      hasArchiveFilesItor = true;
+      //The vid is the destination one
+      ASSERT_EQ(tape2.vid,archiveFilesItor.next().tapeFiles.at(1).vid);
+    }
+    ASSERT_TRUE(hasArchiveFilesItor);
+  }
+  //Check that the old files are in the file recycle logs
+  std::list<common::dataStructures::FileRecycleLog> fileRecycleLogs;
+  {
+    auto fileRecycleLogItor = m_catalogue->getFileRecycleLogItor();
+    while(fileRecycleLogItor.hasMore()){
+      fileRecycleLogs.push_back(fileRecycleLogItor.next());
+    }
+    ASSERT_FALSE(fileRecycleLogs.empty());
+    //Now we check the consistency of what is returned by the file recycle log
+    for(auto fileRecycleLog: fileRecycleLogs){
+      //First, get the correct file written to tape associated to the current fileRecycleLog
+      auto tapeFilesWrittenCopy1Itor = std::find_if(tapeFilesWrittenCopy1.begin(), tapeFilesWrittenCopy1.end(),[fileRecycleLog](const cta::catalogue::TapeItemWrittenPointer & item){
+        cta::catalogue::TapeFileWritten * fileWrittenPtr = static_cast<cta::catalogue::TapeFileWritten *>(item.get());
+        return fileWrittenPtr->archiveFileId == fileRecycleLog.archiveFileId;
+      });
+      
+      ASSERT_NE(tapeFilesWrittenCopy1.end(), tapeFilesWrittenCopy1Itor);
+      cta::catalogue::TapeFileWritten * fileWrittenPtr = static_cast<cta::catalogue::TapeFileWritten *>(tapeFilesWrittenCopy1Itor->get());
+      ASSERT_EQ(fileRecycleLog.vid,tape1.vid);
+      ASSERT_EQ(fileRecycleLog.fSeq,fileWrittenPtr->fSeq);
+      ASSERT_EQ(fileRecycleLog.blockId,fileWrittenPtr->blockId);
+      ASSERT_EQ(fileRecycleLog.logicalSizeInBytes,fileWrittenPtr->size);
+      ASSERT_EQ(fileRecycleLog.copyNb,fileWrittenPtr->copyNb);
+      ASSERT_EQ(fileRecycleLog.archiveFileId,fileWrittenPtr->archiveFileId);
+      ASSERT_EQ(fileRecycleLog.diskInstanceName,fileWrittenPtr->diskInstance);
+      ASSERT_EQ(fileRecycleLog.diskFileId,fileWrittenPtr->diskFileId);
+      ASSERT_EQ(fileRecycleLog.diskFileIdWhenDeleted,fileWrittenPtr->diskFileId);
+      ASSERT_EQ(fileRecycleLog.diskFileUid,fileWrittenPtr->diskFileOwnerUid);
+      ASSERT_EQ(fileRecycleLog.diskFileGid,fileWrittenPtr->diskFileGid);
+      ASSERT_EQ(fileRecycleLog.sizeInBytes,fileWrittenPtr->size);
+      ASSERT_EQ(fileRecycleLog.checksumBlob,fileWrittenPtr->checksumBlob);
+      ASSERT_EQ(fileRecycleLog.storageClassName,fileWrittenPtr->storageClassName);
+      ASSERT_EQ(fileRecycleLog.reconciliationTime,fileRecycleLog.archiveFileCreationTime);
+      ASSERT_EQ(cta::nullopt, fileRecycleLog.collocationHint);
+      ASSERT_EQ(cta::nullopt, fileRecycleLog.diskFilePath);\
+      ASSERT_EQ(cta::catalogue::InsertFileRecycleLog::getRepackReasonLog(),fileRecycleLog.reasonLog);
+    }
+  }
+}
+
+TEST_P(cta_catalogue_CatalogueTest, sameFileWrittenToSameTapePutThePreviousCopyOnTheFileRecycleLog) {
+  using namespace cta;
+
+  const bool logicalLibraryIsDisabled= false;
+  const std::string tapePoolName1 = "tape_pool_name_1";
+  const std::string tapePoolName2 = "tape_pool_name_2";
+  const uint64_t nbPartialTapes = 1;
+  const bool isEncrypted = true;
+  const cta::optional<std::string> supply("value for the supply pool mechanism");
+  const std::string diskInstance = "disk_instance";
+  const std::string tapeDrive = "tape_drive";
+
+  m_catalogue->createMediaType(m_admin, m_mediaType);
+  m_catalogue->createLogicalLibrary(m_admin, m_tape1.logicalLibraryName, logicalLibraryIsDisabled, "Create logical library");
+  m_catalogue->createVirtualOrganization(m_admin, m_vo);
+  m_catalogue->createTapePool(m_admin, tapePoolName1, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
+  m_catalogue->createTapePool(m_admin, tapePoolName2, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
+  m_catalogue->createStorageClass(m_admin, m_storageClassSingleCopy);
+
+  auto tape1 = m_tape1;
+  tape1.tapePoolName = tapePoolName1;
+
+  m_catalogue->createTape(m_admin, tape1);
+
+  ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore());
+  const uint64_t archiveFileSize = 2 * 1000 * 1000 * 1000;
+
+  std::set<catalogue::TapeItemWrittenPointer> tapeFilesWrittenCopy1;
+  
+  std::ostringstream diskFileId;
+  diskFileId << 12345677;
+
+  std::ostringstream diskFilePath;
+  diskFilePath << "/test/file1";
+
+  // Two files same archiveFileId and CopyNb on the same tape
+  auto fileWrittenUP=cta::make_unique<cta::catalogue::TapeFileWritten>();
+  auto & fileWritten = *fileWrittenUP;
+  fileWritten.archiveFileId = 1;
+  fileWritten.diskInstance = diskInstance;
+  fileWritten.diskFileId = diskFileId.str();
+  fileWritten.diskFilePath = diskFilePath.str();
+  fileWritten.diskFileOwnerUid = PUBLIC_DISK_USER;
+  fileWritten.diskFileGid = PUBLIC_DISK_GROUP;
+  fileWritten.size = archiveFileSize;
+  fileWritten.checksumBlob.insert(checksum::ADLER32, "1357");
+  fileWritten.storageClassName = m_storageClassSingleCopy.name;
+  fileWritten.vid = tape1.vid;
+  fileWritten.fSeq = 1;
+  fileWritten.blockId = 1 * 100;
+  fileWritten.copyNb = 1;
+  fileWritten.tapeDrive = tapeDrive;
+  tapeFilesWrittenCopy1.emplace(fileWrittenUP.release());
+  
+  m_catalogue->filesWrittenToTape(tapeFilesWrittenCopy1);
+  
+  auto & tapeItemWritten =  *(tapeFilesWrittenCopy1.begin());
+  cta::catalogue::TapeFileWritten * tapeItem = static_cast<cta::catalogue::TapeFileWritten *>(tapeItemWritten.get());
+  tapeItem->fSeq = 2;
+  tapeItem->blockId = 2 *100 + 1;
+  
+  m_catalogue->filesWrittenToTape(tapeFilesWrittenCopy1);
+  {
+    auto archiveFilesItor = m_catalogue->getArchiveFilesItor();
+    ASSERT_TRUE(archiveFilesItor.hasMore());
+    //The file with fseq 2 is on the active archive files of CTA
+    ASSERT_EQ(2,archiveFilesItor.next().tapeFiles.at(1).fSeq);
+    auto fileRecycleLogItor = m_catalogue->getFileRecycleLogItor();
+    ASSERT_TRUE(fileRecycleLogItor.hasMore());
+    //The previous file (fSeq = 1) is on the recycle log
+    ASSERT_EQ(1,fileRecycleLogItor.next().fSeq);
+  }  
+}
+
 } // namespace unitTests

catalogue/DummyCatalogue.hpp

@@ -83,7 +83,9 @@ public:
   common::dataStructures::ArchiveFile getArchiveFileById(const uint64_t id) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
   ArchiveFileItor getArchiveFilesItor(const TapeFileSearchCriteria& searchCriteria) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
   DeletedArchiveFileItor getDeletedArchiveFilesItor() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
+  FileRecycleLogItor getFileRecycleLogItor() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
   void deleteFileFromRecycleBin(const uint64_t archiveFileId, log::LogContext &lc) {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
+  void deleteFilesFromRecycleLog(const std::string & vid, log::LogContext & lc) {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");}
   std::list<common::dataStructures::ArchiveFile> getFilesForRepack(const std::string &vid, const uint64_t startFSeq, const uint64_t maxNbFiles) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
   ArchiveFileItor getArchiveFilesForRepackItor(const std::string &vid, const uint64_t startFSeq) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
   std::list<common::dataStructures::ArchiveRoute> getArchiveRoutes() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }

catalogue/InsertFileRecycleLog.hpp

@@ -25,29 +25,22 @@
 
 namespace cta { namespace catalogue {
 
-  struct InsertFileRecycleLog {
+  class InsertFileRecycleLog {
+  public:
     std::string vid;
     uint64_t fseq;
     uint64_t blockId;
     uint64_t logicalSizeInBytes;
-    uint64_t copyNb;
+    uint8_t copyNb;
     time_t tapeFileCreationTime;
     uint64_t archiveFileId;
-    std::string diskInstanceName;
-    uint64_t diskFileId;
-    uint64_t diskFileIdWhenDeleted;
-    uint64_t diskFileUid;
-    uint64_t diskFileGid;
-    uint64_t sizeInBytes;
-    checksum::ChecksumBlob checksumBlob;
-    uint32_t checksumAdler32;
-    uint64_t storageClassId;
-    time_t archiveFileCreationTime;
-    time_t reconciliationTime;
-    std::string collocationHint;
     std::string diskFilePath;
     std::string reasonLog;
     time_t recycleLogTime;
+    
+    static std::string getRepackReasonLog(){
+      return "REPACK";
+    }
   };
   
 }}

catalogue/OracleCatalogue.cpp

@@ -572,6 +572,8 @@ void OracleCatalogue::filesWrittenToTape(const std::set<TapeItemWrittenPointer>
       fileSizeAndChecksum.checksumBlob.validate(event.checksumBlob);
     }
 
+    std::list<InsertFileRecycleLog> recycledFiles = insertOldCopiesOfFilesIfAnyOnFileRecycleLog(conn);
+        
     {
       const char *const sql =
         "INSERT INTO TAPE_FILE (VID, FSEQ, BLOCK_ID, LOGICAL_SIZE_IN_BYTES,"              "\n"
@@ -582,8 +584,6 @@ void OracleCatalogue::filesWrittenToTape(const std::set<TapeItemWrittenPointer>
       stmt.executeNonQuery();
     }
     
-    std::list<InsertFileRecycleLog> recycledFiles = insertOldCopiesOfFilesIfAnyOnFileRecycleLog(conn);
-    
     for(auto & recycledFile: recycledFiles){
       const char *const sql =
         "DELETE FROM "
@@ -782,7 +782,7 @@ std::list<cta::catalogue::InsertFileRecycleLog> OracleCatalogue::insertOldCopies
         "ON "
           "TEMP_TAPE_FILE_INSERTION_BATCH.ARCHIVE_FILE_ID = TAPE_FILE.ARCHIVE_FILE_ID AND TEMP_TAPE_FILE_INSERTION_BATCH.COPY_NB = TAPE_FILE.COPY_NB "
         "WHERE "
-          "TAPE_FILE.VID != TEMP_TAPE_FILE_INSERTION_BATCH.VID";
+          "TAPE_FILE.VID != TEMP_TAPE_FILE_INSERTION_BATCH.VID OR TAPE_FILE.FSEQ != TEMP_TAPE_FILE_INSERTION_BATCH.FSEQ";
       auto stmt = conn.createStmt(sql);
       auto rset = stmt.executeQuery();
       while(rset.next()){
@@ -794,7 +794,7 @@ std::list<cta::catalogue::InsertFileRecycleLog> OracleCatalogue::insertOldCopies
         fileRecycleLog.copyNb = rset.columnUint8("COPY_NB");
         fileRecycleLog.tapeFileCreationTime = rset.columnUint64("TAPE_FILE_CREATION_TIME");
         fileRecycleLog.archiveFileId = rset.columnUint64("ARCHIVE_FILE_ID");
-        fileRecycleLog.reasonLog = "Repacked from VID " + fileRecycleLog.vid;
+        fileRecycleLog.reasonLog = InsertFileRecycleLog::getRepackReasonLog();
         fileRecycleLog.recycleLogTime = time(nullptr);
         fileRecycleLogsToInsert.push_back(fileRecycleLog);
       }

catalogue/PostgresCatalogue.cpp

@@ -502,11 +502,7 @@ void PostgresCatalogue::filesWrittenToTape(const std::set<TapeItemWrittenPointer
       "-- :LOGICAL_SIZE_IN_BYTES,"                                                   "\n"
       "-- :COPY_NB,"                                                                 "\n"
       "-- :CREATION_TIME,"                                                           "\n"
-      "-- :ARCHIVE_FILE_ID"                                                          "\n"
-    "INSERT INTO TAPE_FILE (VID, FSEQ, BLOCK_ID, LOGICAL_SIZE_IN_BYTES,"             "\n"
-      "COPY_NB, CREATION_TIME, ARCHIVE_FILE_ID) "                                    "\n"
-    "SELECT VID, FSEQ, BLOCK_ID, LOGICAL_SIZE_IN_BYTES,"                             "\n"
-      "COPY_NB, CREATION_TIME, ARCHIVE_FILE_ID FROM TEMP_TAPE_FILE_INSERTION_BATCH;" "\n";
+      "-- :ARCHIVE_FILE_ID;"                                                         "\n";
     
     auto stmt = conn.createStmt(sql);
     rdbms::wrapper::PostgresStmt &postgresStmt = dynamic_cast<rdbms::wrapper::PostgresStmt &>(stmt.getStmt());
@@ -522,6 +518,16 @@ void PostgresCatalogue::filesWrittenToTape(const std::set<TapeItemWrittenPointer
     
     auto recycledFiles = insertOldCopiesOfFilesIfAnyOnFileRecycleLog(conn);
     
+    {
+      //Insert the tapefiles from the TEMP_TAPE_FILE_INSERTION_BATCH
+      const char * const insertTapeFileSql = 
+      "INSERT INTO TAPE_FILE (VID, FSEQ, BLOCK_ID, LOGICAL_SIZE_IN_BYTES,"            "\n"
+      "COPY_NB, CREATION_TIME, ARCHIVE_FILE_ID) "                                     "\n"
+      "SELECT VID, FSEQ, BLOCK_ID, LOGICAL_SIZE_IN_BYTES,"                            "\n"
+      "COPY_NB, CREATION_TIME, ARCHIVE_FILE_ID FROM TEMP_TAPE_FILE_INSERTION_BATCH;"  "\n";
+      conn.executeNonQuery(insertTapeFileSql);
+    }
+    
     for(auto & recycledFile: recycledFiles){
       const char * const deleteTapeFileSql = 
       "DELETE FROM TAPE_FILE WHERE TAPE_FILE.VID = :VID AND TAPE_FILE.FSEQ = :FSEQ";
@@ -685,7 +691,7 @@ std::list<cta::catalogue::InsertFileRecycleLog> PostgresCatalogue::insertOldCopi
         "ON "
           "TEMP_TAPE_FILE_INSERTION_BATCH.ARCHIVE_FILE_ID = TAPE_FILE.ARCHIVE_FILE_ID AND TEMP_TAPE_FILE_INSERTION_BATCH.COPY_NB = TAPE_FILE.COPY_NB "
         "WHERE "
-          "TAPE_FILE.VID != TEMP_TAPE_FILE_INSERTION_BATCH.VID";
+          "TAPE_FILE.VID != TEMP_TAPE_FILE_INSERTION_BATCH.VID OR TAPE_FILE.FSEQ != TEMP_TAPE_FILE_INSERTION_BATCH.FSEQ";
       auto stmt = conn.createStmt(sql);
       auto rset = stmt.executeQuery();
       while(rset.next()){
@@ -697,7 +703,7 @@ std::list<cta::catalogue::InsertFileRecycleLog> PostgresCatalogue::insertOldCopi
         fileRecycleLog.copyNb = rset.columnUint8("COPY_NB");
         fileRecycleLog.tapeFileCreationTime = rset.columnUint64("TAPE_FILE_CREATION_TIME");
         fileRecycleLog.archiveFileId = rset.columnUint64("ARCHIVE_FILE_ID");
-        fileRecycleLog.reasonLog = "Repacked from VID " + fileRecycleLog.vid;
+        fileRecycleLog.reasonLog = InsertFileRecycleLog::getRepackReasonLog();
         fileRecycleLog.recycleLogTime = time(nullptr);
         fileRecycleLogsToInsert.push_back(fileRecycleLog);
       }

catalogue/RdbmsCatalogue.cpp

@@ -37,6 +37,7 @@
 #include "common/utils/utils.hpp"
 #include "rdbms/AutoRollback.hpp"
 #include "version.h"
+#include "RdbmsCatalogueGetFileRecycleLogItor.hpp"
 
 #include <ctype.h>
 #include <memory>
@@ -476,7 +477,7 @@ void RdbmsCatalogue::modifyVirtualOrganizationName(const common::dataStructures:
         "LAST_UPDATE_HOST_NAME = :LAST_UPDATE_HOST_NAME,"
         "LAST_UPDATE_TIME = :LAST_UPDATE_TIME "
       "WHERE "
-        "VIRTUAL_ORGANIZATION_NAME = :CURRENT_VIRTUAL_ORGANIZATION_NAME";
+        "VIRTUAL_ORGANIZATION_NAME = :CUR_VIRTUAL_ORGANIZATION_NAME";
     auto conn = m_connPool.getConn();
     if(newVoName != currentVoName){
       if(virtualOrganizationExists(conn,newVoName)){
@@ -488,7 +489,7 @@ void RdbmsCatalogue::modifyVirtualOrganizationName(const common::dataStructures:
     stmt.bindString(":LAST_UPDATE_USER_NAME", admin.username);
     stmt.bindString(":LAST_UPDATE_HOST_NAME", admin.host);
     stmt.bindUint64(":LAST_UPDATE_TIME", now);
-    stmt.bindString(":CURRENT_VIRTUAL_ORGANIZATION_NAME", currentVoName);
+    stmt.bindString(":CUR_VIRTUAL_ORGANIZATION_NAME", currentVoName);
     stmt.executeNonQuery();
 
     if(0 == stmt.getNbAffectedRows()) {
@@ -6801,6 +6802,18 @@ Catalogue::DeletedArchiveFileItor RdbmsCatalogue::getDeletedArchiveFilesItor() c
   }
 }
 
+Catalogue::FileRecycleLogItor RdbmsCatalogue::getFileRecycleLogItor() const {
+  try {
+    auto impl = new RdbmsCatalogueGetFileRecycleLogItor(m_log, m_archiveFileListingConnPool);
+    return FileRecycleLogItor(impl);
+  } catch(exception::UserError &) {
+    throw;
+  } catch(exception::Exception &ex) {
+    ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+    throw;
+  }
+}
+
 //------------------------------------------------------------------------------
 // getFilesForRepack
 //------------------------------------------------------------------------------
@@ -7679,120 +7692,51 @@ void RdbmsCatalogue::insertTapeFile(
   const uint64_t archiveFileId) {
   rdbms::AutoRollback autoRollback(conn);
   try{
+    
+      std::list<InsertFileRecycleLog> insertedFilesRecycleLog = insertOldCopiesOfFilesIfAnyOnFileRecycleLog(conn,tapeFile,archiveFileId);
+      {
+        const time_t now = time(nullptr);
+        const char *const sql =
+          "INSERT INTO TAPE_FILE("
+            "VID,"
+            "FSEQ,"
+            "BLOCK_ID,"
+            "LOGICAL_SIZE_IN_BYTES,"
+            "COPY_NB,"
+            "CREATION_TIME,"
+            "ARCHIVE_FILE_ID)"
+          "VALUES("
+            ":VID,"
+            ":FSEQ,"
+            ":BLOCK_ID,"
+            ":LOGICAL_SIZE_IN_BYTES,"
+            ":COPY_NB,"
+            ":CREATION_TIME,"
+            ":ARCHIVE_FILE_ID)";
+        auto stmt = conn.createStmt(sql);
+
+        stmt.bindString(":VID", tapeFile.vid);
+        stmt.bindUint64(":FSEQ", tapeFile.fSeq);
+        stmt.bindUint64(":BLOCK_ID", tapeFile.blockId);
+        stmt.bindUint64(":LOGICAL_SIZE_IN_BYTES", tapeFile.fileSize);
+        stmt.bindUint64(":COPY_NB", tapeFile.copyNb);
+        stmt.bindUint64(":CREATION_TIME", now);
+        stmt.bindUint64(":ARCHIVE_FILE_ID", archiveFileId);
+        stmt.executeNonQuery();
+      }
     {
-      const time_t now = time(nullptr);
-      const char *const sql =
-        "INSERT INTO TAPE_FILE("
-          "VID,"
-          "FSEQ,"
-          "BLOCK_ID,"
-          "LOGICAL_SIZE_IN_BYTES,"
-          "COPY_NB,"
-          "CREATION_TIME,"
-          "ARCHIVE_FILE_ID)"
-        "VALUES("
-          ":VID,"
-          ":FSEQ,"
-          ":BLOCK_ID,"
-          ":LOGICAL_SIZE_IN_BYTES,"
-          ":COPY_NB,"
-          ":CREATION_TIME,"
-          ":ARCHIVE_FILE_ID)";
-      auto stmt = conn.createStmt(sql);
-
-      stmt.bindString(":VID", tapeFile.vid);
-      stmt.bindUint64(":FSEQ", tapeFile.fSeq);
-      stmt.bindUint64(":BLOCK_ID", tapeFile.blockId);
-      stmt.bindUint64(":LOGICAL_SIZE_IN_BYTES", tapeFile.fileSize);
-      stmt.bindUint64(":COPY_NB", tapeFile.copyNb);
-      stmt.bindUint64(":CREATION_TIME", now);
-      stmt.bindUint64(":ARCHIVE_FILE_ID", archiveFileId);
-      stmt.executeNonQuery();
-    }
-    {
-      uint64_t fileRecycleLogId = getNextFileRecyleLogId(conn);
-      const char *const sql = 
-      "INSERT INTO FILE_RECYCLE_LOG ("
-        "FILE_RECYCLE_LOG_ID,"
-        "VID,"
-        "FSEQ,"
-        "BLOCK_ID,"
-        "LOGICAL_SIZE_IN_BYTES,"
-        "COPY_NB,"
-        "TAPE_FILE_CREATION_TIME,"
-        "ARCHIVE_FILE_ID,"
-        "DISK_INSTANCE_NAME,"
-        "DISK_FILE_ID,"
-        "DISK_FILE_ID_WHEN_DELETED,"
-        "DISK_FILE_UID,"
-        "DISK_FILE_GID,"
-        "SIZE_IN_BYTES,"
-        "CHECKSUM_BLOB,"
-        "CHECKSUM_ADLER32,"
-        "STORAGE_CLASS_ID,"
-        "ARCHIVE_FILE_CREATION_TIME,"
-        "RECONCILIATION_TIME,"
-        "COLLOCATION_HINT,"
-        "REASON_LOG,"
-        "RECYCLE_LOG_TIME"
-      ") SELECT "
-        ":FILE_RECYCLE_LOG_ID,"
-        "TAPE_FILE.VID,"
-        "TAPE_FILE.FSEQ,"
-        "TAPE_FILE.BLOCK_ID,"
-        "TAPE_FILE.LOGICAL_SIZE_IN_BYTES,"
-        "TAPE_FILE.COPY_NB,"
-        "TAPE_FILE.CREATION_TIME,"
-        "TAPE_FILE.ARCHIVE_FILE_ID,"
-        "ARCHIVE_FILE.DISK_INSTANCE_NAME AS DISK_INSTANCE_NAME,"
-        "ARCHIVE_FILE.DISK_FILE_ID AS DISK_FILE_ID,"
-        "ARCHIVE_FILE.DISK_FILE_ID AS DISK_FILE_ID_2,"
-        "ARCHIVE_FILE.DISK_FILE_UID AS DISK_FILE_UID,"
-        "ARCHIVE_FILE.DISK_FILE_GID AS DISK_FILE_GID,"
-        "ARCHIVE_FILE.SIZE_IN_BYTES AS SIZE_IN_BYTES,"
-        "ARCHIVE_FILE.CHECKSUM_BLOB AS CHECKSUM_BLOB,"
-        "ARCHIVE_FILE.CHECKSUM_ADLER32 AS CHECKSUM_ADLER32,"
-        "ARCHIVE_FILE.STORAGE_CLASS_ID AS STORAGE_CLASS_ID,"
-        "ARCHIVE_FILE.CREATION_TIME AS ARCHIVE_FILE_CREATION_TIME,"
-        "ARCHIVE_FILE.RECONCILIATION_TIME AS RECONCILIATION_TIME,"