Added the deletion of every superseded tape files during an archive mount

catalogue/Catalogue.hpp

@@ -600,12 +600,13 @@ public:
   
   /**
    * Returns true if non superseded files exist after fSeq in the tape where vid is passed in parameter
+   * If there is only superseded files after fSeq, these tape files will be deleted
    * 
    * @param vid the vid of the tape to check if non superseded files exist after fSeq
    * @param fSeq the fSeq after which we want to check if non superseded files exist
    * @return true if non superseded files exist, false otherwise
    */
-  virtual bool existNonSupersededFilesAfterFSeq(const std::string &vid, const uint64_t fSeq) const  = 0;
+  virtual bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string &vid, const uint64_t fSeq) const  = 0;
 
 }; // class Catalogue
 

catalogue/CatalogueRetryWrapper.hpp

@@ -381,8 +381,8 @@ public:
     return retryOnLostConnection(m_log, [&]{return m_catalogue->tapeExists(vid);}, m_maxTriesToConnect);
   }
   
-  bool existNonSupersededFilesAfterFSeq(const std::string& vid, const uint64_t fSeq) const override {
-    return retryOnLostConnection(m_log,[&]{return m_catalogue->existNonSupersededFilesAfterFSeq(vid,fSeq);},m_maxTriesToConnect);
+  bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const override {
+    return retryOnLostConnection(m_log,[&]{return m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid,fSeq);},m_maxTriesToConnect);
   }
 
 

catalogue/CatalogueTest.cpp

@@ -11532,7 +11532,7 @@ TEST_P(cta_catalogue_CatalogueTest, exist_non_superseded_files_after_fseq) {
     disabledValue, fullValue, createTapeComment);
   
   //A tape with no tape file have no files after FSeq 0
-  ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeq(vid1,0));
+  ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,0));
   
   const uint64_t archiveFileId = 1234;
 
@@ -11578,9 +11578,9 @@ TEST_P(cta_catalogue_CatalogueTest, exist_non_superseded_files_after_fseq) {
     m_catalogue->filesWrittenToTape(file1WrittenSet);
   }
   //One file written : this file is not superseded by another one, existNonSupersededFilesAfterFSeq = true
-  ASSERT_TRUE(m_catalogue->existNonSupersededFilesAfterFSeq(vid1,0));
+  ASSERT_TRUE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,0));
   //No file after the only file inserted, existNonSupersededFilesAfterFseq = false
-  ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeq(vid1,1));
+  ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,1));
   
   //Insert another file in another tape that will supersed the first one in vid1
   {
@@ -11615,8 +11615,8 @@ TEST_P(cta_catalogue_CatalogueTest, exist_non_superseded_files_after_fseq) {
   }
   //The tape files written to tape vid2 are not superseded by any file, but the tape files in vid1 
   //are superseded by the tape files in vid2
-  ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeq(vid1,0));
-  ASSERT_TRUE(m_catalogue->existNonSupersededFilesAfterFSeq(vid2,0));
+  ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,0));
+  ASSERT_TRUE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid2,0));
 }
 
 TEST_P(cta_catalogue_CatalogueTest, ping) {

catalogue/DummyCatalogue.hpp

@@ -114,7 +114,7 @@ public:
   void tapeMountedForArchive(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
   void tapeMountedForRetrieve(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
   bool tapePoolExists(const std::string& tapePoolName) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
-  bool existNonSupersededFilesAfterFSeq(const std::string& vid, const uint64_t fSeq) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
+  bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
 
   // Special functions for unit tests.
   void addEnabledTape(const std::string & vid) {

catalogue/RdbmsCatalogue.cpp

@@ -1791,7 +1791,7 @@ bool RdbmsCatalogue::tapeExists(rdbms::Conn &conn, const std::string &vid) const
   }
 }
 
-bool RdbmsCatalogue::existNonSupersededFilesAfterFSeq(const std::string& vid, const uint64_t fSeq) const {
+bool RdbmsCatalogue::existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const {
   try{
     auto conn = m_connPool.getConn();
     const char *const sql =
@@ -1806,7 +1806,23 @@ bool RdbmsCatalogue::existNonSupersededFilesAfterFSeq(const std::string& vid, co
     stmt.bindString(":VID",vid);
     stmt.bindUint64(":FSEQ",fSeq);
     auto rset = stmt.executeQuery();
-    return rset.next();
+    if(!rset.next()){
+      //No non-superseded files detected, we can delete the tape files
+      const char *const sqlDelete =
+      "DELETE FROM TAPE_FILE "
+      "WHERE "
+        "VID =:VID AND "
+        "FSEQ > :FSEQ AND "
+        "SUPERSEDED_BY_VID IS NOT NULL AND "
+        "SUPERSEDED_BY_FSEQ IS NOT NULL";
+      auto stmtDelete = conn.createStmt(sqlDelete);
+      stmtDelete.bindString(":VID",vid);
+      stmtDelete.bindUint64(":FSEQ",fSeq);
+      stmtDelete.executeNonQuery();
+      return false;
+    }
+    //Non superseded files are present
+    return true;
   } catch(exception::UserError &) {
     throw;
   } catch(exception::Exception &ex) {

catalogue/RdbmsCatalogue.hpp

@@ -716,12 +716,13 @@ protected:
   
   /**
    * Returns true if non superseded files exist after fSeq in the tape where vid is passed in parameter
+   * If there is only superseded files after fSeq, these tape files will be deleted
    * 
    * @param vid the vid of the tape to check if non superseded files exist after fSeq
    * @param fSeq the fSeq after which we want to check if non superseded files exist
    * @return true if non superseded files exist, false otherwise
    */
-  bool existNonSupersededFilesAfterFSeq(const std::string& vid, const uint64_t fSeq) const override;
+  bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const override;
 
 
   /**

continuousintegration/orchestration/tests/archive_retrieve.sh

@@ -44,7 +44,7 @@ kubectl -n ${NAMESPACE} exec ctaeos -- bash /root/grep_xrdlog_mgm_for_error.sh |
 echo
 ./simple_repack.sh -n ${NAMESPACE} || exit 1
 
-NB_FILES=20
+NB_FILES=10000
 FILE_SIZE_KB=15
 
 echo
@@ -53,7 +53,7 @@ echo " Archiving ${NB_FILES} files of ${FILE_SIZE_KB}kB each"
 echo " Archiving files: xrdcp as user1"
 echo " Retrieving them as poweruser1"
 kubectl -n ${NAMESPACE} cp client_ar.sh client:/root/client_ar.sh
-kubectl -n ${NAMESPACE} exec client -- bash /root/client_ar.sh -n ${NB_FILES} -s ${FILE_SIZE_KB} -p 100 -d /eos/ctaeos/preprod -v || exit 1
+kubectl -n ${NAMESPACE} exec client -- bash /root/client_ar.sh -n ${NB_FILES} -s ${FILE_SIZE_KB} -p 100 -d /eos/ctaeos/preprod -v -r || exit 1
 
 kubectl -n ${NAMESPACE} exec ctaeos -- bash /root/grep_xrdlog_mgm_for_error.sh || exit 1
 

continuousintegration/orchestration/tests/simple_repack.sh

@@ -82,4 +82,9 @@ kubectl -n ${NAMESPACE} exec ctacli -ti -- cta-admin ta ls -v V01003 -h
 
 echo
 echo "Summary of the repack requests"
-kubectl -n ${NAMESPACE} exec ctacli -ti -- cta-admin re ls -h
\ No newline at end of file
+kubectl -n ${NAMESPACE} exec ctacli -ti -- cta-admin re ls -h
+
+echo "Reclaiming tape V01003"
+executeReclaim V01003
+
+echo "End of test simple_repack"
\ No newline at end of file

scheduler/ArchiveMount.cpp

@@ -107,8 +107,8 @@ uint32_t cta::ArchiveMount::getNbFiles() const {
 //------------------------------------------------------------------------------
 // checkTapeFSeqForWriting
 //------------------------------------------------------------------------------
-void cta::ArchiveMount::checkTapeFSeqForWriting(uint64_t fSeq) const {
-  if(m_catalogue.existNonSupersededFilesAfterFSeq(getVid(),fSeq)){
+void cta::ArchiveMount::checkTapeFSeqAndDeleteTapeFilesForWriting(uint64_t fSeq) const {
+  if(m_catalogue.existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(getVid(),fSeq)){
     throw cta::exception::Exception("Non superseded files have been detected in the tape "+getVid() +" after "+std::to_string(fSeq));
   }
 }

scheduler/ArchiveMount.hpp

@@ -178,7 +178,7 @@ namespace cta {
      * we throw an exception : We don't want these files to be lost.
      * @param fSeq : The fSeq after which we want to check if the writing is possible
      */
-    void checkTapeFSeqForWriting(uint64_t fSeq) const;
+    void checkTapeFSeqAndDeleteTapeFilesForWriting(uint64_t fSeq) const;
     
     /**
      * Creates a disk reporter for the ArchiveJob (this is a wrapper).

tapeserver/castor/tape/tapeserver/daemon/TapeWriteSingleThread.cpp

@@ -320,7 +320,7 @@ void castor::tape::tapeserver::daemon::TapeWriteSingleThread::run() {
       m_logContext.log(cta::log::INFO, "Tape session started");
       mountTapeReadWrite();
       currentErrorToCount = "Error_tapeFSeqCheckAndTapeFileCleanup";
-      m_archiveMount.checkTapeFSeqForWriting(m_lastFseq);
+      m_archiveMount.checkTapeFSeqAndDeleteTapeFilesForWriting(m_lastFseq);
       currentErrorToCount = "Error_tapeLoad";
       waitForDrive();
       currentErrorToCount = "Error_checkingTapeAlert";