Commit a7a61afc authored by Cedric CAFFY's avatar Cedric CAFFY Committed by Eric Cano
Browse files

Added the deletion of every superseded tape files during an archive mount

parent 6938321d
...@@ -600,12 +600,13 @@ public: ...@@ -600,12 +600,13 @@ public:
/** /**
* Returns true if non superseded files exist after fSeq in the tape where vid is passed in parameter * Returns true if non superseded files exist after fSeq in the tape where vid is passed in parameter
* If there is only superseded files after fSeq, these tape files will be deleted
* *
* @param vid the vid of the tape to check if non superseded files exist after fSeq * @param vid the vid of the tape to check if non superseded files exist after fSeq
* @param fSeq the fSeq after which we want to check if non superseded files exist * @param fSeq the fSeq after which we want to check if non superseded files exist
* @return true if non superseded files exist, false otherwise * @return true if non superseded files exist, false otherwise
*/ */
virtual bool existNonSupersededFilesAfterFSeq(const std::string &vid, const uint64_t fSeq) const = 0; virtual bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string &vid, const uint64_t fSeq) const = 0;
}; // class Catalogue }; // class Catalogue
......
...@@ -381,8 +381,8 @@ public: ...@@ -381,8 +381,8 @@ public:
return retryOnLostConnection(m_log, [&]{return m_catalogue->tapeExists(vid);}, m_maxTriesToConnect); return retryOnLostConnection(m_log, [&]{return m_catalogue->tapeExists(vid);}, m_maxTriesToConnect);
} }
bool existNonSupersededFilesAfterFSeq(const std::string& vid, const uint64_t fSeq) const override { bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const override {
return retryOnLostConnection(m_log,[&]{return m_catalogue->existNonSupersededFilesAfterFSeq(vid,fSeq);},m_maxTriesToConnect); return retryOnLostConnection(m_log,[&]{return m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid,fSeq);},m_maxTriesToConnect);
} }
......
...@@ -11532,7 +11532,7 @@ TEST_P(cta_catalogue_CatalogueTest, exist_non_superseded_files_after_fseq) { ...@@ -11532,7 +11532,7 @@ TEST_P(cta_catalogue_CatalogueTest, exist_non_superseded_files_after_fseq) {
disabledValue, fullValue, createTapeComment); disabledValue, fullValue, createTapeComment);
//A tape with no tape file have no files after FSeq 0 //A tape with no tape file have no files after FSeq 0
ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeq(vid1,0)); ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,0));
const uint64_t archiveFileId = 1234; const uint64_t archiveFileId = 1234;
...@@ -11578,9 +11578,9 @@ TEST_P(cta_catalogue_CatalogueTest, exist_non_superseded_files_after_fseq) { ...@@ -11578,9 +11578,9 @@ TEST_P(cta_catalogue_CatalogueTest, exist_non_superseded_files_after_fseq) {
m_catalogue->filesWrittenToTape(file1WrittenSet); m_catalogue->filesWrittenToTape(file1WrittenSet);
} }
//One file written : this file is not superseded by another one, existNonSupersededFilesAfterFSeq = true //One file written : this file is not superseded by another one, existNonSupersededFilesAfterFSeq = true
ASSERT_TRUE(m_catalogue->existNonSupersededFilesAfterFSeq(vid1,0)); ASSERT_TRUE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,0));
//No file after the only file inserted, existNonSupersededFilesAfterFseq = false //No file after the only file inserted, existNonSupersededFilesAfterFseq = false
ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeq(vid1,1)); ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,1));
//Insert another file in another tape that will supersed the first one in vid1 //Insert another file in another tape that will supersed the first one in vid1
{ {
...@@ -11615,8 +11615,8 @@ TEST_P(cta_catalogue_CatalogueTest, exist_non_superseded_files_after_fseq) { ...@@ -11615,8 +11615,8 @@ TEST_P(cta_catalogue_CatalogueTest, exist_non_superseded_files_after_fseq) {
} }
//The tape files written to tape vid2 are not superseded by any file, but the tape files in vid1 //The tape files written to tape vid2 are not superseded by any file, but the tape files in vid1
//are superseded by the tape files in vid2 //are superseded by the tape files in vid2
ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeq(vid1,0)); ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,0));
ASSERT_TRUE(m_catalogue->existNonSupersededFilesAfterFSeq(vid2,0)); ASSERT_TRUE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid2,0));
} }
TEST_P(cta_catalogue_CatalogueTest, ping) { TEST_P(cta_catalogue_CatalogueTest, ping) {
......
...@@ -114,7 +114,7 @@ public: ...@@ -114,7 +114,7 @@ public:
void tapeMountedForArchive(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } void tapeMountedForArchive(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void tapeMountedForRetrieve(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } void tapeMountedForRetrieve(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
bool tapePoolExists(const std::string& tapePoolName) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } bool tapePoolExists(const std::string& tapePoolName) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
bool existNonSupersededFilesAfterFSeq(const std::string& vid, const uint64_t fSeq) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
// Special functions for unit tests. // Special functions for unit tests.
void addEnabledTape(const std::string & vid) { void addEnabledTape(const std::string & vid) {
......
...@@ -1791,7 +1791,7 @@ bool RdbmsCatalogue::tapeExists(rdbms::Conn &conn, const std::string &vid) const ...@@ -1791,7 +1791,7 @@ bool RdbmsCatalogue::tapeExists(rdbms::Conn &conn, const std::string &vid) const
} }
} }
bool RdbmsCatalogue::existNonSupersededFilesAfterFSeq(const std::string& vid, const uint64_t fSeq) const { bool RdbmsCatalogue::existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const {
try{ try{
auto conn = m_connPool.getConn(); auto conn = m_connPool.getConn();
const char *const sql = const char *const sql =
...@@ -1806,7 +1806,23 @@ bool RdbmsCatalogue::existNonSupersededFilesAfterFSeq(const std::string& vid, co ...@@ -1806,7 +1806,23 @@ bool RdbmsCatalogue::existNonSupersededFilesAfterFSeq(const std::string& vid, co
stmt.bindString(":VID",vid); stmt.bindString(":VID",vid);
stmt.bindUint64(":FSEQ",fSeq); stmt.bindUint64(":FSEQ",fSeq);
auto rset = stmt.executeQuery(); auto rset = stmt.executeQuery();
return rset.next(); if(!rset.next()){
//No non-superseded files detected, we can delete the tape files
const char *const sqlDelete =
"DELETE FROM TAPE_FILE "
"WHERE "
"VID =:VID AND "
"FSEQ > :FSEQ AND "
"SUPERSEDED_BY_VID IS NOT NULL AND "
"SUPERSEDED_BY_FSEQ IS NOT NULL";
auto stmtDelete = conn.createStmt(sqlDelete);
stmtDelete.bindString(":VID",vid);
stmtDelete.bindUint64(":FSEQ",fSeq);
stmtDelete.executeNonQuery();
return false;
}
//Non superseded files are present
return true;
} catch(exception::UserError &) { } catch(exception::UserError &) {
throw; throw;
} catch(exception::Exception &ex) { } catch(exception::Exception &ex) {
......
...@@ -716,12 +716,13 @@ protected: ...@@ -716,12 +716,13 @@ protected:
/** /**
* Returns true if non superseded files exist after fSeq in the tape where vid is passed in parameter * Returns true if non superseded files exist after fSeq in the tape where vid is passed in parameter
* If there is only superseded files after fSeq, these tape files will be deleted
* *
* @param vid the vid of the tape to check if non superseded files exist after fSeq * @param vid the vid of the tape to check if non superseded files exist after fSeq
* @param fSeq the fSeq after which we want to check if non superseded files exist * @param fSeq the fSeq after which we want to check if non superseded files exist
* @return true if non superseded files exist, false otherwise * @return true if non superseded files exist, false otherwise
*/ */
bool existNonSupersededFilesAfterFSeq(const std::string& vid, const uint64_t fSeq) const override; bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const override;
/** /**
......
...@@ -44,7 +44,7 @@ kubectl -n ${NAMESPACE} exec ctaeos -- bash /root/grep_xrdlog_mgm_for_error.sh | ...@@ -44,7 +44,7 @@ kubectl -n ${NAMESPACE} exec ctaeos -- bash /root/grep_xrdlog_mgm_for_error.sh |
echo echo
./simple_repack.sh -n ${NAMESPACE} || exit 1 ./simple_repack.sh -n ${NAMESPACE} || exit 1
NB_FILES=20 NB_FILES=10000
FILE_SIZE_KB=15 FILE_SIZE_KB=15
echo echo
...@@ -53,7 +53,7 @@ echo " Archiving ${NB_FILES} files of ${FILE_SIZE_KB}kB each" ...@@ -53,7 +53,7 @@ echo " Archiving ${NB_FILES} files of ${FILE_SIZE_KB}kB each"
echo " Archiving files: xrdcp as user1" echo " Archiving files: xrdcp as user1"
echo " Retrieving them as poweruser1" echo " Retrieving them as poweruser1"
kubectl -n ${NAMESPACE} cp client_ar.sh client:/root/client_ar.sh kubectl -n ${NAMESPACE} cp client_ar.sh client:/root/client_ar.sh
kubectl -n ${NAMESPACE} exec client -- bash /root/client_ar.sh -n ${NB_FILES} -s ${FILE_SIZE_KB} -p 100 -d /eos/ctaeos/preprod -v || exit 1 kubectl -n ${NAMESPACE} exec client -- bash /root/client_ar.sh -n ${NB_FILES} -s ${FILE_SIZE_KB} -p 100 -d /eos/ctaeos/preprod -v -r || exit 1
kubectl -n ${NAMESPACE} exec ctaeos -- bash /root/grep_xrdlog_mgm_for_error.sh || exit 1 kubectl -n ${NAMESPACE} exec ctaeos -- bash /root/grep_xrdlog_mgm_for_error.sh || exit 1
......
...@@ -83,3 +83,8 @@ kubectl -n ${NAMESPACE} exec ctacli -ti -- cta-admin ta ls -v V01003 -h ...@@ -83,3 +83,8 @@ kubectl -n ${NAMESPACE} exec ctacli -ti -- cta-admin ta ls -v V01003 -h
echo echo
echo "Summary of the repack requests" echo "Summary of the repack requests"
kubectl -n ${NAMESPACE} exec ctacli -ti -- cta-admin re ls -h kubectl -n ${NAMESPACE} exec ctacli -ti -- cta-admin re ls -h
echo "Reclaiming tape V01003"
executeReclaim V01003
echo "End of test simple_repack"
\ No newline at end of file
...@@ -107,8 +107,8 @@ uint32_t cta::ArchiveMount::getNbFiles() const { ...@@ -107,8 +107,8 @@ uint32_t cta::ArchiveMount::getNbFiles() const {
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// checkTapeFSeqForWriting // checkTapeFSeqForWriting
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
void cta::ArchiveMount::checkTapeFSeqForWriting(uint64_t fSeq) const { void cta::ArchiveMount::checkTapeFSeqAndDeleteTapeFilesForWriting(uint64_t fSeq) const {
if(m_catalogue.existNonSupersededFilesAfterFSeq(getVid(),fSeq)){ if(m_catalogue.existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(getVid(),fSeq)){
throw cta::exception::Exception("Non superseded files have been detected in the tape "+getVid() +" after "+std::to_string(fSeq)); throw cta::exception::Exception("Non superseded files have been detected in the tape "+getVid() +" after "+std::to_string(fSeq));
} }
} }
......
...@@ -178,7 +178,7 @@ namespace cta { ...@@ -178,7 +178,7 @@ namespace cta {
* we throw an exception : We don't want these files to be lost. * we throw an exception : We don't want these files to be lost.
* @param fSeq : The fSeq after which we want to check if the writing is possible * @param fSeq : The fSeq after which we want to check if the writing is possible
*/ */
void checkTapeFSeqForWriting(uint64_t fSeq) const; void checkTapeFSeqAndDeleteTapeFilesForWriting(uint64_t fSeq) const;
/** /**
* Creates a disk reporter for the ArchiveJob (this is a wrapper). * Creates a disk reporter for the ArchiveJob (this is a wrapper).
......
...@@ -320,7 +320,7 @@ void castor::tape::tapeserver::daemon::TapeWriteSingleThread::run() { ...@@ -320,7 +320,7 @@ void castor::tape::tapeserver::daemon::TapeWriteSingleThread::run() {
m_logContext.log(cta::log::INFO, "Tape session started"); m_logContext.log(cta::log::INFO, "Tape session started");
mountTapeReadWrite(); mountTapeReadWrite();
currentErrorToCount = "Error_tapeFSeqCheckAndTapeFileCleanup"; currentErrorToCount = "Error_tapeFSeqCheckAndTapeFileCleanup";
m_archiveMount.checkTapeFSeqForWriting(m_lastFseq); m_archiveMount.checkTapeFSeqAndDeleteTapeFilesForWriting(m_lastFseq);
currentErrorToCount = "Error_tapeLoad"; currentErrorToCount = "Error_tapeLoad";
waitForDrive(); waitForDrive();
currentErrorToCount = "Error_checkingTapeAlert"; currentErrorToCount = "Error_checkingTapeAlert";
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment