Commit a7a61afc authored by Cedric CAFFY's avatar Cedric CAFFY Committed by Eric Cano
Browse files

Added the deletion of every superseded tape files during an archive mount

parent 6938321d
......@@ -600,12 +600,13 @@ public:
/**
* Returns true if non superseded files exist after fSeq in the tape where vid is passed in parameter
* If there is only superseded files after fSeq, these tape files will be deleted
*
* @param vid the vid of the tape to check if non superseded files exist after fSeq
* @param fSeq the fSeq after which we want to check if non superseded files exist
* @return true if non superseded files exist, false otherwise
*/
virtual bool existNonSupersededFilesAfterFSeq(const std::string &vid, const uint64_t fSeq) const = 0;
virtual bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string &vid, const uint64_t fSeq) const = 0;
}; // class Catalogue
......
......@@ -381,8 +381,8 @@ public:
return retryOnLostConnection(m_log, [&]{return m_catalogue->tapeExists(vid);}, m_maxTriesToConnect);
}
bool existNonSupersededFilesAfterFSeq(const std::string& vid, const uint64_t fSeq) const override {
return retryOnLostConnection(m_log,[&]{return m_catalogue->existNonSupersededFilesAfterFSeq(vid,fSeq);},m_maxTriesToConnect);
bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const override {
return retryOnLostConnection(m_log,[&]{return m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid,fSeq);},m_maxTriesToConnect);
}
......
......@@ -11532,7 +11532,7 @@ TEST_P(cta_catalogue_CatalogueTest, exist_non_superseded_files_after_fseq) {
disabledValue, fullValue, createTapeComment);
//A tape with no tape file have no files after FSeq 0
ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeq(vid1,0));
ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,0));
const uint64_t archiveFileId = 1234;
......@@ -11578,9 +11578,9 @@ TEST_P(cta_catalogue_CatalogueTest, exist_non_superseded_files_after_fseq) {
m_catalogue->filesWrittenToTape(file1WrittenSet);
}
//One file written : this file is not superseded by another one, existNonSupersededFilesAfterFSeq = true
ASSERT_TRUE(m_catalogue->existNonSupersededFilesAfterFSeq(vid1,0));
ASSERT_TRUE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,0));
//No file after the only file inserted, existNonSupersededFilesAfterFseq = false
ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeq(vid1,1));
ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,1));
//Insert another file in another tape that will supersed the first one in vid1
{
......@@ -11615,8 +11615,8 @@ TEST_P(cta_catalogue_CatalogueTest, exist_non_superseded_files_after_fseq) {
}
//The tape files written to tape vid2 are not superseded by any file, but the tape files in vid1
//are superseded by the tape files in vid2
ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeq(vid1,0));
ASSERT_TRUE(m_catalogue->existNonSupersededFilesAfterFSeq(vid2,0));
ASSERT_FALSE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid1,0));
ASSERT_TRUE(m_catalogue->existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(vid2,0));
}
TEST_P(cta_catalogue_CatalogueTest, ping) {
......
......@@ -114,7 +114,7 @@ public:
void tapeMountedForArchive(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void tapeMountedForRetrieve(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
bool tapePoolExists(const std::string& tapePoolName) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
bool existNonSupersededFilesAfterFSeq(const std::string& vid, const uint64_t fSeq) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
// Special functions for unit tests.
void addEnabledTape(const std::string & vid) {
......
......@@ -1791,7 +1791,7 @@ bool RdbmsCatalogue::tapeExists(rdbms::Conn &conn, const std::string &vid) const
}
}
bool RdbmsCatalogue::existNonSupersededFilesAfterFSeq(const std::string& vid, const uint64_t fSeq) const {
bool RdbmsCatalogue::existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const {
try{
auto conn = m_connPool.getConn();
const char *const sql =
......@@ -1806,7 +1806,23 @@ bool RdbmsCatalogue::existNonSupersededFilesAfterFSeq(const std::string& vid, co
stmt.bindString(":VID",vid);
stmt.bindUint64(":FSEQ",fSeq);
auto rset = stmt.executeQuery();
return rset.next();
if(!rset.next()){
//No non-superseded files detected, we can delete the tape files
const char *const sqlDelete =
"DELETE FROM TAPE_FILE "
"WHERE "
"VID =:VID AND "
"FSEQ > :FSEQ AND "
"SUPERSEDED_BY_VID IS NOT NULL AND "
"SUPERSEDED_BY_FSEQ IS NOT NULL";
auto stmtDelete = conn.createStmt(sqlDelete);
stmtDelete.bindString(":VID",vid);
stmtDelete.bindUint64(":FSEQ",fSeq);
stmtDelete.executeNonQuery();
return false;
}
//Non superseded files are present
return true;
} catch(exception::UserError &) {
throw;
} catch(exception::Exception &ex) {
......
......@@ -716,12 +716,13 @@ protected:
/**
* Returns true if non superseded files exist after fSeq in the tape where vid is passed in parameter
* If there is only superseded files after fSeq, these tape files will be deleted
*
* @param vid the vid of the tape to check if non superseded files exist after fSeq
* @param fSeq the fSeq after which we want to check if non superseded files exist
* @return true if non superseded files exist, false otherwise
*/
bool existNonSupersededFilesAfterFSeq(const std::string& vid, const uint64_t fSeq) const override;
bool existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(const std::string& vid, const uint64_t fSeq) const override;
/**
......
......@@ -44,7 +44,7 @@ kubectl -n ${NAMESPACE} exec ctaeos -- bash /root/grep_xrdlog_mgm_for_error.sh |
echo
./simple_repack.sh -n ${NAMESPACE} || exit 1
NB_FILES=20
NB_FILES=10000
FILE_SIZE_KB=15
echo
......@@ -53,7 +53,7 @@ echo " Archiving ${NB_FILES} files of ${FILE_SIZE_KB}kB each"
echo " Archiving files: xrdcp as user1"
echo " Retrieving them as poweruser1"
kubectl -n ${NAMESPACE} cp client_ar.sh client:/root/client_ar.sh
kubectl -n ${NAMESPACE} exec client -- bash /root/client_ar.sh -n ${NB_FILES} -s ${FILE_SIZE_KB} -p 100 -d /eos/ctaeos/preprod -v || exit 1
kubectl -n ${NAMESPACE} exec client -- bash /root/client_ar.sh -n ${NB_FILES} -s ${FILE_SIZE_KB} -p 100 -d /eos/ctaeos/preprod -v -r || exit 1
kubectl -n ${NAMESPACE} exec ctaeos -- bash /root/grep_xrdlog_mgm_for_error.sh || exit 1
......
......@@ -83,3 +83,8 @@ kubectl -n ${NAMESPACE} exec ctacli -ti -- cta-admin ta ls -v V01003 -h
echo
echo "Summary of the repack requests"
kubectl -n ${NAMESPACE} exec ctacli -ti -- cta-admin re ls -h
echo "Reclaiming tape V01003"
executeReclaim V01003
echo "End of test simple_repack"
\ No newline at end of file
......@@ -107,8 +107,8 @@ uint32_t cta::ArchiveMount::getNbFiles() const {
//------------------------------------------------------------------------------
// checkTapeFSeqForWriting
//------------------------------------------------------------------------------
void cta::ArchiveMount::checkTapeFSeqForWriting(uint64_t fSeq) const {
if(m_catalogue.existNonSupersededFilesAfterFSeq(getVid(),fSeq)){
void cta::ArchiveMount::checkTapeFSeqAndDeleteTapeFilesForWriting(uint64_t fSeq) const {
if(m_catalogue.existNonSupersededFilesAfterFSeqAndDeleteTapeFilesForWriting(getVid(),fSeq)){
throw cta::exception::Exception("Non superseded files have been detected in the tape "+getVid() +" after "+std::to_string(fSeq));
}
}
......
......@@ -178,7 +178,7 @@ namespace cta {
* we throw an exception : We don't want these files to be lost.
* @param fSeq : The fSeq after which we want to check if the writing is possible
*/
void checkTapeFSeqForWriting(uint64_t fSeq) const;
void checkTapeFSeqAndDeleteTapeFilesForWriting(uint64_t fSeq) const;
/**
* Creates a disk reporter for the ArchiveJob (this is a wrapper).
......
......@@ -320,7 +320,7 @@ void castor::tape::tapeserver::daemon::TapeWriteSingleThread::run() {
m_logContext.log(cta::log::INFO, "Tape session started");
mountTapeReadWrite();
currentErrorToCount = "Error_tapeFSeqCheckAndTapeFileCleanup";
m_archiveMount.checkTapeFSeqForWriting(m_lastFseq);
m_archiveMount.checkTapeFSeqAndDeleteTapeFilesForWriting(m_lastFseq);
currentErrorToCount = "Error_tapeLoad";
waitForDrive();
currentErrorToCount = "Error_checkingTapeAlert";
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment