From 4511c655a4f3a83d357d60eaa6c50593049e253a Mon Sep 17 00:00:00 2001
From: Cedric CAFFY <cedric.caffy@cern.ch>
Date: Thu, 30 Jan 2020 15:44:08 +0100
Subject: [PATCH] Honoured NB_MASTER_FILES, MASTER_DATA_IN_BYTES and DIRTY in
the catalogue and in the CTA frontend
---
catalogue/CatalogueTest.cpp | 8 +++++++-
catalogue/MysqlCatalogue.cpp | 7 ++++++-
catalogue/OracleCatalogue.cpp | 6 ++++++
catalogue/PostgresCatalogue.cpp | 6 ++++++
catalogue/RdbmsCatalogue.cpp | 28 +++++++++++++++++++++++++---
catalogue/RdbmsCatalogue.hpp | 7 +++++++
catalogue/SqliteCatalogue.cpp | 6 ++++++
common/dataStructures/Tape.cpp | 5 ++++-
common/dataStructures/Tape.hpp | 5 ++++-
xroot_plugins/XrdCtaTapeLs.hpp | 5 ++++-
10 files changed, 75 insertions(+), 8 deletions(-)
diff --git a/catalogue/CatalogueTest.cpp b/catalogue/CatalogueTest.cpp
index ff57919511..b6d16c23ae 100644
--- a/catalogue/CatalogueTest.cpp
+++ b/catalogue/CatalogueTest.cpp
@@ -10690,6 +10690,7 @@ TEST_P(cta_catalogue_CatalogueTest, filesWrittenToTape_1_archive_file_1_tape_cop
ASSERT_EQ(1, tapes.size());
const common::dataStructures::Tape &tape = tapes.front();
ASSERT_EQ(1, tape.lastFSeq);
+ ASSERT_TRUE(tape.dirty);
}
{
@@ -12503,7 +12504,11 @@ TEST_P(cta_catalogue_CatalogueTest, deleteArchiveFile) {
log::LogContext dummyLc(m_dummyLog);
m_catalogue->deleteArchiveFile("disk_instance", archiveFileId, dummyLc);
-
+
+ for(auto &tape:m_catalogue->getTapes()){
+ ASSERT_TRUE(tape.dirty);
+ }
+
ASSERT_FALSE(m_catalogue->getArchiveFilesItor().hasMore());
}
@@ -14617,6 +14622,7 @@ TEST_P(cta_catalogue_CatalogueTest, reclaimTape_full_lastFSeq_0_no_tape_files) {
ASSERT_FALSE(tape.labelLog);
ASSERT_FALSE(tape.lastReadLog);
ASSERT_FALSE(tape.lastWriteLog);
+ ASSERT_TRUE(tape.dirty);
const common::dataStructures::EntryLog creationLog = tape.creationLog;
ASSERT_EQ(m_admin.username, creationLog.username);
diff --git a/catalogue/MysqlCatalogue.cpp b/catalogue/MysqlCatalogue.cpp
index edd101bdd7..554f0f5bec 100644
--- a/catalogue/MysqlCatalogue.cpp
+++ b/catalogue/MysqlCatalogue.cpp
@@ -524,6 +524,12 @@ void MysqlCatalogue::deleteArchiveFile(const std::string &diskInstanceName, cons
stmt.bindUint64(":ARCHIVE_FILE_ID", archiveFileId);
stmt.executeNonQuery();
}
+
+ for(auto &tapeFile: archiveFile->tapeFiles){
+ //We deleted the TAPE_FILE so the tapes containing them should be set as dirty
+ setTapeDirty(conn,tapeFile.vid);
+ }
+
const auto deleteFromTapeFileTime = t.secs(utils::Timer::resetCounter);
{
@@ -533,7 +539,6 @@ void MysqlCatalogue::deleteArchiveFile(const std::string &diskInstanceName, cons
stmt.executeNonQuery();
}
const auto deleteFromArchiveFileTime = t.secs(utils::Timer::resetCounter);
-
conn.commit();
const auto commitTime = t.secs();
diff --git a/catalogue/OracleCatalogue.cpp b/catalogue/OracleCatalogue.cpp
index ec64025b2f..71cc86aaea 100644
--- a/catalogue/OracleCatalogue.cpp
+++ b/catalogue/OracleCatalogue.cpp
@@ -869,6 +869,12 @@ void OracleCatalogue::deleteArchiveFile(const std::string &diskInstanceName, con
stmt.bindUint64(":ARCHIVE_FILE_ID", archiveFileId);
stmt.executeNonQuery();
}
+
+ for(auto &tapeFile: archiveFile->tapeFiles){
+ //We deleted the TAPE_FILE so the tapes containing them should be set as dirty
+ setTapeDirty(conn,tapeFile.vid);
+ }
+
const auto deleteFromTapeFileTime = t.secs(utils::Timer::resetCounter);
{
diff --git a/catalogue/PostgresCatalogue.cpp b/catalogue/PostgresCatalogue.cpp
index f9d7bc4909..aaa4762d9b 100644
--- a/catalogue/PostgresCatalogue.cpp
+++ b/catalogue/PostgresCatalogue.cpp
@@ -808,6 +808,12 @@ void PostgresCatalogue::deleteArchiveFile(const std::string &diskInstanceName, c
stmt.bindUint64(":ARCHIVE_FILE_ID", archiveFileId);
stmt.executeNonQuery();
}
+
+ for(auto &tapeFile: archiveFile->tapeFiles){
+ //We deleted the TAPE_FILE so the tapes containing them should be set as dirty
+ setTapeDirty(conn,tapeFile.vid);
+ }
+
const auto deleteFromTapeFileTime = t.secs(utils::Timer::resetCounter);
{
diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp
index 9464f340ff..26c8f6aeaf 100644
--- a/catalogue/RdbmsCatalogue.cpp
+++ b/catalogue/RdbmsCatalogue.cpp
@@ -2361,11 +2361,14 @@ std::list<common::dataStructures::Tape> RdbmsCatalogue::getTapes(rdbms::Conn &co
"TAPE.ENCRYPTION_KEY_NAME AS ENCRYPTION_KEY_NAME,"
"TAPE.CAPACITY_IN_BYTES AS CAPACITY_IN_BYTES,"
"TAPE.DATA_IN_BYTES AS DATA_IN_BYTES,"
+ "TAPE.NB_MASTER_FILES AS NB_MASTER_FILES,"
+ "TAPE.MASTER_DATA_IN_BYTES AS MASTER_DATA_IN_BYTES,"
"TAPE.LAST_FSEQ AS LAST_FSEQ,"
"TAPE.IS_DISABLED AS IS_DISABLED,"
"TAPE.IS_FULL AS IS_FULL,"
"TAPE.IS_READ_ONLY AS IS_READ_ONLY,"
"TAPE.IS_FROM_CASTOR AS IS_FROM_CASTOR,"
+ "TAPE.DIRTY AS DIRTY,"
"TAPE.LABEL_DRIVE AS LABEL_DRIVE,"
"TAPE.LABEL_TIME AS LABEL_TIME,"
@@ -2517,12 +2520,15 @@ std::list<common::dataStructures::Tape> RdbmsCatalogue::getTapes(rdbms::Conn &co
tape.encryptionKeyName = rset.columnOptionalString("ENCRYPTION_KEY_NAME");
tape.capacityInBytes = rset.columnUint64("CAPACITY_IN_BYTES");
tape.dataOnTapeInBytes = rset.columnUint64("DATA_IN_BYTES");
+ tape.nbMasterFiles = rset.columnUint64("NB_MASTER_FILES");
+ tape.masterDataInBytes = rset.columnUint64("MASTER_DATA_IN_BYTES");
tape.lastFSeq = rset.columnUint64("LAST_FSEQ");
tape.disabled = rset.columnBool("IS_DISABLED");
tape.full = rset.columnBool("IS_FULL");
tape.readOnly = rset.columnBool("IS_READ_ONLY");
tape.isFromCastor = rset.columnBool("IS_FROM_CASTOR");
-
+ tape.dirty = rset.columnBool("DIRTY");
+
tape.labelLog = getTapeLogFromRset(rset, "LABEL_DRIVE", "LABEL_TIME");
tape.lastReadLog = getTapeLogFromRset(rset, "LAST_READ_DRIVE", "LAST_READ_TIME");
tape.lastWriteLog = getTapeLogFromRset(rset, "LAST_WRITE_DRIVE", "LAST_WRITE_TIME");
@@ -2857,6 +2863,20 @@ void RdbmsCatalogue::deleteTapeFiles(rdbms::Conn& conn, const std::string& vid)
}
}
+void RdbmsCatalogue::setTapeDirty(rdbms::Conn& conn, const std::string& vid) const {
+ try {
+ const char * const sql =
+ "UPDATE TAPE SET DIRTY='1' WHERE VID = :VID";
+ auto stmt = conn.createStmt(sql);
+ stmt.bindString(":VID", vid);
+ stmt.executeNonQuery();
+ } catch(exception::Exception &ex) {
+ ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+ throw;
+ }
+}
+
+
void RdbmsCatalogue::resetTapeCounters(rdbms::Conn& conn, const common::dataStructures::SecurityIdentity& admin, const std::string& vid) const {
try {
const time_t now = time(nullptr);
@@ -2867,7 +2887,8 @@ void RdbmsCatalogue::resetTapeCounters(rdbms::Conn& conn, const common::dataStru
"IS_FULL = '0',"
"LAST_UPDATE_USER_NAME = :LAST_UPDATE_USER_NAME,"
"LAST_UPDATE_HOST_NAME = :LAST_UPDATE_HOST_NAME,"
- "LAST_UPDATE_TIME = :LAST_UPDATE_TIME "
+ "LAST_UPDATE_TIME = :LAST_UPDATE_TIME,"
+ "DIRTY = '1' "
"WHERE "
"VID = :VID";
auto stmt = conn.createStmt(sql);
@@ -6112,7 +6133,8 @@ void RdbmsCatalogue::updateTape(
"LAST_FSEQ = :LAST_FSEQ,"
"DATA_IN_BYTES = DATA_IN_BYTES + :DATA_IN_BYTES,"
"LAST_WRITE_DRIVE = :LAST_WRITE_DRIVE,"
- "LAST_WRITE_TIME = :LAST_WRITE_TIME "
+ "LAST_WRITE_TIME = :LAST_WRITE_TIME,"
+ "DIRTY='1' "
"WHERE "
"VID = :VID";
auto stmt = conn.createStmt(sql);
diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp
index cd4863683d..961dc70833 100644
--- a/catalogue/RdbmsCatalogue.hpp
+++ b/catalogue/RdbmsCatalogue.hpp
@@ -425,6 +425,13 @@ public:
*/
void deleteTapeFiles(rdbms::Conn &conn, const std::string& vid) const;
+ /**
+ * Set the DIRTY flag to true
+ * @param conn the database connection
+ * @param vid the vid in which we want to set it as dirty
+ */
+ void setTapeDirty(rdbms::Conn &conn, const std::string &vid) const;
+
/**
* Reset the counters of a tape
* @param conn the database connection
diff --git a/catalogue/SqliteCatalogue.cpp b/catalogue/SqliteCatalogue.cpp
index 5432b89552..f468098f20 100644
--- a/catalogue/SqliteCatalogue.cpp
+++ b/catalogue/SqliteCatalogue.cpp
@@ -127,6 +127,12 @@ void SqliteCatalogue::deleteArchiveFile(const std::string &diskInstanceName, con
stmt.bindUint64(":ARCHIVE_FILE_ID", archiveFileId);
stmt.executeNonQuery();
}
+
+ for(auto &tapeFile: archiveFile->tapeFiles){
+ //We deleted the TAPE_FILE so the tapes containing them should be set as dirty
+ setTapeDirty(conn,tapeFile.vid);
+ }
+
const auto deleteFromTapeFileTime = t.secs(utils::Timer::resetCounter);
{
diff --git a/common/dataStructures/Tape.cpp b/common/dataStructures/Tape.cpp
index b8afb13c43..93dcb94293 100644
--- a/common/dataStructures/Tape.cpp
+++ b/common/dataStructures/Tape.cpp
@@ -31,9 +31,12 @@ Tape::Tape():
lastFSeq(0),
capacityInBytes(0),
dataOnTapeInBytes(0),
+ nbMasterFiles(0),
+ masterDataInBytes(0),
full(false),
disabled(false),
- readOnly(false) {}
+ readOnly(false)
+ {}
//------------------------------------------------------------------------------
// operator==
diff --git a/common/dataStructures/Tape.hpp b/common/dataStructures/Tape.hpp
index fadecc15bc..433df119aa 100644
--- a/common/dataStructures/Tape.hpp
+++ b/common/dataStructures/Tape.hpp
@@ -51,6 +51,8 @@ struct Tape {
std::string vo;
uint64_t capacityInBytes;
uint64_t dataOnTapeInBytes;
+ uint64_t nbMasterFiles;
+ uint64_t masterDataInBytes;
/**
* The optional name of the encryption key.
@@ -63,7 +65,8 @@ struct Tape {
bool full;
bool disabled;
bool readOnly;
- bool isFromCastor;
+ bool isFromCastor;
+ bool dirty;
uint64_t readMountCount;
uint64_t writeMountCount;
EntryLog creationLog;
diff --git a/xroot_plugins/XrdCtaTapeLs.hpp b/xroot_plugins/XrdCtaTapeLs.hpp
index f6455ee084..ecb8723586 100644
--- a/xroot_plugins/XrdCtaTapeLs.hpp
+++ b/xroot_plugins/XrdCtaTapeLs.hpp
@@ -114,7 +114,10 @@ int TapeLsStream::fillBuffer(XrdSsiPb::OStreamBuffer<Data> *streambuf) {
tape_item->set_from_castor(tape.isFromCastor);
tape_item->set_read_mount_count(tape.readMountCount);
tape_item->set_write_mount_count(tape.writeMountCount);
-
+ tape_item->set_nb_master_files(tape.nbMasterFiles);
+ tape_item->set_master_data_in_bytes(tape.masterDataInBytes);
+ tape_item->set_dirty(tape.dirty);
+
if(tape.labelLog) {
::cta::common::TapeLog * labelLog = tape_item->mutable_label_log();
labelLog->set_drive(tape.labelLog.value().drive);
--
GitLab