Commit c717a874 authored by Cedric CAFFY's avatar Cedric CAFFY
Browse files

cta-statistics-update tool is now working

parent f74372f7
......@@ -464,6 +464,9 @@ public:
virtual void setTapeIsFromCastorInUnitTests(const std::string &vid) = 0;
virtual void setTapeDisabled(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const bool disabledValue) = 0;
virtual void setTapeDirty(const std::string & vid) = 0;
virtual void modifyTapeComment(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const std::string &comment) = 0;
virtual void modifyRequesterMountRulePolicy(const common::dataStructures::SecurityIdentity &admin, const std::string &instanceName, const std::string &requesterName, const std::string &mountPolicy) = 0;
......
......@@ -299,6 +299,10 @@ public:
void setTapeDisabled(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const bool disabledValue) override {
return retryOnLostConnection(m_log, [&]{return m_catalogue->setTapeDisabled(admin, vid, disabledValue);}, m_maxTriesToConnect);
}
void setTapeDirty(const std::string & vid) override {
return retryOnLostConnection(m_log,[&]{ return m_catalogue->setTapeDirty(vid);}, m_maxTriesToConnect);
}
void modifyTapeComment(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const std::string &comment) override {
return retryOnLostConnection(m_log, [&]{return m_catalogue->modifyTapeComment(admin, vid, comment);}, m_maxTriesToConnect);
......
......@@ -130,6 +130,7 @@ public:
uint64_t getNbFilesOnTape(const std::string& vid) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void setTapeDisabled(const common::dataStructures::SecurityIdentity& admin, const std::string& vid, const bool disabledValue) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void setTapeFull(const common::dataStructures::SecurityIdentity& admin, const std::string& vid, const bool fullValue) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void setTapeDirty(const std::string & vid) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void setTapeReadOnly(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const bool readOnlyValue) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void setTapeReadOnlyOnError(const std::string &vid) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void setTapeIsFromCastorInUnitTests(const std::string &vid) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
......
......@@ -428,6 +428,7 @@ void MysqlCatalogue::deleteArchiveFile(const std::string &diskInstanceName, cons
rdbms::Rset selectRset = selectStmt.executeQuery();
const auto selectFromArchiveFileTime = t.secs();
std::unique_ptr<common::dataStructures::ArchiveFile> archiveFile;
std::set<std::string> vidsToSetDirty;
while(selectRset.next()) {
if(nullptr == archiveFile.get()) {
archiveFile = cta::make_unique<common::dataStructures::ArchiveFile>();
......@@ -450,6 +451,7 @@ void MysqlCatalogue::deleteArchiveFile(const std::string &diskInstanceName, cons
// Add the tape file to the archive file's in-memory structure
common::dataStructures::TapeFile tapeFile;
tapeFile.vid = selectRset.columnString("VID");
vidsToSetDirty.insert(tapeFile.vid);
tapeFile.fSeq = selectRset.columnUint64("FSEQ");
tapeFile.blockId = selectRset.columnUint64("BLOCK_ID");
tapeFile.fileSize = selectRset.columnUint64("LOGICAL_SIZE_IN_BYTES");
......@@ -525,9 +527,9 @@ void MysqlCatalogue::deleteArchiveFile(const std::string &diskInstanceName, cons
stmt.executeNonQuery();
}
for(auto &tapeFile: archiveFile->tapeFiles){
for(auto &vidToSetDirty: vidsToSetDirty){
//We deleted the TAPE_FILE so the tapes containing them should be set as dirty
setTapeDirty(conn,tapeFile.vid);
setTapeDirty(conn,vidToSetDirty);
}
const auto deleteFromTapeFileTime = t.secs(utils::Timer::resetCounter);
......
......@@ -776,6 +776,7 @@ void OracleCatalogue::deleteArchiveFile(const std::string &diskInstanceName, con
rdbms::Rset selectRset = selectStmt.executeQuery();
const auto selectFromArchiveFileTime = t.secs();
std::unique_ptr<common::dataStructures::ArchiveFile> archiveFile;
std::set<std::string> vidsToSetDirty;
while(selectRset.next()) {
if(nullptr == archiveFile.get()) {
archiveFile = cta::make_unique<common::dataStructures::ArchiveFile>();
......@@ -798,6 +799,7 @@ void OracleCatalogue::deleteArchiveFile(const std::string &diskInstanceName, con
// Add the tape file to the archive file's in-memory structure
common::dataStructures::TapeFile tapeFile;
tapeFile.vid = selectRset.columnString("VID");
vidsToSetDirty.insert(tapeFile.vid);
tapeFile.fSeq = selectRset.columnUint64("FSEQ");
tapeFile.blockId = selectRset.columnUint64("BLOCK_ID");
tapeFile.fileSize = selectRset.columnUint64("LOGICAL_SIZE_IN_BYTES");
......@@ -870,9 +872,9 @@ void OracleCatalogue::deleteArchiveFile(const std::string &diskInstanceName, con
stmt.executeNonQuery();
}
for(auto &tapeFile: archiveFile->tapeFiles){
//We deleted the TAPE_FILE so the tapes containing them should be set as dirty
setTapeDirty(conn,tapeFile.vid);
//Set the tapes where the files have been deleted to dirty
for(auto &vidToSetDirty: vidsToSetDirty){
setTapeDirty(conn,vidToSetDirty);
}
const auto deleteFromTapeFileTime = t.secs(utils::Timer::resetCounter);
......
......@@ -714,6 +714,7 @@ void PostgresCatalogue::deleteArchiveFile(const std::string &diskInstanceName, c
rdbms::Rset selectRset = selectStmt.executeQuery();
const auto selectFromArchiveFileTime = t.secs();
std::unique_ptr<common::dataStructures::ArchiveFile> archiveFile;
std::set<std::string> vidsToSetDirty;
while(selectRset.next()) {
if(nullptr == archiveFile.get()) {
archiveFile = cta::make_unique<common::dataStructures::ArchiveFile>();
......@@ -736,6 +737,7 @@ void PostgresCatalogue::deleteArchiveFile(const std::string &diskInstanceName, c
// Add the tape file to the archive file's in-memory structure
common::dataStructures::TapeFile tapeFile;
tapeFile.vid = selectRset.columnString("VID");
vidsToSetDirty.insert(tapeFile.vid);
tapeFile.fSeq = selectRset.columnUint64("FSEQ");
tapeFile.blockId = selectRset.columnUint64("BLOCK_ID");
tapeFile.fileSize = selectRset.columnUint64("LOGICAL_SIZE_IN_BYTES");
......@@ -809,9 +811,9 @@ void PostgresCatalogue::deleteArchiveFile(const std::string &diskInstanceName, c
stmt.executeNonQuery();
}
for(auto &tapeFile: archiveFile->tapeFiles){
for(auto &vidToSetDirty: vidsToSetDirty){
//We deleted the TAPE_FILE so the tapes containing them should be set as dirty
setTapeDirty(conn,tapeFile.vid);
setTapeDirty(conn,vidToSetDirty);
}
const auto deleteFromTapeFileTime = t.secs(utils::Timer::resetCounter);
......
......@@ -3588,6 +3588,17 @@ void RdbmsCatalogue::setTapeDisabled(const common::dataStructures::SecurityIdent
}
}
void RdbmsCatalogue::setTapeDirty(const std::string& vid) {
try {
auto conn = m_connPool.getConn();
setTapeDirty(conn,vid);
} catch(exception::Exception &ex) {
ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
throw;
}
}
//------------------------------------------------------------------------------
// modifyTapeComment
//------------------------------------------------------------------------------
......
......@@ -487,6 +487,7 @@ public:
void setTapeIsFromCastorInUnitTests(const std::string &vid) override;
void setTapeDisabled(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const bool disabledValue) override;
void setTapeDirty(const std::string & vid) override;
void modifyTapeComment(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const std::string &comment) override;
void modifyRequesterMountRulePolicy(const common::dataStructures::SecurityIdentity &admin, const std::string &instanceName, const std::string &requesterName, const std::string &mountPolicy) override;
......
......@@ -128,9 +128,15 @@ void SqliteCatalogue::deleteArchiveFile(const std::string &diskInstanceName, con
stmt.executeNonQuery();
}
std::set<std::string> vidsToSetDirty;
//We will insert the vids to set dirty in a set so that
//we limit the calls to setTapeDirty to the number of tapes that contained the deleted tape files
for(auto &tapeFile: archiveFile->tapeFiles){
//We deleted the TAPE_FILE so the tapes containing them should be set as dirty
setTapeDirty(conn,tapeFile.vid);
vidsToSetDirty.insert(tapeFile.vid);
}
for(auto &vidToSetDirty: vidsToSetDirty){
setTapeDirty(conn,vidToSetDirty);
}
const auto deleteFromTapeFileTime = t.secs(utils::Timer::resetCounter);
......
......@@ -208,7 +208,7 @@ void cta::ArchiveMount::reportJobsBatchTransferred(std::queue<std::unique_ptr<ct
validatedSuccessfulDBArchiveJobs.emplace_back(std::move(schJob->m_dbJob));
}
validatedSuccessfulArchiveJobs.clear();
updateCatalogueWithTapeFilesWritten(tapeItemsWritten);
catalogueTime=t.secs(utils::Timer::resetCounter);
{
......
......@@ -4645,6 +4645,9 @@ objectstore::ArchiveRequest::RepackInfo OStoreDB::ArchiveJob::getRepackInfoAfter
// OStoreDB::RepackArchiveSuccessesReportBatch::report()
//------------------------------------------------------------------------------
void OStoreDB::RepackArchiveSuccessesReportBatch::report(log::LogContext& lc) {
objectstore::RepackRequest req(m_repackRequest.getAddressIfSet(),this->m_oStoreDb.m_objectStore);
req.fetchNoLock();
this->m_oStoreDb.m_catalogue.setTapeDirty(req.getInfo().vid);
OStoreDB::RepackArchiveReportBatch::report(lc);
}
......
......@@ -24,6 +24,7 @@ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow")
set (STATISTICS_LIB_SRC_FILES
StatisticsSchema.cpp
TapeStatisticsUpdater.cpp
)
add_library (ctastatistics SHARED
......
......@@ -21,6 +21,8 @@
#include "statistics/StatisticsUpdateCmd.hpp"
#include "catalogue/SchemaChecker.hpp"
#include "statistics/StatisticsUpdateCmdLineArgs.hpp"
#include "common/Timer.hpp"
#include "TapeStatisticsUpdater.hpp"
namespace cta {
namespace statistics {
......@@ -55,10 +57,6 @@ int StatisticsUpdateCmd::exceptionThrowingMain(const int argc, char *const *cons
auto loginCatalogue = rdbms::Login::parseFile(cmdLineArgs.catalogueDbConfigPath);
rdbms::ConnPool catalogueConnPool(loginCatalogue, maxNbConns);
auto catalogueConn = catalogueConnPool.getConn();
/*auto loginStatistics = rdbms::Login::parseFile(cmdLineArgs.statisticsDbConfigPath);
rdbms::ConnPool statisticsConnPool(loginStatistics, maxNbConns);
auto statisticsConn = statisticsConnPool.getConn();*/
SchemaChecker::Builder catalogueCheckerBuilder("catalogue",loginCatalogue.dbType,catalogueConn);
std::unique_ptr<cta::catalogue::SchemaChecker> catalogueChecker;
......@@ -71,14 +69,12 @@ int StatisticsUpdateCmd::exceptionThrowingMain(const int argc, char *const *cons
if(tapeTableStatus == SchemaChecker::Status::FAILURE || tapeFileTableStatus == SchemaChecker::Status::FAILURE || archiveFileTableStatus == SchemaChecker::Status::FAILURE){
return EXIT_FAILURE;
}
/*SchemaChecker::Builder statisticsCheckerBuilder("statistics",loginStatistics.dbType,statisticsConn);
cta::statistics::MysqlStatisticsSchema mysqlSchema;
std::unique_ptr<SchemaChecker> statisticsChecker =
statisticsCheckerBuilder.useCppSchemaStatementsReader(mysqlSchema)
.useSQLiteSchemaComparer()
.build();
statisticsChecker->compareTablesLocatedInSchema();*/
std::unique_ptr<TapeStatisticsUpdater> updater = TapeStatisticsUpdaterFactory::create(loginCatalogue.dbType,catalogueConn);
std::cout<<"Updating tape statistics in the catalogue..."<<std::endl;
cta::utils::Timer t;
updater->updateTapeStatistics();
std::cout<<"Updated catalogue tape statistics in "<<t.secs()<<", "<<updater->getNbUpdatedTapes()<<" tape(s) have been updated"<<std::endl;
return EXIT_SUCCESS;
}
......
/**
* The CERN Tape Archive (CTA) project
* Copyright © 2018 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "TapeStatisticsUpdater.hpp"
#include "rdbms/Login.hpp"
namespace cta { namespace statistics {
TapeStatisticsUpdater::TapeStatisticsUpdater(cta::rdbms::Conn &conn):m_conn(conn) {
}
TapeStatisticsUpdater::~TapeStatisticsUpdater() {
}
void TapeStatisticsUpdater::updateTapeStatistics() {
const char * const sql =
"UPDATE TAPE TAPE_TO_UPDATE SET"
"("
"DIRTY,"
"NB_MASTER_FILES,"
"MASTER_DATA_IN_BYTES"
")"
" = "
"("
"SELECT "
"'0' AS DIRTY,"
"COALESCE(NON_EMPTY_TAPE_STATS.NB_MASTER_FILES, 0) AS NB_MASTER_FILES,"
"COALESCE(NON_EMPTY_TAPE_STATS.MASTER_DATA_IN_BYTES, 0) AS MASTER_DATA_IN_BYTES "
"FROM "
"TAPE T "
"LEFT OUTER JOIN "
"("
"SELECT "
"TAPE_FILE.VID,"
"'0' AS DIRTY,"
"COUNT(TAPE_FILE.FSEQ) AS NB_MASTER_FILES,"
"SUM(ARCHIVE_FILE.SIZE_IN_BYTES) AS MASTER_DATA_IN_BYTES "
"FROM "
"TAPE_FILE "
"INNER JOIN ARCHIVE_FILE ON "
"TAPE_FILE.ARCHIVE_FILE_ID = ARCHIVE_FILE.ARCHIVE_FILE_ID "
"WHERE "
"TAPE_FILE.SUPERSEDED_BY_VID IS NULL AND "
"TAPE_FILE.SUPERSEDED_BY_FSEQ IS NULL "
"GROUP BY TAPE_FILE.VID"
") NON_EMPTY_TAPE_STATS ON "
"T.VID = NON_EMPTY_TAPE_STATS.VID "
"WHERE T.VID = TAPE_TO_UPDATE.VID"
") "
"WHERE TAPE_TO_UPDATE.DIRTY='1'";
try {
auto stmt = m_conn.createStmt(sql);
stmt.executeNonQuery();
m_nbUpdatedTapes = stmt.getNbAffectedRows();
} catch(cta::exception::Exception &ex) {
ex.getMessage().str(std::string(__PRETTY_FUNCTION__) + ": " + ex.getMessage().str());
throw;
}
}
uint64_t TapeStatisticsUpdater::getNbUpdatedTapes() {
return m_nbUpdatedTapes;
}
std::unique_ptr<TapeStatisticsUpdater> TapeStatisticsUpdaterFactory::create(cta::rdbms::Login::DbType dbType, cta::rdbms::Conn& conn){
typedef cta::rdbms::Login::DbType DbType;
std::unique_ptr<TapeStatisticsUpdater> ret;
switch(dbType){
case DbType::DBTYPE_IN_MEMORY:
case DbType::DBTYPE_SQLITE:
case DbType::DBTYPE_MYSQL:
throw cta::exception::Exception("In TapeStatisticsUpdaterFactory::create(), the "+cta::rdbms::Login::dbTypeToString(dbType)+" database type is not supported.");
case DbType::DBTYPE_ORACLE:
case DbType::DBTYPE_POSTGRESQL:
ret.reset(new TapeStatisticsUpdater(conn));
return std::move(ret);
default:
throw cta::exception::Exception("In TapeStatisticsUpdaterFactory::create(), unknown database type.");
}
}
}}
/**
* The CERN Tape Archive (CTA) project
* Copyright © 2018 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "rdbms/Conn.hpp"
#include "rdbms/Login.hpp"
namespace cta { namespace statistics {
class TapeStatisticsUpdater {
public:
TapeStatisticsUpdater(cta::rdbms::Conn &conn);
virtual void updateTapeStatistics();
virtual uint64_t getNbUpdatedTapes();
virtual ~TapeStatisticsUpdater();
protected:
cta::rdbms::Conn &m_conn;
uint64_t m_nbUpdatedTapes = 0;
};
//TODO if necessary
class MySQLStatisticsUpdater: public TapeStatisticsUpdater {
public:
MySQLStatisticsUpdater(cta::rdbms::Conn &conn);
virtual ~MySQLStatisticsUpdater();
void updateTapeStatistics() override;
};
class TapeStatisticsUpdaterFactory {
public:
static std::unique_ptr<TapeStatisticsUpdater> create(cta::rdbms::Login::DbType dbType, cta::rdbms::Conn &conn);
};
}}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment