From 4434a6f8bb6762a510fcf4b6822d2b35855e4a47 Mon Sep 17 00:00:00 2001
From: Cedric Caffy <cedric.caffy@cern.ch>
Date: Fri, 26 Feb 2021 09:36:27 +0100
Subject: [PATCH] [checksumBlob] The first checksum information in a
checksumBlob can be added to log parameters
---
catalogue/MysqlCatalogue.cpp | 6 ++----
catalogue/OracleCatalogue.cpp | 4 ++--
catalogue/PostgresCatalogue.cpp | 4 ++--
catalogue/RdbmsCatalogue.cpp | 4 ++--
catalogue/SqliteCatalogue.cpp | 4 ++--
common/checksum/ChecksumBlob.cpp | 11 +++++++++++
common/checksum/ChecksumBlob.hpp | 8 ++++++++
scheduler/Scheduler.cpp | 4 ++--
.../castor/tape/tapeserver/daemon/DiskReadTask.cpp | 9 +++++----
.../tape/tapeserver/daemon/MigrationReportPacker.cpp | 5 ++++-
10 files changed, 40 insertions(+), 19 deletions(-)
diff --git a/catalogue/MysqlCatalogue.cpp b/catalogue/MysqlCatalogue.cpp
index c5bc490ed4..fb2095ef8f 100644
--- a/catalogue/MysqlCatalogue.cpp
+++ b/catalogue/MysqlCatalogue.cpp
@@ -629,7 +629,6 @@ void MysqlCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string &
.add("diskFileInfo.owner_uid", archiveFile->diskFileInfo.owner_uid)
.add("diskFileInfo.gid", archiveFile->diskFileInfo.gid)
.add("fileSize", std::to_string(archiveFile->fileSize))
- .add("checksumBlob", archiveFile->checksumBlob)
.add("creationTime", std::to_string(archiveFile->creationTime))
.add("reconciliationTime", std::to_string(archiveFile->reconciliationTime))
.add("storageClass", archiveFile->storageClass)
@@ -644,9 +643,9 @@ void MysqlCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string &
<< " blockId: " << it->blockId
<< " creationTime: " << it->creationTime
<< " fileSize: " << it->fileSize
- << " checksumBlob: " << it->checksumBlob //this shouldn't be here: repeated field
<< " copyNb: " << it->copyNb;
spc.add("TAPE FILE", tapeCopyLogStream.str());
+ archiveFile->checksumBlob.addFirstChecksumToLog(spc);
}
lc.log(log::WARNING, "Failed to delete archive file because the disk instance of the request does not match that "
"of the archived file");
@@ -694,7 +693,6 @@ void MysqlCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string &
.add("diskFileInfo.owner_uid", archiveFile->diskFileInfo.owner_uid)
.add("diskFileInfo.gid", archiveFile->diskFileInfo.gid)
.add("fileSize", std::to_string(archiveFile->fileSize))
- .add("checksumBlob", archiveFile->checksumBlob)
.add("creationTime", std::to_string(archiveFile->creationTime))
.add("reconciliationTime", std::to_string(archiveFile->reconciliationTime))
.add("storageClass", archiveFile->storageClass)
@@ -705,6 +703,7 @@ void MysqlCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string &
.add("deleteFromArchiveFileTime", deleteFromArchiveFileTime)
.add("setTapeDirtyTime",setTapeDirtyTime)
.add("commitTime", commitTime);
+ archiveFile->checksumBlob.addFirstChecksumToLog(spc);
for(auto it=archiveFile->tapeFiles.begin(); it!=archiveFile->tapeFiles.end(); it++) {
std::stringstream tapeCopyLogStream;
tapeCopyLogStream << "copy number: " << it->copyNb
@@ -713,7 +712,6 @@ void MysqlCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string &
<< " blockId: " << it->blockId
<< " creationTime: " << it->creationTime
<< " fileSize: " << it->fileSize
- << " checksumBlob: " << it->checksumBlob //this shouldn't be here: repeated field
<< " copyNb: " << static_cast<int>(it->copyNb); //this shouldn't be here: repeated field
spc.add("TAPE FILE", tapeCopyLogStream.str());
}
diff --git a/catalogue/OracleCatalogue.cpp b/catalogue/OracleCatalogue.cpp
index 30915eedb1..a0679a3203 100644
--- a/catalogue/OracleCatalogue.cpp
+++ b/catalogue/OracleCatalogue.cpp
@@ -956,13 +956,13 @@ void OracleCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string
.add("diskFileInfo.owner_uid", archiveFile->diskFileInfo.owner_uid)
.add("diskFileInfo.gid", archiveFile->diskFileInfo.gid)
.add("fileSize", std::to_string(archiveFile->fileSize))
- .add("checksumBlob", archiveFile->checksumBlob)
.add("creationTime", std::to_string(archiveFile->creationTime))
.add("reconciliationTime", std::to_string(archiveFile->reconciliationTime))
.add("storageClass", archiveFile->storageClass)
.add("getConnTime", getConnTime)
.add("createStmtTime", createStmtTime)
.add("selectFromArchiveFileTime", selectFromArchiveFileTime);
+ archiveFile->checksumBlob.addFirstChecksumToLog(spc);
for(auto it=archiveFile->tapeFiles.begin(); it!=archiveFile->tapeFiles.end(); it++) {
std::stringstream tapeCopyLogStream;
tapeCopyLogStream << "copy number: " << static_cast<int>(it->copyNb)
@@ -1020,7 +1020,6 @@ void OracleCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string
.add("diskFileInfo.owner_uid", archiveFile->diskFileInfo.owner_uid)
.add("diskFileInfo.gid", archiveFile->diskFileInfo.gid)
.add("fileSize", std::to_string(archiveFile->fileSize))
- .add("checksumBlob", archiveFile->checksumBlob)
.add("creationTime", std::to_string(archiveFile->creationTime))
.add("reconciliationTime", std::to_string(archiveFile->reconciliationTime))
.add("storageClass", archiveFile->storageClass)
@@ -1031,6 +1030,7 @@ void OracleCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string
.add("setTapeDirtyTime",setTapeDirtyTime)
.add("deleteFromArchiveFileTime", deleteFromArchiveFileTime)
.add("commitTime", commitTime);
+ archiveFile->checksumBlob.addFirstChecksumToLog(spc);
for(auto it=archiveFile->tapeFiles.begin(); it!=archiveFile->tapeFiles.end(); it++) {
std::stringstream tapeCopyLogStream;
tapeCopyLogStream << "copy number: " << it->copyNb
diff --git a/catalogue/PostgresCatalogue.cpp b/catalogue/PostgresCatalogue.cpp
index e179a58c6b..449dbc9f6a 100644
--- a/catalogue/PostgresCatalogue.cpp
+++ b/catalogue/PostgresCatalogue.cpp
@@ -900,13 +900,13 @@ void PostgresCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::strin
.add("diskFileInfo.owner_uid", archiveFile->diskFileInfo.owner_uid)
.add("diskFileInfo.gid", archiveFile->diskFileInfo.gid)
.add("fileSize", std::to_string(archiveFile->fileSize))
- .add("checksumBlob", archiveFile->checksumBlob)
.add("creationTime", std::to_string(archiveFile->creationTime))
.add("reconciliationTime", std::to_string(archiveFile->reconciliationTime))
.add("storageClass", archiveFile->storageClass)
.add("getConnTime", getConnTime)
.add("createStmtTime", createStmtTime)
.add("selectFromArchiveFileTime", selectFromArchiveFileTime);
+ archiveFile->checksumBlob.addFirstChecksumToLog(spc);
for(auto it=archiveFile->tapeFiles.begin(); it!=archiveFile->tapeFiles.end(); it++) {
std::stringstream tapeCopyLogStream;
tapeCopyLogStream << "copy number: " << it->copyNb
@@ -965,7 +965,6 @@ void PostgresCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::strin
.add("diskFileInfo.owner_uid", archiveFile->diskFileInfo.owner_uid)
.add("diskFileInfo.gid", archiveFile->diskFileInfo.gid)
.add("fileSize", std::to_string(archiveFile->fileSize))
- .add("checksumBlob", archiveFile->checksumBlob)
.add("creationTime", std::to_string(archiveFile->creationTime))
.add("reconciliationTime", std::to_string(archiveFile->reconciliationTime))
.add("storageClass", archiveFile->storageClass)
@@ -976,6 +975,7 @@ void PostgresCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::strin
.add("deleteFromArchiveFileTime", deleteFromArchiveFileTime)
.add("setTapeDirtyTime",setTapeDirtyTime)
.add("commitTime", commitTime);
+ archiveFile->checksumBlob.addFirstChecksumToLog(spc);
for(auto it=archiveFile->tapeFiles.begin(); it!=archiveFile->tapeFiles.end(); it++) {
std::stringstream tapeCopyLogStream;
tapeCopyLogStream << "copy number: " << it->copyNb
diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp
index 903451754f..2af09a8a7d 100644
--- a/catalogue/RdbmsCatalogue.cpp
+++ b/catalogue/RdbmsCatalogue.cpp
@@ -8618,12 +8618,12 @@ void RdbmsCatalogue::moveArchiveFileToRecycleLog(const common::dataStructures::D
.add("diskFileInfo.owner_uid", archiveFile.diskFileInfo.owner_uid)
.add("diskFileInfo.gid", archiveFile.diskFileInfo.gid)
.add("fileSize", std::to_string(archiveFile.fileSize))
- .add("checksumBlob", archiveFile.checksumBlob)
.add("creationTime", std::to_string(archiveFile.creationTime))
.add("reconciliationTime", std::to_string(archiveFile.reconciliationTime))
.add("diskFilePath",request.diskFilePath)
.add("errorMessage",ex.getMessageValue())
.add("storageClass", archiveFile.storageClass);
+ archiveFile.checksumBlob.addFirstChecksumToLog(spc);
for(auto it=archiveFile.tapeFiles.begin(); it!=archiveFile.tapeFiles.end(); it++) {
std::stringstream tapeCopyLogStream;
tapeCopyLogStream << "copy number: " << static_cast<int>(it->copyNb)
@@ -8656,10 +8656,10 @@ void RdbmsCatalogue::moveArchiveFileToRecycleLog(const common::dataStructures::D
.add("diskFileInfo.owner_uid", archiveFile.diskFileInfo.owner_uid)
.add("diskFileInfo.gid", archiveFile.diskFileInfo.gid)
.add("fileSize", std::to_string(archiveFile.fileSize))
- .add("checksumBlob", archiveFile.checksumBlob)
.add("creationTime", std::to_string(archiveFile.creationTime))
.add("reconciliationTime", std::to_string(archiveFile.reconciliationTime))
.add("storageClass", archiveFile.storageClass);
+ archiveFile.checksumBlob.addFirstChecksumToLog(spc);
for(auto it=archiveFile.tapeFiles.begin(); it!=archiveFile.tapeFiles.end(); it++) {
std::stringstream tapeCopyLogStream;
tapeCopyLogStream << "copy number: " << static_cast<int>(it->copyNb)
diff --git a/catalogue/SqliteCatalogue.cpp b/catalogue/SqliteCatalogue.cpp
index 365232ffa8..9120b2a3d5 100644
--- a/catalogue/SqliteCatalogue.cpp
+++ b/catalogue/SqliteCatalogue.cpp
@@ -85,12 +85,12 @@ void SqliteCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string
.add("diskFileInfo.owner_uid", archiveFile->diskFileInfo.owner_uid)
.add("diskFileInfo.gid", archiveFile->diskFileInfo.gid)
.add("fileSize", std::to_string(archiveFile->fileSize))
- .add("checksumBlob", archiveFile->checksumBlob)
.add("creationTime", std::to_string(archiveFile->creationTime))
.add("reconciliationTime", std::to_string(archiveFile->reconciliationTime))
.add("storageClass", archiveFile->storageClass)
.add("getConnTime", getConnTime)
.add("getArchiveFileTime", getArchiveFileTime);
+ archiveFile->checksumBlob.addFirstChecksumToLog(spc);
for(auto it=archiveFile->tapeFiles.begin(); it!=archiveFile->tapeFiles.end(); it++) {
std::stringstream tapeCopyLogStream;
tapeCopyLogStream << "copy number: " << it->copyNb
@@ -159,7 +159,6 @@ void SqliteCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string
.add("diskFileInfo.owner_uid", archiveFile->diskFileInfo.owner_uid)
.add("diskFileInfo.gid", archiveFile->diskFileInfo.gid)
.add("fileSize", std::to_string(archiveFile->fileSize))
- .add("checksumBlob", archiveFile->checksumBlob)
.add("creationTime", std::to_string(archiveFile->creationTime))
.add("reconciliationTime", std::to_string(archiveFile->reconciliationTime))
.add("storageClass", archiveFile->storageClass)
@@ -169,6 +168,7 @@ void SqliteCatalogue::DO_NOT_USE_deleteArchiveFile_DO_NOT_USE(const std::string
.add("deleteFromArchiveFileTime", deleteFromArchiveFileTime)
.add("setTapeDirtyTime",setTapeDirtyTime)
.add("commitTime", commitTime);
+ archiveFile->checksumBlob.addFirstChecksumToLog(spc);
for(auto it=archiveFile->tapeFiles.begin(); it!=archiveFile->tapeFiles.end(); it++) {
std::stringstream tapeCopyLogStream;
tapeCopyLogStream << "copy number: " << it->copyNb
diff --git a/common/checksum/ChecksumBlob.cpp b/common/checksum/ChecksumBlob.cpp
index 507243fefc..68583c6dc3 100644
--- a/common/checksum/ChecksumBlob.cpp
+++ b/common/checksum/ChecksumBlob.cpp
@@ -150,6 +150,17 @@ std::string ChecksumBlob::ByteArrayToHex(const std::string &bytearray) {
return value.str();
}
+void ChecksumBlob::addFirstChecksumToLog(cta::log::ScopedParamContainer &spc) const{
+ const auto & csItor = m_cs.begin();
+ if(csItor != m_cs.end()){
+ auto & cs = *csItor;
+ std::string checksumTypeParam = "checksumType";
+ std::string checksumValueParam = "checksumValue";
+ spc.add(checksumTypeParam,ChecksumTypeName.at(cs.first))
+ .add(checksumValueParam,ByteArrayToHex(cs.second));
+ }
+}
+
std::ostream &operator<<(std::ostream &os, const ChecksumBlob &csb) {
os << "[ ";
auto num_els = csb.m_cs.size();
diff --git a/common/checksum/ChecksumBlob.hpp b/common/checksum/ChecksumBlob.hpp
index 12c4b4d26d..adc404779f 100644
--- a/common/checksum/ChecksumBlob.hpp
+++ b/common/checksum/ChecksumBlob.hpp
@@ -23,6 +23,7 @@
#include <common/exception/ChecksumBlobSizeMismatch.hpp>
#include <common/exception/ChecksumTypeMismatch.hpp>
#include <common/exception/ChecksumValueMismatch.hpp>
+#include "common/log/LogContext.hpp"
namespace cta {
namespace checksum {
@@ -189,6 +190,13 @@ public:
}
return true;
}
+
+ /**
+ * Adds the first checksum of the list of checksums
+ * in the log params container given in parameter
+ * @param spc the log param container to put the first checksum in
+ */
+ void addFirstChecksumToLog(cta::log::ScopedParamContainer &spc) const;
/*!
* Returns false if all the checksums in the blob match
diff --git a/scheduler/Scheduler.cpp b/scheduler/Scheduler.cpp
index 95555ea3ca..680d3c8751 100644
--- a/scheduler/Scheduler.cpp
+++ b/scheduler/Scheduler.cpp
@@ -164,7 +164,6 @@ std::string Scheduler::queueArchiveWithGivenId(const uint64_t archiveFileId, con
.add("diskFilePath", request.diskFileInfo.path)
.add("diskFileOwnerUid", request.diskFileInfo.owner_uid)
.add("diskFileGid", request.diskFileInfo.gid)
- .add("checksumBlob", request.checksumBlob)
.add("archiveReportURL", midEllipsis(request.archiveReportURL, 50, 15))
.add("archiveErrorReportURL", midEllipsis(request.archiveErrorReportURL, 50, 15))
.add("creationHost", request.creationLog.host)
@@ -175,6 +174,7 @@ std::string Scheduler::queueArchiveWithGivenId(const uint64_t archiveFileId, con
.add("srcURL", midEllipsis(request.srcURL, 50, 15))
.add("catalogueTime", catalogueTime)
.add("schedulerDbTime", schedulerDbTime);
+ request.checksumBlob.addFirstChecksumToLog(spc);
lc.log(log::INFO, "Queued archive request");
return archiveReqAddr;
}
@@ -227,7 +227,6 @@ std::string Scheduler::queueRetrieve(
.add("requesterName", request.requester.name)
.add("requesterGroup", request.requester.group)
.add("criteriaArchiveFileId", queueCriteria.archiveFile.archiveFileID)
- .add("criteriaChecksumBlob", queueCriteria.archiveFile.checksumBlob)
.add("criteriaCreationTime", queueCriteria.archiveFile.creationTime)
.add("criteriaDiskFileId", queueCriteria.archiveFile.diskFileId)
.add("criteriaDiskFileOwnerUid", queueCriteria.archiveFile.diskFileInfo.owner_uid)
@@ -235,6 +234,7 @@ std::string Scheduler::queueRetrieve(
.add("criteriaFileSize", queueCriteria.archiveFile.fileSize)
.add("reconciliationTime", queueCriteria.archiveFile.reconciliationTime)
.add("storageClass", queueCriteria.archiveFile.storageClass);
+ queueCriteria.archiveFile.checksumBlob.addFirstChecksumToLog(spc);
uint32_t copyNumber=0;
for (auto & tf:queueCriteria.archiveFile.tapeFiles) {
std::stringstream tc;
diff --git a/tapeserver/castor/tape/tapeserver/daemon/DiskReadTask.cpp b/tapeserver/castor/tape/tapeserver/daemon/DiskReadTask.cpp
index eb1399d028..61b5e0143e 100644
--- a/tapeserver/castor/tape/tapeserver/daemon/DiskReadTask.cpp
+++ b/tapeserver/castor/tape/tapeserver/daemon/DiskReadTask.cpp
@@ -165,10 +165,11 @@ void DiskReadTask::execute(cta::log::LogContext& lc, cta::disk::DiskFileFactory
m_nextTask.pushDataBlock(mb);
mb=NULL;
- LogContext::ScopedParam sp(lc, Param("blockID",blockId));
- LogContext::ScopedParam sp0(lc, Param("exceptionMessage", e.getMessageValue()));
- LogContext::ScopedParam sp1(lc,Param("fileSize",m_archiveJob->archiveFile.fileSize));
- LogContext::ScopedParam sp2(lc,Param("checksumBlob",m_archiveJob->archiveFile.checksumBlob));
+ cta::log::ScopedParamContainer spc(lc);
+ spc.add("blockID",blockId)
+ .add("exceptionMessage", e.getMessageValue())
+ .add("fileSize",m_archiveJob->archiveFile.fileSize);
+ m_archiveJob->archiveFile.checksumBlob.addFirstChecksumToLog(spc);
lc.log(cta::log::ERR,"Exception while reading a file");
//deal here the number of mem block
diff --git a/tapeserver/castor/tape/tapeserver/daemon/MigrationReportPacker.cpp b/tapeserver/castor/tape/tapeserver/daemon/MigrationReportPacker.cpp
index 2c7bba9c61..7424d5e902 100644
--- a/tapeserver/castor/tape/tapeserver/daemon/MigrationReportPacker.cpp
+++ b/tapeserver/castor/tape/tapeserver/daemon/MigrationReportPacker.cpp
@@ -198,8 +198,11 @@ void MigrationReportPacker::ReportSkipped::execute(MigrationReportPacker& report
{
cta::log::ScopedParamContainer params(reportPacker.m_lc);
params.add("failureLog", m_failureLog)
+ .add("fileSize",m_skippedArchiveJob->archiveFile.fileSize)
.add("fileId", m_skippedArchiveJob->archiveFile.archiveFileID);
- reportPacker.m_lc.log(cta::log::ERR,"In MigrationReportPacker::ReportSkipped::execute(): skipping archive job after exception.");
+ m_skippedArchiveJob->archiveFile.checksumBlob.addFirstChecksumToLog(params);
+
+ reportPacker.m_lc.log(cta::log::DEBUG,"In MigrationReportPacker::ReportSkipped::execute(): skipping archive job after exception.");
}
try {
m_skippedArchiveJob->transferFailed(m_failureLog, reportPacker.m_lc);
--
GitLab