From b814798ffd60f36f754971f46f59e7217984c06d Mon Sep 17 00:00:00 2001
From: Lasse Tjernaes Wardenaer <lasse.tjernaes.wardenaer@cern.ch>
Date: Tue, 1 Nov 2022 13:34:07 +0100
Subject: [PATCH] Resolve "Allow to submit multiple files for verification"
---
ReleaseNotes.md | 1 +
.../standalone_cli_tools/CtaVerifyFile.cpp | 74 ++++++++++++-------
.../common/CmdLineArgs.cpp | 19 ++++-
common/utils/utils.cpp | 16 ++++
common/utils/utils.hpp | 7 ++
5 files changed, 86 insertions(+), 31 deletions(-)
diff --git a/ReleaseNotes.md b/ReleaseNotes.md
index 03f4349a61..e4d75e3b31 100644
--- a/ReleaseNotes.md
+++ b/ReleaseNotes.md
@@ -7,6 +7,7 @@
- cta/CTA#78 - Tool to update the storage class
- cta/CTA#153 - Allow verification status to be cleared with cta-admin
- cta/CTA#173 - Update release notes and small changes to refactoring of operation tools cmd line parsing - Compatible with operations 0.4-95 or later
+- cta/CTA#180 - Allow to submit multiple files for verification
### Continuous Integration
- cta/CTA#118 - Add unit tests for OSM label
- cta/CTA#191 - Block merge until cta_valgrind success
diff --git a/cmdline/standalone_cli_tools/CtaVerifyFile.cpp b/cmdline/standalone_cli_tools/CtaVerifyFile.cpp
index 965a8370e4..f799306ae0 100644
--- a/cmdline/standalone_cli_tools/CtaVerifyFile.cpp
+++ b/cmdline/standalone_cli_tools/CtaVerifyFile.cpp
@@ -19,9 +19,10 @@
#include <iostream>
#include <map>
+#include "common/CmdLineArgs.hpp"
+#include "common/utils/utils.hpp"
#include "CtaFrontendApi.hpp"
#include "version.h"
-#include "common/CmdLineArgs.hpp"
using namespace cta::cliTool;
@@ -49,11 +50,10 @@ typedef std::map<std::string, std::string> AttrMap;
* Fill a Notification message from the command-line parameters and stdin
*
* @param[out] notification The protobuf to fill
- * @param[in] argc Number of arguments passed on the command line
- * @param[in] argv Command line arguments array
+ * @param[in] cmdLineArgs Command line arguments
+ * @param[in] archiveFileId Archive file id to verify
*/
-void fillNotification(cta::eos::Notification ¬ification, const int argc, char *const *const argv, const CmdLineArgs &cmdLineArgs)
-{
+void fillNotification(cta::eos::Notification ¬ification, const CmdLineArgs &cmdLineArgs, const std::string &archiveFileId) {
XrdSsiPb::Config config(config_file, "eos");
for (const auto &conf_option : std::vector<std::string>({ "instance", "requester.user", "requester.group" })) {
if (!config.getOptionValueStr(conf_option).first) {
@@ -63,25 +63,16 @@ void fillNotification(cta::eos::Notification ¬ification, const int argc, char
notification.mutable_wf()->mutable_instance()->set_name(config.getOptionValueStr("instance").second);
notification.mutable_cli()->mutable_user()->set_username(config.getOptionValueStr("requester.user").second);
notification.mutable_cli()->mutable_user()->set_groupname(config.getOptionValueStr("requester.group").second);
-
- if(cmdLineArgs.m_help) { cmdLineArgs.printUsage(std::cout); exit(0); }
-
- if(!cmdLineArgs.m_archiveFileId || !cmdLineArgs.m_vid) {
- cmdLineArgs.printUsage(std::cout);
- throw std::runtime_error("ERROR: Usage");
- }
if (cmdLineArgs.m_diskInstance) {
notification.mutable_wf()->mutable_instance()->set_name(cmdLineArgs.m_diskInstance.value());
}
if (cmdLineArgs.m_requestUser) {
- notification.mutable_cli()->mutable_user()->set_username(cmdLineArgs.m_requestUser.value());
+ notification.mutable_cli()->mutable_user()->set_username(cmdLineArgs.m_requestUser.value());
}
if (cmdLineArgs.m_requestGroup) {
notification.mutable_cli()->mutable_user()->set_groupname(cmdLineArgs.m_requestGroup.value());
- }
-
- const std::string archiveFileId(cmdLineArgs.m_archiveFileId.value());
+ }
// WF
notification.mutable_wf()->set_event(cta::eos::Workflow::PREPARE);
@@ -89,7 +80,7 @@ void fillNotification(cta::eos::Notification ¬ification, const int argc, char
notification.mutable_wf()->set_verify_only(true);
notification.mutable_wf()->set_vid(cmdLineArgs.m_vid.value());
-
+
// Transport
notification.mutable_transport()->set_dst_url("file://dummy");
@@ -106,18 +97,9 @@ void fillNotification(cta::eos::Notification ¬ification, const int argc, char
}
}
-
-/*
- * Sends a Notification to the CTA XRootD SSI server
- */
-int exceptionThrowingMain(int argc, char *const *const argv)
-{
- using namespace cta::cliTool;
-
+void sendVerifyRequest(const CmdLineArgs &cmdLineArgs, const std::string &archiveFileId) {
std::string vid;
- cta::cliTool::CmdLineArgs cmdLineArgs(argc, argv, StandaloneCliTool::CTA_VERIFY_FILE);
-
// Verify that the Google Protocol Buffer header and linked library versions are compatible
GOOGLE_PROTOBUF_VERIFY_VERSION;
@@ -143,7 +125,7 @@ int exceptionThrowingMain(int argc, char *const *const argv)
config.getEnv("log", "XrdSsiPbLogLevel");
// Parse the command line arguments: fill the Notification fields
- fillNotification(notification, argc, argv, cmdLineArgs);
+ fillNotification(notification, cmdLineArgs, archiveFileId);
// Obtain a Service Provider
XrdSsiPbServiceType cta_service(config);
@@ -166,6 +148,42 @@ int exceptionThrowingMain(int argc, char *const *const argv)
// Delete all global objects allocated by libprotobuf
google::protobuf::ShutdownProtobufLibrary();
+}
+
+/*
+ * Sends a Notification to the CTA XRootD SSI server
+ */
+int exceptionThrowingMain(int argc, char *const *const argv)
+{
+ using namespace cta::cliTool;
+
+ cta::cliTool::CmdLineArgs cmdLineArgs(argc, argv, StandaloneCliTool::CTA_VERIFY_FILE);
+
+ if(cmdLineArgs.m_help) { cmdLineArgs.printUsage(std::cout); exit(0); }
+
+ std::vector<std::string> archiveFileIds;
+
+ if((!cmdLineArgs.m_archiveFileId && !cmdLineArgs.m_archiveFileIds) || !cmdLineArgs.m_vid) {
+ cmdLineArgs.printUsage(std::cout);
+ throw std::runtime_error("Error: Usage");
+ }
+
+ if(cmdLineArgs.m_archiveFileId) {
+ const std::vector<std::string> ids = cta::utils::commaSeparatedStringToVector(cmdLineArgs.m_archiveFileId.value());
+ for (const auto &id : ids) {
+ archiveFileIds.push_back(id);
+ }
+ }
+
+ if(cmdLineArgs.m_archiveFileIds) {
+ for (const auto &id : cmdLineArgs.m_archiveFileIds.value()) {
+ archiveFileIds.push_back(id);
+ }
+ }
+
+ for(const auto &archiveFileId : archiveFileIds) {
+ sendVerifyRequest(cmdLineArgs, archiveFileId);
+ }
return 0;
}
diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp
index 22a036c30b..22bbb28be7 100644
--- a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp
+++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp
@@ -53,6 +53,7 @@ static struct option sendFileLongOption[] = {
static struct option verifyFileLongOption[] = {
{"id", required_argument, nullptr, 'I'},
+ {"filename", required_argument, nullptr, 'F'},
{"instance", required_argument, nullptr, 'i'},
{"request.user", required_argument, nullptr, 'u'},
{"request.group", required_argument, nullptr, 'g'},
@@ -80,7 +81,7 @@ std::map<StandaloneCliTool, const option*> longopts = {
std::map<StandaloneCliTool, const char*> shortopts = {
{StandaloneCliTool::RESTORE_FILES, "I:i:f:F:v:c:hd:"},
{StandaloneCliTool::CTA_SEND_EVENT, "i:e:u:g:"},
- {StandaloneCliTool::CTA_VERIFY_FILE, "I:i:u:g:v:h:"},
+ {StandaloneCliTool::CTA_VERIFY_FILE, "I:F:i:u:g:v:h:"},
{StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS, "I:F:n:t:h:"},
};
@@ -212,7 +213,10 @@ void CmdLineArgs::readIdListFromFile(const std::string &filename) {
m_archiveFileIds.value().push_back(line);
break;
case StandaloneCliTool::CTA_VERIFY_FILE:
- m_fxIds.value().push_back(line);
+ if (!m_archiveFileIds) {
+ m_archiveFileIds = std::list<std::string>();
+ }
+ m_archiveFileIds.value().push_back(line);
break;
case StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS:
if (!m_archiveFileIds) {
@@ -245,7 +249,16 @@ void CmdLineArgs::printUsage(std::ostream &os) const {
break;
case StandaloneCliTool::CTA_VERIFY_FILE :
os << " Usage:" << std::endl <<
- " cta-verify-file --id/-I <archiveFileID> --vid/-v <vid> [--instance/-i <instance>] [--request.user/-u <user>] [request.group/-g <group>]" << std::endl;
+ " cta-verify-file --id/-I <archiveFileID,archiveFileID,...,archiveFileID> | --filename/-F <filename> " << std::endl <<
+ " --vid/-v <vid>" << std::endl <<
+ " [--instance/-i <instance>]" << std::endl <<
+ " [--request.user/-u <user>]" << std::endl <<
+ " [request.group/-g <group>]" << std::endl << std::endl <<
+ " If a filename is used to provide archive file ids, it should be following the format:" << std::endl << std::endl <<
+ " <archiveFileId_1>" << std::endl <<
+ " <archiveFileId_2>" << std::endl <<
+ " ..." << std::endl <<
+ " <archiveFileId_n>" << std::endl;
break;
case StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS :
os << " Usage:" << std::endl <<
diff --git a/common/utils/utils.cpp b/common/utils/utils.cpp
index 580797b5b2..36698c46e5 100644
--- a/common/utils/utils.cpp
+++ b/common/utils/utils.cpp
@@ -998,5 +998,21 @@ std::string getEnv(const std::string& variableName){
return std::string(envVarC);
}
+std::vector<std::string> commaSeparatedStringToVector(const std::string &commaSeparated) {
+ std::string str = commaSeparated;
+ std::vector<std::string> result;
+ // Remove white spaces
+ str.erase(std::remove_if(str.begin(), str.end(), isspace), str.end());
+
+ // Separate the string by ,
+ std::istringstream ss(str);
+ while(ss.good()) {
+ std::string substr;
+ std::getline(ss, substr, ',' );
+ result.push_back( substr );
+ }
+ return result;
+}
+
} // namespace utils
} // namespace cta
diff --git a/common/utils/utils.hpp b/common/utils/utils.hpp
index b953de9eb6..1b11cf2f10 100644
--- a/common/utils/utils.hpp
+++ b/common/utils/utils.hpp
@@ -479,6 +479,13 @@ namespace utils {
*/
std::string getEnv(const std::string & variableName);
+ /**
+ * Transorms a comma separated list to a vector
+ * @param commaSeparated the string hat will be transformed to a vector
+ * @return the transformed vecor
+ */
+ std::vector<std::string> commaSeparatedStringToVector(const std::string &commaSeparated);
+
} // namespace utils
} // namespace cta
--
GitLab