Resolve "Allow to submit multiple files for verification"

ReleaseNotes.md

@@ -7,6 +7,7 @@
 - cta/CTA#78 - Tool to update the storage class
 - cta/CTA#153 - Allow verification status to be cleared with cta-admin
 - cta/CTA#173 - Update release notes and small changes to refactoring of operation tools cmd line parsing - Compatible with operations 0.4-95 or later
+- cta/CTA#180 - Allow to submit multiple files for verification
 ### Continuous Integration
 - cta/CTA#118 - Add unit tests for OSM label
 - cta/CTA#191 - Block merge until cta_valgrind success

cmdline/standalone_cli_tools/CtaVerifyFile.cpp

@@ -19,9 +19,10 @@
 #include <iostream>
 #include <map>
 
+#include "common/CmdLineArgs.hpp"
+#include "common/utils/utils.hpp"
 #include "CtaFrontendApi.hpp"
 #include "version.h"
-#include "common/CmdLineArgs.hpp"
 
 using namespace cta::cliTool;
 
@@ -49,11 +50,10 @@ typedef std::map<std::string, std::string> AttrMap;
  * Fill a Notification message from the command-line parameters and stdin
  *
  * @param[out]   notification    The protobuf to fill
- * @param[in]    argc            Number of arguments passed on the command line
- * @param[in]    argv            Command line arguments array
+ * @param[in]    cmdLineArgs     Command line arguments
+ * @param[in]    archiveFileId   Archive file id to verify
  */
-void fillNotification(cta::eos::Notification &notification, const int argc, char *const *const argv, const CmdLineArgs &cmdLineArgs)
-{   
+void fillNotification(cta::eos::Notification &notification, const CmdLineArgs &cmdLineArgs, const std::string &archiveFileId) {
   XrdSsiPb::Config config(config_file, "eos");
   for (const auto &conf_option : std::vector<std::string>({ "instance", "requester.user", "requester.group" })) {
     if (!config.getOptionValueStr(conf_option).first) {
@@ -63,25 +63,16 @@ void fillNotification(cta::eos::Notification &notification, const int argc, char
   notification.mutable_wf()->mutable_instance()->set_name(config.getOptionValueStr("instance").second);
   notification.mutable_cli()->mutable_user()->set_username(config.getOptionValueStr("requester.user").second);
   notification.mutable_cli()->mutable_user()->set_groupname(config.getOptionValueStr("requester.group").second);
-  
-  if(cmdLineArgs.m_help) { cmdLineArgs.printUsage(std::cout); exit(0); }
-
-  if(!cmdLineArgs.m_archiveFileId || !cmdLineArgs.m_vid) { 
-    cmdLineArgs.printUsage(std::cout);
-    throw std::runtime_error("ERROR: Usage");
-  }
 
   if (cmdLineArgs.m_diskInstance) {
     notification.mutable_wf()->mutable_instance()->set_name(cmdLineArgs.m_diskInstance.value());
   }
   if (cmdLineArgs.m_requestUser) {
-    notification.mutable_cli()->mutable_user()->set_username(cmdLineArgs.m_requestUser.value());  
+    notification.mutable_cli()->mutable_user()->set_username(cmdLineArgs.m_requestUser.value());
   }
   if (cmdLineArgs.m_requestGroup) {
     notification.mutable_cli()->mutable_user()->set_groupname(cmdLineArgs.m_requestGroup.value());
-  }  
-
-  const std::string archiveFileId(cmdLineArgs.m_archiveFileId.value());
+  }
 
   // WF
   notification.mutable_wf()->set_event(cta::eos::Workflow::PREPARE);
@@ -89,7 +80,7 @@ void fillNotification(cta::eos::Notification &notification, const int argc, char
   notification.mutable_wf()->set_verify_only(true);
   notification.mutable_wf()->set_vid(cmdLineArgs.m_vid.value());
 
-  
+
   // Transport
   notification.mutable_transport()->set_dst_url("file://dummy");
 
@@ -106,18 +97,9 @@ void fillNotification(cta::eos::Notification &notification, const int argc, char
   }
 }
 
-
-/*
- * Sends a Notification to the CTA XRootD SSI server
- */
-int exceptionThrowingMain(int argc, char *const *const argv)
-{
-  using namespace cta::cliTool;
-
+void sendVerifyRequest(const CmdLineArgs &cmdLineArgs, const std::string &archiveFileId) {
   std::string vid;
 
-  cta::cliTool::CmdLineArgs cmdLineArgs(argc, argv, StandaloneCliTool::CTA_VERIFY_FILE);
-
   // Verify that the Google Protocol Buffer header and linked library versions are compatible
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
@@ -143,7 +125,7 @@ int exceptionThrowingMain(int argc, char *const *const argv)
   config.getEnv("log", "XrdSsiPbLogLevel");
 
   // Parse the command line arguments: fill the Notification fields
-  fillNotification(notification, argc, argv, cmdLineArgs);
+  fillNotification(notification, cmdLineArgs, archiveFileId);
 
   // Obtain a Service Provider
   XrdSsiPbServiceType cta_service(config);
@@ -166,6 +148,42 @@ int exceptionThrowingMain(int argc, char *const *const argv)
 
   // Delete all global objects allocated by libprotobuf
   google::protobuf::ShutdownProtobufLibrary();
+}
+
+/*
+ * Sends a Notification to the CTA XRootD SSI server
+ */
+int exceptionThrowingMain(int argc, char *const *const argv)
+{
+  using namespace cta::cliTool;
+
+  cta::cliTool::CmdLineArgs cmdLineArgs(argc, argv, StandaloneCliTool::CTA_VERIFY_FILE);
+
+  if(cmdLineArgs.m_help) { cmdLineArgs.printUsage(std::cout); exit(0); }
+
+  std::vector<std::string> archiveFileIds;
+
+  if((!cmdLineArgs.m_archiveFileId && !cmdLineArgs.m_archiveFileIds) || !cmdLineArgs.m_vid) {
+    cmdLineArgs.printUsage(std::cout);
+    throw std::runtime_error("Error: Usage");
+  }
+
+  if(cmdLineArgs.m_archiveFileId) {
+    const std::vector<std::string> ids = cta::utils::commaSeparatedStringToVector(cmdLineArgs.m_archiveFileId.value());
+    for (const auto &id : ids) {
+      archiveFileIds.push_back(id);
+    }
+  }
+
+  if(cmdLineArgs.m_archiveFileIds) {
+    for (const auto &id : cmdLineArgs.m_archiveFileIds.value()) {
+      archiveFileIds.push_back(id);
+    }
+  }
+
+  for(const auto &archiveFileId : archiveFileIds) {
+    sendVerifyRequest(cmdLineArgs, archiveFileId);
+  }
 
   return 0;
 }

cmdline/standalone_cli_tools/common/CmdLineArgs.cpp

@@ -53,6 +53,7 @@ static struct option sendFileLongOption[] = {
 
 static struct option verifyFileLongOption[] = {
   {"id", required_argument, nullptr, 'I'},
+  {"filename", required_argument, nullptr, 'F'},
   {"instance", required_argument, nullptr, 'i'},
   {"request.user", required_argument, nullptr, 'u'},
   {"request.group", required_argument, nullptr, 'g'},
@@ -80,7 +81,7 @@ std::map<StandaloneCliTool, const option*> longopts = {
 std::map<StandaloneCliTool, const char*> shortopts = {
   {StandaloneCliTool::RESTORE_FILES, "I:i:f:F:v:c:hd:"},
   {StandaloneCliTool::CTA_SEND_EVENT, "i:e:u:g:"},
-  {StandaloneCliTool::CTA_VERIFY_FILE, "I:i:u:g:v:h:"},
+  {StandaloneCliTool::CTA_VERIFY_FILE, "I:F:i:u:g:v:h:"},
   {StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS, "I:F:n:t:h:"},
 };
 
@@ -212,7 +213,10 @@ void CmdLineArgs::readIdListFromFile(const std::string &filename) {
         m_archiveFileIds.value().push_back(line);
         break;
       case StandaloneCliTool::CTA_VERIFY_FILE:
-        m_fxIds.value().push_back(line);
+        if (!m_archiveFileIds) {
+          m_archiveFileIds = std::list<std::string>();
+        }
+        m_archiveFileIds.value().push_back(line);
         break;
       case StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS:
         if (!m_archiveFileIds) {
@@ -245,7 +249,16 @@ void CmdLineArgs::printUsage(std::ostream &os) const {
     break;
   case StandaloneCliTool::CTA_VERIFY_FILE :
     os << "    Usage:" << std::endl <<
-    "    cta-verify-file --id/-I <archiveFileID> --vid/-v <vid> [--instance/-i <instance>] [--request.user/-u <user>] [request.group/-g <group>]" << std::endl;
+    "    cta-verify-file --id/-I <archiveFileID,archiveFileID,...,archiveFileID> | --filename/-F <filename> " << std::endl <<
+    "                            --vid/-v <vid>" << std::endl <<
+    "                            [--instance/-i <instance>]" << std::endl <<
+    "                            [--request.user/-u <user>]" << std::endl <<
+    "                            [request.group/-g <group>]" << std::endl << std::endl <<
+    "                            If a filename is used to provide archive file ids, it should be following the format:" << std::endl << std::endl <<
+    "                            <archiveFileId_1>" << std::endl <<
+    "                            <archiveFileId_2>" << std::endl <<
+    "                            ..." << std::endl <<
+    "                            <archiveFileId_n>" << std::endl;
     break;
   case StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS :
     os << "    Usage:" << std::endl <<

common/utils/utils.cpp

@@ -998,5 +998,21 @@ std::string getEnv(const std::string& variableName){
   return std::string(envVarC);
 }
 
+std::vector<std::string> commaSeparatedStringToVector(const std::string &commaSeparated) {
+  std::string str = commaSeparated;
+  std::vector<std::string> result;
+  // Remove white spaces
+  str.erase(std::remove_if(str.begin(), str.end(), isspace), str.end());
+
+  // Separate the string by ,
+  std::istringstream ss(str);
+  while(ss.good()) {
+      std::string substr;
+      std::getline(ss, substr, ',' );
+      result.push_back( substr );
+  }
+  return result;
+}
+
 } // namespace utils
 } // namespace cta

common/utils/utils.hpp

@@ -479,6 +479,13 @@ namespace utils {
    */
   std::string getEnv(const std::string & variableName);
 
+   /**
+   * Transorms a comma separated list to a vector
+   * @param commaSeparated the string hat will be transformed to a vector
+   * @return the transformed vecor
+   */
+  std::vector<std::string> commaSeparatedStringToVector(const std::string &commaSeparated);
+
 } // namespace utils
 
 } // namespace cta