Resolve "Switch to json format when providing arguments from a file...

Resolve "Switch to json format when providing arguments from a file (cta-restore-deleted-files, cta-change-storage-class)"

ReleaseNotes.md

@@ -11,6 +11,7 @@
 - cta/CTA#248 - Clean up output from cta-readtp
 - cta/CTA#251 - Increase free drive STALE threshold to 4 hours
 - cta/CTA#218 - Do not retry during repack requests
+- cta/CTA#252 - Update cta-change-storage-class to accept json file as input
 ### Bug Fixes
 - cta/CTA#181 - cta-statistics-update can fail for catalogues in postgres
 - cta/CTA#189 - Avoid postgres logging frequent warnings about no transaction in progress

cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt

@@ -27,7 +27,7 @@ include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include)
 # Compiled protocol buffers
 include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS})
 
-add_executable(cta-change-storage-class ChangeStorageClass.cpp ChangeStorageClassMain.cpp ../../CtaAdminCmdParse.cpp)
+add_executable(cta-change-storage-class ChangeStorageClass.cpp ChangeStorageClassMain.cpp ../../CtaAdminCmdParse.cpp JsonFileData.cpp)
 target_link_libraries(cta-change-storage-class XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient stdc++fs ctaCmdlineToolsCommon)
 set_property (TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
 

cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp

@@ -29,6 +29,7 @@
 
 #include "cmdline/CtaAdminCmdParse.hpp"
 #include "cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp"
+#include "cmdline/standalone_cli_tools/change_storage_class/JsonFileData.hpp"
 #include "cmdline/standalone_cli_tools/common/CatalogueFetch.hpp"
 #include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp"
 #include "cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp"
@@ -63,19 +64,42 @@ ChangeStorageClass::ChangeStorageClass(
 // exceptionThrowingMain
 //------------------------------------------------------------------------------
 int ChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) {
-    CmdLineArgs cmdLineArgs(argc, argv, StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS);
+  CmdLineArgs cmdLineArgs(argc, argv, StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS);
+  handleArguments(cmdLineArgs);
 
+  auto [serviceProvider, endpointmap] = ConnConfiguration::readAndSetConfiguration(m_log, getUsername(), cmdLineArgs);
+  m_serviceProviderPtr = std::move(serviceProvider);
+  m_endpointMapPtr = std::move(endpointmap);
+
+  storageClassExists();
+  updateStorageClassInEosNamespace();
+  updateStorageClassInCatalogue();
+  writeSkippedArchiveIdsToFile();
+  return 0;
+}
+
+//------------------------------------------------------------------------------
+// handleArguments
+//------------------------------------------------------------------------------
+void ChangeStorageClass::handleArguments(const CmdLineArgs &cmdLineArgs) {
   if (cmdLineArgs.m_help) {
     cmdLineArgs.printUsage(std::cout);
     throw exception::UserError("");
   }
 
+  if(cmdLineArgs.m_json) {
+    JsonFileData jsonFilaData(cmdLineArgs.m_json.value());
+    for(const auto &jsonFileDataObject : jsonFilaData.m_jsonArgumentsCollection) {
+      m_archiveFileIds.push_back(jsonFileDataObject.archiveId);
+    }
+  }
+
   if (!cmdLineArgs.m_storageClassName) {
     cmdLineArgs.printUsage(std::cout);
     throw exception::UserError("Missing requried option: storage.class.name");
   }
 
-  if (!cmdLineArgs.m_archiveFileId && !cmdLineArgs.m_archiveFileIds) {
+  if (!cmdLineArgs.m_archiveFileId && !cmdLineArgs.m_json) {
     cmdLineArgs.printUsage(std::cout);
     throw exception::UserError("filename or id must be provided");
   }
@@ -86,27 +110,11 @@ int ChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const
     m_archiveFileIds.push_back(cmdLineArgs.m_archiveFileId.value());
   }
 
-  if (cmdLineArgs.m_archiveFileIds) {
-    for (const auto& archiveFileId : cmdLineArgs.m_archiveFileIds.value() ) {
-      m_archiveFileIds.push_back(archiveFileId);
-    }
-  }
-
   if (cmdLineArgs.m_frequency) {
     m_eosUpdateFrequency = cmdLineArgs.m_frequency.value();
   } else {
     m_eosUpdateFrequency = 100;
   }
-
-  auto [serviceProvider, endpointmap] = ConnConfiguration::readAndSetConfiguration(m_log, getUsername(), cmdLineArgs);
-  m_serviceProviderPtr = std::move(serviceProvider);
-  m_endpointMapPtr = std::move(endpointmap);
-
-  storageClassExists();
-  updateStorageClassInEosNamespace();
-  updateStorageClassInCatalogue();
-  writeSkippedArchiveIdsToFile();
-  return 0;
 }
 
 //------------------------------------------------------------------------------
@@ -143,6 +151,11 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() {
     }
 
     const auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId);
+    {
+      std::list<cta::log::Param> params;
+      params.push_back(cta::log::Param("path", path));
+      m_log(cta::log::INFO, "Path found", params);
+    }
     const auto status = m_endpointMapPtr->setXAttr(diskInstance, path, "sys.archive.storage_class", m_storageClassName);
     if (status != 0) {
       m_archiveIdsNotUpdatedInEos.push_back(archiveFileId);
@@ -197,6 +210,12 @@ void ChangeStorageClass::storageClassExists() const {
   // wait until the data stream has been processed before exiting
   stream_future.wait();
 
+  for(const auto &storageClass : g_storageClasses) {
+    std::list<cta::log::Param> params;
+    params.push_back(cta::log::Param("storageClass", storageClass));
+    m_log(cta::log::INFO, "Storage class found", params);
+  }
+
   if (g_storageClasses.empty()){
     throw(exception::UserError("The storage class provided has not been defined."));
   }
@@ -261,7 +280,7 @@ void ChangeStorageClass::updateStorageClassInCatalogue() const {
 // writeSkippedArchiveIdsToFile
 //------------------------------------------------------------------------------
 void ChangeStorageClass::writeSkippedArchiveIdsToFile() const {
-  const std::filesystem::path filePath = "/var/log/skippedArchiveIds.txt";
+  const std::filesystem::path filePath = "/tmp/skippedArchiveIds.txt";
   std::ofstream archiveIdFile(filePath);
 
   if (archiveIdFile.fail()) {
@@ -270,7 +289,7 @@ void ChangeStorageClass::writeSkippedArchiveIdsToFile() const {
 
   if (archiveIdFile.is_open()) {
     for (const auto& archiveId : m_archiveIdsNotUpdatedInEos) {
-      archiveIdFile << archiveId << std::endl;
+      archiveIdFile << "{ archiveId : " << archiveId << " }" << std::endl;
     }
     archiveIdFile.close();
     std::cout << m_archiveIdsNotUpdatedInEos.size() << " files did not update the storage class." << std::endl;

cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp

@@ -33,6 +33,7 @@ class StdoutLogger;
 }
 
 namespace cliTool {
+class CmdLineArgs;
 
 class ChangeStorageClass: public CmdLineTool {
 public:
@@ -120,6 +121,11 @@ private:
   */
   bool fileInFlight(const google::protobuf::RepeatedField<unsigned int> &locations) const;
 
+  /**
+  * Fills the member variables with data, based on the arguments that were provided
+  */
+  void handleArguments(const CmdLineArgs &cmdLineArgs);
+
   /**
    * An exception throwing version of main().
    *

cmdline/standalone_cli_tools/change_storage_class/JsonFileData.cpp

+/*
+ * @project      The CERN Tape Archive (CTA)
+ * @copyright    Copyright © 2021-2022 CERN
+ * @license      This program is free software, distributed under the terms of the GNU General Public
+ *               Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can
+ *               redistribute it and/or modify it under the terms of the GPL Version 3, or (at your
+ *               option) any later version.
+ *
+ *               This program is distributed in the hope that it will be useful, but WITHOUT ANY
+ *               WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ *               PARTICULAR PURPOSE. See the GNU General Public License for more details.
+ *
+ *               In applying this licence, CERN does not waive the privileges and immunities
+ *               granted to it by virtue of its status as an Intergovernmental Organization or
+ *               submit itself to any jurisdiction.
+ */
+
+#include <filesystem>
+#include <iostream>
+#include <fstream>
+
+#include "cmdline/standalone_cli_tools/change_storage_class/JsonFileData.hpp"
+#include "common/exception/UserError.hpp"
+
+namespace cta::cliTool {
+
+JsonFileData::JsonFileData(const std::filesystem::path &jsonPath) {
+  readJson(jsonPath);
+}
+
+void JsonFileData::readJson(const std::filesystem::path &jsonPath) {
+  std::ifstream infile(jsonPath);
+  if(infile.fail()) {
+    throw exception::UserError("Could not open " + jsonPath.generic_string());
+  }
+
+  std::string line;
+  while (std::getline(infile, line)) {
+    if(!line.empty()) {
+      cta::utils::json::object::JSONCObject jsonObject;
+      buildFromJSON(line);
+
+      JsonFileDataObject JsonFileDataObject;
+      JsonFileDataObject.archiveId = jsonGetValue<std::string>("archiveId");
+
+      m_jsonArgumentsCollection.push_back(JsonFileDataObject);
+    }
+  }
+  infile.close();
+}
+
+} // namespace cta::cliTool
\ No newline at end of file

cmdline/standalone_cli_tools/change_storage_class/JsonFileData.hpp

+/*
+ * @project      The CERN Tape Archive (CTA)
+ * @copyright    Copyright © 2021-2022 CERN
+ * @license      This program is free software, distributed under the terms of the GNU General Public
+ *               Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can
+ *               redistribute it and/or modify it under the terms of the GPL Version 3, or (at your
+ *               option) any later version.
+ *
+ *               This program is distributed in the hope that it will be useful, but WITHOUT ANY
+ *               WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ *               PARTICULAR PURPOSE. See the GNU General Public License for more details.
+ *
+ *               In applying this licence, CERN does not waive the privileges and immunities
+ *               granted to it by virtue of its status as an Intergovernmental Organization or
+ *               submit itself to any jurisdiction.
+ */
+
+#pragma once
+
+#include <filesystem>
+#include <string>
+#include <list>
+
+#include "common/json/object/JSONCObject.hpp"
+
+namespace cta::cliTool {
+
+struct JsonFileDataObject {
+  std::string archiveId;
+};
+
+class JsonFileData : public cta::utils::json::object::JSONCObject{
+  public:
+    /**
+    * Constructor
+    */
+    JsonFileData(const std::filesystem::path &jsonPath);
+
+    /**
+    * List of argument objects
+    */
+    std::list<JsonFileDataObject> m_jsonArgumentsCollection;
+  private:
+    /**
+    * Reads the provided json file
+    */
+    void readJson(const std::filesystem::path &path);
+ };
+ } // namespace cta::cliTool
\ No newline at end of file

cmdline/standalone_cli_tools/change_storage_class/README.md

+# How to use the cta-change-storage-class
+The cta-change-storage-class can be used to change the storaegclass of files. To storage class update of the eos containers must be done manually by an operator.
+
+```cta-change-storage-class --id/-I <archiveFileID> | --json/-j <path> --storageclassname/-n <storageClassName> [--frequenzy/-t <eosRequestFrequency>]```
+
+where the json file is a text file with one json object for each line, example:
+
+```
+{"archiveId": <archiveId>}
+{"archiveId": <archiveId>}
+{"archiveId": <archiveId>}
+```
+
+The tool must be run from the frontend as it needs access to both eos and the catalogue. This means that and ```cta-cli.conf``` and a ```eos.grpc.keytab``` must be copied to the frontend. ```cta-cli``` should be placed in /etc/cta, while the location of ```eos.grpc.keytab``` should be specified in ```cta-frontend-xrootd.conf```. If the tool fails to update any of the files, the ids can be found in ```/tmp/skippedArchiveIds.txt``` in json format.
\ No newline at end of file

cmdline/standalone_cli_tools/common/CmdLineArgs.cpp

@@ -63,8 +63,8 @@ static struct option verifyFileLongOption[] = {
 
 static struct option changeStorageClassLongOption[] = {
   {"id", required_argument, nullptr, 'I'},
-  {"filename", required_argument, nullptr, 'F'},
-  {"storage.class.name", required_argument, nullptr, 'n'},
+  {"json", required_argument, nullptr, 'j'},
+  {"storageclassname", required_argument, nullptr, 'n'},
   {"frequenzy", required_argument, nullptr, 't'},
   {"help", no_argument, nullptr, 'h'},
   {nullptr, 0, nullptr, 0}
@@ -88,7 +88,7 @@ std::map<StandaloneCliTool, const char*> shortopts = {
   {StandaloneCliTool::RESTORE_FILES, "I:i:f:F:v:c:hd:"},
   {StandaloneCliTool::CTA_SEND_EVENT, "i:e:u:g:"},
   {StandaloneCliTool::CTA_VERIFY_FILE, "I:F:i:u:g:v:h:"},
-  {StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS, "I:F:n:t:h:"},
+  {StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS, "I:j:n:t:h:"},
   {StandaloneCliTool::EOS_NAMESPACE_INJECTION, "j:h:"},
 };
 
@@ -270,7 +270,7 @@ void CmdLineArgs::printUsage(std::ostream &os) const {
     break;
   case StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS :
     os << "    Usage:" << std::endl <<
-    "    cta-change-storage-class --id/-I <archiveFileID> | --filename/-F <filename> --storage.class.name/-n <storageClassName> [--frequenzy/-t <eosRequestFrequency>]" << std::endl << std::endl;
+    "    cta-change-storage-class --id/-I <archiveFileID> | --json/-j <path> --storageclassname/-n <storageClassName> [--frequenzy/-t <eosRequestFrequency>]" << std::endl << std::endl;
     break;
   case StandaloneCliTool::EOS_NAMESPACE_INJECTION :
     os << "    Usage:" << std::endl <<

cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp

@@ -145,10 +145,14 @@ int RestoreFilesCmd::exceptionThrowingMain(const int argc, char *const *const ar
   m_copyNumber = cmdLineArgs.m_copyNumber;
   m_archiveFileId = cmdLineArgs.m_archiveFileId;
 
-  if (m_fids && !m_diskInstance) {
-    throw XrdSsiPb::UserException("Disk instance must be provided when fids are used as input.");
+  if(argc == 1) {
+    cmdLineArgs.printUsage(m_out);
+    throw RestoreFilesCmdException("No arguments were provided");
+  }
+  if(m_fids && !m_diskInstance) {
+    cmdLineArgs.printUsage(m_out);
+    throw RestoreFilesCmdException("Disk instance must be provided when fids are used as input.");
   }
-
 
   if (cmdLineArgs.m_debug) {
     m_log.setLogMask("DEBUG");

continuousintegration/orchestration/tests/changeStorageClass.sh

@@ -82,15 +82,16 @@ cd ~
 IDS_FILEPATH=~/archiveFileIds.txt
 rm ${IDS_FILEPATH}
 touch ${IDS_FILEPATH}
-echo ${EOS_ARCHIVE_ID_1} | tee -a ${IDS_FILEPATH}
-echo ${EOS_ARCHIVE_ID_2} | tee -a ${IDS_FILEPATH}
+echo '{"archiveId": '${EOS_ARCHIVE_ID_1}'}' >> ${IDS_FILEPATH}
+echo '{"archiveId": '${EOS_ARCHIVE_ID_2}'}' >> ${IDS_FILEPATH}
+sleep 1
 
 echo
 kubectl cp ~/CTA-build/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class ${NAMESPACE}/ctafrontend:/usr/bin/
 echo "kubectl cp ${IDS_FILEPATH} ${NAMESPACE}/ctafrontend:~/"
 kubectl cp ${IDS_FILEPATH} ${NAMESPACE}/ctafrontend:/root/
-echo "kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/cta/eos.sss.keytab cta-change-storage-class --storage.class.name ${NEW_STORAGE_CLASS_NAME} --filename ${IDS_FILEPATH}"
-kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c "XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/cta/eos.sss.keytab cta-change-storage-class --storage.class.name ${NEW_STORAGE_CLASS_NAME} --filename ${IDS_FILEPATH} -t 1"
+echo "kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/cta/eos.sss.keytab cta-change-storage-class --storageclassname ${NEW_STORAGE_CLASS_NAME} --json ${IDS_FILEPATH}"
+kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c "XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/cta/eos.sss.keytab cta-change-storage-class --storageclassname ${NEW_STORAGE_CLASS_NAME} --json ${IDS_FILEPATH} -t 1"
 
 EOS_METADATA_PATH_AFTER_CHANGE_1=$(mktemp -d).json
 echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH_AFTER_CHANGE_1}"