[migration] Adds eos-test-file-inject to cta-migration RPM

cta.spec.in

@@ -308,6 +308,7 @@ directory metadata into the EOS namespace.
 %attr(0755,root,root) %{_bindir}/eos-import-dirs
 %attr(0755,root,root) %{_bindir}/eos-import-files
 %attr(0755,root,root) %{_bindir}/eos-test-dir-inject
+%attr(0755,root,root) %{_bindir}/eos-test-file-inject
 %attr(0755,root,root) %{_bindir}/eos-test-inject.sh
 
 %package -n cta-rmcd

migration/gRPC/CMakeLists.txt

@@ -32,12 +32,17 @@ add_executable(eos-import-files EosImportFiles.cpp GrpcClient.cpp)
 target_link_libraries(eos-import-files EosMigration ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} ctacatalogue)
 set_property(TARGET eos-import-files APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
 
-# Standalone test tool
+# Standalone test tools
 add_executable(eos-test-dir-inject EosTestDirInject.cpp GrpcClient.cpp)
 target_link_libraries(eos-test-dir-inject EosMigration ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} ctacatalogue)
 set_property(TARGET eos-test-dir-inject APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
 
+add_executable(eos-test-file-inject EosTestFileInject.cpp GrpcClient.cpp)
+target_link_libraries(eos-test-file-inject EosMigration ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} ctacatalogue)
+set_property(TARGET eos-test-file-inject APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH})
+
 install(TARGETS eos-import-dirs DESTINATION usr/bin)
 install(TARGETS eos-import-files DESTINATION usr/bin)
 install(TARGETS eos-test-dir-inject DESTINATION usr/bin)
+install(TARGETS eos-test-file-inject DESTINATION usr/bin)
 install(FILES ${CMAKE_SOURCE_DIR}/migration/gRPC/eos-test-inject.sh DESTINATION usr/bin)

migration/gRPC/EosTestDirInject.cpp

@@ -126,10 +126,10 @@ void EosTestDirInject::inject(const std::string &path, uint64_t fileid)
   }
 
   // Inject directories into EOS
-    int retc = m_eosgrpc->ContainerInsert(dirs);
-    if(retc != 0) {
-      throw std::runtime_error("EosTestDirInject::processBatch(): ContainerInsert failed with error " + std::to_string(retc));
-    }
+  int retc = m_eosgrpc->ContainerInsert(dirs);
+  if(retc != 0) {
+    throw std::runtime_error("EosTestDirInject::inject(): ContainerInsert failed with error " + std::to_string(retc));
+  }
 
   auto elapsed_time = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - m_start_time);
   std::cerr << "Processed 1 directory in " << elapsed_time.count() << "s" << std::endl;

migration/gRPC/EosTestFileInject.cpp

+/*!
+ * @project        The CERN Tape Archive (CTA)
+ * @brief          Inject directory metadata into EOS using gRPC
+ * @copyright      Copyright 2019 CERN
+ * @license        This program is free software: you can redistribute it and/or modify
+ *                 it under the terms of the GNU General Public License as published by
+ *                 the Free Software Foundation, either version 3 of the License, or
+ *                 (at your option) any later version.
+ *
+ *                 This program is distributed in the hope that it will be useful,
+ *                 but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *                 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *                 GNU General Public License for more details.
+ *
+ *                 You should have received a copy of the GNU General Public License
+ *                 along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <iostream>
+#include <sstream>
+
+#include <XrdSsiPbConfig.hpp>
+#include <XrdSsiPbLog.hpp>
+#include "GrpcClient.hpp"
+
+
+
+namespace cta {
+namespace migration {
+
+class EosTestFileInject
+{
+public:
+  EosTestFileInject(const std::string &configfile);
+
+  void inject(const std::string &path, uint64_t fileid);
+
+  std::string ping(const std::string &payload) {
+    return m_eosgrpc->ping(payload);
+  }
+
+  struct Dirname {
+    std::string basename;                                //!< Just the directory name
+    std::string pathname;                                //!< The full path including the directory name
+  };
+
+private:
+  static std::string convertChecksum(uint32_t adler32);
+  Dirname manglePathname(const std::string &pathname);
+
+  std::chrono::steady_clock::time_point m_start_time;    //!< Start the clock
+
+  bool m_is_json;                                        //!< Display results in JSON format for debugging
+  std::unique_ptr<eos::client::GrpcClient> m_eosgrpc;    //!< EOS gRPC API interface
+  std::string m_castor_prefix;                           //!< CASTOR namespace prefix to strip
+  std::string m_eos_prefix;                              //!< EOS namespace prefix to prepend
+
+  std::map<unsigned int, std::string> m_storageClass;    //!< Mapping of CASTOR file class IDs to CTA storage classes
+};
+
+
+EosTestFileInject::EosTestFileInject(const std::string &configfile) :
+  m_start_time(std::chrono::steady_clock::now())
+{
+  // Parse configuration file
+  XrdSsiPb::Config config(configfile);
+
+  auto is_json       = config.getOptionValueBool("castor.json");
+  auto castor_prefix = config.getOptionValueStr("castor.prefix");
+  auto eos_prefix    = config.getOptionValueStr("eos.prefix");
+  auto endpoint      = config.getOptionValueStr("eos.endpoint");
+  auto token         = config.getOptionValueStr("eos.token");
+
+  // Connect to EOS
+  m_eosgrpc = eos::client::GrpcClient::Create(endpoint.first ? endpoint.second : "localhost:50051", token.second);
+
+  // Set parameters and defaults
+  m_is_json       = is_json.first       ? is_json.second       : false;
+  m_castor_prefix = castor_prefix.first ? castor_prefix.second : "/castor/cern.ch";
+  m_eos_prefix    = eos_prefix.first    ? eos_prefix.second    : "/eos/grpc";
+}
+
+
+void EosTestFileInject::inject(const std::string &path, uint64_t fileid)
+{
+  // See EOS common/LayoutId.hh for definitions of constants
+  const int kAdler         =  0x2;
+  const int kReplica       = (0x1 <<  4);
+  const int kStripeSize    = (0x0 <<  8); // 1 stripe
+  const int kStripeWidth   = (0x0 << 16); // 4K blocks
+  const int kBlockChecksum = (0x1 << 20);
+  
+  // Layout id should be 00100012
+  const uint64_t layout = kReplica | kAdler | kStripeSize | kStripeWidth | kBlockChecksum;
+
+  eos::rpc::FileMdProto file;
+
+  file.set_id(fileid);
+  file.set_cont_id(12345);
+  file.set_uid(1000);
+  file.set_gid(1000);
+  file.set_size(42);
+  file.set_layout_id(layout);
+
+  file.set_flags(0755);
+  file.mutable_checksum()->set_type("adler");
+  file.mutable_checksum()->set_value(convertChecksum(0xaabbccdd));
+
+  // Timestamps
+  file.mutable_ctime()->set_sec(1553900400);
+  file.mutable_mtime()->set_sec(1553900400);
+  // we don't care about file.stime (sync time, used for CERNBox)
+  // BTIME is set as an extended attribute (see below)
+
+  // Filename and path
+  file.set_name(path);
+  file.set_path(m_eos_prefix + "/" + file.name());
+  // we don't care about link_name
+
+  // Extended attributes:
+  //
+  // 1. Archive File ID
+  std::string archiveId(std::to_string(file.id()));
+  file.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("CTA_ArchiveFileId", archiveId));
+  // 2. Storage Class
+  file.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("CTA_StorageClass", "my_storage_class"));
+  // 3. Birth Time
+  // POSIX ATIME (Access Time) is used by CASTOR to store the file creation time. EOS calls this "birth time",
+  // but there is no place in the namespace to store it, so it is stored as an extended attribute.
+  file.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("eos.btime", "1553900400"));
+
+  // Indicate that there is a tape-resident replica of this file
+  file.mutable_locations()->Add(65535);
+  // we don't care about unlink_locations (placeholder for files scheduled for deletion)
+
+  std::vector<eos::rpc::FileMdProto> files;
+  files.push_back(file);
+
+  // Put results on stdout for debugging
+  if(m_is_json) {
+    char delim = '[';
+    for(auto &file : files) {
+      std::cout << delim << XrdSsiPb::Log::DumpProtobuf(&file);
+      delim = ',';
+    }
+    if(delim == ',') std::cout << "]";
+  }
+
+  // Inject file into EOS
+  int retc = m_eosgrpc->FileInsert(files);
+  if(retc != 0) {
+    throw std::runtime_error("EosTestFileInject::inject(): FileInsert failed with error " + std::to_string(retc));
+  }
+
+  auto elapsed_time = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - m_start_time);
+  std::cerr << "Processed 1 file in " << elapsed_time.count() << "s" << std::endl;
+}
+
+
+std::string EosTestFileInject::convertChecksum(uint32_t adler32)
+{
+  char bytes[4];
+  for(int i = 3; i >= 0; --i, adler32 >>= 8) { bytes[i] = adler32 & 0xFF; }
+  return std::string(bytes, 4);
+}
+
+
+EosTestFileInject::Dirname EosTestFileInject::manglePathname(const std::string &pathname)
+{
+  Dirname dir;
+
+  size_t clip = (pathname.find(m_castor_prefix) == 0) ? m_castor_prefix.length() : 0;
+  dir.pathname = m_eos_prefix + pathname.substr(clip);
+  clip = dir.pathname.find_last_of('/');
+  dir.basename = (clip == std::string::npos) ? dir.pathname : dir.pathname.substr(clip+1);
+
+  return dir;
+}
+
+}} // namespace cta::migration
+
+
+void throwUsage(const std::string &program, const std::string &error_txt)
+{
+  std::stringstream help;
+
+  help << program << ": " << error_txt << std::endl
+       << "Usage: " << program << " [--config <config_file>] [--fileid <fileid>] ping|--path <path>";
+
+  throw std::runtime_error(help.str());
+}
+
+
+int main(int argc, const char* argv[])
+{
+  std::string configfile = "/etc/cta/castor-migration.conf";
+  std::string path;
+  uint64_t fileid = 0;
+
+  bool doPing = false;
+
+  try {
+    for(auto i = 1; i < argc; ++i) {
+      std::string option(argv[i]);
+
+      if(option == "--config" && argc > ++i) {
+        configfile = argv[i];
+        continue;
+      } else if(option == "--path" && argc > ++i) {
+        path = argv[i];
+        continue;
+      } else if(option == "--fileid" && argc > ++i) {
+        fileid = strtoul(argv[i], NULL, 0);
+        continue;
+      } else if(option == "ping") {
+        doPing = true;
+        continue;
+      }
+      throwUsage(argv[0], "invalid option " + option);
+    }
+    cta::migration::EosTestFileInject testDirInject(configfile);
+
+    if(doPing) {
+      std::cout << "Pinging EOS MGM using gRPC API..." << std::endl;
+      auto pingStr = testDirInject.ping("Ping from EosTestFileInject");
+      std::cout << "Ping successful. Server responded with: " << pingStr << std::endl;
+    } else {
+      if(path.empty()) throwUsage(argv[0], "path not specified");
+      testDirInject.inject(path, fileid);
+    }
+  } catch(std::runtime_error &ex) {
+    std::cerr << ex.what() << std::endl;
+    return -1;
+  }
+  return 0;
+}