From 1244e75b7f474ecb087cc75567a803bcf05bd34d Mon Sep 17 00:00:00 2001 From: Lasse Tjernaes Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 19 Apr 2023 16:39:09 +0200 Subject: [PATCH] Resolve "Create a tool to change the storage class of data which has already been written to tape" --- ReleaseNotes.md | 2 + .../change_storage_class/CMakeLists.txt | 10 +-- .../ChangeStorageClass.cpp | 88 +------------------ .../ChangeStorageClass.hpp | 35 -------- ...ta-change-storage-class-in-catalogue.1cta} | 0 ...ss.sh => changeStorageClassInCatalogue.sh} | 59 +++---------- cta.spec.in | 4 +- 7 files changed, 23 insertions(+), 175 deletions(-) rename cmdline/standalone_cli_tools/change_storage_class/{cta-change-storage-class.1cta => cta-change-storage-class-in-catalogue.1cta} (100%) rename continuousintegration/orchestration/tests/{changeStorageClass.sh => changeStorageClassInCatalogue.sh} (70%) diff --git a/ReleaseNotes.md b/ReleaseNotes.md index b821295493..942502a0bb 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,6 +1,8 @@ # v4.NEXT ### Features +- cta/CTA#78 - Remove gRPC and ability to change the storage class in EOS from the cta-change-storage-class C++ script + ### Bug Fixes - cta/CTA#259 - cta-rmcd should not exit if /dev/sg0 is missing diff --git a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt index 4009a702a4..712efdac20 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt +++ b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt @@ -27,9 +27,9 @@ include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) # Compiled protocol buffers include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) -add_executable(cta-change-storage-class ChangeStorageClass.cpp ChangeStorageClassMain.cpp ../../CtaAdminCmdParse.cpp JsonFileData.cpp) -target_link_libraries(cta-change-storage-class XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient stdc++fs ctaCmdlineToolsCommon) -set_property (TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) +add_executable(cta-change-storage-class-in-catalogue ChangeStorageClass.cpp ChangeStorageClassMain.cpp ../../CtaAdminCmdParse.cpp JsonFileData.cpp) +target_link_libraries(cta-change-storage-class-in-catalogue XrdSsiLib XrdUtils ctacommon stdc++fs ctaCmdlineToolsCommon) +set_property (TARGET cta-change-storage-class-in-catalogue APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) -install(TARGETS cta-change-storage-class DESTINATION usr/bin) -install(FILES cta-change-storage-class.1cta DESTINATION usr/share/man/man1) \ No newline at end of file +install(TARGETS cta-change-storage-class-in-catalogue DESTINATION usr/bin) +install(FILES cta-change-storage-class-in-catalogue.1cta DESTINATION usr/share/man/man1) \ No newline at end of file diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index c926924e5d..f43f9b9a9b 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -35,12 +35,10 @@ #include "cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp" #include "common/checksum/ChecksumBlob.hpp" #include "common/exception/CommandLineNotParsed.hpp" -#include "common/exception/GrpcError.hpp" #include "common/exception/UserError.hpp" #include "common/log/StdoutLogger.hpp" #include "common/utils/utils.hpp" #include "CtaFrontendApi.hpp" -#include "eos_grpc_client/GrpcEndpoint.hpp" // GLOBAL VARIABLES : used to pass information between main thread and stream handler thread @@ -80,14 +78,11 @@ int ChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const auto [serviceProvider, endpointmap] = ConnConfiguration::readAndSetConfiguration(m_log, getUsername(), cmdLineArgs); m_serviceProviderPtr = std::move(serviceProvider); - m_endpointMapPtr = std::move(endpointmap); handleArguments(cmdLineArgs); storageClassExists(); - updateStorageClassInEosNamespace(); updateStorageClassInCatalogue(); - writeSkippedArchiveIdsToFile(); return 0; } @@ -128,68 +123,8 @@ void ChangeStorageClass::handleArguments(const CmdLineArgs &cmdLineArgs) { throw exception::UserError("Archive id does not match with disk file id or disk instance, are you sure the correct file metadata was provided?"); } } - - if (cmdLineArgs.m_frequency) { - m_eosUpdateFrequency = cmdLineArgs.m_frequency.value(); - } else { - m_eosUpdateFrequency = 100; - } } -//------------------------------------------------------------------------------ -// fileInFlight -//------------------------------------------------------------------------------ -bool ChangeStorageClass::fileInFlight(const google::protobuf::RepeatedField<uint32_t> &locations) const { - // file is not in flight if fsid==65535 - return std::all_of(std::begin(locations), std::end(locations), [](const int &location) { return location != 65535; }); -} - -//------------------------------------------------------------------------------ -// updateStorageClassInEosNamespace -//------------------------------------------------------------------------------ -void ChangeStorageClass::updateStorageClassInEosNamespace() { - uint64_t requestCounter = 0; - for(const auto &archiveFileId : m_archiveFileIds) { - requestCounter++; - if(requestCounter >= m_eosUpdateFrequency) { - requestCounter = 0; - sleep(1); - } - - const auto [diskInstance, diskFileId] = CatalogueFetch::getInstanceAndFid(archiveFileId, m_serviceProviderPtr, m_log); - - // No files in flight should change storage class - const bool showJson = false; - if(const auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, cta::utils::toUint64(diskFileId), "", showJson); - fileInFlight(md_response.fmd().locations())) { - if (md_response.fmd().locations().empty()){ - throw ChangeStorageClassError("Metadata from EOS could not be fetched: disk file id " + diskFileId + " does not exist or authentication is failing"); - } - m_archiveIdsNotUpdatedInEos.push_back(archiveFileId); - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("archiveFileId", archiveFileId)); - m_log(cta::log::WARNING, "File did not change storage class because the file was in flight", params); - continue; - } - - const auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId); - { - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("path", path)); - m_log(cta::log::INFO, "Path found", params); - } - - if(auto status = m_endpointMapPtr->setXAttr(diskInstance, path, "sys.archive.storage_class", m_storageClassName); status.ok()) { - m_archiveIdsUpdatedInEos.push_back(archiveFileId); - } else { - m_archiveIdsNotUpdatedInEos.push_back(archiveFileId); - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("archiveFileId", archiveFileId)); - params.push_back(cta::log::Param("error", status.error_message())); - m_log(cta::log::WARNING, "File did not change storage class because query to EOS failed", params); - } - } -} //------------------------------------------------------------------------------ // storageClassExists @@ -267,7 +202,7 @@ void ChangeStorageClass::updateStorageClassInCatalogue() const { const auto key = cta::admin::OptionStrList::FILE_ID; const auto new_opt = admincmd->add_option_str_list(); new_opt->set_key(key); - for (const auto &archiveFileId : m_archiveIdsUpdatedInEos) { + for (const auto &archiveFileId : m_archiveFileIds) { new_opt->add_item(archiveFileId); } } @@ -298,27 +233,6 @@ void ChangeStorageClass::updateStorageClassInCatalogue() const { } } -//------------------------------------------------------------------------------ -// writeSkippedArchiveIdsToFile -//------------------------------------------------------------------------------ -void ChangeStorageClass::writeSkippedArchiveIdsToFile() const { - const std::filesystem::path filePath = "/tmp/skippedArchiveIds.txt"; - std::ofstream archiveIdFile(filePath); - - if (archiveIdFile.fail()) { - throw std::runtime_error("Unable to open file " + filePath.string()); - } - - if (archiveIdFile.is_open()) { - for (const auto& archiveId : m_archiveIdsNotUpdatedInEos) { - archiveIdFile << "{ \"archiveId\" : " << archiveId << " }" << std::endl; - } - archiveIdFile.close(); - std::cout << m_archiveIdsNotUpdatedInEos.size() << " files did not update the storage class." << std::endl; - std::cout << "The skipped archive ids can be found here: " << filePath << std::endl; - } -} - bool ChangeStorageClass::validateUserInputFileMetadata(const std::string& archiveId, const std::string& operatorProvidedFid, const std::string& operatorProvidedInstance) { const auto [diskDiskInstance, diskFileId] = CatalogueFetch::getInstanceAndFid(archiveId, m_serviceProviderPtr, m_log); return ((operatorProvidedInstance == diskDiskInstance) && (operatorProvidedFid == diskFileId)); diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index 1484270d46..2112a0adf8 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -76,31 +76,6 @@ private: */ std::unique_ptr<XrdSsiPbServiceType> m_serviceProviderPtr; - /** - * Endpoint map for interaction with EOS - */ - std::unique_ptr<::eos::client::EndpointMap> m_endpointMapPtr; - - /** - * Endpoint map for interaction with EOS - */ - std::vector<std::string> m_archiveIdsNotUpdatedInEos; - - /** - * Endpoint map for interaction with EOS - */ - std::vector<std::string> m_archiveIdsUpdatedInEos; - - /** - * Endpoint map for interaction with EOS - */ - uint64_t m_eosUpdateFrequency; - - /** - * Updates the storage class name in the EOS namespace - */ - void updateStorageClassInEosNamespace(); - /** * Checks if the storage class provided to the tool is defined, * and throws an exception::UserError if it is not found @@ -112,16 +87,6 @@ private: */ void updateStorageClassInCatalogue() const; - /** - * Writes the skipped archive ids to file - */ - void writeSkippedArchiveIdsToFile() const; - - /** - * Checks if a file is in flight - */ - bool fileInFlight(const google::protobuf::RepeatedField<unsigned int> &locations) const; - /** * Fills the member variables with data, based on the arguments that were provided */ diff --git a/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta b/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class-in-catalogue.1cta similarity index 100% rename from cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta rename to cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class-in-catalogue.1cta diff --git a/continuousintegration/orchestration/tests/changeStorageClass.sh b/continuousintegration/orchestration/tests/changeStorageClassInCatalogue.sh similarity index 70% rename from continuousintegration/orchestration/tests/changeStorageClass.sh rename to continuousintegration/orchestration/tests/changeStorageClassInCatalogue.sh index 05d72f72ae..20e008f068 100755 --- a/continuousintegration/orchestration/tests/changeStorageClass.sh +++ b/continuousintegration/orchestration/tests/changeStorageClassInCatalogue.sh @@ -29,7 +29,6 @@ fi EOSINSTANCE=ctaeos NEW_STORAGE_CLASS_NAME=newStorageClassName -TMP_DIR=$(mktemp -d) FILE_1=`uuidgen` FILE_2=`uuidgen` @@ -37,10 +36,13 @@ echo echo "Creating files: ${FILE_1} ${FILE_2}" FRONTEND_IP=$(kubectl -n ${NAMESPACE} get pods ctafrontend -o json | jq .status.podIP | tr -d '"') + + +############## CREAT TEMPORARY DIRECTORY ############## +TMP_DIR=$(mktemp -d) kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c "mkdir -p ${TMP_DIR}" -echo -echo "ENABLE CTAFRONTEND TO EXECUTE CTA ADMIN COMMANDS" +############## ENABLE CTAFRONTEND TO EXECUTE CTA ADMIN COMMANDS ############## kubectl --namespace=${NAMESPACE} exec kdc -- cat /root/ctaadmin2.keytab | kubectl --namespace=${NAMESPACE} exec -i ctafrontend -- bash -c "cat > /root/ctaadmin2.keytab; mkdir -p /tmp/ctaadmin2" kubectl -n ${NAMESPACE} cp client_helper.sh ctafrontend:/root/client_helper.sh touch ${TMP_DIR}/init_kerb.sh @@ -48,30 +50,24 @@ echo '. /root/client_helper.sh; admin_kinit' >> ${TMP_DIR}/init_kerb.sh kubectl -n ${NAMESPACE} cp ${TMP_DIR}/init_kerb.sh ctafrontend:${TMP_DIR}/init_kerb.sh kubectl -n ${NAMESPACE} exec ctafrontend -- bash ${TMP_DIR}/init_kerb.sh - -echo -echo "ADD FRONTEND GATEWAY TO EOS" -echo "kubectl -n ${NAMESPACE} exec ctaeos -- bash eos root://${EOSINSTANCE} -r 0 0 vid add gateway ${FRONTEND_IP} grpc" -kubectl -n ${NAMESPACE} exec ctaeos -- eos -r 0 0 vid add gateway ${FRONTEND_IP} grpc - -echo -echo "ADD STORAGE CLASS WITH ONE COPIES ${NEW_STORAGE_CLASS_NAME}" +############## ADD STORAGE CLASS WITH TWO COPIES ############## kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc add --name ${NEW_STORAGE_CLASS_NAME} --numberofcopies 2 --virtualorganisation vo --comment "comment" kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc ls -echo -echo "COPY REQUIRED FILES TO FRONTEND POD" +############## COPY REQUIRED FILES TO FRONTEND POD ############## echo "kubectl cp ${NAMESPACE}/ctacli:/etc/cta/cta-cli.conf ${TMP_DIR}/cta-cli.conf" echo "kubectl cp ${TMP_DIR}/cta-cli.conf ${NAMESPACE}/ctafrontend:/etc/cta/cta-cli.conf" kubectl cp ${NAMESPACE}/ctacli:/etc/cta/cta-cli.conf ${TMP_DIR}/cta-cli.conf kubectl cp ${TMP_DIR}/cta-cli.conf ${NAMESPACE}/ctafrontend:/etc/cta/cta-cli.conf +############## ARCHIVE FILES ############## kubectl -n ${NAMESPACE} exec client -- bash -c "mkdir -p ${TMP_DIR}" kubectl -n ${NAMESPACE} cp common/archive_file.sh client:${TMP_DIR}/ kubectl -n ${NAMESPACE} cp client_helper.sh client:/root/ kubectl -n ${NAMESPACE} exec client -- bash ${TMP_DIR}/archive_file.sh -f ${FILE_1} || exit 1 kubectl -n ${NAMESPACE} exec client -- bash ${TMP_DIR}/archive_file.sh -f ${FILE_2} || exit 1 +############## GET EOS AND ARCHVIVE FILE IDS FROM ARCHIVED FILES ############## EOS_METADATA_PATH_1=$(mktemp -d).json echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH}" touch ${EOS_METADATA_PATH_1} @@ -79,7 +75,6 @@ kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /e EOS_ARCHIVE_ID_1=$(jq -r '.xattr | .["sys.archive.file_id"]' ${EOS_METADATA_PATH_1}) EOS_FILE_ID_1=$(jq -r '.id' ${EOS_METADATA_PATH_1}) - EOS_METADATA_PATH_2=$(mktemp -d).json echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH_2}" touch ${EOS_METADATA_PATH_2} @@ -87,35 +82,17 @@ kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /e EOS_ARCHIVE_ID_2=$(jq -r '.xattr | .["sys.archive.file_id"]' ${EOS_METADATA_PATH_2}) EOS_FILE_ID_2=$(jq -r '.id' ${EOS_METADATA_PATH_2}) -echo -echo "CHANGE FILES WITH IDS" -cd ~ +############## CREATE INPUT FILE TO USE FOR TOOL ############## IDS_FILEPATH=${TMP_DIR}/archiveFileIds.txt touch ${IDS_FILEPATH} echo '{"archiveId": '${EOS_ARCHIVE_ID_1}', "fid": '${EOS_FILE_ID_1}', "instance": "'${EOSINSTANCE}'"}' >> ${IDS_FILEPATH} echo '{"archiveId": '${EOS_ARCHIVE_ID_2}', "fid": '${EOS_FILE_ID_2}', "instance": "'${EOSINSTANCE}'"}' >> ${IDS_FILEPATH} -sleep 1 -echo -echo "kubectl cp ${IDS_FILEPATH} ${NAMESPACE}/ctafrontend:${IDS_FILEPATH}" +############## RUN TOOL ############## kubectl cp ${IDS_FILEPATH} ${NAMESPACE}/ctafrontend:${IDS_FILEPATH} -echo "kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c XrdSecPROTOCOL=krb5 KRB5CCNAME=/tmp/ctaadmin2/krb5cc_0 cta-change-storage-class --storageclassname ${NEW_STORAGE_CLASS_NAME} --json ${IDS_FILEPATH}" -kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c "XrdSecPROTOCOL=krb5 KRB5CCNAME=/tmp/ctaadmin2/krb5cc_0 cta-change-storage-class --storageclassname ${NEW_STORAGE_CLASS_NAME} --json ${IDS_FILEPATH} -t 1" - -EOS_METADATA_PATH_AFTER_CHANGE_1=$(mktemp -d).json -echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH_AFTER_CHANGE_1}" -touch ${EOS_METADATA_PATH_AFTER_CHANGE_1} -kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_1} | jq . | tee ${EOS_METADATA_PATH_AFTER_CHANGE_1} -EOS_STORAGE_CLASS_1=$(jq -r '.xattr | .["sys.archive.storage_class"]' ${EOS_METADATA_PATH_AFTER_CHANGE_1}) -rm -r ${EOS_METADATA_PATH_AFTER_CHANGE_1} - -EOS_METADATA_PATH_AFTER_CHANGE_2=$(mktemp -d).json -echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH_AFTER_CHANGE_2}" -touch ${EOS_METADATA_PATH_AFTER_CHANGE_2} -kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_2} | jq . | tee ${EOS_METADATA_PATH_AFTER_CHANGE_2} -EOS_STORAGE_CLASS_2=$(jq -r '.xattr | .["sys.archive.storage_class"]' ${EOS_METADATA_PATH_AFTER_CHANGE_2}) -rm -r ${EOS_METADATA_PATH_AFTER_CHANGE_2} +kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c "XrdSecPROTOCOL=krb5 KRB5CCNAME=/tmp/ctaadmin2/krb5cc_0 cta-change-storage-class-in-catalogue --storageclassname ${NEW_STORAGE_CLASS_NAME} --json ${IDS_FILEPATH} -t 1" +############## CHECK THAT VALUES WERE CHANGED IN THE CATALOGUE ############## CATALOGUE_METADATA_PATH_AFTER_CHANGE_1=$(mktemp -d).json echo "SEND CATALOGUE METADATA TO JSON FILE: ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1}" touch ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1} @@ -130,21 +107,11 @@ kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tf ls --id ${EOS_ARCHIVE CATALOGUE_STORAGE_CLASS_2=$(jq . ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2} | jq '.[0]' | jq -r '.af | .["storageClass"]') rm -r ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2} -if test ${EOS_STORAGE_CLASS_1} != ${NEW_STORAGE_CLASS_NAME}; then - echo "ERROR: File ${FILE_1} did not change the storage class in EOS" - exit 1 -fi - if test ${CATALOGUE_STORAGE_CLASS_1} != ${NEW_STORAGE_CLASS_NAME}; then echo "ERROR: File ${FILE_1} did not change the storage class in the Catalogue" exit 1 fi -if test ${EOS_STORAGE_CLASS_2} != ${NEW_STORAGE_CLASS_NAME}; then - echo "ERROR: File ${FILE_2} did not change the storage class in EOS" - exit 1 -fi - if test ${CATALOGUE_STORAGE_CLASS_2} != ${NEW_STORAGE_CLASS_NAME}; then echo "ERROR: File ${FILE_2} did not change the storage class in the Catalogue" exit 1 diff --git a/cta.spec.in b/cta.spec.in index 1c884c4f71..d56ac49f9a 100644 --- a/cta.spec.in +++ b/cta.spec.in @@ -233,9 +233,9 @@ The command line utilities %attr(0755,root,root) %{_bindir}/cta-send-event %attr(0755,root,root) %{_bindir}/cta-send-closew.sh %attr(0755,root,root) %{_bindir}/cta-verify-file -%attr(0755,root,root) %{_bindir}/cta-change-storage-class +%attr(0755,root,root) %{_bindir}/cta-change-storage-class-in-catalogue %attr(0755,root,root) %{_bindir}/cta-eos-namespace-inject -%attr(0644,root,root) %doc /usr/share/man/man1/cta-change-storage-class.1cta.gz +%attr(0644,root,root) %doc /usr/share/man/man1/cta-change-storage-class-in-catalogue.1cta.gz %attr(0644,root,root) %config(noreplace) %{_sysconfdir}/cta/cta-cli.conf.example %posttrans -n cta-cli -- GitLab