From 54dd8cad1e2867c151ccae3b90affe6e008df666 Mon Sep 17 00:00:00 2001
From: Eric Cano <Eric.Cano@cern.ch>
Date: Wed, 28 Jun 2017 10:45:33 +0200
Subject: [PATCH] Created an objectstore utility cleaning up the root entry
after deleting archive or retrieve queues.
This utility is to be used for cleaning up the pre-prod environement following a schema change in the archive queues.
---
cta.spec.in | 1 +
objectstore/CMakeLists.txt | 8 +-
objectstore/RootEntry.cpp | 8 ++
objectstore/RootEntry.hpp | 4 +
...objectstore-dereference-removed-queues.cpp | 85 +++++++++++++++++++
5 files changed, 105 insertions(+), 1 deletion(-)
create mode 100644 objectstore/cta-objectstore-dereference-removed-queues.cpp
diff --git a/cta.spec.in b/cta.spec.in
index 0ab7b14593..5205d47077 100644
--- a/cta.spec.in
+++ b/cta.spec.in
@@ -246,6 +246,7 @@ Tools allowing initialization and inspection of the object store.
%attr(0755,root,root) %{_bindir}/cta-objectstore-list
%attr(0755,root,root) %{_bindir}/cta-objectstore-dump-object
%attr(0755,root,root) %{_bindir}/cta-objectstore-unfollow-agent
+%attr(0755,root,root) %{_bindir}/cta-objectstore-dereference-removed-queues
#cta-systemtests installs libraries so we need ldconfig.
%post -n cta-systemtests -p /sbin/ldconfig
diff --git a/objectstore/CMakeLists.txt b/objectstore/CMakeLists.txt
index 6e7d0c0b0f..d240ae4679 100644
--- a/objectstore/CMakeLists.txt
+++ b/objectstore/CMakeLists.txt
@@ -106,5 +106,11 @@ set_target_properties(cta-objectstore-unfollow-agent PROPERTIES INSTALL_RPATH ${
target_link_libraries(cta-objectstore-unfollow-agent
${PROTOBUF3_LIBRARIES} ctaobjectstore ctacommon)
-install(TARGETS cta-objectstore-initialize cta-objectstore-list cta-objectstore-dump-object cta-objectstore-unfollow-agent
+add_executable(cta-objectstore-dereference-removed-queues cta-objectstore-dereference-removed-queues.cpp)
+set_target_properties(cta-objectstore-dereference-removed-queues PROPERTIES INSTALL_RPATH ${PROTOBUF3_RPATH})
+target_link_libraries(cta-objectstore-dereference-removed-queues
+ ${PROTOBUF3_LIBRARIES} ctaobjectstore ctacommon)
+
+install(TARGETS cta-objectstore-initialize cta-objectstore-list cta-objectstore-dump-object cta-objectstore-unfollow-agent
+ cta-objectstore-dereference-removed-queues
DESTINATION usr/bin)
diff --git a/objectstore/RootEntry.cpp b/objectstore/RootEntry.cpp
index 6e1890ff6d..52f8c52211 100644
--- a/objectstore/RootEntry.cpp
+++ b/objectstore/RootEntry.cpp
@@ -181,6 +181,10 @@ void RootEntry::removeArchiveQueueAndCommit(const std::string& tapePool) {
}
}
+void RootEntry::removeMissingArchiveQueueReference(const std::string& tapePool) {
+ serializers::removeOccurences(m_payload.mutable_archivequeuepointers(), tapePool);
+}
+
std::string RootEntry::getArchiveQueueAddress(const std::string& tapePool) {
checkPayloadReadable();
try {
@@ -248,6 +252,10 @@ std::string RootEntry::addOrGetRetrieveQueueAndCommit(const std::string& vid, Ag
return retrieveQueueAddress;
}
+void RootEntry::removeMissingRetrieveQueueReference(const std::string& vid) {
+ serializers::removeOccurences(m_payload.mutable_retrievequeuepointers(), vid);
+}
+
void RootEntry::removeRetrieveQueueAndCommit(const std::string& vid) {
checkPayloadWritable();
// find the address of the retrieve queue object
diff --git a/objectstore/RootEntry.hpp b/objectstore/RootEntry.hpp
index 0215a41cfe..7ca3d78be5 100644
--- a/objectstore/RootEntry.hpp
+++ b/objectstore/RootEntry.hpp
@@ -67,6 +67,8 @@ public:
* Fails if it not empty*/
CTA_GENERATE_EXCEPTION_CLASS(NoSuchArchiveQueue);
void removeArchiveQueueAndCommit(const std::string & tapePool);
+ /** This function is used in a cleanup utility. Removes unconditionally the reference to the archive queue */
+ void removeMissingArchiveQueueReference(const std::string & tapePool);
void removeArchiveQueueIfAddressMatchesAndCommit(const std::string & tapePool, const std::string & archiveQueueAddress);
std::string getArchiveQueueAddress(const std::string & tapePool);
struct ArchiveQueueDump {
@@ -80,6 +82,8 @@ public:
/** This function implicitly creates the retrieve queue structure and updates
* the pointer to it. It will implicitly commit the object to the store. */
std::string addOrGetRetrieveQueueAndCommit(const std::string & vid, AgentReference & agentRef);
+ /** This function is used in a cleanup utility. Removes unconditionally the reference to the retrieve queue */
+ void removeMissingRetrieveQueueReference(const std::string & address);
CTA_GENERATE_EXCEPTION_CLASS(NoSuchRetrieveQueue);
void removeRetrieveQueueAndCommit(const std::string & vid);
std::string getRetrieveQueue(const std::string & vid);
diff --git a/objectstore/cta-objectstore-dereference-removed-queues.cpp b/objectstore/cta-objectstore-dereference-removed-queues.cpp
new file mode 100644
index 0000000000..a4667c4368
--- /dev/null
+++ b/objectstore/cta-objectstore-dereference-removed-queues.cpp
@@ -0,0 +1,85 @@
+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * This program will make sure every queue listed in the root entry does exist and
+ * will remove reference for the ones that do not. This utility was created to quickly
+ * unblock tape servers after changing the ArchiveQueue schema during development.
+ */
+
+#include "BackendFactory.hpp"
+#include "BackendVFS.hpp"
+#include "Agent.hpp"
+#include "RootEntry.hpp"
+#include "AgentRegister.hpp"
+#include <iostream>
+#include <stdexcept>
+
+int main(int argc, char ** argv) {
+ try {
+ std::unique_ptr<cta::objectstore::Backend> be;
+ if (2 == argc) {
+ be.reset(cta::objectstore::BackendFactory::createBackend(argv[1]).release());
+ } else {
+ throw std::runtime_error("Wrong number of arguments: expected 1");
+ }
+ // If the backend is a VFS, make sure we don't delete it on exit.
+ // If not, nevermind.
+ try {
+ dynamic_cast<cta::objectstore::BackendVFS &>(*be).noDeleteOnExit();
+ } catch (std::bad_cast &){}
+ std::cout << "Object store path: " << be->getParams()->toURL() << std::endl;
+ // Open the root entry RW
+ cta::objectstore::RootEntry re(*be);
+ cta::objectstore::ScopedExclusiveLock rel(re);
+ re.fetch();
+ std::list<std::string> missingArchiveQueues, missingRetrieveQueues;
+ for (auto & aq: re.dumpArchiveQueues()) {
+ if (!be->exists(aq.address)) {
+ missingArchiveQueues.emplace_back(aq.tapePool);
+ std::cout << "The archive queue for tape pool " << aq.tapePool << " at address " << aq.address
+ << " is missing and will be dereferenced." << std::endl;
+ }
+ }
+ for (auto & rq: re.dumpRetrieveQueues()) {
+ if (!be->exists(rq.address)) {
+ missingRetrieveQueues.emplace_back(rq.vid);
+ std::cout << "The retrieve queue for vid " << rq.vid << " at address " << rq.address
+ << " is missing and will be dereferenced." << std::endl;
+ }
+ }
+ // Actually do the job
+ for (auto & tp: missingArchiveQueues) {
+ re.removeMissingArchiveQueueReference(tp);
+ std::cout << "Archive queue for tape pool " << tp << "dereferenced." << std::endl;
+ }
+ for (auto & vid: missingRetrieveQueues) {
+ re.removeMissingRetrieveQueueReference(vid);
+ std::cout << "Retrieve queue for vid " << vid << "dereferenced." << std::endl;
+ }
+ if (missingArchiveQueues.size() || missingRetrieveQueues.size()) {
+ re.commit();
+ std::cout << "Root entry committed." << std::endl;
+ } else {
+ std::cout << "Nothing to clean up from root entry." << std::endl;
+ }
+ } catch (std::exception & e) {
+ std::cerr << "Failed to cleanup root entry: "
+ << std::endl << e.what() << std::endl;
+ }
+}
\ No newline at end of file
--
GitLab