Created an objectstore utility cleaning up the root entry after deleting...

Created an objectstore utility cleaning up the root entry after deleting archive or retrieve queues.

This utility is to be used for cleaning up the pre-prod environement following a schema change in the archive queues.

cta.spec.in

@@ -246,6 +246,7 @@ Tools allowing initialization and inspection of the object store.
 %attr(0755,root,root) %{_bindir}/cta-objectstore-list
 %attr(0755,root,root) %{_bindir}/cta-objectstore-dump-object
 %attr(0755,root,root) %{_bindir}/cta-objectstore-unfollow-agent
+%attr(0755,root,root) %{_bindir}/cta-objectstore-dereference-removed-queues
 
 #cta-systemtests installs libraries so we need ldconfig.
 %post -n cta-systemtests -p /sbin/ldconfig

objectstore/CMakeLists.txt

@@ -106,5 +106,11 @@ set_target_properties(cta-objectstore-unfollow-agent PROPERTIES INSTALL_RPATH ${
 target_link_libraries(cta-objectstore-unfollow-agent
   ${PROTOBUF3_LIBRARIES} ctaobjectstore ctacommon)
 
-install(TARGETS cta-objectstore-initialize cta-objectstore-list cta-objectstore-dump-object cta-objectstore-unfollow-agent
+add_executable(cta-objectstore-dereference-removed-queues cta-objectstore-dereference-removed-queues.cpp)
+set_target_properties(cta-objectstore-dereference-removed-queues PROPERTIES INSTALL_RPATH ${PROTOBUF3_RPATH})
+target_link_libraries(cta-objectstore-dereference-removed-queues
+  ${PROTOBUF3_LIBRARIES} ctaobjectstore ctacommon)
+
+install(TARGETS cta-objectstore-initialize cta-objectstore-list cta-objectstore-dump-object cta-objectstore-unfollow-agent 
+  cta-objectstore-dereference-removed-queues
   DESTINATION usr/bin)

objectstore/RootEntry.cpp

@@ -181,6 +181,10 @@ void RootEntry::removeArchiveQueueAndCommit(const std::string& tapePool) {
   }
 }
 
+void RootEntry::removeMissingArchiveQueueReference(const std::string& tapePool) {
+  serializers::removeOccurences(m_payload.mutable_archivequeuepointers(), tapePool);
+}
+
 std::string RootEntry::getArchiveQueueAddress(const std::string& tapePool) {
   checkPayloadReadable();
   try {
@@ -248,6 +252,10 @@ std::string RootEntry::addOrGetRetrieveQueueAndCommit(const std::string& vid, Ag
   return retrieveQueueAddress;
 }
 
+void RootEntry::removeMissingRetrieveQueueReference(const std::string& vid) {
+  serializers::removeOccurences(m_payload.mutable_retrievequeuepointers(), vid);
+}
+
 void RootEntry::removeRetrieveQueueAndCommit(const std::string& vid) {
   checkPayloadWritable();
   // find the address of the retrieve queue object

objectstore/RootEntry.hpp

@@ -67,6 +67,8 @@ public:
    * Fails if it not empty*/
   CTA_GENERATE_EXCEPTION_CLASS(NoSuchArchiveQueue);
   void removeArchiveQueueAndCommit(const std::string & tapePool);
+  /** This function is used in a cleanup utility. Removes unconditionally the reference to the archive queue */
+  void removeMissingArchiveQueueReference(const std::string & tapePool);
   void removeArchiveQueueIfAddressMatchesAndCommit(const std::string & tapePool, const std::string & archiveQueueAddress);
   std::string getArchiveQueueAddress(const std::string & tapePool);
   struct ArchiveQueueDump {
@@ -80,6 +82,8 @@ public:
   /** This function implicitly creates the retrieve queue structure and updates 
    * the pointer to it. It will implicitly commit the object to the store. */
   std::string addOrGetRetrieveQueueAndCommit(const std::string & vid, AgentReference & agentRef);
+  /** This function is used in a cleanup utility. Removes unconditionally the reference to the retrieve queue */
+  void removeMissingRetrieveQueueReference(const std::string & address);
   CTA_GENERATE_EXCEPTION_CLASS(NoSuchRetrieveQueue);
   void removeRetrieveQueueAndCommit(const std::string & vid);
   std::string getRetrieveQueue(const std::string & vid);

objectstore/cta-objectstore-dereference-removed-queues.cpp

+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015  CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * This program will make sure every queue listed in the root entry does exist and 
+ * will remove reference for the ones that do not. This utility was created to quickly
+ * unblock tape servers after changing the ArchiveQueue schema during development.
+ */
+
+#include "BackendFactory.hpp"
+#include "BackendVFS.hpp"
+#include "Agent.hpp"
+#include "RootEntry.hpp"
+#include "AgentRegister.hpp"
+#include <iostream>
+#include <stdexcept>
+
+int main(int argc, char ** argv) {
+  try {
+    std::unique_ptr<cta::objectstore::Backend> be;
+    if (2 == argc) {
+      be.reset(cta::objectstore::BackendFactory::createBackend(argv[1]).release());
+    } else {
+      throw std::runtime_error("Wrong number of arguments: expected 1");
+    }
+    // If the backend is a VFS, make sure we don't delete it on exit.
+    // If not, nevermind.
+    try {
+      dynamic_cast<cta::objectstore::BackendVFS &>(*be).noDeleteOnExit();
+    } catch (std::bad_cast &){}
+    std::cout << "Object store path: " << be->getParams()->toURL() << std::endl;
+    // Open the root entry RW
+    cta::objectstore::RootEntry re(*be);
+    cta::objectstore::ScopedExclusiveLock rel(re);
+    re.fetch();
+    std::list<std::string> missingArchiveQueues, missingRetrieveQueues;
+    for (auto & aq: re.dumpArchiveQueues()) {
+      if (!be->exists(aq.address)) {
+        missingArchiveQueues.emplace_back(aq.tapePool);
+        std::cout << "The archive queue for tape pool " << aq.tapePool << " at address " << aq.address
+            << " is missing and will be dereferenced." << std::endl;
+      }
+    }
+    for (auto & rq: re.dumpRetrieveQueues()) {
+      if (!be->exists(rq.address)) {
+        missingRetrieveQueues.emplace_back(rq.vid);
+        std::cout << "The retrieve queue for vid " << rq.vid << " at address " << rq.address
+            << " is missing and will be dereferenced." << std::endl;
+      }
+    }
+    // Actually do the job
+    for (auto & tp: missingArchiveQueues) {
+      re.removeMissingArchiveQueueReference(tp);
+      std::cout << "Archive queue for tape pool " << tp << "dereferenced." << std::endl;
+    }
+    for (auto & vid: missingRetrieveQueues) {
+      re.removeMissingRetrieveQueueReference(vid);
+      std::cout << "Retrieve queue for vid " << vid << "dereferenced." << std::endl;
+    }
+    if (missingArchiveQueues.size() || missingRetrieveQueues.size()) {
+      re.commit();
+      std::cout << "Root entry committed." << std::endl;
+    } else {
+      std::cout << "Nothing to clean up from root entry." << std::endl;
+    }
+  } catch (std::exception & e) {
+    std::cerr << "Failed to cleanup root entry: "
+        << std::endl << e.what() << std::endl;
+  }
+}
\ No newline at end of file