cta/CTA#186 CTA should cope with idle database connections being dropped by the database server

OracleCatalogue::filesWrittenToTape() should now transparently re-connect to the database.

catalogue/OracleCatalogue.cpp

@@ -18,8 +18,9 @@
 
 #include "catalogue/ArchiveFileRow.hpp"
 #include "catalogue/OracleCatalogue.hpp"
-#include "common/exception/UserError.hpp"
 #include "common/exception/Exception.hpp"
+#include "common/exception/LostDatabaseConnection.hpp"
+#include "common/exception/UserError.hpp"
 #include "common/make_unique.hpp"
 #include "common/threading/MutexLocker.hpp"
 #include "common/Timer.hpp"
@@ -591,6 +592,36 @@ common::dataStructures::Tape OracleCatalogue::selectTapeForUpdate(rdbms::Conn &c
 // filesWrittenToTape
 //------------------------------------------------------------------------------
 void OracleCatalogue::filesWrittenToTape(const std::set<TapeFileWritten> &events) {
+  try {
+    const uint32_t maxTries = 3;
+    for (uint32_t tryNb = 1; tryNb <= maxTries; tryNb++) {
+      try {
+        filesWrittenToTapeInternal(events);
+      } catch (exception::LostDatabaseConnection &lc) {
+        // Ignore lost connection
+        std::list<log::Param> params = {
+          {"maxTries", maxTries},
+          {"tryNb", tryNb},
+          {"msg", lc.getMessage()}
+        };
+        m_log(cta::log::WARNING, "Lost database connection", params);
+      }
+    }
+
+    exception::Exception ex;
+    ex.getMessage() << " failed: Lost the database connection after trying " << maxTries << " times";
+    throw ex;
+  } catch(exception::Exception &ex) {
+    throw exception::Exception(std::string(__FUNCTION__) +  " failed: " + ex.getMessage().str());
+  } catch(std::exception &se) {
+    throw exception::Exception(std::string(__FUNCTION__) + " failed: " + se.what());
+  }
+}
+
+//------------------------------------------------------------------------------
+// filesWrittenToTapeInternal
+//------------------------------------------------------------------------------
+void OracleCatalogue::filesWrittenToTapeInternal(const std::set<TapeFileWritten> &events) {
   try {
     if (events.empty()) {
       return;
@@ -692,6 +723,8 @@ void OracleCatalogue::filesWrittenToTape(const std::set<TapeFileWritten> &events
 
     conn.commit();
 
+  } catch(exception::LostDatabaseConnection &) {
+    throw;
   } catch(exception::Exception &ex) {
     throw exception::Exception(std::string(__FUNCTION__) +  " failed: " + ex.getMessage().str());
   } catch(std::exception &se) {

catalogue/OracleCatalogue.hpp

@@ -127,6 +127,16 @@ public:
 
 private:
 
+  /**
+   * Notifies the catalogue that the specified files have been written to tape.
+   *
+   * This internal method can be re-tried if it throws a LostDatabaseConnection
+   * exception.
+   *
+   * @param events The tape file written events.
+   */
+  void filesWrittenToTapeInternal(const std::set<TapeFileWritten> &events);
+
   /**
    * Selects the specified tape within the Tape table for update.
    *