Renamed files and started implermenting middletier for object store.

The implementation is now good enough to run the unit tests (they do not pass yet).

middletier/CMakeLists.txt

@@ -25,8 +25,8 @@ add_library (ctamiddletiersqliteunittests SHARED
 include_directories(${CMAKE_BINARY_DIR})
 
 add_library(ctamiddletierobjectstore
-  objectstore/ObjectStoreMiddleTierAdmin.cpp
-  objectstore/ObjectStoreMiddleTierUser.cpp)
+  objectstore/OStoreMiddleTierAdmin.cpp
+  objectstore/OStoreMiddleTierUser.cpp)
 
 add_library (ctamiddletierobjectstoreunittests SHARED
-  objectstore/ObjectStoreMiddleTierTest.cpp)
+  objectstore/OStoreMiddleTierTest.cpp)

middletier/objectstore/ObjectStoreMiddleTierAdmin.cpp → middletier/objectstore/OStoreMiddleTierAdmin.cpp

@@ -17,7 +17,7 @@
  */
 
 #include "common/exception/Exception.hpp"
-#include "middletier/objectstore/ObjectStoreMiddleTierAdmin.hpp"
+#include "middletier/objectstore/OStoreMiddleTierAdmin.hpp"
 #include "objectstore/Backend.hpp"
 #include "objectstore/RootEntry.hpp"
 #include "objectstore/AdminUsersList.hpp"
@@ -27,8 +27,9 @@ namespace cta {
 //------------------------------------------------------------------------------
 // constructor
 //------------------------------------------------------------------------------
-OStoreMiddleTierAdmin::OStoreMiddleTierAdmin(objectstore::Backend& backend):
-  m_backend(backend) {
+OStoreMiddleTierAdmin::OStoreMiddleTierAdmin(objectstore::Backend& backend,
+  objectstore::Agent & agent):
+  m_backend(backend), m_agent(agent) {
   // check that we can at least access the root entry
   objectstore::RootEntry re(m_backend);
   objectstore::ScopedSharedLock reLock(re);
@@ -109,7 +110,14 @@ void OStoreMiddleTierAdmin::createStorageClass(
   const std::string &name,
   const uint16_t nbCopies,
   const std::string &comment) {
-  throw cta::exception::Exception("TODO");
+  // Get the root entry
+  objectstore::RootEntry re(m_backend);
+  {
+    objectstore::ScopedSharedLock sl(re);
+    re.fetch();
+  }
+  // Get the storage class list
+  re.allocateOrGetStorageClassList(m_agent);
 }
 
 void OStoreMiddleTierAdmin::deleteStorageClass(

middletier/objectstore/ObjectStoreMiddleTierAdmin.hpp → middletier/objectstore/OStoreMiddleTierAdmin.hpp

@@ -24,6 +24,7 @@ namespace cta {
 
 namespace objectstore {
   class Backend;
+  class Agent;
 }
   
 /**
@@ -35,7 +36,8 @@ public:
   /**
    * Constructor
    */
-  OStoreMiddleTierAdmin(objectstore::Backend & backend);
+  OStoreMiddleTierAdmin(objectstore::Backend & backend,
+    objectstore::Agent & agent);
   
   /**
    * Destructor
@@ -301,6 +303,11 @@ private:
    * Reference to the backend used for storing objects
    */
   objectstore::Backend & m_backend;
+  
+  /**
+   * Reference to the agent register entry for this execution
+   */
+  objectstore::Agent & m_agent;
 };
 
 }

middletier/objectstore/ObjectStoreMiddleTierTest.cpp → middletier/objectstore/OStoreMiddleTierTest.cpp

@@ -17,18 +17,20 @@
  */
 
 #include "middletier/sharedtest/MiddleTierAbstractTest.hpp"
-#include "middletier/objectstore/ObjectStoreMiddleTierAdmin.hpp"
-#include "middletier/objectstore/ObjectStoreMiddleTierUser.hpp"
+#include "middletier/objectstore/OStoreMiddleTierAdmin.hpp"
+#include "middletier/objectstore/OStoreMiddleTierUser.hpp"
 #include "objectstore/BackendVFS.hpp"
 #include "objectstore/RootEntry.hpp"
+#include "objectstore/Agent.hpp"
 #include "scheduler/MiddleTierAdmin.hpp"
 #include "scheduler/MiddleTierUser.hpp"
+#include "nameserver/MockNameServer.hpp"
 
 namespace unitTests {
   
-class InitializedVFS: public cta::objectstore::BackendVFS {
+class InitializedVFSOStore: public cta::objectstore::BackendVFS {
 public:
-  InitializedVFS(): cta::objectstore::BackendVFS() {
+  InitializedVFSOStore(): cta::objectstore::BackendVFS() {
     // Create the root entry
     cta::objectstore::RootEntry re(*this);
     re.initialize();
@@ -38,11 +40,17 @@ public:
 
 class OStoreVfsMiddleTier: public localMiddleTier {
 public:
-  OStoreVfsMiddleTier(): m_vfs(), m_admin(m_vfs), m_user(m_vfs) {}
+  OStoreVfsMiddleTier(): m_vfsOStore(), m_agent(m_vfsOStore),
+    m_mockNameServer(),
+    m_admin(m_vfsOStore, m_agent), m_user(m_vfsOStore, m_mockNameServer) {
+    m_agent.generateName("OStoreVfsMiddleTier");
+  }
   virtual cta::MiddleTierAdmin & admin () { return m_admin; }
   virtual cta::MiddleTierUser & user () { return m_user; }
 private:
-  InitializedVFS m_vfs;
+  InitializedVFSOStore m_vfsOStore;
+  cta::objectstore::Agent m_agent;
+  cta::MockNameServer m_mockNameServer;
   cta::OStoreMiddleTierAdmin m_admin;
   cta::OStoreMiddleTierUser m_user;
 };

middletier/objectstore/ObjectStoreMiddleTierUser.cpp → middletier/objectstore/OStoreMiddleTierUser.cpp

@@ -17,58 +17,59 @@
  */
 
 #include "common/exception/Exception.hpp"
-#include "middletier/objectstore/ObjectStoreMiddleTierUser.hpp"
+#include "middletier/objectstore/OStoreMiddleTierUser.hpp"
 #include "objectstore/Backend.hpp"
 #include "objectstore/RootEntry.hpp"
+#include "nameserver/NameServer.hpp"
 
 namespace cta {
 
-OStoreMiddleTierUser::OStoreMiddleTierUser(objectstore::Backend& backend):
-  m_backend(backend) {}
+OStoreMiddleTierUser::OStoreMiddleTierUser(objectstore::Backend& backend,
+    NameServer & nameserver):
+  m_backend(backend), m_nameserver(nameserver) {}
 
 OStoreMiddleTierUser::~OStoreMiddleTierUser() throw() { }
 
 void OStoreMiddleTierUser::createDir(
   const SecurityIdentity& requester,
   const std::string& dirPath) {
-  throw cta::exception::Exception("TODO");
-
+  m_nameserver.createDir(requester, dirPath, 0777);
 }
 void OStoreMiddleTierUser::deleteDir(
   const SecurityIdentity& requester,
   const std::string& dirPath) {
-  throw cta::exception::Exception("TODO");
+  m_nameserver.deleteDir(requester, dirPath);
 }
 
 DirIterator OStoreMiddleTierUser::getDirContents(
   const SecurityIdentity& requester,
   const std::string& dirPath) const {
-  throw cta::exception::Exception("TODO");
+  return m_nameserver.getDirContents(requester, dirPath);
 }
 
 DirEntry OStoreMiddleTierUser::stat(
   const SecurityIdentity& requester,
   const std::string path) const {
-  throw cta::exception::Exception("TODO");
+  return m_nameserver.statDirEntry(requester, path);
 }
 
 void OStoreMiddleTierUser::setDirStorageClass(
   const SecurityIdentity& requester,
   const std::string& dirPath,
   const std::string& storageClassName) {
-  throw cta::exception::Exception("TODO");
+  m_nameserver.setDirStorageClass(requester, dirPath, storageClassName);
 }
 
 void OStoreMiddleTierUser::clearDirStorageClass(
   const SecurityIdentity& requester,
   const std::string& dirPath) {
-  throw cta::exception::Exception("TODO");
+  m_nameserver.clearDirStorageClass(requester, dirPath);
 }
 
 std::string OStoreMiddleTierUser::getDirStorageClass(
   const SecurityIdentity& requester,
   const std::string& dirPath) const {
-  throw cta::exception::Exception("TODO");
+  return m_nameserver.getDirStorageClass(requester, dirPath);
 }
 
 void OStoreMiddleTierUser::archive(

middletier/objectstore/ObjectStoreMiddleTierUser.hpp → middletier/objectstore/OStoreMiddleTierUser.hpp

@@ -22,10 +22,13 @@
 
 namespace cta {
 
+// Forward declaration of classes for references.
 namespace objectstore {
   class Backend;
 }
 
+class NameServer;
+
 /**
  * The user API of the the middle-tier.
  */
@@ -34,7 +37,7 @@ public:
   /**
    * Constructor
    */
-  OStoreMiddleTierUser(objectstore::Backend & backend);
+  OStoreMiddleTierUser(objectstore::Backend & backend, NameServer & nameserver);
   
   /**
    * Destructor
@@ -245,6 +248,11 @@ private:
    * Reference to the backend used for storing objects
    */
   objectstore::Backend & m_backend;
+  
+  /**
+   * Reference to the name server  for 
+   */
+  NameServer & m_nameserver;
 
 }; // class OStoreMiddleTierUser
 

objectstore/RootEntry.cpp

@@ -204,6 +204,68 @@ void cta::objectstore::RootEntry::setAdminUsersList(const std::string& name) {
   throw cta::exception::Exception("TODO");
 }
 
+// Get the name of the admin user list (or exception if not available)
+std::string cta::objectstore::RootEntry::getStorageClassList() {
+  // Check that the fetch was done
+  if (!m_payloadInterpreted)
+    throw ObjectOpsBase::NotFetched("In RootEntry::getStorageClassList: object not yet fetched");
+  // If the registry is defined, return it, job done.
+  if (m_payload.storageclasslist().size())
+    return m_payload.storageclasslist();
+  throw NotAllocatedEx("In RootEntry::getStorageClassList: StorageClassList not yet allocated");
+}
+
+std::string cta::objectstore::RootEntry::allocateOrGetStorageClassList(Agent& agent) {
+//  // Check if the job pool exists
+//  try {
+//    return getStorageClassList();
+//  } catch (NotAllocatedEx &) {
+//    // If we get here, the job pool is not created yet, so we have to do it:
+//    // lock the entry again, for writing
+//    ScopedExclusiveLock lock(*this);
+//    fetch();
+//    // If the registry is already defined, somebody was faster. We're done.
+//    if (m_payload.storageclasslist().size()) {
+//      lock.release();
+//      return m_payload.storageclasslist();
+//    }
+//    // We will really create the register
+//    // decide on the object's name
+//    std::string sclName (agent.nextId("storageClassList"));
+//    // Record the agent in the intent log
+//    addIntendedStorageClassList(sclName);
+//    // Create and populate the object
+//    StorageClassList scl(sclName, m_objectStore);
+//    scl.initialize();
+//    scl.setOwner("");
+//    scl.setBackupOwner("");
+//    scl.insert();
+//    // Take a lock on the newly created job pool
+//    ScopedExclusiveLock sclLock(scl);
+//    // Move job pool from intent to official
+//    setStorageClassList(scl);
+//    commit();
+//    // Record completion on the job pool
+//    scl.setOwner(getNameIfSet());
+//    scl.setBackupOwner(getNameIfSet());
+//    scl.commit();
+//    // and we are done
+//    return scl;
+//  }
+  throw exception::Exception("TODO");
+}
+
+void cta::objectstore::RootEntry::addIntendedStorageClassList(const std::string& name) {
+  throw cta::exception::Exception("TODO");
+}
+
+void cta::objectstore::RootEntry::deleteFromIntendedStorageClassList(const std::string& name) {
+  throw cta::exception::Exception("TODO");
+}
+
+void cta::objectstore::RootEntry::setStorageClassList(const std::string& name) {
+  throw cta::exception::Exception("TODO");
+}
 
 // Dump the root entry
 std::string cta::objectstore::RootEntry::dump () {

objectstore/RootEntry.hpp

@@ -81,6 +81,21 @@ private:
   
   void setAdminUsersList(const std::string & name);
   
+public:
+  // Get the name of the StorageClassList (or exception if not available)
+  std::string getStorageClassList();
+  
+  // Get the name of a (possibly freshly created) StorageClassList
+  std::string allocateOrGetStorageClassList(Agent & agent);
+  
+private:
+  void addIntendedStorageClassList(const std::string & name);
+  
+  void deleteFromIntendedStorageClassList(const std::string & name);
+  
+  void setStorageClassList(const std::string & name);
+  
+  
 public:
   // Dump the root entry
   std::string dump ();

objectstore/StorageClassList.hpp

+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2015  CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "Backend.hpp"
+#include "ObjectOps.hpp"
+#include <sstream>
+
+namespace cta { namespace objectstore {
+
+class StorageClassList: public ObjectOps<serializers::JobPool> {
+public:
+  StorageClassList(const std::string & name, Backend & os):
+  ObjectOps<serializers::StorageClassList>(os, name) {}
+  
+  void addStorageClass (
+    const std::string & name,
+    
+    Agent & agent) {
+    throw cta::exception::Exception("TODO");
+  }
+  
+  uint32_t getStorageClassCount
+  
+  class NotAllocatedEx: public cta::exception::Exception {
+  public:
+    NotAllocatedEx(const std::string & context): cta::exception::Exception(context) {}
+  };
+  
+  std::string dump(Agent & agent) {
+    checkPayloadReadable();
+    std::stringstream ret;
+    ret << "<<<< StorageClassList " << getNameIfSet() << " dump start" << std::endl
+        << "Migration=" << m_payload.migration() << std::endl
+        << "Recall=" << m_payload.recall() << std::endl
+        << "RecallCounter=" << m_payload.recallcounter() << std::endl;
+    ret << ">>>> JobPool " << getNameIfSet() << " dump end" << std::endl;
+    return ret.str();
+  }
+  
+  std::string getRecallFIFO () {
+    checkPayloadReadable();
+    if (m_payload.recall().size())
+      return m_payload.recall();
+    throw NotAllocatedEx("In RootEntry::getJobPool: jobPool not yet allocated");
+  }
+  
+  // Get the name of a (possibly freshly created) recall FIFO
+  std::string allocateOrGetRecallFIFO(Agent & agent) {
+    // Check if the job pool exists
+    try {
+      return getRecallFIFO();
+    } catch (NotAllocatedEx &) {
+      throw;
+      /*// If we get here, the job pool is not created yet, so we have to do it:
+      // lock the entry again, for writing
+      serializers::JobPool res;
+      ContextHandle & ctx = agent.getFreeContext();
+      lockExclusiveAndRead(res, ctx, __func__);
+      // If the registry is already defined, somebody was faster. We're done.
+      if (res.recall().size()) {
+        unlock(ctx);
+        return res.recall();
+      }
+      // We will really create the register
+      // decide on the object's name
+      std::string FIFOName (agent.nextId("recallFIFO"));
+      // Record the FIFO in the intent log
+      agent.addToIntend(selfName(), FIFOName, serializers::RecallFIFO_t);
+      // The potential object can now be garbage collected if we die from here.
+      // Create the object, then lock. The name should be unique, so no race.
+      serializers::FIFO rfs;
+      rfs.set_readpointer(0);
+      writeChild(FIFOName, rfs);
+      // If we lived that far, we can update the jop pool to point to the FIFO
+      res.set_recall(FIFOName);
+      agent.removeFromIntent(selfName(), FIFOName, serializers::RecallFIFO_t);
+      write(res);
+      // release the lock, and return the register name
+      unlock(ctx);
+      return FIFOName;*/
+    }
+  }
+  
+  std::string getRecallCounter (Agent & agent) {
+    checkPayloadReadable();
+    if (m_payload.recallcounter().size())
+      return m_payload.recallcounter();
+    throw NotAllocatedEx("In RootEntry::getRecallCounter: recallCounter not yet allocated");
+  }
+  
+  std::string allocateOrGetRecallCounter(Agent & agent) {
+    // Check if the counter exists
+    try {
+      return getRecallCounter(agent);
+    } catch (NotAllocatedEx &) {
+      throw;/*
+      // If we get here, the job pool is not created yet, so we have to do it:
+      // lock the entry again, for writing
+      serializers::JobPool res;
+      ContextHandle & ctx = agent.getFreeContext();
+      lockExclusiveAndRead(res, ctx, __func__);
+      // If the registry is already defined, somebody was faster. We're done.
+      if (res.recallcounter().size()) {
+        unlock(ctx);
+        return res.recallcounter();
+      }
+      // We will really create the register
+      // decide on the object's name
+      std::string recallCounterName (agent.nextId("recallCounter"));
+      // Record the FIFO in the intent log
+      agent.addToIntend(selfName(), recallCounterName, serializers::RecallFIFO_t);
+      // The potential object can now be garbage collected if we die from here.
+      // Create the object, then lock. The name should be unique, so no race.
+      serializers::Counter cs;
+      cs.set_count(0);
+      writeChild(recallCounterName, cs);
+      // If we lived that far, we can update the jop pool to point to the FIFO
+      res.set_recallcounter(recallCounterName);
+      agent.removeFromIntent(selfName(), recallCounterName, serializers::RecallFIFO_t);
+      write(res);
+      // release the lock, and return the register name
+      unlock(ctx);
+      return recallCounterName;*/
+    }
+  }
+  
+  // The following functions are hidden from the user in order to provide
+  // higher level functionnality
+  //std::string allocateOrGetMigrationFIFO
+  
+};
+
+}}

objectstore/cta.proto

@@ -13,6 +13,7 @@ enum ObjectType {
   Counter_t = 7;
   FIFO_t = 8;
   AdminUsersList_t = 9;
+  StorageClassList_t = 10;
   GenericObject_t = 1000;
 }
 
@@ -65,24 +66,26 @@ message Agent {
 // should be very rare (besides first register creation). All other objects
 // can be covered by the agents's ownership logs.
 message RootEntry {
-  optional string agentregister = 91;
-  repeated string agentregisterintentlog = 92;
-  optional string jobpool = 93;
-  repeated string jobpoolintentlog = 94;
-  optional string adminuserslist = 95;
-  repeated string adminuserslistintentlog = 96;
+  optional string agentregister = 100;
+  repeated string agentregisterintentlog = 101;
+  optional string jobpool = 102;
+  repeated string jobpoolintentlog = 103;
+  optional string adminuserslist = 104;
+  repeated string adminuserslistintentlog = 105;
+  optional string storageclasslist = 106;
+  repeated string storageclasslistintentlog = 107;
 }
 
 // The registers (simple name arrays)
 message Register {
-  repeated string elements = 100;
+  repeated string elements = 150;
 }
 
 // The agent register holds 2 lists:
 // a full list, and a list of agents not yet watched
 message AgentRegister {
-  repeated string agents = 110;
-  repeated string untrackedagents = 111;
+  repeated string agents = 160;
+  repeated string untrackedagents = 161;
 }
 
 // A basic FIFO
@@ -169,4 +172,19 @@ message AdminUsersList {
   repeated AdminUser element = 8020;
 }
 
+message ConfigurationItem {
+  required UserIdentity creator = 9002;
+  optional string comment = 9003;
+}
+
+message StorageClass {
+  required string name = 9000;
+  required uint32 nbCopies = 9001;
+
+}
+
+message StorageClassList {
+  repeated StorageClass element = 9050;
+}
+
 

tests/CMakeLists.txt

@@ -16,7 +16,7 @@ target_link_libraries(unittests
   ctamiddletierunittests
   ctamiddletiersqlite
   ctamiddletiersqliteunittests
-  #ctamiddletierobjectstoreunittests
+  ctamiddletierobjectstoreunittests
   ctamiddletierobjectstore
   ctamocknameservertest
   ctamocknameserver