diff --git a/common/exception/Exception.hpp b/common/exception/Exception.hpp index 1aff8fe2c61185e7abe46e3535b379aa1bc619cd..a8cf7e5142844a719ace239e23b167e0cd00470f 100644 --- a/common/exception/Exception.hpp +++ b/common/exception/Exception.hpp @@ -160,4 +160,10 @@ namespace cta { } // end of castor namespace +#define CTA_GENERATE_EXCEPTION_CLASS(A) \ +class A: public cta::exception::Exception { \ +public: \ + A(const std::string & w): cta::exception::Exception(w) {} \ +} + diff --git a/objectstore/ProtcolBuffersAlgorithms.hpp b/objectstore/ProtcolBuffersAlgorithms.hpp index a4b940cbb02b7d0ccacc779e7b958b476fc08772..1ca012054be81dc1c76d35b424f17990f3d2adce 100644 --- a/objectstore/ProtcolBuffersAlgorithms.hpp +++ b/objectstore/ProtcolBuffersAlgorithms.hpp @@ -25,6 +25,22 @@ namespace cta { namespace objectstore { namespace serializers { void removeString(::google::protobuf::RepeatedPtrField< ::std::string>* field, const std::string & value); +template <class C1, class C2> +void removeOccurences(::google::protobuf::RepeatedPtrField<C1>* field, + const C2 & value) { + bool found; + do { + found=false; + for (size_t i=0; i<(size_t)field->size(); i++) { + if (value == field->Get(i)) { + found = true; + field->SwapElements(i, field->size()-1); + field->RemoveLast(); + break; + } + } + } while (found); +} class NotFound: public cta::exception::Exception { public: NotFound(const std::string & w): cta::exception::Exception(w) {} @@ -36,4 +52,34 @@ size_t findString(::google::protobuf::RepeatedPtrField< ::std::string>* field, size_t findStringFrom(::google::protobuf::RepeatedPtrField< ::std::string>* field, size_t fromIndex, const std::string & value); +template <class C1, class C2> +C1 & findElement(::google::protobuf::RepeatedPtrField<C1>* field, const C2 & value) { + for (auto i=field->begin(); i!= field->end(); i++) { + if (value == *i) { + return *i; + } + } + throw NotFound("In cta::objectsotre::serializers::findElement(non-const): element not found"); +} + +template <class C1, class C2> +const C1 & findElement(const ::google::protobuf::RepeatedPtrField<C1>& field, const C2 & value) { + for (auto i=field.begin(); i!= field.end(); i++) { + if (value == *i) { + return *i; + } + } + throw NotFound("In cta::objectsotre::serializers::findElement(const): element not found"); +} + +template <class C1, class C2> +bool isElementPresent(const ::google::protobuf::RepeatedPtrField<C1>& field, const C2 & value) { + for (auto i=field.begin(); i!= field.end(); i++) { + if (value == *i) { + return true; + } + } + return false; +} + }}} \ No newline at end of file diff --git a/objectstore/ProtocolBuffersAlgorithms.cpp b/objectstore/ProtocolBuffersAlgorithms.cpp index 4579bca62abc6df069fa82449930676fb30d3730..79b1195efc8e73b6b9c0de34beaaef041e22fa85 100644 --- a/objectstore/ProtocolBuffersAlgorithms.cpp +++ b/objectstore/ProtocolBuffersAlgorithms.cpp @@ -20,18 +20,19 @@ void cta::objectstore::serializers::removeString(::google::protobuf::RepeatedPtrField< ::std::string>* field, const std::string & value) { - bool found; - do { - found = false; - for (size_t i=0; i<(size_t)field->size(); i++) { - if (value == field->Get(i)) { - found = true; - field->SwapElements(i, field->size()-1); - field->RemoveLast(); - break; - } - } - } while (found); +// bool found; +// do { +// found = false; +// for (size_t i=0; i<(size_t)field->size(); i++) { +// if (value == field->Get(i)) { +// found = true; +// field->SwapElements(i, field->size()-1); +// field->RemoveLast(); +// break; +// } +// } +// } while (found); + removeOccurences(field, value); } size_t cta::objectstore::serializers::findString( diff --git a/objectstore/RootEntry.cpp b/objectstore/RootEntry.cpp index 4e04dde450196a7c92a559744bfe761d62cc7704..86fb78434d8b343c502083ede9a6ef0adf58601c 100644 --- a/objectstore/RootEntry.cpp +++ b/objectstore/RootEntry.cpp @@ -36,117 +36,75 @@ void cta::objectstore::RootEntry::initialize() { } // ============================================================================= -// ================ Agent register manipulation ================================ +// ================ Admin Hosts manipulations ================================== // ============================================================================= -// Get the name of the agent register (or exception if not available) -std::string cta::objectstore::RootEntry::getAgentRegisterPointer() { - // Check that the fetch was done - if (!m_payloadInterpreted) - throw ObjectOpsBase::NotFetched("In RootEntry::getAgentRegisterPointer: object not yet fetched"); - // If the registry is defined, return it, job done. - if (m_payload.agentregisterpointer().address().size()) - return m_payload.agentregisterpointer().address(); - throw NotAllocatedEx("In RootEntry::getAgentRegister: agentRegister not yet allocated"); -} -// Get the name of a (possibly freshly created) agent register -std::string cta::objectstore::RootEntry::addOrGetAgentRegisterPointer(Agent & agent, - const CreationLog & log) { - // Check if the agent register exists - try { - return getAgentRegisterPointer(); - } catch (NotAllocatedEx &) { - // If we get here, the agent register is not created yet, so we have to do it: - // lock the entry again, for writing. We take the lock ourselves if needed - // This will make an autonomous transaction - std::auto_ptr<ScopedExclusiveLock> lockPtr; - if (!m_locksForWriteCount) - lockPtr.reset(new ScopedExclusiveLock(*this)); - // If the registry is already defined, somebody was faster. We're done. - fetch(); - if (m_payload.agentregisterpointer().address().size()) { - lockPtr.reset(NULL); - return m_payload.agentregisterpointer().address(); - } - // We will really create the register - // decide on the object's name - std::string arName (agent.nextId("agentRegister")); - // Record the agent registry in our own intent - addIntendedAgentRegistry(arName); - commit(); - // Create the agent registry - AgentRegister ar(arName, m_objectStore); - ar.initialize(); - // There is no garbage collection for an agent registry: if it is not - // plugged to the root entry, it does not exist. - ar.setOwner(""); - ar.setBackupOwner(""); - ar.insert(); - // Take a lock on agent registry - ScopedExclusiveLock arLock(ar); - // Move agent registry from intent to official - setAgentRegistry(arName, log); - deleteIntendedAgentRegistry(); - commit(); - // Record completion in agent registry - ar.setOwner(getNameIfSet()); - ar.setBackupOwner(getNameIfSet()); - ar.commit(); - // And we are done. Release locks - lockPtr.reset(NULL); - arLock.release(); - return arName; +// This operator will be used in the following usage of the templated +// findElement and removeOccurences +namespace { + bool operator==(const std::string & hostName, + const cta::objectstore::serializers::AdminHost & ah) { + return ah.hostname() == hostName; } } -void cta::objectstore::RootEntry::addIntendedAgentRegistry(const std::string& address) { +void cta::objectstore::RootEntry::addAdminHost(const std::string& hostname, + const CreationLog& log) { checkPayloadWritable(); - // We are supposed to have only one intended agent registry at a time. - // If we got the lock and there is one entry, this means the previous - // attempt to create one did not succeed. - // When getting here, having a set pointer to the registry is an error. - if (m_payload.agentregisterpointer().address().size()) { - throw exception::Exception("In cta::objectstore::RootEntry::addIntendedAgentRegistry:" - " pointer to registry already set"); - } - if (m_payload.agentregisterintent().size()) { - // The intended object might not have made it to the official pointer. - // If it did, we just clean up the intent. - // If it did not, we clean up the object if present, clean up the intent - // and replace it with the new one. - // We do not recycle the object, as the state is doubtful. - if (ObjectOps<serializers::RootEntry>::m_objectStore.exists(m_payload.agentregisterintent())) { - ObjectOps<serializers::RootEntry>::m_objectStore.read(m_payload.agentregisterintent()); - } - } - m_payload.set_agentregisterintent(address); + // Check that the host is not listed already + try { + serializers::findElement(m_payload.adminhosts(), hostname); + throw DuplicateEntry("In RootEntry::addAdminHost: entry already exists"); + } catch (serializers::NotFound &) {} + // Add the host + auto * ah = m_payload.add_adminhosts(); + ah->set_hostname(hostname); + log.serialize(*ah->mutable_log()); } -void cta::objectstore::RootEntry::deleteIntendedAgentRegistry() { +void cta::objectstore::RootEntry::removeAdminHost(const std::string & hostname) { checkPayloadWritable(); - m_payload.set_agentregisterintent(""); + serializers::removeOccurences(m_payload.mutable_adminhosts(), hostname); } -void cta::objectstore::RootEntry::setAgentRegistry(const std::string& address, - const CreationLog & log) { - checkPayloadWritable(); - m_payload.mutable_agentregisterpointer()->set_address(address); - log.serialize(*m_payload.mutable_agentregisterpointer()->mutable_log()); +bool cta::objectstore::RootEntry::isAdminHost(const std::string& hostname) { + checkPayloadReadable(); + return serializers::isElementPresent(m_payload.adminhosts(), hostname); +} + +auto cta::objectstore::RootEntry::dumpAdminHosts() -> std::list<AdminHostDump> { + checkPayloadReadable(); + std::list<AdminHostDump> ret; + auto &list=m_payload.adminhosts(); + for (auto i=list.begin();i!=list.end(); i++) { + ret.push_back(AdminHostDump()); + ret.back().hostname = i->hostname(); + ret.back().log.deserialize(i->log()); + } + return ret; } // ============================================================================= // ================ Admin Users manipulations ================================== // ============================================================================= +// This operator will be used in the following usage of the templated +// findElement and removeOccurences +namespace { + bool operator==(const cta::objectstore::UserIdentity & user, + const cta::objectstore::serializers::AdminUser & au) { + return au.user().uid() == user.uid; + } +} + void cta::objectstore::RootEntry::addAdminUser(const UserIdentity& user, const CreationLog& log) { checkPayloadWritable(); // Check that the user does not exists already - for (size_t i=0; i<(size_t)m_payload.adminusers_size(); i++) { - if (user.uid == m_payload.adminusers(i).user().uid()) { - throw DuplicateEntry("In RootEntry::addAdminUser: entry already exists"); - } - } + try { + serializers::findElement(m_payload.adminusers(), user); + throw DuplicateEntry("In RootEntry::addAdminUser: entry already exists"); + } catch (...) {} serializers::AdminUser * au = m_payload.add_adminusers(); user.serialize(*au->mutable_user()); log.serialize(*au->mutable_log()); @@ -154,34 +112,17 @@ void cta::objectstore::RootEntry::addAdminUser(const UserIdentity& user, void cta::objectstore::RootEntry::removeAdminUser(const UserIdentity& user) { checkPayloadWritable(); - bool found; - auto *list = m_payload.mutable_adminusers(); - do { - found = false; - for (size_t i=0; i<(size_t)list->size(); i++) { - if (list->Get(i).user().uid() == user.uid) { - found = true; - list->SwapElements(i, list->size()-1); - list->ReleaseLast(); - break; - } - } - } while (found); + serializers::removeOccurences(m_payload.mutable_adminusers(), user); } bool cta::objectstore::RootEntry::isAdminUser(const UserIdentity& user) { checkPayloadReadable(); - auto &list=m_payload.adminusers(); - for (auto i=list.begin(); i != list.end(); i++) { - if (i->user().uid() == user.uid) { - return true; - } - } - return false; + return serializers::isElementPresent(m_payload.adminusers(), user); } std::list<cta::objectstore::RootEntry::AdminUserDump> cta::objectstore::RootEntry::dumpAdminUsers() { + checkPayloadReadable(); std::list<cta::objectstore::RootEntry::AdminUserDump> ret; auto &list=m_payload.adminusers(); for (auto i=list.begin(); i != list.end(); i++) { @@ -192,6 +133,128 @@ cta::objectstore::RootEntry::dumpAdminUsers() { return ret; } +// ============================================================================= +// ========== Storage Class and archival routes manipulations ================== +// ============================================================================= + +// This operator will be used in the following usage of the templated +// findElement and removeOccurences +namespace { + bool operator==(const std::string & scName, + const cta::objectstore::serializers::StorageClass & ssc) { + return ssc.name() == scName; + } +} + +void cta::objectstore::RootEntry::addStorageClass(const std::string storageClass, + uint16_t copyCount, const CreationLog & log) { + checkPayloadWritable(); + // Check the storage class does not exist already + try { + serializers::findElement(m_payload.storageclasses(), storageClass); + throw DuplicateEntry("In RootEntry::addStorageClass: trying to create duplicate entry"); + } catch (...) {} + // Insert the storage class + auto * sc = m_payload.mutable_storageclasses()->Add(); + sc->set_name(storageClass); + sc->set_copycount(copyCount); + log.serialize(*sc->mutable_log()); +} + +void cta::objectstore::RootEntry::removeStorageClass(const std::string storageClass) { + checkPayloadWritable(); + serializers::removeOccurences(m_payload.mutable_storageclasses(), storageClass); +} + +void cta::objectstore::RootEntry::setStorageClassCopyCount( + const std::string& storageClass, uint16_t copyCount) { + checkPayloadWritable(); + auto & sc = serializers::findElement(m_payload.mutable_storageclasses(), storageClass); + // If we reduce the number of routes, we have to remove the extra ones. + if (copyCount < sc.copycount()) { + for (size_t i = copyCount; i<sc.copycount(); i++) { + serializers::removeOccurences(sc.mutable_routes(), i); + } + } + // and set the count + sc.set_copycount(copyCount); +} + +uint16_t cta::objectstore::RootEntry::getStorageClassCopyCount ( + std::string & storageClass) { + checkPayloadReadable(); + auto & sc = serializers::findElement(m_payload.storageclasses(), storageClass); + return sc.copycount(); +} + +// This operator will be used in the following usage of the findElement +// removeOccurences +namespace { + bool operator==(uint16_t copyNb, + const cta::objectstore::serializers::ArchivalRoute & ar) { + return ar.copynb() == copyNb; + } +} + +void cta::objectstore::RootEntry::setArchiveRoute(const std::string& storageClass, + uint16_t copyNb, const std::string& tapePool, const CreationLog& cl) { + checkPayloadWritable(); + // Find the storageClass entry + if (copyNb > maxCopyCount) { + std::stringstream ss; + ss << "In RootEntry::setArchiveRoute: invalid copy number: " << + copyNb << " > " << maxCopyCount; + throw InvalidCopyNumber(ss.str()); + } + auto & sc = serializers::findElement(m_payload.mutable_storageclasses(), storageClass); + if (copyNb >= sc.copycount()) { + std::stringstream ss; + ss << "In RootEntry::setArchiveRoute: copy number out of range: " << + copyNb << " >= " << sc.copycount(); + throw ss.str(); + } + // Find the archival route (if it exists) + try { + // It does: update in place. + auto &ar = serializers::findElement(sc.mutable_routes(), copyNb); + // Sanity check: is it the right route? + if (ar.copynb() != copyNb) { + throw exception::Exception( + "In RootEntry::setArchiveRoute: internal error: extracted wrong route"); + } + cl.serialize(*ar.mutable_log()); + ar.set_tapepool(tapePool); + } catch (serializers::NotFound &) { + // The route is not present yet. Add it. + auto *ar = sc.mutable_routes()->Add(); + cl.serialize(*ar->mutable_log()); + ar->set_copynb(copyNb); + ar->set_tapepool(tapePool); + } +} + +std::vector<std::string> cta::objectstore::RootEntry::getArchiveRoutes(const std::string storageClass) { + checkPayloadReadable(); + auto & sc = serializers::findElement(m_payload.storageclasses(), storageClass); + std::vector<std::string> ret; + ret.resize(sc.copycount()); + auto & list = sc.routes(); + for (auto i = list.begin(); i!= list.end(); i++) { + ret[i->copynb()] = i->tapepool(); + } + return ret; +} + +// TODO +//auto cta::objectstore::RootEntry::dumpStorageClasses() -> std::list<StorageClassDump> { +// std::list<StorageClassDump> ret; +// +//} + +//void cta::objectstore::RootEntry:: + + + //// Get the name of the JobPool (or exception if not available) //std::string cta::objectstore::RootEntry::getJobPool() { // checkPayloadReadable(); @@ -347,6 +410,106 @@ cta::objectstore::RootEntry::dumpAdminUsers() { // throw cta::exception::Exception("TODO"); //} +// ============================================================================= +// ================ Agent register manipulation ================================ +// ============================================================================= +// Get the name of the agent register (or exception if not available) +std::string cta::objectstore::RootEntry::getAgentRegisterPointer() { + // Check that the fetch was done + if (!m_payloadInterpreted) + throw ObjectOpsBase::NotFetched("In RootEntry::getAgentRegisterPointer: object not yet fetched"); + // If the registry is defined, return it, job done. + if (m_payload.agentregisterpointer().address().size()) + return m_payload.agentregisterpointer().address(); + throw NotAllocatedEx("In RootEntry::getAgentRegister: agentRegister not yet allocated"); +} + +// Get the name of a (possibly freshly created) agent register +std::string cta::objectstore::RootEntry::addOrGetAgentRegisterPointer(Agent & agent, + const CreationLog & log) { + // Check if the agent register exists + try { + return getAgentRegisterPointer(); + } catch (NotAllocatedEx &) { + // If we get here, the agent register is not created yet, so we have to do it: + // lock the entry again, for writing. We take the lock ourselves if needed + // This will make an autonomous transaction + std::auto_ptr<ScopedExclusiveLock> lockPtr; + if (!m_locksForWriteCount) + lockPtr.reset(new ScopedExclusiveLock(*this)); + // If the registry is already defined, somebody was faster. We're done. + fetch(); + if (m_payload.agentregisterpointer().address().size()) { + lockPtr.reset(NULL); + return m_payload.agentregisterpointer().address(); + } + // We will really create the register + // decide on the object's name + std::string arName (agent.nextId("agentRegister")); + // Record the agent registry in our own intent + addIntendedAgentRegistry(arName); + commit(); + // Create the agent registry + AgentRegister ar(arName, m_objectStore); + ar.initialize(); + // There is no garbage collection for an agent registry: if it is not + // plugged to the root entry, it does not exist. + ar.setOwner(""); + ar.setBackupOwner(""); + ar.insert(); + // Take a lock on agent registry + ScopedExclusiveLock arLock(ar); + // Move agent registry from intent to official + setAgentRegistry(arName, log); + deleteIntendedAgentRegistry(); + commit(); + // Record completion in agent registry + ar.setOwner(getNameIfSet()); + ar.setBackupOwner(getNameIfSet()); + ar.commit(); + // And we are done. Release locks + lockPtr.reset(NULL); + arLock.release(); + return arName; + } +} + +void cta::objectstore::RootEntry::addIntendedAgentRegistry(const std::string& address) { + checkPayloadWritable(); + // We are supposed to have only one intended agent registry at a time. + // If we got the lock and there is one entry, this means the previous + // attempt to create one did not succeed. + // When getting here, having a set pointer to the registry is an error. + if (m_payload.agentregisterpointer().address().size()) { + throw exception::Exception("In cta::objectstore::RootEntry::addIntendedAgentRegistry:" + " pointer to registry already set"); + } + if (m_payload.agentregisterintent().size()) { + // The intended object might not have made it to the official pointer. + // If it did, we just clean up the intent. + // If it did not, we clean up the object if present, clean up the intent + // and replace it with the new one. + // We do not recycle the object, as the state is doubtful. + if (ObjectOps<serializers::RootEntry>::m_objectStore.exists(m_payload.agentregisterintent())) { + ObjectOps<serializers::RootEntry>::m_objectStore.read(m_payload.agentregisterintent()); + } + } + m_payload.set_agentregisterintent(address); +} + +void cta::objectstore::RootEntry::deleteIntendedAgentRegistry() { + checkPayloadWritable(); + m_payload.set_agentregisterintent(""); +} + +void cta::objectstore::RootEntry::setAgentRegistry(const std::string& address, + const CreationLog & log) { + checkPayloadWritable(); + m_payload.mutable_agentregisterpointer()->set_address(address); + log.serialize(*m_payload.mutable_agentregisterpointer()->mutable_log()); +} + + // Dump the root entry std::string cta::objectstore::RootEntry::dump () { checkPayloadReadable(); diff --git a/objectstore/RootEntry.hpp b/objectstore/RootEntry.hpp index 66f859a6c46cba48dcfcb3a4cede966d3d0c74f9..89d64ea08bde207e4f16d9785782d7c030853d12 100644 --- a/objectstore/RootEntry.hpp +++ b/objectstore/RootEntry.hpp @@ -43,7 +43,7 @@ public: // In memory initialiser void initialize(); - // Manipulations of AdminHosts + // Manipulations of AdminHosts =============================================== void addAdminHost(const std::string & hostname, const CreationLog & log); void removeAdminHost(const std::string & hostname); bool isAdminHost(const std::string & hostname); @@ -54,12 +54,9 @@ public: }; std::list<AdminHostDump> dumpAdminHosts(); - class DuplicateEntry: public cta::exception::Exception { - public: - DuplicateEntry(const std::string & w): cta::exception::Exception(w) {} - }; + CTA_GENERATE_EXCEPTION_CLASS(DuplicateEntry); - // Manipulations of Admin Users + // Manipulations of Admin Users ============================================== void addAdminUser(const UserIdentity & user, const CreationLog & log); void removeAdminUser(const UserIdentity & user); bool isAdminUser(const UserIdentity & user); @@ -70,9 +67,24 @@ public: }; std::list<AdminUserDump> dumpAdminUsers(); - // Manipulations of Storage Classes and migration routes - void addStorageClass(const std::string storageClass, uint16_t copyCount, const CreationLog & log); + // Manipulations of Storage Classes and archival routes ====================== + CTA_GENERATE_EXCEPTION_CLASS(MissingEntry); + CTA_GENERATE_EXCEPTION_CLASS(IncompleteEntry); + CTA_GENERATE_EXCEPTION_CLASS(NoSuchStorageClass); + CTA_GENERATE_EXCEPTION_CLASS(InvalidCopyNumber); + CTA_GENERATE_EXCEPTION_CLASS(CopyNumberOutOfRange); +private: + // Totally arbitrary (but ridiculously high) copy number + static const uint16_t maxCopyCount=100; +public: + void addStorageClass(const std::string storageClass, uint16_t copyCount, + const CreationLog & log); void removeStorageClass(const std::string storageClass); + void setStorageClassCopyCount(const std::string & storageClass, uint16_t copyCount); + uint16_t getStorageClassCopyCount(std::string & storageClass); + void setArchiveRoute(const std::string & storageClass, uint16_t copyNb, + const std::string & tapePool, const CreationLog & cl); + /** Ordered vector of archive routes */ std::vector<std::string> getArchiveRoutes (const std::string storageClass); class StorageClassDump { @@ -89,7 +101,7 @@ public: }; std::list<StorageClassDump> dumpStorageClasses(); - // Manipulations of libraries + // Manipulations of libraries ================================================ void addLibrary(const std::string & library); void removeLibrary(const std::string & library); bool libraryExists(const std::string & library); @@ -111,12 +123,12 @@ public: }; std::list<TapePoolDump> dumpTapePool(); - // Drive register manipulations + // Drive register manipulations ============================================== std::string getDriveRegisterPointer(); std::string addOrGetDriveRegisterPointer(const CreationLog & log, Agent & agent); std::string removeDriveRegister(); - // Agent register manipulations + // Agent register manipulations ============================================== std::string getAgentRegisterPointer(); /** We do pass the agent here even if there is no agent register yet, as it * is used to generate the object name. We have the dedicated agent intent diff --git a/objectstore/UserIdentity.hpp b/objectstore/UserIdentity.hpp index 9fde40c769305450d08d56e323ade1247037c243..8577db2555f3e5d3358e47fcd9ca1b823479ae42 100644 --- a/objectstore/UserIdentity.hpp +++ b/objectstore/UserIdentity.hpp @@ -47,6 +47,17 @@ public: gid = user.gid(); gname = user.gname(); } -}; - + /** + * We can compare the UserIdentities between each other, and with the + * serialized user identities. + * The current actual criteria is numeric uid equility only. + */ + bool operator==(const UserIdentity & o) const { + return uid == o.uid; + } + bool operator==(const cta::objectstore::serializers::UserIdentity & o) const { + return uid == o.uid(); + } +}; + }}