[rao_lto] Added necessary architecture for CTA RAO LTO integration

objectstore/DriveState.cpp

@@ -281,6 +281,8 @@ void DriveState::setConfig(const cta::tape::daemon::TapedConfiguration& tapedCon
   fillConfig(config->mountCriteria);
   fillConfig(config->nbDiskThreads);
   fillConfig(config->useRAO);
+  fillConfig(config->raoLtoAlgorithm);
+  fillConfig(config->raoLtoOptions);
   fillConfig(config->wdScheduleMaxSecs);
   fillConfig(config->wdMountMaxSecs);
   fillConfig(config->wdNoBlockMoveMaxSecs);

tapeserver/castor/tape/tapeserver/CMakeLists.txt

@@ -41,6 +41,7 @@ add_subdirectory(drive)
 add_subdirectory(system)
 add_subdirectory(file)
 add_subdirectory(daemon)
+add_subdirectory(RAO)
 
 # .. and of course, the tests (last to use the variable definition)
 add_subdirectory(test)

tapeserver/castor/tape/tapeserver/RAO/CMakeLists.txt

+cmake_minimum_required (VERSION 2.6)
+
+include_directories(${PROJECT_SOURCE_DIR}/tapeserver)
+include_directories(${PROJECT_SOURCE_DIR}/tapeserver/h)
+
+set(CTARAO_LIBRARY_SRCS
+  RAOConfig.cpp
+  RAOManager.cpp
+  RAOAlgorithm.cpp
+  EnterpriseRAOAlgorithm.cpp
+  RAOAlgorithmFactory.cpp
+  EnterpriseRAOAlgorithmFactory.cpp
+  LinearRAOAlgorithm.cpp
+  RandomRAOAlgorithm.cpp
+  NoParamRAOAlgorithmFactory.cpp
+  RAOAlgorithmFactoryFactory.cpp
+)
+
+add_library (ctarao SHARED
+  ${CTARAO_LIBRARY_SRCS})
+
+install(TARGETS ctarao DESTINATION usr/${CMAKE_INSTALL_LIBDIR})
\ No newline at end of file

tapeserver/castor/tape/tapeserver/RAO/EnterpriseRAOAlgorithm.cpp

+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2019  CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <list>
+
+#include "EnterpriseRAOAlgorithm.hpp"
+#include "castor/tape/tapeserver/SCSI/Structures.hpp"
+
+namespace castor { namespace tape { namespace tapeserver { namespace rao {
+
+EnterpriseRAOAlgorithm::EnterpriseRAOAlgorithm(castor::tape::tapeserver::drive::DriveInterface * drive, const uint64_t maxFilesSupported):m_drive(drive), m_maxFilesSupported(maxFilesSupported) {
+}
+
+EnterpriseRAOAlgorithm::~EnterpriseRAOAlgorithm() {
+}
+
+std::vector<uint64_t> EnterpriseRAOAlgorithm::performRAO(const std::vector<std::unique_ptr<cta::RetrieveJob> >& jobs) {
+  std::vector<uint64_t> raoOrder;
+  uint64_t njobs = jobs.size();
+  uint32_t block_size = 262144;
+  std::list<castor::tape::SCSI::Structures::RAO::blockLims> files;
+  for (uint32_t i = 0; i < njobs; i++) {
+    cta::RetrieveJob *job = jobs.at(i).get();
+    castor::tape::SCSI::Structures::RAO::blockLims lims;
+    strncpy((char*)lims.fseq, std::to_string(i).c_str(), sizeof(i));
+    lims.begin = job->selectedTapeFile().blockId;
+    lims.end = job->selectedTapeFile().blockId + 8 +
+               /* ceiling the number of blocks */
+               ((job->archiveFile.fileSize + block_size - 1) / block_size);
+
+    files.push_back(lims);
+    if ((files.size() == m_maxFilesSupported) ||
+            ((i == njobs - 1) && (files.size() > 1))) {
+      /* We do a RAO query if:
+       *  1. the maximum number of files supported by the drive
+       *     for RAO query has been reached
+       *  2. the end of the jobs list has been reached and there are at least
+       *     2 unordered files
+       */
+      m_drive->queryRAO(files, m_maxFilesSupported);
+
+      /* Add the RAO sorted files to the new list*/
+      for (auto fit = files.begin(); fit != files.end(); fit++) {
+        uint64_t id = atoi((char*)fit->fseq);
+        raoOrder.push_back(id);
+      }
+      files.clear();
+    }
+  }
+  for (auto fit = files.begin(); fit != files.end(); fit++) {
+    uint64_t id = atoi((char*)fit->fseq);
+    raoOrder.push_back(id);
+  }
+  files.clear();
+  return raoOrder;
+}
+
+
+}}}}
\ No newline at end of file

tapeserver/castor/tape/tapeserver/RAO/EnterpriseRAOAlgorithm.hpp

+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2019  CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "RAOAlgorithm.hpp"
+#include "castor/tape/tapeserver/drive/DriveInterface.hpp"
+#include "EnterpriseRAOAlgorithmFactory.hpp"
+
+namespace castor { namespace tape { namespace tapeserver { namespace rao {
+
+class EnterpriseRAOAlgorithmFactory;
+  
+/**
+ * This class represents an EnterpriseRAOAlgorithm. 
+ */
+class EnterpriseRAOAlgorithm : public RAOAlgorithm{
+public:
+  friend EnterpriseRAOAlgorithmFactory;
+  
+  virtual ~EnterpriseRAOAlgorithm();
+
+  /**
+   * Asks the Enteprise drive to perform a RAO query in order to get the RAO of the 
+   * files represented by the jobs passed in parameter
+   * @param jobs the jobs representing the files we want to perform the RAO on
+   * @return the vector of the indexes of the jobs passed in parameters rearranged by the RAO query
+   */
+  std::vector<uint64_t> performRAO(const std::vector<std::unique_ptr<cta::RetrieveJob>> & jobs) override;
+  
+private:
+  /**
+   * Constructs an EnterpriseRAOAlgorithm
+   * @param drive the drive in order to call its RAO via its DriveInterface
+   * @param maxFilesSupported the maximum number of files supported by the drive to perform the RAO
+   */
+  EnterpriseRAOAlgorithm(castor::tape::tapeserver::drive::DriveInterface * drive, const uint64_t maxFilesSupported);
+  
+  //Interface to the drive
+  castor::tape::tapeserver::drive::DriveInterface * m_drive;
+  //Maximum number of files supported by the drive to perform the RAO
+  uint64_t m_maxFilesSupported;
+};
+
+}}}}

tapeserver/castor/tape/tapeserver/RAO/EnterpriseRAOAlgorithmFactory.cpp

+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2019  CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "EnterpriseRAOAlgorithmFactory.hpp"
+#include "EnterpriseRAOAlgorithm.hpp"
+
+namespace castor { namespace tape { namespace tapeserver { namespace rao {
+  
+EnterpriseRAOAlgorithmFactory::EnterpriseRAOAlgorithmFactory(castor::tape::tapeserver::drive::DriveInterface * drive, const uint64_t maxFilesSupported):
+m_drive(drive), m_maxFilesSupported(maxFilesSupported) {
+}
+
+EnterpriseRAOAlgorithmFactory::~EnterpriseRAOAlgorithmFactory() {
+}
+
+std::unique_ptr<RAOAlgorithm> EnterpriseRAOAlgorithmFactory::createRAOAlgorithm() {
+  return std::unique_ptr<RAOAlgorithm>(new EnterpriseRAOAlgorithm(m_drive,m_maxFilesSupported));
+}
+
+
+}}}}
\ No newline at end of file

tapeserver/castor/tape/tapeserver/RAO/EnterpriseRAOAlgorithmFactory.hpp

+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2019  CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "RAOAlgorithmFactory.hpp"
+#include "castor/tape/tapeserver/drive/DriveInterface.hpp"
+
+namespace castor { namespace tape { namespace tapeserver { namespace rao {
+
+/**
+ * Factory of EnterpriseRAOAlgorithm. 
+ */
+class EnterpriseRAOAlgorithmFactory : public RAOAlgorithmFactory{
+public:
+  /**
+   * Constructor of this factory
+   * @param drive the DriveInterface to perform a RAO query 
+   * @param maxFilesSupported the maximum number of files the drive supports to perform a RAO query
+   */
+  EnterpriseRAOAlgorithmFactory(castor::tape::tapeserver::drive::DriveInterface * drive, const uint64_t maxFilesSupported);
+  
+  /**
+   * Returns an Enteprise RAO Algorithm
+   */
+  std::unique_ptr<RAOAlgorithm> createRAOAlgorithm() override;
+  virtual ~EnterpriseRAOAlgorithmFactory();
+private:
+  drive::DriveInterface * m_drive;
+  uint64_t m_maxFilesSupported;
+};
+
+}}}}
\ No newline at end of file

tapeserver/castor/tape/tapeserver/RAO/LinearRAOAlgorithm.cpp

+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2019  CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <algorithm>
+#include <numeric>
+
+#include "LinearRAOAlgorithm.hpp"
+
+namespace castor { namespace tape { namespace tapeserver { namespace rao {
+
+LinearRAOAlgorithm::LinearRAOAlgorithm() {
+}
+
+LinearRAOAlgorithm::~LinearRAOAlgorithm() {
+}
+
+std::vector<uint64_t> LinearRAOAlgorithm::performRAO(const std::vector<std::unique_ptr<cta::RetrieveJob> >& jobs) {
+  std::vector<uint64_t> raoIndices(jobs.size());
+  //Initialize the vector of indices
+  std::iota(raoIndices.begin(),raoIndices.end(),0);
+  //Sort the indices regarding the fseq of the jobs located in the vector passed in parameter
+  std::stable_sort(raoIndices.begin(),raoIndices.end(),[&jobs](const uint64_t index1, const uint64_t index2){
+    return jobs[index1]->selectedTapeFile().fSeq < jobs[index2]->selectedTapeFile().fSeq;
+  });
+  return raoIndices;
+}
+
+
+}}}}
\ No newline at end of file

tapeserver/castor/tape/tapeserver/RAO/LinearRAOAlgorithm.hpp

+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2019  CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "RAOAlgorithm.hpp"
+#include "NoParamRAOAlgorithmFactory.hpp"
+
+
+namespace castor { namespace tape { namespace tapeserver { namespace rao {
+
+class NoParamRAOAlgorithmFactory;
+  
+/**
+ * This class represents a LinearRAOAlgorithm 
+ */
+class LinearRAOAlgorithm : public RAOAlgorithm {
+public:
+  friend NoParamRAOAlgorithmFactory;
+  
+  /**
+   * This method will return the indexes of the jobs that are reoreded in a linear way (sorted by fseq ascendant)
+   * Example : if the fseqs of jobs in parameter are arranged like this [2, 3, 1, 4] the 
+   * algorithm will return the following indexes vector : [2, 0, 1, 3]
+   * @param jobs the jobs to perform the linear RAO query
+   * @return the indexes of the jobs ordered by fseq ascendant
+   */
+  std::vector<uint64_t> performRAO(const std::vector<std::unique_ptr<cta::RetrieveJob> >& jobs) override;
+  virtual ~LinearRAOAlgorithm();
+private:
+  LinearRAOAlgorithm();
+};
+
+}}}}
\ No newline at end of file

tapeserver/castor/tape/tapeserver/RAO/NoParamRAOAlgorithmFactory.cpp

+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2019  CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "NoParamRAOAlgorithmFactory.hpp"
+#include "LinearRAOAlgorithm.hpp"
+#include "RandomRAOAlgorithm.hpp"
+
+namespace castor { namespace tape { namespace tapeserver { namespace rao {
+
+NoParamRAOAlgorithmFactory::NoParamRAOAlgorithmFactory(const RAOConfig::RAOAlgorithmType & type) : m_type(type) {
+}
+
+NoParamRAOAlgorithmFactory::~NoParamRAOAlgorithmFactory() {
+}
+
+std::unique_ptr<RAOAlgorithm> NoParamRAOAlgorithmFactory::createRAOAlgorithm() {
+  std::unique_ptr<RAOAlgorithm> ret;
+  switch(m_type){
+    case RAOConfig::linear:{
+      ret.reset(new LinearRAOAlgorithm());
+      break;
+    }
+    case RAOConfig::random:{
+      ret.reset(new RandomRAOAlgorithm());
+      break;
+    }
+    default:
+    {
+      throw cta::exception::Exception("In NoParamRAOAlgorithmFactory::createRAOAlgorithm(): unknown type of algorithm");
+    }
+  }
+  return ret;
+}
+
+
+}}}}
\ No newline at end of file

tapeserver/castor/tape/tapeserver/RAO/NoParamRAOAlgorithmFactory.hpp

+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2019  CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once 
+
+#include "RAOConfig.hpp"
+#include "RAOAlgorithmFactory.hpp"
+
+namespace castor { namespace tape { namespace tapeserver { namespace rao {
+  
+/**
+ * This factory allows to instanciate RAO algorithm that do not need any
+ * parameter to work. E.G the linear algorithm just does a sort of the fseqs,
+ * it does not need any parameter.
+ */
+class NoParamRAOAlgorithmFactory : public RAOAlgorithmFactory {
+public:
+  /**
+   * Constructor
+   * @param type the type given will be used by the createRAOAlgorithm() method
+   * to instanciate the correct algorithm regarding its type
+   */
+  NoParamRAOAlgorithmFactory(const RAOConfig::RAOAlgorithmType & type);
+  /**
+   * Returns the correct instance of RAO algorithm regarding the type
+   * given while constructing this factory.
+   * @throws Exception if the type is unknown
+   */
+  std::unique_ptr<RAOAlgorithm> createRAOAlgorithm() override;
+  virtual ~NoParamRAOAlgorithmFactory();
+private:
+  RAOConfig::RAOAlgorithmType m_type;
+};
+
+}}}}
\ No newline at end of file

tapeserver/castor/tape/tapeserver/RAO/RAOAlgorithm.cpp

+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2019  CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "RAOAlgorithm.hpp"
+
+namespace castor { namespace tape { namespace tapeserver { namespace rao {
+
+RAOAlgorithm::RAOAlgorithm() {
+}
+
+RAOAlgorithm::~RAOAlgorithm() {
+}
+
+}}}}
\ No newline at end of file

tapeserver/castor/tape/tapeserver/RAO/RAOAlgorithm.hpp

+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2019  CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <vector>
+#include <memory>
+#include "scheduler/RetrieveJob.hpp"
+
+
+namespace castor { namespace tape { namespace tapeserver { namespace rao {
+  
+/**
+ * Abstract class that represents an RAO algorithm
+ */
+class RAOAlgorithm {
+public:
+  RAOAlgorithm();
+  virtual ~RAOAlgorithm();
+  
+  /**
+   * Returns the vector of indexes of the jobs passed in parameter
+   * sorted according to an algorithm
+   * @param jobs the jobs to perform RAO on
+   * @return the vector of indexes sorted by an algorithm applied on the jobs passed in parameter
+   */
+  virtual std::vector<uint64_t> performRAO(const std::vector<std::unique_ptr<cta::RetrieveJob>> & jobs) = 0;
+
+};
+
+}}}}
\ No newline at end of file

tapeserver/castor/tape/tapeserver/RAO/RAOAlgorithmFactory.cpp

+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2019  CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "RAOAlgorithmFactory.hpp"
+
+namespace castor { namespace tape { namespace tapeserver { namespace rao {
+
+
+
+RAOAlgorithmFactory::~RAOAlgorithmFactory() {
+}
+
+}}}}
\ No newline at end of file

tapeserver/castor/tape/tapeserver/RAO/RAOAlgorithmFactory.hpp

+/*
+ * The CERN Tape Archive (CTA) project
+ * Copyright (C) 2019  CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once