Commit 02042f88 authored by Cedric Caffy's avatar Cedric Caffy
Browse files

[rao_lto] Added necessary architecture for CTA RAO LTO integration

parent 6cdb86a4
......@@ -281,6 +281,8 @@ void DriveState::setConfig(const cta::tape::daemon::TapedConfiguration& tapedCon
fillConfig(config->mountCriteria);
fillConfig(config->nbDiskThreads);
fillConfig(config->useRAO);
fillConfig(config->raoLtoAlgorithm);
fillConfig(config->raoLtoOptions);
fillConfig(config->wdScheduleMaxSecs);
fillConfig(config->wdMountMaxSecs);
fillConfig(config->wdNoBlockMoveMaxSecs);
......
......@@ -41,6 +41,7 @@ add_subdirectory(drive)
add_subdirectory(system)
add_subdirectory(file)
add_subdirectory(daemon)
add_subdirectory(RAO)
# .. and of course, the tests (last to use the variable definition)
add_subdirectory(test)
......
cmake_minimum_required (VERSION 2.6)
include_directories(${PROJECT_SOURCE_DIR}/tapeserver)
include_directories(${PROJECT_SOURCE_DIR}/tapeserver/h)
set(CTARAO_LIBRARY_SRCS
RAOConfig.cpp
RAOManager.cpp
RAOAlgorithm.cpp
EnterpriseRAOAlgorithm.cpp
RAOAlgorithmFactory.cpp
EnterpriseRAOAlgorithmFactory.cpp
LinearRAOAlgorithm.cpp
RandomRAOAlgorithm.cpp
NoParamRAOAlgorithmFactory.cpp
RAOAlgorithmFactoryFactory.cpp
)
add_library (ctarao SHARED
${CTARAO_LIBRARY_SRCS})
install(TARGETS ctarao DESTINATION usr/${CMAKE_INSTALL_LIBDIR})
\ No newline at end of file
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <list>
#include "EnterpriseRAOAlgorithm.hpp"
#include "castor/tape/tapeserver/SCSI/Structures.hpp"
namespace castor { namespace tape { namespace tapeserver { namespace rao {
EnterpriseRAOAlgorithm::EnterpriseRAOAlgorithm(castor::tape::tapeserver::drive::DriveInterface * drive, const uint64_t maxFilesSupported):m_drive(drive), m_maxFilesSupported(maxFilesSupported) {
}
EnterpriseRAOAlgorithm::~EnterpriseRAOAlgorithm() {
}
std::vector<uint64_t> EnterpriseRAOAlgorithm::performRAO(const std::vector<std::unique_ptr<cta::RetrieveJob> >& jobs) {
std::vector<uint64_t> raoOrder;
uint64_t njobs = jobs.size();
uint32_t block_size = 262144;
std::list<castor::tape::SCSI::Structures::RAO::blockLims> files;
for (uint32_t i = 0; i < njobs; i++) {
cta::RetrieveJob *job = jobs.at(i).get();
castor::tape::SCSI::Structures::RAO::blockLims lims;
strncpy((char*)lims.fseq, std::to_string(i).c_str(), sizeof(i));
lims.begin = job->selectedTapeFile().blockId;
lims.end = job->selectedTapeFile().blockId + 8 +
/* ceiling the number of blocks */
((job->archiveFile.fileSize + block_size - 1) / block_size);
files.push_back(lims);
if ((files.size() == m_maxFilesSupported) ||
((i == njobs - 1) && (files.size() > 1))) {
/* We do a RAO query if:
* 1. the maximum number of files supported by the drive
* for RAO query has been reached
* 2. the end of the jobs list has been reached and there are at least
* 2 unordered files
*/
m_drive->queryRAO(files, m_maxFilesSupported);
/* Add the RAO sorted files to the new list*/
for (auto fit = files.begin(); fit != files.end(); fit++) {
uint64_t id = atoi((char*)fit->fseq);
raoOrder.push_back(id);
}
files.clear();
}
}
for (auto fit = files.begin(); fit != files.end(); fit++) {
uint64_t id = atoi((char*)fit->fseq);
raoOrder.push_back(id);
}
files.clear();
return raoOrder;
}
}}}}
\ No newline at end of file
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "RAOAlgorithm.hpp"
#include "castor/tape/tapeserver/drive/DriveInterface.hpp"
#include "EnterpriseRAOAlgorithmFactory.hpp"
namespace castor { namespace tape { namespace tapeserver { namespace rao {
class EnterpriseRAOAlgorithmFactory;
/**
* This class represents an EnterpriseRAOAlgorithm.
*/
class EnterpriseRAOAlgorithm : public RAOAlgorithm{
public:
friend EnterpriseRAOAlgorithmFactory;
virtual ~EnterpriseRAOAlgorithm();
/**
* Asks the Enteprise drive to perform a RAO query in order to get the RAO of the
* files represented by the jobs passed in parameter
* @param jobs the jobs representing the files we want to perform the RAO on
* @return the vector of the indexes of the jobs passed in parameters rearranged by the RAO query
*/
std::vector<uint64_t> performRAO(const std::vector<std::unique_ptr<cta::RetrieveJob>> & jobs) override;
private:
/**
* Constructs an EnterpriseRAOAlgorithm
* @param drive the drive in order to call its RAO via its DriveInterface
* @param maxFilesSupported the maximum number of files supported by the drive to perform the RAO
*/
EnterpriseRAOAlgorithm(castor::tape::tapeserver::drive::DriveInterface * drive, const uint64_t maxFilesSupported);
//Interface to the drive
castor::tape::tapeserver::drive::DriveInterface * m_drive;
//Maximum number of files supported by the drive to perform the RAO
uint64_t m_maxFilesSupported;
};
}}}}
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "EnterpriseRAOAlgorithmFactory.hpp"
#include "EnterpriseRAOAlgorithm.hpp"
namespace castor { namespace tape { namespace tapeserver { namespace rao {
EnterpriseRAOAlgorithmFactory::EnterpriseRAOAlgorithmFactory(castor::tape::tapeserver::drive::DriveInterface * drive, const uint64_t maxFilesSupported):
m_drive(drive), m_maxFilesSupported(maxFilesSupported) {
}
EnterpriseRAOAlgorithmFactory::~EnterpriseRAOAlgorithmFactory() {
}
std::unique_ptr<RAOAlgorithm> EnterpriseRAOAlgorithmFactory::createRAOAlgorithm() {
return std::unique_ptr<RAOAlgorithm>(new EnterpriseRAOAlgorithm(m_drive,m_maxFilesSupported));
}
}}}}
\ No newline at end of file
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "RAOAlgorithmFactory.hpp"
#include "castor/tape/tapeserver/drive/DriveInterface.hpp"
namespace castor { namespace tape { namespace tapeserver { namespace rao {
/**
* Factory of EnterpriseRAOAlgorithm.
*/
class EnterpriseRAOAlgorithmFactory : public RAOAlgorithmFactory{
public:
/**
* Constructor of this factory
* @param drive the DriveInterface to perform a RAO query
* @param maxFilesSupported the maximum number of files the drive supports to perform a RAO query
*/
EnterpriseRAOAlgorithmFactory(castor::tape::tapeserver::drive::DriveInterface * drive, const uint64_t maxFilesSupported);
/**
* Returns an Enteprise RAO Algorithm
*/
std::unique_ptr<RAOAlgorithm> createRAOAlgorithm() override;
virtual ~EnterpriseRAOAlgorithmFactory();
private:
drive::DriveInterface * m_drive;
uint64_t m_maxFilesSupported;
};
}}}}
\ No newline at end of file
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <algorithm>
#include <numeric>
#include "LinearRAOAlgorithm.hpp"
namespace castor { namespace tape { namespace tapeserver { namespace rao {
LinearRAOAlgorithm::LinearRAOAlgorithm() {
}
LinearRAOAlgorithm::~LinearRAOAlgorithm() {
}
std::vector<uint64_t> LinearRAOAlgorithm::performRAO(const std::vector<std::unique_ptr<cta::RetrieveJob> >& jobs) {
std::vector<uint64_t> raoIndices(jobs.size());
//Initialize the vector of indices
std::iota(raoIndices.begin(),raoIndices.end(),0);
//Sort the indices regarding the fseq of the jobs located in the vector passed in parameter
std::stable_sort(raoIndices.begin(),raoIndices.end(),[&jobs](const uint64_t index1, const uint64_t index2){
return jobs[index1]->selectedTapeFile().fSeq < jobs[index2]->selectedTapeFile().fSeq;
});
return raoIndices;
}
}}}}
\ No newline at end of file
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "RAOAlgorithm.hpp"
#include "NoParamRAOAlgorithmFactory.hpp"
namespace castor { namespace tape { namespace tapeserver { namespace rao {
class NoParamRAOAlgorithmFactory;
/**
* This class represents a LinearRAOAlgorithm
*/
class LinearRAOAlgorithm : public RAOAlgorithm {
public:
friend NoParamRAOAlgorithmFactory;
/**
* This method will return the indexes of the jobs that are reoreded in a linear way (sorted by fseq ascendant)
* Example : if the fseqs of jobs in parameter are arranged like this [2, 3, 1, 4] the
* algorithm will return the following indexes vector : [2, 0, 1, 3]
* @param jobs the jobs to perform the linear RAO query
* @return the indexes of the jobs ordered by fseq ascendant
*/
std::vector<uint64_t> performRAO(const std::vector<std::unique_ptr<cta::RetrieveJob> >& jobs) override;
virtual ~LinearRAOAlgorithm();
private:
LinearRAOAlgorithm();
};
}}}}
\ No newline at end of file
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "NoParamRAOAlgorithmFactory.hpp"
#include "LinearRAOAlgorithm.hpp"
#include "RandomRAOAlgorithm.hpp"
namespace castor { namespace tape { namespace tapeserver { namespace rao {
NoParamRAOAlgorithmFactory::NoParamRAOAlgorithmFactory(const RAOConfig::RAOAlgorithmType & type) : m_type(type) {
}
NoParamRAOAlgorithmFactory::~NoParamRAOAlgorithmFactory() {
}
std::unique_ptr<RAOAlgorithm> NoParamRAOAlgorithmFactory::createRAOAlgorithm() {
std::unique_ptr<RAOAlgorithm> ret;
switch(m_type){
case RAOConfig::linear:{
ret.reset(new LinearRAOAlgorithm());
break;
}
case RAOConfig::random:{
ret.reset(new RandomRAOAlgorithm());
break;
}
default:
{
throw cta::exception::Exception("In NoParamRAOAlgorithmFactory::createRAOAlgorithm(): unknown type of algorithm");
}
}
return ret;
}
}}}}
\ No newline at end of file
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "RAOConfig.hpp"
#include "RAOAlgorithmFactory.hpp"
namespace castor { namespace tape { namespace tapeserver { namespace rao {
/**
* This factory allows to instanciate RAO algorithm that do not need any
* parameter to work. E.G the linear algorithm just does a sort of the fseqs,
* it does not need any parameter.
*/
class NoParamRAOAlgorithmFactory : public RAOAlgorithmFactory {
public:
/**
* Constructor
* @param type the type given will be used by the createRAOAlgorithm() method
* to instanciate the correct algorithm regarding its type
*/
NoParamRAOAlgorithmFactory(const RAOConfig::RAOAlgorithmType & type);
/**
* Returns the correct instance of RAO algorithm regarding the type
* given while constructing this factory.
* @throws Exception if the type is unknown
*/
std::unique_ptr<RAOAlgorithm> createRAOAlgorithm() override;
virtual ~NoParamRAOAlgorithmFactory();
private:
RAOConfig::RAOAlgorithmType m_type;
};
}}}}
\ No newline at end of file
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "RAOAlgorithm.hpp"
namespace castor { namespace tape { namespace tapeserver { namespace rao {
RAOAlgorithm::RAOAlgorithm() {
}
RAOAlgorithm::~RAOAlgorithm() {
}
}}}}
\ No newline at end of file
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <vector>
#include <memory>
#include "scheduler/RetrieveJob.hpp"
namespace castor { namespace tape { namespace tapeserver { namespace rao {
/**
* Abstract class that represents an RAO algorithm
*/
class RAOAlgorithm {
public:
RAOAlgorithm();
virtual ~RAOAlgorithm();
/**
* Returns the vector of indexes of the jobs passed in parameter
* sorted according to an algorithm
* @param jobs the jobs to perform RAO on
* @return the vector of indexes sorted by an algorithm applied on the jobs passed in parameter
*/
virtual std::vector<uint64_t> performRAO(const std::vector<std::unique_ptr<cta::RetrieveJob>> & jobs) = 0;
};
}}}}
\ No newline at end of file
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "RAOAlgorithmFactory.hpp"
namespace castor { namespace tape { namespace tapeserver { namespace rao {
RAOAlgorithmFactory::~RAOAlgorithmFactory() {
}
}}}}
\ No newline at end of file
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once