Commit bda08bbb authored by Cedric Caffy's avatar Cedric Caffy
Browse files

[tpsrv] If configured, backpressure now calls an external script to get EOS free space

parent 32987202
......@@ -26,7 +26,7 @@ JSONCObject::JSONCObject():JSONObject() {
initializeJSONCObject();
}
void JSONCObject::setAttributesFromJSON(const std::string& json){
void JSONCObject::buildFromJSON(const std::string& json){
//DO JSON_C deinitialization
if(m_jsonObject != nullptr){
destroyJSONCObject();
......@@ -34,6 +34,10 @@ void JSONCObject::setAttributesFromJSON(const std::string& json){
m_jsonObject = json_tokener_parse(json.c_str());
}
std::string JSONCObject::getExpectedJSONToBuildObject() const {
return "{}";
}
std::string JSONCObject::getJSON() {
return std::string(json_object_to_json_string_ext(m_jsonObject, JSON_C_TO_STRING_PLAIN));
}
......
......@@ -23,25 +23,76 @@
namespace cta { namespace utils { namespace json { namespace object {
/**
* This class allows to build the inherited object or to generate the
* JSON output of the inherited object by using the JSON-C library
* https://github.com/json-c/json-c/wiki
*
* The same json_object pointer is used to read from a JSON string
* and to create a JSON string. Hence this pointer is reinitialized at each call
* to buildFromJSON() and getJSON()
*/
class JSONCObject : public JSONObject {
public:
JSONCObject();
virtual void setAttributesFromJSON(const std::string & json);
/**
* !!! This method must be called in the first line of same method in the inherited object
* Constructs the inherited object from the json passed in parameter
* @param json the json to build the object from
* @throws JSONObjectException if the json provided does not allow to build this object
*/
virtual void buildFromJSON(const std::string & json);
/**
* Return the inherited object expected JSON structure allowing to set its attributes
* via the buildFromJSON() method
* @return an example of JSON allowing to build the object e.g {"freeSpace",42}
*/
virtual std::string getExpectedJSONToBuildObject() const;
/**
* Returns the json representation of the inherited object
* or null if the json cannot be generated from the inherited object attributes
*/
virtual std::string getJSON();
virtual ~JSONCObject();
protected:
json_object * m_jsonObject = nullptr;
/**
* Initialize the JSON representation of this object
*
*/
void initializeJSONCObject();
/**
* Destroy the JSON representation of this object
*/
void destroyJSONCObject();
/**
* Destroy then initialize the JSON-C representation
*/
void reinitializeJSONCObject();
/**
* This method allows to get the value from the JSON-C representation of the object
* @params key the key to get the object from it
* T is the type of the value associated to the key
*/
template<typename T>
T jsonGetValue(const std::string & key);
/**
* This method allows to create or set an object on this JSON-C object representation
* @param key the key to create
* @param value the value associated to the key
* T is the type of the value associated to the key
*/
template<typename T>
void jsonSetValue(const std::string & key, const T & value);
/**
* Returns a pointer to the JSON-C representation of the object associated to the key passed in parameter
* @param key the key to return the JSON-C representation of the object associated to it
* @return the JSON-C representation of the object associated to the key passed in parameter
*/
json_object * getJSONObject(const std::string & key);
};
......
......@@ -22,10 +22,28 @@
namespace cta { namespace utils { namespace json { namespace object {
/**
* Interface that allows the objects that inherits from the implementations of this interface
* to be built from JSON or to generate the json string representation from its attributes
*/
class JSONObject {
public:
/**
* Return the JSON representation of this object
*/
virtual std::string getJSON() = 0;
virtual void setAttributesFromJSON(const std::string & json) = 0;
/**
* Set the inherited object attributes from the json passed in parameter
* @param json the json string used to set the inherited object attributes
* @throws JSONObjectException if the json does not contain the correct key-value attributes
*/
virtual void buildFromJSON(const std::string & json) = 0;
/**
* Return the inherited object expected JSON structure allowing to set its attributes
* via the buildFromJSON() method
* @return an example of JSON allowing to build the object e.g {"freeSpace",42}
*/
virtual std::string getExpectedJSONToBuildObject() const = 0;
virtual ~JSONObject();
private:
......
......@@ -20,6 +20,10 @@
namespace cta { namespace exception {
/**
* This exception should be used by JSONObject inherited classes
* to inform about a problem linked to the creation or json serialization of an object
*/
class JSONObjectException : public Exception {
using Exception::Exception;
};
......
......@@ -36,7 +36,7 @@ TEST(JSONCObjectTest, testJSONGenerationFromObject) {
TEST(JSONCObjectTest, testObjectGenerationFromJSON){
std::string json = "{\"integer_number\":42,\"str\":\"forty two\",\"double_number\":42.000000}";
JSONCTestObject to;
to.setAttributesFromJSON(json);
to.buildFromJSON(json);
ASSERT_EQ(42,to.integer_number);
ASSERT_EQ("forty two",to.str);
ASSERT_EQ(42.000000,to.double_number);
......@@ -49,7 +49,7 @@ TEST(JSONCObjectTest, testJSONCParserGetJSONShouldReturnDefaultValues){
TEST(JSONCObjectTest, testJSONCParserSetJSONToBeParsedWrongJSONFormat){
JSONCTestObject to;
ASSERT_THROW(to.setAttributesFromJSON("WRONG_JSON_STRING"),cta::exception::JSONObjectException);
ASSERT_THROW(to.buildFromJSON("WRONG_JSON_STRING"),cta::exception::JSONObjectException);
}
}
\ No newline at end of file
......@@ -25,8 +25,8 @@ JSONCTestObject::JSONCTestObject():JSONCObject(), TestObject() {
}
void JSONCTestObject::setAttributesFromJSON(const std::string & json){
JSONCObject::setAttributesFromJSON(json);
void JSONCTestObject::buildFromJSON(const std::string & json){
JSONCObject::buildFromJSON(json);
double_number = jsonGetValue<double>("double_number");
integer_number = jsonGetValue<uint64_t>("integer_number");
str = jsonGetValue<std::string>("str");
......@@ -40,6 +40,10 @@ std::string JSONCTestObject::getJSON() {
return JSONCObject::getJSON();
}
std::string JSONCTestObject::getExpectedJSONToBuildObject() const {
return "{\"integer_number\":42,\"str\":\"forty two\",\"double_number\":42.000000}";
}
JSONCTestObject::~JSONCTestObject() {
}
......
......@@ -25,10 +25,14 @@ using namespace cta::utils::json;
namespace unitTests {
/**
* This class is only use to unit test the JSONCObject class
*/
class JSONCTestObject : public object::JSONCObject, public TestObject {
public:
JSONCTestObject();
void setAttributesFromJSON(const std::string & json) override;
void buildFromJSON(const std::string & json) override;
std::string getExpectedJSONToBuildObject() const override;
std::string getJSON() override;
virtual ~JSONCTestObject();
private:
......
......@@ -22,6 +22,10 @@
namespace unitTests {
/**
* Object that will be used
* to unit test the JSONCTestObject
*/
struct TestObject {
uint64_t integer_number;
std::string str;
......
......@@ -29,6 +29,7 @@ add_library(ctadisk SHARED
RadosStriperPool.cpp
DiskSystem.cpp
JSONDiskSystem.cpp
JSONFreeSpace.cpp
)
target_link_libraries (ctadisk XrdCl cryptopp radosstriper)
......
......@@ -23,6 +23,8 @@
#include "common/threading/SubProcess.hpp"
#include "common/exception/Errnum.hpp"
#include "common/utils/utils.hpp"
#include "JSONFreeSpace.hpp"
#include "common/json/object/JSONObjectException.hpp"
namespace cta {
namespace disk {
......@@ -65,6 +67,9 @@ void DiskSystemList::setFetchEosFreeSpaceScript(const std::string& path){
m_fetchEosFreeSpaceScript = path;
}
//------------------------------------------------------------------------------
// DiskSystemList::getFetchEosFreeSpaceScript()
//------------------------------------------------------------------------------
std::string DiskSystemList::getFetchEosFreeSpaceScript() const{
return m_fetchEosFreeSpaceScript;
}
......@@ -86,16 +91,24 @@ void DiskSystemFreeSpaceList::fetchDiskSystemFreeSpace(const std::set<std::strin
auto & currentDiskSystem = m_systemList.at(ds);
regexResult = eosDiskSystem.exec(currentDiskSystem.freeSpaceQueryURL);
if (regexResult.size()) {
//Script, then EOS free space query
if(!m_systemList.getFetchEosFreeSpaceScript().empty()){
//Script is provided
try {
//TODO: TO BE CONTINUED
cta::disk::JSONDiskSystem jsoncDiskSystem(currentDiskSystem);
freeSpace = fetchEosFreeSpaceWithScript(m_systemList.getFetchEosFreeSpaceScript(),jsoncDiskSystem.getJSON(),lc);
} catch(const cta::disk::FetchEosFreeSpaceScriptException &ex){
cta::disk::JSONDiskSystem jsoncDiskSystem(currentDiskSystem);
freeSpace = fetchEosFreeSpaceWithScript(m_systemList.getFetchEosFreeSpaceScript(),jsoncDiskSystem.getJSON(),lc);
goto found;
} catch(const cta::disk::FetchEosFreeSpaceScriptException &ex){
cta::log::ScopedParamContainer spc(lc);
spc.add("exceptionMsg",ex.getMessageValue());
std::string errorMsg = "In DiskSystemFreeSpaceList::fetchDiskSystemFreeSpace(), unable to get the EOS free space with the script "
+ m_systemList.getFetchEosFreeSpaceScript() + ". Will run eos space ls -m to fetch the free space for backpressure";
lc.log(cta::log::INFO,errorMsg);
}
}
freeSpace = fetchEosFreeSpace(regexResult.at(1), regexResult.at(2), lc);
goto found;
}
}
regexResult = constantFreeSpaceDiskSystem.exec(m_systemList.at(ds).freeSpaceQueryURL);
if (regexResult.size()) {
freeSpace = fetchConstantFreeSpace(regexResult.at(1), lc);
......@@ -173,9 +186,8 @@ uint64_t DiskSystemFreeSpaceList::fetchConstantFreeSpace(const std::string& inst
// DiskSystemFreeSpaceList::fetchEosFreeSpaceWithScript()
//------------------------------------------------------------------------------
uint64_t DiskSystemFreeSpaceList::fetchEosFreeSpaceWithScript(const std::string& scriptPath, const std::string& jsonInput, log::LogContext& lc){
//TODO A CONTINUER
cta::threading::SubProcess sp(scriptPath,{scriptPath},jsonInput);
sp.wait();
sp.wait();
try {
std::string errMsg = "In DiskSystemFreeSpaceList::fetchEosFreeSpaceWithScript(), failed to call \"" + scriptPath;
exception::Errnum::throwOnNonZero(sp.exitValue(),errMsg);
......@@ -189,10 +201,21 @@ uint64_t DiskSystemFreeSpaceList::fetchEosFreeSpaceWithScript(const std::string&
ex.getMessage() << utils::toString(sp.killSignal());
throw cta::disk::FetchEosFreeSpaceScriptException(ex.getMessage().str());
}
// Look for the result line for default space.
//Get the JSON result from stdout and return the free space
JSONFreeSpace jsonFreeSpace;
std::istringstream spStdoutIss(sp.stdout());
lc.log(log::CRIT,spStdoutIss.str());
throw cta::disk::FetchEosFreeSpaceScriptException("Test backpressure with script");
std::string stdoutScript = spStdoutIss.str();
try {
jsonFreeSpace.buildFromJSON(stdoutScript);
std::string logMessage = "In DiskSystemFreeSpaceList::fetchEosFreeSpaceWithScript(), freeSpace returned from the script is: " + std::to_string(jsonFreeSpace.m_freeSpace);
lc.log(log::DEBUG,logMessage);
return jsonFreeSpace.m_freeSpace;
} catch(const cta::exception::JSONObjectException &ex){
std::string errMsg = "In DiskSystemFreeSpaceList::fetchEosFreeSpaceWithScript(): the json received from the script "+ scriptPath +
" json=" + stdoutScript + " could not be used to get the FreeSpace, the json to receive from the script should have the following format: " +
jsonFreeSpace.getExpectedJSONToBuildObject() + ".";
throw cta::disk::FetchEosFreeSpaceScriptException(errMsg);
}
}
}} // namespace cta::disk
\ No newline at end of file
......@@ -63,9 +63,10 @@ public:
/** Get the file system parameters from a file system name */
const DiskSystem & at(const std::string &name) const;
/** Get the fetch EOS free space script path */
/** Get the fetch EOS free space script path. This script will be used by the backpressure */
std::string getFetchEosFreeSpaceScript() const;
/** Sets the fetch EOS free space script path. This script will be used by the backpressure */
void setFetchEosFreeSpaceScript(const std::string & path);
private:
......
......@@ -34,8 +34,8 @@ JSONDiskSystem::JSONDiskSystem(const DiskSystem& diskSystem):JSONCObject(){
}
}
void JSONDiskSystem::setAttributesFromJSON(const std::string& json) {
JSONCObject::setAttributesFromJSON(json);
void JSONDiskSystem::buildFromJSON(const std::string& json) {
JSONCObject::buildFromJSON(json);
name = jsonGetValue<std::string>("name");
fileRegexp = jsonGetValue<std::string>("fileRegexp");
freeSpaceQueryURL = jsonGetValue<std::string>("freeSpaceQueryURL");
......
......@@ -25,11 +25,23 @@ using namespace cta::utils::json::object;
namespace cta { namespace disk {
/**
* This class allows to transform a DiskSystem object into a JSON string
* and to build a DiskSystem object from a JSON string
*/
class JSONDiskSystem : public DiskSystem, public JSONCObject {
public:
JSONDiskSystem();
JSONDiskSystem(const DiskSystem & diskSystem);
void setAttributesFromJSON(const std::string & json) override;
/**
* Builds the DiskSystem object with the json passed in parameter
* @param json
*/
void buildFromJSON(const std::string & json) override;
/**
* Get the json string representation of the inherited DiskSystem object
* @return
*/
std::string getJSON() override;
virtual ~JSONDiskSystem();
private:
......
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "JSONFreeSpace.hpp"
namespace cta { namespace disk {
JSONFreeSpace::JSONFreeSpace(): JSONCObject() {
}
void JSONFreeSpace::buildFromJSON(const std::string& json) {
JSONCObject::buildFromJSON(json);
m_freeSpace = jsonGetValue<uint64_t>("freeSpace");
}
std::string JSONFreeSpace::getJSON(){
reinitializeJSONCObject();
jsonSetValue("freeSpace",m_freeSpace);
return JSONCObject::getJSON();
}
std::string JSONFreeSpace::getExpectedJSONToBuildObject() const {
return "{\"freeSpace\":42}";
}
JSONFreeSpace::~JSONFreeSpace() {
}
}}
\ No newline at end of file
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "common/json/object/JSONCObject.hpp"
namespace cta { namespace disk {
/**
* This class allows JSON-represent a FreeSpace object that is only a uint64_t value
* {"freeSpace",42}
*/
class JSONFreeSpace: public cta::utils::json::object::JSONCObject {
public:
JSONFreeSpace();
void buildFromJSON(const std::string & json) override;
std::string getJSON() override;
std::string getExpectedJSONToBuildObject() const override;
virtual ~JSONFreeSpace();
uint64_t m_freeSpace = 0;
};
}}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment