Commit 32987202 authored by Cedric Caffy's avatar Cedric Caffy
Browse files

DiskSystem jsonified and sent to stdin of the script

parent 9f227007
......@@ -105,7 +105,8 @@ set (COMMON_LIB_SRC_FILES
exception/TimeOut.cpp
exception/UserError.cpp
exception/XrootCl.cpp
json/parser/JSONCParser.cpp
json/object/JSONObject.cpp
json/object/JSONCObject.cpp
log/DummyLogger.cpp
log/FileLogger.cpp
log/LogContext.cpp
......@@ -204,7 +205,8 @@ set (COMMON_UNIT_TESTS_LIB_SRC_FILES
utils/UtilsTest.cpp
optionalTest.cpp
rangeTest.cpp
json/parser/JSONCParserTest.cpp
json/test/JSONCTestObject.cpp
json/test/JSONCObjectTest.cpp
)
add_library (ctacommonunittests SHARED
......
......@@ -16,100 +16,89 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <json-c/json.h>
#include <json/json_object.h>
#include "JSONCObject.hpp"
#include "JSONCParser.hpp"
#include "common/json/object/SchedulerHints.hpp"
#include "common/json/object/TestObject.hpp"
#include "common/exception/Exception.hpp"
#include "JSONParserException.hpp"
#include "JSONObjectException.hpp"
namespace cta { namespace utils { namespace json { namespace parser {
JSONCParser::JSONCParser() {
m_jsonObject = json_object_new_object();
namespace cta { namespace utils { namespace json { namespace object {
JSONCObject::JSONCObject():JSONObject() {
initializeJSONCObject();
}
void JSONCParser::setJSONToBeParsed(const std::string& json){
void JSONCObject::setAttributesFromJSON(const std::string& json){
//DO JSON_C deinitialization
if(m_jsonObject != nullptr){
json_object_put(m_jsonObject);
m_jsonObject = nullptr;
destroyJSONCObject();
}
m_jsonObject = json_tokener_parse(json.c_str());
}
std::string JSONCParser::getJSON() const {
std::string JSONCObject::getJSON() {
return std::string(json_object_to_json_string_ext(m_jsonObject, JSON_C_TO_STRING_PLAIN));
}
JSONCParser::~JSONCParser() {
//Free the JSON object if initialized
if(m_jsonObject != nullptr){
json_object_put(m_jsonObject);
m_jsonObject = nullptr;
}
void JSONCObject::initializeJSONCObject() {
m_jsonObject = json_object_new_object();
}
void JSONCObject::destroyJSONCObject() {
json_object_put(m_jsonObject);
m_jsonObject = nullptr;
}
json_object * JSONCParser::getJSONObject(const std::string& key){
void JSONCObject::reinitializeJSONCObject() {
destroyJSONCObject();
initializeJSONCObject();
}
json_object * JSONCObject::getJSONObject(const std::string& key){
json_object * objectRet;
if(json_object_object_get_ex(m_jsonObject,key.c_str(),&objectRet)){
return objectRet;
}
std::string errMsg = "In JSONCParser::getJSONObject(), the provided json does not contain any key named \""+key+"\".";
throw cta::exception::JSONParserException(errMsg);
throw cta::exception::JSONObjectException(errMsg);
}
////////////////////////////////////////////////////////////////////
// JSONCParser::getValue() implementation START
////////////////////////////////////////////////////////////////////
template<>
std::string JSONCParser::getValue(const std::string& key){
std::string JSONCObject::jsonGetValue(const std::string& key){
json_object * jsonObj = getJSONObject(key);
return std::string(json_object_get_string(jsonObj));
}
template<>
uint64_t JSONCParser::getValue(const std::string & key){
uint64_t JSONCObject::jsonGetValue(const std::string & key){
json_object * jsonObj = getJSONObject(key);
return json_object_get_int64(jsonObj);
}
template<>
double JSONCParser::getValue(const std::string & key){
double JSONCObject::jsonGetValue(const std::string & key){
json_object * jsonObj = getJSONObject(key);
return json_object_get_double(jsonObj);
}
////////////////////////////////////////////////////////////////////
// JSONCParser::getValue() implementation END
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
// json::object::TestObject implementation START
////////////////////////////////////////////////////////////////////
template<>
void JSONCParser::generateJSONFromObject(const object::TestObject & value){
json_object_object_add(m_jsonObject,"integer_number",json_object_new_int64(value.integer_number));
json_object_object_add(m_jsonObject,"str",json_object_new_string(value.str.c_str()));
json_object_object_add(m_jsonObject,"double_number",json_object_new_double(value.double_number));
void JSONCObject::jsonSetValue(const std::string& key, const std::string & value){
json_object_object_add(m_jsonObject,key.c_str(),json_object_new_string(value.c_str()));
}
template<>
object::TestObject JSONCParser::getObjectFromJSON(){
object::TestObject ret;
ret.str = getValue<std::string>("str");
ret.integer_number = getValue<uint64_t>("integer_number");
ret.double_number = getValue<double>("double_number");
return ret;
void JSONCObject::jsonSetValue(const std::string& key, const double & value){
json_object_object_add(m_jsonObject,key.c_str(),json_object_new_double(value));
}
////////////////////////////////////////////////////////////////////
// json::object::TestObject implementation END
////////////////////////////////////////////////////////////////////
template <>
void JSONCParser::generateJSONFromObject(const object::SchedulerHints & value){
template<>
void JSONCObject::jsonSetValue(const std::string& key, const uint64_t & value){
json_object_object_add(m_jsonObject,key.c_str(),json_object_new_int64(value));
}
JSONCObject::~JSONCObject() {
//Free the JSON object if initialized
if(m_jsonObject != nullptr){
destroyJSONCObject();
}
}
}}}}
\ No newline at end of file
......@@ -18,36 +18,32 @@
#pragma once
#include <string>
#include "JSONObject.hpp"
#include <json-c/json.h>
#include <memory>
namespace cta { namespace utils { namespace json { namespace parser {
class JSONCParser {
namespace cta { namespace utils { namespace json { namespace object {
class JSONCObject : public JSONObject {
public:
JSONCParser();
void setJSONToBeParsed(const std::string & json);
JSONCObject();
virtual void setAttributesFromJSON(const std::string & json);
virtual std::string getJSON();
virtual ~JSONCObject();
protected:
json_object * m_jsonObject = nullptr;
std::string getJSON() const;
void initializeJSONCObject();
void destroyJSONCObject();
void reinitializeJSONCObject();
template<typename T>
void generateJSONFromObject(const T & value);
T jsonGetValue(const std::string & key);
template<typename T>
T getObjectFromJSON();
virtual ~JSONCParser();
private:
json_object * m_jsonObject = nullptr;
template<typename T>
T getValue(const std::string & key);
void jsonSetValue(const std::string & key, const T & value);
json_object * getJSONObject(const std::string & key);
};
}}}}
}}}}
\ No newline at end of file
......@@ -16,15 +16,11 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <string>
#include "JSONObject.hpp"
namespace cta { namespace utils { namespace json { namespace object {
struct SchedulerHints {
int test;
std::string test2;
};
}}}}
JSONObject::~JSONObject() {
}
}}}}
\ No newline at end of file
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <string>
namespace cta { namespace utils { namespace json { namespace object {
class JSONObject {
public:
virtual std::string getJSON() = 0;
virtual void setAttributesFromJSON(const std::string & json) = 0;
virtual ~JSONObject();
private:
};
}}}}
......@@ -20,7 +20,7 @@
namespace cta { namespace exception {
class JSONParserException : public Exception {
class JSONObjectException : public Exception {
using Exception::Exception;
};
......
......@@ -18,52 +18,38 @@
#include <gtest/gtest.h>
#include "common/json/parser/JSONCParser.hpp"
#include "common/json/object/TestObject.hpp"
#include "JSONParserException.hpp"
#include "JSONCTestObject.hpp"
#include "common/json/object/JSONObjectException.hpp"
namespace unitTests {
using namespace cta::utils;
TEST(JSONCParserTest, testJSONGenerationFromObject) {
json::object::TestObject to;
TEST(JSONCObjectTest, testJSONGenerationFromObject) {
JSONCTestObject to;
to.double_number = 42.0;
to.integer_number = 42;
to.str = "forty two";
json::parser::JSONCParser parser;
parser.generateJSONFromObject(to);
ASSERT_EQ("{\"integer_number\":42,\"str\":\"forty two\",\"double_number\":42.000000}",parser.getJSON());
ASSERT_EQ("{\"integer_number\":42,\"str\":\"forty two\",\"double_number\":42.000000}",to.getJSON());
}
TEST(JSONCParserTest, testObjectGenerationFromJSON){
TEST(JSONCObjectTest, testObjectGenerationFromJSON){
std::string json = "{\"integer_number\":42,\"str\":\"forty two\",\"double_number\":42.000000}";
json::parser::JSONCParser parser;
parser.setJSONToBeParsed(json);
json::object::TestObject to = parser.getObjectFromJSON<decltype(to)>();
JSONCTestObject to;
to.setAttributesFromJSON(json);
ASSERT_EQ(42,to.integer_number);
ASSERT_EQ("forty two",to.str);
ASSERT_EQ(42.000000,to.double_number);
}
TEST(JSONCParserTest, testJSONCParserGetObjectFromUninitializedJSON){
json::parser::JSONCParser parser;
ASSERT_THROW(parser.getObjectFromJSON<json::object::TestObject>(),cta::exception::JSONParserException);
TEST(JSONCObjectTest, testJSONCParserGetJSONShouldReturnDefaultValues){
JSONCTestObject to;
ASSERT_EQ("{\"integer_number\":0,\"str\":\"\",\"double_number\":0.000000}",to.getJSON());
}
TEST(JSONCParserTest, testJSONCParserGetJSONShouldReturnEmptyJSON){
json::parser::JSONCParser parser;
ASSERT_EQ("{}",parser.getJSON());
}
TEST(JSONCParserTest, testJSONCParserSetJSONToBeParsedWrongJSONFormat){
json::parser::JSONCParser parser;
parser.setJSONToBeParsed("WRONG_JSON_STRING");
ASSERT_EQ("null",parser.getJSON());
parser.setJSONToBeParsed("{\"test");
ASSERT_EQ("null",parser.getJSON());
ASSERT_THROW(parser.getObjectFromJSON<json::object::TestObject>(),cta::exception::JSONParserException);
TEST(JSONCObjectTest, testJSONCParserSetJSONToBeParsedWrongJSONFormat){
JSONCTestObject to;
ASSERT_THROW(to.setAttributesFromJSON("WRONG_JSON_STRING"),cta::exception::JSONObjectException);
}
}
\ No newline at end of file
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "JSONCTestObject.hpp"
namespace unitTests {
JSONCTestObject::JSONCTestObject():JSONCObject(), TestObject() {
}
void JSONCTestObject::setAttributesFromJSON(const std::string & json){
JSONCObject::setAttributesFromJSON(json);
double_number = jsonGetValue<double>("double_number");
integer_number = jsonGetValue<uint64_t>("integer_number");
str = jsonGetValue<std::string>("str");
}
std::string JSONCTestObject::getJSON() {
reinitializeJSONCObject();
jsonSetValue("integer_number",integer_number);
jsonSetValue("str",str);
jsonSetValue("double_number",double_number);
return JSONCObject::getJSON();
}
JSONCTestObject::~JSONCTestObject() {
}
}
\ No newline at end of file
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "common/json/object/JSONCObject.hpp"
#include "TestObject.hpp"
using namespace cta::utils::json;
namespace unitTests {
class JSONCTestObject : public object::JSONCObject, public TestObject {
public:
JSONCTestObject();
void setAttributesFromJSON(const std::string & json) override;
std::string getJSON() override;
virtual ~JSONCTestObject();
private:
};
}
......@@ -20,7 +20,7 @@
#include <string>
namespace cta { namespace utils { namespace json { namespace object {
namespace unitTests {
struct TestObject {
uint64_t integer_number;
......@@ -28,4 +28,4 @@ namespace cta { namespace utils { namespace json { namespace object {
double double_number;
};
}}}}
\ No newline at end of file
}
\ No newline at end of file
......@@ -28,6 +28,7 @@ add_library(ctadisk SHARED
DiskFile.cpp
RadosStriperPool.cpp
DiskSystem.cpp
JSONDiskSystem.cpp
)
target_link_libraries (ctadisk XrdCl cryptopp radosstriper)
......
......@@ -17,7 +17,7 @@
*/
#include "DiskSystem.hpp"
#include "JSONDiskSystem.hpp"
#include <algorithm>
#include "common/exception/Exception.hpp"
#include "common/threading/SubProcess.hpp"
......@@ -37,9 +37,9 @@ const DiskSystem& DiskSystemList::at(const std::string& name) const {
}
//------------------------------------------------------------------------------
// DiskSystemList::getFSNAme()
// DiskSystemList::getDSName()
//------------------------------------------------------------------------------
std::string DiskSystemList::getDSNAme(const std::string& fileURL) const {
std::string DiskSystemList::getDSName(const std::string& fileURL) const {
// First if the regexes have not been created yet, do so.
if (m_pointersAndRegexes.empty() && size()) {
for (const auto &ds: *this) {
......@@ -58,6 +58,17 @@ std::string DiskSystemList::getDSNAme(const std::string& fileURL) const {
throw std::out_of_range("In DiskSystemList::getDSNAme(): not match for fileURL");
}
//------------------------------------------------------------------------------
// DiskSystemList::setFetchEosFreeSpaceScript()
//------------------------------------------------------------------------------
void DiskSystemList::setFetchEosFreeSpaceScript(const std::string& path){
m_fetchEosFreeSpaceScript = path;
}
std::string DiskSystemList::getFetchEosFreeSpaceScript() const{
return m_fetchEosFreeSpaceScript;
}
//------------------------------------------------------------------------------
// DiskSystemFreeSpaceList::fetchFileSystemFreeSpace()
//------------------------------------------------------------------------------
......@@ -72,8 +83,16 @@ void DiskSystemFreeSpaceList::fetchDiskSystemFreeSpace(const std::set<std::strin
uint64_t freeSpace = 0;
try {
std::vector<std::string> regexResult;
regexResult = eosDiskSystem.exec(m_systemList.at(ds).freeSpaceQueryURL);
auto & currentDiskSystem = m_systemList.at(ds);
regexResult = eosDiskSystem.exec(currentDiskSystem.freeSpaceQueryURL);
if (regexResult.size()) {
try {
//TODO: TO BE CONTINUED
cta::disk::JSONDiskSystem jsoncDiskSystem(currentDiskSystem);
freeSpace = fetchEosFreeSpaceWithScript(m_systemList.getFetchEosFreeSpaceScript(),jsoncDiskSystem.getJSON(),lc);
} catch(const cta::disk::FetchEosFreeSpaceScriptException &ex){
}
freeSpace = fetchEosFreeSpace(regexResult.at(1), regexResult.at(2), lc);
goto found;
}
......@@ -150,6 +169,30 @@ uint64_t DiskSystemFreeSpaceList::fetchConstantFreeSpace(const std::string& inst
return utils::toUint64(instanceAddress);
}
//------------------------------------------------------------------------------
// DiskSystemFreeSpaceList::fetchEosFreeSpaceWithScript()
//------------------------------------------------------------------------------
uint64_t DiskSystemFreeSpaceList::fetchEosFreeSpaceWithScript(const std::string& scriptPath, const std::string& jsonInput, log::LogContext& lc){
//TODO A CONTINUER
cta::threading::SubProcess sp(scriptPath,{scriptPath},jsonInput);
sp.wait();
try {
std::string errMsg = "In DiskSystemFreeSpaceList::fetchEosFreeSpaceWithScript(), failed to call \"" + scriptPath;
exception::Errnum::throwOnNonZero(sp.exitValue(),errMsg);
} catch (exception::Exception & ex) {
ex.getMessage() << " scriptPath: " << scriptPath << " stderr: " << sp.stderr();
throw cta::disk::FetchEosFreeSpaceScriptException(ex.getMessage().str());
}
if (sp.wasKilled()) {
std::string errMsg = "In DiskSystemFreeSpaceList::fetchEosFreeSpaceWithScript(): " + scriptPath + " killed by signal: ";
exception::Exception ex(errMsg);
ex.getMessage() << utils::toString(sp.killSignal());
throw cta::disk::FetchEosFreeSpaceScriptException(ex.getMessage().str());
}
// Look for the result line for default space.
std::istringstream spStdoutIss(sp.stdout());
lc.log(log::CRIT,spStdoutIss.str());
throw cta::disk::FetchEosFreeSpaceScriptException("Test backpressure with script");
}
}} // namespace cta::disk
\ No newline at end of file
......@@ -39,6 +39,7 @@ namespace cta { namespace disk {
* interval), and the expected external bandwidth from sources external to CTA.
*/
CTA_GENERATE_EXCEPTION_CLASS(FetchEosFreeSpaceException);
CTA_GENERATE_EXCEPTION_CLASS(FetchEosFreeSpaceScriptException);
struct DiskSystem {
std::string name;
......@@ -57,11 +58,16 @@ class DiskSystemList: public std::list<DiskSystem> {
public:
/** Get the filesystem for a given destination URL */
std::string getDSNAme(const std::string &fileURL) const;
std::string getDSName(const std::string &fileURL) const;
/** Get the file system parameters from a file system name */
const DiskSystem & at(const std::string &name) const;
/** Get the fetch EOS free space script path */
std::string getFetchEosFreeSpaceScript() const;
void setFetchEosFreeSpaceScript(const std::string & path);
private:
struct PointerAndRegex {
PointerAndRegex(const DiskSystem & dsys, const std::string &re): ds(dsys), regex(re) {}
......@@ -70,6 +76,7 @@ private:
};
mutable std::list<PointerAndRegex> m_pointersAndRegexes;
std::string m_fetchEosFreeSpaceScript;
};
......@@ -95,6 +102,7 @@ private:
DiskSystemList &m_systemList;
uint64_t fetchEosFreeSpace(const std::string & instanceAddress, const std::string & spaceName,log::LogContext & lc);
uint64_t fetchConstantFreeSpace(const std::string & instanceAddress, log::LogContext & lc);
uint64_t fetchEosFreeSpaceWithScript(const std::string & scriptPath, const std::string & jsonInput, log::LogContext &lc);
};
}} // namespace cta::common
......
......@@ -106,15 +106,15 @@ namespace unitTests {
std::string dstURL = "root://ctaeos.archiveretrieve-1215709git0e38ccd0-xi98.svc.cluster.local//eos/ctaeos/cta/54065a67-a3ea-4a44-b213-6f6a6f4e2cf4?eos.lfn=fxid:7&eos.ruid=0&eos.rgid=0&eos.injection=1&eos.workflow=retrieve_written&eos.space=spinners&toto=5";
ASSERT_EQ(m_diskSystemSpinner.name,allDiskSystem.getDSNAme(dstURL));
ASSERT_EQ(m_diskSystemSpinner.name,allDiskSystem.getDSName(dstURL));
dstURL = "root://ctaeos.archiveretrieve-1215709git0e38ccd0-xi98.svc.cluster.local//eos/ctaeos/cta/54065a67-a3ea-4a44-b213-6f6a6f4e2cf4?eos.lfn=fxid:7&eos.ruid=0&eos.rgid=0&eos.injection=1&eos.workflow=retrieve_written&eos.space=default";
ASSERT_EQ(m_diskSystemDefault.name,allDiskSystem.getDSNAme(dstURL));
ASSERT_EQ(m_diskSystemDefault.name,allDiskSystem.getDSName(dstURL));
dstURL = "root://ctaeos.archiveretrieve-1215709git0e38ccd0-xi98.svc.cluster.local//eos/ctaeos/cta/54065a67-a3ea-4a44-b213-6f6a6f4e2cf4?eos.lfn=fxid:7&eos.ruid=0&eos.rgid=0&eos.injection=1&eos.workflow=retrieve_written";
ASSERT_THROW(allDiskSystem.getDSNAme(dstURL),std::out_of_range);
ASSERT_THROW(allDiskSystem.getDSName(dstURL),std::out_of_range);