DiskSystem.cpp 9.09 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/*
 * The CERN Tape Archive (CTA) project
 * Copyright (C) 2015  CERN
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "DiskSystem.hpp"
20
#include "JSONDiskSystem.hpp"
21
#include <algorithm>
22
23
24
25
#include "common/exception/Exception.hpp"
#include "common/threading/SubProcess.hpp"
#include "common/exception/Errnum.hpp"
#include "common/utils/utils.hpp"
26

27
namespace cta {
28
namespace disk {
29

30
31
32
//------------------------------------------------------------------------------
// DiskSystemList::at()
//------------------------------------------------------------------------------
33
const DiskSystem& DiskSystemList::at(const std::string& name) const {
34
35
36
37
38
  auto dsi = std::find_if(begin(), end(), [&](const DiskSystem& ds){ return ds.name == name;});
  if (dsi != end()) return *dsi;
  throw std::out_of_range("In DiskSystemList::at(): name not found.");
}

39
//------------------------------------------------------------------------------
40
// DiskSystemList::getDSName()
41
//------------------------------------------------------------------------------
42
std::string DiskSystemList::getDSName(const std::string& fileURL) const {
43
44
45
  // First if the regexes have not been created yet, do so.
  if (m_pointersAndRegexes.empty() && size()) {
    for (const auto &ds: *this) {
46
      m_pointersAndRegexes.emplace_back(ds, ds.fileRegexp);
47
48
49
50
51
52
53
54
55
56
57
    }
  }
  // Try and find the fileURL
  auto pri = std::find_if(m_pointersAndRegexes.begin(), m_pointersAndRegexes.end(), 
      [&](const PointerAndRegex & pr){ return !pr.regex.exec(fileURL).empty(); });
  if (pri != m_pointersAndRegexes.end()) {
    // We found a match. Let's move the pointer and regex to the front so next file will be faster (most likely).
    if (pri != m_pointersAndRegexes.begin())
      m_pointersAndRegexes.splice(m_pointersAndRegexes.begin(), m_pointersAndRegexes, pri);
    return pri->ds.name;
  }
58
  throw std::out_of_range("In DiskSystemList::getDSNAme(): not match for fileURL");
59
60
}

61
62
63
64
65
66
67
68
69
70
71
//------------------------------------------------------------------------------
// DiskSystemList::setFetchEosFreeSpaceScript()
//------------------------------------------------------------------------------
void DiskSystemList::setFetchEosFreeSpaceScript(const std::string& path){
  m_fetchEosFreeSpaceScript = path;
}

std::string DiskSystemList::getFetchEosFreeSpaceScript() const{
  return m_fetchEosFreeSpaceScript;
}

72
73
74
//------------------------------------------------------------------------------
// DiskSystemFreeSpaceList::fetchFileSystemFreeSpace()
//------------------------------------------------------------------------------
75
void DiskSystemFreeSpaceList::fetchDiskSystemFreeSpace(const std::set<std::string>& diskSystems, log::LogContext & lc) {
76
  // The real deal: go fetch the file system's free space.
77
  cta::utils::Regex eosDiskSystem("^eos:(.*):(.*)$");
78
  // For testing purposes
79
  cta::utils::Regex constantFreeSpaceDiskSystem("^constantFreeSpace:(.*)");
80
81
  //Key = diskSystemName, Value = failureReason
  std::map<std::string, cta::exception::Exception> failedToFetchDiskSystems;
82
  for (auto const & ds: diskSystems) {
83
84
85
    uint64_t freeSpace = 0;
    try {
      std::vector<std::string> regexResult;
86
87
      auto & currentDiskSystem = m_systemList.at(ds);
      regexResult = eosDiskSystem.exec(currentDiskSystem.freeSpaceQueryURL);
88
      if (regexResult.size()) {
89
90
91
92
93
94
95
          try {
            //TODO:  TO BE CONTINUED
          cta::disk::JSONDiskSystem jsoncDiskSystem(currentDiskSystem);
          freeSpace = fetchEosFreeSpaceWithScript(m_systemList.getFetchEosFreeSpaceScript(),jsoncDiskSystem.getJSON(),lc);
        } catch(const cta::disk::FetchEosFreeSpaceScriptException &ex){
          
        }
96
97
98
99
100
101
102
103
104
105
106
        freeSpace = fetchEosFreeSpace(regexResult.at(1), regexResult.at(2), lc);
        goto found;
      }
      regexResult = constantFreeSpaceDiskSystem.exec(m_systemList.at(ds).freeSpaceQueryURL);
      if (regexResult.size()) {
        freeSpace = fetchConstantFreeSpace(regexResult.at(1), lc);
        goto found;
      }
      throw cta::disk::FetchEosFreeSpaceException("In DiskSystemFreeSpaceList::fetchDiskSystemFreeSpace(): could not interpret free space query URL.");
    } catch (const cta::disk::FetchEosFreeSpaceException &ex) {
      failedToFetchDiskSystems[ds] = ex;
107
108
109
110
111
112
113
    }
  found:
    DiskSystemFreeSpace & entry = operator [](ds);
    entry.freeSpace = freeSpace;
    entry.fetchTime = ::time(nullptr);
    entry.targetedFreeSpace = m_systemList.at(ds).targetedFreeSpace;
  }
114
115
116
117
118
  if(failedToFetchDiskSystems.size()){
    cta::disk::DiskSystemFreeSpaceListException ex;
    ex.m_failedDiskSystems = failedToFetchDiskSystems;
    throw ex;
  }
119
120
121
122
123
}

//------------------------------------------------------------------------------
// DiskSystemFreeSpaceList::fetchFileSystemFreeSpace()
//------------------------------------------------------------------------------
124
uint64_t DiskSystemFreeSpaceList::fetchEosFreeSpace(const std::string& instanceAddress, const std::string &spaceName, log::LogContext & lc) {
125
  threading::SubProcess sp("/usr/bin/eos", {"/usr/bin/eos", std::string("root://")+instanceAddress, "space", "ls", "-m"});
126
127
  sp.wait();
  try {
128
129
130
    exception::Errnum::throwOnNonZero(sp.exitValue(),
        std::string("In DiskSystemFreeSpaceList::fetchEosFreeSpace(), failed to call \"eos root://") + 
        instanceAddress + " space ls -m\"");
131
  } catch (exception::Exception & ex) {
132
    ex.getMessage() << " instanceAddress: " << instanceAddress << " stderr: " << sp.stderr();
133
    throw cta::disk::FetchEosFreeSpaceException(ex.getMessage().str());
134
135
136
137
  }
  if (sp.wasKilled()) {
    exception::Exception ex("In DiskSystemFreeSpaceList::fetchEosFreeSpace(): eos space ls -m killed by signal: ");
    ex.getMessage() << utils::toString(sp.killSignal());
138
    throw cta::disk::FetchEosFreeSpaceException(ex.getMessage().str());
139
140
141
142
  }
  // Look for the result line for default space.
  std::istringstream spStdoutIss(sp.stdout());
  std::string defaultSpaceLine;
143
  utils::Regex defaultSpaceRe("^.*name="+spaceName+" .*$");
144
145
146
147
148
149
  do {
    std::string spStdoutLine;
    std::getline(spStdoutIss, spStdoutLine);
    auto res = defaultSpaceRe.exec(spStdoutLine);
    if (res.size()) {
      defaultSpaceLine = res.at(0);
150
      goto spaceNameFound;
151
152
    }
  } while (!spStdoutIss.eof());
153
  throw cta::disk::FetchEosFreeSpaceException("In DiskSystemFreeSpaceList::fetchEosFreeSpace(): could not find the \""+spaceName+"\" in the eos space ls -m result.");
154
  
155
spaceNameFound:
156
  // Look for the parameters in the result line.
157
  utils::Regex rwSpaceRegex("sum.stat.statfs.freebytes\\?configstatus@rw=([0-9]+) ");
158
159
  auto rwSpaceRes = rwSpaceRegex.exec(defaultSpaceLine);
  if (rwSpaceRes.empty())
160
    throw cta::disk::FetchEosFreeSpaceException(
161
        "In DiskSystemFreeSpaceList::fetchEosFreeSpace(): failed to parse parameter sum.stat.statfs.capacity?configstatus@rw.");
162
  return utils::toUint64(rwSpaceRes.at(1));
163
164
}

165
166
167
//------------------------------------------------------------------------------
// DiskSystemFreeSpaceList::fetchFileSystemFreeSpace()
//------------------------------------------------------------------------------
168
uint64_t DiskSystemFreeSpaceList::fetchConstantFreeSpace(const std::string& instanceAddress, log::LogContext & lc) {
169
170
171
  return utils::toUint64(instanceAddress);
}

172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
//------------------------------------------------------------------------------
// DiskSystemFreeSpaceList::fetchEosFreeSpaceWithScript()
//------------------------------------------------------------------------------
uint64_t DiskSystemFreeSpaceList::fetchEosFreeSpaceWithScript(const std::string& scriptPath, const std::string& jsonInput, log::LogContext& lc){
  //TODO A CONTINUER
  cta::threading::SubProcess sp(scriptPath,{scriptPath},jsonInput);
   sp.wait();
  try {
    std::string errMsg = "In DiskSystemFreeSpaceList::fetchEosFreeSpaceWithScript(), failed to call \"" + scriptPath;
    exception::Errnum::throwOnNonZero(sp.exitValue(),errMsg);
  } catch (exception::Exception & ex) {
    ex.getMessage() << " scriptPath: " << scriptPath << " stderr: " << sp.stderr();
    throw cta::disk::FetchEosFreeSpaceScriptException(ex.getMessage().str());
  }
  if (sp.wasKilled()) {
    std::string errMsg = "In DiskSystemFreeSpaceList::fetchEosFreeSpaceWithScript(): " + scriptPath + " killed by signal: ";
    exception::Exception ex(errMsg);
    ex.getMessage() << utils::toString(sp.killSignal());
    throw cta::disk::FetchEosFreeSpaceScriptException(ex.getMessage().str());
  }
  // Look for the result line for default space.
  std::istringstream spStdoutIss(sp.stdout());
  lc.log(log::CRIT,spStdoutIss.str());
  throw cta::disk::FetchEosFreeSpaceScriptException("Test backpressure with script");
}
197

198
}} // namespace cta::disk