Commit 603b0c8b authored by David COME's avatar David COME
Browse files

Merge remote branch 'origin/master'

parents e34ae786 258c329a
......@@ -302,7 +302,7 @@
# you can use a list like /your/mount/point/01/ /your/mount/point/02/ ...
#DiskManager MountPoints
# List of dataPools the DiskManager daemon should monitor
# List of dataPools the DiskManager daemon should monitor, garbage collect and synchronize
# you can use a list like pool1 pool2 ...
#DiskManager DataPools
......@@ -335,6 +335,13 @@
# stager catalog. The synchronization with the nameserver is not affected.
#GC DisableStagerSync no
# The period during which new files won't be considered for synchronization
# This protects in particular files being created (eg ongoing recalls) by giving
# time to the stager DB to create the associated DiskCopy. Otherwise, we would
# have a time window where the file exist on disk and can be considered by
# synchronization, while it does not exist on the stager. Thus it may be dropped
# Default is one day
#GC SyncGracePeriod 86400
## Security Configuration ######################################################
......
......@@ -906,12 +906,10 @@ INITRANS 50 PCTFREE 50 ENABLE ROW MOVEMENT;
CREATE INDEX I_DiskCopy_Castorfile ON DiskCopy (castorFile);
CREATE INDEX I_DiskCopy_FileSystem ON DiskCopy (fileSystem);
CREATE INDEX I_DiskCopy_DataPool ON DiskCopy (dataPool);
CREATE INDEX I_DiskCopy_FS_GCW ON DiskCopy (fileSystem, gcWeight);
CREATE INDEX I_DiskCopy_DP_GCW ON DiskCopy (dataPool, gcWeight);
CREATE INDEX I_DiskCopy_FS_DP_GCW ON DiskCopy (nvl(fileSystem,0)+nvl(dataPool,0), gcWeight);
-- for queries on active statuses
CREATE INDEX I_DiskCopy_Status_6 ON DiskCopy (decode(status,6,status,NULL));
CREATE INDEX I_DiskCopy_Status_7_FS ON DiskCopy (decode(status,7,status,NULL), fileSystem);
CREATE INDEX I_DiskCopy_Status_7_DP ON DiskCopy (decode(status,7,status,NULL), dataPool);
CREATE INDEX I_DiskCopy_Status_7_FS_DP ON DiskCopy (decode(status,7,status,NULL), nvl(fileSystem,0)+nvl(dataPool,0));
CREATE INDEX I_DiskCopy_Status_9 ON DiskCopy (decode(status,9,status,NULL));
-- to speed up deleteOutOfDateStageOutDCs
CREATE INDEX I_DiskCopy_Status_Open ON DiskCopy (decode(status,6,status,decode(status,5,status,decode(status,11,status,NULL))));
......
......@@ -256,12 +256,12 @@ BEGIN
-- Loop on all concerned fileSystems/DataPools in a random order.
totalCount := 0;
FOR fs IN (SELECT * FROM (SELECT FileSystem.id AS fsId, 0 AS dpId
FOR fs IN (SELECT * FROM (SELECT FileSystem.id AS fsId
FROM FileSystem, DiskServer
WHERE FileSystem.diskServer = DiskServer.id
AND DiskServer.name = diskServerName
UNION ALL
SELECT 0 AS fsId, DiskServer.dataPool AS dpId
SELECT DiskServer.dataPool AS fsId
FROM DiskServer
WHERE DiskServer.name = diskServerName)
ORDER BY dbms_random.value) LOOP
......@@ -271,7 +271,7 @@ BEGIN
SELECT totalCount + count(*), nvl(sum(DiskCopy.diskCopySize), 0)
INTO totalCount, freed
FROM DiskCopy
WHERE (fileSystem = fs.fsId OR dataPool = fs.dpId)
WHERE (fileSystem = fs.fsId OR dataPool = fs.fsId)
AND decode(status, 9, status, NULL) = 9; -- BEINGDELETED (decode used to use function-based index)
-- estimate the number of GC running the "long" query, that is the one dealing with the GCing of
......@@ -281,10 +281,10 @@ BEGIN
WHERE s.sql_id = t.sql_id AND t.sql_text LIKE '%I_DiskCopy_FS_GCW%';
-- Process diskcopies that are in an INVALID state.
UPDATE /*+ INDEX_RS_ASC(DiskCopy I_DiskCopy_Status_7_FS)) */ DiskCopy
UPDATE /*+ INDEX_RS_ASC(DiskCopy I_DiskCopy_Status_7_FS_DP)) */ DiskCopy
SET status = 9, -- BEINGDELETED
gcType = decode(gcType, NULL, dconst.GCTYPE_USER, gcType)
WHERE (fileSystem = fs.fsId OR dataPool = fs.dpId)
WHERE (nvl(fileSystem,0)+nvl(dataPool,0) = fs.fsId)
AND decode(status, 7, status, NULL) = 7 -- INVALID (decode used to use function-based index)
AND rownum <= 10000 - totalCount
RETURNING id BULK COLLECT INTO dcIds;
......@@ -315,7 +315,7 @@ BEGIN
ELSE
SELECT DiskServer.id INTO unused
FROM DiskServer
WHERE dataPool = fs.dpId
WHERE dataPool = fs.fsId
AND status IN (dconst.DISKSERVER_PRODUCTION, dconst.DISKSERVER_READONLY)
AND hwOnline = 1
AND ROWNUM < 2;
......@@ -342,15 +342,15 @@ BEGIN
SELECT decode(sign(maxFreeSpace * totalSize - free), -1, 0, maxFreeSpace * totalSize - free)
INTO toBeFreed
FROM DataPool
WHERE id = fs.dpId;
WHERE id = fs.fsId;
END IF;
-- If space is still required even after removal of INVALID files, consider
-- removing VALID files until we are below the free space watermark
IF freed < toBeFreed THEN
-- Loop on file deletions
FOR dc IN (SELECT /*+ INDEX_RS_ASC(DiskCopy I_DiskCopy_FS_GCW) */ DiskCopy.id, castorFile
FOR dc IN (SELECT /*+ INDEX_RS_ASC(DiskCopy I_DiskCopy_FS_DP_GCW) */ DiskCopy.id, castorFile
FROM DiskCopy, CastorFile
WHERE (fileSystem = fs.fsId OR dataPool = fs.dpId)
WHERE (nvl(fileSystem,0)+nvl(dataPool,0) = fs.fsId)
AND status = dconst.DISKCOPY_VALID
AND CastorFile.id = DiskCopy.castorFile
AND CastorFile.tapeStatus IN (dconst.CASTORFILE_DISKONLY, dconst.CASTORFILE_ONTAPE)
......@@ -416,12 +416,13 @@ BEGIN
AND DiskServer.name = diskServerName
UNION ALL
SELECT DiskCopy.castorFile,
DiskCopy.path AS path, DiskCopy.id,
DataPool.name || '/' || DiskCopy.path AS path, DiskCopy.id,
DiskCopy.lastAccessTime, DiskCopy.nbCopyAccesses, DiskCopy.gcWeight,
getObjStatusName('DiskCopy', 'gcType', DiskCopy.gcType) AS gcType,
getSvcClassListDP(DiskServer.dataPool) AS svcClassList
FROM DiskServer, DiskCopy
FROM DiskServer, DiskCopy, DataPool
WHERE decode(DiskCopy.status, 9, DiskCopy.status, NULL) = 9 -- BEINGDELETED
AND DiskCopy.dataPool = DataPool.id
AND DiskCopy.dataPool = DiskServer.dataPool
AND DiskServer.name = diskServerName
AND ROWNUM < 2) DC
......
......@@ -1439,7 +1439,7 @@ BEGIN
UPDATE /*+ INDEX(Subrequest PK_Subrequest_Id)*/ SubRequest
SET status = dconst.SUBREQUEST_FAILED,
errorCode = serrno.ENOSPC, -- No space left on device
errorMessage = 'File creation canceled since diskPool is full'
errorMessage = 'File creation canceled since pool is full'
WHERE id = srId;
RETURN;
END IF;
......
......@@ -24,13 +24,15 @@ cmake_minimum_required (VERSION 2.6)
################################################################################
# Rules to build and install gc
################################################################################
find_package (ceph REQUIRED)
include_directories(${RADOS_INCLUDE_DIR})
set(GCBIN_SRCS
GcDaemon.cpp
DeletionThread.cpp
SynchronizationThread.cpp)
SynchronizationThread.cpp
CephGlobals.cpp)
add_executable (gcd ${GCBIN_SRCS})
target_link_libraries (gcd castorcommon castordlf castorns castorclient)
target_link_libraries (gcd castorcommon castordlf castorns castorclient ${RADOS_LIBS})
install (TARGETS gcd DESTINATION ${CASTOR_DEST_BIN_DIR})
################################################################################
......
/******************************************************************************
* CephGlobals.cpp
*
* This file is part of the Castor project.
* See http://castor.web.cern.ch/castor
*
* Copyright (C) 2003 CERN
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* couple of global maps needed for interfacing with ceph libraries.
* Basically a cache of connections to ceph
*
* @author Dennis Waldron
*****************************************************************************/
#include "castor/gc/CephGlobals.hpp"
#include <map>
/// global variables holding ioCtx and stripers for each ceph pool
std::map<std::string, libradosstriper::RadosStriper*> g_radosStripers;
std::map<std::string, librados::IoCtx*> g_ioCtx;
librados::IoCtx* castor::gc::getRadosIoCtx(std::string pool) {
libradosstriper::RadosStriper* striper = castor::gc::getRadosStriper(pool);
if (0 == striper) return 0;
return g_ioCtx[pool];
}
libradosstriper::RadosStriper* castor::gc::getRadosStriper(std::string pool) {
std::map<std::string, libradosstriper::RadosStriper*>::iterator it =
g_radosStripers.find(pool);
if (it == g_radosStripers.end()) {
// we need to create a new radosStriper
librados::Rados cluster;
int rc = cluster.init(0);
if (rc) return 0;
rc = cluster.conf_read_file(NULL);
if (rc) {
cluster.shutdown();
return 0;
}
cluster.conf_parse_env(NULL);
rc = cluster.connect();
if (rc) {
cluster.shutdown();
return 0;
}
g_ioCtx[pool] = new librados::IoCtx();
librados::IoCtx *ioctx = g_ioCtx[pool];
rc = cluster.ioctx_create(pool.c_str(), *ioctx);
if (rc != 0) {
g_ioCtx.erase(pool);
return 0;
}
libradosstriper::RadosStriper *newStriper = new libradosstriper::RadosStriper;
rc = libradosstriper::RadosStriper::striper_create(*ioctx, newStriper);
if (rc != 0) {
delete newStriper;
g_ioCtx.erase(pool);
return 0;
}
it = g_radosStripers.insert(std::pair<std::string, libradosstriper::RadosStriper*>
(pool, newStriper)).first;
}
return it->second;
}
/******************************************************************************
* CephGlobals.hpp
*
* This file is part of the Castor project.
* See http://castor.web.cern.ch/castor
*
* Copyright (C) 2003 CERN
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* couple of global maps needed for interfacing with ceph libraries.
* Basically a cache of connections to ceph
*
* @author castor dev team
*****************************************************************************/
#pragma once
#include "castor/gc/CephGlobals.hpp"
#include <rados/librados.hpp>
#include <radosstriper/libradosstriper.hpp>
namespace castor {
namespace gc {
/// gets an IoCtx object for a given pool
librados::IoCtx* getRadosIoCtx(std::string pool);
/// gets a Striper object for a given pool
libradosstriper::RadosStriper* getRadosStriper(std::string pool);
}
}
......@@ -25,12 +25,14 @@
// Include files
#include "castor/gc/DeletionThread.hpp"
#include "castor/gc/CephGlobals.hpp"
#include "castor/Services.hpp"
#include "castor/Constants.hpp"
#include "castor/stager/IGCSvc.hpp"
#include "castor/stager/GCLocalFile.hpp"
#include "castor/System.hpp"
#include "getconfent.h"
#include <radosstriper/libradosstriper.hpp>
#include <vector>
#include <errno.h>
......@@ -273,24 +275,52 @@ void castor::gc::DeletionThread::run(void*) {
} // End of loop
}
//-----------------------------------------------------------------------------
// gcRemoveFilePath
//-----------------------------------------------------------------------------
void castor::gc::DeletionThread::gcRemoveFilePath
(std::string filepath, u_signed64 &filesize, u_signed64 &fileage)
{
struct stat64 fileinfo;
if (::stat64(filepath.c_str(), &fileinfo) ) {
castor::exception::Exception e(errno);
e.getMessage() << "Failed to stat file " << filepath;
throw e;
}
filesize = fileinfo.st_size;
fileage = time(NULL) - fileinfo.st_ctime;
if (unlink(filepath.c_str()) < 0) {
castor::exception::Exception e(errno);
e.getMessage() << "Failed to unlink file " << filepath;
throw e;
(std::string filepath, u_signed64 &filesize, u_signed64 &fileage) {
if (!filepath.empty() and filepath[0] == '/') {
// regular file, call POSIX API
struct stat64 fileinfo;
if (::stat64(filepath.c_str(), &fileinfo) ) {
castor::exception::Exception e(errno);
e.getMessage() << "Failed to stat file " << filepath;
throw e;
}
filesize = fileinfo.st_size;
fileage = time(NULL) - fileinfo.st_ctime;
if (unlink(filepath.c_str()) < 0) {
castor::exception::Exception e(errno);
e.getMessage() << "Failed to unlink file " << filepath;
throw e;
}
} else {
// ceph file
int slashPos = filepath.find('/');
std::string pool = filepath.substr(0,slashPos);
std::string filename = filepath.substr(slashPos+1);
libradosstriper::RadosStriper *striper = getRadosStriper(pool);
if (0 == striper) {
castor::exception::Exception e(EINVAL);
e.getMessage() << "Failed to connect to ceph while unlinking file " << filepath;
throw e;
}
time_t pmtime;
uint64_t pmsize;
int rc = striper->stat(filename, &pmsize, &pmtime);
if (rc) {
castor::exception::Exception e(-rc);
e.getMessage() << "Failed to stat ceph file " << filepath;
throw e;
}
filesize = pmsize;
fileage = time(NULL) - pmtime;
rc = striper->remove(filename);
if (rc) {
castor::exception::Exception e(-rc);
e.getMessage() << "Failed to unlink ceph file " << filepath;
throw e;
}
}
}
......@@ -140,7 +140,7 @@ castor::gc::GcDaemon::GcDaemon(std::ostream &stdOut, std::ostream &stdErr,
{ 23, "Unable to retrieve mountpoints, giving up with synchronization" },
{ 24, "Could not list filesystem directories, giving up with filesystem's synchronisation" },
{ 25, "Could not list filesystem subdirectory, ignoring it for synchronization" },
{ 27, "Deleting local file which is no longer in the nameserver" },
{ 27, "Deleting file which is no longer in the nameserver" },
{ 28, "Deletion of orphaned local file failed" },
{ 29, "Memory allocation failure" },
{ 30, "Synchronization configuration" },
......@@ -155,6 +155,10 @@ castor::gc::GcDaemon::GcDaemon(std::ostream &stdOut, std::ostream &stdErr,
{ 40, "Unexpected exception caught in synchronizeFiles" },
{ 41, "Failed to stat file" },
{ 43, "Could not get fileid from filepath, giving up for this file" },
{ 44, "Unable to get RadosStriper object. Ignoring file" },
{ 45, "Unable to retrieve IoCtx for DataPool" },
{ 46, "New synchronization grace period" },
{ 47, "Invalid GC/SyncGracePeriod option, using default" },
{ -1, "" }};
dlfInit(messages);
}
......@@ -26,12 +26,15 @@
// Include files
#include "castor/gc/SynchronizationThread.hpp"
#include "castor/gc/CephGlobals.hpp"
#include "castor/Services.hpp"
#include "castor/Constants.hpp"
#include "castor/stager/IGCSvc.hpp"
#include "castor/System.hpp"
#include "castor/exception/Exception.hpp"
#include "getconfent.h"
#include "serrno.h"
#include <radosstriper/libradosstriper.hpp>
#include <sys/types.h>
#include <sys/stat.h>
......@@ -49,222 +52,323 @@
#define DEFAULT_CHUNKINTERVAL 1800
#define DEFAULT_CHUNKSIZE 2000
#define DEFAULT_DISABLESTAGERSYNC false
#define DEFAULT_GRACEPERIOD 86400
/// current ceph pool. Needed as an extra backdoor argument to POSIX APIs
std::string g_pool;
//-----------------------------------------------------------------------------
// Constructor
//-----------------------------------------------------------------------------
castor::gc::SynchronizationThread::SynchronizationThread(int startDelay) :
m_startDelay(startDelay) { };
m_startDelay(startDelay), m_chunkInterval(DEFAULT_CHUNKINTERVAL),
m_chunkSize(DEFAULT_CHUNKSIZE), m_gracePeriod(DEFAULT_GRACEPERIOD),
m_disableStagerSync(DEFAULT_DISABLESTAGERSYNC)
{};
//-----------------------------------------------------------------------------
// Run
// syncLocalFile
//-----------------------------------------------------------------------------
void castor::gc::SynchronizationThread::run(void*) {
bool castor::gc::SynchronizationThread::syncLocalFile
(const std::string &path,
const char* fileName,
std::map<std::string, std::map<u_signed64, std::string> > &paths) {
// Ignore non regular files and files closed too recently
// This protects in particular recently created files by giving time
// to the stager DB to create the associated DiskCopy. Otherwise,
// we would have a time window where the file exist on disk and can
// be considered by us, while it does not exist on the stager. Thus
// we would drop it
struct stat64 filebuf;
std::string filepath (path + "/" + fileName);
if (stat64(filepath.c_str(), &filebuf) < 0) {
return false;
} else if (!(filebuf.st_mode & S_IFREG)) {
return false; // not a file
} else if (filebuf.st_mtime > time(NULL) - m_gracePeriod) {
return false;
}
// Extract the nameserver host and diskcopy id from the filename
std::pair<std::string, u_signed64> fid;
try {
fid = diskCopyIdFromFileName(fileName);
} catch (castor::exception::Exception& e) {
// "Ignoring filename that does not conform to castor naming
// conventions"
castor::dlf::Param params[] =
{castor::dlf::Param("Filename", fileName)};
castor::dlf::dlf_writep(nullCuuid, DLF_LVL_DEBUG, 39, 1, params);
return false;
}
// "Starting synchronization thread"
castor::dlf::dlf_writep(nullCuuid, DLF_LVL_SYSTEM, 18);
sleep(m_startDelay);
paths[fid.first][fid.second] = filepath;
// Get the synchronization interval and chunk size
unsigned int chunkInterval = DEFAULT_CHUNKINTERVAL;
unsigned int chunkSize = DEFAULT_CHUNKSIZE;
bool disableStagerSync = DEFAULT_DISABLESTAGERSYNC;
readConfigFile(&chunkInterval, &chunkSize, &disableStagerSync, true);
// In the case of a large number of files, synchronize them in
// chunks so to not overwhelm central services
return checkAndSyncChunk(fid.first, paths, m_chunkSize);
}
// Endless loop
for (;;) {
//-----------------------------------------------------------------------------
// syncCephFile
//-----------------------------------------------------------------------------
bool castor::gc::SynchronizationThread::syncCephFile
(const std::string fileName,
std::map<std::string, std::map<u_signed64, std::string> > &paths) {
// Ignore files closed too recently
// This protects in particular recently recalled files by giving time
// to the stager DB to create the associated DiskCopy. Otherwise,
// we would have a time window where the file exist on disk and can
// be considered by us, while it does not exist on the stager. Thus
// we would drop it
libradosstriper::RadosStriper *striper = getRadosStriper(g_pool);
if (0 == striper) {
castor::dlf::Param params[] =
{castor::dlf::Param("FileName", fileName)};
// log "Unable to get RadosStriper object. Ignoring file"
castor::dlf::dlf_writep(nullCuuid, DLF_LVL_ERROR, 44, 1, params);
return false;
}
time_t pmtime;
uint64_t fsize;
if (striper->stat(fileName.c_str(), &fsize, &pmtime) != 0) {
return false;
} else if (pmtime > time(NULL) - m_gracePeriod) {
return false;
}
// Extract the nameserver host and diskcopy id from the filename
std::pair<std::string, u_signed64> fid;
try {
fid = diskCopyIdFromFileName(fileName);
} catch (castor::exception::Exception& e) {
// "Ignoring filename that does not conform to castor naming conventions"
castor::dlf::Param params[] =
{castor::dlf::Param("Filename", fileName)};
castor::dlf::dlf_writep(nullCuuid, DLF_LVL_DEBUG, 39, 1, params);
return false;
}
paths[fid.first][fid.second] = fileName;
// In the case of a large number of files, synchronize them in
// chunks so to not overwhelm central services
return checkAndSyncChunk(fid.first, paths, m_chunkSize);
}
// Get the synchronization interval and chunk size these may have changed
// since the last iteration
readConfigFile(&chunkInterval, &chunkSize, &disableStagerSync);
if (chunkInterval <= 0) {
// just do nothing if interval = 0
sleep(300);
return;
}
//-----------------------------------------------------------------------------
// syncFileSystems
//-----------------------------------------------------------------------------
void castor::gc::SynchronizationThread::syncFileSystems() {
// Get the list of filesystem to be checked
char** fs;
int nbFs;
if (getconfent_multi("DiskManager", "MountPoints", 1, &fs, &nbFs) < 0) {
// "Unable to retrieve mountpoints, giving up with synchronization"
castor::dlf::dlf_writep(nullCuuid, DLF_LVL_ERROR, 23);
sleep(m_chunkInterval);
return;
}
// Get the list of filesystem to be checked
char** fs;
int nbFs;
if (getconfent_multi("DiskManager", "MountPoints", 1, &fs, &nbFs) < 0) {
// "Unable to retrieve mountpoints, giving up with synchronization"
castor::dlf::dlf_writep(nullCuuid, DLF_LVL_ERROR, 23);
sleep(chunkInterval);
// Loop over the fileSystems starting in a random place
std::map<std::string, std::map<u_signed64, std::string> > paths;
int fsIt = (int) (nbFs * (rand() / (RAND_MAX + 1.0)));
for (int i = 0; i < nbFs; i++) {
// List the filesystem directories in random order
std::vector<std::string> directories;
DIR *dirs = opendir(fs[fsIt]);
if (0 == dirs) {
// "Could not list filesystem directories"
castor::dlf::Param params[] =
{castor::dlf::Param("FileSystem", fs[fsIt]),
castor::dlf::Param("Error", strerror(errno))};
castor::dlf::dlf_writep(nullCuuid, DLF_LVL_ERROR, 24, 2, params);
sleep(m_chunkInterval);
continue;
}
struct dirent *dir;
while ((dir = readdir(dirs))) {
struct stat64 file;
std::ostringstream filepath;
filepath << fs[fsIt] << dir->d_name;
if (stat64(filepath.str().c_str(), &file) < 0) {
continue;
} else if (!(file.st_mode & S_IFDIR)) {
continue; // not a directory
} else if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) {
continue;
} else if (strspn(dir->d_name, "0123456789") != strlen(dir->d_name)
|| (strlen(dir->d_name) != 2)) {
continue; // not a numbered directory name between 00 and 99
}
int offset = (int) ((1 + directories.size()) *
(rand() / (RAND_MAX + 1.0)));
directories.insert
(directories.begin() + offset, filepath.str().c_str());
}
closedir(dirs);
// Initialize random number generator
srand(time(0));
// Loop over the fileSystems starting in a random place
std::map<std::string, std::map<u_signed64, std::string> > paths;
int fsIt = (int) (nbFs * (rand() / (RAND_MAX + 1.0)));
for (int i = 0; i < nbFs; i++) {
// Loop over the directories
for (std::vector<std::string>::const_iterator it =
directories.begin();
it != directories.end();
it++) {
// List the filesystem directories in random order
std::vector<std::string> directories;
DIR *dirs = opendir(fs[fsIt]);
if (0 == dirs) {
// "Could not list filesystem directories"
// Loop over files inside a directory
DIR *files = opendir(it->c_str());
if (0 == files) {
// "Could not list filesystem subdirectory"
castor::dlf::Param params[] =