Commit 8aa7b55d authored by Steven Murray's avatar Steven Murray
Browse files

bug #90313: RFE: tapebridged should request more files to transfer in bulk

The implementation of this RFE is going to take a relatively long period of
time in that there will be several commits to SVN before it is completed.

This commit adds some helper classes to the tapebridged source-code base.
These classes are tested by the unit-tests of the test/unittest directory but
are not actively used by the tapebridged daemon and so will not effect any
release that may need to be made of the trunk.
parent c5d01b5c
/******************************************************************************
* castor/tape/tapebridge/BulkRequestConfigParams.cpp
*
* This file is part of the Castor project.
* See http://castor.web.cern.ch/castor
*
* Copyright (C) 2003 CERN
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
*
*
* @author Steven.Murray@cern.ch
*****************************************************************************/
#include "castor/tape/tapebridge/BulkRequestConfigParams.hpp"
#include "castor/tape/tapebridge/Constants.hpp"
//-----------------------------------------------------------------------------
// constructor
//-----------------------------------------------------------------------------
castor::tape::tapebridge::BulkRequestConfigParams::BulkRequestConfigParams():
m_bulkRequestMigrationMaxBytes("TAPEBRIDGE", "BULKREQUESTMIGRATIONMAXBYTES",
(uint64_t)0, "UNKNOWN"),
m_bulkRequestMigrationMaxFiles("TAPEBRIDGE", "BULKREQUESTMIGRATIONMAXFILES",
(uint64_t)0, "UNKNOWN"),
m_bulkRequestRecallMaxBytes("TAPEBRIDGE", "BULKREQUESTRECALLMAXBYTES",
(uint64_t)0, "UNKNOWN"),
m_bulkRequestRecallMaxFiles("TAPEBRIDGE", "BULKREQUESTRECALLMAXFILES",
(uint64_t)0, "UNKNOWN") {
// Do nothing
}
//-----------------------------------------------------------------------------
// getBulkRequestMigrationMaxBytes
//-----------------------------------------------------------------------------
const castor::tape::tapebridge::ConfigParamAndSource<uint64_t> &
castor::tape::tapebridge::BulkRequestConfigParams::
getBulkRequestMigrationMaxBytes() const {
return m_bulkRequestMigrationMaxBytes;
}
//-----------------------------------------------------------------------------
// getBulkRequestMigrationMaxFiles
//-----------------------------------------------------------------------------
const castor::tape::tapebridge::ConfigParamAndSource<uint64_t> &
castor::tape::tapebridge::BulkRequestConfigParams::
getBulkRequestMigrationMaxFiles() const {
return m_bulkRequestMigrationMaxFiles;
}
//-----------------------------------------------------------------------------
// getBulkRequestRecallMaxBytes
//-----------------------------------------------------------------------------
const castor::tape::tapebridge::ConfigParamAndSource<uint64_t> &
castor::tape::tapebridge::BulkRequestConfigParams::
getBulkRequestRecallMaxBytes() const {
return m_bulkRequestRecallMaxBytes;
}
//-----------------------------------------------------------------------------
// getBulkRequestRecallMaxFiles
//-----------------------------------------------------------------------------
const castor::tape::tapebridge::ConfigParamAndSource<uint64_t> &
castor::tape::tapebridge::BulkRequestConfigParams::
getBulkRequestRecallMaxFiles() const {
return m_bulkRequestRecallMaxFiles;
}
//-----------------------------------------------------------------------------
// determineConfigParams
//-----------------------------------------------------------------------------
void castor::tape::tapebridge::BulkRequestConfigParams::determineConfigParams()
throw(castor::exception::Exception) {
determineBulkRequestMigrationMaxBytes();
determineBulkRequestMigrationMaxFiles();
determineBulkRequestRecallMaxBytes();
determineBulkRequestRecallMaxFiles();
}
//-----------------------------------------------------------------------------
// determineBulkRequestMigrationMaxBytes
//-----------------------------------------------------------------------------
void castor::tape::tapebridge::BulkRequestConfigParams::
determineBulkRequestMigrationMaxBytes()
throw(castor::exception::Exception) {
determineUint64ConfigParam(m_bulkRequestMigrationMaxBytes,
TAPEBRIDGE_BULKREQUESTMIGRATIONMAXBYTES);
}
//-----------------------------------------------------------------------------
// determineBulkRequestMigrationMaxFiles
//-----------------------------------------------------------------------------
void castor::tape::tapebridge::BulkRequestConfigParams::
determineBulkRequestMigrationMaxFiles()
throw(castor::exception::Exception) {
determineUint64ConfigParam(m_bulkRequestMigrationMaxFiles,
TAPEBRIDGE_BULKREQUESTMIGRATIONMAXFILES);
}
//-----------------------------------------------------------------------------
// determineBulkRequestRecallMaxBytes
//-----------------------------------------------------------------------------
void castor::tape::tapebridge::BulkRequestConfigParams::
determineBulkRequestRecallMaxBytes()
throw(castor::exception::Exception) {
determineUint64ConfigParam(m_bulkRequestRecallMaxBytes,
TAPEBRIDGE_BULKREQUESTRECALLMAXBYTES);
}
//-----------------------------------------------------------------------------
// determineBulkRequestRecallMaxFiles
//-----------------------------------------------------------------------------
void castor::tape::tapebridge::BulkRequestConfigParams::
determineBulkRequestRecallMaxFiles()
throw(castor::exception::Exception) {
determineUint64ConfigParam(m_bulkRequestRecallMaxFiles,
TAPEBRIDGE_BULKREQUESTRECALLMAXFILES);
}
/******************************************************************************
* castor/tape/tapebridge/BulkRequestConfigParams.hpp
*
* This file is part of the Castor project.
* See http://castor.web.cern.ch/castor
*
* Copyright (C) 2003 CERN
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
*
*
* @author Steven.Murray@cern.ch
*****************************************************************************/
#ifndef CASTOR_TAPE_TAPEBRIDGE_BULKREQUESTCONFIGPARAMS_HPP
#define CASTOR_TAPE_TAPEBRIDGE_BULKREQUESTCONFIGPARAMS_HPP 1
#include "castor/exception/Exception.hpp"
#include "castor/tape/tapebridge/ConfigParamAndSource.hpp"
#include "castor/tape/tapebridge/ConfigParams.hpp"
#include <stdint.h>
namespace castor {
namespace tape {
namespace tapebridge {
/**
* Class used to determine and store the configuration parameters used by the
* tapebridged daemon to request in bulk files for migration and recall.
*/
class BulkRequestConfigParams: public ConfigParams {
public:
/**
* Constructor.
*/
BulkRequestConfigParams();
/**
* Determines the values of the tape-bridge configuration parameters by
* searching the environment variables, followed by castor.conf file and then
* by using the compile-time defaults.
*/
void determineConfigParams() throw(castor::exception::Exception);
/**
* Returns the bulkRequestMigrationMaxBytes configuration parameter.
*/
const ConfigParamAndSource<uint64_t> &getBulkRequestMigrationMaxBytes() const;
/**
* Returns the bulkRequestMigrationMaxFiles configuration parameter.
*/
const ConfigParamAndSource<uint64_t> &getBulkRequestMigrationMaxFiles() const;
/**
* Returns the bulkRequestRecallMaxBytes configuration parameter.
*/
const ConfigParamAndSource<uint64_t> &getBulkRequestRecallMaxBytes() const;
/**
* Returns the bulkRequestRecallMaxFiles configuration parameter.
*/
const ConfigParamAndSource<uint64_t> &getBulkRequestRecallMaxFiles() const;
protected:
/**
* Determines the value of the bulkRequestMigrationMaxBytes configuration
* parameter.
*
* All users of this class accept for unit testers should not call this
* method, but instead call determineConfigParams().
*
* This method determines the required value by first reading the
* environment variables, then if unsuccessful by reading castor.conf and
* finally if still unsuccessfull by using the compile-time default.
*/
void determineBulkRequestMigrationMaxBytes()
throw(castor::exception::Exception);
/**
* Determines the value of the bulkRequestMigrationMaxFiles configuration
* parameter.
*
* All users of this class accept for unit testers should not call this
* method, but instead call determineConfigParams().
*
* This method determines the required value by first reading the
* environment variables, then if unsuccessful by reading castor.conf and
* finally if still unsuccessfull by using the compile-time default.
*/
void determineBulkRequestMigrationMaxFiles()
throw(castor::exception::Exception);
/**
* Determines the value of the bulkRequestRecallMaxBytes configuration
* parameter.
*
* All users of this class accept for unit testers should not call this
* method, but instead call determineConfigParams().
*
* This method determines the required value by first reading the
* environment variables, then if unsuccessful by reading castor.conf and
* finally if still unsuccessfull by using the compile-time default.
*/
void determineBulkRequestRecallMaxBytes()
throw(castor::exception::Exception);
/**
* Determines the value of the bulkRequestRecallMaxFiles configuration
* parameter.
*
* All users of this class accept for unit testers should not call this
* method, but instead call determineConfigParams().
*
* This method determines the required value by first reading the
* environment variables, then if unsuccessful by reading castor.conf and
* finally if still unsuccessfull by using the compile-time default.
*/
void determineBulkRequestRecallMaxFiles()
throw(castor::exception::Exception);
private:
/**
* When the tapegatewayd daemon is asked for a set of files to migrate to
* tape, this configuration parameter specifies the maximum number of bytes
* the resulting set can represent. This number may be exceeded when the set
* contains a single file.
*/
ConfigParamAndSource<uint64_t> m_bulkRequestMigrationMaxBytes;
/**
* When the tapegatewayd daemon is asked for a set of files to migrate to
* tape, this configuration parameter specifies the maximum number of files
* that can be in that set.
*/
ConfigParamAndSource<uint64_t> m_bulkRequestMigrationMaxFiles;
/**
* When the tapegatewayd daemon is asked for a set of files to recall from
* tape, this configuration parameter specifies the maximum number of bytes
* the resulting set can represent. This number may be exceeded when the set
* contains a single file.
*/
ConfigParamAndSource<uint64_t> m_bulkRequestRecallMaxBytes;
/**
* When the tapegatewayd daemon is asked for a set of files to recall from
* tape, this configuration parameter specifies the maximum number of files
* that can be in that set.
*/
ConfigParamAndSource<uint64_t> m_bulkRequestRecallMaxFiles;
}; // class BulkRequestConfigParams
} // namespace tapebridge
} // namespace tape
} // namespace castor
#endif // CASTOR_TAPE_TAPEBRIDGE_BULKREQUESTCONFIGPARAMS_HPP
/******************************************************************************
* castor/tape/tapebridge/ConfigParams.cpp
*
* This file is part of the Castor project.
* See http://castor.web.cern.ch/castor
*
* Copyright (C) 2003 CERN
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
*
*
* @author Steven.Murray@cern.ch
*****************************************************************************/
#include "castor/exception/InvalidArgument.hpp"
#include "castor/tape/tapebridge/ConfigParams.hpp"
#include "castor/tape/utils/utils.hpp"
#include "h/getconfent.h"
#include "h/u64subr.h"
//------------------------------------------------------------------------------
// destructor
//------------------------------------------------------------------------------
castor::tape::tapebridge::ConfigParams::~ConfigParams() {
// Do nothing
}
//------------------------------------------------------------------------------
// determineUint64ConfigParam
//------------------------------------------------------------------------------
void
castor::tape::tapebridge::ConfigParams::determineUint64ConfigParam(
ConfigParamAndSource<uint64_t> &param,
const uint64_t compileTimeDefault)
throw(castor::exception::Exception) {
const std::string envVarName = param.category + "_" + param.name;
const char *valueCStr = NULL;
// Try to get the value from the environment variables, else try to get it
// from castor.conf
if(NULL != (valueCStr = getenv(envVarName.c_str()))) {
param.source = "environment variable";
} else if(NULL != (valueCStr = getconfent(param.category.c_str(),
param.name.c_str(), 0))) {
param.source = "castor.conf";
}
// If we got the value, then try to convert it to a uint64_t, else use the
// compile-time default
if(NULL != valueCStr) {
if(!utils::isValidUInt(valueCStr)) {
castor::exception::InvalidArgument ex;
ex.getMessage() <<
"Configuration parameter is not a valid unsigned integer"
": category=" << param.category <<
" name=" << param.name <<
" value=" << valueCStr <<
" source=" << param.source;
throw(ex);
}
param.value = strtou64(valueCStr);
} else {
param.source = "compile-time default";
param.value = compileTimeDefault;
}
}
/******************************************************************************
* castor/tape/tapebridge/ConfigParams.hpp
*
* This file is part of the Castor project.
* See http://castor.web.cern.ch/castor
*
* Copyright (C) 2003 CERN
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
*
*
* @author Steven.Murray@cern.ch
*****************************************************************************/
#ifndef CASTOR_TAPE_TAPEBRIDGE_CONFIGPARAMS_HPP
#define CASTOR_TAPE_TAPEBRIDGE_CONFIGPARAMS_HPP 1
#include "castor/exception/Exception.hpp"
#include "castor/tape/tapebridge/ConfigParamAndSource.hpp"
#include <stdint.h>
namespace castor {
namespace tape {
namespace tapebridge {
/**
* Abstract base-class of objects used to determine and store
* tapebridged-daemon configuration-parameters.
*/
class ConfigParams {
public:
/**
* Virtual destructor.
*/
virtual ~ConfigParams();
protected:
/**
* Determines the values of the tapebridged-daemon configuration-parameters
* for which this object is responsible.
*
* This method determines each required value by first reading the
* environment variables, then if unsuccessful by reading castor.conf and
* finally if still unsuccessfull by using the compile-time default.
*/
virtual void determineConfigParams() throw(castor::exception::Exception) = 0;
/**
* Determines the value of the specified configuration parameter.
*
* This method determines the required value by first reading the
* environment variables, then if unsuccessful by reading castor.conf and
* finally if still unsuccessfull by using the compile-time default.
*
* @param param In/Out Parameter: The configuration parameter to
* be determined and set.
* @param compileTimeDefault The compile-time default value for the
* configuration parameter.
*/
void determineUint64ConfigParam(
ConfigParamAndSource<uint64_t> &param,
const uint64_t compileTimeDefault)
throw(castor::exception::Exception);
}; // class ConfigParams
} // namespace tapebridge
} // namespace tape
} // namespace castor
#endif // CASTOR_TAPE_TAPEBRIDGE_CONFIGPARAMS_HPP
......@@ -36,7 +36,7 @@
namespace castor {
namespace tape {
namespace tapebridge {
/**
* The minumim timeout in seconds between pings to the tapebridge clients.
* Please note that the tape gateway is not pinged.
......@@ -164,6 +164,36 @@ namespace tapebridge {
*/
const uint64_t TAPEBRIDGE_MAXFILESBEFOREFLUSH = 1;
/**
* When the tapegatewayd daemon is asked for a set of files to migrate to
* tape, this is the compile-time default for the maximum number of bytes
* the resulting set can represent. This number may be exceeded when the set
* contains a single file.
*/
const uint64_t TAPEBRIDGE_BULKREQUESTMIGRATIONMAXBYTES = 1;
/**
* When the tapegatewayd daemon is asked for a set of files to migrate to
* tape, this is the compile-time default for the maximum number of files
* that can be in that set.
*/
const uint64_t TAPEBRIDGE_BULKREQUESTMIGRATIONMAXFILES = 1;
/**
* When the tapegatewayd daemon is asked for a set of files to recall from
* tape, this is the compile-time default for the maximum number of bytes
* the resulting set can represent. This number may be exceeded when the set
* contains a single file.
*/
const uint64_t TAPEBRIDGE_BULKREQUESTRECALLMAXBYTES = 1;
/**
* When the tapegatewayd daemon is asked for a set of files to recall from
* tape, this is the compile-time default for the maximum number of files
* that can be in that set.
*/
const uint64_t TAPEBRIDGE_BULKREQUESTRECALLMAXFILES = 1;
} // namespace tapebridge
} // namespace tape
} // namespace castor
......
/******************************************************************************
* castor/tape/tapebridge/FileToMigrate.cpp
*
* This file is part of the Castor project.
* See http://castor.web.cern.ch/castor
*
* Copyright (C) 2003 CERN
* This program is free software, you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation, either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY, without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program, if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
*
*
* @author Steven.Murray@cern.ch
*****************************************************************************/
#include "castor/tape/tapebridge/FileToMigrate.hpp"
//-----------------------------------------------------------------------------
// constructor
//-----------------------------------------------------------------------------
castor::tape::tapebridge::FileToMigrate::FileToMigrate():
fileTransactionId(0),
nsHost(""),
fileId(0),
fseq(0),
positionMethod(0),
fileSize(0),
lastKnownFilename(""),
lastModificationTime(0),
path(""),
umask(0) {
// Do nothing
}
/******************************************************************************
* castor/tape/tapebridge/FileToMigrate.hpp
*
* This file is part of the Castor project.
* See http://castor.web.cern.ch/castor
*
* Copyright (C) 2003 CERN
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
*
*
* @author Steven.Murray@cern.ch
*****************************************************************************/
#ifndef CASTOR_TAPE_TAPEBRIDGE_FILETOMIGRATE_HPP
#define CASTOR_TAPE_TAPEBRIDGE_FILETOMIGRATE_HPP 1
#include <stdint.h>
#include <string>
namespace castor {
namespace tape {
namespace tapebridge {
/**
* A file to be recalled from tape.
*/
struct FileToMigrate {
/**
* Constructor that initialises all integer member-values to 0 and string
* member-values to empty strings.
*/