Commit ab9ee163 authored by David COME's avatar David COME
Browse files

Skeleton is in place for compressed file size

parent 407125d3
......@@ -26,6 +26,7 @@
#include "castor/tape/tapeserver/daemon/MigrationReportPacker.hpp"
#include "castor/tape/tapegateway/FileErrorReportStruct.hpp"
#include "castor/tape/tapegateway/FileMigratedNotificationStruct.hpp"
#include <numeric>
#include <cstdio>
namespace{
......@@ -73,10 +74,10 @@ void MigrationReportPacker::reportFailedJob(const tapegateway::FileToMigrateStru
}
//------------------------------------------------------------------------------
//reportFlush
//------------------------------------------------------------------------------
void MigrationReportPacker::reportFlush() {
//------------------------------------------------------------------------------
void MigrationReportPacker::reportFlush(uint64_t nbByteWritenWithCompression){
castor::server::MutexLocker ml(&m_producterProtection);
m_fifo.push(new ReportFlush());
m_fifo.push(new ReportFlush(nbByteWritenWithCompression));
}
//------------------------------------------------------------------------------
//reportEndOfSession
......@@ -108,18 +109,60 @@ void MigrationReportPacker::ReportSuccessful::execute(MigrationReportPacker& _th
successMigration->setId(m_migratedFile.id());
successMigration->setNshost(m_migratedFile.nshost());
successMigration->setFileid(m_migratedFile.fileid());
//WARNING; Ad-hoc name of the ChecksumName !!");
successMigration->setChecksumName("adler32");
successMigration->setChecksum(m_checksum);
successMigration->setFileSize(m_migratedFile.fileSize());
// successMigration->setCompressedFileSize();
//WARNING; Ad-hoc name of the ChecksumName !!
successMigration->setChecksumName("adler32");
// successMigration->setBlockId0(m_migratedFile.BlockId0());
// successMigration->setBlockId1();
// successMigration->setBlockId2();
// successMigration->setBlockId3();
_this.m_listReports->addSuccessfulMigrations(successMigration.release());
}
//------------------------------------------------------------------------------
//ReportFlush::computeCompressedSize
//------------------------------------------------------------------------------
void MigrationReportPacker::ReportFlush::computeCompressedSize(
std::vector<tapegateway::FileMigratedNotificationStruct*>::iterator beg,
std::vector<tapegateway::FileMigratedNotificationStruct*>::iterator end
)
{
//lets pray for C++11 lamba and std accumulate
uint64_t rawSize = 0;
for(std::vector<tapegateway::FileMigratedNotificationStruct*>::iterator it = beg;
it != end ;++it){
rawSize+=(*it)->fileSize();
}
//we dont want compressionRatio to be equal to zero not to have a division by zero
double compressionRatio = nbByteWritenWithCompression>0 && rawSize >0 ?
1.0*rawSize/nbByteWritenWithCompression : 1.;
for(std::vector<tapegateway::FileMigratedNotificationStruct*>::iterator it = beg;
it != end ;++it){
const uint64_t compressedFileSize =
static_cast<uint64_t>((*it)->fileSize() * compressionRatio);
// The compressed file size should never be reported as being less than 1
// byte
uint64_t validCompressedFileSize = 0 < compressedFileSize ? compressedFileSize : 1;
(*it)->setCompressedFileSize(validCompressedFileSize);
}
}
//------------------------------------------------------------------------------
//ReportFlush::execute
//------------------------------------------------------------------------------
void MigrationReportPacker::ReportFlush::execute(MigrationReportPacker& _this){
computeCompressedSize(_this.m_listReports->successfulMigrations().begin(),
_this.m_listReports->successfulMigrations().end());
if(!_this.m_errorHappened){
_this.logReport(_this.m_listReports->successfulMigrations(),"A file was successfully written on the tape");
tapeserver::client::ClientInterface::RequestReport chrono;
......
......@@ -67,7 +67,7 @@ public:
/**
* Create into the MigrationReportPacker a report for the signaling a flusing on tape
*/
void reportFlush();
void reportFlush(uint64_t nbByteWritenWithCompression);
/**
* Create into the MigrationReportPacker a report for the nominal end of session
......@@ -106,7 +106,30 @@ private:
virtual void execute(MigrationReportPacker& _this);
};
class ReportFlush : public Report {
uint64_t nbByteWritenWithCompression;
/**
* This function will approximate the compressed size of the files which
* have been migrated. The idea is to compute the average ration
* logicalSize/nbByteWritenWithCompression for the whole batch
* and apply that ratio to the whole set of files
* We currently computing it only to the file that have been successfully
* migrated
* @param beg Beginning of the upper class' successfulMigrations()
* @param end End of upper class' successfulMigrations()
*/
void computeCompressedSize(
std::vector<tapegateway::FileMigratedNotificationStruct*>::iterator beg,
std::vector<tapegateway::FileMigratedNotificationStruct*>::iterator end);
public:
/* We only can compute the compressed size once we have flushed on the drive
* We can get from the drive the number of byte it really wrote to tape
* @param nbByte the number of byte it really wrote to tape between
* this flush and the previous one
* */
ReportFlush(uint64_t nbByte):nbByteWritenWithCompression(nbByte){}
void execute(MigrationReportPacker& _this);
};
class ReportError : public Report {
......
......@@ -45,7 +45,7 @@ TEST(castor_tape_tapeserver_daemon, MigrationReportPackerNominal) {
mrp.reportCompletedJob(migratedFile,0);
mrp.reportCompletedJob(migratedFile,0);
mrp.reportFlush();
mrp.reportFlush(0);
mrp.reportEndOfSession();
mrp.waitThread();
......@@ -71,7 +71,7 @@ TEST(castor_tape_tapeserver_daemon, MigrationReportPackerFaillure) {
mrp.reportCompletedJob(migratedFile,0);
mrp.reportCompletedJob(migratedFile,0);
mrp.reportFailedJob(failed,error,-1);
mrp.reportFlush();
mrp.reportFlush(0);
mrp.reportEndOfSessionWithErrors(error,-1);
mrp.waitThread();
......@@ -96,7 +96,7 @@ TEST(castor_tape_tapeserver_daemon, MigrationReportPackerFaillureGoodEnd) {
mrp.reportCompletedJob(migratedFile,0);
mrp.reportCompletedJob(migratedFile,0);
mrp.reportFailedJob(failed,error,-1);
mrp.reportFlush();
mrp.reportFlush(0);
mrp.reportEndOfSession();
mrp.waitThread();
......@@ -123,7 +123,7 @@ TEST(castor_tape_tapeserver_daemon, MigrationReportPackerGoodBadEnd) {
mrp.reportCompletedJob(migratedFile,0);
mrp.reportCompletedJob(migratedFile,0);
mrp.reportFlush();
mrp.reportFlush(0);
mrp.reportEndOfSessionWithErrors(error,-1);
mrp.waitThread();
......
......@@ -106,10 +106,11 @@ void RecallReportPacker::ReportSuccessful::execute(RecallReportPacker& parent){
successRecall->setId(m_recalledFile.id());
successRecall->setNshost(m_recalledFile.nshost());
successRecall->setFileid(m_recalledFile.fileid());
successRecall->setChecksum(m_checksum);
//WARNING : ad hoc name of checksum algorithm
successRecall->setChecksumName("adler32");
successRecall->setChecksum(m_checksum);
parent.m_listReports->addSuccessfulRecalls(successRecall.release());
}
//------------------------------------------------------------------------------
......
......@@ -95,7 +95,8 @@ tapeFlush(const std::string& message,uint64_t bytes,uint64_t files,
.add("bytes", bytes)
.add("flushTime", flushTime);
m_logContext.log(LOG_INFO,message);
m_reportPacker.reportFlush();
uint64_t nbByteWritenWithCompression = m_drive.nbByteWritenWithCompression();
m_reportPacker.reportFlush(nbByteWritenWithCompression);
m_stats.flushTime += flushTime;
}
//------------------------------------------------------------------------------
......
......@@ -637,6 +637,11 @@ void drives::DriveGeneric::SCSI_inquiry() {
<< SCSI::Structures::toString(*((SCSI::Structures::inquiryData_t *) dataBuff));
}
uint64_t drives::DriveGeneric::nbByteWritenWithCompression(){
return 0;
}
drives::compressionStats drives::DriveT10000::getCompression() {
compressionStats driveCompressionStats;
......
......@@ -303,6 +303,7 @@ namespace drives {
/**
*/
virtual uint64_t nbByteWritenWithCompression();
protected:
SCSI::DeviceInfo m_SCSIInfo;
int m_tapeFD;
......
......@@ -167,6 +167,8 @@ namespace drives {
virtual bool isAtEOD() = 0;
virtual bool isTapeBlank() = 0;
virtual bool hasTapeInPlace() = 0;
virtual uint64_t nbByteWritenWithCompression() = 0;
/**
* Member string allowing the convenient storage of the string describing
* drive location for the mount system (we get the information from TPCONFIG
......
......@@ -191,3 +191,7 @@ bool castor::tape::tapeserver::drives::FakeDrive::isTapeBlank() {
bool castor::tape::tapeserver::drives::FakeDrive::hasTapeInPlace() {
return true;
}
uint64_t castor::tape::tapeserver::drives::FakeDrive::nbByteWritenWithCompression(){
return 0;
}
......@@ -71,6 +71,7 @@ namespace drives {
virtual bool isAtEOD() ;
virtual bool isTapeBlank();
virtual bool hasTapeInPlace();
virtual uint64_t nbByteWritenWithCompression();
};
}}}}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment