From 7c47bd03f991e5a20a191a3f46572ee007cddda8 Mon Sep 17 00:00:00 2001 From: Daniele Kruse <dkruse@cern.ch> Date: Wed, 11 Mar 2015 15:55:07 +0100 Subject: [PATCH] WIP: VFS and Sqlite backend --- libs/middletier/SqliteDatabase.cpp | 180 ++++++++++++++++++++--- libs/middletier/SqliteDatabase.hpp | 15 +- libs/middletier/SqliteMiddleTierUser.cpp | 26 ++-- libs/middletier/Tape.cpp | 7 + libs/middletier/Tape.hpp | 7 + libs/middletier/Vfs.cpp | 7 + libs/middletier/Vfs.hpp | 2 + 7 files changed, 207 insertions(+), 37 deletions(-) diff --git a/libs/middletier/SqliteDatabase.cpp b/libs/middletier/SqliteDatabase.cpp index 569fd39446..9786ee106c 100644 --- a/libs/middletier/SqliteDatabase.cpp +++ b/libs/middletier/SqliteDatabase.cpp @@ -280,10 +280,12 @@ void cta::SqliteDatabase::createArchivalJobTable() { "STATE INTEGER," "SRCURL TEXT," "DSTPATH TEXT," + "TAPEPOOL_NAME TEXT," "UID INTEGER," "GID INTEGER," "CREATIONTIME INTEGER," - "PRIMARY KEY (DSTPATH)" + "PRIMARY KEY (DSTPATH)," + "FOREIGN KEY (TAPEPOOL_NAME) REFERENCES TAPEPOOL(NAME)" ");", 0, 0, &zErrMsg); if(rc!=SQLITE_OK){ @@ -304,10 +306,12 @@ void cta::SqliteDatabase::createRetrievalJobTable() { "STATE INTEGER," "SRCPATH TEXT," "DSTURL TEXT," + "VID TEXT," "UID INTEGER," "GID INTEGER," "CREATIONTIME INTEGER," - "PRIMARY KEY (DSTURL)" + "PRIMARY KEY (DSTURL)," + "FOREIGN KEY (VID) REFERENCES TAPE(VID)" ");", 0, 0, &zErrMsg); if(rc!=SQLITE_OK){ @@ -386,10 +390,10 @@ void cta::SqliteDatabase::insertAdminHost(const SecurityIdentity &requester, con //------------------------------------------------------------------------------ // insertArchivalJob //------------------------------------------------------------------------------ -void cta::SqliteDatabase::insertArchivalJob(const SecurityIdentity &requester, const std::string &srcUrl, const std::string &dstPath) { +void cta::SqliteDatabase::insertArchivalJob(const SecurityIdentity &requester, const std::string &tapepool, const std::string &srcUrl, const std::string &dstPath) { char *zErrMsg = 0; std::ostringstream query; - query << "INSERT INTO ARCHIVALJOB VALUES(0,'" << srcUrl << "','" << dstPath << "',"<< requester.user.getUid() << "," << requester.user.getGid() << "," << (int)time(NULL) << ");"; + query << "INSERT INTO ARCHIVALJOB VALUES(" << (int)cta::ArchivalJobState::PENDING << ",'" << srcUrl << "','" << dstPath << "','" << tapepool << "',"<< requester.user.getUid() << "," << requester.user.getGid() << "," << (int)time(NULL) << ");"; int rc = sqlite3_exec(m_dbHandle, query.str().c_str(), 0, 0, &zErrMsg); if(rc!=SQLITE_OK){ std::ostringstream message; @@ -402,10 +406,10 @@ void cta::SqliteDatabase::insertArchivalJob(const SecurityIdentity &requester, c //------------------------------------------------------------------------------ // insertRetrievalJob //------------------------------------------------------------------------------ -void cta::SqliteDatabase::insertRetrievalJob(const SecurityIdentity &requester, const std::string &srcPath, const std::string &dstUrl) { +void cta::SqliteDatabase::insertRetrievalJob(const SecurityIdentity &requester, const std::string &vid, const std::string &srcPath, const std::string &dstUrl) { char *zErrMsg = 0; std::ostringstream query; - query << "INSERT INTO RETRIEVALJOB VALUES(0,'" << srcPath << "','" << dstUrl << "',"<< requester.user.getUid() << "," << requester.user.getGid() << "," << (int)time(NULL) << ");"; + query << "INSERT INTO RETRIEVALJOB VALUES(" << (int)cta::RetrievalJobState::PENDING << ",'" << srcPath << "','" << dstUrl << "','" << vid << "',"<< requester.user.getUid() << "," << requester.user.getGid() << "," << (int)time(NULL) << ");"; int rc = sqlite3_exec(m_dbHandle, query.str().c_str(), 0, 0, &zErrMsg); if(rc!=SQLITE_OK){ std::ostringstream message; @@ -1267,10 +1271,10 @@ std::list<cta::ArchiveRoute> cta::SqliteDatabase::selectAllArchiveRoutes(const sqlite3_stmt *statement; int rc = sqlite3_prepare(m_dbHandle, query.str().c_str(), -1, &statement, 0 ); if(rc!=SQLITE_OK){ - std::ostringstream message; - message << "selectAllArchiveRoutes() - SQLite error: " << zErrMsg; - sqlite3_free(zErrMsg); - throw(Exception(message.str())); + std::ostringstream message; + message << "selectAllArchiveRoutes() - SQLite error: " << zErrMsg; + sqlite3_free(zErrMsg); + throw(Exception(message.str())); } while(sqlite3_step(statement)==SQLITE_ROW) { routes.push_back(cta::ArchiveRoute( @@ -1286,6 +1290,48 @@ std::list<cta::ArchiveRoute> cta::SqliteDatabase::selectAllArchiveRoutes(const return routes; } +//------------------------------------------------------------------------------ +// getArchiveRouteOfStorageClass +//------------------------------------------------------------------------------ +cta::ArchiveRoute cta::SqliteDatabase::getArchiveRouteOfStorageClass(const SecurityIdentity &requester, const std::string &storageClassName, const uint16_t copyNb) { + char *zErrMsg = 0; + std::ostringstream query; + query << "SELECT TAPEPOOL_NAME FROM ARCHIVEROUTE WHERE STORAGECLASS_NAME='"<< storageClassName <<"' AND COPYNB="<< (int)copyNb <<";"; + sqlite3_stmt *statement; + int rc = sqlite3_prepare(m_dbHandle, query.str().c_str(), -1, &statement, 0 ); + if(rc!=SQLITE_OK){ + std::ostringstream message; + message << "getArchiveRouteOfStorageClass() - SQLite error: " << zErrMsg; + sqlite3_free(zErrMsg); + throw(Exception(message.str())); + } + cta::ArchiveRoute route; + int res = sqlite3_step(statement); + if(res==SQLITE_ROW) { + route = cta::ArchiveRoute( + std::string((char *)sqlite3_column_text(statement,0)), + sqlite3_column_int(statement,1), + std::string((char *)sqlite3_column_text(statement,2)), + cta::UserIdentity(sqlite3_column_int(statement,3),sqlite3_column_int(statement,4)), + time_t(sqlite3_column_int(statement,5)), + std::string((char *)sqlite3_column_text(statement,6)) + ); + } + else if(res==SQLITE_DONE) { + std::ostringstream message; + message << "getArchiveRouteOfStorageClass() - No archive route found for storage class: " << storageClassName << " and copynb: "<< (int)copyNb; + throw(Exception(message.str())); + } + else { + std::ostringstream message; + message << "getArchiveRouteOfStorageClass() - SQLite error: " << zErrMsg; + sqlite3_free(zErrMsg); + throw(Exception(message.str())); + } + sqlite3_finalize(statement); + return route; +} + //------------------------------------------------------------------------------ // selectAllTapes //------------------------------------------------------------------------------ @@ -1377,10 +1423,10 @@ std::list<cta::AdminHost> cta::SqliteDatabase::selectAllAdminHosts(const Securit //------------------------------------------------------------------------------ // selectAllArchivalJobs //------------------------------------------------------------------------------ -std::list<cta::ArchivalJob> cta::SqliteDatabase::selectAllArchivalJobs(const SecurityIdentity &requester) { +std::map<cta::TapePool, std::list<cta::ArchivalJob> > cta::SqliteDatabase::selectAllArchivalJobs(const SecurityIdentity &requester) { char *zErrMsg = 0; std::ostringstream query; - std::list<cta::ArchivalJob> list; + std::map<cta::TapePool, std::list<cta::ArchivalJob> > map; query << "SELECT * FROM ARCHIVALJOB ORDER BY DSTPATH;"; sqlite3_stmt *statement; int rc = sqlite3_prepare(m_dbHandle, query.str().c_str(), -1, &statement, 0 ); @@ -1391,25 +1437,115 @@ std::list<cta::ArchivalJob> cta::SqliteDatabase::selectAllArchivalJobs(const Sec throw(Exception(message.str())); } while(sqlite3_step(statement)==SQLITE_ROW) { - list.push_back(cta::ArchivalJob( + map[getTapePoolByName(requester, std::string((char *)sqlite3_column_text(statement,3)))].push_back(cta::ArchivalJob( (cta::ArchivalJobState::Enum)sqlite3_column_int(statement,0), std::string((char *)sqlite3_column_text(statement,1)), std::string((char *)sqlite3_column_text(statement,2)), - cta::UserIdentity(sqlite3_column_int(statement,3),sqlite3_column_int(statement,4)), - time_t(sqlite3_column_int(statement,5)) + cta::UserIdentity(sqlite3_column_int(statement,4),sqlite3_column_int(statement,5)), + time_t(sqlite3_column_int(statement,6)) )); } sqlite3_finalize(statement); - return list; + return map; +} + +//------------------------------------------------------------------------------ +// getTapePoolByName +//------------------------------------------------------------------------------ +cta::TapePool cta::SqliteDatabase::getTapePoolByName(const SecurityIdentity &requester, const std::string &name) { + char *zErrMsg = 0; + std::ostringstream query; + cta::TapePool pool; + query << "SELECT * FROM TAPEPOOL WHERE NAME='" << name << "';"; + sqlite3_stmt *statement; + int rc = sqlite3_prepare(m_dbHandle, query.str().c_str(), -1, &statement, 0 ); + if(rc!=SQLITE_OK){ + std::ostringstream message; + message << "getTapePoolByName() - SQLite error: " << zErrMsg; + sqlite3_free(zErrMsg); + throw(Exception(message.str())); + } + int res = sqlite3_step(statement); + if(res==SQLITE_ROW) { + pool = cta::TapePool( + std::string((char *)sqlite3_column_text(statement,0)), + sqlite3_column_int(statement,1), + sqlite3_column_int(statement,2), + cta::UserIdentity(sqlite3_column_int(statement,3),sqlite3_column_int(statement,4)), + time_t(sqlite3_column_int(statement,5)), + std::string((char *)sqlite3_column_text(statement,6)) + ); + } + else if(res==SQLITE_DONE) { + std::ostringstream message; + message << "getTapePoolByName() - No tape pool found with name: " << name; + sqlite3_finalize(statement); + throw(Exception(message.str())); + } + else { + std::ostringstream message; + message << "getTapePoolByName() - SQLite error: " << zErrMsg; + sqlite3_free(zErrMsg); + sqlite3_finalize(statement); + throw(Exception(message.str())); + } + sqlite3_finalize(statement); + return pool; +} + +//------------------------------------------------------------------------------ +// getTapeByVid +//------------------------------------------------------------------------------ +cta::Tape cta::SqliteDatabase::getTapeByVid(const SecurityIdentity &requester, const std::string &vid) { + char *zErrMsg = 0; + std::ostringstream query; + cta::Tape tape; + query << "SELECT * FROM TAPE WHERE VID='" << vid << "';"; + sqlite3_stmt *statement; + int rc = sqlite3_prepare(m_dbHandle, query.str().c_str(), -1, &statement, 0 ); + if(rc!=SQLITE_OK){ + std::ostringstream message; + message << "getTapeByVid() - SQLite error: " << zErrMsg; + sqlite3_free(zErrMsg); + throw(Exception(message.str())); + } + int res = sqlite3_step(statement); + if(res==SQLITE_ROW) { + tape = cta::Tape( + std::string((char *)sqlite3_column_text(statement,0)), + std::string((char *)sqlite3_column_text(statement,1)), + std::string((char *)sqlite3_column_text(statement,2)), + sqlite3_column_int(statement,3), + sqlite3_column_int(statement,4), + cta::UserIdentity(sqlite3_column_int(statement,5),sqlite3_column_int(statement,6)), + time_t(sqlite3_column_int(statement,7)), + std::string((char *)sqlite3_column_text(statement,8)) + ); + } + else if(res==SQLITE_DONE) { + std::ostringstream message; + message << "getTapeByVid() - No tape found with vid: " << vid; + sqlite3_finalize(statement); + throw(Exception(message.str())); + } + else { + std::ostringstream message; + message << "getTapeByVid() - SQLite error: " << zErrMsg; + sqlite3_free(zErrMsg); + sqlite3_finalize(statement); + throw(Exception(message.str())); + } + sqlite3_finalize(statement); + return tape; } //------------------------------------------------------------------------------ // selectAllRetrievalJobs //------------------------------------------------------------------------------ -std::list<cta::RetrievalJob> cta::SqliteDatabase::selectAllRetrievalJobs(const SecurityIdentity &requester) { +std::map<cta::Tape, std::list<cta::RetrievalJob> > cta::SqliteDatabase::selectAllRetrievalJobs(const SecurityIdentity &requester) { char *zErrMsg = 0; std::ostringstream query; - std::list<cta::RetrievalJob> list; + std::map<cta::Tape, std::list<cta::RetrievalJob> > map; query << "SELECT * FROM RETRIEVALJOB ORDER BY DSTURL;"; sqlite3_stmt *statement; int rc = sqlite3_prepare(m_dbHandle, query.str().c_str(), -1, &statement, 0 ); @@ -1420,16 +1556,16 @@ std::list<cta::RetrievalJob> cta::SqliteDatabase::selectAllRetrievalJobs(const S throw(Exception(message.str())); } while(sqlite3_step(statement)==SQLITE_ROW) { - list.push_back(cta::RetrievalJob( + map[getTapeByVid(requester, std::string((char *)sqlite3_column_text(statement,3)))].push_back(cta::RetrievalJob( (cta::RetrievalJobState::Enum)sqlite3_column_int(statement,0), std::string((char *)sqlite3_column_text(statement,1)), std::string((char *)sqlite3_column_text(statement,2)), - cta::UserIdentity(sqlite3_column_int(statement,3),sqlite3_column_int(statement,4)), - time_t(sqlite3_column_int(statement,5)) + cta::UserIdentity(sqlite3_column_int(statement,4),sqlite3_column_int(statement,5)), + time_t(sqlite3_column_int(statement,6)) )); } sqlite3_finalize(statement); - return list; + return map; } //------------------------------------------------------------------------------ diff --git a/libs/middletier/SqliteDatabase.hpp b/libs/middletier/SqliteDatabase.hpp index 3edca48a62..13117b811e 100644 --- a/libs/middletier/SqliteDatabase.hpp +++ b/libs/middletier/SqliteDatabase.hpp @@ -1,6 +1,7 @@ #pragma once #include <list> +#include <map> #include <sqlite3.h> @@ -46,9 +47,9 @@ public: void insertAdminHost(const SecurityIdentity &requester, const std::string &hostName, const std::string &comment); - void insertArchivalJob(const SecurityIdentity &requester, const std::string &srcUrl, const std::string &dstPath); + void insertArchivalJob(const SecurityIdentity &requester, const std::string &tapepool, const std::string &srcUrl, const std::string &dstPath); - void insertRetrievalJob(const SecurityIdentity &requester, const std::string &srcPath, const std::string &dstUrl); + void insertRetrievalJob(const SecurityIdentity &requester, const std::string &vid, const std::string &srcPath, const std::string &dstUrl); void insertLogicalLibrary(const SecurityIdentity &requester, const std::string &name, const std::string &comment); @@ -82,12 +83,18 @@ public: std::list<cta::AdminHost> selectAllAdminHosts(const SecurityIdentity &requester); - std::list<cta::ArchivalJob> selectAllArchivalJobs(const SecurityIdentity &requester); + std::map<cta::TapePool, std::list<cta::ArchivalJob> > selectAllArchivalJobs(const SecurityIdentity &requester); - std::list<cta::RetrievalJob> selectAllRetrievalJobs(const SecurityIdentity &requester); + std::map<cta::Tape, std::list<cta::RetrievalJob> > selectAllRetrievalJobs(const SecurityIdentity &requester); std::list<cta::LogicalLibrary> selectAllLogicalLibraries(const SecurityIdentity &requester); + cta::ArchiveRoute getArchiveRouteOfStorageClass(const SecurityIdentity &requester, const std::string &storageClassName, const uint16_t copyNb); + + cta::TapePool getTapePoolByName(const SecurityIdentity &requester, const std::string &name); + + cta::Tape getTapeByVid(const SecurityIdentity &requester, const std::string &vid); + diff --git a/libs/middletier/SqliteMiddleTierUser.cpp b/libs/middletier/SqliteMiddleTierUser.cpp index 1b3fae144b..1d98f79e0f 100644 --- a/libs/middletier/SqliteMiddleTierUser.cpp +++ b/libs/middletier/SqliteMiddleTierUser.cpp @@ -109,10 +109,11 @@ void cta::SqliteMiddleTierUser::archiveToDirectory( const std::string &dstFileName = *itor; m_vfs.createFile(requester, dstDir+dstFileName, 0666); } - + std::string storageClass = m_vfs.getDirectoryStorageClass(requester, dstDir); + cta::ArchiveRoute route = m_sqlite_db.getArchiveRouteOfStorageClass(requester, storageClass, 1); for(std::list<std::string>::const_iterator itor = srcUrls.begin(); itor != srcUrls.end(); itor++) { const std::string &srcFileName = *itor; - m_sqlite_db.insertArchivalJob(requester, srcFileName, dstDir); + m_sqlite_db.insertArchivalJob(requester, route.getTapePoolName(), srcFileName, dstDir); } } @@ -131,7 +132,9 @@ void cta::SqliteMiddleTierUser::archiveToFile( const std::string &srcFileName = srcUrls.front(); m_vfs.createFile(requester, dstFile, 0666); - m_sqlite_db.insertArchivalJob(requester, srcFileName, dstFile); + std::string storageClass = m_vfs.getDirectoryStorageClass(requester, cta::Utils::getEnclosingDirPath(dstFile)); + cta::ArchiveRoute route = m_sqlite_db.getArchiveRouteOfStorageClass(requester, storageClass, 1); + m_sqlite_db.insertArchivalJob(requester, route.getTapePoolName(), srcFileName, dstFile); } //------------------------------------------------------------------------------ @@ -140,7 +143,7 @@ void cta::SqliteMiddleTierUser::archiveToFile( std::map<cta::TapePool, std::list<cta::ArchivalJob> > cta::SqliteMiddleTierUser::getArchivalJobs( const SecurityIdentity &requester) const { - throw(Exception("Not Implemented!")); + return m_sqlite_db.selectAllArchivalJobs(requester); } //------------------------------------------------------------------------------ @@ -149,8 +152,7 @@ std::map<cta::TapePool, std::list<cta::ArchivalJob> > std::list<cta::ArchivalJob> cta::SqliteMiddleTierUser::getArchivalJobs( const SecurityIdentity &requester, const std::string &tapePoolName) const { - //TODO - return m_sqlite_db.selectAllArchivalJobs(requester); + return (m_sqlite_db.selectAllArchivalJobs(requester))[m_sqlite_db.getTapePoolByName(requester, tapePoolName)]; } //------------------------------------------------------------------------------ @@ -168,8 +170,11 @@ void cta::SqliteMiddleTierUser::deleteArchivalJob( void cta::SqliteMiddleTierUser::retrieve( const SecurityIdentity &requester, const std::list<std::string> &srcPaths, - const std::string &dstUrl) { - //TODO + const std::string &dstUrl) { //we consider only the case in which dstUrl is a directory so that we accept multiple source files + for(std::list<std::string>::const_iterator it=srcPaths.begin(); it!=srcPaths.end(); it++) { + std::string vid = m_vfs.getVidOfFile(requester, *it, 1); //we only consider 1st copy + m_sqlite_db.insertRetrievalJob(requester, vid, *it, dstUrl); + } } //------------------------------------------------------------------------------ @@ -178,7 +183,7 @@ void cta::SqliteMiddleTierUser::retrieve( std::map<cta::Tape, std::list<cta::RetrievalJob> > cta::SqliteMiddleTierUser::getRetrievalJobs( const SecurityIdentity &requester) const { - throw(Exception("Not Implemented!")); + return m_sqlite_db.selectAllRetrievalJobs(requester); } //------------------------------------------------------------------------------ @@ -187,8 +192,7 @@ std::map<cta::Tape, std::list<cta::RetrievalJob> > std::list<cta::RetrievalJob> cta::SqliteMiddleTierUser::getRetrievalJobs( const SecurityIdentity &requester, const std::string &vid) const { - //TODO - return m_sqlite_db.selectAllRetrievalJobs(requester); + return m_sqlite_db.selectAllRetrievalJobs(requester)[m_sqlite_db.getTapeByVid(requester, vid)]; } //------------------------------------------------------------------------------ diff --git a/libs/middletier/Tape.cpp b/libs/middletier/Tape.cpp index ca7db53695..a4a66f28b0 100644 --- a/libs/middletier/Tape.cpp +++ b/libs/middletier/Tape.cpp @@ -31,6 +31,13 @@ cta::Tape::Tape( m_comment(comment) { } +//------------------------------------------------------------------------------ +// operator< +//------------------------------------------------------------------------------ +bool cta::Tape::operator<(const Tape &rhs) const throw() { + return m_vid < rhs.m_vid; +} + //------------------------------------------------------------------------------ // getVid //------------------------------------------------------------------------------ diff --git a/libs/middletier/Tape.hpp b/libs/middletier/Tape.hpp index 79f2f3d242..0d79bd970a 100644 --- a/libs/middletier/Tape.hpp +++ b/libs/middletier/Tape.hpp @@ -38,6 +38,13 @@ public: const time_t creationTime, const std::string &comment); + /** + * Less than operator. + * + * @param rhs The right-hand side of the operator. + */ + bool operator<(const Tape &rhs) const throw(); + /** * Returns the volume identifier of the tape. * diff --git a/libs/middletier/Vfs.cpp b/libs/middletier/Vfs.cpp index 29e731c229..68c5441e81 100644 --- a/libs/middletier/Vfs.cpp +++ b/libs/middletier/Vfs.cpp @@ -304,4 +304,11 @@ cta::DirectoryIterator cta::Vfs::getDirectoryContents(const SecurityIdentity &re cta::Utils::checkAbsolutePathSyntax(dirPath); checkDirectoryExists(dirPath); return cta::DirectoryIterator(getDirectoryEntries(requester, dirPath)); +} + +//------------------------------------------------------------------------------ +// getDirectoryContents +//------------------------------------------------------------------------------ +std::string cta::Vfs::getVidOfFile(const SecurityIdentity &requester, const std::string &pathname, uint16_t copyNb) { + return "T00001"; //everything is on one tape for the moment:) } \ No newline at end of file diff --git a/libs/middletier/Vfs.hpp b/libs/middletier/Vfs.hpp index 21dcf020a3..a87436e931 100644 --- a/libs/middletier/Vfs.hpp +++ b/libs/middletier/Vfs.hpp @@ -41,6 +41,8 @@ public: cta::DirectoryIterator getDirectoryContents(const SecurityIdentity &requester, const std::string &dirPath); bool isExistingDirectory(const SecurityIdentity &requester, const std::string &dirPath); + + std::string getVidOfFile(const SecurityIdentity &requester, const std::string &pathname, uint16_t copyNb); private: -- GitLab