Commit 02be202e authored by David Smith's avatar David Smith
Browse files

Added PostgreSQL support for the CTA catalogue (issue 355)

parent 512f492d
......@@ -38,6 +38,7 @@ set (CATALOGUE_LIB_SRC_FILES
MysqlCatalogueSchema.cpp
OracleCatalogue.cpp
OracleCatalogueFactory.cpp
PostgresCatalogue.cpp
PostgresqlCatalogueFactory.cpp
SqliteCatalogueSchema.cpp
TapeFileWritten.cpp
......@@ -78,7 +79,7 @@ target_link_libraries (ctacatalogue
ctacommon
ctardbms)
add_custom_command (OUTPUT sqlite_catalogue_schema.sql mysql_catalogue_schema.sql oracle_catalogue_schema.sql
add_custom_command (OUTPUT sqlite_catalogue_schema.sql mysql_catalogue_schema.sql oracle_catalogue_schema.sql postgres_catalogue_schema.sql
COMMAND cat
${CMAKE_CURRENT_SOURCE_DIR}/sqlite_catalogue_schema_header.sql
${CMAKE_CURRENT_SOURCE_DIR}/common_catalogue_schema.sql
......@@ -96,12 +97,19 @@ add_custom_command (OUTPUT sqlite_catalogue_schema.sql mysql_catalogue_schema.sq
${CMAKE_CURRENT_SOURCE_DIR}/oracle_catalogue_schema_trailer.sql
| sed 's/VARCHAR/VARCHAR2/g'
> oracle_catalogue_schema.sql
COMMAND cat
${CMAKE_CURRENT_SOURCE_DIR}/postgres_catalogue_schema_header.sql
${CMAKE_CURRENT_SOURCE_DIR}/common_catalogue_schema.sql
${CMAKE_CURRENT_SOURCE_DIR}/postgres_catalogue_schema_trailer.sql
> postgres_catalogue_schema.sql
DEPENDS
${CMAKE_CURRENT_SOURCE_DIR}/common_catalogue_schema.sql
${CMAKE_CURRENT_SOURCE_DIR}/sqlite_catalogue_schema_header.sql
${CMAKE_CURRENT_SOURCE_DIR}/sqlite_catalogue_schema_trailer.sql
${CMAKE_CURRENT_SOURCE_DIR}/mysql_catalogue_schema_header.sql
${CMAKE_CURRENT_SOURCE_DIR}/mysql_catalogue_schema_trailer.sql
${CMAKE_CURRENT_SOURCE_DIR}/postgres_catalogue_schema_header.sql
${CMAKE_CURRENT_SOURCE_DIR}/postgres_catalogue_schema_trailer.sql
${CMAKE_CURRENT_SOURCE_DIR}/oracle_catalogue_schema_header.sql
${CMAKE_CURRENT_SOURCE_DIR}/oracle_catalogue_schema_trailer.sql)
......@@ -127,6 +135,15 @@ add_custom_command(OUTPUT MysqlCatalogueSchema.cpp
COMMAND sed -e '/CTA_SQL_SCHEMA/r mysql_catalogue_schema.cpp' -e '/CTA_SQL_TRIGGER/r mysql_catalogue_schema_trigger.cpp' ${CMAKE_CURRENT_SOURCE_DIR}/MysqlCatalogueSchema.before_SQL.cpp > MysqlCatalogueSchema.cpp
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/MysqlCatalogueSchema.before_SQL.cpp mysql_catalogue_schema.cpp mysql_catalogue_schema_trigger.cpp)
# For Postgres
add_custom_command(OUTPUT postgres_catalogue_schema.cpp
COMMAND sed 's/^/\ \ \"/' postgres_catalogue_schema.sql | sed 's/$$/\"/' > postgres_catalogue_schema.cpp
DEPENDS postgres_catalogue_schema.sql)
add_custom_command(OUTPUT PostgresCatalogueSchema.cpp
COMMAND sed -e '/CTA_SQL_SCHEMA/r postgres_catalogue_schema.cpp' ${CMAKE_CURRENT_SOURCE_DIR}/PostgresCatalogueSchema.before_SQL.cpp > PostgresCatalogueSchema.cpp
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/PostgresCatalogueSchema.before_SQL.cpp postgres_catalogue_schema.cpp)
set(IN_MEMORY_CATALOGUE_UNIT_TESTS_LIB_SRC_FILES
CatalogueTest.cpp
......@@ -175,6 +192,7 @@ add_executable(cta-catalogue-schema-create
CreateSchemaCmdMain.cpp
OracleCatalogueSchema.cpp
SqliteCatalogueSchema.cpp
PostgresCatalogueSchema.cpp
MysqlCatalogueSchema.cpp)
target_link_libraries (cta-catalogue-schema-create
......
This diff is collapsed.
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2015 CERN
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
......@@ -20,6 +20,7 @@
#include "catalogue/CreateSchemaCmdLineArgs.hpp"
#include "catalogue/MysqlCatalogueSchema.hpp"
#include "catalogue/OracleCatalogueSchema.hpp"
#include "catalogue/PostgresCatalogueSchema.hpp"
#include "catalogue/SqliteCatalogueSchema.hpp"
#include "common/exception/Exception.hpp"
#include "rdbms/ConnPool.hpp"
......@@ -73,6 +74,12 @@ int CreateSchemaCmd::exceptionThrowingMain(const int argc, char *const *const ar
conn.executeNonQueries(schema.sql);
}
break;
case rdbms::Login::DBTYPE_POSTGRESQL:
{
PostgresCatalogueSchema schema;
conn.executeNonQueries(schema.sql);
}
break;
case rdbms::Login::DBTYPE_MYSQL:
{
MysqlCatalogueSchema schema;
......
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2015 CERN
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
......@@ -106,6 +106,9 @@ void DropSchemaCmd::dropCatalogueSchema(const rdbms::Login::DbType &dbType, rdbm
case rdbms::Login::DBTYPE_MYSQL:
dropMysqlCatalogueSchema(conn);
break;
case rdbms::Login::DBTYPE_POSTGRESQL:
dropPostgresCatalogueSchema(conn);
break;
case rdbms::Login::DBTYPE_ORACLE:
dropOracleCatalogueSchema(conn);
break;
......@@ -243,6 +246,37 @@ void DropSchemaCmd::dropOracleCatalogueSchema(rdbms::Conn &conn) {
}
}
//------------------------------------------------------------------------------
// dropPostgresCatalogueSchema
//------------------------------------------------------------------------------
void DropSchemaCmd::dropPostgresCatalogueSchema(rdbms::Conn &conn) {
try {
std::list<std::string> tablesInDb = conn.getTableNames();
std::list<std::string> tablesToDrop = {
"CTA_CATALOGUE",
"ARCHIVE_ROUTE",
"TAPE_FILE",
"ARCHIVE_FILE",
"TAPE",
"REQUESTER_MOUNT_RULE",
"REQUESTER_GROUP_MOUNT_RULE",
"ADMIN_USER",
"ADMIN_HOST",
"STORAGE_CLASS",
"TAPE_POOL",
"LOGICAL_LIBRARY",
"MOUNT_POLICY"
};
dropDatabaseTables(conn, tablesToDrop);
std::list<std::string> sequencesToDrop = {"ARCHIVE_FILE_ID_SEQ", "STORAGE_CLASS_ID_SEQ"};
dropDatabaseSequences(conn, sequencesToDrop);
} catch(exception::Exception &ex) {
throw exception::Exception(std::string(__FUNCTION__) + " failed: " + ex.getMessage().str());
}
}
//------------------------------------------------------------------------------
// dropDatabaseSequences
//------------------------------------------------------------------------------
......
......@@ -98,6 +98,14 @@ private:
*/
void dropMysqlCatalogueSchema(rdbms::Conn &conn);
/**
* Unconditionally drops the schema of the catalogue database associated with
* the specified database connection.
*
* @param conn The database connection.
*/
void dropPostgresCatalogueSchema(rdbms::Conn &conn);
/**
* Drops the database tables with the specified names.
*
......
This diff is collapsed.
/*
* The CERN Tape Archive(CTA) project
* Copyright(C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "catalogue/RdbmsCatalogue.hpp"
#include "rdbms/Conn.hpp"
namespace cta {
namespace catalogue {
/**
* An Postgres based implementation of the CTA catalogue.
*/
class PostgresCatalogue: public RdbmsCatalogue {
public:
/**
* Constructor.
*
* @param log Object representing the API to the CTA logging system.
* @param username The database username.
* @param password The database password.
* @param database The database name.
* @param nbConns The maximum number of concurrent connections to the
* underlying relational database for all operations accept listing archive
* files which can be relatively long operations.
* @param nbArchiveFileListingConns The maximum number of concurrent
* connections to the underlying relational database for the sole purpose of
* listing archive files.
*/
PostgresCatalogue(
log::Logger &log,
const rdbms::Login &login,
const uint64_t nbConns,
const uint64_t nbArchiveFileListingConns);
/**
* Destructor.
*/
~PostgresCatalogue() override;
/**
* Deletes the specified archive file and its associated tape copies from the
* catalogue.
*
* Please note that the name of the disk instance is specified in order to
* prevent a disk instance deleting an archive file that belongs to another
* disk instance.
*
* Please note that this method is idempotent. If the file to be deleted does
* not exist in the CTA catalogue then this method returns without error.
*
* @param instanceName The name of the instance from where the deletion request
* originated
* @param archiveFileId The unique identifier of the archive file.
* @param lc The log context.
* @return The metadata of the deleted archive file including the metadata of
* the associated and also deleted tape copies.
*/
void deleteArchiveFile(const std::string &instanceName, const uint64_t archiveFileId,
log::LogContext &lc) override;
/**
* Notifies the catalogue that the specified files have been written to tape.
*
* @param events The tape file written events.
*/
void filesWrittenToTape(const std::set<TapeItemWrittenPointer> &events) override;
/**
* Returns a unique archive ID that can be used by a new archive file within
* the catalogue.
*
* This method must be implemented by the sub-classes of RdbmsCatalogue
* because different database technologies propose different solution to the
* problem of generating ever increasing numeric identifiers.
*
* @param conn The database connection.
* @return A unique archive ID that can be used by a new archive file within
* the catalogue.
*/
uint64_t getNextArchiveFileId(rdbms::Conn &conn) override;
/**
* Returns a unique storage class ID that can be used by a new storage class
* within the catalogue.
*
* This method must be implemented by the sub-classes of RdbmsCatalogue
* because different database technologies propose different solution to the
* problem of generating ever increasing numeric identifiers.
*
* @param conn The database connection.
* @return a unique archive ID that can be used by a new archive file within
* within the catalogue.
*/
uint64_t getNextStorageClassId(rdbms::Conn &conn) override;
private:
/**
* Start a database transaction and then create the temporary
* tables TEMP_ARCHIVE_FILE_BATCH and TEMP_TAPE_FILE_BATCH.
* Sets deferred mode for one of the db constraints to avoid
* violations during concurrent bulk insert.
*
* @parm conn The database connection.
*/
void beginCreateTemporarySetDeferred(rdbms::Conn &conn) const;
/**
* Selects the specified tape within the Tape table for update.
*
* @param conn The database connection.
* @param vid The volume identifier of the tape.
*/
common::dataStructures::Tape selectTapeForUpdate(rdbms::Conn &conn, const std::string &vid) const;
/**
* Batch inserts rows into the ARCHIVE_FILE table that correspond to the
* specified TapeFileWritten events.
*
* This method has idempotent behaviour in the case where an ARCHIVE_FILE
* already exists. Such a situation will occur when a file has more than one
* copy on tape. The first tape copy will cause two successful inserts, one
* into the ARCHIVE_FILE table and one into the TAPE_FILE table. The second
* tape copy will try to do the same, but the insert into the ARCHIVE_FILE
* table will fail or simply bounce as the row will already exists. The
* insert into the TABLE_FILE table will succeed because the two TAPE_FILE
* rows will be unique.
*
* @param conn The database connection.
* @param events The tape file written events.
*/
void idempotentBatchInsertArchiveFiles(rdbms::Conn &conn, const std::set<TapeFileWritten> &events) const;
/**
* The size and checksum of a file.
*/
struct FileSizeAndChecksum {
uint64_t fileSize;
std::string checksumType;
std::string checksumValue;
};
/**
* Returns the sizes and checksums of the specified archive files.
*
* @param conn The database connection.
* @param events The tape file written events that identify the archive files.
* @return A map from the identifier of each archive file to its size and checksum.
*/
std::map<uint64_t, FileSizeAndChecksum> selectArchiveFileSizesAndChecksums(rdbms::Conn &conn,
const std::set<TapeFileWritten> &events) const;
/**
* Batch inserts rows into the TAPE_FILE_BATCH temporary table that correspond
* to the specified TapeFileWritten events.
*
* @param conn The database connection.
* @param events The tape file written events.
*/
void insertTapeFileBatchIntoTempTable(rdbms::Conn &conn, const std::set<TapeFileWritten> &events) const;
}; // class PostgresCatalogue
} // namespace catalogue
} // namespace cta
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "catalogue/PostgresCatalogueSchema.hpp"
namespace cta {
namespace catalogue {
//------------------------------------------------------------------------------
// constructor
//------------------------------------------------------------------------------
PostgresCatalogueSchema::PostgresCatalogueSchema(): sql(
// CTA_SQL_SCHEMA - The contents of postgres_catalogue_schema.cpp go here
) {
}
} // namespace catalogue
} // namespace cta
/*
* The CERN Tape Archive (CTA) project
* Copyright (C) 2019 CERN
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <string>
namespace cta {
namespace catalogue {
/**
* Structure containing the SQL to create the schema of the in memory CTA
* database.
*
* The CMakeLists.txt file of this directory instructs cmake to generate
* PostgresCatalogueSchema.cpp from:
* - PostgresCatalogueSchema.before_SQL.cpp
* - postgres_catalogue_schema.sql
*
* The PostgresSchema.before_SQL.cpp file is not compilable and is therefore
* difficult for Integrated Developent Environments (IDEs) to handle.
*
* The purpose of this class is to help IDEs by isolating the "non-compilable"
* issues into a small cpp file.
*/
struct PostgresCatalogueSchema {
/**
* Constructor.
*/
PostgresCatalogueSchema();
/**
* The schema.
*/
const std::string sql;
};
} // namespace catalogue
} // namespace cta
......@@ -18,6 +18,7 @@
#include "catalogue/CatalogueRetryWrapper.hpp"
#include "catalogue/PostgresqlCatalogueFactory.hpp"
#include "catalogue/PostgresCatalogue.hpp"
#include "common/exception/Exception.hpp"
#include "common/make_unique.hpp"
......@@ -50,7 +51,12 @@ PostgresqlCatalogueFactory::PostgresqlCatalogueFactory(
// create
//------------------------------------------------------------------------------
std::unique_ptr<Catalogue> PostgresqlCatalogueFactory::create() {
throw exception::Exception(std::string(__FUNCTION__) + ": Not implemented");
try {
auto c = cta::make_unique<PostgresCatalogue>(m_log, m_login, m_nbConns, m_nbArchiveFileListingConns);
return cta::make_unique<CatalogueRetryWrapper>(m_log, std::move(c), m_maxTriesToConnect);
} catch(exception::Exception &ex) {
throw exception::Exception(std::string(__FUNCTION__) + " failed: " + ex.getMessage().str());
}
}
} // namespace catalogue
......
CREATE SEQUENCE ARCHIVE_FILE_ID_SEQ
INCREMENT BY 1
START WITH 1
NO MAXVALUE
MINVALUE 1
NO CYCLE
CACHE 20;
CREATE SEQUENCE STORAGE_CLASS_ID_SEQ
INCREMENT BY 1
START WITH 1
NO MAXVALUE
MINVALUE 1
NO CYCLE
CACHE 20;
ALTER TABLE ARCHIVE_FILE DROP CONSTRAINT
ARCHIVE_FILE_DIN_DFI_UN;
ALTER TABLE ARCHIVE_FILE ADD CONSTRAINT
ARCHIVE_FILE_DIN_DFI_UN UNIQUE(DISK_INSTANCE_NAME, DISK_FILE_ID) DEFERRABLE INITIALLY IMMEDIATE;
# The CERN Tape Archive(CTA) project
# Copyright(C) 2018 CERN
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# This module will set the following variables:
# POSTGRES_FOUND
# POSTGRES_INCLUDE_DIRS
# POSTGRES_LIBRARIES
find_path(POSTGRES_INCLUDE_DIRS
libpq-fe.h
PATHS /usr/include
NO_DEFAULT_PATH)
find_library(POSTGRES_LIBRARIES
NAME pq
PATHS /usr/lib64/
NO_DEFAULT_PATH)
message (STATUS "POSTGRES_INCLUDE_DIRS = ${POSTGRES_INCLUDE_DIRS}")
message (STATUS "POSTGRES_LIBRARIES = ${POSTGRES_LIBRARIES}")
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(mysql DEFAULT_MSG
POSTGRES_INCLUDE_DIRS POSTGRES_LIBRARIES)
......@@ -86,6 +86,7 @@ RUN yum-config-manager --enable epel --setopt="epel.priority=4" \
sudo \
zeromq \
mariadb-devel \
postgresql-libs \
&& \
yum clean all \
&& \
......
......@@ -55,6 +55,7 @@ BuildRequires: json-c-devel >= 0.11
BuildRequires: libattr-devel >= 2.4.44
BuildRequires: oracle-instantclient12.2-devel
BuildRequires: mariadb-devel
BuildRequires: postgresql-devel
BuildRequires: valgrind
BuildRequires: valgrind-devel
%{?systemd_requires}
......@@ -228,6 +229,7 @@ Group: Application/CTA
Requires: oracle-instantclient12.2-basic
Requires: librados2 = %{radosVersion}
Requires: mariadb-libs
Requires: postgresql-libs
Requires: zeromq
%description -n cta-lib
CERN Tape Archive:
......
......@@ -30,7 +30,12 @@ namespace rdbms {
//------------------------------------------------------------------------------
// s_fileFormat
//------------------------------------------------------------------------------
const char *Login::s_fileFormat = "either in_memory or oracle:username/password@database or sqlite:filename";
const char *Login::s_fileFormat = "one of "
"in_memory, "
"oracle:username/password@database, "
"sqlite:filename, "
"mysql://<username>:<password>@<host>:<port>/<db_name> or "
"postgresql:[connectinfo | URI]";
//------------------------------------------------------------------------------
// constructor
......@@ -348,8 +353,7 @@ Login Login::parseMySql(const std::string &connectionDetails) {
// parsePostgresql
//------------------------------------------------------------------------------
Login Login::parsePostgresql(const std::string &connectionDetails) {
//return Login(DBTYPE_POSTGRESQL, "", "", connectionDetails, "", 0);
throw exception::Exception(std::string(__FUNCTION__) + " not implemented");
return Login(DBTYPE_POSTGRESQL, "", "", connectionDetails, "", 0);
}
} // namespace catalogue
......
......@@ -215,7 +215,25 @@ struct Login {
static Login parseMySql(const std::string &connectionDetails);
/**
* Parses the specified connection details.
* Parses the specified connection details for the Postgres database.
* The postgres selection type in the config file is "postgres".
*
* Possible database login configuration lines for postgres:
*
* postgresql:[connectinfo]
* or
* postgresql:[URI]
*
* connectinfo or URI are the connectionDetails and may be empty.
* Otherwise connectinfo is a series of 1 or more keyword=value
* separated by whitespace. See doucmentaion, e.g.
* https://www.postgresql.org/docs/11/libpq-connect.html#LIBPQ-CONNSTRING
*
* an example configuraiton using the URI is
* postgresql:postgresql://user:secret@localhost/mydb
*
* an example configuraiton using connect info
* postgresql:user=user password=secret host=localhost db=mydb
*
* @param connectionDetails The database connection details.
*/
......
......@@ -19,8 +19,10 @@ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow")
find_package (sqlite REQUIRED)
find_package (mysql REQUIRED)
find_package (postgres REQUIRED)
find_package (oracle-instantclient REQUIRED)
include_directories (${MYSQL_INCLUDE_DIRS})
include_directories (${POSTGRES_INCLUDE_DIRS})
include_directories (${ORACLE-INSTANTCLIENT_INCLUDE_DIRS})
set (RDBMS_WRAPPER_LIB_SRC_FILES
......@@ -45,6 +47,14 @@ set (RDBMS_WRAPPER_LIB_SRC_FILES
MysqlRset.cpp
MysqlStmt.cpp)
set (RDBMS_WRAPPER_LIB_SRC_FILES
${RDBMS_WRAPPER_LIB_SRC_FILES}
PostgresConn.cpp
PostgresConnFactory.cpp
PostgresRset.cpp
PostgresStmt.cpp
PostgresColumn.cpp)
set (RDBMS_WRAPPER_LIB_SRC_FILES
${RDBMS_WRAPPER_LIB_SRC_FILES}
ConnFactoryFactory.cpp
......@@ -65,6 +75,7 @@ target_link_libraries (ctardbmswrapper
ctacommon
${SQLITE_LIBRARIES}
${MYSQL_LIBRARIES}
${POSTGRES_LIBRARIES}
${ORACLE-INSTANTCLIENT_LIBRARIES})
install (TARGETS ctardbmswrapper DESTINATION usr/${CMAKE_INSTALL_LIBDIR})
......@@ -73,6 +84,7 @@ set(RDBMS_WRAPPER_UNIT_TESTS_LIB_SRC_FILES