cta-catalogue-schema-verify now checks for table names, columns and types

Added a class SchemaChecker that has the responsiblity of checking :
- the schema against the .sql defined in the SCHEMA_VERSION/xxx_catalogue_schema.sql
- the presence of PARALLEL tables in ORACLE (not implemented yet)

catalogue/CMakeLists.txt

@@ -290,6 +290,7 @@ add_executable(cta-catalogue-schema-verify
   SQLiteSchemaComparer.cpp
   DbToSQLiteStatementTransformer.cpp
   SchemaComparerResult.cpp
+  SchemaChecker.cpp
   CatalogueMetadataGetter.cpp
   VerifySchemaCmd.cpp
   VerifySchemaCmdLineArgs.cpp

catalogue/CatalogueMetadataGetter.cpp

@@ -17,10 +17,19 @@
  */
 
 #include "CatalogueMetadataGetter.hpp"
+#include <algorithm>
 
 namespace cta {
 namespace catalogue {
 
+void CatalogueMetadataGetter::removeObjectNameContaining(std::list<std::string>& objects, const std::list<std::string> &wordsToTriggerRemoval){
+  objects.remove_if([&wordsToTriggerRemoval](const std::string &object){
+    return std::find_if(wordsToTriggerRemoval.begin(), wordsToTriggerRemoval.end(),[&object](const std::string &wordTriggeringRemoval){
+      return object.find(wordTriggeringRemoval) != std::string::npos;
+    }) != wordsToTriggerRemoval.end();
+  });
+}
+  
 CatalogueMetadataGetter::CatalogueMetadataGetter(cta::rdbms::Conn& conn):m_conn(conn){}
 
 std::string CatalogueMetadataGetter::getCatalogueVersion(){
@@ -52,32 +61,38 @@ SQLiteCatalogueMetadataGetter::~SQLiteCatalogueMetadataGetter(){}
 
 std::list<std::string> SQLiteCatalogueMetadataGetter::getIndexNames() {
   std::list<std::string> indexNames = m_conn.getIndexNames();
-  indexNames.remove_if([](std::string& indexName){
-    return ((indexName.find("sqlite_autoindex") != std::string::npos)); 
-  });
+  removeObjectNameContaining(indexNames,{"sqlite_autoindex"});
   return indexNames;
 }
 
 std::list<std::string> SQLiteCatalogueMetadataGetter::getTableNames(){
-  return std::list<std::string>();
+  std::list<std::string> tableNames = m_conn.getTableNames();
+  removeObjectNameContaining(tableNames,{"sqlite_sequence"});
+  return tableNames;
+}
+
+std::map<std::string, std::string> SQLiteCatalogueMetadataGetter::getColumns(const std::string& tableName){
+  return m_conn.getColumns(tableName);
 }
 
+
 OracleCatalogueMetadataGetter::OracleCatalogueMetadataGetter(cta::rdbms::Conn & conn):CatalogueMetadataGetter(conn){}
 OracleCatalogueMetadataGetter::~OracleCatalogueMetadataGetter(){}
 
 std::list<std::string> OracleCatalogueMetadataGetter::getIndexNames() {
   std::list<std::string> indexNames = m_conn.getIndexNames();
-  indexNames.remove_if([](std::string& indexName){
-    return ((indexName.find("_UN") != std::string::npos) || (indexName.find("PK") != std::string::npos) || (indexName.find("_LLN") != std::string::npos)); 
-  });
+  removeObjectNameContaining(indexNames,{"_UN","PK","_LLN"});
   return indexNames;
 }
 
 std::list<std::string> OracleCatalogueMetadataGetter::getTableNames() {
-  return std::list<std::string>();
+  std::list<std::string> tableNames = m_conn.getTableNames();
+  return tableNames;
 }
 
-
+std::map<std::string, std::string> OracleCatalogueMetadataGetter::getColumns(const std::string& tableName){
+  return m_conn.getColumns(tableName);
+}
 
 CatalogueMetadataGetter * CatalogueMetadataGetterFactory::create(const rdbms::Login::DbType dbType, cta::rdbms::Conn & conn) {
   typedef rdbms::Login::DbType DbType;

catalogue/CatalogueMetadataGetter.hpp

@@ -27,15 +27,16 @@ namespace cta {
 namespace catalogue {
   
 class CatalogueMetadataGetter {
-public:
   protected:
     rdbms::Conn& m_conn;
+    void removeObjectNameContaining(std::list<std::string>& objects, const std::list<std::string> &wordsToTriggerRemoval);
   public:
     CatalogueMetadataGetter(cta::rdbms::Conn & conn);
     virtual ~CatalogueMetadataGetter();
     std::string getCatalogueVersion();
     virtual std::list<std::string> getIndexNames() = 0;
     virtual std::list<std::string> getTableNames() = 0;
+    virtual std::map<std::string,std::string> getColumns(const std::string& tableName) = 0;
 };
 
 class SQLiteCatalogueMetadataGetter: public CatalogueMetadataGetter{
@@ -44,6 +45,7 @@ public:
   virtual ~SQLiteCatalogueMetadataGetter();
   std::list<std::string> getIndexNames() override;
   std::list<std::string> getTableNames() override;
+  std::map<std::string,std::string> getColumns(const std::string& tableName) override;
 };
 
 class OracleCatalogueMetadataGetter: public CatalogueMetadataGetter{
@@ -52,6 +54,7 @@ class OracleCatalogueMetadataGetter: public CatalogueMetadataGetter{
   virtual ~OracleCatalogueMetadataGetter();
   std::list<std::string> getIndexNames() override;
   std::list<std::string> getTableNames() override;
+  std::map<std::string,std::string> getColumns(const std::string& tableName) override;
 };
 
 class CatalogueMetadataGetterFactory {

catalogue/SQLiteSchemaComparer.cpp

@@ -35,6 +35,7 @@ SQLiteSchemaComparer::SQLiteSchemaComparer(const cta::rdbms::Login::DbType &cata
 }
 
 SQLiteSchemaComparer::~SQLiteSchemaComparer() {
+  m_sqliteSchemaMetadataGetter.release();
 }
 
 SchemaComparerResult SQLiteSchemaComparer::compare(){
@@ -48,15 +49,8 @@ SchemaComparerResult SQLiteSchemaComparer::compare(){
 SchemaComparerResult SQLiteSchemaComparer::compareTables(){
   std::list<std::string> catalogueTables = m_catalogueMetadataGetter->getTableNames();
   std::list<std::string> schemaTables = m_sqliteSchemaMetadataGetter->getTableNames();
-  std::cout<<"catalogue tables = "<<std::endl;
-  for(auto& catalogueTable: catalogueTables){
-    std::cout<<catalogueTable<<std::endl;
-  }
-  std::cout<<"schema tables = "<<std::endl;
-  for(auto& schemaTable: schemaTables){
-    std::cout<<schemaTable<<std::endl;
-  }
-  return SchemaComparerResult();
+  SchemaComparerResult res = compareTables(catalogueTables,schemaTables);
+  return res;
 }
 
 void SQLiteSchemaComparer::insertSchemaInSQLite() {
@@ -65,17 +59,69 @@ void SQLiteSchemaComparer::insertSchemaInSQLite() {
 }
 
 SchemaComparerResult SQLiteSchemaComparer::compareIndexes(){
-  SchemaComparerResult result;
   std::list<std::string> catalogueIndexes = m_catalogueMetadataGetter->getIndexNames();
   std::list<std::string> schemaIndexes = m_sqliteSchemaMetadataGetter->getIndexNames();
-  for(auto &ci: catalogueIndexes){
-    if(std::find(schemaIndexes.begin(),schemaIndexes.end(),ci) == schemaIndexes.end()){
-      result.addDiff("INDEX "+ci+" is missing in the schema but defined in the catalogue");
+  return compareItems("INDEX", catalogueIndexes, schemaIndexes);
+}
+
+SchemaComparerResult SQLiteSchemaComparer::compareItems(const std::string &itemType, const std::list<std::string>& itemsFromCatalogue, const std::list<std::string>& itemsFromSQLite){
+  SchemaComparerResult result;
+  for(auto &catalogueItem: itemsFromCatalogue){
+    if(std::find(itemsFromSQLite.begin(),itemsFromSQLite.end(),catalogueItem) == itemsFromSQLite.end()){
+      result.addDiff(itemType+" "+catalogueItem+" is missing in the schema but defined in the catalogue");
+    }
+  }
+  for(auto &sqliteItem: itemsFromSQLite){
+    if(std::find(itemsFromCatalogue.begin(),itemsFromCatalogue.end(),sqliteItem) == itemsFromCatalogue.end()){
+      result.addDiff(itemType+" "+sqliteItem+" is missing in the catalogue but is defined in the schema");
     }
   }
-  for(auto &si: schemaIndexes){
-    if(std::find(catalogueIndexes.begin(),catalogueIndexes.end(),si) == catalogueIndexes.end()){
-      result.addDiff("INDEX "+si+" is missing in the catalogue but is defined in the schema");
+  return result;
+}
+
+SchemaComparerResult SQLiteSchemaComparer::compareTables(const std::list<std::string>& catalogueTables, const std::list<std::string>& schemaTables){
+  SchemaComparerResult result;
+  std::map<std::string, std::map<std::string, std::string>> catalogueTableColumns;
+  std::map<std::string, std::map<std::string, std::string>> schemaTableColumns;
+  for(auto &catalogueTable: catalogueTables){
+    catalogueTableColumns[catalogueTable] = m_catalogueMetadataGetter->getColumns(catalogueTable);
+  }
+  
+  for(auto &schemaTable: schemaTables){
+    schemaTableColumns[schemaTable] = m_sqliteSchemaMetadataGetter->getColumns(schemaTable);
+  }
+  
+  result +=  compareTableColumns(catalogueTableColumns,"catalogue",schemaTableColumns,"schema");
+  result +=  compareTableColumns(schemaTableColumns,"schema",catalogueTableColumns,"catalogue");
+  
+  return result;
+}
+
+SchemaComparerResult SQLiteSchemaComparer::compareTableColumns(const TableColumns & firstSchemaTableColumns, const std::string &schema1Type,const TableColumns & secondSchemaTableColumns, const std::string &schema2Type){
+  SchemaComparerResult result;
+  for(auto &kvFirstSchemaTableColumns: firstSchemaTableColumns){
+    //For each firstSchema table, get the corresponding secondSchema table
+    //If it does not exist, add a difference and go ahead
+    //otherwise, get the columns/types of the table from the firstSchema and do the comparison
+    //against the columns/types of the same table from the secondSchema
+    std::string firstSchemaTableName = kvFirstSchemaTableColumns.first;
+    try {
+      std::map<std::string, std::string> mapSecondSchemaColumnType = secondSchemaTableColumns.at(firstSchemaTableName);
+      std::map<std::string, std::string> mapFirstSchemaColumnType = kvFirstSchemaTableColumns.second;
+      for(auto &kvFirstSchemaColumn: mapFirstSchemaColumnType){
+        std::string firstSchemaColumnName = kvFirstSchemaColumn.first;
+        std::string firstSchemaColumnType = kvFirstSchemaColumn.second;
+        try {
+          std::string schemaColumnType = mapSecondSchemaColumnType.at(firstSchemaColumnName);
+          if( firstSchemaColumnType != schemaColumnType){
+            result.addDiff("TABLE "+firstSchemaTableName+" from "+schema1Type+" has a column named "+firstSchemaColumnName+" that has a type "+firstSchemaColumnType+" that does not match the column type from the "+schema2Type+" ("+schemaColumnType+")");
+          }
+        } catch (const std::out_of_range &) {
+          result.addDiff("TABLE "+firstSchemaTableName+" from "+schema1Type+" has a column named " + firstSchemaColumnName + " that is missing in the "+schema2Type+".");
+        }
+      }
+    } catch (const std::out_of_range &) {
+      result.addDiff("TABLE "+firstSchemaTableName+" is missing in the "+schema2Type+" but is defined in the "+schema1Type+".");
     }
   }
   return result;

catalogue/SQLiteSchemaComparer.hpp

@@ -24,7 +24,7 @@ namespace catalogue {
   
 class SQLiteSchemaComparer: public SchemaComparer {
 public:
-  SQLiteSchemaComparer(const cta::rdbms::Login::DbType &catalogueDbType, rdbms::Conn &catalogueConnPool);
+  SQLiteSchemaComparer(const cta::rdbms::Login::DbType &catalogueDbType, rdbms::Conn &catalogueConn);
   SchemaComparerResult compare() override;
   virtual ~SQLiteSchemaComparer();
   
@@ -32,6 +32,10 @@ private:
   void insertSchemaInSQLite();
   SchemaComparerResult compareIndexes();
   SchemaComparerResult compareTables() override;
+  SchemaComparerResult compareItems(const std::string &itemType, const std::list<std::string>& itemsFromCatalogue, const std::list<std::string>& itemsFromSQLite);
+  SchemaComparerResult compareTables(const std::list<std::string> &catalogueTables, const std::list<std::string> &schemaTables);
+  typedef std::map<std::string, std::map<std::string, std::string>> TableColumns;
+  SchemaComparerResult compareTableColumns(const TableColumns & tableColumns1, const std::string &originTableColumns1,const TableColumns & tableColumns2, const std::string &originTableColumns2);
   rdbms::Conn m_sqliteConn;
   std::unique_ptr<SQLiteCatalogueMetadataGetter> m_sqliteSchemaMetadataGetter;  
 };

catalogue/SchemaChecker.cpp

+/**
+ * The CERN Tape Archive (CTA) project
+ * Copyright © 2018 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "SchemaChecker.hpp"
+#include "SQLiteSchemaComparer.hpp"
+
+namespace cta{
+namespace catalogue{
+  
+SchemaChecker::SchemaChecker(const cta::rdbms::Login::DbType &catalogueDbType,cta::rdbms::Conn &conn):m_dbType(catalogueDbType),m_conn(conn) {
+  m_schemaComparer.reset(new SQLiteSchemaComparer(m_dbType,m_conn));
+}
+
+SchemaChecker::~SchemaChecker() {
+}
+
+void SchemaChecker::setSchemaComparer(std::unique_ptr<SchemaComparer> schemaComparer){
+  if(m_schemaComparer != nullptr){
+    m_schemaComparer.release();
+  }
+  m_schemaComparer = std::move(schemaComparer);
+}
+
+void SchemaChecker::compareSchema(){
+  cta::catalogue::SchemaComparerResult res = m_schemaComparer->compare();
+  std::cout << "Schema version : " << m_schemaComparer->getCatalogueVersion() << std::endl;
+  std::cout << "Status of the checking : " << cta::catalogue::SchemaComparerResult::StatusToString(res.getStatus()) << std::endl;
+  res.printDiffs();
+}
+
+void SchemaChecker::checkNoParallelTables(){
+  
+}
+
+}}
\ No newline at end of file

catalogue/SchemaChecker.hpp

+/**
+ * The CERN Tape Archive (CTA) project
+ * Copyright © 2018 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+#include "rdbms/Login.hpp"
+#include "rdbms/Conn.hpp"
+#include "SchemaComparer.hpp"
+
+namespace cta{
+namespace catalogue{
+    
+class SchemaChecker {
+public:
+  SchemaChecker(const cta::rdbms::Login::DbType &catalogueDbType,cta::rdbms::Conn &conn);
+  virtual ~SchemaChecker();
+  void setSchemaComparer(std::unique_ptr<SchemaComparer> schemaComparer);
+  void compareSchema();
+  void checkNoParallelTables();
+private:
+  const cta::rdbms::Login::DbType &m_dbType;
+  cta::rdbms::Conn &m_conn;
+  std::unique_ptr<SchemaComparer> m_schemaComparer;
+};
+
+}}
+
+

catalogue/SchemaComparer.hpp

@@ -35,6 +35,10 @@ class SchemaComparer {
 public:
   SchemaComparer(const cta::rdbms::Login::DbType &catalogueDbType,cta::rdbms::Conn &conn);
   virtual ~SchemaComparer();
+  /**
+   * Compare the schema
+   * @return 
+   */
   virtual SchemaComparerResult compare() = 0;
   std::string getCatalogueVersion();
 protected:

catalogue/VerifySchemaCmd.cpp

@@ -29,8 +29,7 @@
 #include "rdbms/AutocommitMode.hpp"
 #include "common/log/DummyLogger.hpp"
 #include "Catalogue.hpp"
-#include "SchemaComparer.hpp"
-#include "SQLiteSchemaComparer.hpp"
+#include "SchemaChecker.hpp"
 
 #include <algorithm>
 #include <map>
@@ -74,13 +73,10 @@ int VerifySchemaCmd::exceptionThrowingMain(const int argc, char *const *const ar
     return 1;
   }
   
-  std::unique_ptr<cta::catalogue::SchemaComparer> schemaComparer;
-  schemaComparer.reset(new cta::catalogue::SQLiteSchemaComparer(login.dbType,conn));
-  cta::catalogue::SchemaComparerResult res = schemaComparer->compare();
-  std::cout << "Schema version : " << schemaComparer->getCatalogueVersion() << std::endl;
-  std::cout << "Status of the checking : " << cta::catalogue::SchemaComparerResult::StatusToString(res.getStatus()) << std::endl;
-  res.printDiffs();
-  schemaComparer.release();
+  cta::catalogue::SchemaChecker schemaChecker(login.dbType,conn);
+  schemaChecker.compareSchema();
+  schemaChecker.checkNoParallelTables();
+
   /*std::unique_ptr<CatalogueSchema> schema;
   switch(login.dbType) {
   case rdbms::Login::DBTYPE_IN_MEMORY:

rdbms/wrapper/SqliteConn.cpp

@@ -249,7 +249,8 @@ std::map<std::string, std::string> SqliteConn::getColumns(const std::string &tab
     "CHAR|"
     "VARCHAR|"
     "VARCHAR2|"
-    "BLOB";
+    "BLOB|"
+    "RAW";
     
     auto stmt = createStmt(sql);
     stmt->bindString(":TABLE_NAME", tableName);