Commit 7a6229d0 authored by Giuseppe Lo Presti's avatar Giuseppe Lo Presti
Browse files

[migration] Reworked the logic to handle dual copy tapepools.

Now the export is performed for both tapepools holding the two tape copies,
and the process aborts if a single 2nd copy tape pool is not found
(cf. README.md for more details).
parent 5449bd48
......@@ -35,7 +35,7 @@ The tools are designed to work as follows, for any given VO:
3. In case of errors, `export_production_tapepool_to_cta.sh` stops and the operator is expected to fix the case and rerun the export, possibly re-executing by hand one of the commands documented above. Errors are accumulated in suitable Oracle tables both for the database migration and the EOS namespace injection tools. For the latter, please refer to their specific instructions.
> [warning] In order to efficiently import tape pools holding dual tape copies, and avoid a double pass over the EOS metadata import, such tape pools are imported in a single operation: assuming that a tape pool holding the 1st copies is requested to be exported, the corresponding tape pool holding the 2nd copies is **also exported**. Furthermore, to protect CTA, `tapepool_castor_to_cta.py` will abort if some files are found having their 1st tape copy and other files having their 2nd tape copy, all in the given tape pool.
> [warning] In order to efficiently import tape pools holding dual tape copies, and avoid a double pass over the EOS metadata import, such tape pools are imported in a single operation: assuming that a tape pool holding the 1st copies is requested to be exported, the corresponding tape pool holding the 2nd copies is **also exported**. Furthermore, to protect CTA, `tapepool_castor_to_cta.py` will abort if some files are found having their 1st tape copy and other files having their 2nd tape copy, all in the given tape pool. The process will also abort if 2nd copies exist but a single tape pool holding them cannot be identified (e.g. because there are two, or it was not found).
In addition, the following tools are provided, which can be used as part of a restore/recovery procedure. Such procedure has deliberately **not** been automated and will have to be dealt with on a case by case basis.
......
......@@ -29,12 +29,16 @@ ACCEPT ctaSchema CHAR PROMPT 'Enter the username of the CTA schema: ';
-- Helper tables to store intermediate data for the export
CREATE TABLE CTAFilesHelper (fileid INTEGER NOT NULL PRIMARY KEY, parent_fileid INTEGER, filename VARCHAR2(255), disk_uid INTEGER, disk_gid INTEGER,
filemode INTEGER, btime INTEGER, ctime INTEGER, mtime INTEGER, classid INTEGER,
filesize INTEGER, checksum NUMBER, copyno INTEGER, VID VARCHAR2(6), fseq INTEGER,
filesize INTEGER, checksum NUMBER, copyno INTEGER, vid VARCHAR2(6), fseq INTEGER,
blockId INTEGER, s_mtime INTEGER);
CREATE INDEX I_CTAFiles_parent_id ON CTAFilesHelper (parent_fileid);
CREATE INDEX I_CTAFiles_vid_filesize ON CTAFilesHelper (vid, filesize);
CREATE INDEX I_CTAFiles_copyno ON CTAFilesHelper (copyno);
CREATE TABLE CTAFiles2ndCopyHelper (fileid INTEGER NOT NULL PRIMARY KEY, filesize INTEGER,
vid VARCHAR2(6), fseq INTEGER, blockId INTEGER, s_mtime INTEGER);
CREATE INDEX I_CTAFiles2_vid_filesize ON CTAFiles2ndCopyHelper (vid, filesize);
CREATE TABLE CTAFilesFailed (fileid INTEGER NOT NULL PRIMARY KEY, parent_fileid INTEGER, filename VARCHAR2(255), disk_uid INTEGER, disk_gid INTEGER,
filemode INTEGER, btime INTEGER, ctime INTEGER, mtime INTEGER, classid INTEGER,
filesize INTEGER, checksum NUMBER, copyno INTEGER, VID VARCHAR2(6), fseq INTEGER,
......@@ -76,6 +80,7 @@ CREATE TABLE CTAMigrationLog (timestamp NUMBER, client VARCHAR2(100), tapepool V
-- Enable parallel access in all concerned tables
ALTER TABLE CTAFilesHelper PARALLEL;
ALTER TABLE CTAFiles2ndCopyHelper PARALLEL;
ALTER TABLE CTADeltaDirsHelper PARALLEL;
ALTER TABLE Cns_file_metadata PARALLEL;
ALTER TABLE Cns_seg_metadata PARALLEL;
......@@ -87,7 +92,7 @@ CREATE OR REPLACE PROCEDURE ctaLog(inTapePool IN VARCHAR2, inMsg IN VARCHAR2) AS
PRAGMA AUTONOMOUS_TRANSACTION;
BEGIN
INSERT INTO CTAMigrationLog (timestamp, client, tapepool, message)
VALUES (getTime(), (SELECT SYS_CONTEXT('USERENV', 'HOST', 15) FROM Dual), inTapePool, inMsg);
VALUES (getTime(), (SELECT SYS_CONTEXT('USERENV', 'HOST', 100) FROM Dual), inTapePool, inMsg);
COMMIT;
END;
/
......@@ -104,12 +109,10 @@ END;
/
/* Procedure to extract the directory names for the export to CTA */
CREATE OR REPLACE PROCEDURE filesAndDirsForCTAExport(inPoolName IN VARCHAR2) AS
/* Procedure to prepare the files and segments metadata for export to CTA */
create or replace PROCEDURE filesForCTAExport(inPoolName IN VARCHAR2, out2ndCopyPoolName OUT VARCHAR2) AS
nbFiles INTEGER;
dirsSnapshotTime INTEGER;
dirIds numList;
varPath VARCHAR2(2048);
var2ndCopy INTEGER;
BEGIN
-- populate the helper table by selecting all required metadata
INSERT /*+ APPEND PARALLEL(CTAFilesHelper) */ INTO CTAFilesHelper (
......@@ -134,6 +137,61 @@ BEGIN
END IF;
ctaLog(inPoolName, 'Intermediate table for files prepared, '|| nbFiles ||' files to be exported. ETA: '||
prettyTime(nbFiles/5200/60));
-- check for dual tape copies if any: first, abort if this tape pool already contains some 2nd copies
SELECT COUNT(*) INTO nbFiles FROM CTAFilesHelper WHERE copyno <> 1;
IF nbFiles > 0 THEN
raise_application_error(-20000, 'Second tape copy detected, please export the first-copy tape pool to also export this one');
END IF;
-- now look for the second copy tape pool if the involved fileclasses require it to exist
out2ndCopyPoolName := NULL;
SELECT MAX(nbcopies) INTO var2ndCopy FROM Cns_class_metadata
WHERE classid IN (SELECT DISTINCT classid FROM CTAFilesHelper);
IF var2ndCopy = 2 THEN
-- yes, work out the tape pool holding the second copies
BEGIN
SELECT DISTINCT(poolName) INTO out2ndCopyPoolName
FROM Vmgr_tape_side T
WHERE T.vid IN (
SELECT S.vid
FROM Cns_seg_metadata S, CTAFilesHelper F
WHERE s_fileid = F.fileid
AND S.copyno = 2
);
EXCEPTION WHEN TOO_MANY_ROWS THEN
-- more than one tape pool found for the second copies of the selected files
raise_application_error(-20000, 'Multiple tape pools found, which hold the second copy for files in '||
inPoolName ||'. This is not supported in CTA, aborting.');
END;
ctaLog(inPoolName, 'Dual tape copies detected, will also export tape pool '|| out2ndCopyPoolName);
ctaLog(out2ndCopyPoolName, 'Preparing export of the second-copy tape pool following export of '|| inPoolName);
-- populate the 2nd copy helper table
INSERT /*+ APPEND PARALLEL(CTAFiles2ndCopyHelper) */ INTO CTAFiles2ndCopyHelper (
SELECT /*+ PARALLEL(F) PARALLEL(S) */
F.fileid, S.segsize as filesize, S.vid, S.fseq,
utl_raw.cast_to_binary_integer(S.blockId), S.lastModificationTime as s_mtime
FROM Cns_file_metadata F, Cns_seg_metadata S
WHERE F.fileid = S.s_fileid
AND S.vid IN (
SELECT vid FROM Vmgr_tape_side
WHERE poolName = out2ndCopyPoolName
AND BITAND(status, 2) = 0 AND BITAND(status, 32) = 0 -- not already EXPORTED or ARCHIVED
)
);
COMMIT;
ctaLog(inPoolName, 'Intermediate table for second-copy files prepared');
END IF;
END;
/
/* Procedure to prepare the directories for export to CTA. The local Dirs_Full_Path cache table is updated */
CREATE OR REPLACE PROCEDURE dirsForCTAExport(inPoolName IN VARCHAR2) AS
varDirsCount INTEGER;
dirsSnapshotTime INTEGER;
dirIds numList;
varPath VARCHAR2(2048);
BEGIN
EXECUTE IMMEDIATE 'TRUNCATE TABLE CTADeltaDirsHelper';
-- estimate the last update time of the Dirs_Full_Path snapshot. To be noted that
-- in case the directory identified by max(fileid) got updated itself, the estimate
......@@ -200,8 +258,8 @@ BEGIN
-- drop null paths due to the above recursive query going to the top /castor level
DELETE FROM CTADeltaDirsHelper WHERE path IS NULL;
COMMIT;
SELECT count(*) INTO nbFiles FROM CTADeltaDirsHelper;
ctaLog(inPoolName, 'Intermediate delta table completed with '|| nbFiles ||' directories');
SELECT count(*) INTO varDirsCount FROM CTADeltaDirsHelper;
ctaLog(inPoolName, 'Intermediate delta table completed with '|| varDirsCount ||' directories');
-- also update the hierarchy snapshot with what was found
MERGE /*+ PARALLEL(Dirs_Full_Path) */ INTO Dirs_Full_Path DP
USING (SELECT /*+ PARALLEL(CTADeltaDirsHelper) */
......@@ -221,37 +279,13 @@ END;
/
/* Procedure to remove all CTAFilesHelper entries in case of a dual copy export */
CREATE OR REPLACE PROCEDURE clean2ndCopyInCTAExport(inPoolName VARCHAR2) AS
varIs2ndCopy INTEGER;
BEGIN
-- check that we are in the dual tape copy case
SELECT COUNT(*) INTO varIs2ndCopy FROM Dual
WHERE EXISTS (SELECT 1 FROM CTAFilesHelper WHERE copyno = 2);
IF varIs2ndCopy = 0 THEN
raise_application_error(-20000, 'No second-copy found in this export');
END IF;
-- drop everything - cf. bottom of populateCTAFilesFromCASTOR() in the CTA schema
-- for the rationale behind this operation
EXECUTE IMMEDIATE 'TRUNCATE TABLE CTAFilesHelper';
ctaLog(inPoolName, 'Removed second-copy files metadata from intermediate table');
-- insert a placeholder in CTAFilesHelper: this is a hack to make sure the rest
-- of the logic still assumes a migration is ongoing and another one cannot be started
INSERT INTO CTAFilesHelper (fileid, parent_fileid, filename, disk_uid, disk_gid,
filemode, btime, ctime, mtime, classid, filesize, checksum)
VALUES (0, 0, 'Placeholder for dual tape copies migration', 0, 0, 0, 0, 0, 0, 0, 0, 1);
COMMIT;
END;
/
/* Procedure to terminate the export to CTA and account it on the statistics */
CREATE OR REPLACE PROCEDURE completeCTAExport AS
CURSOR c IS SELECT fileid FROM CTAFilesHelper;
ids numList;
varUnused INTEGER;
varTapePool VARCHAR2(100);
varMsg VARCHAR2(2048);
varStartMsg VARCHAR2(2048);
varStartTime INTEGER;
BEGIN
BEGIN
......@@ -260,13 +294,13 @@ BEGIN
raise_application_error(-20000, 'No ongoing files export to CTA, nothing to do');
END;
-- Get info from the current export. It must be there as we passed the previous check.
SELECT tapePool, timestamp, message INTO varTapePool, varStartTime, varMsg
SELECT tapePool, timestamp, message INTO varTapePool, varStartTime, varStartMsg
FROM (SELECT tapePool, timestamp, message
FROM CTAMigrationLog
WHERE message LIKE 'CASTOR metadata import started%'
ORDER BY timestamp DESC)
WHERE ROWNUM <= 1;
IF INSTR(varMsg, '[dry-run mode]') = 0 THEN
IF INSTR(varStartMsg, '[dry-run mode]') = 0 THEN
ctaLog(varTapePool, 'Marking CASTOR files as exported to CTA');
OPEN c;
LOOP
......@@ -281,7 +315,8 @@ BEGIN
CLOSE c;
END IF;
EXECUTE IMMEDIATE 'TRUNCATE TABLE CTAFilesHelper';
IF INSTR(varMsg, '[dry-run mode]') > 0 THEN
EXECUTE IMMEDIATE 'TRUNCATE TABLE CTAFiles2ndCopyHelper';
IF INSTR(varStartMsg, '[dry-run mode]') > 0 THEN
ctaLog(varTapePool, 'Export from CASTOR fully completed in '|| prettyTime((getTime()-varStartTime)/60) ||' [dry-run mode]');
ELSE
ctaLog(varTapePool, 'Export from CASTOR fully completed in '|| prettyTime((getTime()-varStartTime)/60));
......@@ -292,10 +327,11 @@ END;
-- Need to grant access from the CTA catalogue schema
GRANT SELECT ON CTAFilesHelper TO &ctaSchema;
GRANT SELECT ON CTAFiles2ndCopyHelper TO &ctaSchema;
GRANT SELECT ON CTADirsHelper TO &ctaSchema;
GRANT SELECT ON CTAMigrationLog TO &ctaSchema;
GRANT SELECT ON Cns_class_metadata TO &ctaSchema;
GRANT SELECT ON Dirs_Full_Path TO &ctaSchema;
GRANT EXECUTE ON filesAndDirsForCTAExport TO &ctaSchema;
GRANT EXECUTE ON clean2ndCopyInCTAExport TO &ctaSchema;
GRANT EXECUTE ON filesForCTAExport TO &ctaSchema;
GRANT EXECUTE ON dirsForCTAExport TO &ctaSchema;
GRANT EXECUTE ON ctaLog TO &ctaSchema;
......@@ -26,14 +26,15 @@
UNDEF castornsSchema
ACCEPT castornsSchema CHAR DEFAULT 'castor' PROMPT 'Enter the name of the CASTOR Nameserver schema (default castor): ';
CREATE OR REPLACE SYNONYM CNS_CTAFilesHelper FOR &castornsSchema..CTAFilesHelper;
CREATE OR REPLACE SYNONYM CNS_CTAFiles2ndCopyHelper FOR &castornsSchema..CTAFiles2ndCopyHelper;
CREATE OR REPLACE SYNONYM CNS_CTADirsHelper FOR &castornsSchema..CTADirsHelper;
CREATE OR REPLACE SYNONYM CNS_CTAMigrationLog FOR &castornsSchema..CTAMigrationLog;
CREATE OR REPLACE SYNONYM CNS_Class_Metadata FOR &castornsSchema..Cns_class_metadata;
CREATE OR REPLACE SYNONYM CNS_Dirs_Full_Path FOR &castornsSchema..Dirs_Full_Path;
CREATE OR REPLACE SYNONYM cns_filesAndDirsForCTAExport FOR &castornsSchema..filesAndDirsForCTAExport;
CREATE OR REPLACE SYNONYM cns_clean2ndCopyInCTAExport FOR &castornsSchema..clean2ndCopyInCTAExport;
CREATE OR REPLACE SYNONYM cns_ctaLog FOR &castornsSchema..ctaLog;
CREATE OR REPLACE SYNONYM CNS_filesForCTAExport FOR &castornsSchema..filesForCTAExport;
CREATE OR REPLACE SYNONYM CNS_dirsForCTAExport FOR &castornsSchema..dirsForCTAExport;
CREATE OR REPLACE SYNONYM CNS_ctaLog FOR &castornsSchema..ctaLog;
UNDEF vmgrSchema
ACCEPT vmgrSchema CHAR DEFAULT 'vmgr' PROMPT 'Enter the name of the VMGR schema (default vmgr): ';
......@@ -41,41 +42,12 @@ CREATE OR REPLACE SYNONYM Vmgr_tape_side FOR &vmgrSchema..Vmgr_tape_side;
CREATE OR REPLACE SYNONYM vmgr_tape_info FOR &vmgrSchema..Vmgr_tape_info;
CREATE OR REPLACE SYNONYM Vmgr_tape_dgnmap FOR &vmgrSchema..Vmgr_tape_dgnmap;
-- Enable parallel inserts
ALTER TABLE Archive_File PARALLEL;
ALTER TABLE Tape_File PARALLEL;
-- Used by removeCASTORMetadata
CREATE OR REPLACE TYPE NUMLIST IS TABLE OF INTEGER;
-- Perform some validation checks in the case of dual copy tape pools
CREATE OR REPLACE PROCEDURE validateDualCopies(inTapePool VARCHAR2) AS
varCount INTEGER;
BEGIN
SELECT COUNT(*) INTO varCount FROM CNS_CTAFilesHelper WHERE copyno = 2;
IF varCount > 0 THEN
-- first check: enforce that all files being imported are either copynb = 1 xor copynb = 2
SELECT COUNT(*) INTO varCount FROM CNS_CTAFilesHelper WHERE copyno = 1;
IF varCount > 0 THEN
raise_application_error(-20000, 'This tape pool contains both files having copynb = 1 and '||
'files having copynb = 2, which is forbidden in CTA. Aborting');
END IF;
-- second check: if we're importing copynb = 2 files, make sure their copynb = 1 entries
-- are all already present in the CTA catalogue
SELECT COUNT(*) INTO varCount FROM CNS_CTAFilesHelper F
WHERE NOT EXISTS (SELECT 1 FROM Tape_File WHERE archive_file_id = F.fileid AND copy_nb = 1);
IF varCount > 0 THEN
raise_application_error(-20000, 'This tape pool contains '|| varCount ||' 2nd copy files, '||
'for which the 1st copy has not yet been imported in CTA. Aborting');
END IF;
-- all fine, log a message and move on
CNS_ctaLog(inTapePool, 'Validation of dual tape copies completed');
END IF;
END;
/
-- Enable parallel inserts
ALTER TABLE Archive_File PARALLEL;
ALTER TABLE Tape_File PARALLEL;
-- Import a tapepool and its tapes from CASTOR
......@@ -93,7 +65,7 @@ BEGIN
creation_log_user_name, creation_log_host_name, creation_log_time,
last_update_user_name, last_update_host_name, last_update_time)
VALUES (
(SELECT Logical_Library_id_seq.NEXTVAL FROM Dual),
Logical_Library_id_seq.NEXTVAL,
DGN.dgn,
'Imported from CASTOR',
'CASTOR', 'CASTOR', getTime(),
......@@ -225,37 +197,32 @@ BEGIN
COMMIT;
CNS_ctaLog(inTapePool, 'Storage classes import completed');
-- Check if we're dealing with 2nd copies
SELECT COUNT(*) INTO varIs2ndCopy FROM Dual
WHERE EXISTS (SELECT 1 FROM CNS_CTAFilesHelper WHERE copyno = 2);
IF varIs2ndCopy = 0 THEN
-- Populate the CTA catalogue with the CASTOR file metadata
IF instr(inEOSCTAInstance, 'eoscta') = 1 THEN
varNamespacePrefix := substr(inEOSCTAInstance, 7);
ELSIF instr(inEOSCTAInstance, 'eos') = 1 THEN
varNamespacePrefix := substr(inEOSCTAInstance, 4);
ELSE
varNamespacePrefix := inEOSCTAInstance;
END IF;
INSERT /*+ APPEND PARALLEL(Archive_File) */ INTO Archive_File
(archive_file_id, disk_instance_name, disk_file_id, disk_file_path,
disk_file_uid, disk_file_gid, size_in_bytes, checksum_adler32,
storage_class_id, creation_time, reconciliation_time, is_deleted) (
SELECT /*+ PARALLEL(F) PARALLEL(D) */
F.fileid, inEOSCTAInstance, F.fileid,
'/eos/' || varNamespacePrefix
|| substr(D.path, varLenCastorPrefix) || '/' || F.filename,
F.disk_uid, F.disk_gid, F.filesize, F.checksum, F.classid, F.btime, 0, '0'
FROM CNS_CTAFilesHelper F, CNS_Dirs_Full_Path D
WHERE F.parent_fileid = D.fileid
-- no need to exclude already existing files because of dual copies
-- thanks to the pre-checks in validateDualCopies()
);
COMMIT;
CNS_ctaLog(inTapePool, 'Archive files import completed');
-- Populate the CTA catalogue with the CASTOR file metadata
IF instr(inEOSCTAInstance, 'eoscta') = 1 THEN
varNamespacePrefix := substr(inEOSCTAInstance, 7);
ELSIF instr(inEOSCTAInstance, 'eos') = 1 THEN
varNamespacePrefix := substr(inEOSCTAInstance, 4);
ELSE
varNamespacePrefix := inEOSCTAInstance;
END IF;
INSERT /*+ APPEND PARALLEL(Archive_File) */ INTO Archive_File
(archive_file_id, disk_instance_name, disk_file_id, disk_file_path,
disk_file_uid, disk_file_gid, size_in_bytes, checksum_adler32,
storage_class_id, creation_time, reconciliation_time, is_deleted) (
SELECT /*+ PARALLEL(F) PARALLEL(D) */
F.fileid, inEOSCTAInstance, F.fileid,
'/eos/' || varNamespacePrefix
|| substr(D.path, varLenCastorPrefix) || '/' || F.filename,
F.disk_uid, F.disk_gid, F.filesize, F.checksum, F.classid, F.btime, 0, '0'
FROM CNS_CTAFilesHelper F, CNS_Dirs_Full_Path D
WHERE F.parent_fileid = D.fileid
-- no need to exclude already existing files because of dual copies
-- as this is validated before hand in the cns_filesForCTAExport procedure
);
COMMIT;
CNS_ctaLog(inTapePool, 'Archive files import completed');
-- And now the tape file metadata: here we also store the constraint_violation
-- Import the tape file metadata: here we also store the constraint_violation
-- errors for later manual analysis
SELECT COUNT(*) INTO nbPreviousErrors FROM Err$_Tape_File;
INSERT /*+ APPEND PARALLEL(Tape_File) */ INTO Tape_File
......@@ -264,6 +231,18 @@ BEGIN
SELECT /*+ PARALLEL(F) */
fileid, filesize, vid, fseq, blockId, copyno, s_mtime FROM CNS_CTAFilesHelper F
) LOG ERRORS INTO Err$_Tape_File ('Importing '|| inTapePool) REJECT LIMIT UNLIMITED;
-- Check if we're dealing with dual copy files
SELECT COUNT(*) INTO varIs2ndCopy FROM Dual
WHERE EXISTS (SELECT 1 FROM CNS_CTAFiles2ndCopyHelper);
IF varIs2ndCopy = 1 THEN
-- Yes, do the same import for the 2nd copies
INSERT /*+ APPEND PARALLEL(Tape_File) */ INTO Tape_File
(archive_file_id, logical_size_in_bytes,
vid, fseq, block_id, copy_nb, creation_time) (
SELECT /*+ PARALLEL(F) */
fileid, filesize, vid, fseq, blockId, 2, s_mtime FROM CNS_CTAFiles2ndCopyHelper F
) LOG ERRORS INTO Err$_Tape_File ('Importing '|| inTapePool) REJECT LIMIT UNLIMITED;
END IF;
COMMIT;
SELECT COUNT(*) INTO nbMissingImports FROM Err$_Tape_File;
IF nbMissingImports = nbPreviousErrors THEN
......@@ -278,25 +257,17 @@ BEGIN
||' files are missing: please check the Err$_Tape_File table');
END IF;
-- Update data counter on the Tape table: this does NOT include deleted segments,
-- Update data counters on the Tape table: this does NOT include deleted segments,
-- but it's the best approximation we have here
FOR v IN (SELECT vid, sum(filesize) totalpervid FROM CNS_CTAFilesHelper GROUP BY vid) LOOP
FOR v IN (SELECT vid, sum(filesize) totalpervid FROM CNS_CTAFilesHelper GROUP BY vid
UNION ALL
SELECT vid, sum(filesize) totalpervid FROM CNS_CTAFiles2ndCopyHelper GROUP BY vid) LOOP
UPDATE Tape
SET data_in_bytes = data_in_bytes + v.totalpervid
WHERE vid = v.vid;
END LOOP;
COMMIT;
CNS_ctaLog(inTapePool, 'Updated Tape data counters');
IF varIs2ndCopy = 1 THEN
-- Clean up the metadata related to dual tape copy files, as they had previously been imported:
-- this is to speed up the subsequent EOS metadata import by avoiding the double import.
-- Note that we only remove 2nd copy metadata for which 1st copy also exists, in the assumption
-- that 1) 2nd copy files are stored on dedicated tapepools (cf. CASTOR routing in castorrepack),
-- and 2) a 2nd-copy tapepool is imported *after* its corresponding 1st-copy one.
-- Those assumptions are enforced in the validateDualCopies() procedure.
CNS_clean2ndCopyInCTAExport(inTapePool);
END IF;
END;
/
......@@ -316,6 +287,7 @@ END;
-- Entry point to import metadata from the CASTOR namespace
CREATE OR REPLACE PROCEDURE importFromCASTOR(inTapePool VARCHAR2, inVO VARCHAR2, inEOSCTAInstance VARCHAR2, inDryRun INTEGER) AS
nbFiles INTEGER;
var2ndTapePool VARCHAR2(100);
BEGIN
-- First check if there's anything already ongoing and fail early:
SELECT COUNT(*) INTO nbFiles FROM CNS_CTAFilesHelper;
......@@ -329,17 +301,23 @@ BEGIN
CNS_ctaLog(inTapePool, 'CASTOR metadata import started [dry-run mode]');
END IF;
BEGIN
-- extract all relevant metadata; can raise exceptions
CNS_filesAndDirsForCTAExport(inTapePool);
-- validate dual tape copies, stop if acceptance criteria are not met
validateDualCopies(inTapePool);
-- extract all relevant files metadata and work out if dual tape copies are to be imported; can raise exceptions
CNS_filesForCTAExport(inTapePool, var2ndTapePool);
-- extract all directories metadata
CNS_dirsForCTAExport(inTapePool);
-- import tapes; can raise exceptions
importTapePool(inTapePool, inVO);
IF var2ndTapePool IS NOT NULL THEN
importTapePool(var2ndTapePool, inVO);
END IF;
-- import metadata into the CTA catalogue
populateCTAFilesFromCASTOR(inEOSCTAInstance, inTapePool);
IF inDryRun = 0 THEN
-- mark tapes as exported, only when executed for real
markCASTORTapePoolExported(inTapePool);
IF var2ndTapePool IS NOT NULL THEN
markCASTORTapePoolExported(var2ndTapePool);
END IF;
CNS_ctaLog(inTapePool, 'CASTOR metadata import completed successfully');
ELSE
CNS_ctaLog(inTapePool, 'CASTOR metadata import completed successfully [dry-run mode]');
......@@ -399,6 +377,7 @@ BEGIN
WHERE archive_file_id = varTempId)
);
CNS_ctaLog(inTapePool, 'Dual tape copies detected, will ALSO remove files metadata from tape pool '|| var2ndTapePool);
CNS_ctaLog(var2ndTapePool, 'Removal of CASTOR files and tapes metadata due to the removal of tape pool '|| inTapePool);
END IF;
-- efficiently delete all Tape_File and Archive_File entries in multiple bulks
OPEN temp_data_cur;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment