Commit 725de502 authored by Giuseppe Lo Presti's avatar Giuseppe Lo Presti
Browse files

Implemented latest decisions as documented in #488. The PL/SQL code is ready...

Implemented latest decisions as documented in #488. The PL/SQL code is ready for testing once the schema changes are available.
parent 8200ee55
......@@ -61,14 +61,15 @@ BEGIN
WHERE F.fileid = S.s_fileid
AND vid IN (
SELECT vid FROM Vmgr_tape_side
WHERE poolName = inPoolName AND BITAND(status, 2) = 0 -- not already EXPORTED
WHERE poolName = inPoolName
AND BITAND(status, 2) = 0 AND BITAND(status, 32) = 0 -- not already EXPORTED or ARCHIVED
);
-- XXX TODO log fileids.LENGTH
-- Populate the helper tables by selecting all required metadata
EXECUTE IMMEDIATE 'TRUNCATE TABLE CTADirsHelper';
INSERT /*+ APPEND */ INTO CTADirsHelper (
-- strip the /castor/cern.ch prefix from all paths
SELECT F.fileid, substr(nvl(D.path, getPathForFileid(F.fileid)), length('/castor/cern.ch')) as path,
SELECT F.fileid, substr(nvl(D.path, getPathForFileid(F.fileid)), length('/castor/cern.ch/')) as path,
F.owner_uid disk_uid, F.gid disk_gid,
F.filemode, F.atime, F.mtime, C.name classname
FROM Cns_file_metadata F, Cns_class_metadata C, Dirs_Full_Path D,
......@@ -76,11 +77,12 @@ BEGIN
WHERE DirIdsTable.column_value = F.fileid
AND F.fileid(+) = D.fileid
);
-- XXX TODO log "CTADirsHelper filled"
COMMIT;
INSERT /*+ APPEND */ INTO CTAFilesHelper (
SELECT F.fileid, decode(D.path, NULL, getPathForFileid(F.fileid), D.path || '/' || F.name) as path,
F.owner_uid, F.gid, F.filemode, F.atime, F.mtime, C.name classname,
S.segsize, S.checksum, S.copyno, S.vid, S.fseq, utl_raw.cast_to_binary_integer(S.blockId),
S.segsize as filesize, S.checksum, S.copyno, S.vid, S.fseq, utl_raw.cast_to_binary_integer(S.blockId),
S.lastModificationTime as s_mtime
FROM Cns_file_metadata F, Cns_seg_metadata S, Cns_class_metadata C, Dirs_Full_Path D,
(SELECT * FROM TABLE(fileIds)) FileIdsTable
......@@ -89,6 +91,7 @@ BEGIN
AND F.parent_fileid(+) = D.fileid
AND F.fileclass = C.classid
);
-- XXX TODO log "CTAFilesHelper filled"
COMMIT;
END;
/
......@@ -117,7 +120,6 @@ END;
-- XXX TO BE CHECKED
EXECUTE IMMEDIATE 'GRANT EXECUTE ON prepareCTAExport TO CTA';
EXECUTE IMMEDIATE 'GRANT EXECUTE ON dirsForCTAExport TO CTA';
EXECUTE IMMEDIATE 'GRANT EXECUTE ON filesForCTAExport TO CTA';
EXECUTE IMMEDIATE 'GRANT EXECUTE ON completeCTAExport TO CTA';
EXECUTE IMMEDIATE 'GRANT EXECUTE ON prepareCTAExport TO ' || &ctaSchema;
EXECUTE IMMEDIATE 'GRANT EXECUTE ON filesAndDirsForCTAExport TO ' || &ctaSchema;
EXECUTE IMMEDIATE 'GRANT EXECUTE ON completeCTAExport TO ' || &ctaSchema;
......@@ -33,6 +33,12 @@ fi
# check that no migrations are pending/ongoing for this tapepool
printmigrationstatus | grep $1 && echo 'Migrations still ongoing, aborting' && exit 1
# check that all tapes are good for export, that is no BUSY or RDONLY
busytapes=`vmgrlisttape -P $1 | grep -c BUSY`
[[ $busytapes -gt 0 ]] && echo 'Found' $busytapes 'tape(s) in BUSY state, aborting' && exit 1
rotapes=`vmgrlisttape -P $1 | grep -c RDONLY`
[[ $rotapes -gt 0 ]] && echo 'Found' $rotapes 'tape(s) in RDONLY state, aborting' && exit 1
# backup relevant metadata
mkdir -p ~/ctaexport
cd ~/ctaexport
......@@ -42,15 +48,16 @@ cd ~/ctaexport
stager_removeprivilege -U:
# on the stager, make the tapepool unusable (the tapepool metadata can stay)
[[ ! -x migrationroute_$1 }} && printmigrationroute | grep $1 > migrationroutes_$1
printmigrationroute | grep $1 | awk '{print $1}' | xargs -i deletemigrationroute {}
[[ ! -x migrationroute_$1 ]] && printmigrationroute | grep -w $1 > migrationroutes_$1
printmigrationroute | grep -w $1 | awk '{print $1}' | xargs -i deletemigrationroute {}
# on the VMGR, mark all tapes as Exported for the tape pool
# on the VMGR, mark all tapes as EXPORTED for the tape pool
vmgrlisttape -P $1 | awk '{print $1}' | xargs -i vmgrmodifytape -V {} --st EXPORTED
# execute the CTA DB extraction from the CTA DB
# (remote-linked with the CASTOR Nameserver)
# ...
# (linked with the CASTOR Nameserver)
# ... code from Michael to invoke the PL/SQL part + feed EOS ...
# empty the CASTOR disk cache (not necessary)
#for h in `printdiskserver | grep cern.ch | awk '{print $1}'`; do
......
......@@ -116,8 +116,7 @@ BEGIN
JOB_NAME => 'StatsJob',
JOB_TYPE => 'PLSQL_BLOCK',
JOB_ACTION => 'BEGIN gatherCatalogueStats(); END;',
JOB_CLASS => 'CASTOR_JOB_CLASS',
START_DATE => SYSDATE + 60/1440,
START_DATE => SYSDATE,
REPEAT_INTERVAL => 'FREQ=DAILY; INTERVAL=1',
ENABLED => TRUE,
COMMENTS => 'Gathering of catalogue usage statistics');
......@@ -134,6 +133,30 @@ END;
-- Create synonyms for all relevant tables
-- XXX TBD XXX
/* Get current time as a time_t (Unix time) */
CREATE OR REPLACE FUNCTION getTime RETURN NUMBER IS
epoch TIMESTAMP WITH TIME ZONE;
now TIMESTAMP WITH TIME ZONE;
interval INTERVAL DAY(9) TO SECOND;
interval_days NUMBER;
interval_hours NUMBER;
interval_minutes NUMBER;
interval_seconds NUMBER;
BEGIN
epoch := TO_TIMESTAMP_TZ('01-JAN-1970 00:00:00 00:00',
'DD-MON-YYYY HH24:MI:SS TZH:TZM');
now := SYSTIMESTAMP AT TIME ZONE '00:00';
interval := now - epoch;
interval_days := EXTRACT(DAY FROM (interval));
interval_hours := EXTRACT(HOUR FROM (interval));
interval_minutes := EXTRACT(MINUTE FROM (interval));
interval_seconds := EXTRACT(SECOND FROM (interval));
RETURN interval_days * 24 * 60 * 60 + interval_hours * 60 * 60 +
interval_minutes * 60 + interval_seconds;
END;
/
/* Function to convert seconds into a time string using the format:
* DD-MON-YYYY HH24:MI:SS. If seconds is not defined then the current time
* will be returned. Note that the time is converted from UTC to the
......@@ -153,17 +176,19 @@ END;
CREATE OR REPLACE PROCEDURE importFromCASTOR(inTapePool VARCHAR2, inVO VARCHAR2, inEOSCTAInstance VARCHAR2,
Dirs OUT SYS_REFCURSOR, Files OUT SYS_REFCURSOR) AS
BEGIN
-- first import tapes
-- first import tapes; can raise exceptions
importTapePool(inTapePool, inVO);
-- XXX error handling is missing
-- extract all relevant metadata; can raise exceptions
castor.filesAndDirsForCTAExport(inTapePool);
-- import metadata into the CTA catalogue
populateCTAFromCASTOR(inEOSCTAInstance);
-- Get all metadata for the EOS-side namespace
-- return all metadata for the EOS-side namespace
OPEN Dirs FOR
SELECT * FROM castor.CTADirsHelper;
OPEN Files FOR
SELECT * FROM castor.CTAFilesHelper;
SELECT F.fileid, F.path, F.owner_uid, F.gid, F.filemode,
F.filesize, F.checksum, F.atime, F.mtime, F.classname
FROM castor.CTAFilesHelper F;
END;
/
......@@ -179,7 +204,7 @@ BEGIN
disk_gid, size_in_bytes, checksum_type, checksum_value,
storage_class_id, creation_time, reconciliation_time)
VALUES (f.fileId, inEOSCTAInstance, f.fileId, pathInEos, f.gid,
f.filesize, 'AD', f.checksum, f.classname, f.atime, 0);
f.filesize, 'AD', f.checksum, f.classname, f.atime, 0); -- XXX 'AD'?
-- insert tape metadata
INSERT INTO Tape_File (archive_file_id, vid, fseq, block_id, copy_nb, creation_time)
VALUES (f.fileId, f.vid, f.fseq, f.blockId, f.copyno, f.s_mtime);
......@@ -201,49 +226,64 @@ END;
-- Import a tapepool and its tapes from CASTOR
-- Raises constraint_violation if the tapepool and/or some tapes were already imported
CREATE OR REPLACE PROCEDURE importTapePool(inTapePool VARCHAR2, inVO VARCHAR2) AS
varTapePoolName VARCHAR2 := inTapePool;
BEGIN
-- strip the 'r_' legacy prefix if present
IF substr(varTapePoolName, 0, 2) = 'r_' THEN
varTapePoolName := substr(varTapePoolName, 3);
END IF;
INSERT INTO Tape_Pool (tape_pool_name, vo, nb_partial_tapes, is_encrypted, user_comment,
creation_log_user_name, creation_log_host_name, creation_log_time, last_update_user_name,
last_update_host_name, last_update_time) VALUES (
inTapePool,
varTapePoolName,
inVO,
0, -- nb_partial_tapes?
0, -- nb_partial_tapes, to be filled afterwards
'F',
'Imported from CASTOR',
'CASTOR', 'CASTOR', getTime,
'CASTOR', 'CASTOR', getTime
'CASTOR', 'CASTOR', getTime(),
'CASTOR', 'CASTOR', getTime()
);
FOR T in (SELECT T.vid, T.library, TS.status, TS.nbfiles
FOR T in (SELECT T.vid, T.model, T.manufacturer, T.library, TS.status, TS.nbfiles,
TS.rcount, TS.wcount, TS.rhost, TS.whost, TS.rtime, TS.wtime
FROM castor.Vmgr_tape_info T, castor.Vmgr_tape_side TS
WHERE T.vid = TS.vid and TS.poolname = 'r_' || inTapePool) LOOP
WHERE T.vid = TS.vid AND TS.poolname = inTapePool) LOOP
INSERT INTO Tape (vid, media_type, vendor, logical_library_name, tape_pool_name,
encryption_key, capacity_in_bytes, data_in_bytes, last_fseq, is_disabled, is_full,
label_drive, label_time, last_read_drive, last_read_time, last_write_drive, last_write_time,
label_drive, label_time, last_read_drive, last_read_time, read_mount_count,
last_write_drive, last_write_time, write_mount_count,
user_comment, creation_log_user_name, creation_log_host_name, creation_log_time,
last_update_user_name, last_update_host_name, last_update_time) VALUES (
T.vid,
'', '', -- media_type & vendor from model/media_letter/manufacturer
T.model,
T.manufacturer,
T.library,
inTapePool,
'',
0, -- capacity?
0, -- data: will be filled afterwards
0, -- last_fseq?
False,
BITAND(TS.status, "FULL") > 0, -- XXX find the right value for FULL
'CASTOR', 0, -- label_drive and time
'CASTOR', 0, -- last read drive and time
'CASTOR', 0, -- last write drive and time
varTapePoolName,
'', -- blank encryption key
decode(T.density, -- capacity: only one of those options
'7000GC', 7000000000000,
'8000GC', 8000000000000,
'9TC', 9000000000000,
'10TC', 10000000000000,
'12TC', 12000000000000,
'15TC', 15000000000000,
0),
0, -- total data: will be filled by populateCTAFromCASTOR()
TS.nbfiles,
decode(BITAND(TS.status, 1) != 0, 'T', 'F'), -- DISABLED flag
decode(BITAND(TS.status, 8) != 0, 'T', 'F'), -- FULL flag
'CASTOR', 0, -- label drive and time (unknown)
TS.rhost, TS.rtime, TS.rcount, -- last read drive/time and count
TS.whost, TS.wtime, TS.wcount, -- last write drive/time and count
'Imported from CASTOR',
'CASTOR', 'CASTOR', getTime,
'CASTOR', 'CASTOR', getTime
'CASTOR', 'CASTOR', getTime(),
'CASTOR', 'CASTOR', getTime()
);
END LOOP;
COMMIT;
END;
/
-- Complete an import session. This procedure is idempotent.
CREATE OR REPLACE PROCEDURE completeImportFromCASTOR AS
BEGIN
castor.completeCTAExport;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment