diff --git a/castor/db/oracleRepack.sql b/castor/db/oracleRepack.sql
index f57812c9611dda26df8762efd9f18c8be8240126..0f8603fd2c3383362b87701aef1b95253bd1fd28 100644
--- a/castor/db/oracleRepack.sql
+++ b/castor/db/oracleRepack.sql
@@ -450,12 +450,12 @@ CREATE OR REPLACE PROCEDURE triggerRepackRecall
(inCfId IN INTEGER, inFileId IN INTEGER, inNsHost IN VARCHAR2, inBlock IN RAW,
inFseq IN INTEGER, inCopynb IN INTEGER, inEuid IN INTEGER, inEgid IN INTEGER,
inRecallGroupId IN INTEGER, inSvcClassId IN INTEGER, inVid IN VARCHAR2, inFileSize IN INTEGER,
- inFileClass IN INTEGER, inAllSegments IN VARCHAR2, inReqUUID IN VARCHAR2,
+ inFileClass IN INTEGER, inAllValidSegments IN VARCHAR2, inReqUUID IN VARCHAR2,
inSubReqUUID IN VARCHAR2, inRecallGroupName IN VARCHAR2) AS
varLogParam VARCHAR2(2048);
- varAllCopyNbs "numList" := "numList"();
- varAllVIDs strListTable := strListTable();
- varAllSegments strListTable;
+ varAllValidCopyNbs "numList" := "numList"();
+ varAllValidVIDs strListTable := strListTable();
+ varAllValidSegments strListTable;
varFileClassId INTEGER;
BEGIN
-- create recallJob for the given VID, copyNb, etc.
@@ -470,18 +470,22 @@ BEGIN
varLogParam || ' fileClass=' || TO_CHAR(inFileClass) || ' copyNb=' || TO_CHAR(inCopynb)
|| ' TPVID=' || inVid || ' fseq=' || TO_CHAR(inFseq) || ' FileSize=' || TO_CHAR(inFileSize));
-- create missing segments if needed
- SELECT * BULK COLLECT INTO varAllSegments
- FROM TABLE(strTokenizer(inAllSegments));
- FOR i IN 1 .. varAllSegments.COUNT/2 LOOP
- varAllCopyNbs.EXTEND;
- varAllCopyNbs(i) := TO_NUMBER(varAllSegments(2*i-1));
- varAllVIDs.EXTEND;
- varAllVIDs(i) := varAllSegments(2*i);
+ SELECT * BULK COLLECT INTO varAllValidSegments
+ FROM TABLE(strTokenizer(inAllValidSegments));
+ FOR i IN 1 .. varAllValidSegments.COUNT/2 LOOP
+ varAllValidCopyNbs.EXTEND;
+ varAllValidCopyNbs(i) := TO_NUMBER(varAllValidSegments(2*i-1));
+ varAllValidVIDs.EXTEND;
+ varAllValidVIDs(i) := varAllValidSegments(2*i);
END LOOP;
SELECT id INTO varFileClassId
FROM FileClass WHERE classId = inFileClass;
- createMJForMissingSegments(inCfId, inFileSize, varFileClassId, varAllCopyNbs,
- varAllVIDs, inFileId, inNsHost, varLogParam);
+ -- Note that the number given here for the number of existing segments is the total number of
+ -- segment, without removing the ones on EXPORTED tapes. This means that repack will not
+ -- recreate new segments for files that have some copies on EXPORTED tapes.
+ -- This is different from standard recalls that would recreate segments on EXPORTED tapes.
+ createMJForMissingSegments(inCfId, inFileSize, varFileClassId, varAllValidCopyNbs,
+ varAllValidVIDs, varAllValidCopyNbs.COUNT, inFileId, inNsHost, varLogParam);
END;
/
diff --git a/castor/db/oracleStager.sql b/castor/db/oracleStager.sql
index 7eb1b0d75944626c0a5f3f603935ee38f64648e6..5b36f057e7f96fe9e209530bdb0d497f1ad8a3f0 100644
--- a/castor/db/oracleStager.sql
+++ b/castor/db/oracleStager.sql
@@ -2624,8 +2624,9 @@ END;
CREATE OR REPLACE PROCEDURE createMJForMissingSegments(inCfId IN INTEGER,
inFileSize IN INTEGER,
inFileClassId IN INTEGER,
- inAllCopyNbs IN "numList",
- inAllVIDs IN strListTable,
+ inAllValidCopyNbs IN "numList",
+ inAllValidVIDs IN strListTable,
+ inNbExistingSegments IN INTEGER,
inFileId IN INTEGER,
inNsHost IN VARCHAR2,
inLogParams IN VARCHAR2) AS
@@ -2636,7 +2637,7 @@ CREATE OR REPLACE PROCEDURE createMJForMissingSegments(inCfId IN INTEGER,
BEGIN
-- check whether there are missing segments and whether we should create new ones
SELECT nbCopies INTO varExpectedNbCopies FROM FileClass WHERE id = inFileClassId;
- IF varExpectedNbCopies > inAllCopyNbs.COUNT THEN
+ IF varExpectedNbCopies > inNbExistingSegments THEN
-- some copies are missing
DECLARE
unused INTEGER;
@@ -2648,7 +2649,7 @@ BEGIN
-- another recall group.
-- log "detected missing copies on tape, but migrations ongoing"
logToDLF(NULL, dlf.LVL_DEBUG, dlf.RECALL_MISSING_COPIES_NOOP, inFileId, inNsHost, 'stagerd',
- inLogParams || ' nbMissingCopies=' || TO_CHAR(varExpectedNbCopies-inAllCopyNbs.COUNT));
+ inLogParams || ' nbMissingCopies=' || TO_CHAR(varExpectedNbCopies-inNbExistingSegments));
RETURN;
EXCEPTION WHEN NO_DATA_FOUND THEN
-- we need to remigrate this file
@@ -2656,12 +2657,12 @@ BEGIN
END;
-- log "detected missing copies on tape"
logToDLF(NULL, dlf.LVL_SYSTEM, dlf.RECALL_MISSING_COPIES, inFileId, inNsHost, 'stagerd',
- inLogParams || ' nbMissingCopies=' || TO_CHAR(varExpectedNbCopies-inAllCopyNbs.COUNT));
+ inLogParams || ' nbMissingCopies=' || TO_CHAR(varExpectedNbCopies-inNbExistingSegments));
-- copies are missing, try to recreate them
- WHILE varExpectedNbCopies > inAllCopyNbs.COUNT + varCreatedMJs AND varNextCopyNb <= varExpectedNbCopies LOOP
+ WHILE varExpectedNbCopies > inNbExistingSegments + varCreatedMJs AND varNextCopyNb <= varExpectedNbCopies LOOP
BEGIN
-- check whether varNextCopyNb is already in use by a valid copy
- SELECT * INTO varNb FROM TABLE(inAllCopyNbs) WHERE COLUMN_VALUE=varNextCopyNb;
+ SELECT * INTO varNb FROM TABLE(inAllValidCopyNbs) WHERE COLUMN_VALUE=varNextCopyNb;
-- this copy number is in use, go to next one
EXCEPTION WHEN NO_DATA_FOUND THEN
-- copy number is not in use, create a migrationJob using it (may throw exceptions)
@@ -2674,19 +2675,19 @@ BEGIN
varNextCopyNb := varNextCopyNb + 1;
END LOOP;
-- Did we create new copies ?
- IF varExpectedNbCopies > inAllCopyNbs.COUNT + varCreatedMJs THEN
+ IF varExpectedNbCopies > inNbExistingSegments + varCreatedMJs THEN
-- We did not create enough new copies, this means that we did not find enough
-- valid copy numbers. Odd... Log something !
logToDLF(NULL, dlf.LVL_ERROR, dlf.RECALL_COPY_STILL_MISSING, inFileId, inNsHost, 'stagerd',
inLogParams || ' nbCopiesStillMissing=' ||
- TO_CHAR(varExpectedNbCopies - inAllCopyNbs.COUNT - varCreatedMJs));
+ TO_CHAR(varExpectedNbCopies - inAllValidCopyNbs.COUNT - varCreatedMJs));
ELSE
-- Yes, then create migrated segments for the existing segments if there are none
SELECT count(*) INTO varNb FROM MigratedSegment WHERE castorFile = inCfId;
IF varNb = 0 THEN
- FOR i IN inAllCopyNbs.FIRST .. inAllCopyNbs.LAST LOOP
+ FOR i IN inAllValidCopyNbs.FIRST .. inAllValidCopyNbs.LAST LOOP
INSERT INTO MigratedSegment (castorFile, copyNb, VID)
- VALUES (inCfId, inAllCopyNbs(i), inAllVIDs(i));
+ VALUES (inCfId, inAllValidCopyNbs(i), inAllValidVIDs(i));
END LOOP;
END IF;
END IF;
@@ -2710,8 +2711,9 @@ CREATE OR REPLACE FUNCTION createRecallJobs(inCfId IN INTEGER,
inRequestTime IN NUMBER,
inLogParams IN VARCHAR2) RETURN INTEGER AS
-- list of all valid segments, whatever the tape status. Used to trigger remigrations
- varAllCopyNbs "numList" := "numList"();
- varAllVIDs strListTable := strListTable();
+ varAllValidCopyNbs "numList" := "numList"();
+ varAllValidVIDs strListTable := strListTable();
+ varNbExistingSegments INTEGER := 0;
-- whether we found a segment at all (valid or not). Used to detect potentially lost files
varFoundSeg boolean := FALSE;
varI INTEGER := 1;
@@ -2736,10 +2738,10 @@ BEGIN
BITAND(varSeg.tapeStatus, tconst.TAPE_EXPORTED) = 0 AND
BITAND(varSeg.tapeStatus, tconst.TAPE_ARCHIVED) = 0 THEN
-- remember the copy number and tape
- varAllCopyNbs.EXTEND;
- varAllCopyNbs(varI) := varSeg.copyno;
- varAllVIDs.EXTEND;
- varAllVIDs(varI) := varSeg.vid;
+ varAllValidCopyNbs.EXTEND;
+ varAllValidCopyNbs(varI) := varSeg.copyno;
+ varAllValidVIDs.EXTEND;
+ varAllValidVIDs(varI) := varSeg.vid;
varI := varI + 1;
-- create recallJob
INSERT INTO RecallJob (id, castorFile, copyNb, recallGroup, svcClass, euid, egid,
@@ -2747,15 +2749,29 @@ BEGIN
VALUES (ids_seq.nextval, inCfId, varSeg.copyno, inRecallGroupId, inSvcClassId,
inEuid, inEgid, varSeg.vid, varSeg.fseq, tconst.RECALLJOB_PENDING, inFileSize, inRequestTime,
varSeg.blockId, NULL);
+ varNbExistingSegments := varNbExistingSegments + 1;
-- log "created new RecallJob"
logToDLF(NULL, dlf.LVL_SYSTEM, dlf.RECALL_CREATING_RECALLJOB, inFileId, inNsHost, 'stagerd',
inLogParams || ' copyNb=' || TO_CHAR(varSeg.copyno) || ' TPVID=' || varSeg.vid ||
' fseq=' || TO_CHAR(varSeg.fseq || ' FileSize=' || TO_CHAR(inFileSize)));
ELSE
- -- invalid tape found. Log it.
- -- "createRecallCandidate: found segment on unusable tape"
- logToDLF(NULL, dlf.LVL_DEBUG, dlf.RECALL_UNUSABLE_TAPE, inFileId, inNsHost, 'stagerd',
- inLogParams || ' segStatus=OK tapeStatus=' || tapeStatusToString(varSeg.tapeStatus));
+ -- Should the segment be counted in the count of existing segments ?
+ -- In other terms, should we recreate a segment for replacing this one ?
+ -- Yes if the segment in on an EXPORTED tape.
+ IF BITAND(varSeg.tapeStatus, tconst.TAPE_EXPORTED) = 0 THEN
+ -- invalid tape found with segments that are counting for the total count.
+ -- "createRecallCandidate: found segment on unusable tape"
+ logToDLF(NULL, dlf.LVL_SYSTEM, dlf.RECALL_UNUSABLE_TAPE, inFileId, inNsHost, 'stagerd',
+ inLogParams || ' segStatus=OK tapeStatus=' || tapeStatusToString(varSeg.tapeStatus) ||
+ ' recreatingSegment=No');
+ varNbExistingSegments := varNbExistingSegments + 1;
+ ELSE
+ -- invalid tape found with segments that will be completely ignored.
+ -- "createRecallCandidate: found segment on unusable tape"
+ logToDLF(NULL, dlf.LVL_DEBUG, dlf.RECALL_UNUSABLE_TAPE, inFileId, inNsHost, 'stagerd',
+ inLogParams || ' segStatus=OK tapeStatus=' || tapeStatusToString(varSeg.tapeStatus) ||
+ ' recreatingSegment=Yes');
+ END IF;
END IF;
ELSE
-- invalid segment tape found. Log it.
@@ -2780,15 +2796,15 @@ BEGIN
RETURN serrno.ESTNOSEGFOUND;
END IF;
-- If we found no valid segment (but some disabled ones), log a warning
- IF varAllCopyNbs.COUNT = 0 THEN
+ IF varAllValidCopyNbs.COUNT = 0 THEN
-- log "createRecallCandidate: no valid segment to recall found"
logToDLF(NULL, dlf.LVL_WARNING, dlf.RECALL_NO_SEG_FOUND, inFileId, inNsHost, 'stagerd', inLogParams);
RETURN serrno.ESTNOSEGFOUND;
END IF;
BEGIN
-- create missing segments if needed
- createMJForMissingSegments(inCfId, inFileSize, inFileClassId, varAllCopyNbs,
- varAllVIDs, inFileId, inNsHost, inLogParams);
+ createMJForMissingSegments(inCfId, inFileSize, inFileClassId, varAllValidCopyNbs,
+ varAllValidVIDs, varNbExistingSegments, inFileId, inNsHost, inLogParams);
EXCEPTION WHEN NO_TAPE_ROUTE THEN
-- there's at least a missing segment and we cannot recreate it!
-- log a "no route to tape defined for missing copy" error, but don't fail the recall
diff --git a/castor/db/stager_oracle_create.sql b/castor/db/stager_oracle_create.sql
index eff817f81da1cbe6ba0144e48eb06fff2b8ae96e..861989bd0414fe3444df281d1c38d5019c9da075 100644
--- a/castor/db/stager_oracle_create.sql
+++ b/castor/db/stager_oracle_create.sql
@@ -3489,12 +3489,12 @@ CREATE OR REPLACE PROCEDURE triggerRepackRecall
(inCfId IN INTEGER, inFileId IN INTEGER, inNsHost IN VARCHAR2, inBlock IN RAW,
inFseq IN INTEGER, inCopynb IN INTEGER, inEuid IN INTEGER, inEgid IN INTEGER,
inRecallGroupId IN INTEGER, inSvcClassId IN INTEGER, inVid IN VARCHAR2, inFileSize IN INTEGER,
- inFileClass IN INTEGER, inAllSegments IN VARCHAR2, inReqUUID IN VARCHAR2,
+ inFileClass IN INTEGER, inAllValidSegments IN VARCHAR2, inReqUUID IN VARCHAR2,
inSubReqUUID IN VARCHAR2, inRecallGroupName IN VARCHAR2) AS
varLogParam VARCHAR2(2048);
- varAllCopyNbs "numList" := "numList"();
- varAllVIDs strListTable := strListTable();
- varAllSegments strListTable;
+ varAllValidCopyNbs "numList" := "numList"();
+ varAllValidVIDs strListTable := strListTable();
+ varAllValidSegments strListTable;
varFileClassId INTEGER;
BEGIN
-- create recallJob for the given VID, copyNb, etc.
@@ -3509,18 +3509,22 @@ BEGIN
varLogParam || ' fileClass=' || TO_CHAR(inFileClass) || ' copyNb=' || TO_CHAR(inCopynb)
|| ' TPVID=' || inVid || ' fseq=' || TO_CHAR(inFseq) || ' FileSize=' || TO_CHAR(inFileSize));
-- create missing segments if needed
- SELECT * BULK COLLECT INTO varAllSegments
- FROM TABLE(strTokenizer(inAllSegments));
- FOR i IN 1 .. varAllSegments.COUNT/2 LOOP
- varAllCopyNbs.EXTEND;
- varAllCopyNbs(i) := TO_NUMBER(varAllSegments(2*i-1));
- varAllVIDs.EXTEND;
- varAllVIDs(i) := varAllSegments(2*i);
+ SELECT * BULK COLLECT INTO varAllValidSegments
+ FROM TABLE(strTokenizer(inAllValidSegments));
+ FOR i IN 1 .. varAllValidSegments.COUNT/2 LOOP
+ varAllValidCopyNbs.EXTEND;
+ varAllValidCopyNbs(i) := TO_NUMBER(varAllValidSegments(2*i-1));
+ varAllValidVIDs.EXTEND;
+ varAllValidVIDs(i) := varAllValidSegments(2*i);
END LOOP;
SELECT id INTO varFileClassId
FROM FileClass WHERE classId = inFileClass;
- createMJForMissingSegments(inCfId, inFileSize, varFileClassId, varAllCopyNbs,
- varAllVIDs, inFileId, inNsHost, varLogParam);
+ -- Note that the number given here for the number of existing segments is the total number of
+ -- segment, without removing the ones on EXPORTED tapes. This means that repack will not
+ -- recreate new segments for files that have some copies on EXPORTED tapes.
+ -- This is different from standard recalls that would recreate segments on EXPORTED tapes.
+ createMJForMissingSegments(inCfId, inFileSize, varFileClassId, varAllValidCopyNbs,
+ varAllValidVIDs, varAllValidCopyNbs.COUNT, inFileId, inNsHost, varLogParam);
END;
/
@@ -6243,8 +6247,9 @@ END;
CREATE OR REPLACE PROCEDURE createMJForMissingSegments(inCfId IN INTEGER,
inFileSize IN INTEGER,
inFileClassId IN INTEGER,
- inAllCopyNbs IN "numList",
- inAllVIDs IN strListTable,
+ inAllValidCopyNbs IN "numList",
+ inAllValidVIDs IN strListTable,
+ inNbExistingSegments IN INTEGER,
inFileId IN INTEGER,
inNsHost IN VARCHAR2,
inLogParams IN VARCHAR2) AS
@@ -6255,7 +6260,7 @@ CREATE OR REPLACE PROCEDURE createMJForMissingSegments(inCfId IN INTEGER,
BEGIN
-- check whether there are missing segments and whether we should create new ones
SELECT nbCopies INTO varExpectedNbCopies FROM FileClass WHERE id = inFileClassId;
- IF varExpectedNbCopies > inAllCopyNbs.COUNT THEN
+ IF varExpectedNbCopies > inNbExistingSegments THEN
-- some copies are missing
DECLARE
unused INTEGER;
@@ -6267,7 +6272,7 @@ BEGIN
-- another recall group.
-- log "detected missing copies on tape, but migrations ongoing"
logToDLF(NULL, dlf.LVL_DEBUG, dlf.RECALL_MISSING_COPIES_NOOP, inFileId, inNsHost, 'stagerd',
- inLogParams || ' nbMissingCopies=' || TO_CHAR(varExpectedNbCopies-inAllCopyNbs.COUNT));
+ inLogParams || ' nbMissingCopies=' || TO_CHAR(varExpectedNbCopies-inNbExistingSegments));
RETURN;
EXCEPTION WHEN NO_DATA_FOUND THEN
-- we need to remigrate this file
@@ -6275,12 +6280,12 @@ BEGIN
END;
-- log "detected missing copies on tape"
logToDLF(NULL, dlf.LVL_SYSTEM, dlf.RECALL_MISSING_COPIES, inFileId, inNsHost, 'stagerd',
- inLogParams || ' nbMissingCopies=' || TO_CHAR(varExpectedNbCopies-inAllCopyNbs.COUNT));
+ inLogParams || ' nbMissingCopies=' || TO_CHAR(varExpectedNbCopies-inNbExistingSegments));
-- copies are missing, try to recreate them
- WHILE varExpectedNbCopies > inAllCopyNbs.COUNT + varCreatedMJs AND varNextCopyNb <= varExpectedNbCopies LOOP
+ WHILE varExpectedNbCopies > inNbExistingSegments + varCreatedMJs AND varNextCopyNb <= varExpectedNbCopies LOOP
BEGIN
-- check whether varNextCopyNb is already in use by a valid copy
- SELECT * INTO varNb FROM TABLE(inAllCopyNbs) WHERE COLUMN_VALUE=varNextCopyNb;
+ SELECT * INTO varNb FROM TABLE(inAllValidCopyNbs) WHERE COLUMN_VALUE=varNextCopyNb;
-- this copy number is in use, go to next one
EXCEPTION WHEN NO_DATA_FOUND THEN
-- copy number is not in use, create a migrationJob using it (may throw exceptions)
@@ -6293,19 +6298,19 @@ BEGIN
varNextCopyNb := varNextCopyNb + 1;
END LOOP;
-- Did we create new copies ?
- IF varExpectedNbCopies > inAllCopyNbs.COUNT + varCreatedMJs THEN
+ IF varExpectedNbCopies > inNbExistingSegments + varCreatedMJs THEN
-- We did not create enough new copies, this means that we did not find enough
-- valid copy numbers. Odd... Log something !
logToDLF(NULL, dlf.LVL_ERROR, dlf.RECALL_COPY_STILL_MISSING, inFileId, inNsHost, 'stagerd',
inLogParams || ' nbCopiesStillMissing=' ||
- TO_CHAR(varExpectedNbCopies - inAllCopyNbs.COUNT - varCreatedMJs));
+ TO_CHAR(varExpectedNbCopies - inAllValidCopyNbs.COUNT - varCreatedMJs));
ELSE
-- Yes, then create migrated segments for the existing segments if there are none
SELECT count(*) INTO varNb FROM MigratedSegment WHERE castorFile = inCfId;
IF varNb = 0 THEN
- FOR i IN inAllCopyNbs.FIRST .. inAllCopyNbs.LAST LOOP
+ FOR i IN inAllValidCopyNbs.FIRST .. inAllValidCopyNbs.LAST LOOP
INSERT INTO MigratedSegment (castorFile, copyNb, VID)
- VALUES (inCfId, inAllCopyNbs(i), inAllVIDs(i));
+ VALUES (inCfId, inAllValidCopyNbs(i), inAllValidVIDs(i));
END LOOP;
END IF;
END IF;
@@ -6329,8 +6334,9 @@ CREATE OR REPLACE FUNCTION createRecallJobs(inCfId IN INTEGER,
inRequestTime IN NUMBER,
inLogParams IN VARCHAR2) RETURN INTEGER AS
-- list of all valid segments, whatever the tape status. Used to trigger remigrations
- varAllCopyNbs "numList" := "numList"();
- varAllVIDs strListTable := strListTable();
+ varAllValidCopyNbs "numList" := "numList"();
+ varAllValidVIDs strListTable := strListTable();
+ varNbExistingSegments INTEGER := 0;
-- whether we found a segment at all (valid or not). Used to detect potentially lost files
varFoundSeg boolean := FALSE;
varI INTEGER := 1;
@@ -6355,10 +6361,10 @@ BEGIN
BITAND(varSeg.tapeStatus, tconst.TAPE_EXPORTED) = 0 AND
BITAND(varSeg.tapeStatus, tconst.TAPE_ARCHIVED) = 0 THEN
-- remember the copy number and tape
- varAllCopyNbs.EXTEND;
- varAllCopyNbs(varI) := varSeg.copyno;
- varAllVIDs.EXTEND;
- varAllVIDs(varI) := varSeg.vid;
+ varAllValidCopyNbs.EXTEND;
+ varAllValidCopyNbs(varI) := varSeg.copyno;
+ varAllValidVIDs.EXTEND;
+ varAllValidVIDs(varI) := varSeg.vid;
varI := varI + 1;
-- create recallJob
INSERT INTO RecallJob (id, castorFile, copyNb, recallGroup, svcClass, euid, egid,
@@ -6366,15 +6372,29 @@ BEGIN
VALUES (ids_seq.nextval, inCfId, varSeg.copyno, inRecallGroupId, inSvcClassId,
inEuid, inEgid, varSeg.vid, varSeg.fseq, tconst.RECALLJOB_PENDING, inFileSize, inRequestTime,
varSeg.blockId, NULL);
+ varNbExistingSegments := varNbExistingSegments + 1;
-- log "created new RecallJob"
logToDLF(NULL, dlf.LVL_SYSTEM, dlf.RECALL_CREATING_RECALLJOB, inFileId, inNsHost, 'stagerd',
inLogParams || ' copyNb=' || TO_CHAR(varSeg.copyno) || ' TPVID=' || varSeg.vid ||
' fseq=' || TO_CHAR(varSeg.fseq || ' FileSize=' || TO_CHAR(inFileSize)));
ELSE
- -- invalid tape found. Log it.
- -- "createRecallCandidate: found segment on unusable tape"
- logToDLF(NULL, dlf.LVL_DEBUG, dlf.RECALL_UNUSABLE_TAPE, inFileId, inNsHost, 'stagerd',
- inLogParams || ' segStatus=OK tapeStatus=' || tapeStatusToString(varSeg.tapeStatus));
+ -- Should the segment be counted in the count of existing segments ?
+ -- In other terms, should we recreate a segment for replacing this one ?
+ -- Yes if the segment in on an EXPORTED tape.
+ IF BITAND(varSeg.tapeStatus, tconst.TAPE_EXPORTED) = 0 THEN
+ -- invalid tape found with segments that are counting for the total count.
+ -- "createRecallCandidate: found segment on unusable tape"
+ logToDLF(NULL, dlf.LVL_SYSTEM, dlf.RECALL_UNUSABLE_TAPE, inFileId, inNsHost, 'stagerd',
+ inLogParams || ' segStatus=OK tapeStatus=' || tapeStatusToString(varSeg.tapeStatus) ||
+ ' recreatingSegment=No');
+ varNbExistingSegments := varNbExistingSegments + 1;
+ ELSE
+ -- invalid tape found with segments that will be completely ignored.
+ -- "createRecallCandidate: found segment on unusable tape"
+ logToDLF(NULL, dlf.LVL_DEBUG, dlf.RECALL_UNUSABLE_TAPE, inFileId, inNsHost, 'stagerd',
+ inLogParams || ' segStatus=OK tapeStatus=' || tapeStatusToString(varSeg.tapeStatus) ||
+ ' recreatingSegment=Yes');
+ END IF;
END IF;
ELSE
-- invalid segment tape found. Log it.
@@ -6399,15 +6419,15 @@ BEGIN
RETURN serrno.ESTNOSEGFOUND;
END IF;
-- If we found no valid segment (but some disabled ones), log a warning
- IF varAllCopyNbs.COUNT = 0 THEN
+ IF varAllValidCopyNbs.COUNT = 0 THEN
-- log "createRecallCandidate: no valid segment to recall found"
logToDLF(NULL, dlf.LVL_WARNING, dlf.RECALL_NO_SEG_FOUND, inFileId, inNsHost, 'stagerd', inLogParams);
RETURN serrno.ESTNOSEGFOUND;
END IF;
BEGIN
-- create missing segments if needed
- createMJForMissingSegments(inCfId, inFileSize, inFileClassId, varAllCopyNbs,
- varAllVIDs, inFileId, inNsHost, inLogParams);
+ createMJForMissingSegments(inCfId, inFileSize, inFileClassId, varAllValidCopyNbs,
+ varAllValidVIDs, varNbExistingSegments, inFileId, inNsHost, inLogParams);
EXCEPTION WHEN NO_TAPE_ROUTE THEN
-- there's at least a missing segment and we cannot recreate it!
-- log a "no route to tape defined for missing copy" error, but don't fail the recall
diff --git a/client/src/stager/stager_get.man b/client/src/stager/stager_get.man
index 25ec35b020b4078f75991b6ef87482e638ce5dd2..3cb75c08eb0738037be3beb56c731062d04cb900 100644
--- a/client/src/stager/stager_get.man
+++ b/client/src/stager/stager_get.man
@@ -39,6 +39,12 @@ pre-stages a list of CASTOR HSM files from tape to disk. The command
only supports asychronous stage-in operation meaning that it returns immediately
after the stager has stored and acknowledged the request. The command does not wait for the
files to be staged in and it is supposed to be used in conjunction with the
+
+Note that this command will also trigger the remigration/repair of missing segments for the involved
+files be repacked. A typical case is when the second copy of a two copies file has been lost and
+stager_get recalls the first copy. Also note that segments on EXPORTED tapes will be considered as
+lost and thus remigrated/repaired. This is different behavior repack for which they will not be.
+
.B stager_qry(1castor)
command to check the progress of the request.
@@ -99,6 +105,7 @@ failed.
.SH SEE ALSO
.BR stager_qry(1castor) ,
.BR stager_update(1castor)
+.BR repack(1castor)
.SH AUTHOR
\fBCASTOR\fP Team <castor.support@cern.ch>
diff --git a/hsmtools/repack.man b/hsmtools/repack.man
index 452fecc6aa5465cbb696e4c56e58ce1946d9988d..23ee8ccc57189ba0d2600adb8f5fa641343aa768 100644
--- a/hsmtools/repack.man
+++ b/hsmtools/repack.man
@@ -61,6 +61,12 @@ status may be repacked.
When the repack process has been successfully completed (FINISHED status), the volume can be
reclaimed using the reclaim command.
+Note that repack will also trigger the remigration/repair of missing segments for the files that will
+be repacked. A typical case is when the second copy of a two copies file has been lost and
+the tape containing the first copy is repacked.
+However, segments on EXPORTED tapes will not be considered as lost. This is different behavior
+from standard recalls triggered e.g. by stager_get for which they will be.
+
.SH OPTIONS
.TP
@@ -210,5 +216,8 @@ Output restricted to 2 errors. There are more errors for this tape
This command requires database client access to the stager catalogue.
Configuration for the database access is taken from castor.conf.
+.SH SEE ALSO
+.BR stager_get(1castor) ,
+
.SH AUTHOR
\fBCASTOR\fP Team <castor.support@cern.ch
diff --git a/upgrades/stager_2.1.14-15_to_2.1.14-15-1.sql b/upgrades/stager_2.1.14-15_to_2.1.14-15-1.sql
new file mode 100644
index 0000000000000000000000000000000000000000..16dfbb46f563c8437e07277de8c3deb6fb52513a
--- /dev/null
+++ b/upgrades/stager_2.1.14-15_to_2.1.14-15-1.sql
@@ -0,0 +1,321 @@
+/******************************************************************************
+ * stager_2.1.14-15_to_2.1.14-15-1.sql
+ *
+ * This file is part of the Castor project.
+ * See http://castor.web.cern.ch/castor
+ *
+ * Copyright (C) 2003 CERN
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * This script upgrades a CASTOR v2.1.14-15 STAGER database to v2.1.14-15-1
+ *
+ * @author Castor Dev team, castor-dev@cern.ch
+ *****************************************************************************/
+
+/* Stop on errors */
+WHENEVER SQLERROR EXIT FAILURE
+BEGIN
+ -- If we have encountered an error rollback any previously non committed
+ -- operations. This prevents the UPDATE of the UpgradeLog from committing
+ -- inconsistent data to the database.
+ ROLLBACK;
+ UPDATE UpgradeLog
+ SET failureCount = failureCount + 1
+ WHERE schemaVersion = '2_1_14_2'
+ AND release = '2_1_14_15_1'
+ AND state != 'COMPLETE';
+ COMMIT;
+END;
+/
+
+/* Verify that the script is running against the correct schema and version */
+DECLARE
+ unused VARCHAR(100);
+BEGIN
+ SELECT release INTO unused FROM CastorVersion
+ WHERE schemaName = 'STAGER'
+ AND release LIKE '2_1_14_15%';
+EXCEPTION WHEN NO_DATA_FOUND THEN
+ -- Error, we cannot apply this script
+ raise_application_error(-20000, 'PL/SQL release mismatch. Please run previous upgrade scripts for the STAGER before this one.');
+END;
+/
+
+INSERT INTO UpgradeLog (schemaVersion, release, type)
+VALUES ('2_1_14_2', '2_1_14_15_1', 'TRANSPARENT');
+COMMIT;
+
+/* Schema changes go here */
+/**************************/
+
+/* Update and revalidation of PL-SQL code */
+/******************************************/
+
+/* PL/SQL method creating MigrationJobs for missing segments of a file if needed */
+/* Can throw a -20100 exception when no route to tape is found for the missing segments */
+CREATE OR REPLACE PROCEDURE createMJForMissingSegments(inCfId IN INTEGER,
+ inFileSize IN INTEGER,
+ inFileClassId IN INTEGER,
+ inAllValidCopyNbs IN "numList",
+ inAllValidVIDs IN strListTable,
+ inNbExistingSegments IN INTEGER,
+ inFileId IN INTEGER,
+ inNsHost IN VARCHAR2,
+ inLogParams IN VARCHAR2) AS
+ varExpectedNbCopies INTEGER;
+ varCreatedMJs INTEGER := 0;
+ varNextCopyNb INTEGER := 1;
+ varNb INTEGER;
+BEGIN
+ -- check whether there are missing segments and whether we should create new ones
+ SELECT nbCopies INTO varExpectedNbCopies FROM FileClass WHERE id = inFileClassId;
+ IF varExpectedNbCopies > inNbExistingSegments THEN
+ -- some copies are missing
+ DECLARE
+ unused INTEGER;
+ BEGIN
+ -- check presence of migration jobs for this file
+ SELECT id INTO unused FROM MigrationJob WHERE castorFile=inCfId AND ROWNUM < 2;
+ -- there are MigrationJobs already, so remigrations were already handled. Nothing to be done
+ -- we typically are in a situation where the file was already waiting for recall for
+ -- another recall group.
+ -- log "detected missing copies on tape, but migrations ongoing"
+ logToDLF(NULL, dlf.LVL_DEBUG, dlf.RECALL_MISSING_COPIES_NOOP, inFileId, inNsHost, 'stagerd',
+ inLogParams || ' nbMissingCopies=' || TO_CHAR(varExpectedNbCopies-inNbExistingSegments));
+ RETURN;
+ EXCEPTION WHEN NO_DATA_FOUND THEN
+ -- we need to remigrate this file
+ NULL;
+ END;
+ -- log "detected missing copies on tape"
+ logToDLF(NULL, dlf.LVL_SYSTEM, dlf.RECALL_MISSING_COPIES, inFileId, inNsHost, 'stagerd',
+ inLogParams || ' nbMissingCopies=' || TO_CHAR(varExpectedNbCopies-inNbExistingSegments));
+ -- copies are missing, try to recreate them
+ WHILE varExpectedNbCopies > inNbExistingSegments + varCreatedMJs AND varNextCopyNb <= varExpectedNbCopies LOOP
+ BEGIN
+ -- check whether varNextCopyNb is already in use by a valid copy
+ SELECT * INTO varNb FROM TABLE(inAllValidCopyNbs) WHERE COLUMN_VALUE=varNextCopyNb;
+ -- this copy number is in use, go to next one
+ EXCEPTION WHEN NO_DATA_FOUND THEN
+ -- copy number is not in use, create a migrationJob using it (may throw exceptions)
+ initMigration(inCfId, inFileSize, NULL, NULL, varNextCopyNb, tconst.MIGRATIONJOB_WAITINGONRECALL);
+ varCreatedMJs := varCreatedMJs + 1;
+ -- log "create new MigrationJob to migrate missing copy"
+ logToDLF(NULL, dlf.LVL_SYSTEM, dlf.RECALL_MJ_FOR_MISSING_COPY, inFileId, inNsHost, 'stagerd',
+ inLogParams || ' copyNb=' || TO_CHAR(varNextCopyNb));
+ END;
+ varNextCopyNb := varNextCopyNb + 1;
+ END LOOP;
+ -- Did we create new copies ?
+ IF varExpectedNbCopies > inNbExistingSegments + varCreatedMJs THEN
+ -- We did not create enough new copies, this means that we did not find enough
+ -- valid copy numbers. Odd... Log something !
+ logToDLF(NULL, dlf.LVL_ERROR, dlf.RECALL_COPY_STILL_MISSING, inFileId, inNsHost, 'stagerd',
+ inLogParams || ' nbCopiesStillMissing=' ||
+ TO_CHAR(varExpectedNbCopies - inAllValidCopyNbs.COUNT - varCreatedMJs));
+ ELSE
+ -- Yes, then create migrated segments for the existing segments if there are none
+ SELECT count(*) INTO varNb FROM MigratedSegment WHERE castorFile = inCfId;
+ IF varNb = 0 THEN
+ FOR i IN inAllValidCopyNbs.FIRST .. inAllValidCopyNbs.LAST LOOP
+ INSERT INTO MigratedSegment (castorFile, copyNb, VID)
+ VALUES (inCfId, inAllValidCopyNbs(i), inAllValidVIDs(i));
+ END LOOP;
+ END IF;
+ END IF;
+ END IF;
+END;
+/
+
+/* PL/SQL method creating RecallJobs
+ * It also creates MigrationJobs for eventually missing segments
+ * It returns 0 if successful, else an error code
+ */
+CREATE OR REPLACE FUNCTION createRecallJobs(inCfId IN INTEGER,
+ inFileId IN INTEGER,
+ inNsHost IN VARCHAR2,
+ inFileSize IN INTEGER,
+ inFileClassId IN INTEGER,
+ inRecallGroupId IN INTEGER,
+ inSvcClassId IN INTEGER,
+ inEuid IN INTEGER,
+ inEgid IN INTEGER,
+ inRequestTime IN NUMBER,
+ inLogParams IN VARCHAR2) RETURN INTEGER AS
+ -- list of all valid segments, whatever the tape status. Used to trigger remigrations
+ varAllValidCopyNbs "numList" := "numList"();
+ varAllValidVIDs strListTable := strListTable();
+ varNbExistingSegments INTEGER := 0;
+ -- whether we found a segment at all (valid or not). Used to detect potentially lost files
+ varFoundSeg boolean := FALSE;
+ varI INTEGER := 1;
+ NO_TAPE_ROUTE EXCEPTION;
+ PRAGMA EXCEPTION_INIT(NO_TAPE_ROUTE, -20100);
+ varErrorMsg VARCHAR2(2048);
+BEGIN
+ BEGIN
+ -- loop over the existing segments
+ FOR varSeg IN (SELECT s_fileId as fileId, 0 as lastModTime, copyNo, segSize, 0 as comprSize,
+ Cns_seg_metadata.vid, fseq, blockId, checksum_name, nvl(checksum, 0) as checksum,
+ Cns_seg_metadata.s_status as segStatus, Vmgr_tape_status_view.status as tapeStatus
+ FROM Cns_seg_metadata@remotens, Vmgr_tape_status_view@remotens
+ WHERE Cns_seg_metadata.s_fileid = inFileId
+ AND Vmgr_tape_status_view.VID = Cns_seg_metadata.VID
+ ORDER BY copyno, fsec) LOOP
+ varFoundSeg := TRUE;
+ -- Is the segment valid
+ IF varSeg.segStatus = '-' THEN
+ -- Is the segment on a valid tape from recall point of view ?
+ IF BITAND(varSeg.tapeStatus, tconst.TAPE_DISABLED) = 0 AND
+ BITAND(varSeg.tapeStatus, tconst.TAPE_EXPORTED) = 0 AND
+ BITAND(varSeg.tapeStatus, tconst.TAPE_ARCHIVED) = 0 THEN
+ -- remember the copy number and tape
+ varAllValidCopyNbs.EXTEND;
+ varAllValidCopyNbs(varI) := varSeg.copyno;
+ varAllValidVIDs.EXTEND;
+ varAllValidVIDs(varI) := varSeg.vid;
+ varI := varI + 1;
+ -- create recallJob
+ INSERT INTO RecallJob (id, castorFile, copyNb, recallGroup, svcClass, euid, egid,
+ vid, fseq, status, fileSize, creationTime, blockId, fileTransactionId)
+ VALUES (ids_seq.nextval, inCfId, varSeg.copyno, inRecallGroupId, inSvcClassId,
+ inEuid, inEgid, varSeg.vid, varSeg.fseq, tconst.RECALLJOB_PENDING, inFileSize, inRequestTime,
+ varSeg.blockId, NULL);
+ varNbExistingSegments := varNbExistingSegments + 1;
+ -- log "created new RecallJob"
+ logToDLF(NULL, dlf.LVL_SYSTEM, dlf.RECALL_CREATING_RECALLJOB, inFileId, inNsHost, 'stagerd',
+ inLogParams || ' copyNb=' || TO_CHAR(varSeg.copyno) || ' TPVID=' || varSeg.vid ||
+ ' fseq=' || TO_CHAR(varSeg.fseq || ' FileSize=' || TO_CHAR(inFileSize)));
+ ELSE
+ -- Should the segment be counted in the count of existing segments ?
+ -- In other terms, should we recreate a segment for replacing this one ?
+ -- Yes if the segment in on an EXPORTED tape.
+ IF BITAND(varSeg.tapeStatus, tconst.TAPE_EXPORTED) = 0 THEN
+ -- invalid tape found with segments that are counting for the total count.
+ -- "createRecallCandidate: found segment on unusable tape"
+ logToDLF(NULL, dlf.LVL_SYSTEM, dlf.RECALL_UNUSABLE_TAPE, inFileId, inNsHost, 'stagerd',
+ inLogParams || ' segStatus=OK tapeStatus=' || tapeStatusToString(varSeg.tapeStatus) ||
+ ' recreatingSegment=No');
+ varNbExistingSegments := varNbExistingSegments + 1;
+ ELSE
+ -- invalid tape found with segments that will be completely ignored.
+ -- "createRecallCandidate: found segment on unusable tape"
+ logToDLF(NULL, dlf.LVL_DEBUG, dlf.RECALL_UNUSABLE_TAPE, inFileId, inNsHost, 'stagerd',
+ inLogParams || ' segStatus=OK tapeStatus=' || tapeStatusToString(varSeg.tapeStatus) ||
+ ' recreatingSegment=Yes');
+ END IF;
+ END IF;
+ ELSE
+ -- invalid segment tape found. Log it.
+ -- "createRecallCandidate: found unusable segment"
+ logToDLF(NULL, dlf.LVL_NOTICE, dlf.RECALL_INVALID_SEGMENT, inFileId, inNsHost, 'stagerd',
+ inLogParams || ' segStatus=' ||
+ CASE varSeg.segStatus WHEN '-' THEN 'OK'
+ WHEN 'D' THEN 'DISABLED'
+ ELSE 'UNKNOWN:' || varSeg.segStatus END);
+ END IF;
+ END LOOP;
+ EXCEPTION WHEN OTHERS THEN
+ -- log "error when retrieving segments from namespace"
+ logToDLF(NULL, dlf.LVL_ERROR, dlf.RECALL_UNKNOWN_NS_ERROR, inFileId, inNsHost, 'stagerd',
+ inLogParams || ' ErrorMessage=' || SQLERRM);
+ RETURN serrno.SEINTERNAL;
+ END;
+ -- If we did not find any valid segment to recall, log a critical error as the file is probably lost
+ IF NOT varFoundSeg THEN
+ -- log "createRecallCandidate: no segment found for this file. File is probably lost"
+ logToDLF(NULL, dlf.LVL_CRIT, dlf.RECALL_NO_SEG_FOUND_AT_ALL, inFileId, inNsHost, 'stagerd', inLogParams);
+ RETURN serrno.ESTNOSEGFOUND;
+ END IF;
+ -- If we found no valid segment (but some disabled ones), log a warning
+ IF varAllValidCopyNbs.COUNT = 0 THEN
+ -- log "createRecallCandidate: no valid segment to recall found"
+ logToDLF(NULL, dlf.LVL_WARNING, dlf.RECALL_NO_SEG_FOUND, inFileId, inNsHost, 'stagerd', inLogParams);
+ RETURN serrno.ESTNOSEGFOUND;
+ END IF;
+ BEGIN
+ -- create missing segments if needed
+ createMJForMissingSegments(inCfId, inFileSize, inFileClassId, varAllValidCopyNbs,
+ varAllValidVIDs, varNbExistingSegments, inFileId, inNsHost, inLogParams);
+ EXCEPTION WHEN NO_TAPE_ROUTE THEN
+ -- there's at least a missing segment and we cannot recreate it!
+ -- log a "no route to tape defined for missing copy" error, but don't fail the recall
+ logToDLF(NULL, dlf.LVL_ALERT, dlf.RECALL_MISSING_COPY_NO_ROUTE, inFileId, inNsHost, 'stagerd', inLogParams);
+ WHEN OTHERS THEN
+ -- some other error happened, log "unexpected error when creating missing copy", but don't fail the recall
+ varErrorMsg := 'Oracle error caught : ' || SQLERRM;
+ logToDLF(NULL, dlf.LVL_ERROR, dlf.RECALL_MISSING_COPY_ERROR, inFileId, inNsHost, 'stagerd',
+ 'errorCode=' || to_char(SQLCODE) ||' errorMessage="' || varErrorMsg
+ ||'" stackTrace="' || dbms_utility.format_error_backtrace ||'"');
+ END;
+ RETURN 0;
+END;
+/
+
+/* PL/SQL procedure implementing triggerRepackRecall
+ * this triggers a recall in the repack context
+ */
+CREATE OR REPLACE PROCEDURE triggerRepackRecall
+(inCfId IN INTEGER, inFileId IN INTEGER, inNsHost IN VARCHAR2, inBlock IN RAW,
+ inFseq IN INTEGER, inCopynb IN INTEGER, inEuid IN INTEGER, inEgid IN INTEGER,
+ inRecallGroupId IN INTEGER, inSvcClassId IN INTEGER, inVid IN VARCHAR2, inFileSize IN INTEGER,
+ inFileClass IN INTEGER, inAllValidSegments IN VARCHAR2, inReqUUID IN VARCHAR2,
+ inSubReqUUID IN VARCHAR2, inRecallGroupName IN VARCHAR2) AS
+ varLogParam VARCHAR2(2048);
+ varAllValidCopyNbs "numList" := "numList"();
+ varAllValidVIDs strListTable := strListTable();
+ varAllValidSegments strListTable;
+ varFileClassId INTEGER;
+BEGIN
+ -- create recallJob for the given VID, copyNb, etc.
+ INSERT INTO RecallJob (id, castorFile, copyNb, recallGroup, svcClass, euid, egid,
+ vid, fseq, status, fileSize, creationTime, blockId, fileTransactionId)
+ VALUES (ids_seq.nextval, inCfId, inCopynb, inRecallGroupId, inSvcClassId,
+ inEuid, inEgid, inVid, inFseq, tconst.RECALLJOB_PENDING, inFileSize, getTime(),
+ inBlock, NULL);
+ -- log "created new RecallJob"
+ varLogParam := 'SUBREQID=' || inSubReqUUID || ' RecallGroup=' || inRecallGroupName;
+ logToDLF(inReqUUID, dlf.LVL_SYSTEM, dlf.RECALL_CREATING_RECALLJOB, inFileId, inNsHost, 'stagerd',
+ varLogParam || ' fileClass=' || TO_CHAR(inFileClass) || ' copyNb=' || TO_CHAR(inCopynb)
+ || ' TPVID=' || inVid || ' fseq=' || TO_CHAR(inFseq) || ' FileSize=' || TO_CHAR(inFileSize));
+ -- create missing segments if needed
+ SELECT * BULK COLLECT INTO varAllValidSegments
+ FROM TABLE(strTokenizer(inAllValidSegments));
+ FOR i IN 1 .. varAllValidSegments.COUNT/2 LOOP
+ varAllValidCopyNbs.EXTEND;
+ varAllValidCopyNbs(i) := TO_NUMBER(varAllValidSegments(2*i-1));
+ varAllValidVIDs.EXTEND;
+ varAllValidVIDs(i) := varAllValidSegments(2*i);
+ END LOOP;
+ SELECT id INTO varFileClassId
+ FROM FileClass WHERE classId = inFileClass;
+ -- Note that the number given here for the number of existing segments is the total number of
+ -- segment, without removing the ones on EXPORTED tapes. This means that repack will not
+ -- recreate new segments for files that have some copies on EXPORTED tapes.
+ -- This is different from standard recalls that would recreate segments on EXPORTED tapes.
+ createMJForMissingSegments(inCfId, inFileSize, varFileClassId, varAllValidCopyNbs,
+ varAllValidVIDs, varAllValidCopyNbs.COUNT, inFileId, inNsHost, varLogParam);
+END;
+/
+
+/* revalidate all code */
+BEGIN
+ recompileAll();
+END;
+/
+
+/* Flag the schema upgrade as COMPLETE */
+/***************************************/
+UPDATE UpgradeLog SET endDate = systimestamp, state = 'COMPLETE'
+ WHERE release = '2_1_14_15_1';
+COMMIT;