cta.proto

// The CERN Tape Archive (CTA) project
// Copyright (C) 2015  CERN
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.
syntax = "proto2";
package cta.objectstore.serializers;

// The types of the objects. It will be used to allow introspection
// for the contents.
enum ObjectType {
  RootEntry_t = 0;
  AgentRegister_t = 1;
  Agent_t = 2;
  DriveRegister_t = 3;
  DriveState_t = 4;
  SchedulerGlobalLock_t = 6;
  ArchiveRequest_t = 7;
  RetrieveRequest_t = 8;
  ArchiveQueue_t = 9;
  ArchiveQueueShard_t = 90;
  RetrieveQueue_t = 10;
  RetrieveQueueShard_t = 100;
  RepackRequest_t = 11;
  RepackIndex_t = 12;
  RepackQueue_t = 13;
  GenericObject_t = 1000;
}

// The base object header. This will allow introspection and automatic
// "garbage collection", i.e. returning an unprocessed object belonging
// to a dead agent to the right queue or container.
// - The type allows automatic management of the content without knowing its
// payload.
// - The version allows multi schema to be used at the object level (smooth schema 
// evolution, on an object by object basis).
// - The owner is the authoritative source of ownership for the object.
// It allows arbitration of the actual owner in case the object is pointed
// to by several containers (during a transition, or after a failure).
// - The backup owner allows the object to be returned to a previous container
// in case of failure of a owner (when it is an agent).
message ObjectHeader {
  required ObjectType type = 1;
  required uint64 version = 2;
  required string owner = 3;
  required string backupowner = 4;
  required bytes payload = 5;
}

// A placeholder object for the implementation of neutral object handlers
message GenericObject {
}

// ===========================  Root Entry =====================================
// The objects making up the root entry.

// A user information record
message UserIdentity {
  required string name = 10;
  required string group = 11;
}

// Pointer to the drive register (register defined further)
message DriveRegisterPointer {
  required string address = 90;
  required EntryLog log = 91;
}

// Pointer to the agent register (register defined further)
message AgentRegisterPointer {
  required string address = 100;
  required EntryLog log = 101;
}

message RepackIndexPointer {
  required string address = 105;
}

message RepackQueuePointer {
  required string address = 107;
}


// Pointer to the scheduler global lock
message SchedulerGlobalLockPointer {
  required string address = 110;
  required EntryLog log = 111;
}

// Pointer to the archive queue
message ArchiveQueuePointer {
  required string address = 120;
  required string name = 121;
}

// Pointer to the tape queue
message RetrieveQueuePointer {
  required string address = 130;
  required string vid = 131;
}

// The root entry. This entry contains all the most static information, i.e.
// the admin handled configuration information
message RootEntry {
  repeated ArchiveQueuePointer archive_queue_to_transfer_for_user_pointers = 1050;
  repeated ArchiveQueuePointer archive_queue_failed_pointers = 1062;
  repeated ArchiveQueuePointer archive_queue_to_report_for_user_pointers = 1068;
  repeated ArchiveQueuePointer archive_queue_to_transfer_for_repack_pointers = 1069;
  repeated ArchiveQueuePointer archive_queue_to_report_to_repack_for_success_pointers = 1072;
  repeated ArchiveQueuePointer archive_queue_to_report_to_repack_for_failure_pointers = 1073;
  repeated RetrieveQueuePointer retrieve_queue_to_transfer_for_user_pointers = 1060;
  repeated RetrieveQueuePointer retrieve_queue_to_report_for_user_pointers = 1063;
  repeated RetrieveQueuePointer retrieve_queue_failed_pointers = 1065;
  repeated RetrieveQueuePointer retrieve_queue_to_report_to_repack_for_success_pointers = 1066;
  repeated RetrieveQueuePointer retrieve_queue_to_report_to_repack_for_failure_pointers = 1067;
  repeated RetrieveQueuePointer retrieve_queue_to_transfer_for_repack_pointers = 1071;
  optional DriveRegisterPointer driveregisterpointer = 1070;
  optional AgentRegisterPointer agentregisterpointer = 1080;
  optional RepackIndexPointer repackindexpointer = 1085;
  optional RepackQueuePointer repackrequestspendingqueuepointer = 1086;
  optional RepackQueuePointer repackrequeststoexpandqueuepointer = 1088;
  optional string agentregisterintent = 1090;
  optional SchedulerGlobalLockPointer schedulerlockpointer = 1100;
}

//=========== Sub-objects ======================================================

// ------------- Agent handling ------------------------------------------------

// The agent object represents a running process. It is a payload to an object
// itself, and it can be owned by a watchdog or a global agent registry, which
// lists the agents not yet watched by a watchdog.
// - The description is a freetext string used for logging.
// - The watchdogs themselves own the agents they are watching, so the agents can
// be returned to the agent registry when they fail.
// - The heartbeat is incremented from time to time by the agent.
// - The timeout_us is the amount of microseconds after which the absence of
// change to the heartbeat can be interpreted as agent failure.
// - The ownedobject list is an ownership intent log that points to the objects
// that the agent is about to create, intends to own, or fully owns.
// The objects in this list can be considered for being returned to a backup
// owner.
message Agent {
  required string description = 2000;
  required uint64 heartbeat = 2001;
  required uint64 timeout_us = 2002;
  repeated string ownedobjects = 2003;
  optional bool being_garbage_collected = 2004 [default = false];
}

message AgentRegister {
  repeated string agents = 2100;
  repeated string untrackedagents = 2101;
}

// ------------- Mount criteria and quota  -------------------------------------

message MountCriteria {
  required uint64 maxFilesBeforeMount = 3200;
  required uint64 maxBytesBeforeMount = 3201;
  required uint64 maxSecondsBeforeMount = 3202;
  required uint32 quota = 3203;
}

message TapeFile {
  required string vid = 9120;
  required uint64 fseq = 9121;
  required uint64 blockid = 9122;
  required uint64 compressedsize = 9123;
  required uint32 copynb = 9124;
  required uint64 creationtime = 9125;
  required string checksumtype = 9126;
  required string checksumvalue = 9127;
}

message DiskFileInfo {
  required bytes recoveryblob = 8900;
  required string group = 8910;
  required string owner = 8930;
  required string path = 8940;
}

message ArchiveFile {
  required uint64 archivefileid = 4351;
  required uint64 filesize = 4352;
  required string diskfileid = 4353;
  required string diskinstance= 4354;
  required DiskFileInfo diskfileinfo= 4355;
  required string checksumtype = 4356;
  required string checksumvalue = 4357;
  required uint64 creationtime = 4358;
  repeated TapeFile tapefiles = 4359;
  required uint64 reconciliationtime = 4360;
  required string storageclass = 4361;
}

// ------------- Drives handling  ----------------------------------------------

message DriveState {
  required string drivename = 5000;
  required string host = 5001;
  required string logicallibrary = 5002;
  optional uint64 sessionid = 5003;
  optional uint64 bytestransferedinsession = 5004;
  optional uint64 filestransferedinsession = 5005;
  optional double latestbandwidth = 5006;
  optional uint64 sessionstarttime = 5007;
  optional uint64 mountstarttime = 5008;
  optional uint64 transferstarttime = 5009;
  optional uint64 unloadstarttime = 5010;
  optional uint64 unmountstarttime = 5011;
  optional uint64 drainingstarttime = 5012;
  optional uint64 downorupstarttime = 5013;
  optional uint64 probestarttime = 5026;
  optional uint64 cleanupstarttime = 5014;
  optional uint64 lastupdatetime = 5015;
  optional uint64 startstarttime = 5016;
  optional uint64 shutdowntime = 5027;
  required uint32 mounttype = 5017;
  required uint32 drivestatus = 5018;
  required bool desiredUp = 5019;
  required bool desiredForceDown = 5020;
  optional string currentvid = 5021;
  optional string currenttapepool = 5022;
  optional uint32 nextmounttype = 5023;
  optional string nextvid = 5024;
  optional string nexttapepool = 5025;
// TODO: implement or remove  required EntryLog creationlog = 5023;
}

message DriveStatePointer {
  required string drivename = 7010;
  required string drivestateaddress = 7011;
}

message DriveRegister {
  repeated DriveStatePointer drives = 7001;
}

// ------------- Scheduler global lock handling  -------------------------------

message SchedulerGlobalLock {
  required uint64 nextmountid = 8000;
}

message EntryLog {
  required string username = 8950;
  required string host = 8960;
  required uint64 time = 8970;
}

message MountPolicy {
  required string name = 8980;
  required uint64 archivepriority = 8981;
  required uint64 archiveminrequestage = 8982;
  required uint64 retrievepriority = 8983;
  required uint64 retieveminrequestage = 8984;
  required uint64 maxdrivesallowed = 8985;
  required EntryLog creationlog = 8986;
  required EntryLog lastmodificationlog = 8987;
  required string comment = 8988;
}

// ------------- Archive Jobs --------------------------------------------------

// The status of the individual archive jobs. The jobs are materialised 
// by table entries in the ArchiveRequest.
// This life cycle represented by the following enum

enum ArchiveJobStatus {
  AJS_ToTransferForUser = 1;
  AJS_ToReportToUserForTransfer = 2;
  AJS_Complete = 3;
  AJS_ToReportToUserForFailure = 997;
  AJS_Failed = 998;
  AJS_Abandoned = 999;
  AJS_ToTransferForRepack = 1001;
  AJS_ToReportToRepackForSuccess = 1005;
  AJS_ToReportToRepackForFailure = 1006;
}

message ArchiveJob {
  required uint32 copynb = 4400;
  required string tapepool = 4401;
  required string archivequeueaddress = 4402;
  required string owner = 4403;
  required ArchiveJobStatus status = 4404;
  required uint32 totalretries = 4405;
  required uint32 retrieswithinmount = 4406;
  required uint64 lastmountwithfailure = 4407;
  required uint32 maxtotalretries = 4408;
  required uint32 maxretrieswithinmount = 4409;
  repeated string failurelogs = 4410;
  required uint32 maxreportretries = 4411;
  required uint32 totalreportretries = 4412;
  repeated string reportfailurelogs = 4413;
}

message ArchiveRequestRepackInfo {
  required string repack_request_address = 4450;
  required string file_buffer_url = 4453;
  required uint64 fseq = 4455;
}

message ArchiveRequest {
  required uint64 archivefileid = 8990;
  required MountPolicy mountpolicy = 8995;
  required string checksumtype = 9000;
  required string checksumvalue = 9010;
  required uint64 creationtime = 9015;
  required uint64 reconcilationtime = 9017;
  required DiskFileInfo diskfileinfo = 9040;
  required string diskfileid = 9050;
  required string diskinstance = 9055;
  required string archivereporturl = 9057;
  required string archiveerrorreporturl = 9058;
  required uint64 filesize = 9060;
  required UserIdentity requester = 9070;
  required string srcurl = 9080;
  required string storageclass = 9090;
  required EntryLog creationlog = 9091;
  repeated ArchiveJob jobs = 9092;
  required bool reportdecided = 9093;
  required bool isrepack = 9095;
  optional ArchiveRequestRepackInfo repack_info = 9097;
}

// ------------- Retrieve Jobs -------------------------------------------------

// The status of the individual retrieve jobs. The jobs are materialised 
// by table entries in the RetrieveRequest.
// This life cycle represented by the following enum
// There is no complete state as the completion of one jobs implies the 
// completion of the whole requests, and leads to the immediate deletion
// of the request.

enum RetrieveJobStatus {
  RJS_ToTransferForUser = 1;
  RJS_ToReportToUserForFailure = 997;
  RJS_Failed = 998;
  RJS_ToReportToRepackForSuccess = 1002; //For Retrieve request created by a Repack request
  RJS_ToReportToRepackForFailure = 1003;
  RJS_ToTransferForRepack = 1004;
}

message SchedulerRetrieveRequest {
  required UserIdentity requester = 9100;
  required uint64 ArchiveFileId = 9101;
  required string dstURL = 9102;
  required DiskFileInfo diskfileinfo = 9103;
  required EntryLog entrylog = 9106;
  required string retrieveerrorreporturl = 9110;
}

message RetrieveJob {
  required uint32 copynb = 9200;
  required uint32 maxtotalretries = 9201;
  required uint32 maxretrieswithinmount = 9202;
  required uint32 retrieswithinmount = 9203;
  required uint32 totalretries = 9204;
  required RetrieveJobStatus status = 9205;
  required uint64 lastmountwithfailure = 9206;
  repeated string failurelogs = 9207;
  required uint32 maxreportretries = 9208;
  required uint32 totalreportretries = 9209;
  repeated string reportfailurelogs = 9210;
}

message RetrieveRequestArchiveRoute {
  required uint32 copynb = 9400;
  required string tapepool = 9410;
}

message RetrieveRequestRepackInfo {
  repeated RetrieveRequestArchiveRoute archive_routes = 9500;
  repeated uint32 copy_nbs_to_rearchive = 9510;
  required string repack_request_address = 9520;
  required string file_buffer_url = 9530;
  required uint64 fseq = 9540;
}

// The different timings of the lifecycle of a RetrieveRequest (creation time, first select time, request complete)
message LifecycleTimings{
  optional uint64 creation_time = 9160 [default = 0];
  optional uint64 first_selected_time = 9161 [default = 0];
  optional uint64 completed_time = 9162 [default = 0];
}

message RetrieveRequest {
  required SchedulerRetrieveRequest schedulerrequest = 9150;
  required MountPolicy mountpolicy = 9151;
  required ArchiveFile archivefile = 9152;
  required uint32 activecopynb = 9153;
  repeated RetrieveJob jobs = 9154;
  required string failurereporturl = 9155;
  required string failurereportlog = 9156;
  required bool isrepack = 9157;
  optional RetrieveRequestRepackInfo repack_info = 9158;
  optional LifecycleTimings lifecycle_timings = 9159;
}

message ValueCountPair {
  required uint64 value = 9301;
  required uint64 count = 9302;
}

message ArchiveJobPointer {
  required uint64 fileid = 3000;
  required uint64 size = 3001;
  required string address = 3002;
  required uint32 copynb = 3003;
  required uint64 priority = 3004;
  required uint64 minarchiverequestage = 3005;
  required uint64 maxdrivesallowed = 3006;
  required uint64 starttime = 3007;
}

message ArchiveQueueShardPointer {
  required string address = 10200;
  required uint64 shardjobscount = 10201;
  required uint64 shardbytescount = 10202;
}

message ArchiveQueueShard {
  repeated ArchiveJobPointer archivejobs = 10300;
  required uint64 archivejobstotalsize = 10301;
}

message ArchiveQueue {
  required string tapepool = 10000;
  repeated ArchiveQueueShardPointer archivequeueshards = 10010;
  repeated ValueCountPair prioritymap = 10031;
  repeated ValueCountPair minarchiverequestagemap = 10032;
  repeated ValueCountPair maxdrivesallowedmap = 10033;
  required uint64 archivejobstotalsize = 10040;
  required uint64 archivejobscount = 10045;
  required uint64 oldestjobcreationtime = 10050;
  required uint64 mapsrebuildcount = 10060;
}

message RetrieveJobPointer {
  required uint64 size = 3101;
  required string address = 3102;
  required uint32 copynb = 3103;
  required uint64 fseq = 3107;
  required uint64 priority = 3104;
  required uint64 minretrieverequestage = 3105;
  required uint64 maxdrivesallowed = 3106;
  required uint64 starttime = 3108;
}

message RetrieveQueueShardPointer {
  required string address = 10400;
  required uint64 shardjobscount = 10401;
  required uint64 shardbytescount = 10402;
  required uint64 minfseq = 10403;
  required uint64 maxfseq = 10404;
}

message RetrieveQueueShard {
  repeated RetrieveJobPointer retrievejobs = 10500;
  required uint64 retrievejobstotalsize = 10501;
}

message RetrieveQueue {
  required string vid = 10100;
  repeated RetrieveQueueShardPointer retrievequeueshards = 10111;
  repeated ValueCountPair prioritymap = 10131;
  repeated ValueCountPair minretrieverequestagemap = 10132;
  repeated ValueCountPair maxdrivesallowedmap = 10133;
  required uint64 retrievejobstotalsize = 10140;
  required uint64 retrievejobscount = 10145;
  required uint64 oldestjobcreationtime = 10150;
  required uint64 mapsrebuildcount = 10160;
  required uint64 maxshardsize = 10170;
}

// ------------- Repack data strcutures ----------------------------------------

enum RepackRequestStatus {
  // those value are matching the dataStructures/RepackInfo.hpp: RepackInfo::Status values.
  RRS_Pending = 1;
  RRS_ToExpand = 2;
  RRS_Starting = 3;
  RRS_Running = 4;
  RRS_Complete = 5;
  RRS_Failed = 6;
}

// In order to properly handle retries in case of failure during reporting, we hold
// a series of booleans tracking the status of the statistics recording and deletion of the
// sub requests. Subrequest deleted records whether a sub request is expected to be present or not.
// The process deleting the sub request will
// first mark it as delete with this status and then actually delete it. In case of crash in this
// process, the switch of this status will be re-done and the sub request deleted on retry.
// On the opposite side, this information will allow the creation process to know if a reference to the
// sub request should be interpreted as an unfulfilled creation intent (deleted=false) and create the
// missing sub request or the completion of the request (which can happen anytime after sub request
// creation).
// Likewise, the "accounted" booleans or copyNbs will prevent double counting in case a report (for success or failure)
// need to be retried after a process failure. The same flag is used for both success and failure. Archive requires a
// set of copyNbs as a single repack can lead to several archives (in case we create new copies).
message RepackSubRequestPointer {
  required string address = 10500;
  required uint64 fseq = 10510;
  required bool retrieve_accounted = 10530;
  repeated uint32 archive_copynb_accounted = 10534;
  required bool subrequest_deleted = 10540;
}

message RepackRequest {
  required string vid = 11000;
  required string buffer_url = 11005;
  required RepackRequestStatus status = 11010;
  required bool add_copies_mode = 11400;
  required bool move_mode = 11410;
  required uint64 totalfilestoretrieve = 11420;
  required uint64 totalbytestoretrieve = 11430;
  // We can have more files to archive than we have to retrieve if we plan to also archive a missing
  // copy of a multi copy file.
  required uint64 totalfilestoarchive = 11440;
  required uint64 totalbytestoarchive = 11450;
  required uint64 userprovidedfiles = 11460;
  required uint64 userprovidedbytes = 11470;
  required uint64 retrievedfiles = 11480;
  required uint64 retrievedbytes = 11490;
  required uint64 archivedfiles = 11500;
  required uint64 archivedbytes = 11510;
  required uint64 failedtoretrievefiles = 11520;
  required uint64 failedtoretrievebytes = 11530;
  required uint64 failedtoarchivefiles = 11540;
  required uint64 failedtoarchivebytes = 11550;
  required uint64 lastexpandedfseq = 11560;
  repeated RepackSubRequestPointer subrequests = 11570;
}

message RepackRequestIndexPointer {
  required string vid = 12000;
  required string address = 12010;
}

message RepackIndex {
  repeated RepackRequestIndexPointer repackrequestpointers = 12100;
}

message RepackRequestQueuePointer {
  required string address = 12101;
}

message RepackQueue {
  repeated RepackRequestQueuePointer repackrequestpointers = 12200;
}