Added lifecycle timings for a RetrieveRequest

common/CMakeLists.txt

@@ -44,6 +44,7 @@ set (COMMON_LIB_SRC_FILES
   dataStructures/DriveStatus.cpp
   dataStructures/EntryLog.cpp
   dataStructures/FrontendReturnCode.cpp
+  dataStructures/LifecycleTimings.cpp
   dataStructures/ListStorageClassRequest.cpp
   dataStructures/LogicalLibrary.cpp
   dataStructures/MountType.cpp

common/dataStructures/LifecycleTimings.cpp

+/**
+ * The CERN Tape Archive (CTA) project
+ * Copyright © 2018 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "LifecycleTimings.hpp"
+namespace cta{
+namespace common{
+namespace dataStructures{
+
+LifecycleTimings::LifecycleTimings() : creation_time(0), first_selected_time(0), completed_time(0) {}
+
+LifecycleTimings::LifecycleTimings(const LifecycleTimings& orig) {
+  creation_time = orig.creation_time;
+  first_selected_time = orig.first_selected_time;
+  completed_time = orig.completed_time;
+}
+
+LifecycleTimings::~LifecycleTimings() {
+}
+
+time_t LifecycleTimings::getTimeForSelection(){
+  if(first_selected_time != 0 && creation_time != 0){
+    return first_selected_time - creation_time;
+  }
+  return 0;
+}
+
+time_t LifecycleTimings::getTimeForCompletion(){
+  if(completed_time != 0 && creation_time != 0){
+    return completed_time - creation_time;
+  }
+  return 0;
+}
+
+}}}
\ No newline at end of file

common/dataStructures/LifecycleTimings.hpp

+/**
+ * The CERN Tape Archive (CTA) project
+ * Copyright © 2018 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <ctime>
+
+namespace cta{
+namespace common{
+namespace dataStructures{
+    
+/**
+ * This class stores the informations about a request's lifecycle timings
+ */
+class LifecycleTimings {
+public:
+  LifecycleTimings();
+  LifecycleTimings(const LifecycleTimings& orig);
+  virtual ~LifecycleTimings();
+  /**
+   * Returns the elapsed time between the creation of the request
+   * and the selection of the request for mounting
+   */
+  time_t getTimeForSelection();
+  /**
+   * Returns the elapsed time between the creation of the request and
+   * its completion
+   */
+  time_t getTimeForCompletion();
+  time_t creation_time;
+  time_t first_selected_time;
+  time_t completed_time;
+};
+
+}}}

common/dataStructures/RetrieveRequest.hpp

@@ -27,6 +27,7 @@
 #include "common/dataStructures/EntryLog.hpp"
 #include "common/dataStructures/UserIdentity.hpp"
 #include "common/dataStructures/ArchiveRoute.hpp"
+#include "LifecycleTimings.hpp"
 
 namespace cta {
 namespace common {
@@ -49,6 +50,7 @@ struct RetrieveRequest {
   std::string errorReportURL;
   DiskFileInfo diskFileInfo;
   EntryLog creationLog;
+  LifecycleTimings lifecycleTimings;
 }; // struct RetrieveRequest
 
 std::ostream &operator<<(std::ostream &os, const RetrieveRequest &obj);

objectstore/LifecycleTimingsSerDeser.hpp

+/**
+ * The CERN Tape Archive (CTA) project
+ * Copyright © 2018 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+namespace cta{
+namespace objectstore{
+  class LifecycleTimingsSerDeser: public cta::common::dataStructures::LifecycleTimings{
+    public:
+      LifecycleTimingsSerDeser() : cta::common::dataStructures::LifecycleTimings() {}
+      LifecycleTimingsSerDeser(const cta::common::dataStructures::LifecycleTimings& lifecycleTimings) : cta::common::dataStructures::LifecycleTimings(lifecycleTimings) {}
+      operator cta::common::dataStructures::LifecycleTimings() {
+	return cta::common::dataStructures::LifecycleTimings(*this);
+      } 
+
+      void deserialize(const cta::objectstore::serializers::LifecycleTimings& ostoreLifecycleTimings){
+	completed_time = ostoreLifecycleTimings.completed_time();
+	creation_time = ostoreLifecycleTimings.creation_time();
+	first_selected_time = ostoreLifecycleTimings.first_selected_time();
+      }
+      
+      void serialize(cta::objectstore::serializers::LifecycleTimings &lifecycleTimings){
+	lifecycleTimings.set_completed_time(completed_time);
+	lifecycleTimings.set_creation_time(creation_time);
+	lifecycleTimings.set_first_selected_time(first_selected_time);
+      }
+  };
+}}

objectstore/RetrieveRequest.cpp

@@ -26,6 +26,7 @@
 #include "objectstore/cta.pb.h"
 #include "Helpers.hpp"
 #include "common/utils/utils.hpp"
+#include "LifecycleTimingsSerDeser.hpp"
 #include <google/protobuf/util/json_util.h>
 #include <cmath>
 
@@ -724,6 +725,22 @@ serializers::RetrieveJobStatus RetrieveRequest::getJobStatus(uint32_t copyNumber
   throw exception::Exception(err.str());
 }
 
+void RetrieveRequest::updateLifecycleTiming(serializers::RetrieveRequest& payload, const cta::objectstore::serializers::RetrieveJob& retrieveJob){
+  typedef ::cta::objectstore::serializers::RetrieveJobStatus RetrieveJobStatus;
+  LifecycleTimingsSerDeser lifeCycleSerDeser;
+  lifeCycleSerDeser.deserialize(payload.lifecycle_timings());
+  switch(retrieveJob.status() == RetrieveJobStatus::RJS_ToTransferForUser){
+    case RetrieveJobStatus::RJS_ToTransferForUser:
+      if(retrieveJob.totalretries() == 0){
+        //totalretries = 0 then this is the first selection of the request
+        lifeCycleSerDeser.first_selected_time = time(nullptr);
+      }
+      break;
+    default:
+      break;
+  }
+  lifeCycleSerDeser.serialize(*payload.mutable_lifecycle_timings());
+}
 //------------------------------------------------------------------------------
 // RetrieveRequest::asyncUpdateJobOwner()
 //------------------------------------------------------------------------------
@@ -783,6 +800,10 @@ auto RetrieveRequest::asyncUpdateJobOwner(uint32_t copyNumber, const std::string
             af.deserialize(payload.archivefile());
             retRef.m_archiveFile = af;
             retRef.m_jobStatus = j.status();
+            RetrieveRequest::updateLifecycleTiming(payload,j);
+            LifecycleTimingsSerDeser lifeCycleSerDeser;
+            lifeCycleSerDeser.deserialize(payload.lifecycle_timings());
+            retRef.m_retrieveRequest.lifecycleTimings = lifeCycleSerDeser;
             if (payload.isrepack()) {
               RetrieveRequest::RepackInfo & ri = retRef.m_repackInfo;
               for (auto &ar: payload.repack_info().archive_routes()) {
@@ -836,6 +857,28 @@ const common::dataStructures::RetrieveRequest& RetrieveRequest::AsyncJobOwnerUpd
   return m_retrieveRequest;
 }
 
+cta::common::dataStructures::LifecycleTimings RetrieveRequest::getLifecycleTimings(){
+  checkPayloadReadable();
+  LifecycleTimingsSerDeser serDeser;
+  serDeser.deserialize(m_payload.lifecycle_timings());
+  return serDeser;
+}
+
+void RetrieveRequest::setCreationTime(const uint64_t creationTime){
+  checkPayloadWritable();
+  m_payload.mutable_lifecycle_timings()->set_creation_time(creationTime);
+}
+
+void RetrieveRequest::setFirstSelectedTime(const uint64_t firstSelectedTime){
+  checkPayloadWritable();
+  m_payload.mutable_lifecycle_timings()->set_first_selected_time(firstSelectedTime);
+}
+
+void RetrieveRequest::setCompletedTime(const uint64_t completedTime){
+  checkPayloadWritable();
+  m_payload.mutable_lifecycle_timings()->set_completed_time(completedTime);
+}
+
 //------------------------------------------------------------------------------
 // RetrieveRequest::setActiveCopyNumber()
 //------------------------------------------------------------------------------

objectstore/RetrieveRequest.hpp

@@ -30,6 +30,7 @@
 #include "common/dataStructures/ArchiveFile.hpp"
 #include "common/dataStructures/RetrieveRequest.hpp"
 #include "common/dataStructures/RetrieveFileQueueCriteria.hpp"
+#include "common/dataStructures/LifecycleTimings.hpp"
 #include "AgentReference.hpp"
 
 namespace cta { 
@@ -51,6 +52,7 @@ public:
   void addJob(uint32_t copyNumber, uint16_t maxRetriesWithinMount, uint16_t maxTotalRetries, uint16_t maxReportRetries);
   std::string getLastActiveVid();
   void setFailureReason(const std::string & reason);
+  static void updateLifecycleTiming(serializers::RetrieveRequest& payload, const cta::objectstore::serializers::RetrieveJob & retrieveJob);
   class JobDump {
   public:
     uint32_t copyNb;
@@ -235,7 +237,11 @@ public:
   cta::common::dataStructures::RetrieveFileQueueCriteria getRetrieveFileQueueCriteria();
   cta::common::dataStructures::ArchiveFile getArchiveFile();
   cta::common::dataStructures::EntryLog getEntryLog();
-  
+  cta::common::dataStructures::LifecycleTimings getLifecycleTimings();
+  void setCreationTime(const uint64_t creationTime);
+  void setFirstSelectedTime(const uint64_t firstSelectedTime);
+  void setCompletedTime(const uint64_t completedTime);
+  void setReportedTime(const uint64_t reportedTime);
   void setActiveCopyNumber(uint32_t activeCopyNb);
   uint32_t getActiveCopyNumber();
   // ===========================================================================

objectstore/cta.proto

@@ -388,6 +388,13 @@ message RetrieveRequestRepackInfo {
   required uint64 fseq = 9540;
 }
 
+// The different timings of the lifecycle of a RetrieveRequest (creation time, first select time, request complete)
+message LifecycleTimings{
+  optional uint64 creation_time = 9160 [default = 0];
+  optional uint64 first_selected_time = 9161 [default = 0];
+  optional uint64 completed_time = 9162 [default = 0];
+}
+
 message RetrieveRequest {
   required SchedulerRetrieveRequest schedulerrequest = 9150;
   required MountPolicy mountpolicy = 9151;
@@ -398,6 +405,7 @@ message RetrieveRequest {
   required string failurereportlog = 9156;
   required bool isrepack = 9157;
   optional RetrieveRequestRepackInfo repack_info = 9158;
+  optional LifecycleTimings lifecycle_timings = 9159;
 }
 
 message ValueCountPair {

scheduler/OStoreDB/OStoreDB.cpp

@@ -1115,6 +1115,7 @@ std::string OStoreDB::queueRetrieve(const cta::common::dataStructures::RetrieveR
   rReq->initialize();
   rReq->setSchedulerRequest(rqst);
   rReq->setRetrieveFileQueueCriteria(criteria);
+  rReq->setCreationTime(rqst.creationLog.time);
   // Find the job corresponding to the vid (and check we indeed have one).
   auto jobs = rReq->getJobs();
   objectstore::RetrieveRequest::JobDump job;
@@ -3459,7 +3460,19 @@ void OStoreDB::RetrieveMount::flushAsyncSuccessReports(std::list<cta::SchedulerD
     } else {
       try {
         osdbJob->m_jobDelete->wait();
-        rjToUnown.push_back(osdbJob->m_retrieveRequest.getAddressIfSet());
+        osdbJob->retrieveRequest.lifecycleTimings.completed_time = time(nullptr);
+        std::string requestAddress = osdbJob->m_retrieveRequest.getAddressIfSet();
+        
+        rjToUnown.push_back(requestAddress);
+        
+        cta::common::dataStructures::LifecycleTimings requestTimings = osdbJob->retrieveRequest.lifecycleTimings;
+        log::ScopedParamContainer params(lc);
+        params.add("requestAddress",requestAddress)
+              .add("archiveFileID",osdbJob->archiveFile.archiveFileID)
+              .add("vid",osdbJob->m_retrieveMount->mountInfo.vid)
+              .add("timeForSelection",requestTimings.getTimeForSelection())
+              .add("timeForCompletion", requestTimings.getTimeForCompletion());
+        lc.log(log::INFO, "Retrieve job successfully deleted");
       } catch (cta::exception::Exception & ex) {
         log::ScopedParamContainer params(lc);
         params.add("fileId", osdbJob->archiveFile.archiveFileID)