DiskReadThreadPool.hpp 7.73 KB
Newer Older
1
2
3
4
5
6
7
/*
 * @project        The CERN Tape Archive (CTA)
 * @copyright      Copyright(C) 2021 CERN
 * @license        This program is free software: you can redistribute it and/or modify
 *                 it under the terms of the GNU General Public License as published by
 *                 the Free Software Foundation, either version 3 of the License, or
 *                 (at your option) any later version.
8
 *
9
10
11
12
 *                 This program is distributed in the hope that it will be useful,
 *                 but WITHOUT ANY WARRANTY; without even the implied warranty of
 *                 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *                 GNU General Public License for more details.
13
 *
14
15
16
 *                 You should have received a copy of the GNU General Public License
 *                 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
17
18
19
20

#pragma once

#include "castor/tape/tapeserver/daemon/DiskReadTask.hpp"
21
#include "castor/tape/tapeserver/daemon/TaskWatchDog.hpp"
22
#include "disk/RadosStriperPool.hpp"
23
#include "common/threading/BlockingQueue.hpp"
24
#include "common/threading/Thread.hpp"
25
#include "common/threading/AtomicCounter.hpp"
Victor Kotlyar's avatar
Victor Kotlyar committed
26
#include "common/log/LogContext.hpp"
Victor Kotlyar's avatar
Victor Kotlyar committed
27
#include "common/Timer.hpp"
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
#include <vector>
#include <stdint.h>

namespace castor {
namespace tape {
namespace tapeserver {
namespace daemon {
  class MigrationTaskInjector;
class DiskReadThreadPool {
public:
  /**
   * Constructor. The constructor creates the threads for the pool, but does not
   * start them.
   * @param nbThread Number of thread for reading files 
   * @param maxFilesReq maximal number of files we might require 
43
   * within a single request to the task injector
44
   * @param maxBytesReq maximal number of bytes we might require
45
46
   *  within a single request a single request to the task injector
   * @param lc log context for logging purpose
47
48
   */
  DiskReadThreadPool(int nbThread, uint64_t maxFilesReq,uint64_t maxBytesReq, 
49
          castor::tape::tapeserver::daemon::MigrationWatchDog & migrationWatchDog,
50
51
          cta::log::LogContext lc, const std::string & xrootPrivateKeyPath, 
          uint16_t xrootTimeout);
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
  
  /**
   * Destructor.
   * Simply destroys the thread, which should have been joined by the caller
   * before using waitThreads()
   */
  ~DiskReadThreadPool();
  
  /**
   * Starts the threads which were created at construction time.
   */
  void startThreads();
  
  /**
   * Waits for threads completion of all threads. Should be called before
   * destructor
   */
  void waitThreads();
  
  /**
   * Adds a DiskReadTask to the tape pool's queue of tasks.
   * @param task pointer to the new task. The thread pool takes ownership of the
   * task and will delete it after execution. Push() is not protected against races
   * with finish() as the task injection is done from a single thread (the task
   * injector)
   */
  void push(DiskReadTask *task);
  
  /**
   * Injects as many "end" tasks as there are threads in the thread pool.
   * A thread getting such an end task will exit. This method is called by the
   * task injector at the end of the tape session. It is not race protected. 
   * See push()
   */
  void finish();
  
  /**
   * Sets up the pointer to the task injector. This cannot be done at
   * construction time as both task injector and read thread pool refer to 
   * each other. This function should be called before starting the threads.
   * This is used for the feedback loop where the injector is requested to
   * fetch more work by the read thread pool when the task queue of the thread
   * pool starts to run low.
   */
  void setTaskInjector(MigrationTaskInjector* injector){
      m_injector = injector;
  }
private:
  /**
   * When the last thread finish, we log all m_pooldStat members + message
   * at the given level
   * @param level
   * @param message
   */
  void logWithStat(int level, const std::string& message);
  /** 
   * Get the next task to execute and if there is not enough tasks in queue,
   * it will ask the TaskInjector to get more jobs.
   * @return the next task to execute
   */
Victor Kotlyar's avatar
Victor Kotlyar committed
112
  DiskReadTask* popAndRequestMore(cta::log::LogContext & lc);
113
114
115
116
117
118
119
120
121
122
123
  
  /**
   * When a thread finishm it call this function to Add its stats to one one of the
   * Threadpool
   * @param threadStats
   */
  void addThreadStats(const DiskStats& stats);
  
  /**
   To protect addThreadStats from concurrent calls
   */
124
  cta::threading::Mutex m_statAddingProtection;
125
126
127
128
129
130
  
  /**
   * Aggregate all threads' stats 
   */
  DiskStats m_pooldStat;
  
131
132
133
  /**
   * Measure the thread pool's lifetime
   */
Victor Kotlyar's avatar
Victor Kotlyar committed
134
  cta::utils::Timer m_totalTime;
135
  
136
137
138
  /**
   * Subclass of the thread pool's worker thread.
   */
139
  class DiskReadWorkerThread: private cta::threading::Thread {
140
141
  public:
    DiskReadWorkerThread(DiskReadThreadPool & parent):
142
    m_parent(parent),m_threadID(parent.m_nbActiveThread++),m_lc(parent.m_lc),
143
    m_diskFileFactory(parent.m_xrootPrivateKeyPath,
144
      parent.m_xrootTimeout, parent.m_striperPool){
Victor Kotlyar's avatar
Victor Kotlyar committed
145
146
       cta::log::LogContext::ScopedParam param(m_lc, cta::log::Param("threadID", m_threadID));
       m_lc.log(cta::log::INFO,"DisReadThread created");
147
    }
148
149
    void start() { cta::threading::Thread::start(); }
    void wait() { cta::threading::Thread::wait(); }
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
  private:
    void logWithStat(int level, const std::string& message);
    /*
     * For measuring how long  are the the different steps 
     */
    DiskStats m_threadStat;
    
    /** Pointer to the thread pool, allowing calls to popAndRequestMore,
     * and calling finish() on the task injector when the last thread
     * is finishing (thanks to the actomic counter m_parent.m_nbActiveThread) */
    DiskReadThreadPool & m_parent;
    
    /** The sequential ID of the thread, used in logs */
    const int m_threadID;
    
    /** The local copy of the log context, allowing race-free logging with context
     between threads. */
Victor Kotlyar's avatar
Victor Kotlyar committed
167
    cta::log::LogContext m_lc;
168
169
170
171
    
    /** The execution thread: pops and executes tasks (potentially asking for
     more) and calls task injector's finish() on exit of the last thread. */
    virtual void run();
172
173
174
175
176

    /**
     * A disk file factory, that will create the proper type of file access class,
     * depending on the received path
     */
177
    cta::disk::DiskFileFactory m_diskFileFactory;
178
179
180
181
182
183
184
  };
  
  /** Container for the threads */
  std::vector<DiskReadWorkerThread *> m_threads;
  
  /** The queue of pointer to tasks to be executed. We own the tasks (they are 
   * deleted by the threads after execution) */
185
  cta::threading::BlockingQueue<DiskReadTask *> m_tasks;
186
187
188
189
190
191
192
193
194
195
  
  /**
   * Parameter: path to xroot private key
   */
  std::string m_xrootPrivateKeyPath;
  
  /**
   * Parameter: xroot timeout
   */
  uint16_t m_xrootTimeout;
196
197
198
199

  /**
   * A pool of rados striper connections, to be shared by all threads
   */
200
  cta::disk::RadosStriperPool m_striperPool;
201

202
203
204
205
206
  /**
   * Reference to the watchdog, for error reporting.
   */
  castor::tape::tapeserver::daemon::MigrationWatchDog & m_watchdog;
  
207
208
209
  /** The log context. This is copied on construction to prevent interferences
   * between threads.
   */
Victor Kotlyar's avatar
Victor Kotlyar committed
210
  cta::log::LogContext m_lc;
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
  
  /** Pointer to the task injector allowing request for more work, and 
   * termination signaling */
  MigrationTaskInjector* m_injector;
  
  /** The maximum number of files we ask per request. This value is also used as
   * a threshold (half of it, indeed) to trigger the request for more work. 
   * Another request for more work is also triggered when the task FIFO gets empty.*/
  const uint64_t m_maxFilesReq;
  
  /** Same as m_maxFilesReq for size per request. */
  const uint64_t m_maxBytesReq;
  
  /** An atomic (i.e. thread safe) counter of the current number of thread (they
   are counted up at creation time and down at completion time) */
226
  cta::threading::AtomicCounter<int> m_nbActiveThread;
227
228
229
};

}}}}