OStoreDB.cpp 49.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/*
 * The CERN Tape Archive (CTA) project
 * Copyright (C) 2015  CERN
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "OStoreDB.hpp"
20
#include "common/SecurityIdentity.hpp"
21
#include "objectstore/RootEntry.hpp"
22
23
#include "objectstore/TapePool.hpp"
#include "objectstore/Tape.hpp"
24
#include "objectstore/DriveRegister.hpp"
25
#include "objectstore/ArchiveToFileRequest.hpp"
26
#include "objectstore/RetrieveToFileRequest.hpp"
27
#include "common/exception/Exception.hpp"
28
29
30
#include "common/admin/AdminHost.hpp"
#include "common/admin/AdminUser.hpp"
#include "common/archiveRoutes/ArchiveRoute.hpp"
31
32
#include "scheduler/ArchiveRequest.hpp"
#include "scheduler/ArchiveToFileRequest.hpp"
33
#include "scheduler/LogicalLibrary.hpp"
34
#include "scheduler/RetrieveToFileRequest.hpp"
35
36
37
#include "common/archiveNS/StorageClass.hpp"
#include "common/TapePool.hpp"
#include "common/archiveNS/Tape.hpp"
38
39
#include "ArchiveToDirRequest.hpp"
#include "RetrieveToFileRequest.hpp"
40
#include "common/archiveNS/TapeFileLocation.hpp"
41
#include "RetrieveToDirRequest.hpp"
42
#include "ArchiveToTapeCopyRequest.hpp"
43
#include "common/archiveNS/ArchiveFile.hpp"
44
#include <algorithm>
45
46
#include <stdlib.h>     /* srand, rand */
#include <time.h>       /* time */
47
#include <stdexcept>
48
#include <set>
49
50
51
52
53
54

namespace cta {
  
using namespace objectstore;

OStoreDB::OStoreDB(objectstore::Backend& be):
55
56
57
58
59
60
61
62
63
64
65
66
  m_objectStore(be), m_agent(NULL) {}


OStoreDB::~OStoreDB() throw() {}

void OStoreDB::setAgent(objectstore::Agent& agent) {
  m_agent = & agent;
  }

void OStoreDB::assertAgentSet() {
  if (!m_agent)
    throw AgentNotSet("In OStoreDB::assertAgentSet: Agent pointer not set");
67
68
  }

69
70
std::unique_ptr<SchedulerDatabase::TapeMountDecisionInfo> 
  OStoreDB::getMountInfo() {
71
  //Allocate the getMountInfostructure to return.
72
73
74
  assertAgentSet();
  std::unique_ptr<TapeMountDecisionInfo> privateRet (new TapeMountDecisionInfo(
    m_objectStore, *m_agent));
75
  TapeMountDecisionInfo & tmdi=*privateRet;
76
  // Get all the tape pools and tapes with queues (potential mounts)
77
78
79
  objectstore::RootEntry re(m_objectStore);
  objectstore::ScopedSharedLock rel(re);
  re.fetch();
80
  // Take an exclusive lock on the scheduling and fetch it.
81
82
83
84
  tmdi.m_schedulerGlobalLock.reset(
    new SchedulerGlobalLock(re.getSchedulerGlobalLock(), m_objectStore));
  tmdi.m_lockOnSchedulerGlobalLock.lock(*tmdi.m_schedulerGlobalLock);
  tmdi.m_lockTaken = true;
85
  tmdi.m_schedulerGlobalLock->fetch();
86
87
88
  auto tpl = re.dumpTapePools();
  for (auto tpp=tpl.begin(); tpp!=tpl.end(); tpp++) {
    // Get the tape pool object
89
    objectstore::TapePool tpool(tpp->address, m_objectStore);
90
91
    // debug utility variable
    std::string __attribute__((__unused__)) poolName = tpp->tapePool;
92
93
    objectstore::ScopedSharedLock tpl(tpool);
    tpool.fetch();
94
95
    // If there are files queued, we create an entry for this tape pool in the
    // mount candidates list.
96
    if (tpool.getJobsSummary().files) {
97
      tmdi.potentialMounts.push_back(SchedulerDatabase::PotentialMount());
98
      auto & m = tmdi.potentialMounts.back();
99
      m.tapePool = tpp->tapePool;
100
101
102
103
104
      m.type = cta::MountType::ARCHIVE;
      m.bytesQueued = tpool.getJobsSummary().bytes;
      m.filesQueued = tpool.getJobsSummary().files;      
      m.oldestJobStartTime = tpool.getJobsSummary().oldestJobStartTime;
      m.priority = tpool.getJobsSummary().priority;
105
      
106
      m.mountCriteria.maxFilesQueued = 
107
          tpool.getMountCriteriaByDirection().archive.maxFilesQueued;
108
      m.mountCriteria.maxBytesQueued = 
109
          tpool.getMountCriteriaByDirection().archive.maxBytesQueued;
110
      m.mountCriteria.maxAge = 
111
112
113
          tpool.getMountCriteriaByDirection().archive.maxAge;
      m.mountCriteria.quota = 
          tpool.getMountCriteriaByDirection().archive.quota;
114
115
116
117
      m.logicalLibrary = "";

    }
    // For each tape in the pool, list the tapes with work
Eric Cano's avatar
Eric Cano committed
118
    auto tl = tpool.dumpTapesAndFetchStatus();
119
120
121
122
123
124
125
126
127
128
129
130
    for (auto tp = tl.begin(); tp!= tl.end(); tp++) {
      objectstore::Tape t(tp->address, m_objectStore);
      objectstore::ScopedSharedLock tl(t);
      t.fetch();
      if (t.getJobsSummary().files) {
        tmdi.potentialMounts.push_back(PotentialMount());
        auto & m = tmdi.potentialMounts.back();
        m.type = cta::MountType::RETRIEVE;
        m.bytesQueued = t.getJobsSummary().bytes;
        m.filesQueued = t.getJobsSummary().files;
        m.oldestJobStartTime = t.getJobsSummary().oldestJobStartTime;
        m.priority = t.getJobsSummary().priority;
131
132
        m.vid = t.getVid();
        m.logicalLibrary = t.getLogicalLibrary();
133
134
        
        m.mountCriteria.maxFilesQueued = 
135
            tpool.getMountCriteriaByDirection().retrieve.maxFilesQueued;
136
        m.mountCriteria.maxBytesQueued = 
137
            tpool.getMountCriteriaByDirection().retrieve.maxBytesQueued;
138
        m.mountCriteria.maxAge = 
139
140
141
            tpool.getMountCriteriaByDirection().retrieve.maxAge;
        m.mountCriteria.quota = 
            tpool.getMountCriteriaByDirection().retrieve.quota;
142
143
        m.logicalLibrary = t.getLogicalLibrary();
      }
144
145
    }
  }
146
147
148
149
150
151
152
153
  // Dedication information comes here
  // TODO
  // 
  // Collect information about the existing mounts
  objectstore::DriveRegister dr(re.getDriveRegisterAddress(), m_objectStore);
  objectstore::ScopedSharedLock drl(dr);
  dr.fetch();
  auto dl = dr.dumpDrives();
154
155
  std::set<int> activeDriveStatuses = {
    (int)objectstore::DriveRegister::DriveStatus::Starting,
156
157
158
159
160
    (int)objectstore::DriveRegister::DriveStatus::Mounting,
    (int)objectstore::DriveRegister::DriveStatus::Transfering,
    (int)objectstore::DriveRegister::DriveStatus::Unloading,
    (int)objectstore::DriveRegister::DriveStatus::Unmounting,
    (int)objectstore::DriveRegister::DriveStatus::DrainingToDisk };
161
  for (auto d=dl.begin(); d!= dl.end(); d++) {
162
    if (activeDriveStatuses.count((int)d->status)) {
163
      tmdi.existingMounts.push_back(ExistingMount());
164
165
      switch (d->mountType) {
        case objectstore::DriveRegister::MountType::Archive:
166
167
          tmdi.existingMounts.back().type = cta::MountType::ARCHIVE;
          break;
168
169
        case objectstore::DriveRegister::MountType::Retrieve:
          tmdi.existingMounts.back().type = cta::MountType::RETRIEVE;
170
171
172
173
174
175
176
177
178
          break;
        default:
          throw exception::Exception("In OStoreDB::getMountInfo(): got drive with unexpected mount type");
      }
      tmdi.existingMounts.back().tapePool = d->currentTapePool;
    }
  }
  std::unique_ptr<SchedulerDatabase::TapeMountDecisionInfo> ret(std::move(privateRet));
  return ret;
179
180
}

181
182
void OStoreDB::createAdminHost(const std::string& hostName,
    const cta::CreationLog & creationLog) {
183
184
185
  RootEntry re(m_objectStore);
  ScopedExclusiveLock rel(re);
  re.fetch();
186
  re.addAdminHost(hostName, creationLog);
187
  re.commit();
188
}
189

190
191
192
193
194
195
std::list<AdminHost> OStoreDB::getAdminHosts() const {
  RootEntry re(m_objectStore);
  ScopedSharedLock rel(re);
  re.fetch();
  std::list<AdminHost> ret;
  auto hl=re.dumpAdminHosts();
196
  rel.release();
197
198
  for (auto h=hl.begin(); h!=hl.end(); h++) {
    ret.push_back(AdminHost(h->hostname, 
199
        h->log));
200
  }
201
202
  return ret;
}
203
204
205
206
207
208
209
210
211
212
213
214
215


void OStoreDB::deleteAdminHost(const SecurityIdentity& requester, 
  const std::string& hostName) {
  RootEntry re(m_objectStore);
  ScopedExclusiveLock rel(re);
  re.fetch();
  re.removeAdminHost(hostName);
  re.commit();
}

void OStoreDB::createAdminUser(const SecurityIdentity& requester, 
  const cta::UserIdentity& user, const std::string& comment) {
216
217
218
  RootEntry re(m_objectStore);
  ScopedExclusiveLock rel(re);
  re.fetch();
219
220
  objectstore::CreationLog cl(requester.getUser(), requester.getHost(),
    time(NULL), comment);
221
  re.addAdminUser(objectstore::UserIdentity(user.uid, user.gid), cl);
222
  re.commit();
223
224
225
226
}

void OStoreDB::deleteAdminUser(const SecurityIdentity& requester, 
  const cta::UserIdentity& user) {
227
228
229
  RootEntry re(m_objectStore);
  ScopedExclusiveLock rel(re);
  re.fetch();
230
  re.removeAdminUser(objectstore::UserIdentity(user.uid, user.gid));
231
  re.commit();
232
233
234
}

std::list<AdminUser> OStoreDB::getAdminUsers() const {
235
236
237
238
239
240
241
242
243
  std::list<AdminUser> ret;
  RootEntry re(m_objectStore);
  ScopedSharedLock rel(re);
  re.fetch();
  auto aul = re.dumpAdminUsers();
  for (auto au=aul.begin(); au!=aul.end(); au++) {
    ret.push_back(
      AdminUser(
        cta::UserIdentity(au->user.uid, au->user.gid),
244
        au->log
245
246
247
248
    ));
  }
  rel.release();
  return ret;
249
250
251
}

void OStoreDB::assertIsAdminOnAdminHost(const SecurityIdentity& id) const {
252
253
254
  RootEntry re(m_objectStore);
  ScopedSharedLock rel(re);
  re.fetch();
255
256
  if (!re.isAdminUser(objectstore::UserIdentity(id.getUser().uid,
      id.getUser().gid))) {
257
    std::ostringstream msg;
258
259
    msg << "User uid=" << id.getUser().uid 
        << " gid=" << id.getUser().gid
260
261
262
263
264
265
266
267
        << " is not an administrator";
    throw exception::Exception(msg.str());
  }
  if (!re.isAdminHost(id.getHost())) {
    std::ostringstream msg;
    msg << "Host " << id.getHost() << " is not an administration host";
    throw exception::Exception(msg.str());
  }
268
269
}

270
271
void OStoreDB::createStorageClass(const std::string& name,
  const uint16_t nbCopies, const cta::CreationLog& creationLog) {
272
273
274
  RootEntry re(m_objectStore);
  ScopedExclusiveLock rel(re);
  re.fetch();
275
  re.addStorageClass(name, nbCopies, creationLog);
276
  re.commit();
277
278
279
}

StorageClass OStoreDB::getStorageClass(const std::string& name) const {
280
281
282
283
284
285
  RootEntry re(m_objectStore);
  ScopedSharedLock rel(re);
  re.fetch();
  auto sc = re.dumpStorageClass(name);
  return cta::StorageClass(name,
    sc.copyCount,
286
    sc.log);
287
288
289
}

std::list<StorageClass> OStoreDB::getStorageClasses() const {
290
291
292
293
294
295
296
  RootEntry re(m_objectStore);
  ScopedSharedLock rel(re);
  re.fetch();
  auto scl = re.dumpStorageClasses();
  decltype (getStorageClasses()) ret;
  for (auto sc = scl.begin(); sc != scl.end(); sc++) {
    ret.push_back(StorageClass(sc->storageClass, 
297
        sc->copyCount, sc->log));
298
  }
299
  ret.sort();
300
  return ret;
301
302
303
304
}

void OStoreDB::deleteStorageClass(const SecurityIdentity& requester, 
  const std::string& name) {
305
306
307
308
309
  RootEntry re(m_objectStore);
  ScopedExclusiveLock rel(re);
  re.fetch();
  re.removeStorageClass(name);
  re.commit();
310
311
}

312
void OStoreDB::createArchiveRoute(const std::string& storageClassName,
313
314
  const uint16_t copyNb, const std::string& tapePoolName,
  const cta::CreationLog& creationLog) {
315
316
317
  RootEntry re(m_objectStore);
  ScopedExclusiveLock rel(re);
  re.fetch();
318
  re.addArchiveRoute(storageClassName, copyNb, tapePoolName, 
319
    objectstore::CreationLog(creationLog));
320
  re.commit();
321
322
}

323
324
std::list<ArchiveRoute> 
  OStoreDB::getArchiveRoutes(const std::string& storageClassName) const {
325
326
327
328
329
  RootEntry re(m_objectStore);
  ScopedExclusiveLock rel(re);
  re.fetch();
  auto scd = re.dumpStorageClass(storageClassName);
  rel.release();
330
  if (scd.routes.size() != scd.copyCount) {
331
    throw IncompleteRouting("In OStoreDB::getArchiveRoutes: routes are incomplete");
332
  }
333
  std::list<ArchiveRoute> ret;
334
  for (auto r=scd.routes.begin(); r!=scd.routes.end(); r++) {
335
    ret.push_back(ArchiveRoute(storageClassName, 
336
337
338
      r->copyNumber, r->tapePool, r->log));
  }
  return ret;
339
340
}

341
std::list<ArchiveRoute> OStoreDB::getArchiveRoutes() const {
342
343
344
345
346
  RootEntry re(m_objectStore);
  ScopedExclusiveLock rel(re);
  re.fetch();
  auto scd = re.dumpStorageClasses();
  rel.release();
347
  std::list<ArchiveRoute> ret;
348
349
  for (auto sc=scd.begin(); sc!=scd.end(); sc++) {
    for (auto r=sc->routes.begin(); r!=sc->routes.end(); r++) {
350
      ret.push_back(ArchiveRoute(sc->storageClass, 
351
352
353
354
        r->copyNumber, r->tapePool, r->log));
    }
  }
  return ret;
355
356
}

357
void OStoreDB::deleteArchiveRoute(const SecurityIdentity& requester, 
358
  const std::string& storageClassName, const uint16_t copyNb) {
359
360
361
  RootEntry re(m_objectStore);
  ScopedExclusiveLock rel(re);
  re.fetch();
362
  re.removeArchiveRoute(storageClassName, copyNb);
363
  re.commit();
364
365
}

366
367
368
void OStoreDB::ArchiveToFileRequestCreation::complete() {
  // We inherited all the objects from the creation.
  // Lock is still here at that point.
369
  // First, record that we are fine for next step.
370
  m_request.setAllJobsLinkingToTapePool();
371
  m_request.commit();
372
373
374
375
376
377
378
379
  objectstore::RootEntry re(m_objectStore);
  // We can now plug the request onto its tape pools.
  // We can discover at that point that a tape pool is actually not
  // really owned by the root entry, and hence a dangling pointer
  // We should then unlink the jobs from that already connected
  // tape pools and abort the job creation.
  // The list of done tape pools is held here for this purpose
  // Reconstruct the job list
380
  auto jl = m_request.dumpJobs();
381
382
383
384
385
386
387
388
389
  std::list<std::string> linkedTapePools;
  try {
    for (auto j=jl.begin(); j!=jl.end(); j++) {
      objectstore::TapePool tp(j->tapePoolAddress, m_objectStore);
      ScopedExclusiveLock tpl(tp);
      tp.fetch();
      if (tp.getOwner() != re.getAddressIfSet())
        throw NoSuchTapePool("In OStoreDB::queue: non-existing tape pool found "
            "(dangling pointer): cancelling request creation.");
390
      tp.addJob(*j, m_request.getAddressIfSet(), m_request.getArchiveFile(), 
391
392
        m_request.getRemoteFile().status.size, m_request.getPriority(),
        m_request.getCreationLog().time);
393
394
395
396
      // Now that we have the tape pool handy, get the retry limits from it and 
      // assign them to the job
      m_request.setJobFailureLimits(j->copyNb, tp.getMaxRetriesWithinMount(), 
        tp.getMaxTotalRetries());
397
398
399
400
401
402
403
404
405
      tp.commit();
      linkedTapePools.push_back(j->tapePoolAddress);
    }
  } catch (NoSuchTapePool &) {
    // Unlink the request from already connected tape pools
    for (auto tpa=linkedTapePools.begin(); tpa!=linkedTapePools.end(); tpa++) {
      objectstore::TapePool tp(*tpa, m_objectStore);
      ScopedExclusiveLock tpl(tp);
      tp.fetch();
406
      tp.removeJob(m_request.getAddressIfSet());
407
      tp.commit();
408
      m_request.remove();
409
410
411
    }
    throw;
  }
412
413
  // The request is now fully set. As it's multi-owned, we do not set the owner,
  // just to disown it from the agent.
414
415
416
  m_request.setOwner("");
  m_request.commit();
  m_lock.release();
417
418
419
420
  // And remove reference from the agent
  {
    objectstore::ScopedExclusiveLock al(*m_agent);
    m_agent->fetch();
421
    m_agent->removeFromOwnership(m_request.getAddressIfSet());
422
423
    m_agent->commit();
  }
424
  m_closed=true;
425
  return;
426
427
428
429
430
431
432
}

void OStoreDB::ArchiveToFileRequestCreation::cancel() {
  // We inherited everything from the creation, and all we have to
  // do here is to delete the request from storage and dereference it from
  // the agent's entry
  if (m_closed) {
433
    throw ArchiveRequestAlreadyCompleteOrCanceled(
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
      "In OStoreDB::ArchiveToFileRequestCreation::cancel: trying the close "
      "the request creation twice");
  }
  m_request.remove();
  {
    objectstore::ScopedExclusiveLock al(*m_agent);
    m_agent->fetch();
    m_agent->removeFromOwnership(m_request.getAddressIfSet());
    m_agent->commit();
  }
  m_closed=true;
  return;
}

OStoreDB::ArchiveToFileRequestCreation::~ArchiveToFileRequestCreation() {
  // We have to determine whether complete() or cancel() were called, in which
  // case there is nothing to do, or not, in which case we have to garbage
  // collect the archive to file request. This will queue it to the appropriate
  // tape pool(s) orphanesArchiveToFileCreations. The schedule will then 
  // determine its fate depending on the status of the NS entry creation
  // (no entry, just cancel, already created in NS, carry on).
  if (m_closed)
    return;
  try {
    m_request.garbageCollect(m_agent->getAddressIfSet());
    {
      objectstore::ScopedExclusiveLock al(*m_agent);
      m_agent->fetch();
      m_agent->removeFromOwnership(m_request.getAddressIfSet());
      m_agent->commit();
    }
    m_closed=true;
  } catch (...) {}
}

469
470
471
472
473
474
void OStoreDB::createTapePool(const std::string& name,
  const uint32_t nbPartialTapes, const cta::CreationLog &creationLog) {
  RootEntry re(m_objectStore);
  ScopedExclusiveLock rel(re);
  re.fetch();
  assertAgentSet();
475
  re.addOrGetTapePoolAndCommit(name, nbPartialTapes, 5, 5, *m_agent, creationLog);
476
  re.commit();
477
478
479
480
481
482
483
484
485
486
487
488
489
  }

void OStoreDB::setTapePoolMountCriteria(const std::string& tapePool,
  const MountCriteriaByDirection& mountCriteriaByDirection) {
  RootEntry re(m_objectStore);
  ScopedSharedLock rel(re);
  re.fetch();
  objectstore::TapePool tp(re.getTapePoolAddress(tapePool),m_objectStore);
  rel.release();
  ScopedExclusiveLock tplock(tp);
  tp.fetch();
  tp.setMountCriteriaByDirection(mountCriteriaByDirection);
  tp.commit();
490
491
}

492

493
std::list<cta::TapePool> OStoreDB::getTapePools() const {
494
495
496
497
498
  RootEntry re(m_objectStore);
  ScopedSharedLock rel(re);
  re.fetch();
  auto tpd = re.dumpTapePools();
  rel.release();
499
  std::list<cta::TapePool> ret;
500
  for (auto tp=tpd.begin(); tp!=tpd.end(); tp++) {
501
502
    ret.push_back(cta::TapePool(tp->tapePool, tp->nbPartialTapes,
      tp->mountCriteriaByDirection, tp->log));
503
504
  }
  return ret;
505
506
507
508
}

void OStoreDB::deleteTapePool(const SecurityIdentity& requester, 
  const std::string& name) {
509
510
511
512
513
  RootEntry re(m_objectStore);
  ScopedExclusiveLock rel(re);
  re.fetch();
  re.removeTapePoolAndCommit(name);
  re.commit();
514
515
}

516
517
void OStoreDB::createTape(const std::string& vid, 
  const std::string& logicalLibraryName, 
518
  const std::string& tapePoolName, const uint64_t capacityInBytes, 
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
  const cta::CreationLog& creationLog) {
  // To create a tape, we have to
  // - Find the storage class and lock for write.
  // - Create the tape object.
  // - Connect the tape object to the tape pool.
  RootEntry re(m_objectStore);
  ScopedSharedLock rel(re);
  re.fetch();
  // Check the library exists
  auto libs = re.dumpLibraries();
  for (auto l=libs.begin(); l!=libs.end(); l++) {
    if (l->library == logicalLibraryName)
      goto found;
  }
  throw NoSuchLibrary("In OStoreDB::createTape: trying to create a tape in a non-existing library");
  found:
  std::string tpAddress = re.getTapePoolAddress(tapePoolName);
  // Take hold of the tape pool
  objectstore::TapePool tp(tpAddress, m_objectStore);
  ScopedExclusiveLock tpl(tp);
  tp.fetch();
540
541
542
543
544
  // Check the tape pool is owned by the root entry. If not, it should be
  // considered as a dangling pointer.
  if (tp.getOwner() != re.getAddressIfSet())
    throw NoSuchTapePool("In OStoreDB::createTape: trying to create a tape in a"
      " non-existing tape pool (dangling pointer)");
545
  // Check that the tape exists and throw an exception if it does.
546
  // TODO: we should check in all tape pools (or have a central index)
547
548
549
550
551
552
553
554
555
556
  try {
    tp.getTapeAddress(vid);
    throw TapeAlreadyExists("In OStoreDB::createTape: trying to create an existing tape.");
  } catch (cta::exception::Exception &) {}
  // Create the tape. The tape pool method takes care of the gory details for us.
  tp.addOrGetTapeAndCommit(vid, logicalLibraryName, capacityInBytes, 
      *m_agent, creationLog);
  tp.commit();
}

557
cta::Tape OStoreDB::getTape(const std::string &vid) const {
558
559
560
561
562
563
564
565
566
567
568
569
570
  // Got through all tape pools. Get the list of them
  RootEntry re(m_objectStore);
  ScopedSharedLock rel(re);
  re.fetch();
  auto tpl = re.dumpTapePools();
  for (auto tpi=tpl.begin(); tpi!=tpl.end(); tpi++) {
    objectstore::TapePool tp(tpi->address, m_objectStore);
    ScopedSharedLock tpl(tp);
    tp.fetch();
    // Check the tape pool is owned by the root entry. If not, it should be
    // considered as a dangling pointer (and skip it)
    if (tp.getOwner() != re.getAddressIfSet())
      continue;
Eric Cano's avatar
Eric Cano committed
571
    auto tl=tp.dumpTapesAndFetchStatus();
572
573
574
575
576
577
    for (auto ti=tl.begin(); ti!=tl.end(); ti++) {
      if (vid == ti->vid) {
        objectstore::Tape t(ti->address, m_objectStore);
        ScopedSharedLock tl(t);
        t.fetch();
        const uint64_t nbFiles = 0; // TO BE DONE
Eric Cano's avatar
Eric Cano committed
578
        cta::Tape::Status status;
579
        return cta::Tape(ti->vid, nbFiles, ti->logicalLibraryName,
Eric Cano's avatar
Eric Cano committed
580
          tpi->tapePool, ti->capacityInBytes, t.getStoredData(), ti->log, status);
581
582
583
584
      }
    }
  }
  throw NoSuchTape("In OStoreDB::getTape: No such tape");
585
586
}

587
588
589
590
591
592
593
594
595
596
597
std::list<cta::Tape> OStoreDB::getTapes() const {
  std::list<cta::Tape> ret;
  // Got through all tape pools. Get the list of them
  RootEntry re(m_objectStore);
  ScopedSharedLock rel(re);
  re.fetch();
  auto tpl = re.dumpTapePools();
  for (auto tpi=tpl.begin(); tpi!=tpl.end(); tpi++) {
    objectstore::TapePool tp(tpi->address, m_objectStore);
    ScopedSharedLock tpl(tp);
    tp.fetch();
598
599
600
601
    // Check the tape pool is owned by the root entry. If not, it should be
    // considered as a dangling pointer (and skip it)
    if (tp.getOwner() != re.getAddressIfSet())
      continue;
Eric Cano's avatar
Eric Cano committed
602
    auto tl=tp.dumpTapesAndFetchStatus();
603
604
605
606
    for (auto ti=tl.begin(); ti!=tl.end(); ti++) {
      objectstore::Tape t(ti->address, m_objectStore);
      ScopedSharedLock tl(t);
      t.fetch();
607
608
      const uint64_t nbFiles = 0; // TO BE DONE
      ret.push_back(cta::Tape(ti->vid, nbFiles, ti->logicalLibraryName,
Eric Cano's avatar
Eric Cano committed
609
610
        tpi->tapePool, ti->capacityInBytes, t.getStoredData(), ti->log,
        ti->status));
611
612
613
    }
  }
  return ret;
614
615
616
617
618
619
620
}

void OStoreDB::deleteTape(const SecurityIdentity& requester, 
  const std::string& vid) {
  throw exception::Exception("Not Implemented");
}

621
622
623
624
625
626
627
void OStoreDB::createLogicalLibrary(const std::string& name, 
  const cta::CreationLog& creationLog) {
  RootEntry re(m_objectStore);
  ScopedExclusiveLock rel(re);
  re.fetch();
  re.addLibrary(name, creationLog);
  re.commit();
628
629
630
}

std::list<LogicalLibrary> OStoreDB::getLogicalLibraries() const {
631
632
633
634
635
636
637
638
639
640
  RootEntry re(m_objectStore);
  ScopedSharedLock rel(re);
  re.fetch();
  auto ll = re.dumpLibraries();
  rel.release();
  std::list<LogicalLibrary> ret;
  for (auto l=ll.begin(); l!=ll.end(); l++) {
    ret.push_back(LogicalLibrary(l->library, l->log));
  }
  return ret;
641
642
643
644
}

void OStoreDB::deleteLogicalLibrary(const SecurityIdentity& requester, 
  const std::string& name) {
645
646
  RootEntry re(m_objectStore);
  ScopedExclusiveLock rel(re);
647
648
649
650
651
652
  // Check we are not deleting a non-empty library
  auto tpl = re.dumpTapePools();
  for (auto tpp=tpl.begin(); tpp!=tpl.end(); tpp++) {
    objectstore::TapePool tp(tpp->address, m_objectStore);
    ScopedSharedLock tplock(tp);
    tp.fetch();
653
654
655
656
    // Check the tape pool is owned by the root entry. If not, it should be
    // considered as a dangling pointer.
    if (tp.getOwner() != re.getAddressIfSet())
      continue;
Eric Cano's avatar
Eric Cano committed
657
    auto tl=tp.dumpTapesAndFetchStatus();
658
659
660
661
662
    for (auto t=tl.begin(); t!=tl.end(); t++) {
      if (t->logicalLibraryName == name)
        throw LibraryInUse("In OStoreDB::deleteLogicalLibrary: trying to delete a library used by a tape.");
    }
  }
663
664
665
  re.fetch();
  re.removeLibrary(name);
  re.commit();
666
667
}

668
669
std::unique_ptr<cta::SchedulerDatabase::ArchiveToFileRequestCreation>
  OStoreDB::queue(const cta::ArchiveToFileRequest& rqst) {
670
  assertAgentSet();
671
  // Construct the return value immediately
672
673
674
675
  std::unique_ptr<cta::OStoreDB::ArchiveToFileRequestCreation> 
    internalRet(new cta::OStoreDB::ArchiveToFileRequestCreation(m_agent, m_objectStore));
  cta::objectstore::ArchiveToFileRequest & atfr = internalRet->m_request;
  atfr.setAddress(m_agent->nextId("ArchiveToFileRequest"));
676
  atfr.initialize();
677
  atfr.setArchiveFile(rqst.archiveFile);
678
  atfr.setRemoteFile(rqst.remoteFile);
679
680
  atfr.setPriority(rqst.priority);
  atfr.setCreationLog(rqst.creationLog);
681
682
683
684
  // We will need to identity tapepools is order to construct the request
  RootEntry re(m_objectStore);
  ScopedSharedLock rel(re);
  re.fetch();
685
  auto & cl = rqst.copyNbToPoolMap;
686
  std::list<cta::objectstore::ArchiveToFileRequest::JobDump> jl;
687
688
689
  for (auto copy=cl.begin(); copy != cl.end(); copy++) {
    std::string tpaddr = re.getTapePoolAddress(copy->second);
    atfr.addJob(copy->first, copy->second, tpaddr);
690
691
692
693
    jl.push_back(cta::objectstore::ArchiveToFileRequest::JobDump());
    jl.back().copyNb = copy->first;
    jl.back().tapePool = copy->second;
    jl.back().tapePoolAddress = tpaddr;
694
695
696
697
  }
  if (!jl.size()) {
    throw ArchiveRequestHasNoCopies("In OStoreDB::queue: the archive to file request has no copy");
  }
698
  // We create the object here
699
700
701
702
703
704
  {
    objectstore::ScopedExclusiveLock al(*m_agent);
    m_agent->fetch();
    m_agent->addToOwnership(atfr.getAddressIfSet());
    m_agent->commit();
  }
705
  atfr.setOwner(m_agent->getAddressIfSet());
706
707
  atfr.insert();
  internalRet->m_lock.lock(atfr);  
708
709
710
711
712
713
714
715
  // We successfully prepared the object. It will remain attached to the agent 
  // entry for the time being and get plugged to the tape pools on a second
  // pass. 
  // TODO: this can be improved by passing an opaque set of data to the called
  // (including the lock) in order to optimise the acesses to the object store.
  // In the mean time, the step 2 of this insertion will be done by finding the
  // archiveRequest from the agent's owned object. Crude, but should not be too
  // bad as the agent is not supposed to own many objects in this place.
716
717
718
  std::unique_ptr<cta::SchedulerDatabase::ArchiveToFileRequestCreation> ret;
  ret.reset(internalRet.release());
  return ret;
719
720
721
}

void OStoreDB::queue(const ArchiveToDirRequest& rqst) {
722
723
724
725
  auto & archiveToFileRequests = rqst.getArchiveToFileRequests();
  for(auto req=archiveToFileRequests.begin(); req!=archiveToFileRequests.end(); req++) {
    queue(*req);
  }
726
727
}

728
void OStoreDB::deleteArchiveRequest(const SecurityIdentity& requester, 
729
  const std::string& archiveFile) {
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
  // First of, find the archive request form all the tape pools.
  objectstore::RootEntry re(m_objectStore);
  objectstore::ScopedSharedLock rel(re);
  re.fetch();
  auto tpl = re.dumpTapePools();
  rel.release();
  for (auto tpp=tpl.begin(); tpp!= tpl.end(); tpp++) {
    objectstore::TapePool tp(tpp->address, m_objectStore);
    ScopedSharedLock tplock(tp);
    tp.fetch();
    auto ajl=tp.dumpJobs();
    tplock.release();
    for (auto ajp=ajl.begin(); ajp!=ajl.end(); ajp++) {
      objectstore::ArchiveToFileRequest atfr(ajp->address, m_objectStore);
      ScopedSharedLock atfrl(atfr);
      atfr.fetch();
      if (atfr.getArchiveFile() == archiveFile) {
        atfrl.release();
        objectstore::ScopedExclusiveLock al(*m_agent);
        m_agent->fetch();
        m_agent->addToOwnership(atfr.getAddressIfSet());
        m_agent->commit();
        ScopedExclusiveLock atfrxl(atfr);
        atfr.fetch();
754
        atfr.setAllJobsFailed();
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
        atfr.setOwner(m_agent->getAddressIfSet());
        atfr.commit();
        auto jl = atfr.dumpJobs();
        for (auto j=jl.begin(); j!=jl.end(); j++) {
          try {
            objectstore::TapePool tp(j->tapePoolAddress, m_objectStore);
            ScopedExclusiveLock tpl(tp);
            tp.fetch();
            tp.removeJob(atfr.getAddressIfSet());
            tp.commit();
          } catch (...) {}
        }
        atfr.remove();
        m_agent->removeFromOwnership(atfr.getAddressIfSet());
        m_agent->commit();
      }
    }
  }
  throw NoSuchArchiveRequest("In OStoreDB::deleteArchiveRequest: ArchiveToFileRequest not found");
774
775
}

776
777
std::unique_ptr<SchedulerDatabase::ArchiveToFileRequestCancelation>
  OStoreDB::markArchiveRequestForDeletion(const SecurityIdentity& requester,
778
  const std::string& archiveFile) {
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
  assertAgentSet();
  // Construct the return value immediately
  std::unique_ptr<cta::OStoreDB::ArchiveToFileRequestCancelation>
    internalRet(new cta::OStoreDB::ArchiveToFileRequestCancelation(m_agent, m_objectStore));
  cta::objectstore::ArchiveToFileRequest & atfr = internalRet->m_request;
  cta::objectstore::ScopedExclusiveLock & atfrl = internalRet->m_lock;
  // Attempt to find the request
  objectstore::RootEntry re(m_objectStore);
  ScopedSharedLock rel(re);
  re.fetch();
  auto tpl=re.dumpTapePools();
  rel.release();
  for (auto tpp=tpl.begin(); tpp!=tpl.end(); tpp++) {
    try {
      objectstore::TapePool tp(tpp->address, m_objectStore);
      ScopedSharedLock tpl(tp);
      tp.fetch();
      auto arl = tp.dumpJobs();
      tpl.release();
      for (auto arp=arl.begin(); arp!=arl.end(); arp++) {
        objectstore::ArchiveToFileRequest tatfr(arp->address, m_objectStore);
        objectstore::ScopedSharedLock tatfrl(tatfr);
        tatfr.fetch();
        if (tatfr.getArchiveFile() == archiveFile) {
          // Point the agent to the request
          ScopedExclusiveLock agl(*m_agent);
          m_agent->fetch();
          m_agent->addToOwnership(arp->address);
          m_agent->commit();
          agl.release();
          // Mark all jobs are being pending NS deletion (for being deleted them selves) 
          tatfrl.release();
          atfr.setAddress(arp->address);
          atfrl.lock(atfr);
          atfr.fetch();
814
          atfr.setAllJobsPendingNSdeletion();
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
          atfr.commit();
          // Unlink the jobs from the tape pools (it is safely referenced in the agent)
          auto atpl=atfr.dumpJobs();
          for (auto atpp=atpl.begin(); atpp!=atpl.end(); atpp++) {
            objectstore::TapePool atp(atpp->tapePoolAddress, m_objectStore);
            objectstore::ScopedExclusiveLock atpl(atp);
            atp.fetch();
            atp.removeJob(arp->address);
            atp.commit();
          }
          // Return the object to the caller, so complete() can be called later.
          std::unique_ptr<cta::SchedulerDatabase::ArchiveToFileRequestCancelation> ret;
          ret.reset(internalRet.release());
          return ret;
        }
      }
    } catch (...) {}
  }
  throw NoSuchArchiveRequest("In OStoreDB::markArchiveRequestForDeletion: ArchiveToFileRequest no found");
  }

void OStoreDB::ArchiveToFileRequestCancelation::complete() {
  if (m_closed)
    throw ArchiveRequestAlreadyDeleted("OStoreDB::ArchiveToFileRequestCancelation::complete(): called twice");
  // We just need to delete the object and forget it
  m_request.remove();
  objectstore::ScopedExclusiveLock al (*m_agent);
  m_agent->fetch();
  m_agent->removeFromOwnership(m_request.getAddressIfSet());
  m_agent->commit();
  m_closed = true;
  }

OStoreDB::ArchiveToFileRequestCancelation::~ArchiveToFileRequestCancelation() {
  if (!m_closed) {
850
851
852
853
854
855
856
    try {
      m_request.garbageCollect(m_agent->getAddressIfSet());
      objectstore::ScopedExclusiveLock al (*m_agent);
      m_agent->fetch();
      m_agent->removeFromOwnership(m_request.getAddressIfSet());
      m_agent->commit();
    } catch (...) {}
857
  }
858
859
}

860
861


862
std::map<cta::TapePool, std::list<ArchiveToTapeCopyRequest> >
863
  OStoreDB::getArchiveRequests() const {
864
865
866
867
868
869
870
871
872
873
  objectstore::RootEntry re(m_objectStore);
  objectstore::ScopedSharedLock rel(re);
  re.fetch();
  std::map<cta::TapePool, std::list<ArchiveToTapeCopyRequest> > ret;
  auto tpl = re.dumpTapePools();
  rel.release();
  for (auto tpp=tpl.begin(); tpp!=tpl.end(); tpp++) {
    objectstore::TapePool ostp(tpp->address, m_objectStore);
    ScopedSharedLock ostpl(ostp);
    ostp.fetch();
874
875
    cta::TapePool tp(tpp->tapePool, tpp->nbPartialTapes,
      ostp.getMountCriteriaByDirection(), tpp->log);    
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
    auto arl = ostp.dumpJobs();
    ostpl.release();
    for (auto ar=arl.begin(); ar!=arl.end(); ar++) {
      objectstore::ArchiveToFileRequest osar(ar->address, m_objectStore);
      ScopedSharedLock osarl(osar);
      osar.fetch();
      // Find which copy number is for this tape pool.
      // skip the request if not found
      auto jl = osar.dumpJobs();
      uint16_t copynb;
      bool copyndFound=false;
      for (auto j=jl.begin(); j!=jl.end(); j++) {
        if (j->tapePool == tpp->tapePool) {
          copynb = j->copyNb;
          copyndFound = true;
          break;
        }
      }
      if (!copyndFound) continue;
      ret[tp].push_back(cta::ArchiveToTapeCopyRequest(
        osar.getRemoteFile(),
        osar.getArchiveFile(),
        copynb,
        tpp->tapePool,
        osar.getPriority(),
        osar.getCreationLog()));
    }
  }
  return ret;
905
906
907
908
}

std::list<ArchiveToTapeCopyRequest>
  OStoreDB::getArchiveRequests(const std::string& tapePoolName) const {
909
910
911
912
913
914
915
916
917
918
919
  objectstore::RootEntry re(m_objectStore);
  objectstore::ScopedSharedLock rel(re);
  re.fetch();
  auto tpl = re.dumpTapePools();
  rel.release();
  for (auto tpp=tpl.begin(); tpp!=tpl.end(); tpp++) {
    if (tpp->tapePool != tapePoolName) continue;
    std::list<ArchiveToTapeCopyRequest> ret;
    objectstore::TapePool ostp(tpp->address, m_objectStore);
    ScopedSharedLock ostpl(ostp);
    ostp.fetch();
920
921
    cta::TapePool tp(tpp->tapePool, tpp->nbPartialTapes, 
      ostp.getMountCriteriaByDirection(), tpp->log);
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
    auto arl = ostp.dumpJobs();
    ostpl.release();
    for (auto ar=arl.begin(); ar!=arl.end(); ar++) {
      objectstore::ArchiveToFileRequest osar(ar->address, m_objectStore);
      ScopedSharedLock osarl(osar);
      osar.fetch();
      // Find which copy number is for this tape pool.
      // skip the request if not found
      auto jl = osar.dumpJobs();
      uint16_t copynb;
      bool copyndFound=false;
      for (auto j=jl.begin(); j!=jl.end(); j++) {
        if (j->tapePool == tpp->tapePool) {
          copynb = j->copyNb;
          copyndFound = true;
          break;
        }
      }
      if (!copyndFound) continue;
      ret.push_back(cta::ArchiveToTapeCopyRequest(
        osar.getRemoteFile(),
        osar.getArchiveFile(),
        copynb,
        tpp->tapePool,
        osar.getPriority(),
        osar.getCreationLog()));
    }
    return ret;
  }
  throw NoSuchTapePool("In OStoreDB::getArchiveRequests: tape pool not found");
952
953
}

954
void OStoreDB::queue(const cta::RetrieveToFileRequest& rqst) {
955
  assertAgentSet();
956
  // Check at least one potential tape copy is provided.
957
958
959
960
961
  // In order to post the job, construct it first.
  objectstore::RetrieveToFileRequest rtfr(m_agent->nextId("RetrieveToFileRequest"), m_objectStore);
  rtfr.initialize();
  rtfr.setArchiveFile(rqst.getArchiveFile());
  rtfr.setRemoteFile(rqst.getRemoteFile());
962
963
  rtfr.setPriority(rqst.priority);
  rtfr.setCreationLog(rqst.creationLog);
964
  rtfr.setSize(rqst.getSize());
965
966
967
  // We will need to identity tapes is order to construct the request.
  // First load all the tapes information in a memory map
  std::map<std::string, std::string> vidToAddress;
968
969
970
971
972
973
  RootEntry re(m_objectStore);
  ScopedSharedLock rel(re);
  re.fetch();
  auto tapePools = re.dumpTapePools();
  for(auto pool=tapePools.begin(); pool!=tapePools.end(); pool++) {
    objectstore::TapePool tp(pool->address, m_objectStore);
974
975
    objectstore::ScopedSharedLock tpl(tp);
    tp.fetch();
Eric Cano's avatar
Eric Cano committed
976
    auto tapes = tp.dumpTapesAndFetchStatus();
977
978
979
980
981
982
    for(auto tape=tapes.begin(); tape!=tapes.end(); tape++)
      vidToAddress[tape->vid] = tape->address;
  }
  // Now add all the candidate tape copies to the request. With validation
  for (auto tc=rqst.getTapeCopies().begin(); tc!=rqst.getTapeCopies().end(); tc++) {
    // Check the tape copy copynumber (range = [1 - copyCount] )
983
    if (tc->copyNb > rqst.getTapeCopies().size() || tc->copyNb < 1) {
984
985
986
987
988
989
      throw TapeCopyNumberOutOfRange("In OStoreDB::queue(RetrieveToFile): copy number out of range");
    }
  }
  // Add all the tape copies to the request
  try {
    for (auto tc=rqst.getTapeCopies().begin(); tc!=rqst.getTapeCopies().end(); tc++) {
990
      rtfr.addJob(tc->copyNb, tc->vid, vidToAddress.at(tc->vid));
991
992
993
994
995
996
997
998
999
1000
    }
  } catch (std::out_of_range &) {
    throw NoSuchTape("In OStoreDB::queue(RetrieveToFile): tape not found");
  }
  // We now need to select the tape from which we will migrate next. This should
  // be the tape with the most jobs already queued.
  // TODO: this will have to look at tape statuses on the long run as well
  uint16_t selectedCopyNumber;
  uint64_t bestTapeQueuedBytes;
  std::string selectedVid;
For faster browsing, not all history is shown. View entire blame