MigrationReportPackerTest.cpp 18.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
/******************************************************************************
 *
 * This file is part of the Castor project.
 * See http://castor.web.cern.ch/castor
 *
 * Copyright (C) 2003  CERN
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * 
 *
 * @author Castor Dev team, castor-dev@cern.ch
 *****************************************************************************/
23

24
#include "common/log/DummyLogger.hpp"
Victor Kotlyar's avatar
Victor Kotlyar committed
25
#include "common/log/StringLogger.hpp"
26
#include "castor/tape/tapeserver/daemon/MigrationReportPacker.hpp"
27
#include "castor/tape/tapeserver/drive/DriveInterface.hpp"
28
#include "catalogue/CatalogueFactoryFactory.hpp"
29
#include "scheduler/testingMocks/MockArchiveMount.hpp"
30

31
32
#include <gtest/gtest.h>

33
34
35
36
using ::testing::_;
using ::testing::Invoke;
using namespace castor::tape;

37
namespace unitTests {
38
39
40
41
42

const uint32_t TEST_USER_1  = 9751;
const uint32_t TEST_GROUP_1 = 9752;
const uint32_t TEST_USER_2  = 9753;
const uint32_t TEST_GROUP_2 = 9754;
43

44
  class castor_tape_tapeserver_daemon_MigrationReportPackerTest: public ::testing::Test {
45
46
  public:
    castor_tape_tapeserver_daemon_MigrationReportPackerTest():
47
      m_dummyLog("dummy", "dummy") {
48
49
    }

50
51
52
  protected:

    void SetUp() {
53
      using namespace cta;
Steven Murray's avatar
Steven Murray committed
54
55
      using namespace cta::catalogue;

56
      rdbms::Login catalogueLogin(rdbms::Login::DBTYPE_IN_MEMORY, "", "", "", "", 0);
57
      const uint64_t nbConns = 1;
58
      const uint64_t nbArchiveFileListingConns = 0;
59
60
61
      auto catalogueFactory = CatalogueFactoryFactory::create(m_dummyLog, catalogueLogin, nbConns,
        nbArchiveFileListingConns);
      m_catalogue = catalogueFactory->create();
62
    }
63
64
65
66
67
68
69
70
71
72
    
    void createMediaType(const std::string & name){
      cta::common::dataStructures::SecurityIdentity admin = cta::common::dataStructures::SecurityIdentity("admin","localhost");
      cta::catalogue::MediaType mediaType;
      mediaType.name = name;
      mediaType.capacityInBytes = 10;
      mediaType.cartridge = "cartridge";
      mediaType.comment = "comment";
      m_catalogue->createMediaType(admin,mediaType);
    }
73
74

    void TearDown() {
Steven Murray's avatar
Steven Murray committed
75
      m_catalogue.reset();
76
    }
77

78
    cta::log::DummyLogger m_dummyLog;
Steven Murray's avatar
Steven Murray committed
79
80
    std::unique_ptr<cta::catalogue::Catalogue> m_catalogue;

81
  }; // class castor_tape_tapeserver_daemon_MigrationReportPackerTest
82

83
84
  class MockArchiveJobExternalStats: public cta::MockArchiveJob {
  public:
85
    MockArchiveJobExternalStats(cta::ArchiveMount & am, cta::catalogue::Catalogue & catalogue, 
86
       int & completes, int &failures):
Eric Cano's avatar
Eric Cano committed
87
    MockArchiveJob(&am, catalogue), completesRef(completes), failuresRef(failures) {}
88

89
    virtual void validate() override {}
90
91
92
    virtual cta::catalogue::TapeItemWrittenPointer validateAndGetTapeFileWritten() override {
      auto fileReportUP=cta::make_unique<cta::catalogue::TapeFileWritten>();
      auto & fileReport = *fileReportUP;
93
94
      fileReport.archiveFileId = archiveFile.archiveFileID;
      fileReport.blockId = tapeFile.blockId;
95
      fileReport.checksumBlob = tapeFile.checksumBlob;
96
97
      fileReport.copyNb = tapeFile.copyNb;
      fileReport.diskFileId = archiveFile.diskFileId;
98
99
      fileReport.diskFileOwnerUid = archiveFile.diskFileInfo.owner_uid;
      fileReport.diskFileGid = archiveFile.diskFileInfo.gid;
100
101
102
103
104
105
      fileReport.diskInstance = archiveFile.diskInstance;
      fileReport.fSeq = tapeFile.fSeq;
      fileReport.size = archiveFile.fileSize;
      fileReport.storageClassName = archiveFile.storageClass;
      fileReport.tapeDrive = std::string("testDrive");
      fileReport.vid = tapeFile.vid;
106
      return cta::catalogue::TapeItemWrittenPointer(fileReportUP.release());
107
    }
108

109

110
    void transferFailed(const std::string& failureReason, cta::log::LogContext& lc) override {
111
112
      failuresRef++;
    }
113

114
115
116
117
    void reportJobSucceeded() override {
      completesRef++;
    }

118
119
120
121
  private:
    int & completesRef;
    int & failuresRef;
  };
122

123
  TEST_F(castor_tape_tapeserver_daemon_MigrationReportPackerTest, MigrationReportPackerNominal) {
Steven Murray's avatar
Steven Murray committed
124
    cta::MockArchiveMount tam(*m_catalogue);
125

126
127
    const std::string vid1 = "VTEST001";
    const std::string vid2 = "VTEST002";
128
129
    const std::string mediaType = "media_type";
    const std::string vendor = "vendor";
130
    const std::string logicalLibraryName = "logical_library_name";
131
    const bool logicalLibraryIsDisabled = false;
132
    const std::string tapePoolName = "tape_pool_name";
133
    const cta::optional<std::string> supply("value for the supply pool mechanism");
134
135
    const bool disabledValue = true;
    const bool fullValue = false;
136
    const bool readOnlyValue = false;
137
    const std::string createTapeComment = "Create tape";
138
139
140
141
    cta::common::dataStructures::VirtualOrganization vo;
    vo.name = "vo";
    vo.comment = "comment";
    
142
    cta::common::dataStructures::SecurityIdentity admin = cta::common::dataStructures::SecurityIdentity("admin","localhost");
143
144
    m_catalogue->createVirtualOrganization(admin,vo);
    
145
    m_catalogue->createLogicalLibrary(admin, logicalLibraryName, logicalLibraryIsDisabled, "Create logical library");
146
    m_catalogue->createTapePool(admin, tapePoolName, vo.name, 2, true, supply, "Create tape pool");
147
    createMediaType(mediaType);
148
149

    {
150
      cta::catalogue::CreateTapeAttributes tape;
151
152
153
154
155
156
157
158
159
160
161
      tape.vid = vid1;
      tape.mediaType = mediaType;
      tape.vendor = vendor;
      tape.logicalLibraryName = logicalLibraryName;
      tape.tapePoolName = tapePoolName;
      tape.full = fullValue;
      tape.disabled = disabledValue;
      tape.readOnly = readOnlyValue;
      tape.comment = createTapeComment;
      m_catalogue->createTape(admin, tape);
    }
162
163

    cta::common::dataStructures::StorageClass storageClass;
164
    
165
166
    storageClass.name = "storage_class";
    storageClass.nbCopies = 1;
167
    storageClass.vo.name = vo.name;
168
169
    storageClass.comment = "Create storage class";
    m_catalogue->createStorageClass(admin, storageClass);
170

171
172
    ::testing::InSequence dummy;
    std::unique_ptr<cta::ArchiveJob> job1;
173
    int job1completes(0), job1failures(0);
174
    {
175
      std::unique_ptr<cta::MockArchiveJob> mockJob(
Steven Murray's avatar
Steven Murray committed
176
        new MockArchiveJobExternalStats(tam, *m_catalogue, job1completes, job1failures));
177
      job1.reset(mockJob.release());
178
    }
179
180
181
182
    job1->archiveFile.archiveFileID=1;
    job1->archiveFile.diskInstance="disk_instance";
    job1->archiveFile.diskFileId="diskFileId1";
    job1->archiveFile.diskFileInfo.path="filePath1";
183
184
    job1->archiveFile.diskFileInfo.owner_uid=TEST_USER_1;
    job1->archiveFile.diskFileInfo.gid=TEST_GROUP_1;
185
    job1->archiveFile.fileSize=1024;        
186
    job1->archiveFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
187
188
189
190
    job1->archiveFile.storageClass="storage_class";
    job1->tapeFile.vid="VTEST001";
    job1->tapeFile.fSeq=1;
    job1->tapeFile.blockId=256;
191
    job1->tapeFile.fileSize=768;
192
    job1->tapeFile.copyNb=1;
193
    job1->tapeFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
194

195
    std::unique_ptr<cta::ArchiveJob> job2;
196
    int job2completes(0), job2failures(0);
197
    {
198
      std::unique_ptr<cta::MockArchiveJob> mockJob(
Steven Murray's avatar
Steven Murray committed
199
        new MockArchiveJobExternalStats(tam, *m_catalogue, job2completes, job2failures));
200
      job2.reset(mockJob.release());
201
    }
202
203
204
205
    job2->archiveFile.archiveFileID=2;
    job2->archiveFile.diskInstance="disk_instance";
    job2->archiveFile.diskFileId="diskFileId2";
    job2->archiveFile.diskFileInfo.path="filePath2";
206
207
    job2->archiveFile.diskFileInfo.owner_uid=TEST_USER_2;
    job2->archiveFile.diskFileInfo.gid=TEST_GROUP_2;
208
    job2->archiveFile.fileSize=1024;        
209
    job2->archiveFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
210
211
212
213
    job2->archiveFile.storageClass="storage_class";
    job2->tapeFile.vid="VTEST001";
    job2->tapeFile.fSeq=2;
    job2->tapeFile.blockId=512;
214
    job2->tapeFile.fileSize=768;
215
    job2->tapeFile.copyNb=1;
216
    job2->tapeFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
217

218
    cta::log::StringLogger log("dummy","castor_tape_tapeserver_daemon_MigrationReportPackerNominal",cta::log::DEBUG);
Victor Kotlyar's avatar
Victor Kotlyar committed
219
    cta::log::LogContext lc(log);
220
221
222
    tapeserver::daemon::MigrationReportPacker mrp(&tam,lc);
    mrp.startThreads();

223
224
    mrp.reportCompletedJob(std::move(job1), lc);
    mrp.reportCompletedJob(std::move(job2), lc);
225
226

    const tapeserver::drive::compressionStats statsCompress;
227
228
229
    mrp.reportFlush(statsCompress, lc);
    mrp.reportEndOfSession(lc);
    mrp.reportTestGoingToEnd(lc);
230
231
232
233
    mrp.waitThread(); //here

    std::string temp = log.getLog();
    ASSERT_NE(std::string::npos, temp.find("Reported to the client that a batch of files was written on tape"));
234
235
236
    ASSERT_EQ(1, tam.completes);
    ASSERT_EQ(1, job1completes);
    ASSERT_EQ(1, job2completes);
237
238
  }

239
  TEST_F(castor_tape_tapeserver_daemon_MigrationReportPackerTest, MigrationReportPackerFailure) {
Steven Murray's avatar
Steven Murray committed
240
    cta::MockArchiveMount tam(*m_catalogue);
241
242
243
244

    ::testing::InSequence dummy;
    std::unique_ptr<cta::ArchiveJob> job1;
    {
Eric Cano's avatar
Eric Cano committed
245
      std::unique_ptr<cta::MockArchiveJob> mockJob(new cta::MockArchiveJob(&tam, *m_catalogue));
246
      job1.reset(mockJob.release());
247
    }
248
249
    std::unique_ptr<cta::ArchiveJob> job2;
    {
Eric Cano's avatar
Eric Cano committed
250
      std::unique_ptr<cta::MockArchiveJob> mockJob(new cta::MockArchiveJob(&tam, *m_catalogue));
251
      job2.reset(mockJob.release());
252
    }
253
    std::unique_ptr<cta::ArchiveJob> job3;
254
    int job3completes(0), job3failures(0);
255
    {
256
      std::unique_ptr<cta::MockArchiveJob> mockJob(
Steven Murray's avatar
Steven Murray committed
257
        new MockArchiveJobExternalStats(tam, *m_catalogue, job3completes, job3failures));
258
259
      job3.reset(mockJob.release());
    }
260

261
    cta::log::StringLogger log("dummy","castor_tape_tapeserver_daemon_MigrationReportPackerFailure",cta::log::DEBUG);
Victor Kotlyar's avatar
Victor Kotlyar committed
262
    cta::log::LogContext lc(log);  
263
264
    tapeserver::daemon::MigrationReportPacker mrp(&tam,lc);
    mrp.startThreads();
265

266
267
    mrp.reportCompletedJob(std::move(job1), lc);
    mrp.reportCompletedJob(std::move(job2), lc);
268
269

    const std::string error_msg = "ERROR_TEST_MSG";
270
    const cta::exception::Exception ex(error_msg);
271
    mrp.reportFailedJob(std::move(job3),ex, lc);
272
273

    const tapeserver::drive::compressionStats statsCompress;
274
275
276
    mrp.reportFlush(statsCompress, lc);
    mrp.reportEndOfSession(lc);
    mrp.reportTestGoingToEnd(lc);
277
278
279
280
    mrp.waitThread();

    std::string temp = log.getLog();
    ASSERT_NE(std::string::npos, temp.find(error_msg));
281
282
    ASSERT_EQ(1, tam.completes);
    ASSERT_EQ(1, job3failures);
283
284
  }

285
  TEST_F(castor_tape_tapeserver_daemon_MigrationReportPackerTest, MigrationReportPackerBadFile) {
Steven Murray's avatar
Steven Murray committed
286
    cta::MockArchiveMount tam(*m_catalogue);
287

288
289
    const std::string vid1 = "VTEST001";
    const std::string vid2 = "VTEST002";
290
291
    const std::string mediaType = "media_type";
    const std::string vendor = "vendor";
292
    const std::string logicalLibraryName = "logical_library_name";
293
    const bool logicalLibraryIsDisabled = false;
294
    const std::string tapePoolName = "tape_pool_name";
295
296
297
    const uint64_t nbPartialTapes = 2;
    const bool isEncrypted = true;
    const cta::optional<std::string> supply("value for the supply pool mechanism");
298
299
    const bool disabledValue = true;
    const bool fullValue = false;
300
    const bool readOnlyValue = false;
301
302
    const std::string createTapeComment = "Create tape";
    cta::common::dataStructures::SecurityIdentity admin = cta::common::dataStructures::SecurityIdentity("admin","localhost");
303

304
305
306
307
308
309
    cta::common::dataStructures::VirtualOrganization vo;
    vo.name = "vo";
    vo.comment = "comment";
    
    m_catalogue->createVirtualOrganization(admin,vo);
    
310
    m_catalogue->createLogicalLibrary(admin, logicalLibraryName, logicalLibraryIsDisabled, "Create logical library");
311
    m_catalogue->createTapePool(admin, tapePoolName, vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
312
    createMediaType(mediaType);
313
314

    {
315
      cta::catalogue::CreateTapeAttributes tape;
316
317
318
319
320
321
322
323
324
325
326
      tape.vid = vid1;
      tape.mediaType = mediaType;
      tape.vendor = vendor;
      tape.logicalLibraryName = logicalLibraryName;
      tape.tapePoolName = tapePoolName;
      tape.full = fullValue;
      tape.disabled = disabledValue;
      tape.readOnly = readOnlyValue;
      tape.comment = createTapeComment;
      m_catalogue->createTape(admin, tape);
    }
327
328

    cta::common::dataStructures::StorageClass storageClass;
329
    
330
331
    storageClass.name = "storage_class";
    storageClass.nbCopies = 1;
332
    storageClass.vo.name = vo.name;
333
334
    storageClass.comment = "Create storage class";
    m_catalogue->createStorageClass(admin, storageClass);
335

336
337
    ::testing::InSequence dummy;
    std::unique_ptr<cta::ArchiveJob> migratedBigFile;
338
    int migratedBigFileCompletes(0), migratedBigFileFailures(0);
339
    {
340
      std::unique_ptr<cta::MockArchiveJob> mockJob(
Steven Murray's avatar
Steven Murray committed
341
        new MockArchiveJobExternalStats(tam, *m_catalogue, migratedBigFileCompletes, migratedBigFileFailures));
342
343
344
      migratedBigFile.reset(mockJob.release());
    }
    std::unique_ptr<cta::ArchiveJob> migratedFileSmall;
345
    int migratedFileSmallCompletes(0), migratedFileSmallFailures(0);
346
    {
347
      std::unique_ptr<cta::MockArchiveJob> mockJob(
Steven Murray's avatar
Steven Murray committed
348
        new MockArchiveJobExternalStats(tam, *m_catalogue, migratedFileSmallCompletes, migratedFileSmallFailures));
349
350
351
      migratedFileSmall.reset(mockJob.release());
    }
    std::unique_ptr<cta::ArchiveJob> migratedNullFile;
352
    int migratedNullFileCompletes(0), migratedNullFileFailures(0);
353
    {
354
      std::unique_ptr<cta::MockArchiveJob> mockJob(
Steven Murray's avatar
Steven Murray committed
355
        new MockArchiveJobExternalStats(tam, *m_catalogue, migratedNullFileCompletes, migratedNullFileFailures));
356
357
358
      migratedNullFile.reset(mockJob.release());
    }

359
360
361
362
    migratedBigFile->archiveFile.archiveFileID=4;
    migratedBigFile->archiveFile.diskInstance="disk_instance";
    migratedBigFile->archiveFile.diskFileId="diskFileId2";
    migratedBigFile->archiveFile.diskFileInfo.path="filePath2";
363
364
    migratedBigFile->archiveFile.diskFileInfo.owner_uid=TEST_USER_2;
    migratedBigFile->archiveFile.diskFileInfo.gid=TEST_GROUP_2;
365
    migratedBigFile->archiveFile.fileSize=100000;        
366
    migratedBigFile->archiveFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
367
368
369
370
    migratedBigFile->archiveFile.storageClass="storage_class";
    migratedBigFile->tapeFile.vid="VTEST001";
    migratedBigFile->tapeFile.fSeq=1;
    migratedBigFile->tapeFile.blockId=256;
371
    migratedBigFile->tapeFile.fileSize=768;
372
    migratedBigFile->tapeFile.copyNb=1;
373
    migratedBigFile->tapeFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
374

375
376
377
378
    migratedFileSmall->archiveFile.archiveFileID=5;
    migratedFileSmall->archiveFile.diskInstance="disk_instance";
    migratedFileSmall->archiveFile.diskFileId="diskFileId3";
    migratedFileSmall->archiveFile.diskFileInfo.path="filePath3";
379
380
    migratedFileSmall->archiveFile.diskFileInfo.owner_uid=TEST_USER_2;
    migratedFileSmall->archiveFile.diskFileInfo.gid=TEST_GROUP_2;
381
    migratedFileSmall->archiveFile.fileSize=1;        
382
    migratedFileSmall->archiveFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
383
384
385
386
    migratedFileSmall->archiveFile.storageClass="storage_class";
    migratedFileSmall->tapeFile.vid="VTEST001";
    migratedFileSmall->tapeFile.fSeq=2;
    migratedFileSmall->tapeFile.blockId=512;
387
    migratedFileSmall->tapeFile.fileSize=1;
388
    migratedFileSmall->tapeFile.copyNb=1;
389
    migratedFileSmall->tapeFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
390

391
392
393
394
    migratedNullFile->archiveFile.archiveFileID=6;
    migratedNullFile->archiveFile.diskInstance="disk_instance";
    migratedNullFile->archiveFile.diskFileId="diskFileId4";
    migratedNullFile->archiveFile.diskFileInfo.path="filePath4";
395
396
    migratedNullFile->archiveFile.diskFileInfo.owner_uid=TEST_USER_2;
    migratedNullFile->archiveFile.diskFileInfo.gid=TEST_GROUP_2;
397
    migratedNullFile->archiveFile.fileSize=0;        
398
    migratedNullFile->archiveFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
399
400
401
402
    migratedNullFile->archiveFile.storageClass="storage_class";
    migratedNullFile->tapeFile.vid="VTEST001";
    migratedNullFile->tapeFile.fSeq=3;
    migratedNullFile->tapeFile.blockId=768;
403
    migratedNullFile->tapeFile.fileSize=0;
404
    migratedNullFile->tapeFile.copyNb=1;
405
    migratedNullFile->tapeFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
406

407
    cta::log::StringLogger log("dummy","castor_tape_tapeserver_daemon_MigrationReportPackerOneByteFile",cta::log::DEBUG);
Victor Kotlyar's avatar
Victor Kotlyar committed
408
    cta::log::LogContext lc(log);  
409
410
411
    tapeserver::daemon::MigrationReportPacker mrp(&tam,lc);
    mrp.startThreads();

412
413
414
    mrp.reportCompletedJob(std::move(migratedBigFile), lc);
    mrp.reportCompletedJob(std::move(migratedFileSmall), lc);
    mrp.reportCompletedJob(std::move(migratedNullFile), lc);
415
416
    tapeserver::drive::compressionStats stats;
    stats.toTape=(100000+1)/3;
417
418
419
    mrp.reportFlush(stats, lc);
    mrp.reportEndOfSession(lc);
    mrp.reportTestGoingToEnd(lc);
420
421
422
    mrp.waitThread();

    std::string temp = log.getLog();
423
    ASSERT_NE(std::string::npos, temp.find("TapeFileWrittenEvent is invalid"));
Eric Cano's avatar
Eric Cano committed
424
    ASSERT_NE(std::string::npos, temp.find("Received a CTA exception while reporting archive mount results"));
425
426
427
428
    ASSERT_EQ(0, tam.completes);
    ASSERT_EQ(0, migratedBigFileCompletes);
    ASSERT_EQ(0, migratedFileSmallCompletes);
    ASSERT_EQ(0, migratedNullFileCompletes);
429
  } 
430
}