MigrationReportPackerTest.cpp 18 KB
Newer Older
1
2
/*
 * @project        The CERN Tape Archive (CTA)
3
 * @copyright      Copyright(C) 2003-2021 CERN
4
5
6
7
 * @license        This program is free software: you can redistribute it and/or modify
 *                 it under the terms of the GNU General Public License as published by
 *                 the Free Software Foundation, either version 3 of the License, or
 *                 (at your option) any later version.
8
 *
9
10
11
12
 *                 This program is distributed in the hope that it will be useful,
 *                 but WITHOUT ANY WARRANTY; without even the implied warranty of
 *                 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *                 GNU General Public License for more details.
13
 *
14
15
16
 *                 You should have received a copy of the GNU General Public License
 *                 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
17

18
#include "common/log/DummyLogger.hpp"
Victor Kotlyar's avatar
Victor Kotlyar committed
19
#include "common/log/StringLogger.hpp"
20
#include "castor/tape/tapeserver/daemon/MigrationReportPacker.hpp"
21
#include "castor/tape/tapeserver/drive/DriveInterface.hpp"
22
#include "catalogue/CatalogueFactoryFactory.hpp"
23
#include "scheduler/testingMocks/MockArchiveMount.hpp"
24

25
26
#include <gtest/gtest.h>

27
28
29
30
using ::testing::_;
using ::testing::Invoke;
using namespace castor::tape;

31
namespace unitTests {
32
33
34
35
36

const uint32_t TEST_USER_1  = 9751;
const uint32_t TEST_GROUP_1 = 9752;
const uint32_t TEST_USER_2  = 9753;
const uint32_t TEST_GROUP_2 = 9754;
37

38
  class castor_tape_tapeserver_daemon_MigrationReportPackerTest: public ::testing::Test {
39
40
  public:
    castor_tape_tapeserver_daemon_MigrationReportPackerTest():
41
      m_dummyLog("dummy", "dummy") {
42
43
    }

44
45
46
  protected:

    void SetUp() {
47
      using namespace cta;
Steven Murray's avatar
Steven Murray committed
48
49
      using namespace cta::catalogue;

50
      rdbms::Login catalogueLogin(rdbms::Login::DBTYPE_IN_MEMORY, "", "", "", "", 0);
51
      const uint64_t nbConns = 1;
52
      const uint64_t nbArchiveFileListingConns = 0;
53
54
55
      auto catalogueFactory = CatalogueFactoryFactory::create(m_dummyLog, catalogueLogin, nbConns,
        nbArchiveFileListingConns);
      m_catalogue = catalogueFactory->create();
56
    }
57
58
59
60
61
62
63
64
65
66
    
    void createMediaType(const std::string & name){
      cta::common::dataStructures::SecurityIdentity admin = cta::common::dataStructures::SecurityIdentity("admin","localhost");
      cta::catalogue::MediaType mediaType;
      mediaType.name = name;
      mediaType.capacityInBytes = 10;
      mediaType.cartridge = "cartridge";
      mediaType.comment = "comment";
      m_catalogue->createMediaType(admin,mediaType);
    }
67
68
69
70
71
72
73
74
75
    
    cta::common::dataStructures::VirtualOrganization getDefaultVo(){
      cta::common::dataStructures::VirtualOrganization vo;
      vo.name = "vo";
      vo.readMaxDrives = 1;
      vo.writeMaxDrives = 1;
      vo.comment = "comment";
      return vo;
    }
76
77

    void TearDown() {
Steven Murray's avatar
Steven Murray committed
78
      m_catalogue.reset();
79
    }
80

81
    cta::log::DummyLogger m_dummyLog;
Steven Murray's avatar
Steven Murray committed
82
83
    std::unique_ptr<cta::catalogue::Catalogue> m_catalogue;

84
  }; // class castor_tape_tapeserver_daemon_MigrationReportPackerTest
85

86
87
  class MockArchiveJobExternalStats: public cta::MockArchiveJob {
  public:
88
    MockArchiveJobExternalStats(cta::ArchiveMount & am, cta::catalogue::Catalogue & catalogue, 
89
       int & completes, int &failures):
Eric Cano's avatar
Eric Cano committed
90
    MockArchiveJob(&am, catalogue), completesRef(completes), failuresRef(failures) {}
91

92
    virtual void validate() override {}
93
94
95
    virtual cta::catalogue::TapeItemWrittenPointer validateAndGetTapeFileWritten() override {
      auto fileReportUP=cta::make_unique<cta::catalogue::TapeFileWritten>();
      auto & fileReport = *fileReportUP;
96
97
      fileReport.archiveFileId = archiveFile.archiveFileID;
      fileReport.blockId = tapeFile.blockId;
98
      fileReport.checksumBlob = tapeFile.checksumBlob;
99
100
      fileReport.copyNb = tapeFile.copyNb;
      fileReport.diskFileId = archiveFile.diskFileId;
101
102
      fileReport.diskFileOwnerUid = archiveFile.diskFileInfo.owner_uid;
      fileReport.diskFileGid = archiveFile.diskFileInfo.gid;
103
104
105
106
107
108
      fileReport.diskInstance = archiveFile.diskInstance;
      fileReport.fSeq = tapeFile.fSeq;
      fileReport.size = archiveFile.fileSize;
      fileReport.storageClassName = archiveFile.storageClass;
      fileReport.tapeDrive = std::string("testDrive");
      fileReport.vid = tapeFile.vid;
109
      return cta::catalogue::TapeItemWrittenPointer(fileReportUP.release());
110
    }
111

112

113
    void transferFailed(const std::string& failureReason, cta::log::LogContext& lc) override {
114
115
      failuresRef++;
    }
116

117
118
119
120
    void reportJobSucceeded() override {
      completesRef++;
    }

121
122
123
124
  private:
    int & completesRef;
    int & failuresRef;
  };
125

126
  TEST_F(castor_tape_tapeserver_daemon_MigrationReportPackerTest, MigrationReportPackerNominal) {
Steven Murray's avatar
Steven Murray committed
127
    cta::MockArchiveMount tam(*m_catalogue);
128

129
130
    const std::string vid1 = "VTEST001";
    const std::string vid2 = "VTEST002";
131
132
    const std::string mediaType = "media_type";
    const std::string vendor = "vendor";
133
    const std::string logicalLibraryName = "logical_library_name";
134
    const bool logicalLibraryIsDisabled = false;
135
    const std::string tapePoolName = "tape_pool_name";
136
    const cta::optional<std::string> supply("value for the supply pool mechanism");
137
138
    const bool fullValue = false;
    const std::string createTapeComment = "Create tape";
139
    cta::common::dataStructures::VirtualOrganization vo = getDefaultVo();
140
    
141
    cta::common::dataStructures::SecurityIdentity admin = cta::common::dataStructures::SecurityIdentity("admin","localhost");
142
143
    m_catalogue->createVirtualOrganization(admin,vo);
    
144
    m_catalogue->createLogicalLibrary(admin, logicalLibraryName, logicalLibraryIsDisabled, "Create logical library");
145
    m_catalogue->createTapePool(admin, tapePoolName, vo.name, 2, true, supply, "Create tape pool");
146
    createMediaType(mediaType);
147
148

    {
149
      cta::catalogue::CreateTapeAttributes tape;
150
151
152
153
154
155
156
      tape.vid = vid1;
      tape.mediaType = mediaType;
      tape.vendor = vendor;
      tape.logicalLibraryName = logicalLibraryName;
      tape.tapePoolName = tapePoolName;
      tape.full = fullValue;
      tape.comment = createTapeComment;
157
158
      tape.state = cta::common::dataStructures::Tape::DISABLED;
      tape.stateReason = "Test";
159
160
      m_catalogue->createTape(admin, tape);
    }
161
162

    cta::common::dataStructures::StorageClass storageClass;
163
    
164
165
    storageClass.name = "storage_class";
    storageClass.nbCopies = 1;
166
    storageClass.vo.name = vo.name;
167
168
    storageClass.comment = "Create storage class";
    m_catalogue->createStorageClass(admin, storageClass);
169

170
171
    ::testing::InSequence dummy;
    std::unique_ptr<cta::ArchiveJob> job1;
172
    int job1completes(0), job1failures(0);
173
    {
174
      std::unique_ptr<cta::MockArchiveJob> mockJob(
Steven Murray's avatar
Steven Murray committed
175
        new MockArchiveJobExternalStats(tam, *m_catalogue, job1completes, job1failures));
176
      job1.reset(mockJob.release());
177
    }
178
179
180
181
    job1->archiveFile.archiveFileID=1;
    job1->archiveFile.diskInstance="disk_instance";
    job1->archiveFile.diskFileId="diskFileId1";
    job1->archiveFile.diskFileInfo.path="filePath1";
182
183
    job1->archiveFile.diskFileInfo.owner_uid=TEST_USER_1;
    job1->archiveFile.diskFileInfo.gid=TEST_GROUP_1;
184
    job1->archiveFile.fileSize=1024;        
185
    job1->archiveFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
186
187
188
189
    job1->archiveFile.storageClass="storage_class";
    job1->tapeFile.vid="VTEST001";
    job1->tapeFile.fSeq=1;
    job1->tapeFile.blockId=256;
190
    job1->tapeFile.fileSize=768;
191
    job1->tapeFile.copyNb=1;
192
    job1->tapeFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
193

194
    std::unique_ptr<cta::ArchiveJob> job2;
195
    int job2completes(0), job2failures(0);
196
    {
197
      std::unique_ptr<cta::MockArchiveJob> mockJob(
Steven Murray's avatar
Steven Murray committed
198
        new MockArchiveJobExternalStats(tam, *m_catalogue, job2completes, job2failures));
199
      job2.reset(mockJob.release());
200
    }
201
202
203
204
    job2->archiveFile.archiveFileID=2;
    job2->archiveFile.diskInstance="disk_instance";
    job2->archiveFile.diskFileId="diskFileId2";
    job2->archiveFile.diskFileInfo.path="filePath2";
205
206
    job2->archiveFile.diskFileInfo.owner_uid=TEST_USER_2;
    job2->archiveFile.diskFileInfo.gid=TEST_GROUP_2;
207
    job2->archiveFile.fileSize=1024;        
208
    job2->archiveFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
209
210
211
212
    job2->archiveFile.storageClass="storage_class";
    job2->tapeFile.vid="VTEST001";
    job2->tapeFile.fSeq=2;
    job2->tapeFile.blockId=512;
213
    job2->tapeFile.fileSize=768;
214
    job2->tapeFile.copyNb=1;
215
    job2->tapeFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
216

217
    cta::log::StringLogger log("dummy","castor_tape_tapeserver_daemon_MigrationReportPackerNominal",cta::log::DEBUG);
Victor Kotlyar's avatar
Victor Kotlyar committed
218
    cta::log::LogContext lc(log);
219
220
221
    tapeserver::daemon::MigrationReportPacker mrp(&tam,lc);
    mrp.startThreads();

222
223
    mrp.reportCompletedJob(std::move(job1), lc);
    mrp.reportCompletedJob(std::move(job2), lc);
224
225

    const tapeserver::drive::compressionStats statsCompress;
226
227
228
    mrp.reportFlush(statsCompress, lc);
    mrp.reportEndOfSession(lc);
    mrp.reportTestGoingToEnd(lc);
229
230
231
232
    mrp.waitThread(); //here

    std::string temp = log.getLog();
    ASSERT_NE(std::string::npos, temp.find("Reported to the client that a batch of files was written on tape"));
233
234
235
    ASSERT_EQ(1, tam.completes);
    ASSERT_EQ(1, job1completes);
    ASSERT_EQ(1, job2completes);
236
237
  }

238
  TEST_F(castor_tape_tapeserver_daemon_MigrationReportPackerTest, MigrationReportPackerFailure) {
Steven Murray's avatar
Steven Murray committed
239
    cta::MockArchiveMount tam(*m_catalogue);
240
241
242
243

    ::testing::InSequence dummy;
    std::unique_ptr<cta::ArchiveJob> job1;
    {
Eric Cano's avatar
Eric Cano committed
244
      std::unique_ptr<cta::MockArchiveJob> mockJob(new cta::MockArchiveJob(&tam, *m_catalogue));
245
      job1.reset(mockJob.release());
246
    }
247
248
    std::unique_ptr<cta::ArchiveJob> job2;
    {
Eric Cano's avatar
Eric Cano committed
249
      std::unique_ptr<cta::MockArchiveJob> mockJob(new cta::MockArchiveJob(&tam, *m_catalogue));
250
      job2.reset(mockJob.release());
251
    }
252
    std::unique_ptr<cta::ArchiveJob> job3;
253
    int job3completes(0), job3failures(0);
254
    {
255
      std::unique_ptr<cta::MockArchiveJob> mockJob(
Steven Murray's avatar
Steven Murray committed
256
        new MockArchiveJobExternalStats(tam, *m_catalogue, job3completes, job3failures));
257
258
      job3.reset(mockJob.release());
    }
259

260
    cta::log::StringLogger log("dummy","castor_tape_tapeserver_daemon_MigrationReportPackerFailure",cta::log::DEBUG);
Victor Kotlyar's avatar
Victor Kotlyar committed
261
    cta::log::LogContext lc(log);  
262
263
    tapeserver::daemon::MigrationReportPacker mrp(&tam,lc);
    mrp.startThreads();
264

265
266
    mrp.reportCompletedJob(std::move(job1), lc);
    mrp.reportCompletedJob(std::move(job2), lc);
267
268

    const std::string error_msg = "ERROR_TEST_MSG";
269
    const cta::exception::Exception ex(error_msg);
270
    mrp.reportFailedJob(std::move(job3),ex, lc);
271
272

    const tapeserver::drive::compressionStats statsCompress;
273
274
275
    mrp.reportFlush(statsCompress, lc);
    mrp.reportEndOfSession(lc);
    mrp.reportTestGoingToEnd(lc);
276
277
278
279
    mrp.waitThread();

    std::string temp = log.getLog();
    ASSERT_NE(std::string::npos, temp.find(error_msg));
280
281
    ASSERT_EQ(1, tam.completes);
    ASSERT_EQ(1, job3failures);
282
283
  }

284
  TEST_F(castor_tape_tapeserver_daemon_MigrationReportPackerTest, MigrationReportPackerBadFile) {
Steven Murray's avatar
Steven Murray committed
285
    cta::MockArchiveMount tam(*m_catalogue);
286

287
288
    const std::string vid1 = "VTEST001";
    const std::string vid2 = "VTEST002";
289
290
    const std::string mediaType = "media_type";
    const std::string vendor = "vendor";
291
    const std::string logicalLibraryName = "logical_library_name";
292
    const bool logicalLibraryIsDisabled = false;
293
    const std::string tapePoolName = "tape_pool_name";
294
295
296
    const uint64_t nbPartialTapes = 2;
    const bool isEncrypted = true;
    const cta::optional<std::string> supply("value for the supply pool mechanism");
297
298
299
    const bool fullValue = false;
    const std::string createTapeComment = "Create tape";
    cta::common::dataStructures::SecurityIdentity admin = cta::common::dataStructures::SecurityIdentity("admin","localhost");
300

301
    cta::common::dataStructures::VirtualOrganization vo = getDefaultVo();
302
303
304
    
    m_catalogue->createVirtualOrganization(admin,vo);
    
305
    m_catalogue->createLogicalLibrary(admin, logicalLibraryName, logicalLibraryIsDisabled, "Create logical library");
306
    m_catalogue->createTapePool(admin, tapePoolName, vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
307
    createMediaType(mediaType);
308
309

    {
310
      cta::catalogue::CreateTapeAttributes tape;
311
312
313
314
315
316
317
      tape.vid = vid1;
      tape.mediaType = mediaType;
      tape.vendor = vendor;
      tape.logicalLibraryName = logicalLibraryName;
      tape.tapePoolName = tapePoolName;
      tape.full = fullValue;
      tape.comment = createTapeComment;
318
319
      tape.state = cta::common::dataStructures::Tape::DISABLED;
      tape.stateReason = "test";
320
321
      m_catalogue->createTape(admin, tape);
    }
322
323

    cta::common::dataStructures::StorageClass storageClass;
324
    
325
326
    storageClass.name = "storage_class";
    storageClass.nbCopies = 1;
327
    storageClass.vo.name = vo.name;
328
329
    storageClass.comment = "Create storage class";
    m_catalogue->createStorageClass(admin, storageClass);
330

331
332
    ::testing::InSequence dummy;
    std::unique_ptr<cta::ArchiveJob> migratedBigFile;
333
    int migratedBigFileCompletes(0), migratedBigFileFailures(0);
334
    {
335
      std::unique_ptr<cta::MockArchiveJob> mockJob(
Steven Murray's avatar
Steven Murray committed
336
        new MockArchiveJobExternalStats(tam, *m_catalogue, migratedBigFileCompletes, migratedBigFileFailures));
337
338
339
      migratedBigFile.reset(mockJob.release());
    }
    std::unique_ptr<cta::ArchiveJob> migratedFileSmall;
340
    int migratedFileSmallCompletes(0), migratedFileSmallFailures(0);
341
    {
342
      std::unique_ptr<cta::MockArchiveJob> mockJob(
Steven Murray's avatar
Steven Murray committed
343
        new MockArchiveJobExternalStats(tam, *m_catalogue, migratedFileSmallCompletes, migratedFileSmallFailures));
344
345
346
      migratedFileSmall.reset(mockJob.release());
    }
    std::unique_ptr<cta::ArchiveJob> migratedNullFile;
347
    int migratedNullFileCompletes(0), migratedNullFileFailures(0);
348
    {
349
      std::unique_ptr<cta::MockArchiveJob> mockJob(
Steven Murray's avatar
Steven Murray committed
350
        new MockArchiveJobExternalStats(tam, *m_catalogue, migratedNullFileCompletes, migratedNullFileFailures));
351
352
353
      migratedNullFile.reset(mockJob.release());
    }

354
355
356
357
    migratedBigFile->archiveFile.archiveFileID=4;
    migratedBigFile->archiveFile.diskInstance="disk_instance";
    migratedBigFile->archiveFile.diskFileId="diskFileId2";
    migratedBigFile->archiveFile.diskFileInfo.path="filePath2";
358
359
    migratedBigFile->archiveFile.diskFileInfo.owner_uid=TEST_USER_2;
    migratedBigFile->archiveFile.diskFileInfo.gid=TEST_GROUP_2;
360
    migratedBigFile->archiveFile.fileSize=100000;        
361
    migratedBigFile->archiveFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
362
363
364
365
    migratedBigFile->archiveFile.storageClass="storage_class";
    migratedBigFile->tapeFile.vid="VTEST001";
    migratedBigFile->tapeFile.fSeq=1;
    migratedBigFile->tapeFile.blockId=256;
366
    migratedBigFile->tapeFile.fileSize=768;
367
    migratedBigFile->tapeFile.copyNb=1;
368
    migratedBigFile->tapeFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
369

370
371
372
373
    migratedFileSmall->archiveFile.archiveFileID=5;
    migratedFileSmall->archiveFile.diskInstance="disk_instance";
    migratedFileSmall->archiveFile.diskFileId="diskFileId3";
    migratedFileSmall->archiveFile.diskFileInfo.path="filePath3";
374
375
    migratedFileSmall->archiveFile.diskFileInfo.owner_uid=TEST_USER_2;
    migratedFileSmall->archiveFile.diskFileInfo.gid=TEST_GROUP_2;
376
    migratedFileSmall->archiveFile.fileSize=1;        
377
    migratedFileSmall->archiveFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
378
379
380
381
    migratedFileSmall->archiveFile.storageClass="storage_class";
    migratedFileSmall->tapeFile.vid="VTEST001";
    migratedFileSmall->tapeFile.fSeq=2;
    migratedFileSmall->tapeFile.blockId=512;
382
    migratedFileSmall->tapeFile.fileSize=1;
383
    migratedFileSmall->tapeFile.copyNb=1;
384
    migratedFileSmall->tapeFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
385

386
387
388
389
    migratedNullFile->archiveFile.archiveFileID=6;
    migratedNullFile->archiveFile.diskInstance="disk_instance";
    migratedNullFile->archiveFile.diskFileId="diskFileId4";
    migratedNullFile->archiveFile.diskFileInfo.path="filePath4";
390
391
    migratedNullFile->archiveFile.diskFileInfo.owner_uid=TEST_USER_2;
    migratedNullFile->archiveFile.diskFileInfo.gid=TEST_GROUP_2;
392
    migratedNullFile->archiveFile.fileSize=0;        
393
    migratedNullFile->archiveFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
394
395
396
397
    migratedNullFile->archiveFile.storageClass="storage_class";
    migratedNullFile->tapeFile.vid="VTEST001";
    migratedNullFile->tapeFile.fSeq=3;
    migratedNullFile->tapeFile.blockId=768;
398
    migratedNullFile->tapeFile.fileSize=0;
399
    migratedNullFile->tapeFile.copyNb=1;
400
    migratedNullFile->tapeFile.checksumBlob.insert(cta::checksum::MD5, cta::checksum::ChecksumBlob::HexToByteArray("b170288bf1f61b26a648358866f4d6c6"));
401

402
    cta::log::StringLogger log("dummy","castor_tape_tapeserver_daemon_MigrationReportPackerOneByteFile",cta::log::DEBUG);
Victor Kotlyar's avatar
Victor Kotlyar committed
403
    cta::log::LogContext lc(log);  
404
405
406
    tapeserver::daemon::MigrationReportPacker mrp(&tam,lc);
    mrp.startThreads();

407
408
409
    mrp.reportCompletedJob(std::move(migratedBigFile), lc);
    mrp.reportCompletedJob(std::move(migratedFileSmall), lc);
    mrp.reportCompletedJob(std::move(migratedNullFile), lc);
410
411
    tapeserver::drive::compressionStats stats;
    stats.toTape=(100000+1)/3;
412
413
414
    mrp.reportFlush(stats, lc);
    mrp.reportEndOfSession(lc);
    mrp.reportTestGoingToEnd(lc);
415
416
417
    mrp.waitThread();

    std::string temp = log.getLog();
418
    ASSERT_NE(std::string::npos, temp.find("TapeFileWrittenEvent is invalid"));
Eric Cano's avatar
Eric Cano committed
419
    ASSERT_NE(std::string::npos, temp.find("Received a CTA exception while reporting archive mount results"));
420
421
422
423
    ASSERT_EQ(0, tam.completes);
    ASSERT_EQ(0, migratedBigFileCompletes);
    ASSERT_EQ(0, migratedFileSmallCompletes);
    ASSERT_EQ(0, migratedNullFileCompletes);
424
  } 
425
}