DataTransferSessionTest.cpp 38 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
/******************************************************************************
 *
 * This file is part of the Castor project.
 * See http://castor.web.cern.ch/castor
 *
 * Copyright (C) 2003  CERN
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * 
 *
 * @author Castor Dev team, castor-dev@cern.ch
 *****************************************************************************/

24
25
26
27
#define __STDC_CONSTANT_MACROS // For using stdint macros (stdint is included
// by inttypes.h, so we shoot first)
#include <stdint.h>
#include <inttypes.h>
28
#include <gtest/gtest.h>
29

30
#include "castor/legacymsg/RmcProxyDummy.hpp"
31
#include "castor/log/StringLogger.hpp"
32
33
34
#include "castor/mediachanger/MediaChangerFacade.hpp"
#include "castor/mediachanger/MmcProxyDummy.hpp"
#include "castor/messages/AcsProxyDummy.hpp"
35
#include "castor/messages/TapeserverProxyDummy.hpp"
36
#include "castor/server/ProcessCapDummy.hpp"
37
#include "castor/server/Threading.hpp"
38
#include "castor/tape/tapeserver/daemon/DataTransferSession.hpp"
39
#include "castor/tape/tapeserver/daemon/VolumeInfo.hpp"
40
41
#include "castor/tape/tapeserver/system/Wrapper.hpp"
#include "castor/tape/tapeserver/file/File.hpp"
42
#include "castor/tape/tapeserver/drive/FakeDrive.hpp"
43
44
#include "common/exception/Exception.hpp"
#include "common/Utils.hpp"
45
#include "Ctape.h"
46
#include "scheduler/Scheduler.hpp"
47
#include "smc_struct.h"
48
49
#include "nameserver/mockNS/MockNameServer.hpp"
#include "remotens/MockRemoteNS.hpp"
50
#include "scheduler/DummyScheduler.hpp"
51
#include "scheduler/mockDB/MockSchedulerDatabase.hpp"
52
#include "scheduler/MountType.hpp"
53

54
#include <dirent.h>
55
#include <fcntl.h>
56
#include <stdexcept>
Eric Cano's avatar
Eric Cano committed
57
58
59
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
60
61
#include <unistd.h>
#include <zlib.h>
Eric Cano's avatar
Eric Cano committed
62

63
using namespace castor::tape::tapeserver;
64
using namespace castor::tape::tapeserver::daemon;
65
66
namespace unitTest {

67
68
class castor_tape_tapeserver_daemon_DataTransferSessionTest: public
  ::testing::Test {
69
70
71
protected:

  void SetUp() {
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
    strncpy(m_tmpDir, "/tmp/DataTransferSessionTestXXXXXX", sizeof(m_tmpDir));
    if(!mkdtemp(m_tmpDir)) {
      const std::string errMsg = cta::Utils::errnoToString(errno);
      std::ostringstream msg;
      msg << "Failed to create directory with template"
        " /tmp/DataTransferSessionTestXXXXXX: " << errMsg;
      bzero(m_tmpDir, sizeof(m_tmpDir));
      throw cta::exception::Exception(msg.str());
    }

    struct stat statBuf;
    bzero(&statBuf, sizeof(statBuf));
    if(stat(m_tmpDir, &statBuf)) {
      const std::string errMsg = cta::Utils::errnoToString(errno);
      std::ostringstream msg;
      msg << "Failed to stat directory " << m_tmpDir << ": " << errMsg;
      throw cta::exception::Exception(msg.str());
    }

    std::ostringstream cmd;
    cmd << "touch " << m_tmpDir << "/hello";
    system(cmd.str().c_str());
94
95
96
  }

  void TearDown() {
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
    // If Setup() created a temporary directory
    if(m_tmpDir) {

      // Openn the directory
      std::unique_ptr<DIR, std::function<int(DIR*)>>
        dir(opendir(m_tmpDir), closedir);
      if(NULL == dir.get()) {
        const std::string errMsg = cta::Utils::errnoToString(errno);
        std::ostringstream msg;
        msg << "Failed to open directory " << m_tmpDir << ": " << errMsg;
        throw cta::exception::Exception(msg.str());
      }

      // Delete each of the files within the directory
      struct dirent *entry = NULL;
      while((entry = readdir(dir.get()))) {
        const std::string entryName(entry->d_name);
        if(entryName != "." && entryName != "..") {
          const std::string entryPath = std::string(m_tmpDir) + "/" + entryName;
          if(unlink(entryPath.c_str())) {
            const std::string errMsg = cta::Utils::errnoToString(errno);
            std::ostringstream msg;
            msg << "Failed to unlink " << entryPath;
            throw cta::exception::Exception(msg.str());
          }
        }
      }

      // Delete the now empty directory
      if(rmdir(m_tmpDir)) {
        const std::string errMsg = cta::Utils::errnoToString(errno);
        std::ostringstream msg;
        msg << "Failed to delete directory " << m_tmpDir << ": " << errMsg;
        throw cta::exception::Exception(msg.str());
      }
    }
133
  }
134

135
136
137
138
139
140
141
  /**
   * Temporary directory created with mkdtemp that will be used to contain the
   * destination remote files of the tests that need to create them.
   *
   * Please note that a new temporary directory is created and deleted for each
   * test by the Setup() and TearDown() methods.
   */
142
143
  char m_tmpDir[100];

144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
  class MockArchiveJob: public cta::ArchiveJob {
  public:
    MockArchiveJob() {
    } 
      
    ~MockArchiveJob() throw() {
    } 
  };

  class MockRetrieveJob: public cta::RetrieveJob {
  public:
    MockRetrieveJob() {
    } 
      
    ~MockRetrieveJob() throw() {
    } 
  };
161
162
};

163
TEST_F(castor_tape_tapeserver_daemon_DataTransferSessionTest, DataTransferSessionGooddayRecall) {
Eric Cano's avatar
Eric Cano committed
164
165
166
  // 0) Prepare the logger for everyone
  castor::log::StringLogger logger("tapeServerUnitTest");
  
167
  // 1) prepare the fake scheduler
168
  std::string vid = "V12345";
169
  // cta::MountType::Enum mountType = cta::MountType::RETRIEVE;
170
  std::string density = "8000GC";
171

172
173
  // 3) Prepare the necessary environment (logger, plus system wrapper), 
  castor::tape::System::mockWrapper mockSys;
174
175
  mockSys.delegateToFake();
  mockSys.disableGMockCallsCounting();
176
  mockSys.fake.setupForVirtualDriveSLC6();
177
178
179
  //delete is unnecessary
  //pointer with ownership will be passed to the application,
  //which will do the delete 
180
  mockSys.fake.m_pathToDrive["/dev/nst0"] = new castor::tape::tapeserver::drive::FakeDrive;
181
182
183
184
185
186
187
188
189
190
191
192
193

  // 4) Create the scheduler
  cta::MockNameServer ns;
  cta::MockRemoteNS rns;
  cta::MockSchedulerDatabase db;
  cta::Scheduler scheduler(ns, db, rns);

  // Always use the same requester
  const cta::SecurityIdentity requester;

  // List to remember the path of each remote file so that the existance of the
  // files can be tested for at the end of the test
  std::list<std::string> remoteFilePaths;
194
  
195
  // 5) Prepare files for reading by writing them to the mock system
196
197
198
  {
    // Label the tape
    castor::tape::tapeFile::LabelSession ls(*mockSys.fake.m_pathToDrive["/dev/nst0"], 
199
        "V12345");
200
201
    mockSys.fake.m_pathToDrive["/dev/nst0"]->rewind();
    // And write to it
202
    castor::tape::tapeserver::daemon::VolumeInfo volInfo;
203
    volInfo.vid="V12345";
204
    castor::tape::tapeFile::WriteSession ws(*mockSys.fake.m_pathToDrive["/dev/nst0"],
205
       volInfo , 0, true);
206
207
208

    // Write a few files on the virtual tape and modify the archive name space
    // so that it is in sync
209
210
211
    uint8_t data[1000];
    castor::tape::SCSI::Structures::zeroStruct(&data);
    for (int fseq=1; fseq <= 10 ; fseq ++) {
212
213
      // Create a path to a remote destination file
      std::ostringstream remoteFilePath;
214
      remoteFilePath << "file:" << m_tmpDir << "/test" << fseq;
215
216
217
218
219
220
221
222
223
224
225
226
227
      remoteFilePaths.push_back(remoteFilePath.str());

      // Create an entry in the archive namespace
      std::ostringstream archiveFilePath;
      archiveFilePath << "/test" << fseq;
      const mode_t archiveFileMode = 0655;
      const uint64_t archiveFileSize = 256*1024;
      ns.createFile(
        requester,
        archiveFilePath.str(),
        archiveFileMode,
        archiveFileSize);

228
229
230
231
// TODO
// We need to add the tape file entry to the name server


232
      // Write the file to tape
233
      std::unique_ptr<cta::RetrieveJob> ftr(new MockRetrieveJob());
234
      std::unique_ptr<cta::ArchiveJob> ftm(new MockArchiveJob());
235
      ftr->tapeFileLocation.fSeq = fseq;
236
      ftm->tapeFileLocation.fSeq = fseq;
237
      ftr->archiveFile.fileId = 1000 + fseq;
238
239
      ftm->archiveFile.fileId = 1000 + fseq;
      castor::tape::tapeFile::WriteFile wf(&ws, *ftm, archiveFileSize);
240
      ftr->tapeFileLocation.blockId = wf.getPosition();
241
      ftr->remotePathAndStatus.path = remoteFilePath.str();
242
243
244
245
      // Write the data (one block)
      wf.write(data, sizeof(data));
      // Close the file
      wf.close();
246
247
248
249
250
251
252
253

      // Schedule the retrieval of the file
      std::list<std::string> archiveFilePaths;
      archiveFilePaths.push_back(archiveFilePath.str());
      scheduler. queueRetrieveRequest(
        requester,
        archiveFilePaths,
        remoteFilePath.str());
254
255
    }
  }
256
257

  // 6) Create the data transfer session
258
  DriveConfig driveConfig("T10D6116", "T10KD6", "/dev/tape_T10D6116", "manual");
259
260
261
262
263
264
  DataTransferConfig castorConf;
  castorConf.bufsz = 1024*1024; // 1 MB memory buffers
  castorConf.nbBufs = 10;
  castorConf.bulkRequestRecallMaxBytes = UINT64_C(100)*1000*1000*1000;
  castorConf.bulkRequestRecallMaxFiles = 1000;
  castorConf.nbDiskThreads = 1;
265
266
267
268
  castor::messages::AcsProxyDummy acs;
  castor::mediachanger::MmcProxyDummy mmc;
  castor::legacymsg::RmcProxyDummy rmc;
  castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
269
  castor::server::ProcessCap capUtils;
270
  castor::messages::TapeserverProxyDummy initialProcess;
271
  DataTransferSession sess("tapeHost", logger, mockSys,
272
    driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
273
274

  // 7) Run the data transfer session
275
  ASSERT_NO_THROW(sess.execute());
276
277

  // 8) Check the session git the correct VID
278
  ASSERT_EQ("V12345", sess.getVid());
279

280
  // 9) Check the remote files exist and have the correct size
281
282
283
284
285
286
287
288
  for(auto pathItor = remoteFilePaths.cbegin(); pathItor !=
    remoteFilePaths.cend(); pathItor++) {
    struct stat statBuf;
    bzero(&statBuf, sizeof(statBuf));
    const int statRc = stat(pathItor->c_str(), &statBuf);
    ASSERT_EQ(0, statRc);
    ASSERT_EQ(256*1024, statBuf.st_size);
  }
289
}
290

291
TEST_F(castor_tape_tapeserver_daemon_DataTransferSessionTest, DataTransferSessionWrongRecall) {
292
293
294
  // This test is the same as the previous one, with 
  // wrong parameters set for the recall, so that we fail 
  // to recall the first file and cancel the second.
295
296

  // 0) Prepare the logger for everyone
297
298
  castor::log::StringLogger logger("tapeServerUnitTest");
  
299
  // 1) prepare the fake scheduler
300
  std::string vid = "V12345";
301
  // cta::MountType::Enum mountType = cta::MountType::RETRIEVE;
302
  std::string density = "8000GC";
303

304
305
306
307
308
309
310
311
  // 3) Prepare the necessary environment (logger, plus system wrapper), 
  castor::tape::System::mockWrapper mockSys;
  mockSys.delegateToFake();
  mockSys.disableGMockCallsCounting();
  mockSys.fake.setupForVirtualDriveSLC6();
  //delete is unnecessary
  //pointer with ownership will be passed to the application,
  //which will do the delete 
312
  mockSys.fake.m_pathToDrive["/dev/nst0"] = new castor::tape::tapeserver::drive::FakeDrive;
313
314
315
316
317
318
319
320
321
322
323
324
325

  // 4) Create the scheduler
  cta::MockNameServer ns;
  cta::MockRemoteNS rns;
  cta::MockSchedulerDatabase db;
  cta::Scheduler scheduler(ns, db, rns);

  // Always use the same requester
  const cta::SecurityIdentity requester;

  // List to remember the path of each remote file so that the existance of the
  // files can be tested for at the end of the test
  std::list<std::string> remoteFilePaths;
326
  
327
  // 5) Prepare files for reading by writing them to the mock system
328
329
330
  {
    // Label the tape
    castor::tape::tapeFile::LabelSession ls(*mockSys.fake.m_pathToDrive["/dev/nst0"], 
331
        "V12345");
332
333
    mockSys.fake.m_pathToDrive["/dev/nst0"]->rewind();
    // And write to it
334
    castor::tape::tapeserver::daemon::VolumeInfo volInfo;
335
336
337
    volInfo.vid="V12345";
    castor::tape::tapeFile::WriteSession ws(*mockSys.fake.m_pathToDrive["/dev/nst0"],
       volInfo , 0, true);
338
339
340

    // Write a few files on the virtual tape and modify the archive name space
    // so that it is in sync
341
342
343
    uint8_t data[1000];
    castor::tape::SCSI::Structures::zeroStruct(&data);
    for (int fseq=1; fseq <= 10 ; fseq ++) {
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
      // Create a path to a remote destination file
      std::ostringstream remoteFilePath;
      remoteFilePath << "file:" << m_tmpDir << "/test" << fseq;
      remoteFilePaths.push_back(remoteFilePath.str());

      // Create an entry in the archive namespace
      std::ostringstream archiveFilePath;
      archiveFilePath << "/test" << fseq;
      const mode_t archiveFileMode = 0655;
      const uint64_t archiveFileSize = 256*1024;
      ns.createFile(
        requester,
        archiveFilePath.str(),
        archiveFileMode,
        archiveFileSize);
// TODO
// We need to add the tape file entry to the name server with at least one entry
// pointing beyond the end of data

      // Write the file to tape
364
365
366
367
368
369
      std::unique_ptr<cta::RetrieveJob> ftr(new MockRetrieveJob());
      std::unique_ptr<cta::ArchiveJob> ftm_temp(new MockArchiveJob());
      ftr->tapeFileLocation.fSeq = fseq;
      ftm_temp->tapeFileLocation.fSeq = fseq;
      ftr->archiveFile.fileId = 1000 + fseq;
      ftm_temp->archiveFile.fileId = 1000 + fseq;
370
      castor::tape::tapeFile::WriteFile wf(&ws, *ftm_temp, archiveFileSize);
371
      ftr->tapeFileLocation.blockId = wf.getPosition();
372
      ftr->remotePathAndStatus.path = remoteFilePath.str();
373
374
375
376
      // Write the data (one block)
      wf.write(data, sizeof(data));
      // Close the file
      wf.close();
377
378
379
380
381
382
383
384

      // Schedule the retrieval of the file
      std::list<std::string> archiveFilePaths;
      archiveFilePaths.push_back(archiveFilePath.str());
      scheduler. queueRetrieveRequest(
        requester,
        archiveFilePaths,
        remoteFilePath.str());
385
386
    }
  }
387
388

  // 6) Create the data transfer session
389
  DriveConfig driveConfig("T10D6116", "T10KD6", "/dev/tape_T10D6116", "manual");
390
391
392
393
394
395
  DataTransferConfig castorConf;
  castorConf.bufsz = 1024*1024; // 1 MB memory buffers
  castorConf.nbBufs = 10;
  castorConf.bulkRequestRecallMaxBytes = UINT64_C(100)*1000*1000*1000;
  castorConf.bulkRequestRecallMaxFiles = 1000;
  castorConf.nbDiskThreads = 1;
396
397
398
399
  castor::messages::AcsProxyDummy acs;
  castor::mediachanger::MmcProxyDummy mmc;
  castor::legacymsg::RmcProxyDummy rmc;
  castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
400
401
  castor::server::ProcessCap capUtils;
  castor::messages::TapeserverProxyDummy initialProcess;
402
  DataTransferSession sess("tapeHost", logger, mockSys,
403
    driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
404
405
406
407
408

  // 7) Run the data transfer session
  ASSERT_NO_THROW(sess.execute());

  // 8) Check the session git the correct VID
409
  ASSERT_EQ("V12345", sess.getVid());
410
411
412
413
414
415
416
417
418
419

  // 9) Check the remote files exist and have the correct size
  for(auto pathItor = remoteFilePaths.cbegin(); pathItor !=
    remoteFilePaths.cend(); pathItor++) {
    struct stat statBuf;
    bzero(&statBuf, sizeof(statBuf));
    const int statRc = stat(pathItor->c_str(), &statBuf);
    ASSERT_EQ(0, statRc);
    ASSERT_EQ(256*1024, statBuf.st_size);
  }
420
}
421

422
/*
423
TEST_F(castor_tape_tapeserver_daemon_DataTransferSessionTest, DataTransferSessionNoSuchDrive) {
424
425
426
427
428
429
430
431
  // TpcpClients only supports 32 bits session number
  // This number has to be less than 2^31 as in addition there is a mix
  // of signed and unsigned numbers
  // As the current ids in prod are ~30M, we are far from overflow (Feb 2013)
  // 1) prepare the client and run it in another thread
  uint32_t volReq = 0xBEEF;
  std::string vid = "V12345";
  std::string density = "8000GC";
432
433
  client::ClientSimulator sim(volReq, vid, density,
    castor::tape::tapegateway::READ_TP, castor::tape::tapegateway::READ);
434
  client::ClientSimulator::ipPort clientAddr = sim.getCallbackAddress();
435
436
437
438
439
440
441
442
443
444
  clientRunner simRun(sim);
  simRun.start();
  
  // 3) Prepare the necessary environment (logger, plus system wrapper), 
  // construct and run the session.
  castor::log::StringLogger logger("tapeServerUnitTest");
  castor::tape::System::mockWrapper mockSys;
  mockSys.delegateToFake();
  mockSys.disableGMockCallsCounting();
  mockSys.fake.setupForVirtualDriveSLC6();
445
  DriveConfig driveConfig("T10D6116", "T10KD6", "/dev/noSuchTape", "manual");
446
447
448
  DataTransferConfig castorConf;
  castorConf.bufsz = 1024;
  castorConf.nbBufs = 10;
449
450
451
452
  castor::messages::AcsProxyDummy acs;
  castor::mediachanger::MmcProxyDummy mmc;
  castor::legacymsg::RmcProxyDummy rmc;
  castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
453
  castor::messages::TapeserverProxyDummy initialProcess;
454
  castor::server::ProcessCapDummy capUtils;
455
456
457
458
  cta::MockNameServer ns;
  cta::MockRemoteNS rns;
  cta::MockSchedulerDatabase db;
  cta::Scheduler scheduler(ns, db, rns);
459
  DataTransferSession sess("tapeHost", logger, mockSys,
460
    driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
461
462
463
464
465
  sess.execute();
  simRun.wait();
  std::string temp = logger.getLog();
  temp += "";
  ASSERT_NE(std::string::npos, logger.getLog().find("Drive not found on this path"));
466
  ASSERT_EQ(SEINTERNAL, sim.m_sessionErrorCode);
467
468
}

469
TEST_F(castor_tape_tapeserver_daemon_DataTransferSessionTest, DataTransferSessionFailtoMount) {
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
  // This test is the same as the previous one, with 
  // wrong parameters set for the recall, so that we fail 
  // to recall the first file and cancel the second.
  castor::log::StringLogger logger("tapeServerUnitTest");
  
  // 1) prepare the client and run it in another thread
  uint32_t volReq = 0xBEEF;
  std::string vid = "V12345";
  std::string density = "8000GC";
  client::ClientSimulator sim(volReq, vid, density,
    castor::tape::tapegateway::TAPE_GATEWAY,
    castor::tape::tapegateway::READ);
  client::ClientSimulator::ipPort clientAddr = sim.getCallbackAddress();
  clientRunner simRun(sim);
  simRun.start();
  
  // 3) Prepare the necessary environment (logger, plus system wrapper), 
  // construct and run the session.
  castor::tape::System::mockWrapper mockSys;
  mockSys.delegateToFake();
  mockSys.disableGMockCallsCounting();
  mockSys.fake.setupForVirtualDriveSLC6();

  //delete is unnecessary
  //pointer with ownership will be passed to the application,
  //which will do the delete 
  const bool failOnMount=true;
  mockSys.fake.m_pathToDrive["/dev/nst0"] = new castor::tape::tapeserver::drive::FakeDrive(failOnMount);
  
  // We can prepare files for reading on the drive
  {
    // Label the tape
    castor::tape::tapeFile::LabelSession ls(*mockSys.fake.m_pathToDrive["/dev/nst0"], 
503
        "V12345");
504
505
506
507
508
509
510
    mockSys.fake.m_pathToDrive["/dev/nst0"]->rewind();
    // And write to it
    castor::tape::tapeserver::client::ClientInterface::VolumeInfo volInfo;
    volInfo.vid="V12345";
    // Prepare a non-empty files to recall list to pass the empty session
    // detection
    for (int fseq=1; fseq <= 10 ; fseq ++) {
511
512
513
      std::unique_ptr<cta::RetrieveJob> ftr(new MockRetrieveJob());
      ftr->tapeFileLocation.fSeq = fseq;
      ftr->archiveFile.fileId = 1000 + fseq;
514
      // Set the recall destination (/dev/null)
515
      ftr->archiveFile.path = "/dev/null";
516
      // Record the file for recall, with an out of tape fSeq
517
      ftr->setFseq(ftr->fseq() + 1000);
518
519
520
      sim.addFileToRecall(ftr, 1000);
    }
  }
521
  DriveConfig driveConfig("T10D6116", "T10KD6", "/dev/tape_T10D6116", "manual");
522
523
524
525
526
527
528
529
530
531
532
533
  DataTransferConfig castorConf;
  castorConf.bufsz = 1024*1024; // 1 MB memory buffers
  castorConf.nbBufs = 10;
  castorConf.bulkRequestRecallMaxBytes = UINT64_C(100)*1000*1000*1000;
  castorConf.bulkRequestRecallMaxFiles = 1000;
  castorConf.nbDiskThreads = 3;
  castor::messages::AcsProxyDummy acs;
  castor::mediachanger::MmcProxyDummy mmc;
  castor::legacymsg::RmcProxyDummy rmc;
  castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
  castor::server::ProcessCap capUtils;
  castor::messages::TapeserverProxyDummy initialProcess;
534
535
536
537
  cta::MockNameServer ns;
  cta::MockRemoteNS rns;
  cta::MockSchedulerDatabase db;
  cta::Scheduler scheduler(ns, db, rns);
538
  DataTransferSession sess("tapeHost", logger, mockSys,
539
    driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
540
541
542
543
544
  sess.execute();
  simRun.wait();
  std::string temp = logger.getLog();
  temp += "";
  ASSERT_EQ("V12345", sess.getVid());
545
546
  // The session is now failing (internal error is reported)
  ASSERT_EQ(1015, sim.m_sessionErrorCode);
547
548
}

549
TEST_F(castor_tape_tapeserver_daemon_DataTransferSessionTest, DataTransferSessionEmptyOnVolReq) {
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
  // This test is the same as the previous one, with 
  // wrong parameters set for the recall, so that we fail 
  // to recall the first file and cancel the second.
  castor::log::StringLogger logger("tapeServerUnitTest");
  
  // 1) prepare the client and run it in another thread
  uint32_t volReq = 0xBEEF;
  std::string vid = "V12345";
  std::string density = "8000GC";
  client::ClientSimulator sim(volReq, vid, density,
    castor::tape::tapegateway::TAPE_GATEWAY,
    castor::tape::tapegateway::READ, client::ClientSimulator::EmptyOnVolReq);
  client::ClientSimulator::ipPort clientAddr = sim.getCallbackAddress();
  clientRunner simRun(sim);
  simRun.start();
  
  // 3) Prepare the necessary environment (logger, plus system wrapper), 
  // construct and run the session.
  castor::tape::System::mockWrapper mockSys;
  mockSys.delegateToFake();
  mockSys.disableGMockCallsCounting();
  mockSys.fake.setupForVirtualDriveSLC6();

  // The drive will not even be opened. so no need for one.
  mockSys.fake.m_pathToDrive["/dev/nst0"] = NULL;
  
576
  DriveConfig driveConfig("T10D6116", "T10KD6", "/dev/tape_T10D6116", "manual");
577
578
579
580
581
582
583
584
585
586
587
588
  DataTransferConfig castorConf;
  castorConf.bufsz = 1024*1024; // 1 MB memory buffers
  castorConf.nbBufs = 10;
  castorConf.bulkRequestRecallMaxBytes = UINT64_C(100)*1000*1000*1000;
  castorConf.bulkRequestRecallMaxFiles = 1000;
  castorConf.nbDiskThreads = 3;
  castor::messages::AcsProxyDummy acs;
  castor::mediachanger::MmcProxyDummy mmc;
  castor::legacymsg::RmcProxyDummy rmc;
  castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
  castor::server::ProcessCap capUtils;
  castor::messages::TapeserverProxyDummy initialProcess;
589
590
591
592
  cta::MockNameServer ns;
  cta::MockRemoteNS rns;
  cta::MockSchedulerDatabase db;
  cta::Scheduler scheduler(ns, db, rns);
593
  DataTransferSession sess("tapeHost", logger, mockSys,
594
    driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
595
596
597
598
599
600
601
602
603
604
605
  sess.execute();
  simRun.wait();
  std::string temp = logger.getLog();
  temp += "";
  ASSERT_EQ("", sess.getVid());
  // Currently, failures are reported by files and recall sessions do not fail.
  ASSERT_EQ(0, sim.m_sessionErrorCode);
  // We should not have logged any error
  ASSERT_EQ(std::string::npos, logger.getLog().find("LVL=E"));
}

606
class TempFileForData {
Eric Cano's avatar
Eric Cano committed
607
public:
608
  TempFileForData(size_t size): m_size(size) {
Eric Cano's avatar
Eric Cano committed
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
    // Create a temporary file
    int randomFd = open("/dev/urandom", 0);
    castor::exception::Errnum::throwOnMinusOne(randomFd, "Error opening /dev/urandom");
    char path[100];
    strncpy(path, "/tmp/castorUnitTestMigrationSourceXXXXXX", 100);
    int tmpFileFd = mkstemp(path);
    castor::exception::Errnum::throwOnMinusOne(tmpFileFd, "Error creating a temporary file");
    m_path = path;
    char * buff = NULL;
    try {
      buff = new char[size];
      if(!buff) { throw castor::exception::Exception("Failed to allocate memory to read /dev/urandom"); }
      castor::exception::Errnum::throwOnMinusOne(read(randomFd, buff, size));
      castor::exception::Errnum::throwOnMinusOne(write(tmpFileFd, buff, size));
      m_checksum = adler32(0L, Z_NULL, 0);
      m_checksum = adler32(m_checksum, (const Bytef *)buff, size);
      close(randomFd);
      close(tmpFileFd);
Eric Cano's avatar
Eric Cano committed
627
      delete[] buff;
Eric Cano's avatar
Eric Cano committed
628
629
630
631
632
633
634
    } catch (...) {
      delete[] buff;
      unlink(m_path.c_str());
      throw;
    }
  }
  
635
  ~TempFileForData() {
Eric Cano's avatar
Eric Cano committed
636
637
638
639
640
641
642
643
644
645
    unlink(m_path.c_str());
  }
  const size_t m_size;
  const std::string path() const { return m_path; }
  uint32_t checksum() const { return m_checksum; }
private:
  std::string m_path;
  uint32_t m_checksum;
};

646
class tempFileVector: public std::vector<TempFileForData *> {
Eric Cano's avatar
Eric Cano committed
647
648
649
650
651
652
653
654
655
656
public:
  ~tempFileVector() {
    while(size()) {
      delete back();
      pop_back();
    }
  }
};

struct expectedResult {
657
658
  expectedResult(int fs, uint32_t cs, int eCode = 0):
    fSeq(fs), checksum(cs), errorCode(eCode) {}
Eric Cano's avatar
Eric Cano committed
659
660
  int fSeq;
  uint32_t checksum;
661
  int errorCode;
Eric Cano's avatar
Eric Cano committed
662
663
};

664
TEST_F(castor_tape_tapeserver_daemon_DataTransferSessionTest, DataTransferSessionGooddayMigration) {
Eric Cano's avatar
Eric Cano committed
665
666
667
668
669
670
671
672
673
674
675
  // TpcpClients only supports 32 bits session number
  // This number has to be less than 2^31 as in addition there is a mix
  // of signed and unsigned numbers
  // As the current ids in prod are ~30M, we are far from overflow (Feb 2013)
  // 0) Prepare the logger for everyone
  castor::log::StringLogger logger("tapeServerUnitTest");
  
  // 1) prepare the client and run it in another thread
  uint32_t volReq = 0xBEEF;
  std::string vid = "V12345";
  std::string density = "8000GC";
676
677
  client::ClientSimulator sim(volReq, vid, density,
    castor::tape::tapegateway::WRITE_TP, castor::tape::tapegateway::WRITE);
Eric Cano's avatar
Eric Cano committed
678
679
680
681
682
683
684
685
686
687
688
689
690
691
  client::ClientSimulator::ipPort clientAddr = sim.getCallbackAddress();
  clientRunner simRun(sim);
  simRun.start();
  
  // 3) Prepare the necessary environment (logger, plus system wrapper), 
  // construct and run the session.
  castor::tape::System::mockWrapper mockSys;
  mockSys.delegateToFake();
  mockSys.disableGMockCallsCounting();
  mockSys.fake.setupForVirtualDriveSLC6();

  //delete is unnecessary
  //pointer with ownership will be passed to the application,
  //which will do the delete 
692
  mockSys.fake.m_pathToDrive["/dev/nst0"] = new castor::tape::tapeserver::drive::FakeDrive;
Eric Cano's avatar
Eric Cano committed
693
694
695
  
  // Just label the tape
  castor::tape::tapeFile::LabelSession ls(*mockSys.fake.m_pathToDrive["/dev/nst0"], 
696
      "V12345");
Eric Cano's avatar
Eric Cano committed
697
698
699
700
701
702
703
  mockSys.fake.m_pathToDrive["/dev/nst0"]->rewind();
  
  tempFileVector tempFiles;
  std::vector<expectedResult> expected;
  // Prepare the files (in real filesystem as they will be opened by the rfio client)
  for (int fseq=1; fseq<=10; fseq++) {
    // Create the file from which we will recall
704
    std::unique_ptr<TempFileForData> tf(new TempFileForData(1000));
Eric Cano's avatar
Eric Cano committed
705
    // Prepare the migrationRequest
706
    MockArchiveJob ftm;
Eric Cano's avatar
Eric Cano committed
707
    ftm.setFileSize(tf->m_size);
708
709
    ftm.archiveFile.fileId = 1000 + fseq;
    ftm.tapeFileLocation.fSeq = fseq;
Eric Cano's avatar
Eric Cano committed
710
711
712
713
714
    ftm.setPath(tf->path());
    sim.addFileToMigrate(ftm);
    expected.push_back(expectedResult(fseq, tf->checksum()));
    tempFiles.push_back(tf.release());
  }
715
  DriveConfig driveConfig("T10D6116", "T10KD6", "/dev/tape_T10D6116", "manual");
716
717
718
719
720
721
  DataTransferConfig castorConf;
  castorConf.bufsz = 1024*1024; // 1 MB memory buffers
  castorConf.nbBufs = 10;
  castorConf.bulkRequestMigrationMaxBytes = UINT64_C(100)*1000*1000*1000;
  castorConf.bulkRequestMigrationMaxFiles = 1000;
  castorConf.nbDiskThreads = 1;
722
723
724
725
  castor::messages::AcsProxyDummy acs;
  castor::mediachanger::MmcProxyDummy mmc;
  castor::legacymsg::RmcProxyDummy rmc;
  castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
726
  castor::messages::TapeserverProxyDummy initialProcess;
727
  castor::server::ProcessCapDummy capUtils;
728
729
730
731
  cta::MockNameServer ns;
  cta::MockRemoteNS rns;
  cta::MockSchedulerDatabase db;
  cta::Scheduler scheduler(ns, db, rns);
732
  DataTransferSession sess("tapeHost", logger, mockSys,
733
    driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
Eric Cano's avatar
Eric Cano committed
734
735
  sess.execute();
  simRun.wait();
736
737
738
739
  for (std::vector<struct expectedResult>::iterator i = expected.begin();
      i != expected.end(); i++) {
    ASSERT_EQ(i->checksum, sim.m_receivedChecksums[i->fSeq]);
  }
740
  ASSERT_EQ(0, sim.m_sessionErrorCode);
Eric Cano's avatar
Eric Cano committed
741
742
}

743
744
745
746
//
// This test is the same as the previous one, except that the files are deleted
// from filesystem immediately. The disk tasks will then fail on open.
///
747
TEST_F(castor_tape_tapeserver_daemon_DataTransferSessionTest, DataTransferSessionMissingFilesMigration) {
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
  // TpcpClients only supports 32 bits session number
  // This number has to be less than 2^31 as in addition there is a mix
  // of signed and unsigned numbers
  // As the current ids in prod are ~30M, we are far from overflow (Feb 2013)
  // 0) Prepare the logger for everyone
  castor::log::StringLogger logger("tapeServerUnitTest");
  
  // 1) prepare the client and run it in another thread
  uint32_t volReq = 0xBEEF;
  std::string vid = "V12345";
  std::string density = "8000GC";
  client::ClientSimulator sim(volReq, vid, density,
    castor::tape::tapegateway::WRITE_TP, castor::tape::tapegateway::WRITE);
  client::ClientSimulator::ipPort clientAddr = sim.getCallbackAddress();
  clientRunner simRun(sim);
  simRun.start();
  
  // 3) Prepare the necessary environment (logger, plus system wrapper), 
  // construct and run the session.
  castor::tape::System::mockWrapper mockSys;
  mockSys.delegateToFake();
  mockSys.disableGMockCallsCounting();
  mockSys.fake.setupForVirtualDriveSLC6();

  //delete is unnecessary
  //pointer with ownership will be passed to the application,
  //which will do the delete 
775
  mockSys.fake.m_pathToDrive["/dev/nst0"] = new castor::tape::tapeserver::drive::FakeDrive;
776
777
778
  
  // Just label the tape
  castor::tape::tapeFile::LabelSession ls(*mockSys.fake.m_pathToDrive["/dev/nst0"], 
779
      "V12345");
780
781
782
783
784
  mockSys.fake.m_pathToDrive["/dev/nst0"]->rewind();
  
  // Prepare the files, but delete them immediately. The migration will fail.
  for (int fseq=1; fseq<=10; fseq++) {
    // Create the file from which we will recall
785
    std::unique_ptr<TempFileForData> tf(new TempFileForData(1000));
786
    // Prepare the migrationRequest
787
    MockArchiveJob ftm;
788
    ftm.setFileSize(tf->m_size);
789
790
    ftm.archiveFile.fileId = 1000 + fseq;
    ftm.tapeFileLocation.fSeq = fseq;
791
792
793
    ftm.setPath(tf->path());
    sim.addFileToMigrate(ftm);
  }
794
  DriveConfig driveConfig("T10D6116", "T10KD6", "/dev/tape_T10D6116", "manual");
795
796
797
798
799
800
  DataTransferConfig castorConf;
  castorConf.bufsz = 1024*1024; // 1 MB memory buffers
  castorConf.nbBufs = 10;
  castorConf.bulkRequestMigrationMaxBytes = UINT64_C(100)*1000*1000*1000;
  castorConf.bulkRequestMigrationMaxFiles = 1000;
  castorConf.nbDiskThreads = 1;
801
802
803
804
  castor::messages::AcsProxyDummy acs;
  castor::mediachanger::MmcProxyDummy mmc;
  castor::legacymsg::RmcProxyDummy rmc;
  castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
805
806
  castor::messages::TapeserverProxyDummy initialProcess;
  castor::server::ProcessCapDummy capUtils;
807
808
809
810
  cta::MockNameServer ns;
  cta::MockRemoteNS rns;
  cta::MockSchedulerDatabase db;
  cta::Scheduler scheduler(ns, db, rns);
811
  DataTransferSession sess("tapeHost", logger, mockSys,
812
    driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
813
814
  sess.execute();
  simRun.wait();
815
  ASSERT_EQ(SEINTERNAL, sim.m_sessionErrorCode);
816
}
817

818
819
820
821
822
//
// This test is identical to the good day migration, but the tape will accept
// only a finite number of bytes and hence we will report a full tape skip the
// last migrations
//
823
TEST_F(castor_tape_tapeserver_daemon_DataTransferSessionTest, DataTransferSessionTapeFullMigration) {
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
  // TpcpClients only supports 32 bits session number
  // This number has to be less than 2^31 as in addition there is a mix
  // of signed and unsigned numbers
  // As the current ids in prod are ~30M, we are far from overflow (Feb 2013)
  // 0) Prepare the logger for everyone
  castor::log::StringLogger logger("tapeServerUnitTest");
  
  // 1) prepare the client and run it in another thread
  uint32_t volReq = 0xBEEF;
  std::string vid = "V12345";
  std::string density = "8000GC";
  client::ClientSimulator sim(volReq, vid, density,
    castor::tape::tapegateway::WRITE_TP, castor::tape::tapegateway::WRITE);
  client::ClientSimulator::ipPort clientAddr = sim.getCallbackAddress();
  clientRunner simRun(sim);
  simRun.start();
  
  // 3) Prepare the necessary environment (logger, plus system wrapper), 
  // construct and run the session.
  castor::tape::System::mockWrapper mockSys;
  mockSys.delegateToFake();
  mockSys.disableGMockCallsCounting();
  mockSys.fake.setupForVirtualDriveSLC6();

  //delete is unnecessary
  //pointer with ownership will be passed to the application,
  //which will do the delete 
  const uint64_t tapeSize = 5000;
852
  mockSys.fake.m_pathToDrive["/dev/nst0"] = new castor::tape::tapeserver::drive::FakeDrive(tapeSize);
853
854
855
  
  // Just label the tape
  castor::tape::tapeFile::LabelSession ls(*mockSys.fake.m_pathToDrive["/dev/nst0"], 
856
      "V12345");
857
858
859
860
861
862
863
864
865
866
  mockSys.fake.m_pathToDrive["/dev/nst0"]->rewind();
  
  tempFileVector tempFiles;
  std::vector<expectedResult> expected;
  uint64_t remainingSpace = tapeSize;
  bool failedFileDone = false;
  // Prepare the files (in real filesystem as they will be opened by the rfio client)
  for (int fseq=1; fseq<=10; fseq++) {
    // Create the file from which we will recall
    const size_t fileSize = 1000;
867
    std::unique_ptr<TempFileForData> tf(new TempFileForData(fileSize));
868
    // Prepare the migrationRequest
869
    MockArchiveJob ftm;
870
    ftm.setFileSize(tf->m_size);
871
872
    ftm.archiveFile.fileId = 1000 + fseq;
    ftm.tapeFileLocation.fSeq = fseq;
873
874
875
876
877
878
879
880
881
882
883
884
    ftm.setPath(tf->path());
    sim.addFileToMigrate(ftm);
    if (fileSize + 6 * 80 < remainingSpace) {
      expected.push_back(expectedResult(fseq, tf->checksum()));
      remainingSpace -= fileSize + 6 * 80;
    } else if (!failedFileDone) {
      // We add also the report for the first file (which will come in error)
      expected.push_back(expectedResult(fseq, 0, ENOSPC));
      failedFileDone = true;
    }
    tempFiles.push_back(tf.release());
  }
885
  DriveConfig driveConfig("T10D6116", "T10KD6", "/dev/tape_T10D6116", "manual");
886
887
888
889
890
891
  DataTransferConfig castorConf;
  castorConf.bufsz = 1024*1024; // 1 MB memory buffers
  castorConf.nbBufs = 10;
  castorConf.bulkRequestMigrationMaxBytes = UINT64_C(100)*1000*1000*1000;
  castorConf.bulkRequestMigrationMaxFiles = 1000;
  castorConf.nbDiskThreads = 1;
892
893
894
895
  castor::messages::AcsProxyDummy acs;
  castor::mediachanger::MmcProxyDummy mmc;
  castor::legacymsg::RmcProxyDummy rmc;
  castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
896
897
  castor::messages::TapeserverProxyDummy initialProcess;
  castor::server::ProcessCapDummy capUtils;
898
899
900
901
  cta::MockNameServer ns;
  cta::MockRemoteNS rns;
  cta::MockSchedulerDatabase db;
  cta::Scheduler scheduler(ns, db, rns);
902
  DataTransferSession sess("tapeHost", logger, mockSys,
903
    driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
904
905
906
907
908
909
910
911
912
913
  sess.execute();
  simRun.wait();
  for (std::vector<struct expectedResult>::iterator i = expected.begin();
      i != expected.end(); i++) {
    if (!i->errorCode) {
      ASSERT_EQ(i->checksum, sim.m_receivedChecksums[i->fSeq]);
    } else {
      ASSERT_EQ(i->errorCode, sim.m_receivedErrorCodes[i->fSeq]);
    }
  }
914
915
916
  ASSERT_EQ(ENOSPC, sim.m_sessionErrorCode);
}

917
TEST_F(castor_tape_tapeserver_daemon_DataTransferSessionTest, DataTransferSessionTapeFullOnFlushMigration) {
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
  // TpcpClients only supports 32 bits session number
  // This number has to be less than 2^31 as in addition there is a mix
  // of signed and unsigned numbers
  // As the current ids in prod are ~30M, we are far from overflow (Feb 2013)
  // 0) Prepare the logger for everyone
  castor::log::StringLogger logger("tapeServerUnitTest");
  
  // 1) prepare the client and run it in another thread
  uint32_t volReq = 0xBEEF;
  std::string vid = "V12345";
  std::string density = "8000GC";
  client::ClientSimulator sim(volReq, vid, density,
    castor::tape::tapegateway::WRITE_TP, castor::tape::tapegateway::WRITE);
  client::ClientSimulator::ipPort clientAddr = sim.getCallbackAddress();
  clientRunner simRun(sim);
  simRun.start();
  
  // 3) Prepare the necessary environment (logger, plus system wrapper), 
  // construct and run the session.
  castor::tape::System::mockWrapper mockSys;
  mockSys.delegateToFake();
  mockSys.disableGMockCallsCounting();
  mockSys.fake.setupForVirtualDriveSLC6();

  //delete is unnecessary
  //pointer with ownership will be passed to the application,
  //which will do the delete 
  const uint64_t tapeSize = 5000;
  mockSys.fake.m_pathToDrive["/dev/nst0"] =
947
948
      new castor::tape::tapeserver::drive::FakeDrive(tapeSize,
        castor::tape::tapeserver::drive::FakeDrive::OnFlush);
949
950
951
  
  // Just label the tape
  castor::tape::tapeFile::LabelSession ls(*mockSys.fake.m_pathToDrive["/dev/nst0"], 
952
      "V12345");
953
954
955
956
957
958
959
960
961
962
  mockSys.fake.m_pathToDrive["/dev/nst0"]->rewind();
  
  tempFileVector tempFiles;
  std::vector<expectedResult> expected;
  uint64_t remainingSpace = tapeSize;
  bool failedFileDone = false;
  // Prepare the files (in real filesystem as they will be opened by the rfio client)
  for (int fseq=1; fseq<=10; fseq++) {
    // Create the file from which we will recall
    const size_t fileSize = 1000;
963
    std::unique_ptr<TempFileForData> tf(new TempFileForData(fileSize));
964
    // Prepare the migrationRequest
965
    MockArchiveJob ftm;
966
    ftm.setFileSize(tf->m_size);
967
968
    ftm.archiveFile.fileId = 1000 + fseq;
    ftm.tapeFileLocation.fSeq = fseq;
969
970
971
972
973
974
975
976
977
978
979
980
    ftm.setPath(tf->path());
    sim.addFileToMigrate(ftm);
    if (fileSize + 6 * 80 < remainingSpace) {
      expected.push_back(expectedResult(fseq, tf->checksum()));
      remainingSpace -= fileSize + 6 * 80;
    } else if (!failedFileDone) {
      // We expect no report for this file anymore (but the report for the 
      // session)
      failedFileDone = true;
    }
    tempFiles.push_back(tf.release());
  }
981
  DriveConfig driveConfig("T10D6116", "T10KD6", "/dev/tape_T10D6116", "manual");
982
983
984
985
986
987
  DataTransferConfig castorConf;
  castorConf.bufsz = 1024*1024; // 1 MB memory buffers
  castorConf.nbBufs = 10;
  castorConf.bulkRequestMigrationMaxBytes = UINT64_C(100)*1000*1000*1000;
  castorConf.bulkRequestMigrationMaxFiles = 1000;
  castorConf.nbDiskThreads = 1;
988
989
990
991
  castor::messages::AcsProxyDummy acs;
  castor::mediachanger::MmcProxyDummy mmc;
  castor::legacymsg::RmcProxyDummy rmc;
  castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
992
993
  castor::messages::TapeserverProxyDummy initialProcess;
  castor::server::ProcessCapDummy capUtils;
994
995
996
997
  cta::MockNameServer ns;
  cta::MockRemoteNS rns;
  cta::MockSchedulerDatabase db;
  cta::Scheduler scheduler(ns, db, rns);
998
  DataTransferSession sess("tapeHost", logger, mockSys,
999
    driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
1000
  sess.execute();