Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
dCache
cta
Commits
44fd91ba
Commit
44fd91ba
authored
Jul 24, 2019
by
Michael Davis
Browse files
[migration] Adds eos-test-file-inject to cta-migration RPM
parent
f067fea6
Changes
4
Hide whitespace changes
Inline
Side-by-side
cta.spec.in
View file @
44fd91ba
...
...
@@ -308,6 +308,7 @@ directory metadata into the EOS namespace.
%attr(0755,root,root) %{_bindir}/eos-import-dirs
%attr(0755,root,root) %{_bindir}/eos-import-files
%attr(0755,root,root) %{_bindir}/eos-test-dir-inject
%attr(0755,root,root) %{_bindir}/eos-test-file-inject
%attr(0755,root,root) %{_bindir}/eos-test-inject.sh
%package -n cta-rmcd
...
...
migration/gRPC/CMakeLists.txt
View file @
44fd91ba
...
...
@@ -32,12 +32,17 @@ add_executable(eos-import-files EosImportFiles.cpp GrpcClient.cpp)
target_link_libraries
(
eos-import-files EosMigration
${
PROTOBUF3_LIBRARIES
}
${
GRPC_LIBRARY
}
${
GRPC_GRPC++_LIBRARY
}
ctacatalogue
)
set_property
(
TARGET eos-import-files APPEND PROPERTY INSTALL_RPATH
${
PROTOBUF3_RPATH
}
)
# Standalone test tool
# Standalone test tool
s
add_executable
(
eos-test-dir-inject EosTestDirInject.cpp GrpcClient.cpp
)
target_link_libraries
(
eos-test-dir-inject EosMigration
${
PROTOBUF3_LIBRARIES
}
${
GRPC_LIBRARY
}
${
GRPC_GRPC++_LIBRARY
}
ctacatalogue
)
set_property
(
TARGET eos-test-dir-inject APPEND PROPERTY INSTALL_RPATH
${
PROTOBUF3_RPATH
}
)
add_executable
(
eos-test-file-inject EosTestFileInject.cpp GrpcClient.cpp
)
target_link_libraries
(
eos-test-file-inject EosMigration
${
PROTOBUF3_LIBRARIES
}
${
GRPC_LIBRARY
}
${
GRPC_GRPC++_LIBRARY
}
ctacatalogue
)
set_property
(
TARGET eos-test-file-inject APPEND PROPERTY INSTALL_RPATH
${
PROTOBUF3_RPATH
}
)
install
(
TARGETS eos-import-dirs DESTINATION usr/bin
)
install
(
TARGETS eos-import-files DESTINATION usr/bin
)
install
(
TARGETS eos-test-dir-inject DESTINATION usr/bin
)
install
(
TARGETS eos-test-file-inject DESTINATION usr/bin
)
install
(
FILES
${
CMAKE_SOURCE_DIR
}
/migration/gRPC/eos-test-inject.sh DESTINATION usr/bin
)
migration/gRPC/EosTestDirInject.cpp
View file @
44fd91ba
...
...
@@ -126,10 +126,10 @@ void EosTestDirInject::inject(const std::string &path, uint64_t fileid)
}
// Inject directories into EOS
int
retc
=
m_eosgrpc
->
ContainerInsert
(
dirs
);
if
(
retc
!=
0
)
{
throw
std
::
runtime_error
(
"EosTestDirInject::
processBatch
(): ContainerInsert failed with error "
+
std
::
to_string
(
retc
));
}
int
retc
=
m_eosgrpc
->
ContainerInsert
(
dirs
);
if
(
retc
!=
0
)
{
throw
std
::
runtime_error
(
"EosTestDirInject::
inject
(): ContainerInsert failed with error "
+
std
::
to_string
(
retc
));
}
auto
elapsed_time
=
std
::
chrono
::
duration_cast
<
std
::
chrono
::
seconds
>
(
std
::
chrono
::
steady_clock
::
now
()
-
m_start_time
);
std
::
cerr
<<
"Processed 1 directory in "
<<
elapsed_time
.
count
()
<<
"s"
<<
std
::
endl
;
...
...
migration/gRPC/EosTestFileInject.cpp
0 → 100644
View file @
44fd91ba
/*!
* @project The CERN Tape Archive (CTA)
* @brief Inject directory metadata into EOS using gRPC
* @copyright Copyright 2019 CERN
* @license This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include
<iostream>
#include
<sstream>
#include
<XrdSsiPbConfig.hpp>
#include
<XrdSsiPbLog.hpp>
#include
"GrpcClient.hpp"
namespace
cta
{
namespace
migration
{
class
EosTestFileInject
{
public:
EosTestFileInject
(
const
std
::
string
&
configfile
);
void
inject
(
const
std
::
string
&
path
,
uint64_t
fileid
);
std
::
string
ping
(
const
std
::
string
&
payload
)
{
return
m_eosgrpc
->
ping
(
payload
);
}
struct
Dirname
{
std
::
string
basename
;
//!< Just the directory name
std
::
string
pathname
;
//!< The full path including the directory name
};
private:
static
std
::
string
convertChecksum
(
uint32_t
adler32
);
Dirname
manglePathname
(
const
std
::
string
&
pathname
);
std
::
chrono
::
steady_clock
::
time_point
m_start_time
;
//!< Start the clock
bool
m_is_json
;
//!< Display results in JSON format for debugging
std
::
unique_ptr
<
eos
::
client
::
GrpcClient
>
m_eosgrpc
;
//!< EOS gRPC API interface
std
::
string
m_castor_prefix
;
//!< CASTOR namespace prefix to strip
std
::
string
m_eos_prefix
;
//!< EOS namespace prefix to prepend
std
::
map
<
unsigned
int
,
std
::
string
>
m_storageClass
;
//!< Mapping of CASTOR file class IDs to CTA storage classes
};
EosTestFileInject
::
EosTestFileInject
(
const
std
::
string
&
configfile
)
:
m_start_time
(
std
::
chrono
::
steady_clock
::
now
())
{
// Parse configuration file
XrdSsiPb
::
Config
config
(
configfile
);
auto
is_json
=
config
.
getOptionValueBool
(
"castor.json"
);
auto
castor_prefix
=
config
.
getOptionValueStr
(
"castor.prefix"
);
auto
eos_prefix
=
config
.
getOptionValueStr
(
"eos.prefix"
);
auto
endpoint
=
config
.
getOptionValueStr
(
"eos.endpoint"
);
auto
token
=
config
.
getOptionValueStr
(
"eos.token"
);
// Connect to EOS
m_eosgrpc
=
eos
::
client
::
GrpcClient
::
Create
(
endpoint
.
first
?
endpoint
.
second
:
"localhost:50051"
,
token
.
second
);
// Set parameters and defaults
m_is_json
=
is_json
.
first
?
is_json
.
second
:
false
;
m_castor_prefix
=
castor_prefix
.
first
?
castor_prefix
.
second
:
"/castor/cern.ch"
;
m_eos_prefix
=
eos_prefix
.
first
?
eos_prefix
.
second
:
"/eos/grpc"
;
}
void
EosTestFileInject
::
inject
(
const
std
::
string
&
path
,
uint64_t
fileid
)
{
// See EOS common/LayoutId.hh for definitions of constants
const
int
kAdler
=
0x2
;
const
int
kReplica
=
(
0x1
<<
4
);
const
int
kStripeSize
=
(
0x0
<<
8
);
// 1 stripe
const
int
kStripeWidth
=
(
0x0
<<
16
);
// 4K blocks
const
int
kBlockChecksum
=
(
0x1
<<
20
);
// Layout id should be 00100012
const
uint64_t
layout
=
kReplica
|
kAdler
|
kStripeSize
|
kStripeWidth
|
kBlockChecksum
;
eos
::
rpc
::
FileMdProto
file
;
file
.
set_id
(
fileid
);
file
.
set_cont_id
(
12345
);
file
.
set_uid
(
1000
);
file
.
set_gid
(
1000
);
file
.
set_size
(
42
);
file
.
set_layout_id
(
layout
);
file
.
set_flags
(
0755
);
file
.
mutable_checksum
()
->
set_type
(
"adler"
);
file
.
mutable_checksum
()
->
set_value
(
convertChecksum
(
0xaabbccdd
));
// Timestamps
file
.
mutable_ctime
()
->
set_sec
(
1553900400
);
file
.
mutable_mtime
()
->
set_sec
(
1553900400
);
// we don't care about file.stime (sync time, used for CERNBox)
// BTIME is set as an extended attribute (see below)
// Filename and path
file
.
set_name
(
path
);
file
.
set_path
(
m_eos_prefix
+
"/"
+
file
.
name
());
// we don't care about link_name
// Extended attributes:
//
// 1. Archive File ID
std
::
string
archiveId
(
std
::
to_string
(
file
.
id
()));
file
.
mutable_xattrs
()
->
insert
(
google
::
protobuf
::
MapPair
<
std
::
string
,
std
::
string
>
(
"CTA_ArchiveFileId"
,
archiveId
));
// 2. Storage Class
file
.
mutable_xattrs
()
->
insert
(
google
::
protobuf
::
MapPair
<
std
::
string
,
std
::
string
>
(
"CTA_StorageClass"
,
"my_storage_class"
));
// 3. Birth Time
// POSIX ATIME (Access Time) is used by CASTOR to store the file creation time. EOS calls this "birth time",
// but there is no place in the namespace to store it, so it is stored as an extended attribute.
file
.
mutable_xattrs
()
->
insert
(
google
::
protobuf
::
MapPair
<
std
::
string
,
std
::
string
>
(
"eos.btime"
,
"1553900400"
));
// Indicate that there is a tape-resident replica of this file
file
.
mutable_locations
()
->
Add
(
65535
);
// we don't care about unlink_locations (placeholder for files scheduled for deletion)
std
::
vector
<
eos
::
rpc
::
FileMdProto
>
files
;
files
.
push_back
(
file
);
// Put results on stdout for debugging
if
(
m_is_json
)
{
char
delim
=
'['
;
for
(
auto
&
file
:
files
)
{
std
::
cout
<<
delim
<<
XrdSsiPb
::
Log
::
DumpProtobuf
(
&
file
);
delim
=
','
;
}
if
(
delim
==
','
)
std
::
cout
<<
"]"
;
}
// Inject file into EOS
int
retc
=
m_eosgrpc
->
FileInsert
(
files
);
if
(
retc
!=
0
)
{
throw
std
::
runtime_error
(
"EosTestFileInject::inject(): FileInsert failed with error "
+
std
::
to_string
(
retc
));
}
auto
elapsed_time
=
std
::
chrono
::
duration_cast
<
std
::
chrono
::
seconds
>
(
std
::
chrono
::
steady_clock
::
now
()
-
m_start_time
);
std
::
cerr
<<
"Processed 1 file in "
<<
elapsed_time
.
count
()
<<
"s"
<<
std
::
endl
;
}
std
::
string
EosTestFileInject
::
convertChecksum
(
uint32_t
adler32
)
{
char
bytes
[
4
];
for
(
int
i
=
3
;
i
>=
0
;
--
i
,
adler32
>>=
8
)
{
bytes
[
i
]
=
adler32
&
0xFF
;
}
return
std
::
string
(
bytes
,
4
);
}
EosTestFileInject
::
Dirname
EosTestFileInject
::
manglePathname
(
const
std
::
string
&
pathname
)
{
Dirname
dir
;
size_t
clip
=
(
pathname
.
find
(
m_castor_prefix
)
==
0
)
?
m_castor_prefix
.
length
()
:
0
;
dir
.
pathname
=
m_eos_prefix
+
pathname
.
substr
(
clip
);
clip
=
dir
.
pathname
.
find_last_of
(
'/'
);
dir
.
basename
=
(
clip
==
std
::
string
::
npos
)
?
dir
.
pathname
:
dir
.
pathname
.
substr
(
clip
+
1
);
return
dir
;
}
}}
// namespace cta::migration
void
throwUsage
(
const
std
::
string
&
program
,
const
std
::
string
&
error_txt
)
{
std
::
stringstream
help
;
help
<<
program
<<
": "
<<
error_txt
<<
std
::
endl
<<
"Usage: "
<<
program
<<
" [--config <config_file>] [--fileid <fileid>] ping|--path <path>"
;
throw
std
::
runtime_error
(
help
.
str
());
}
int
main
(
int
argc
,
const
char
*
argv
[])
{
std
::
string
configfile
=
"/etc/cta/castor-migration.conf"
;
std
::
string
path
;
uint64_t
fileid
=
0
;
bool
doPing
=
false
;
try
{
for
(
auto
i
=
1
;
i
<
argc
;
++
i
)
{
std
::
string
option
(
argv
[
i
]);
if
(
option
==
"--config"
&&
argc
>
++
i
)
{
configfile
=
argv
[
i
];
continue
;
}
else
if
(
option
==
"--path"
&&
argc
>
++
i
)
{
path
=
argv
[
i
];
continue
;
}
else
if
(
option
==
"--fileid"
&&
argc
>
++
i
)
{
fileid
=
strtoul
(
argv
[
i
],
NULL
,
0
);
continue
;
}
else
if
(
option
==
"ping"
)
{
doPing
=
true
;
continue
;
}
throwUsage
(
argv
[
0
],
"invalid option "
+
option
);
}
cta
::
migration
::
EosTestFileInject
testDirInject
(
configfile
);
if
(
doPing
)
{
std
::
cout
<<
"Pinging EOS MGM using gRPC API..."
<<
std
::
endl
;
auto
pingStr
=
testDirInject
.
ping
(
"Ping from EosTestFileInject"
);
std
::
cout
<<
"Ping successful. Server responded with: "
<<
pingStr
<<
std
::
endl
;
}
else
{
if
(
path
.
empty
())
throwUsage
(
argv
[
0
],
"path not specified"
);
testDirInject
.
inject
(
path
,
fileid
);
}
}
catch
(
std
::
runtime_error
&
ex
)
{
std
::
cerr
<<
ex
.
what
()
<<
std
::
endl
;
return
-
1
;
}
return
0
;
}
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment