Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
cta
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container Registry
Harbor Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
dCache
cta
Commits
cbb207ff
Commit
cbb207ff
authored
7 years ago
by
Eric Cano
Browse files
Options
Downloads
Patches
Plain Diff
Cleaned up leftover array in ArchiveJob protobuf.
parent
42d4f178
No related branches found
Branches containing commit
No related tags found
Tags containing commit
No related merge requests found
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
objectstore/ArchiveQueue.cpp
+18
-20
18 additions, 20 deletions
objectstore/ArchiveQueue.cpp
objectstore/cta.proto
+1
-3
1 addition, 3 deletions
objectstore/cta.proto
with
19 additions
and
23 deletions
objectstore/ArchiveQueue.cpp
+
18
−
20
View file @
cbb207ff
...
@@ -69,17 +69,17 @@ void ArchiveQueue::commit() {
...
@@ -69,17 +69,17 @@ void ArchiveQueue::commit() {
ValueCountMap
maxDriveAllowedMap
(
m_payload
.
mutable_maxdrivesallowedmap
());
ValueCountMap
maxDriveAllowedMap
(
m_payload
.
mutable_maxdrivesallowedmap
());
ValueCountMap
priorityMap
(
m_payload
.
mutable_prioritymap
());
ValueCountMap
priorityMap
(
m_payload
.
mutable_prioritymap
());
ValueCountMap
minArchiveRequestAgeMap
(
m_payload
.
mutable_minarchiverequestagemap
());
ValueCountMap
minArchiveRequestAgeMap
(
m_payload
.
mutable_minarchiverequestagemap
());
if
(
maxDriveAllowedMap
.
total
()
!=
(
uint64_t
)
m_payload
.
pending
archivejobs_size
()
||
if
(
maxDriveAllowedMap
.
total
()
!=
(
uint64_t
)
m_payload
.
archivejobs_size
()
||
priorityMap
.
total
()
!=
(
uint64_t
)
m_payload
.
pending
archivejobs_size
()
||
priorityMap
.
total
()
!=
(
uint64_t
)
m_payload
.
archivejobs_size
()
||
minArchiveRequestAgeMap
.
total
()
!=
(
uint64_t
)
m_payload
.
pending
archivejobs_size
())
{
minArchiveRequestAgeMap
.
total
()
!=
(
uint64_t
)
m_payload
.
archivejobs_size
())
{
// The maps counts are off: recompute them.
// The maps counts are off: recompute them.
maxDriveAllowedMap
.
clear
();
maxDriveAllowedMap
.
clear
();
priorityMap
.
clear
();
priorityMap
.
clear
();
minArchiveRequestAgeMap
.
clear
();
minArchiveRequestAgeMap
.
clear
();
for
(
size_t
i
=
0
;
i
<
(
size_t
)
m_payload
.
pending
archivejobs_size
();
i
++
)
{
for
(
size_t
i
=
0
;
i
<
(
size_t
)
m_payload
.
archivejobs_size
();
i
++
)
{
maxDriveAllowedMap
.
incCount
(
m_payload
.
pending
archivejobs
(
i
).
maxdrivesallowed
());
maxDriveAllowedMap
.
incCount
(
m_payload
.
archivejobs
(
i
).
maxdrivesallowed
());
priorityMap
.
incCount
(
m_payload
.
pending
archivejobs
(
i
).
priority
());
priorityMap
.
incCount
(
m_payload
.
archivejobs
(
i
).
priority
());
minArchiveRequestAgeMap
.
incCount
(
m_payload
.
pending
archivejobs
(
i
).
priority
());
minArchiveRequestAgeMap
.
incCount
(
m_payload
.
archivejobs
(
i
).
priority
());
}
}
m_payload
.
set_mapsrebuildcount
(
m_payload
.
mapsrebuildcount
()
+
1
);
m_payload
.
set_mapsrebuildcount
(
m_payload
.
mapsrebuildcount
()
+
1
);
}
}
...
@@ -90,9 +90,7 @@ void ArchiveQueue::commit() {
...
@@ -90,9 +90,7 @@ void ArchiveQueue::commit() {
bool
ArchiveQueue
::
isEmpty
()
{
bool
ArchiveQueue
::
isEmpty
()
{
checkPayloadReadable
();
checkPayloadReadable
();
// Check we have no archive jobs pending
// Check we have no archive jobs pending
if
(
m_payload
.
pendingarchivejobs_size
()
if
(
m_payload
.
archivejobs_size
())
||
m_payload
.
orphanedarchivejobsnscreation_size
()
||
m_payload
.
orphanedarchivejobsnsdeletion_size
())
return
false
;
return
false
;
// If we made it to here, it seems the pool is indeed empty.
// If we made it to here, it seems the pool is indeed empty.
return
true
;
return
true
;
...
@@ -154,7 +152,7 @@ void ArchiveQueue::addJobsAndCommit(std::list<JobToAdd> & jobsToAdd) {
...
@@ -154,7 +152,7 @@ void ArchiveQueue::addJobsAndCommit(std::list<JobToAdd> & jobsToAdd) {
priorityMap
.
incCount
(
jta
.
policy
.
archivePriority
);
priorityMap
.
incCount
(
jta
.
policy
.
archivePriority
);
ValueCountMap
minArchiveRequestAgeMap
(
m_payload
.
mutable_minarchiverequestagemap
());
ValueCountMap
minArchiveRequestAgeMap
(
m_payload
.
mutable_minarchiverequestagemap
());
minArchiveRequestAgeMap
.
incCount
(
jta
.
policy
.
archiveMinRequestAge
);
minArchiveRequestAgeMap
.
incCount
(
jta
.
policy
.
archiveMinRequestAge
);
if
(
m_payload
.
pending
archivejobs_size
())
{
if
(
m_payload
.
archivejobs_size
())
{
if
((
uint64_t
)
jta
.
startTime
<
m_payload
.
oldestjobcreationtime
())
if
((
uint64_t
)
jta
.
startTime
<
m_payload
.
oldestjobcreationtime
())
m_payload
.
set_oldestjobcreationtime
(
jta
.
startTime
);
m_payload
.
set_oldestjobcreationtime
(
jta
.
startTime
);
m_payload
.
set_archivejobstotalsize
(
m_payload
.
archivejobstotalsize
()
+
jta
.
fileSize
);
m_payload
.
set_archivejobstotalsize
(
m_payload
.
archivejobstotalsize
()
+
jta
.
fileSize
);
...
@@ -162,7 +160,7 @@ void ArchiveQueue::addJobsAndCommit(std::list<JobToAdd> & jobsToAdd) {
...
@@ -162,7 +160,7 @@ void ArchiveQueue::addJobsAndCommit(std::list<JobToAdd> & jobsToAdd) {
m_payload
.
set_archivejobstotalsize
(
jta
.
fileSize
);
m_payload
.
set_archivejobstotalsize
(
jta
.
fileSize
);
m_payload
.
set_oldestjobcreationtime
(
jta
.
startTime
);
m_payload
.
set_oldestjobcreationtime
(
jta
.
startTime
);
}
}
auto
*
j
=
m_payload
.
add_
pending
archivejobs
();
auto
*
j
=
m_payload
.
add_archivejobs
();
j
->
set_address
(
jta
.
archiveRequestAddress
);
j
->
set_address
(
jta
.
archiveRequestAddress
);
j
->
set_size
(
jta
.
fileSize
);
j
->
set_size
(
jta
.
fileSize
);
j
->
set_fileid
(
jta
.
archiveFileId
);
j
->
set_fileid
(
jta
.
archiveFileId
);
...
@@ -177,7 +175,7 @@ void ArchiveQueue::addJobsAndCommit(std::list<JobToAdd> & jobsToAdd) {
...
@@ -177,7 +175,7 @@ void ArchiveQueue::addJobsAndCommit(std::list<JobToAdd> & jobsToAdd) {
auto
ArchiveQueue
::
getJobsSummary
()
->
JobsSummary
{
auto
ArchiveQueue
::
getJobsSummary
()
->
JobsSummary
{
checkPayloadReadable
();
checkPayloadReadable
();
JobsSummary
ret
;
JobsSummary
ret
;
ret
.
files
=
m_payload
.
pending
archivejobs_size
();
ret
.
files
=
m_payload
.
archivejobs_size
();
ret
.
bytes
=
m_payload
.
archivejobstotalsize
();
ret
.
bytes
=
m_payload
.
archivejobstotalsize
();
ret
.
oldestJobStartTime
=
m_payload
.
oldestjobcreationtime
();
ret
.
oldestJobStartTime
=
m_payload
.
oldestjobcreationtime
();
if
(
ret
.
files
)
{
if
(
ret
.
files
)
{
...
@@ -202,7 +200,7 @@ ArchiveQueue::AdditionSummary ArchiveQueue::addJobsIfNecessaryAndCommit(std::lis
...
@@ -202,7 +200,7 @@ ArchiveQueue::AdditionSummary ArchiveQueue::addJobsIfNecessaryAndCommit(std::lis
ValueCountMap
minArchiveRequestAgeMap
(
m_payload
.
mutable_minarchiverequestagemap
());
ValueCountMap
minArchiveRequestAgeMap
(
m_payload
.
mutable_minarchiverequestagemap
());
AdditionSummary
ret
;
AdditionSummary
ret
;
for
(
auto
&
jta
:
jobsToAdd
)
{
for
(
auto
&
jta
:
jobsToAdd
)
{
auto
&
jl
=
m_payload
.
pending
archivejobs
();
auto
&
jl
=
m_payload
.
archivejobs
();
for
(
auto
j
=
jl
.
begin
();
j
!=
jl
.
end
();
j
++
)
{
for
(
auto
j
=
jl
.
begin
();
j
!=
jl
.
end
();
j
++
)
{
if
(
j
->
address
()
==
jta
.
archiveRequestAddress
)
if
(
j
->
address
()
==
jta
.
archiveRequestAddress
)
goto
skipInsertion
;
goto
skipInsertion
;
...
@@ -212,7 +210,7 @@ ArchiveQueue::AdditionSummary ArchiveQueue::addJobsIfNecessaryAndCommit(std::lis
...
@@ -212,7 +210,7 @@ ArchiveQueue::AdditionSummary ArchiveQueue::addJobsIfNecessaryAndCommit(std::lis
maxDriveAllowedMap
.
incCount
(
jta
.
policy
.
maxDrivesAllowed
);
maxDriveAllowedMap
.
incCount
(
jta
.
policy
.
maxDrivesAllowed
);
priorityMap
.
incCount
(
jta
.
policy
.
archivePriority
);
priorityMap
.
incCount
(
jta
.
policy
.
archivePriority
);
minArchiveRequestAgeMap
.
incCount
(
jta
.
policy
.
archiveMinRequestAge
);
minArchiveRequestAgeMap
.
incCount
(
jta
.
policy
.
archiveMinRequestAge
);
if
(
m_payload
.
pending
archivejobs_size
())
{
if
(
m_payload
.
archivejobs_size
())
{
if
((
uint64_t
)
jta
.
startTime
<
m_payload
.
oldestjobcreationtime
())
if
((
uint64_t
)
jta
.
startTime
<
m_payload
.
oldestjobcreationtime
())
m_payload
.
set_oldestjobcreationtime
(
jta
.
startTime
);
m_payload
.
set_oldestjobcreationtime
(
jta
.
startTime
);
m_payload
.
set_archivejobstotalsize
(
m_payload
.
archivejobstotalsize
()
+
jta
.
fileSize
);
m_payload
.
set_archivejobstotalsize
(
m_payload
.
archivejobstotalsize
()
+
jta
.
fileSize
);
...
@@ -220,7 +218,7 @@ ArchiveQueue::AdditionSummary ArchiveQueue::addJobsIfNecessaryAndCommit(std::lis
...
@@ -220,7 +218,7 @@ ArchiveQueue::AdditionSummary ArchiveQueue::addJobsIfNecessaryAndCommit(std::lis
m_payload
.
set_archivejobstotalsize
(
jta
.
fileSize
);
m_payload
.
set_archivejobstotalsize
(
jta
.
fileSize
);
m_payload
.
set_oldestjobcreationtime
(
jta
.
startTime
);
m_payload
.
set_oldestjobcreationtime
(
jta
.
startTime
);
}
}
auto
*
j
=
m_payload
.
add_
pending
archivejobs
();
auto
*
j
=
m_payload
.
add_archivejobs
();
j
->
set_address
(
jta
.
archiveRequestAddress
);
j
->
set_address
(
jta
.
archiveRequestAddress
);
j
->
set_size
(
jta
.
fileSize
);
j
->
set_size
(
jta
.
fileSize
);
j
->
set_fileid
(
jta
.
archiveFileId
);
j
->
set_fileid
(
jta
.
archiveFileId
);
...
@@ -243,7 +241,7 @@ void ArchiveQueue::removeJobsAndCommit(const std::list<std::string>& requestsToR
...
@@ -243,7 +241,7 @@ void ArchiveQueue::removeJobsAndCommit(const std::list<std::string>& requestsToR
ValueCountMap
maxDriveAllowedMap
(
m_payload
.
mutable_maxdrivesallowedmap
());
ValueCountMap
maxDriveAllowedMap
(
m_payload
.
mutable_maxdrivesallowedmap
());
ValueCountMap
priorityMap
(
m_payload
.
mutable_prioritymap
());
ValueCountMap
priorityMap
(
m_payload
.
mutable_prioritymap
());
ValueCountMap
minArchiveRequestAgeMap
(
m_payload
.
mutable_minarchiverequestagemap
());
ValueCountMap
minArchiveRequestAgeMap
(
m_payload
.
mutable_minarchiverequestagemap
());
auto
*
jl
=
m_payload
.
mutable_
pending
archivejobs
();
auto
*
jl
=
m_payload
.
mutable_archivejobs
();
bool
jobRemoved
=
false
;
bool
jobRemoved
=
false
;
for
(
auto
&
rrt
:
requestsToRemove
)
{
for
(
auto
&
rrt
:
requestsToRemove
)
{
bool
found
=
false
;
bool
found
=
false
;
...
@@ -275,7 +273,7 @@ void ArchiveQueue::removeJobsAndCommit(const std::list<std::string>& requestsToR
...
@@ -275,7 +273,7 @@ void ArchiveQueue::removeJobsAndCommit(const std::list<std::string>& requestsToR
auto
ArchiveQueue
::
dumpJobs
()
->
std
::
list
<
JobDump
>
{
auto
ArchiveQueue
::
dumpJobs
()
->
std
::
list
<
JobDump
>
{
checkPayloadReadable
();
checkPayloadReadable
();
std
::
list
<
JobDump
>
ret
;
std
::
list
<
JobDump
>
ret
;
auto
&
jl
=
m_payload
.
pending
archivejobs
();
auto
&
jl
=
m_payload
.
archivejobs
();
for
(
auto
j
=
jl
.
begin
();
j
!=
jl
.
end
();
j
++
)
{
for
(
auto
j
=
jl
.
begin
();
j
!=
jl
.
end
();
j
++
)
{
ret
.
push_back
(
JobDump
());
ret
.
push_back
(
JobDump
());
JobDump
&
jd
=
ret
.
back
();
JobDump
&
jd
=
ret
.
back
();
...
@@ -290,8 +288,8 @@ auto ArchiveQueue::getCandidateList(uint64_t maxBytes, uint64_t maxFiles, std::s
...
@@ -290,8 +288,8 @@ auto ArchiveQueue::getCandidateList(uint64_t maxBytes, uint64_t maxFiles, std::s
checkPayloadReadable
();
checkPayloadReadable
();
CandidateJobList
ret
;
CandidateJobList
ret
;
ret
.
remainingBytesAfterCandidates
=
m_payload
.
archivejobstotalsize
();
ret
.
remainingBytesAfterCandidates
=
m_payload
.
archivejobstotalsize
();
ret
.
remainingFilesAfterCandidates
=
m_payload
.
pending
archivejobs_size
();
ret
.
remainingFilesAfterCandidates
=
m_payload
.
archivejobs_size
();
for
(
auto
&
j
:
m_payload
.
pending
archivejobs
())
{
for
(
auto
&
j
:
m_payload
.
archivejobs
())
{
if
(
!
archiveRequestsToSkip
.
count
(
j
.
address
()))
{
if
(
!
archiveRequestsToSkip
.
count
(
j
.
address
()))
{
ret
.
candidates
.
push_back
({
j
.
size
(),
j
.
address
(),
(
uint16_t
)
j
.
copynb
()});
ret
.
candidates
.
push_back
({
j
.
size
(),
j
.
address
(),
(
uint16_t
)
j
.
copynb
()});
ret
.
candidateBytes
+=
j
.
size
();
ret
.
candidateBytes
+=
j
.
size
();
...
...
This diff is collapsed.
Click to expand it.
objectstore/cta.proto
+
1
−
3
View file @
cbb207ff
...
@@ -389,9 +389,7 @@ message ValueCountPair {
...
@@ -389,9 +389,7 @@ message ValueCountPair {
message
ArchiveQueue
{
message
ArchiveQueue
{
required
string
tapepool
=
10000
;
required
string
tapepool
=
10000
;
repeated
ArchiveJobPointer
pendingarchivejobs
=
10010
;
repeated
ArchiveJobPointer
archivejobs
=
10010
;
repeated
ArchiveJobPointer
orphanedarchivejobsnscreation
=
10020
;
repeated
ArchiveJobPointer
orphanedarchivejobsnsdeletion
=
10030
;
repeated
ValueCountPair
prioritymap
=
10031
;
repeated
ValueCountPair
prioritymap
=
10031
;
repeated
ValueCountPair
minarchiverequestagemap
=
10032
;
repeated
ValueCountPair
minarchiverequestagemap
=
10032
;
repeated
ValueCountPair
maxdrivesallowedmap
=
10033
;
repeated
ValueCountPair
maxdrivesallowedmap
=
10033
;
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment