Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
FS-SC
AsapoWorker
Commits
1d6d6754
Commit
1d6d6754
authored
Sep 23, 2020
by
Mikhail Karnevskiy
Browse files
Feat: Add data to metadata stream.
parent
16544352
Changes
3
Hide whitespace changes
Inline
Side-by-side
src/AsapoWorker/asapo_receiver.py
View file @
1d6d6754
...
...
@@ -325,10 +325,17 @@ class AsapoMetadataReceiver:
raise
StreamError
(
"Failed to get substream list"
)
from
err
def
get_substream_metadata
(
self
,
substream
):
"""
Return the last available entry from given substream
of the metadata stream
Parameters
----------
substream : str
Name of substream
"""
try
:
data
,
metadata
=
self
.
broker
.
get_by_id
(
1
,
self
.
group_id
,
substream
=
substream
)
return
metadata
return
self
.
broker
.
get_last
(
self
.
group_id
,
substream
=
substream
)
except
asapo_consumer
.
AsapoEndOfStreamError
as
err
:
raise
EndOfStreamError
(
"End of metadata stream"
)
from
err
except
(
asapo_consumer
.
AsapoUnavailableServiceError
,
...
...
src/AsapoWorker/streamer.py
View file @
1d6d6754
...
...
@@ -65,10 +65,10 @@ class Streamer:
log
.
info
(
format_error
(
err
))
# The substream might have been skipped
self
.
_handle_end_of_stream
()
return
None
return
None
,
None
except
TemporaryError
as
err
:
log
.
warn
(
format_error
(
err
))
return
None
return
None
,
None
def
_get_next
(
self
):
try
:
...
...
@@ -164,7 +164,7 @@ class Streamer:
if
self
.
metadata_receiver
:
log
.
info
(
"Waiting for substream metadata"
)
while
not
self
.
stopped
.
is_set
():
substream_metadata
=
self
.
_get_substream_metadata
()
data
,
substream_metadata
=
self
.
_get_substream_metadata
()
if
substream_metadata
:
self
.
_reset_delay_on_error
()
break
...
...
@@ -176,7 +176,7 @@ class Streamer:
return
log
.
info
(
"Performing pre-scan setup"
)
self
.
worker
.
pre_scan
(
substream_metadata
[
"meta"
]
)
self
.
worker
.
pre_scan
(
data
,
substream_metadata
)
log
.
info
(
"Start stream processing."
)
while
not
self
.
stopped
.
is_set
():
...
...
src/AsapoWorker/worker.py
View file @
1d6d6754
...
...
@@ -26,6 +26,20 @@ class Worker:
def
handle_receiver_critical_error
(
self
):
self
.
handle_receiver_error
()
def
pre_scan
(
self
,
data
,
metadata
):
"""
If metadata stream is given this function is called at the beginning of
each stream before get_next is called.
Parameters
----------
data:
data of the metadata stream
metadata: dict
metadata of the metadata steam
"""
pass
def
send
(
self
,
data
,
metadata
):
if
self
.
sender
:
self
.
sender
.
send_data
(
data
,
metadata
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment