From 86da16ad4f179ecd071a3563bbc7da0e65e10a66 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Fri, 27 Sep 2019 14:14:42 +0200
Subject: [PATCH] allow black and white lists coexist for snooper
---
.../src/eventmon_config.cpp | 9 ---------
.../src/eventmon_config_factory.h | 5 +----
.../src/folder_event_detector.cpp | 6 +++---
.../unittests/test_eventmon_config.cpp | 17 ++---------------
.../producer_send_after_restart/test.json.in | 1 +
.../bug_fixes/receiver_cpu_usage/test.json.in | 1 +
.../simple_chain_filegen/test.json.in | 1 +
.../simple_chain_filegen_batches/test.json.in | 1 +
.../test.json.in | 1 +
.../test.json.in | 1 +
.../test.json.in | 1 +
.../producer/file_monitor_producer/test.json.in | 1 +
.../producer/python_api/producer_api.py | 8 ++++----
.../tests_via_nomad/asapo-test_filemon.nomad.in | 2 ++
.../asapo-test_filemon_batch.nomad.in | 2 ++
.../asapo-test_filemon_multisource.nomad.in | 2 ++
...asapo-test_filemon_producer_tolocal.nomad.in | 2 ++
17 files changed, 26 insertions(+), 35 deletions(-)
diff --git a/producer/event_monitor_producer/src/eventmon_config.cpp b/producer/event_monitor_producer/src/eventmon_config.cpp
index 4d89f1f13..cb591e42e 100644
--- a/producer/event_monitor_producer/src/eventmon_config.cpp
+++ b/producer/event_monitor_producer/src/eventmon_config.cpp
@@ -73,7 +73,6 @@ Error EventMonConfigFactory::CheckConfig() {
(err = CheckMode()) ||
(err = CheckLogLevel()) ||
(err = CheckNThreads()) ||
- (err = CheckBlackWhiteLists()) ||
(err = CheckSubsets());
//todo: check monitored folders exist?
@@ -107,14 +106,6 @@ Error EventMonConfigFactory::CheckLogLevel() {
return err;
}
-Error EventMonConfigFactory::CheckBlackWhiteLists() {
- if (config.whitelisted_extensions.size() && config.ignored_extensions.size() ) {
- return TextError("only one of IgnoreExtensions/WhitelistExtensions can be set");
- }
- return nullptr;
-}
-
-
Error EventMonConfigFactory::CheckNThreads() {
if (config.nthreads == 0 || config.nthreads > kMaxProcessingThreads ) {
return TextError("NThreads should between 1 and " + std::to_string(kMaxProcessingThreads));
diff --git a/producer/event_monitor_producer/src/eventmon_config_factory.h b/producer/event_monitor_producer/src/eventmon_config_factory.h
index c9f6579c5..dd46ce76f 100644
--- a/producer/event_monitor_producer/src/eventmon_config_factory.h
+++ b/producer/event_monitor_producer/src/eventmon_config_factory.h
@@ -19,10 +19,7 @@ class EventMonConfigFactory {
Error CheckSubsets();
Error CheckNThreads();
Error CheckConfig();
- Error CheckBlackWhiteLists();
-
-
- };
+};
}
diff --git a/producer/event_monitor_producer/src/folder_event_detector.cpp b/producer/event_monitor_producer/src/folder_event_detector.cpp
index 1083488f3..c3552368a 100644
--- a/producer/event_monitor_producer/src/folder_event_detector.cpp
+++ b/producer/event_monitor_producer/src/folder_event_detector.cpp
@@ -15,7 +15,7 @@ inline bool ends_with(std::string const& value, std::string const& ending) {
}
-bool FileInList(const std::vector<std::string>&list,const std::string& file) {
+bool FileInList(const std::vector<std::string>& list, const std::string& file) {
for (auto& ext : list) {
if (ends_with(file, ext)) {
return true;
@@ -25,14 +25,14 @@ bool FileInList(const std::vector<std::string>&list,const std::string& file) {
}
bool FolderEventDetector::IgnoreFile(const std::string& file) {
- return FileInList(config_->ignored_extensions,file);
+ return FileInList(config_->ignored_extensions, file);
}
bool FolderEventDetector::FileInWhiteList(const std::string& file) {
if (config_->whitelisted_extensions.empty()) {
return true;
}
- return FileInList(config_->whitelisted_extensions,file);
+ return FileInList(config_->whitelisted_extensions, file);
}
diff --git a/producer/event_monitor_producer/unittests/test_eventmon_config.cpp b/producer/event_monitor_producer/unittests/test_eventmon_config.cpp
index 266ddac4a..65251c0a1 100644
--- a/producer/event_monitor_producer/unittests/test_eventmon_config.cpp
+++ b/producer/event_monitor_producer/unittests/test_eventmon_config.cpp
@@ -65,7 +65,7 @@ TEST_F(ConfigTests, ReadSettingsOK) {
test_config.subset_mode = SubSetMode::kBatch;
test_config.subset_batch_size = 9;
test_config.stream = "stream";
- test_config.whitelisted_extensions = {};
+ test_config.whitelisted_extensions = {"bla"};
auto err = asapo::SetFolderMonConfig(test_config);
@@ -90,7 +90,7 @@ TEST_F(ConfigTests, ReadSettingsOK) {
TEST_F(ConfigTests, ReadSettingsWhiteListOK) {
asapo::EventMonConfig test_config;
- test_config.whitelisted_extensions = {"tmp","test"};
+ test_config.whitelisted_extensions = {"tmp", "test"};
test_config.ignored_extensions = {};
auto err = asapo::SetFolderMonConfig(test_config);
@@ -101,19 +101,6 @@ TEST_F(ConfigTests, ReadSettingsWhiteListOK) {
ASSERT_THAT(config->whitelisted_extensions, ElementsAre("tmp", "test"));
}
-
-TEST_F(ConfigTests, ReadSettingsErrorBothIgnoreAndWhitelistSet) {
- asapo::EventMonConfig test_config;
- test_config.whitelisted_extensions = {"tmp","test"};
- test_config.ignored_extensions = {"tmp","test"};
-
- auto err = asapo::SetFolderMonConfig(test_config);
-
- ASSERT_THAT(err, Ne(nullptr));
-}
-
-
-
TEST_F(ConfigTests, ReadSettingsMultiSourceOK) {
asapo::EventMonConfig test_config;
test_config.subset_mode = SubSetMode::kMultiSource;
diff --git a/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in b/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in
index 10f619262..d74bd52eb 100644
--- a/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in
+++ b/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in
@@ -8,6 +8,7 @@
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test1"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in b/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in
index 2b5bdefa3..7cfebfb26 100644
--- a/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in
+++ b/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in
@@ -8,6 +8,7 @@
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test1"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/automatic/full_chain/simple_chain_filegen/test.json.in b/tests/automatic/full_chain/simple_chain_filegen/test.json.in
index 0d301b561..3c77ba335 100644
--- a/tests/automatic/full_chain/simple_chain_filegen/test.json.in
+++ b/tests/automatic/full_chain/simple_chain_filegen/test.json.in
@@ -8,6 +8,7 @@
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test1","test2"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in
index 686316ee8..f1323b8ee 100644
--- a/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in
+++ b/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in
@@ -8,6 +8,7 @@
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test1","test2"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":false,
"Stream": "",
"Subset": {
diff --git a/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in
index 9ab36c342..eadb0bb3e 100644
--- a/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in
+++ b/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in
@@ -8,6 +8,7 @@
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test@ID@"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in
index eb2324ceb..9addfcceb 100644
--- a/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in
+++ b/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in
@@ -8,6 +8,7 @@
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test1","test2"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in
index eb2324ceb..9addfcceb 100644
--- a/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in
+++ b/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in
@@ -8,6 +8,7 @@
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test1","test2"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/automatic/producer/file_monitor_producer/test.json.in b/tests/automatic/producer/file_monitor_producer/test.json.in
index 16442d63f..d88d98b67 100644
--- a/tests/automatic/producer/file_monitor_producer/test.json.in
+++ b/tests/automatic/producer/file_monitor_producer/test.json.in
@@ -8,6 +8,7 @@
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test1","test2"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/automatic/producer/python_api/producer_api.py b/tests/automatic/producer/python_api/producer_api.py
index a83a6b5e4..b2c80dc01 100644
--- a/tests/automatic/producer/python_api/producer_api.py
+++ b/tests/automatic/producer/python_api/producer_api.py
@@ -44,15 +44,15 @@ producer.send_file(3, local_path = "./not_exist",exposed_path = "./whatever",
data = np.arange(10,dtype=np.float64)
#send data from array
-err = producer.send_data(4, stream+"/"+"file5",data,
+producer.send_data(4, stream+"/"+"file5",data,
ingest_mode = asapo_producer.DEFAULT_INGEST_MODE, callback = callback)
#send data from string
-err = producer.send_data(5, stream+"/"+"file6",b"hello",
+producer.send_data(5, stream+"/"+"file6",b"hello",
ingest_mode = asapo_producer.DEFAULT_INGEST_MODE, callback = callback)
#send metadata only
-err = producer.send_data(6, stream+"/"+"file7",None,
+producer.send_data(6, stream+"/"+"file7",None,
ingest_mode = asapo_producer.INGEST_MODE_TRANSFER_METADATA_ONLY, callback = callback)
@@ -67,6 +67,6 @@ else:
-time.sleep(5)
+time.sleep(10)
diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in
index 5fadfb995..929d44cd0 100644
--- a/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in
+++ b/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in
@@ -39,6 +39,7 @@ job "asapo-filemon" {
"RootMonitoredFolder":"u:\\asapo",
"MonitoredSubFolders":["test_folder"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
@@ -99,6 +100,7 @@ job "asapo-filemon" {
"RootMonitoredFolder":"/run/user",
"MonitoredSubFolders":["data"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in
index a8fa68a91..eb73199f2 100644
--- a/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in
+++ b/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in
@@ -39,6 +39,7 @@ job "asapo-filemon_batch" {
"RootMonitoredFolder":"u:\\asapo",
"MonitoredSubFolders":["test_folder"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
@@ -100,6 +101,7 @@ job "asapo-filemon_batch" {
"RootMonitoredFolder":"/run/user",
"MonitoredSubFolders":["data"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in
index de29de242..9d277950f 100644
--- a/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in
+++ b/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in
@@ -39,6 +39,7 @@ job "asapo-filemon_multisource" {
"RootMonitoredFolder":"u:\\asapo",
"MonitoredSubFolders":["test_folder"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
@@ -101,6 +102,7 @@ job "asapo-filemon_multisource" {
"RootMonitoredFolder":"/run/user",
"MonitoredSubFolders":["data"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in
index 29154419b..b627fe870 100644
--- a/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in
+++ b/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in
@@ -39,6 +39,7 @@ job "asapo-produceronly" {
"RootMonitoredFolder":"c:\\tmp\\asapo\\test_in",
"MonitoredSubFolders":["test_folder"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
@@ -99,6 +100,7 @@ job "asapo-produceronly" {
"RootMonitoredFolder":"/tmp/asapo/test_in",
"MonitoredSubFolders":["test_folder"],
"IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
--
GitLab