diff --git a/producer/event_monitor_producer/src/eventmon_config.cpp b/producer/event_monitor_producer/src/eventmon_config.cpp index 4d89f1f13e2331e3b1e1078d2c3538a02569ad70..cb591e42ec168a2ec5600f7e4a2fc3ceaf3bb04d 100644 --- a/producer/event_monitor_producer/src/eventmon_config.cpp +++ b/producer/event_monitor_producer/src/eventmon_config.cpp @@ -73,7 +73,6 @@ Error EventMonConfigFactory::CheckConfig() { (err = CheckMode()) || (err = CheckLogLevel()) || (err = CheckNThreads()) || - (err = CheckBlackWhiteLists()) || (err = CheckSubsets()); //todo: check monitored folders exist? @@ -107,14 +106,6 @@ Error EventMonConfigFactory::CheckLogLevel() { return err; } -Error EventMonConfigFactory::CheckBlackWhiteLists() { - if (config.whitelisted_extensions.size() && config.ignored_extensions.size() ) { - return TextError("only one of IgnoreExtensions/WhitelistExtensions can be set"); - } - return nullptr; -} - - Error EventMonConfigFactory::CheckNThreads() { if (config.nthreads == 0 || config.nthreads > kMaxProcessingThreads ) { return TextError("NThreads should between 1 and " + std::to_string(kMaxProcessingThreads)); diff --git a/producer/event_monitor_producer/src/eventmon_config_factory.h b/producer/event_monitor_producer/src/eventmon_config_factory.h index c9f6579c53d0ba64ee923cb30c71f1c302ebb94c..dd46ce76f170c406a6a74dbbf7bfee159a886e52 100644 --- a/producer/event_monitor_producer/src/eventmon_config_factory.h +++ b/producer/event_monitor_producer/src/eventmon_config_factory.h @@ -19,10 +19,7 @@ class EventMonConfigFactory { Error CheckSubsets(); Error CheckNThreads(); Error CheckConfig(); - Error CheckBlackWhiteLists(); - - - }; +}; } diff --git a/producer/event_monitor_producer/src/folder_event_detector.cpp b/producer/event_monitor_producer/src/folder_event_detector.cpp index 1083488f39e8c46524cf9d085424fd40714801a4..c3552368abd455416726d399dc65a0d888b76b30 100644 --- a/producer/event_monitor_producer/src/folder_event_detector.cpp +++ b/producer/event_monitor_producer/src/folder_event_detector.cpp @@ -15,7 +15,7 @@ inline bool ends_with(std::string const& value, std::string const& ending) { } -bool FileInList(const std::vector<std::string>&list,const std::string& file) { +bool FileInList(const std::vector<std::string>& list, const std::string& file) { for (auto& ext : list) { if (ends_with(file, ext)) { return true; @@ -25,14 +25,14 @@ bool FileInList(const std::vector<std::string>&list,const std::string& file) { } bool FolderEventDetector::IgnoreFile(const std::string& file) { - return FileInList(config_->ignored_extensions,file); + return FileInList(config_->ignored_extensions, file); } bool FolderEventDetector::FileInWhiteList(const std::string& file) { if (config_->whitelisted_extensions.empty()) { return true; } - return FileInList(config_->whitelisted_extensions,file); + return FileInList(config_->whitelisted_extensions, file); } diff --git a/producer/event_monitor_producer/unittests/test_eventmon_config.cpp b/producer/event_monitor_producer/unittests/test_eventmon_config.cpp index 266ddac4abe788e4f87df3b1e8ae9c213127c2f5..65251c0a1733b907c00cdb0e1b7543baadeff183 100644 --- a/producer/event_monitor_producer/unittests/test_eventmon_config.cpp +++ b/producer/event_monitor_producer/unittests/test_eventmon_config.cpp @@ -65,7 +65,7 @@ TEST_F(ConfigTests, ReadSettingsOK) { test_config.subset_mode = SubSetMode::kBatch; test_config.subset_batch_size = 9; test_config.stream = "stream"; - test_config.whitelisted_extensions = {}; + test_config.whitelisted_extensions = {"bla"}; auto err = asapo::SetFolderMonConfig(test_config); @@ -90,7 +90,7 @@ TEST_F(ConfigTests, ReadSettingsOK) { TEST_F(ConfigTests, ReadSettingsWhiteListOK) { asapo::EventMonConfig test_config; - test_config.whitelisted_extensions = {"tmp","test"}; + test_config.whitelisted_extensions = {"tmp", "test"}; test_config.ignored_extensions = {}; auto err = asapo::SetFolderMonConfig(test_config); @@ -101,19 +101,6 @@ TEST_F(ConfigTests, ReadSettingsWhiteListOK) { ASSERT_THAT(config->whitelisted_extensions, ElementsAre("tmp", "test")); } - -TEST_F(ConfigTests, ReadSettingsErrorBothIgnoreAndWhitelistSet) { - asapo::EventMonConfig test_config; - test_config.whitelisted_extensions = {"tmp","test"}; - test_config.ignored_extensions = {"tmp","test"}; - - auto err = asapo::SetFolderMonConfig(test_config); - - ASSERT_THAT(err, Ne(nullptr)); -} - - - TEST_F(ConfigTests, ReadSettingsMultiSourceOK) { asapo::EventMonConfig test_config; test_config.subset_mode = SubSetMode::kMultiSource; diff --git a/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in b/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in index 10f619262e9b0df690658a9cc083ed604b993bfe..d74bd52ebcf85d75b4e1533dd0288b71337dc4b2 100644 --- a/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in +++ b/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in @@ -8,6 +8,7 @@ "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test1"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in b/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in index 2b5bdefa3bf0fee4fc576222038a74fe1dbe19a3..7cfebfb26d502bb8699b5f199d178457c9a38872 100644 --- a/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in +++ b/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in @@ -8,6 +8,7 @@ "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test1"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/automatic/full_chain/simple_chain_filegen/test.json.in b/tests/automatic/full_chain/simple_chain_filegen/test.json.in index 0d301b561a82f4727bc643fbcaebd5594b2661e2..3c77ba335ac934c2b4ce32e9f531e21b56058709 100644 --- a/tests/automatic/full_chain/simple_chain_filegen/test.json.in +++ b/tests/automatic/full_chain/simple_chain_filegen/test.json.in @@ -8,6 +8,7 @@ "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test1","test2"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in index 686316ee8610a15b9f0bcd267ce20d185094d98e..f1323b8ee05f6d835a632779eb362354d9595b74 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in +++ b/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in @@ -8,6 +8,7 @@ "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test1","test2"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":false, "Stream": "", "Subset": { diff --git a/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in index 9ab36c342129922b4344705247075713fb60662d..eadb0bb3eeb9603bf30bf3e0d0de13376095de82 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in +++ b/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in @@ -8,6 +8,7 @@ "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test@ID@"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in index eb2324ceb330e02960e571e85e116e98ba460fcd..9addfcceb52b30c4449268cb140fa04700306a59 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in +++ b/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in @@ -8,6 +8,7 @@ "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test1","test2"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in index eb2324ceb330e02960e571e85e116e98ba460fcd..9addfcceb52b30c4449268cb140fa04700306a59 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in +++ b/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in @@ -8,6 +8,7 @@ "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test1","test2"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/automatic/producer/file_monitor_producer/test.json.in b/tests/automatic/producer/file_monitor_producer/test.json.in index 16442d63f2334696fdc5a2b5fa7376a35f8a522b..d88d98b67bd0a0e5264c1169071210c13325c746 100644 --- a/tests/automatic/producer/file_monitor_producer/test.json.in +++ b/tests/automatic/producer/file_monitor_producer/test.json.in @@ -8,6 +8,7 @@ "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test1","test2"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/automatic/producer/python_api/producer_api.py b/tests/automatic/producer/python_api/producer_api.py index a83a6b5e42c6c2201b425a822f0298c343330f45..b2c80dc012ca3ef4a8c8585747152ff6de1e775d 100644 --- a/tests/automatic/producer/python_api/producer_api.py +++ b/tests/automatic/producer/python_api/producer_api.py @@ -44,15 +44,15 @@ producer.send_file(3, local_path = "./not_exist",exposed_path = "./whatever", data = np.arange(10,dtype=np.float64) #send data from array -err = producer.send_data(4, stream+"/"+"file5",data, +producer.send_data(4, stream+"/"+"file5",data, ingest_mode = asapo_producer.DEFAULT_INGEST_MODE, callback = callback) #send data from string -err = producer.send_data(5, stream+"/"+"file6",b"hello", +producer.send_data(5, stream+"/"+"file6",b"hello", ingest_mode = asapo_producer.DEFAULT_INGEST_MODE, callback = callback) #send metadata only -err = producer.send_data(6, stream+"/"+"file7",None, +producer.send_data(6, stream+"/"+"file7",None, ingest_mode = asapo_producer.INGEST_MODE_TRANSFER_METADATA_ONLY, callback = callback) @@ -67,6 +67,6 @@ else: -time.sleep(5) +time.sleep(10) diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in index 5fadfb9956522d28bcf99f889133b0165b7e52cd..929d44cd0270da08cabd3c780fb8e5181c61ad36 100644 --- a/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in +++ b/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in @@ -39,6 +39,7 @@ job "asapo-filemon" { "RootMonitoredFolder":"u:\\asapo", "MonitoredSubFolders":["test_folder"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":true, "Stream": "", "Subset": { @@ -99,6 +100,7 @@ job "asapo-filemon" { "RootMonitoredFolder":"/run/user", "MonitoredSubFolders":["data"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in index a8fa68a9149fbaad0be72b6651612d44f1642ae2..eb73199f22c88df43e3534345b7464296a904021 100644 --- a/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in +++ b/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in @@ -39,6 +39,7 @@ job "asapo-filemon_batch" { "RootMonitoredFolder":"u:\\asapo", "MonitoredSubFolders":["test_folder"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":true, "Stream": "", "Subset": { @@ -100,6 +101,7 @@ job "asapo-filemon_batch" { "RootMonitoredFolder":"/run/user", "MonitoredSubFolders":["data"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in index de29de2423b21aa1b6ca6eff9a115a5978643ac7..9d277950f8afc05862ca2a2034698d7a3359776d 100644 --- a/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in +++ b/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in @@ -39,6 +39,7 @@ job "asapo-filemon_multisource" { "RootMonitoredFolder":"u:\\asapo", "MonitoredSubFolders":["test_folder"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":true, "Stream": "", "Subset": { @@ -101,6 +102,7 @@ job "asapo-filemon_multisource" { "RootMonitoredFolder":"/run/user", "MonitoredSubFolders":["data"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in index 29154419bee8939f45c3ba402ab339ea90f57b13..b627fe8701b27a7fb9c3daa8f6ed69c9437dad9e 100644 --- a/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in +++ b/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in @@ -39,6 +39,7 @@ job "asapo-produceronly" { "RootMonitoredFolder":"c:\\tmp\\asapo\\test_in", "MonitoredSubFolders":["test_folder"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":true, "Stream": "", "Subset": { @@ -99,6 +100,7 @@ job "asapo-produceronly" { "RootMonitoredFolder":"/tmp/asapo/test_in", "MonitoredSubFolders":["test_folder"], "IgnoreExtensions":["tmp"], + "WhitelistExtensions":[], "RemoveAfterSend":true, "Stream": "", "Subset": {