diff --git a/producer/event_monitor_producer/src/eventmon_config.cpp b/producer/event_monitor_producer/src/eventmon_config.cpp index 759e36232ad0f1ae2be15417f1493c0a4ecbd8b4..4d89f1f13e2331e3b1e1078d2c3538a02569ad70 100644 --- a/producer/event_monitor_producer/src/eventmon_config.cpp +++ b/producer/event_monitor_producer/src/eventmon_config.cpp @@ -46,7 +46,8 @@ Error EventMonConfigFactory::ParseConfigFile(std::string file_name) { (err = parser.GetString("LogLevel", &config.log_level_str)) || (err = parser.GetBool("RemoveAfterSend", &config.remove_after_send)) || (err = parser.GetArrayString("MonitoredSubFolders", &config.monitored_subfolders)) || - (err = parser.GetArrayString("IgnoreExtentions", &config.ignored_extentions)) || + (err = parser.GetArrayString("IgnoreExtensions", &config.ignored_extensions)) || + (err = parser.GetArrayString("WhitelistExtensions", &config.whitelisted_extensions)) || (err = parser.Embedded("Subset").GetString("Mode", &subset_mode)) || (err = SubsetModeToEnum(subset_mode, &config.subset_mode)); if (err) { @@ -72,6 +73,7 @@ Error EventMonConfigFactory::CheckConfig() { (err = CheckMode()) || (err = CheckLogLevel()) || (err = CheckNThreads()) || + (err = CheckBlackWhiteLists()) || (err = CheckSubsets()); //todo: check monitored folders exist? @@ -105,6 +107,13 @@ Error EventMonConfigFactory::CheckLogLevel() { return err; } +Error EventMonConfigFactory::CheckBlackWhiteLists() { + if (config.whitelisted_extensions.size() && config.ignored_extensions.size() ) { + return TextError("only one of IgnoreExtensions/WhitelistExtensions can be set"); + } + return nullptr; +} + Error EventMonConfigFactory::CheckNThreads() { if (config.nthreads == 0 || config.nthreads > kMaxProcessingThreads ) { diff --git a/producer/event_monitor_producer/src/eventmon_config.h b/producer/event_monitor_producer/src/eventmon_config.h index 12ce592a36c432aea33b2f02608a9fd4f276173e..1f042ca49b786e58fa86fe4ed165aade6c054fef 100644 --- a/producer/event_monitor_producer/src/eventmon_config.h +++ b/producer/event_monitor_producer/src/eventmon_config.h @@ -24,7 +24,8 @@ struct EventMonConfig { RequestHandlerType mode = RequestHandlerType::kTcp; std::string root_monitored_folder; std::vector<std::string> monitored_subfolders; - std::vector<std::string> ignored_extentions; + std::vector<std::string> ignored_extensions; + std::vector<std::string> whitelisted_extensions; bool remove_after_send = false; SubSetMode subset_mode = SubSetMode::kNone; uint64_t subset_batch_size = 1; diff --git a/producer/event_monitor_producer/src/eventmon_config_factory.h b/producer/event_monitor_producer/src/eventmon_config_factory.h index 933feb2b994ceea7e75dfe3b909c39964a086af0..c9f6579c53d0ba64ee923cb30c71f1c302ebb94c 100644 --- a/producer/event_monitor_producer/src/eventmon_config_factory.h +++ b/producer/event_monitor_producer/src/eventmon_config_factory.h @@ -19,8 +19,10 @@ class EventMonConfigFactory { Error CheckSubsets(); Error CheckNThreads(); Error CheckConfig(); + Error CheckBlackWhiteLists(); -}; + + }; } diff --git a/producer/event_monitor_producer/src/folder_event_detector.cpp b/producer/event_monitor_producer/src/folder_event_detector.cpp index ab85d2054f83c35cf492c9663389a53765325f2f..1083488f39e8c46524cf9d085424fd40714801a4 100644 --- a/producer/event_monitor_producer/src/folder_event_detector.cpp +++ b/producer/event_monitor_producer/src/folder_event_detector.cpp @@ -15,8 +15,8 @@ inline bool ends_with(std::string const& value, std::string const& ending) { } -bool FolderEventDetector::IgnoreFile(const std::string& file) { - for (auto& ext : config_->ignored_extentions) { +bool FileInList(const std::vector<std::string>&list,const std::string& file) { + for (auto& ext : list) { if (ends_with(file, ext)) { return true; } @@ -24,6 +24,17 @@ bool FolderEventDetector::IgnoreFile(const std::string& file) { return false; } +bool FolderEventDetector::IgnoreFile(const std::string& file) { + return FileInList(config_->ignored_extensions,file); +} + +bool FolderEventDetector::FileInWhiteList(const std::string& file) { + if (config_->whitelisted_extensions.empty()) { + return true; + } + return FileInList(config_->whitelisted_extensions,file); +} + Error FolderEventDetector::UpdateEventsBuffer() { Error err; @@ -37,7 +48,7 @@ Error FolderEventDetector::UpdateEventsBuffer() { } for (auto& file : files) { - if (!IgnoreFile(file)) { + if (!IgnoreFile(file) && FileInWhiteList(file) ) { events_buffer_.emplace_back(EventHeader{0, 0, file}); } } diff --git a/producer/event_monitor_producer/src/folder_event_detector.h b/producer/event_monitor_producer/src/folder_event_detector.h index 472fc451ae1fb1a8bb92041041b9bf8271a5ccd1..9e6c99f940c314cc58f69432cabe123a1c02a865 100644 --- a/producer/event_monitor_producer/src/folder_event_detector.h +++ b/producer/event_monitor_producer/src/folder_event_detector.h @@ -25,6 +25,7 @@ class FolderEventDetector : public AbstractEventDetector { Error UpdateEventsBuffer(); Error GetHeaderFromBuffer(EventHeader* event_header); bool IgnoreFile(const std::string& event); + bool FileInWhiteList(const std::string& file); bool BufferIsEmpty() const; }; diff --git a/producer/event_monitor_producer/unittests/mock_eventmon_config.cpp b/producer/event_monitor_producer/unittests/mock_eventmon_config.cpp index 3a5112aaae98529a42e33293efbbed74d6bd97b7..35fbf17fb638b94a4e6d4ff8a7f0a869d6277ebc 100644 --- a/producer/event_monitor_producer/unittests/mock_eventmon_config.cpp +++ b/producer/event_monitor_producer/unittests/mock_eventmon_config.cpp @@ -86,7 +86,7 @@ Error SetFolderMonConfig (const EventMonConfig& config) { } std::string ignored_exts; - for (auto ext : config.ignored_extentions) { + for (auto ext : config.ignored_extensions) { ignored_exts += "\"" + ext + "\"" + ","; } if (ignored_exts.size()) { @@ -94,9 +94,19 @@ Error SetFolderMonConfig (const EventMonConfig& config) { } + std::string whitelisted_exts; + for (auto ext : config.whitelisted_extensions) { + whitelisted_exts += "\"" + ext + "\"" + ","; + } + if (whitelisted_exts.size()) { + whitelisted_exts.pop_back(); + } + config_string += "," + std::string("\"MonitoredSubFolders\":") + "[" + mon_folders + "]"; config_string += "," + std::string("\"RootMonitoredFolder\":") + "\"" + config.root_monitored_folder + "\""; - config_string += "," + std::string("\"IgnoreExtentions\":") + "[" + ignored_exts + "]"; + config_string += "," + std::string("\"IgnoreExtensions\":") + "[" + ignored_exts + "]"; + config_string += "," + std::string("\"WhitelistExtensions\":") + "[" + whitelisted_exts + "]"; + config_string += "," + std::string("\"Tag\":") + "\"" + config.tag + "\""; config_string += "," + std::string("\"AsapoEndpoint\":") + "\"" + config.asapo_endpoint + "\""; diff --git a/producer/event_monitor_producer/unittests/test_eventmon_config.cpp b/producer/event_monitor_producer/unittests/test_eventmon_config.cpp index 630ef13e8513c786050ba03ee0d698c3db5201ff..266ddac4abe788e4f87df3b1e8ae9c213127c2f5 100644 --- a/producer/event_monitor_producer/unittests/test_eventmon_config.cpp +++ b/producer/event_monitor_producer/unittests/test_eventmon_config.cpp @@ -60,11 +60,12 @@ TEST_F(ConfigTests, ReadSettingsOK) { test_config.mode = asapo::RequestHandlerType::kTcp; test_config.root_monitored_folder = "tmp"; test_config.monitored_subfolders = {"test1", "test2"}; - test_config.ignored_extentions = {"tmp", "test"}; + test_config.ignored_extensions = {"tmp", "test"}; test_config.remove_after_send = true; test_config.subset_mode = SubSetMode::kBatch; test_config.subset_batch_size = 9; test_config.stream = "stream"; + test_config.whitelisted_extensions = {}; auto err = asapo::SetFolderMonConfig(test_config); @@ -79,15 +80,40 @@ TEST_F(ConfigTests, ReadSettingsOK) { ASSERT_THAT(config->mode, Eq(asapo::RequestHandlerType::kTcp)); ASSERT_THAT(config->monitored_subfolders, ElementsAre("test1", "test2")); ASSERT_THAT(config->root_monitored_folder, Eq("tmp")); - ASSERT_THAT(config->ignored_extentions, ElementsAre("tmp", "test")); + ASSERT_THAT(config->ignored_extensions, ElementsAre("tmp", "test")); ASSERT_THAT(config->remove_after_send, Eq(true)); ASSERT_THAT(config->subset_mode, Eq(SubSetMode::kBatch)); ASSERT_THAT(config->subset_batch_size, Eq(9)); ASSERT_THAT(config->stream, Eq("stream")); +} + + +TEST_F(ConfigTests, ReadSettingsWhiteListOK) { + asapo::EventMonConfig test_config; + test_config.whitelisted_extensions = {"tmp","test"}; + test_config.ignored_extensions = {}; + + auto err = asapo::SetFolderMonConfig(test_config); + + auto config = asapo::GetEventMonConfig(); + + ASSERT_THAT(err, Eq(nullptr)); + ASSERT_THAT(config->whitelisted_extensions, ElementsAre("tmp", "test")); +} +TEST_F(ConfigTests, ReadSettingsErrorBothIgnoreAndWhitelistSet) { + asapo::EventMonConfig test_config; + test_config.whitelisted_extensions = {"tmp","test"}; + test_config.ignored_extensions = {"tmp","test"}; + + auto err = asapo::SetFolderMonConfig(test_config); + + ASSERT_THAT(err, Ne(nullptr)); } + + TEST_F(ConfigTests, ReadSettingsMultiSourceOK) { asapo::EventMonConfig test_config; test_config.subset_mode = SubSetMode::kMultiSource; diff --git a/producer/event_monitor_producer/unittests/test_folder_event_detector.cpp b/producer/event_monitor_producer/unittests/test_folder_event_detector.cpp index b2cce0399f0114a625936f2a0de4bed34c3c5f2a..cc46787dd38854b2d5a98e3f1cac3a4655f40330 100644 --- a/producer/event_monitor_producer/unittests/test_folder_event_detector.cpp +++ b/producer/event_monitor_producer/unittests/test_folder_event_detector.cpp @@ -65,7 +65,7 @@ class FolderEventDetectorTests : public testing::Test { } void MockStartMonitoring(); void MockGetEvents(); - void InitiateAndReadSingleEvent(); + asapo::EventHeader InitiateAndReadSingleEvent(); }; void FolderEventDetectorTests::MockStartMonitoring() { @@ -153,13 +153,14 @@ void FolderEventDetectorTests::MockGetEvents() { )); } -void FolderEventDetectorTests::InitiateAndReadSingleEvent() { +asapo::EventHeader FolderEventDetectorTests::InitiateAndReadSingleEvent() { MockStartMonitoring(); MockGetEvents(); detector.StartMonitoring(); asapo::EventHeader event_header; detector.GetNextEvent(&event_header); Mock::VerifyAndClearExpectations(&mock_system_folder_watch); + return event_header; }; @@ -210,7 +211,7 @@ TEST_F(FolderEventDetectorTests, GetNextEventDoesSystemCallIfListEmpty) { } TEST_F(FolderEventDetectorTests, GetNextIgnoresTmpFiles) { - test_config.ignored_extentions = {"tmp"}; + test_config.ignored_extensions = {"tmp"}; InitiateAndReadSingleEvent(); asapo::EventHeader event_header; err = detector.GetNextEvent(&event_header); @@ -222,6 +223,13 @@ TEST_F(FolderEventDetectorTests, GetNextIgnoresTmpFiles) { ASSERT_THAT(err, Eq(asapo::EventMonitorErrorTemplates::kNoNewEvent)); } +TEST_F(FolderEventDetectorTests, GetNextRespectsWhiteList) { + test_config.whitelisted_extensions = {"tmp"}; + auto event_header = InitiateAndReadSingleEvent(); + ASSERT_THAT(err, Eq(nullptr)); + ASSERT_THAT(event_header.file_name, Eq("test3.tmp")); +} + } diff --git a/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in b/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in index 5e120714393ce7008d62dcea60798b71e6762b91..10f619262e9b0df690658a9cc083ed604b993bfe 100644 --- a/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in +++ b/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in @@ -7,7 +7,7 @@ "LogLevel":"debug", "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test1"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in b/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in index 4f4cfcc124f24e24042daa00aa502d4389f46bd4..2b5bdefa3bf0fee4fc576222038a74fe1dbe19a3 100644 --- a/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in +++ b/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in @@ -7,7 +7,7 @@ "LogLevel":"debug", "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test1"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/automatic/full_chain/simple_chain_filegen/test.json.in b/tests/automatic/full_chain/simple_chain_filegen/test.json.in index d08b0242ebce2b0ee56615bd24825e760c1e6dcb..0d301b561a82f4727bc643fbcaebd5594b2661e2 100644 --- a/tests/automatic/full_chain/simple_chain_filegen/test.json.in +++ b/tests/automatic/full_chain/simple_chain_filegen/test.json.in @@ -7,7 +7,7 @@ "LogLevel":"debug", "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test1","test2"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in index c53eecd99f80ba97264a4a9edf102ff6a7fcd2a4..686316ee8610a15b9f0bcd267ce20d185094d98e 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in +++ b/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in @@ -7,7 +7,7 @@ "LogLevel":"debug", "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test1","test2"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":false, "Stream": "", "Subset": { diff --git a/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in index be12d54a998566051a7e30ae2284a0fab99ecc69..9ab36c342129922b4344705247075713fb60662d 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in +++ b/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in @@ -7,7 +7,7 @@ "LogLevel":"debug", "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test@ID@"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in index 501bcff83fe36e74b5224563e5d78313f85863da..eb2324ceb330e02960e571e85e116e98ba460fcd 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in +++ b/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in @@ -7,7 +7,7 @@ "LogLevel":"debug", "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test1","test2"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in index 501bcff83fe36e74b5224563e5d78313f85863da..eb2324ceb330e02960e571e85e116e98ba460fcd 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in +++ b/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in @@ -7,7 +7,7 @@ "LogLevel":"debug", "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test1","test2"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/automatic/producer/file_monitor_producer/test.json.in b/tests/automatic/producer/file_monitor_producer/test.json.in index 60405050db7de85880a45a7388fa2d9796764c96..16442d63f2334696fdc5a2b5fa7376a35f8a522b 100644 --- a/tests/automatic/producer/file_monitor_producer/test.json.in +++ b/tests/automatic/producer/file_monitor_producer/test.json.in @@ -7,7 +7,7 @@ "LogLevel":"debug", "RootMonitoredFolder":"@ROOT_PATH@test_in", "MonitoredSubFolders":["test1","test2"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in index 480df303f522f2448e2c6c4384fb197538371de5..5fadfb9956522d28bcf99f889133b0165b7e52cd 100644 --- a/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in +++ b/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in @@ -38,7 +38,7 @@ job "asapo-filemon" { "LogLevel":"info", "RootMonitoredFolder":"u:\\asapo", "MonitoredSubFolders":["test_folder"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":true, "Stream": "", "Subset": { @@ -98,7 +98,7 @@ job "asapo-filemon" { "LogLevel":"info", "RootMonitoredFolder":"/run/user", "MonitoredSubFolders":["data"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in index a750fff7410e1e51b6e765045052f793acf11d81..a8fa68a9149fbaad0be72b6651612d44f1642ae2 100644 --- a/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in +++ b/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in @@ -38,7 +38,7 @@ job "asapo-filemon_batch" { "LogLevel":"info", "RootMonitoredFolder":"u:\\asapo", "MonitoredSubFolders":["test_folder"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":true, "Stream": "", "Subset": { @@ -99,7 +99,7 @@ job "asapo-filemon_batch" { "LogLevel":"info", "RootMonitoredFolder":"/run/user", "MonitoredSubFolders":["data"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in index 12d7ea1951796a61e4692c64aca60beb1a333950..de29de2423b21aa1b6ca6eff9a115a5978643ac7 100644 --- a/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in +++ b/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in @@ -38,7 +38,7 @@ job "asapo-filemon_multisource" { "LogLevel":"info", "RootMonitoredFolder":"u:\\asapo", "MonitoredSubFolders":["test_folder"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":true, "Stream": "", "Subset": { @@ -100,7 +100,7 @@ job "asapo-filemon_multisource" { "LogLevel":"info", "RootMonitoredFolder":"/run/user", "MonitoredSubFolders":["data"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":true, "Stream": "", "Subset": { diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in index 0c5c80af7ff95a297f526dcbda867dbcd8390773..29154419bee8939f45c3ba402ab339ea90f57b13 100644 --- a/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in +++ b/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in @@ -38,7 +38,7 @@ job "asapo-produceronly" { "LogLevel":"debug", "RootMonitoredFolder":"c:\\tmp\\asapo\\test_in", "MonitoredSubFolders":["test_folder"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":true, "Stream": "", "Subset": { @@ -98,7 +98,7 @@ job "asapo-produceronly" { "LogLevel":"debug", "RootMonitoredFolder":"/tmp/asapo/test_in", "MonitoredSubFolders":["test_folder"], - "IgnoreExtentions":["tmp"], + "IgnoreExtensions":["tmp"], "RemoveAfterSend":true, "Stream": "", "Subset": {