From a6b3ab5559d478dea41a7bbe2fac10f8792304bb Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Fri, 27 Sep 2019 14:13:05 +0200
Subject: [PATCH] add whitelist to file snooper. Correct extension spelling in
config file
---
.../src/eventmon_config.cpp | 11 ++++++-
.../src/eventmon_config.h | 3 +-
.../src/eventmon_config_factory.h | 4 ++-
.../src/folder_event_detector.cpp | 17 +++++++++--
.../src/folder_event_detector.h | 1 +
.../unittests/mock_eventmon_config.cpp | 14 +++++++--
.../unittests/test_eventmon_config.cpp | 30 +++++++++++++++++--
.../unittests/test_folder_event_detector.cpp | 14 +++++++--
.../producer_send_after_restart/test.json.in | 2 +-
.../bug_fixes/receiver_cpu_usage/test.json.in | 2 +-
.../simple_chain_filegen/test.json.in | 2 +-
.../simple_chain_filegen_batches/test.json.in | 2 +-
.../test.json.in | 2 +-
.../test.json.in | 2 +-
.../test.json.in | 2 +-
.../file_monitor_producer/test.json.in | 2 +-
.../asapo-test_filemon.nomad.in | 4 +--
.../asapo-test_filemon_batch.nomad.in | 4 +--
.../asapo-test_filemon_multisource.nomad.in | 4 +--
...apo-test_filemon_producer_tolocal.nomad.in | 4 +--
20 files changed, 97 insertions(+), 29 deletions(-)
diff --git a/producer/event_monitor_producer/src/eventmon_config.cpp b/producer/event_monitor_producer/src/eventmon_config.cpp
index 759e36232..4d89f1f13 100644
--- a/producer/event_monitor_producer/src/eventmon_config.cpp
+++ b/producer/event_monitor_producer/src/eventmon_config.cpp
@@ -46,7 +46,8 @@ Error EventMonConfigFactory::ParseConfigFile(std::string file_name) {
(err = parser.GetString("LogLevel", &config.log_level_str)) ||
(err = parser.GetBool("RemoveAfterSend", &config.remove_after_send)) ||
(err = parser.GetArrayString("MonitoredSubFolders", &config.monitored_subfolders)) ||
- (err = parser.GetArrayString("IgnoreExtentions", &config.ignored_extentions)) ||
+ (err = parser.GetArrayString("IgnoreExtensions", &config.ignored_extensions)) ||
+ (err = parser.GetArrayString("WhitelistExtensions", &config.whitelisted_extensions)) ||
(err = parser.Embedded("Subset").GetString("Mode", &subset_mode)) ||
(err = SubsetModeToEnum(subset_mode, &config.subset_mode));
if (err) {
@@ -72,6 +73,7 @@ Error EventMonConfigFactory::CheckConfig() {
(err = CheckMode()) ||
(err = CheckLogLevel()) ||
(err = CheckNThreads()) ||
+ (err = CheckBlackWhiteLists()) ||
(err = CheckSubsets());
//todo: check monitored folders exist?
@@ -105,6 +107,13 @@ Error EventMonConfigFactory::CheckLogLevel() {
return err;
}
+Error EventMonConfigFactory::CheckBlackWhiteLists() {
+ if (config.whitelisted_extensions.size() && config.ignored_extensions.size() ) {
+ return TextError("only one of IgnoreExtensions/WhitelistExtensions can be set");
+ }
+ return nullptr;
+}
+
Error EventMonConfigFactory::CheckNThreads() {
if (config.nthreads == 0 || config.nthreads > kMaxProcessingThreads ) {
diff --git a/producer/event_monitor_producer/src/eventmon_config.h b/producer/event_monitor_producer/src/eventmon_config.h
index 12ce592a3..1f042ca49 100644
--- a/producer/event_monitor_producer/src/eventmon_config.h
+++ b/producer/event_monitor_producer/src/eventmon_config.h
@@ -24,7 +24,8 @@ struct EventMonConfig {
RequestHandlerType mode = RequestHandlerType::kTcp;
std::string root_monitored_folder;
std::vector<std::string> monitored_subfolders;
- std::vector<std::string> ignored_extentions;
+ std::vector<std::string> ignored_extensions;
+ std::vector<std::string> whitelisted_extensions;
bool remove_after_send = false;
SubSetMode subset_mode = SubSetMode::kNone;
uint64_t subset_batch_size = 1;
diff --git a/producer/event_monitor_producer/src/eventmon_config_factory.h b/producer/event_monitor_producer/src/eventmon_config_factory.h
index 933feb2b9..c9f6579c5 100644
--- a/producer/event_monitor_producer/src/eventmon_config_factory.h
+++ b/producer/event_monitor_producer/src/eventmon_config_factory.h
@@ -19,8 +19,10 @@ class EventMonConfigFactory {
Error CheckSubsets();
Error CheckNThreads();
Error CheckConfig();
+ Error CheckBlackWhiteLists();
-};
+
+ };
}
diff --git a/producer/event_monitor_producer/src/folder_event_detector.cpp b/producer/event_monitor_producer/src/folder_event_detector.cpp
index ab85d2054..1083488f3 100644
--- a/producer/event_monitor_producer/src/folder_event_detector.cpp
+++ b/producer/event_monitor_producer/src/folder_event_detector.cpp
@@ -15,8 +15,8 @@ inline bool ends_with(std::string const& value, std::string const& ending) {
}
-bool FolderEventDetector::IgnoreFile(const std::string& file) {
- for (auto& ext : config_->ignored_extentions) {
+bool FileInList(const std::vector<std::string>&list,const std::string& file) {
+ for (auto& ext : list) {
if (ends_with(file, ext)) {
return true;
}
@@ -24,6 +24,17 @@ bool FolderEventDetector::IgnoreFile(const std::string& file) {
return false;
}
+bool FolderEventDetector::IgnoreFile(const std::string& file) {
+ return FileInList(config_->ignored_extensions,file);
+}
+
+bool FolderEventDetector::FileInWhiteList(const std::string& file) {
+ if (config_->whitelisted_extensions.empty()) {
+ return true;
+ }
+ return FileInList(config_->whitelisted_extensions,file);
+}
+
Error FolderEventDetector::UpdateEventsBuffer() {
Error err;
@@ -37,7 +48,7 @@ Error FolderEventDetector::UpdateEventsBuffer() {
}
for (auto& file : files) {
- if (!IgnoreFile(file)) {
+ if (!IgnoreFile(file) && FileInWhiteList(file) ) {
events_buffer_.emplace_back(EventHeader{0, 0, file});
}
}
diff --git a/producer/event_monitor_producer/src/folder_event_detector.h b/producer/event_monitor_producer/src/folder_event_detector.h
index 472fc451a..9e6c99f94 100644
--- a/producer/event_monitor_producer/src/folder_event_detector.h
+++ b/producer/event_monitor_producer/src/folder_event_detector.h
@@ -25,6 +25,7 @@ class FolderEventDetector : public AbstractEventDetector {
Error UpdateEventsBuffer();
Error GetHeaderFromBuffer(EventHeader* event_header);
bool IgnoreFile(const std::string& event);
+ bool FileInWhiteList(const std::string& file);
bool BufferIsEmpty() const;
};
diff --git a/producer/event_monitor_producer/unittests/mock_eventmon_config.cpp b/producer/event_monitor_producer/unittests/mock_eventmon_config.cpp
index 3a5112aaa..35fbf17fb 100644
--- a/producer/event_monitor_producer/unittests/mock_eventmon_config.cpp
+++ b/producer/event_monitor_producer/unittests/mock_eventmon_config.cpp
@@ -86,7 +86,7 @@ Error SetFolderMonConfig (const EventMonConfig& config) {
}
std::string ignored_exts;
- for (auto ext : config.ignored_extentions) {
+ for (auto ext : config.ignored_extensions) {
ignored_exts += "\"" + ext + "\"" + ",";
}
if (ignored_exts.size()) {
@@ -94,9 +94,19 @@ Error SetFolderMonConfig (const EventMonConfig& config) {
}
+ std::string whitelisted_exts;
+ for (auto ext : config.whitelisted_extensions) {
+ whitelisted_exts += "\"" + ext + "\"" + ",";
+ }
+ if (whitelisted_exts.size()) {
+ whitelisted_exts.pop_back();
+ }
+
config_string += "," + std::string("\"MonitoredSubFolders\":") + "[" + mon_folders + "]";
config_string += "," + std::string("\"RootMonitoredFolder\":") + "\"" + config.root_monitored_folder + "\"";
- config_string += "," + std::string("\"IgnoreExtentions\":") + "[" + ignored_exts + "]";
+ config_string += "," + std::string("\"IgnoreExtensions\":") + "[" + ignored_exts + "]";
+ config_string += "," + std::string("\"WhitelistExtensions\":") + "[" + whitelisted_exts + "]";
+
config_string += "," + std::string("\"Tag\":") + "\"" + config.tag + "\"";
config_string += "," + std::string("\"AsapoEndpoint\":") + "\"" + config.asapo_endpoint + "\"";
diff --git a/producer/event_monitor_producer/unittests/test_eventmon_config.cpp b/producer/event_monitor_producer/unittests/test_eventmon_config.cpp
index 630ef13e8..266ddac4a 100644
--- a/producer/event_monitor_producer/unittests/test_eventmon_config.cpp
+++ b/producer/event_monitor_producer/unittests/test_eventmon_config.cpp
@@ -60,11 +60,12 @@ TEST_F(ConfigTests, ReadSettingsOK) {
test_config.mode = asapo::RequestHandlerType::kTcp;
test_config.root_monitored_folder = "tmp";
test_config.monitored_subfolders = {"test1", "test2"};
- test_config.ignored_extentions = {"tmp", "test"};
+ test_config.ignored_extensions = {"tmp", "test"};
test_config.remove_after_send = true;
test_config.subset_mode = SubSetMode::kBatch;
test_config.subset_batch_size = 9;
test_config.stream = "stream";
+ test_config.whitelisted_extensions = {};
auto err = asapo::SetFolderMonConfig(test_config);
@@ -79,15 +80,40 @@ TEST_F(ConfigTests, ReadSettingsOK) {
ASSERT_THAT(config->mode, Eq(asapo::RequestHandlerType::kTcp));
ASSERT_THAT(config->monitored_subfolders, ElementsAre("test1", "test2"));
ASSERT_THAT(config->root_monitored_folder, Eq("tmp"));
- ASSERT_THAT(config->ignored_extentions, ElementsAre("tmp", "test"));
+ ASSERT_THAT(config->ignored_extensions, ElementsAre("tmp", "test"));
ASSERT_THAT(config->remove_after_send, Eq(true));
ASSERT_THAT(config->subset_mode, Eq(SubSetMode::kBatch));
ASSERT_THAT(config->subset_batch_size, Eq(9));
ASSERT_THAT(config->stream, Eq("stream"));
+}
+
+
+TEST_F(ConfigTests, ReadSettingsWhiteListOK) {
+ asapo::EventMonConfig test_config;
+ test_config.whitelisted_extensions = {"tmp","test"};
+ test_config.ignored_extensions = {};
+
+ auto err = asapo::SetFolderMonConfig(test_config);
+
+ auto config = asapo::GetEventMonConfig();
+
+ ASSERT_THAT(err, Eq(nullptr));
+ ASSERT_THAT(config->whitelisted_extensions, ElementsAre("tmp", "test"));
+}
+TEST_F(ConfigTests, ReadSettingsErrorBothIgnoreAndWhitelistSet) {
+ asapo::EventMonConfig test_config;
+ test_config.whitelisted_extensions = {"tmp","test"};
+ test_config.ignored_extensions = {"tmp","test"};
+
+ auto err = asapo::SetFolderMonConfig(test_config);
+
+ ASSERT_THAT(err, Ne(nullptr));
}
+
+
TEST_F(ConfigTests, ReadSettingsMultiSourceOK) {
asapo::EventMonConfig test_config;
test_config.subset_mode = SubSetMode::kMultiSource;
diff --git a/producer/event_monitor_producer/unittests/test_folder_event_detector.cpp b/producer/event_monitor_producer/unittests/test_folder_event_detector.cpp
index b2cce0399..cc46787dd 100644
--- a/producer/event_monitor_producer/unittests/test_folder_event_detector.cpp
+++ b/producer/event_monitor_producer/unittests/test_folder_event_detector.cpp
@@ -65,7 +65,7 @@ class FolderEventDetectorTests : public testing::Test {
}
void MockStartMonitoring();
void MockGetEvents();
- void InitiateAndReadSingleEvent();
+ asapo::EventHeader InitiateAndReadSingleEvent();
};
void FolderEventDetectorTests::MockStartMonitoring() {
@@ -153,13 +153,14 @@ void FolderEventDetectorTests::MockGetEvents() {
));
}
-void FolderEventDetectorTests::InitiateAndReadSingleEvent() {
+asapo::EventHeader FolderEventDetectorTests::InitiateAndReadSingleEvent() {
MockStartMonitoring();
MockGetEvents();
detector.StartMonitoring();
asapo::EventHeader event_header;
detector.GetNextEvent(&event_header);
Mock::VerifyAndClearExpectations(&mock_system_folder_watch);
+ return event_header;
};
@@ -210,7 +211,7 @@ TEST_F(FolderEventDetectorTests, GetNextEventDoesSystemCallIfListEmpty) {
}
TEST_F(FolderEventDetectorTests, GetNextIgnoresTmpFiles) {
- test_config.ignored_extentions = {"tmp"};
+ test_config.ignored_extensions = {"tmp"};
InitiateAndReadSingleEvent();
asapo::EventHeader event_header;
err = detector.GetNextEvent(&event_header);
@@ -222,6 +223,13 @@ TEST_F(FolderEventDetectorTests, GetNextIgnoresTmpFiles) {
ASSERT_THAT(err, Eq(asapo::EventMonitorErrorTemplates::kNoNewEvent));
}
+TEST_F(FolderEventDetectorTests, GetNextRespectsWhiteList) {
+ test_config.whitelisted_extensions = {"tmp"};
+ auto event_header = InitiateAndReadSingleEvent();
+ ASSERT_THAT(err, Eq(nullptr));
+ ASSERT_THAT(event_header.file_name, Eq("test3.tmp"));
+}
+
}
diff --git a/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in b/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in
index 5e1207143..10f619262 100644
--- a/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in
+++ b/tests/automatic/bug_fixes/producer_send_after_restart/test.json.in
@@ -7,7 +7,7 @@
"LogLevel":"debug",
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test1"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in b/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in
index 4f4cfcc12..2b5bdefa3 100644
--- a/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in
+++ b/tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in
@@ -7,7 +7,7 @@
"LogLevel":"debug",
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test1"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/automatic/full_chain/simple_chain_filegen/test.json.in b/tests/automatic/full_chain/simple_chain_filegen/test.json.in
index d08b0242e..0d301b561 100644
--- a/tests/automatic/full_chain/simple_chain_filegen/test.json.in
+++ b/tests/automatic/full_chain/simple_chain_filegen/test.json.in
@@ -7,7 +7,7 @@
"LogLevel":"debug",
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test1","test2"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in
index c53eecd99..686316ee8 100644
--- a/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in
+++ b/tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in
@@ -7,7 +7,7 @@
"LogLevel":"debug",
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test1","test2"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":false,
"Stream": "",
"Subset": {
diff --git a/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in
index be12d54a9..9ab36c342 100644
--- a/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in
+++ b/tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in
@@ -7,7 +7,7 @@
"LogLevel":"debug",
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test@ID@"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in
index 501bcff83..eb2324ceb 100644
--- a/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in
+++ b/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in
@@ -7,7 +7,7 @@
"LogLevel":"debug",
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test1","test2"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in b/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in
index 501bcff83..eb2324ceb 100644
--- a/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in
+++ b/tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in
@@ -7,7 +7,7 @@
"LogLevel":"debug",
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test1","test2"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/automatic/producer/file_monitor_producer/test.json.in b/tests/automatic/producer/file_monitor_producer/test.json.in
index 60405050d..16442d63f 100644
--- a/tests/automatic/producer/file_monitor_producer/test.json.in
+++ b/tests/automatic/producer/file_monitor_producer/test.json.in
@@ -7,7 +7,7 @@
"LogLevel":"debug",
"RootMonitoredFolder":"@ROOT_PATH@test_in",
"MonitoredSubFolders":["test1","test2"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in
index 480df303f..5fadfb995 100644
--- a/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in
+++ b/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in
@@ -38,7 +38,7 @@ job "asapo-filemon" {
"LogLevel":"info",
"RootMonitoredFolder":"u:\\asapo",
"MonitoredSubFolders":["test_folder"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
@@ -98,7 +98,7 @@ job "asapo-filemon" {
"LogLevel":"info",
"RootMonitoredFolder":"/run/user",
"MonitoredSubFolders":["data"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in
index a750fff74..a8fa68a91 100644
--- a/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in
+++ b/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in
@@ -38,7 +38,7 @@ job "asapo-filemon_batch" {
"LogLevel":"info",
"RootMonitoredFolder":"u:\\asapo",
"MonitoredSubFolders":["test_folder"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
@@ -99,7 +99,7 @@ job "asapo-filemon_batch" {
"LogLevel":"info",
"RootMonitoredFolder":"/run/user",
"MonitoredSubFolders":["data"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in
index 12d7ea195..de29de242 100644
--- a/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in
+++ b/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in
@@ -38,7 +38,7 @@ job "asapo-filemon_multisource" {
"LogLevel":"info",
"RootMonitoredFolder":"u:\\asapo",
"MonitoredSubFolders":["test_folder"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
@@ -100,7 +100,7 @@ job "asapo-filemon_multisource" {
"LogLevel":"info",
"RootMonitoredFolder":"/run/user",
"MonitoredSubFolders":["data"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in
index 0c5c80af7..29154419b 100644
--- a/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in
+++ b/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in
@@ -38,7 +38,7 @@ job "asapo-produceronly" {
"LogLevel":"debug",
"RootMonitoredFolder":"c:\\tmp\\asapo\\test_in",
"MonitoredSubFolders":["test_folder"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
@@ -98,7 +98,7 @@ job "asapo-produceronly" {
"LogLevel":"debug",
"RootMonitoredFolder":"/tmp/asapo/test_in",
"MonitoredSubFolders":["test_folder"],
- "IgnoreExtentions":["tmp"],
+ "IgnoreExtensions":["tmp"],
"RemoveAfterSend":true,
"Stream": "",
"Subset": {
--
GitLab