allow black and white lists coexist for snooper

producer/event_monitor_producer/src/eventmon_config.cpp

@@ -73,7 +73,6 @@ Error EventMonConfigFactory::CheckConfig() {
     (err = CheckMode()) ||
     (err = CheckLogLevel()) ||
     (err = CheckNThreads()) ||
-    (err = CheckBlackWhiteLists()) ||
     (err = CheckSubsets());
 
 //todo: check monitored folders exist?
@@ -107,14 +106,6 @@ Error EventMonConfigFactory::CheckLogLevel() {
     return err;
 }
 
-Error EventMonConfigFactory::CheckBlackWhiteLists() {
-    if (config.whitelisted_extensions.size() && config.ignored_extensions.size() ) {
-        return  TextError("only one of IgnoreExtensions/WhitelistExtensions can be set");
-    }
-    return nullptr;
-}
-
-
 Error EventMonConfigFactory::CheckNThreads() {
     if (config.nthreads == 0 || config.nthreads > kMaxProcessingThreads ) {
         return  TextError("NThreads should between 1 and " + std::to_string(kMaxProcessingThreads));

producer/event_monitor_producer/src/eventmon_config_factory.h

@@ -19,10 +19,7 @@ class EventMonConfigFactory {
     Error CheckSubsets();
     Error CheckNThreads();
     Error CheckConfig();
-    Error CheckBlackWhiteLists();
-
-
-  };
+};
 
 }
 

producer/event_monitor_producer/src/folder_event_detector.cpp

@@ -15,7 +15,7 @@ inline bool ends_with(std::string const& value, std::string const& ending) {
 }
 
 
-bool FileInList(const std::vector<std::string>&list,const std::string& file) {
+bool FileInList(const std::vector<std::string>& list, const std::string& file) {
     for (auto& ext : list) {
         if (ends_with(file, ext)) {
             return true;
@@ -25,14 +25,14 @@ bool FileInList(const std::vector<std::string>&list,const std::string& file) {
 }
 
 bool FolderEventDetector::IgnoreFile(const std::string& file) {
-    return FileInList(config_->ignored_extensions,file);
+    return FileInList(config_->ignored_extensions, file);
 }
 
 bool FolderEventDetector::FileInWhiteList(const std::string& file) {
     if (config_->whitelisted_extensions.empty()) {
         return true;
     }
-    return FileInList(config_->whitelisted_extensions,file);
+    return FileInList(config_->whitelisted_extensions, file);
 }
 
 

producer/event_monitor_producer/unittests/test_eventmon_config.cpp

@@ -65,7 +65,7 @@ TEST_F(ConfigTests, ReadSettingsOK) {
     test_config.subset_mode = SubSetMode::kBatch;
     test_config.subset_batch_size = 9;
     test_config.stream = "stream";
-    test_config.whitelisted_extensions =  {};
+    test_config.whitelisted_extensions =  {"bla"};
 
     auto err = asapo::SetFolderMonConfig(test_config);
 
@@ -90,7 +90,7 @@ TEST_F(ConfigTests, ReadSettingsOK) {
 
 TEST_F(ConfigTests, ReadSettingsWhiteListOK) {
     asapo::EventMonConfig test_config;
-    test_config.whitelisted_extensions =  {"tmp","test"};
+    test_config.whitelisted_extensions =  {"tmp", "test"};
     test_config.ignored_extensions = {};
 
     auto err = asapo::SetFolderMonConfig(test_config);
@@ -101,19 +101,6 @@ TEST_F(ConfigTests, ReadSettingsWhiteListOK) {
     ASSERT_THAT(config->whitelisted_extensions,  ElementsAre("tmp", "test"));
 }
 
-
-TEST_F(ConfigTests, ReadSettingsErrorBothIgnoreAndWhitelistSet) {
-    asapo::EventMonConfig test_config;
-    test_config.whitelisted_extensions =  {"tmp","test"};
-    test_config.ignored_extensions =  {"tmp","test"};
-
-    auto err = asapo::SetFolderMonConfig(test_config);
-
-    ASSERT_THAT(err, Ne(nullptr));
-}
-
-
-
 TEST_F(ConfigTests, ReadSettingsMultiSourceOK) {
     asapo::EventMonConfig test_config;
     test_config.subset_mode = SubSetMode::kMultiSource;

tests/automatic/bug_fixes/producer_send_after_restart/test.json.in

@@ -8,6 +8,7 @@
  "RootMonitoredFolder":"@ROOT_PATH@test_in",
  "MonitoredSubFolders":["test1"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":true,
  "Stream": "",
  "Subset": {

tests/automatic/bug_fixes/receiver_cpu_usage/test.json.in

@@ -8,6 +8,7 @@
  "RootMonitoredFolder":"@ROOT_PATH@test_in",
  "MonitoredSubFolders":["test1"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":true,
  "Stream": "",
  "Subset": {

tests/automatic/full_chain/simple_chain_filegen/test.json.in

@@ -8,6 +8,7 @@
  "RootMonitoredFolder":"@ROOT_PATH@test_in",
  "MonitoredSubFolders":["test1","test2"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":true,
  "Stream": "",
   "Subset": {

tests/automatic/full_chain/simple_chain_filegen_batches/test.json.in

@@ -8,6 +8,7 @@
  "RootMonitoredFolder":"@ROOT_PATH@test_in",
  "MonitoredSubFolders":["test1","test2"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":false,
  "Stream": "",
  "Subset": {

tests/automatic/full_chain/simple_chain_filegen_multisource/test.json.in

@@ -8,6 +8,7 @@
  "RootMonitoredFolder":"@ROOT_PATH@test_in",
  "MonitoredSubFolders":["test@ID@"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":true,
  "Stream": "",
  "Subset": {

tests/automatic/full_chain/simple_chain_filegen_readdata_cache/test.json.in

@@ -8,6 +8,7 @@
  "RootMonitoredFolder":"@ROOT_PATH@test_in",
  "MonitoredSubFolders":["test1","test2"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":true,
  "Stream": "",
  "Subset": {

tests/automatic/full_chain/simple_chain_filegen_readdata_file/test.json.in

@@ -8,6 +8,7 @@
  "RootMonitoredFolder":"@ROOT_PATH@test_in",
  "MonitoredSubFolders":["test1","test2"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":true,
  "Stream": "",
  "Subset": {

tests/automatic/producer/file_monitor_producer/test.json.in

@@ -8,6 +8,7 @@
  "RootMonitoredFolder":"@ROOT_PATH@test_in",
  "MonitoredSubFolders":["test1","test2"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":true,
  "Stream": "",
  "Subset": {

tests/automatic/producer/python_api/producer_api.py

@@ -44,15 +44,15 @@ producer.send_file(3, local_path = "./not_exist",exposed_path = "./whatever",
 data = np.arange(10,dtype=np.float64)
 
 #send data from array
-err = producer.send_data(4, stream+"/"+"file5",data,
+producer.send_data(4, stream+"/"+"file5",data,
                          ingest_mode = asapo_producer.DEFAULT_INGEST_MODE, callback = callback)
 
 #send data from string
-err = producer.send_data(5, stream+"/"+"file6",b"hello",
+producer.send_data(5, stream+"/"+"file6",b"hello",
                          ingest_mode = asapo_producer.DEFAULT_INGEST_MODE, callback = callback)
 
 #send metadata only
-err = producer.send_data(6, stream+"/"+"file7",None,
+producer.send_data(6, stream+"/"+"file7",None,
                          ingest_mode = asapo_producer.INGEST_MODE_TRANSFER_METADATA_ONLY, callback = callback)
 
 
@@ -67,6 +67,6 @@ else:
 
 
 
-time.sleep(5)
+time.sleep(10)
 
 

tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in

@@ -39,6 +39,7 @@ job "asapo-filemon" {
  "RootMonitoredFolder":"u:\\asapo",
  "MonitoredSubFolders":["test_folder"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":true,
  "Stream": "",
  "Subset": {
@@ -99,6 +100,7 @@ job "asapo-filemon" {
  "RootMonitoredFolder":"/run/user",
  "MonitoredSubFolders":["data"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":true,
  "Stream": "",
  "Subset": {

tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in

@@ -39,6 +39,7 @@ job "asapo-filemon_batch" {
  "RootMonitoredFolder":"u:\\asapo",
  "MonitoredSubFolders":["test_folder"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":true,
  "Stream": "",
  "Subset": {
@@ -100,6 +101,7 @@ job "asapo-filemon_batch" {
  "RootMonitoredFolder":"/run/user",
  "MonitoredSubFolders":["data"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":true,
  "Stream": "",
  "Subset": {

tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in

@@ -39,6 +39,7 @@ job "asapo-filemon_multisource" {
  "RootMonitoredFolder":"u:\\asapo",
  "MonitoredSubFolders":["test_folder"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":true,
  "Stream": "",
  "Subset": {
@@ -101,6 +102,7 @@ job "asapo-filemon_multisource" {
  "RootMonitoredFolder":"/run/user",
  "MonitoredSubFolders":["data"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":true,
  "Stream": "",
  "Subset": {

tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad.in

@@ -39,6 +39,7 @@ job "asapo-produceronly" {
  "RootMonitoredFolder":"c:\\tmp\\asapo\\test_in",
  "MonitoredSubFolders":["test_folder"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":true,
  "Stream": "",
   "Subset": {
@@ -99,6 +100,7 @@ job "asapo-produceronly" {
  "RootMonitoredFolder":"/tmp/asapo/test_in",
  "MonitoredSubFolders":["test_folder"],
  "IgnoreExtensions":["tmp"],
+ "WhitelistExtensions":[],
  "RemoveAfterSend":true,
  "Stream": "",
  "Subset": {