new ssh and some rework

simrunner/config_processing.py

@@ -7,8 +7,7 @@ import os
 BYTE_SIZES = ['16', '32', '64', '128', '256', '512', '1024', '2048', '4096', '8192',
                   '16384', '32768', '65536', '131072', '262144', '524288', '1048576', '2097152',
                   '4194304', '8388608', '16777216', '33554432', '67108864']
-#BYTE_SIZES = ['16', '32', '64', '128', '256', '512', '1024', '2048', '4096', '8192',
-                  #'16384', '32768', '65536', '131072', '262144', '524288']
+#BYTE_SIZES = ['16', '32', '64', '128']
 
 
 # Function to parse command-line arguments
@@ -43,47 +42,42 @@ def read_config():
 # Function to set arguments for various data configurations
 def set_arguments_various(da_config, test_data_path):
     data_config = da_config
-    e_list = []
-    # Loop through each byte size to create corresponding event arguments
-    for event in BYTE_SIZES:
-        event_name = 'Data_Set_' + event
-        arguments = ['-s', event]
-        # Loop through the data configuration and append key-value pairs as arguments
+    e_dict = {}
+    # Loop durch jede Byte-Größe, um entsprechende Ereignisargumente zu erstellen
+    for size in BYTE_SIZES:
+        event_name = f'Data_Set_{size}'
+        # Loop durch die Datenkonfiguration und füge Schlüssel-Wert-Paare als Argumente hinzu
         for outer_key in data_config.keys():
             if outer_key == 'data_config_various':
                 inner_dict = data_config[outer_key]
+                event_arguments = {}
                 for k, v in inner_dict.items():
-                    arguments.append(k)
-                    arguments.append(v)
-                arguments.append('--posixDataDir')
-                arguments.append(test_data_path)
-                arguments.append('-o')
-                arguments.append(event_name)
-                e_list.append(arguments)
-    # Return the list of event arguments for various byte sizes
-    return e_list
+                    event_arguments[k] = v
+                event_arguments['-s'] = size
+                event_arguments['--posixDataDir'] = test_data_path
+                event_arguments['-o'] = event_name
+                e_dict[event_name] = event_arguments
+    # Gib das Dictionary der Ereignisargumente für verschiedene Byte-Größen zurück
+    return e_dict
 
 
 # Function to set arguments for long data configurations
 def set_arguments_long(da_config, test_data_path):
     data_config = da_config
-    e_list = []
+    e_dict = {}
     event_name = 'Data_Set_Long'
-    arguments = []
     # Loop through the data configuration and append key-value pairs as arguments
     for outer_key in data_config.keys():
         if outer_key == 'data_config_long':
             inner_dict = data_config[outer_key]
+            event_arguments = {}
             for k, v in inner_dict.items():
-                arguments.append(k)
-                arguments.append(v)
-            arguments.append('--posixDataDir')
-            arguments.append(test_data_path)
-            arguments.append('-o')
-            arguments.append(event_name)
-            e_list.append(arguments)
+                event_arguments[k] = v
+            event_arguments['--posixDataDir'] = test_data_path
+            event_arguments['-o'] = event_name
+            e_dict[event_name] = event_arguments
     # Return the list of event arguments for long data configuration
-    return e_list
+    return e_dict
 
 
 # Function to create the list of event arguments based on the chosen function

simrunner/data_config.json

 {
   "data_config_various":{
-    "-n": "200",
+    "-n": "100",
     "-i": "0.1s",
-    "-N": "8"},
+    "-N": "4"},
   "data_config_long":{
     "-s": "32",
     "-n": "5000",

simrunner/main.py

 import os
+import time
 import subprocess
 import logging
 import concurrent.futures as cf
@@ -28,35 +29,44 @@ def pull_data(source_path, desti_path):
             subprocess.run(scp_cmd, shell=True, check=True)
 
 
-# Function to run the detector simulation with the given event arguments
-def run_sim(e_list, d_path, td_path, sim_path, main_path):
+def rm_data(t_data_path):
+    if args.hosts:
+        host = args.hosts[0]
+        ssh_target = f'{args.user}@{host}'
+        rm_cmd = f'ssh {ssh_target} rm -f {t_data_path}/*'
+        subprocess.run(rm_cmd, shell=True, check=True)
+
 
+# Function to run the detector simulation with the given event arguments
+def run_sim(e_dict, d_path, sim_path, main_path):
     with cf.ThreadPoolExecutor() as executor:
-        futures = []
-        for event in e_list:
-            folder_name = event[-1]
+        for event in e_dict:
+            futures = []
+            folder_name = e_dict[event]['-o']
+            e_dict[event]['--startTime'] = f'@{int(time.time() + 10)}'
             output_path = os.path.join(d_path, folder_name)
             if args and args.hosts:
                 # Remote execution on multiple machines with specified user
                 for host in args.hosts:
                     ssh_target = f'{args.user}@{host}'
-                    future = executor.submit(sim.remote_simulation, sim_path, event, output_path, td_path,
+                    future = executor.submit(sim.remote_simulation, sim_path, e_dict[event], output_path,
                                              ssh_target, host)
                     futures.append((folder_name, host, future))
             # Local execution
             else:
-                future = executor.submit(sim.local_simulation, sim_path, event, td_path, output_path)
+                future = executor.submit(sim.local_simulation, sim_path, e_dict[event], output_path)
                 futures.append((folder_name, None, future))
 
-        for folder_name, host, future in futures:
-            output, errors, result = future.result()
-            terminal_output(output, errors, result)
-            if host:
-                stat_file_name = f'statistics_{folder_name}_{host}'
-            else:
-                stat_file_name = f'statistics_{folder_name}_lokal'
-            op.write_statistics(main_path, stat_file_name, output)
-            logger.info(f'{folder_name}_{host if host else ""} done!')
+            for folder_name, host, future in futures:
+                output, errors, result = future.result()
+                terminal_output(output, errors, result)
+                if host:
+                    stat_file_name = f'statistics_{folder_name}_{host}'
+                else:
+                    stat_file_name = f'statistics_{folder_name}_lokal'
+                op.write_statistics(main_path, stat_file_name, output)
+                logger.info(f'{folder_name}_{host if host else ""} done!')
+            rm_data(test_data_path)
 
 
 # --- MAIN ---
@@ -71,10 +81,11 @@ if __name__ == '__main__':
     test_data_path = os.path.join(args.data, 'test_data')
 
     # Setup arguments for Sim
-    event_list = con.create_event_list(args, test_data_path)
-
+    #event_dict = con.create_event_list(args, test_data_path)
     # Run detectorSimulation
-    run_sim(event_list, data_path, test_data_path, simulation_path, main_folder_path)
+    #run_sim(event_dict, data_path, simulation_path, main_folder_path)
 
-    # Pull data for visualizer
     pull_data(data_path, main_folder_path)
+
+
+

simrunner/output_processing.py

@@ -25,6 +25,5 @@ def write_statistics(main_path, stat_file_name, output):
     stat_path = os.path.join(main_path, 'output/statistics')
     os.makedirs(stat_path, mode=0o777, exist_ok=True)
     file_path = os.path.join(stat_path, stat_file_name + '.dat')
-    print(file_path)
     with open(file_path, 'w') as stat_file:
         stat_file.write(output)

simrunner/simulation.py

@@ -3,12 +3,17 @@ import output_processing as op
 
 
 # Function to run the detector simulation remote
-def remote_simulation(sim_path, event, output_path, test_data_path, ssh_target, host):
+def remote_simulation(sim_path, event, output_path, ssh_target, host):
+    test_data_path = event['--posixDataDir']
     # Create remote output folders and update the event path accordingly
     remote_output = op.make_folders_remote(test_data_path, output_path, ssh_target, host)
-    event[-1] = remote_output
+    event['-o'] = remote_output
+    e_list = []
+    for k, v in event.items():
+        e_list.append(k)
+        e_list.append(v)
     # Construct the command to run the simulation on the remote machine via SSH
-    command = ['ssh', ssh_target, sim_path] + event
+    command = ['ssh', ssh_target, sim_path] + e_list
     result = subprocess.run(command, capture_output=True, text=True)
     output = result.stdout
     errors = result.stderr
@@ -16,10 +21,15 @@ def remote_simulation(sim_path, event, output_path, test_data_path, ssh_target,
 
 
 # Function to run the detector simulation local
-def local_simulation(sim_path, event, test_path, output_path):
-    output_path_lokal = op.make_folders_lokal(test_path, output_path)
-    event[-1] = output_path_lokal
-    command = [sim_path] + event
+def local_simulation(sim_path, event, output_path):
+    test_data_path = event['--posixDataDir']
+    output_path_lokal = op.make_folders_lokal(test_data_path, output_path)
+    event['-o'] = output_path_lokal
+    e_list = []
+    for k, v in event.items():
+        e_list.append(k)
+        e_list.append(v)
+    command = [sim_path] + e_list
     result = subprocess.run(command, capture_output=True, text=True)
     output = result.stdout
     errors = result.stderr

visualizer/visualizer/SeabornPlotter.py

@@ -62,6 +62,7 @@ class SeabornPlotter:
         plt.plot(mean_x_values, mean_y_values, color='red', markersize=10, label='Mean')
 
         # Set axis labels and formatting
+        ax.set_ylim(1e-8, 1e1)
         ax.set_xscale('log')
         ax.set_yscale('log')
         ax.set_xlabel("Data size [Bytes]")