From 85e66602ba91bae985ec833534b33f330c4cd864 Mon Sep 17 00:00:00 2001
From: Michael Reuscher <michael.reuscher@desy.de>
Date: Tue, 1 Aug 2023 16:47:56 +0200
Subject: [PATCH] refactor simrunner
---
simrunner/simrunner.py | 105 +++++++++++++++++++++--------------------
1 file changed, 55 insertions(+), 50 deletions(-)
diff --git a/simrunner/simrunner.py b/simrunner/simrunner.py
index 1a4b488..8c25d98 100644
--- a/simrunner/simrunner.py
+++ b/simrunner/simrunner.py
@@ -3,6 +3,11 @@ import json
import subprocess
import argparse
import getpass
+import logging
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
# List of byte sizes for the events
BYTE_SIZES = ['16', '32', '64', '128', '256', '512', '1024', '2048', '4096', '8192',
@@ -10,10 +15,18 @@ BYTE_SIZES = ['16', '32', '64', '128', '256', '512', '1024', '2048', '4096', '81
'4194304', '8388608']
+# Function to print terminal output and handle errors
+def terminal_output(output, errors, return_code):
+ logger.info(output)
+ logger.info(errors)
+ if return_code > 0:
+ logger.info(f'return code: {return_code}')
+
+
# Function to parse command-line arguments
def parse_arguments():
main_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- default_sim_path = os.path.join(os.path.dirname(__file__), "../build/src/detsim")
+ default_sim_path = os.path.join(os.path.dirname(__file__), '../build/src/detsim')
current_user = getpass.getuser()
parser = argparse.ArgumentParser(description='Run a subprocess locally or on remote machines.')
parser.add_argument('-s', '--sim', type=str, default=default_sim_path,
@@ -34,7 +47,7 @@ def parse_arguments():
# Function to read the configuration from a JSON file
def read_config():
config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data_config.json')
- with open(config_path, "r") as file:
+ with open(config_path, 'r') as file:
d_config = json.load(file)
return d_config
@@ -45,8 +58,8 @@ def set_arguments_various(da_config):
e_list = []
# Loop through each byte size to create corresponding event arguments
for event in BYTE_SIZES:
- event_name = "Data_Set_" + event
- arguments = ["-s", event]
+ event_name = 'Data_Set_' + event
+ arguments = ['-s', event]
# Loop through the data configuration and append key-value pairs as arguments
for outer_key in data_config.keys():
if outer_key == 'data_config_various':
@@ -67,7 +80,7 @@ def set_arguments_various(da_config):
def set_arguments_long(da_config):
data_config = da_config
e_list = []
- event_name = "Data_Set_Long"
+ event_name = 'Data_Set_Long'
arguments = []
# Loop through the data configuration and append key-value pairs as arguments
for outer_key in data_config.keys():
@@ -95,15 +108,40 @@ def create_event_list():
return e_list
+# Function to create local folders for test data and output
+def make_folders_lokal(test_path, output_path):
+ os.makedirs(test_path, mode=0o777, exist_ok=True)
+ os.makedirs(output_path, mode=0o777, exist_ok=True)
+
+
+# Function to create remote folders for test data and output
+def make_folders_remote(test_path, output_path, ssh_target, host):
+ host_output = output_path + f'_{host}'
+ test_command = f'ssh {ssh_target} "mkdir -p {test_path}"'
+ output_command = f'ssh {ssh_target} "mkdir -p {host_output}"'
+ subprocess.run(test_command, shell=True, check=True)
+ subprocess.run(output_command, shell=True, check=True)
+ return host_output
+
+
+# Function to pull data from remote machines to the local machine
+def pull_data(source_path, desti_path):
+ if args.hosts:
+ for host in args.hosts:
+ ssh_target = f'{args.user}@{host}'
+ scp_cmd = f'scp -r {ssh_target}:{source_path} {desti_path}'
+ subprocess.run(scp_cmd, shell=True, check=True)
+
+
# Function to run the detector simulation with the given event arguments
def run_sim(e_list, d_path, sim_path, main_path):
# Create the 'output/statistics' directory if it doesn't exist
- stat_path = os.path.join(main_path, "output/statistics")
+ stat_path = os.path.join(main_path, 'output/statistics')
os.makedirs(stat_path, mode=0o777, exist_ok=True)
# Loop through each event and run the detector simulation
for event in e_list:
folder_name = event[-1]
- stat_file_name = "statistics_" + folder_name
+ stat_file_name = 'statistics_' + folder_name
output_path = os.path.join(d_path, folder_name)
if args.hosts:
# Remote execution on multiple machines with specified user
@@ -113,17 +151,17 @@ def run_sim(e_list, d_path, sim_path, main_path):
remote_output = make_folders_remote(test_data_path, output_path, ssh_target, host)
event[-1] = remote_output
# Construct the command to run the simulation on the remote machine via SSH
- command = ["ssh", ssh_target, sim_path] + event
+ command = ['ssh', ssh_target, sim_path] + event
result = subprocess.run(command, capture_output=True, text=True)
output = result.stdout
errors = result.stderr
# Print the simulation output and errors, along with the return code if there is any error
terminal_output(output, errors, result.returncode)
# Write the simulation output to a statistics file
- relativ_path_stat = os.path.join(stat_path, stat_file_name + f"_{host}.dat")
- with open(relativ_path_stat, "w") as stat_file:
+ relativ_path_stat = os.path.join(stat_path, stat_file_name + f'_{host}.dat')
+ with open(relativ_path_stat, 'w') as stat_file:
stat_file.write(output)
- print(f"{folder_name}_{host} done!")
+ logger.info(f'{folder_name}_{host} done!')
# Local execution
else:
# Create local output folders and update the event path accordingly
@@ -137,55 +175,22 @@ def run_sim(e_list, d_path, sim_path, main_path):
# Print the simulation output and errors, along with the return code if there is any error
terminal_output(output, errors, result.returncode)
# Write the simulation output to a statistics file
- relativ_path_stat = os.path.join(stat_path, stat_file_name + ".dat")
- with open(relativ_path_stat, "w") as stat_file:
+ relativ_path_stat = os.path.join(stat_path, stat_file_name + '.dat')
+ with open(relativ_path_stat, 'w') as stat_file:
stat_file.write(output)
- print(f"{folder_name} done!")
-
-
-# Function to print terminal output and handle errors
-def terminal_output(output, errors, return_code):
- print(output)
- print(errors)
- if return_code > 0:
- print("return code: ", return_code)
-
-
-# Function to create local folders for test data and output
-def make_folders_lokal(test_path, output_path):
- os.makedirs(test_path, mode=0o777, exist_ok=True)
- os.makedirs(output_path, mode=0o777, exist_ok=True)
-
-
-# Function to create remote folders for test data and output
-def make_folders_remote(test_path, output_path, ssh_target, host):
- host_output = output_path + f"_{host}"
- test_command = f'ssh {ssh_target} "mkdir -p {test_path}"'
- output_command = f'ssh {ssh_target} "mkdir -p {host_output}"'
- subprocess.run(test_command, shell=True, check=True)
- subprocess.run(output_command, shell=True, check=True)
- return host_output
-
-
-# Function to pull data from remote machines to the local machine
-def pull_data(source_path, desti_path):
- if args.hosts:
- for host in args.hosts:
- ssh_target = f'{args.user}@{host}'
- scp_cmd = f'scp -r {ssh_target}:{source_path} {desti_path}'
- subprocess.run(scp_cmd, shell=True, check=True)
+ logger.info(f'{folder_name} done!')
# --- MAIN ---
-if __name__ == "__main__":
+if __name__ == '__main__':
# Parse arguments
args = parse_arguments()
# Setup all paths and folders
main_folder_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
simulation_path = args.sim
- data_path = os.path.join(args.data, "output")
- test_data_path = os.path.join(args.data, "test_data")
+ data_path = os.path.join(args.data, 'output')
+ test_data_path = os.path.join(args.data, 'test_data')
# Setup arguments for Sim
event_list = create_event_list()
--
GitLab