Skip to content
Snippets Groups Projects
Commit 85e66602 authored by Michael Reuscher's avatar Michael Reuscher
Browse files

refactor simrunner

parent 7271fac8
No related branches found
No related tags found
1 merge request!4Python driver and data analysis part
Hide whitespace changes
Inline Side-by-side
Showing
with 55 additions and 50 deletions
simrunner/simrunner.py
+ 55
− 50
View file @ 85e66602
......@@ -3,6 +3,11 @@ import json
import subprocess
import argparse
import getpass
import logging
# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
# List of byte sizes for the events
BYTE_SIZES = ['16', '32', '64', '128', '256', '512', '1024', '2048', '4096', '8192',
......@@ -10,10 +15,18 @@ BYTE_SIZES = ['16', '32', '64', '128', '256', '512', '1024', '2048', '4096', '81
'4194304', '8388608']
# Function to print terminal output and handle errors
def terminal_output(output, errors, return_code):
logger.info(output)
logger.info(errors)
if return_code > 0:
logger.info(f'return code: {return_code}')
# Function to parse command-line arguments
def parse_arguments():
main_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
default_sim_path = os.path.join(os.path.dirname(__file__), "../build/src/detsim")
default_sim_path = os.path.join(os.path.dirname(__file__), '../build/src/detsim')
current_user = getpass.getuser()
parser = argparse.ArgumentParser(description='Run a subprocess locally or on remote machines.')
parser.add_argument('-s', '--sim', type=str, default=default_sim_path,
......@@ -34,7 +47,7 @@ def parse_arguments():
# Function to read the configuration from a JSON file
def read_config():
config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data_config.json')
with open(config_path, "r") as file:
with open(config_path, 'r') as file:
d_config = json.load(file)
return d_config
......@@ -45,8 +58,8 @@ def set_arguments_various(da_config):
e_list = []
# Loop through each byte size to create corresponding event arguments
for event in BYTE_SIZES:
event_name = "Data_Set_" + event
arguments = ["-s", event]
event_name = 'Data_Set_' + event
arguments = ['-s', event]
# Loop through the data configuration and append key-value pairs as arguments
for outer_key in data_config.keys():
if outer_key == 'data_config_various':
......@@ -67,7 +80,7 @@ def set_arguments_various(da_config):
def set_arguments_long(da_config):
data_config = da_config
e_list = []
event_name = "Data_Set_Long"
event_name = 'Data_Set_Long'
arguments = []
# Loop through the data configuration and append key-value pairs as arguments
for outer_key in data_config.keys():
......@@ -95,15 +108,40 @@ def create_event_list():
return e_list
# Function to create local folders for test data and output
def make_folders_lokal(test_path, output_path):
os.makedirs(test_path, mode=0o777, exist_ok=True)
os.makedirs(output_path, mode=0o777, exist_ok=True)
# Function to create remote folders for test data and output
def make_folders_remote(test_path, output_path, ssh_target, host):
host_output = output_path + f'_{host}'
test_command = f'ssh {ssh_target} "mkdir -p {test_path}"'
output_command = f'ssh {ssh_target} "mkdir -p {host_output}"'
subprocess.run(test_command, shell=True, check=True)
subprocess.run(output_command, shell=True, check=True)
return host_output
# Function to pull data from remote machines to the local machine
def pull_data(source_path, desti_path):
if args.hosts:
for host in args.hosts:
ssh_target = f'{args.user}@{host}'
scp_cmd = f'scp -r {ssh_target}:{source_path} {desti_path}'
subprocess.run(scp_cmd, shell=True, check=True)
# Function to run the detector simulation with the given event arguments
def run_sim(e_list, d_path, sim_path, main_path):
# Create the 'output/statistics' directory if it doesn't exist
stat_path = os.path.join(main_path, "output/statistics")
stat_path = os.path.join(main_path, 'output/statistics')
os.makedirs(stat_path, mode=0o777, exist_ok=True)
# Loop through each event and run the detector simulation
for event in e_list:
folder_name = event[-1]
stat_file_name = "statistics_" + folder_name
stat_file_name = 'statistics_' + folder_name
output_path = os.path.join(d_path, folder_name)
if args.hosts:
# Remote execution on multiple machines with specified user
......@@ -113,17 +151,17 @@ def run_sim(e_list, d_path, sim_path, main_path):
remote_output = make_folders_remote(test_data_path, output_path, ssh_target, host)
event[-1] = remote_output
# Construct the command to run the simulation on the remote machine via SSH
command = ["ssh", ssh_target, sim_path] + event
command = ['ssh', ssh_target, sim_path] + event
result = subprocess.run(command, capture_output=True, text=True)
output = result.stdout
errors = result.stderr
# Print the simulation output and errors, along with the return code if there is any error
terminal_output(output, errors, result.returncode)
# Write the simulation output to a statistics file
relativ_path_stat = os.path.join(stat_path, stat_file_name + f"_{host}.dat")
with open(relativ_path_stat, "w") as stat_file:
relativ_path_stat = os.path.join(stat_path, stat_file_name + f'_{host}.dat')
with open(relativ_path_stat, 'w') as stat_file:
stat_file.write(output)
print(f"{folder_name}_{host} done!")
logger.info(f'{folder_name}_{host} done!')
# Local execution
else:
# Create local output folders and update the event path accordingly
......@@ -137,55 +175,22 @@ def run_sim(e_list, d_path, sim_path, main_path):
# Print the simulation output and errors, along with the return code if there is any error
terminal_output(output, errors, result.returncode)
# Write the simulation output to a statistics file
relativ_path_stat = os.path.join(stat_path, stat_file_name + ".dat")
with open(relativ_path_stat, "w") as stat_file:
relativ_path_stat = os.path.join(stat_path, stat_file_name + '.dat')
with open(relativ_path_stat, 'w') as stat_file:
stat_file.write(output)
print(f"{folder_name} done!")
# Function to print terminal output and handle errors
def terminal_output(output, errors, return_code):
print(output)
print(errors)
if return_code > 0:
print("return code: ", return_code)
# Function to create local folders for test data and output
def make_folders_lokal(test_path, output_path):
os.makedirs(test_path, mode=0o777, exist_ok=True)
os.makedirs(output_path, mode=0o777, exist_ok=True)
# Function to create remote folders for test data and output
def make_folders_remote(test_path, output_path, ssh_target, host):
host_output = output_path + f"_{host}"
test_command = f'ssh {ssh_target} "mkdir -p {test_path}"'
output_command = f'ssh {ssh_target} "mkdir -p {host_output}"'
subprocess.run(test_command, shell=True, check=True)
subprocess.run(output_command, shell=True, check=True)
return host_output
# Function to pull data from remote machines to the local machine
def pull_data(source_path, desti_path):
if args.hosts:
for host in args.hosts:
ssh_target = f'{args.user}@{host}'
scp_cmd = f'scp -r {ssh_target}:{source_path} {desti_path}'
subprocess.run(scp_cmd, shell=True, check=True)
logger.info(f'{folder_name} done!')
# --- MAIN ---
if __name__ == "__main__":
if __name__ == '__main__':
# Parse arguments
args = parse_arguments()
# Setup all paths and folders
main_folder_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
simulation_path = args.sim
data_path = os.path.join(args.data, "output")
test_data_path = os.path.join(args.data, "test_data")
data_path = os.path.join(args.data, 'output')
test_data_path = os.path.join(args.data, 'test_data')
# Setup arguments for Sim
event_list = create_event_list()
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment

imprint | privacy | Conditions of Use