From c57cd0f7ab23bb0bd2e9354e87f105f8ff9de806 Mon Sep 17 00:00:00 2001
From: Michael Reuscher <michael.reuscher@desy.de>
Date: Tue, 1 Aug 2023 17:46:13 +0200
Subject: [PATCH] more refactor simrunner
---
simrunner/{simrunner.py => main.py} | 68 +++++++----------------------
simrunner/output_processing.py | 32 ++++++++++++++
simrunner/simulation.py | 24 ++++++++++
3 files changed, 71 insertions(+), 53 deletions(-)
rename simrunner/{simrunner.py => main.py} (65%)
create mode 100644 simrunner/output_processing.py
create mode 100644 simrunner/simulation.py
diff --git a/simrunner/simrunner.py b/simrunner/main.py
similarity index 65%
rename from simrunner/simrunner.py
rename to simrunner/main.py
index 8c25d98..3a903fa 100644
--- a/simrunner/simrunner.py
+++ b/simrunner/main.py
@@ -3,16 +3,19 @@ import json
import subprocess
import argparse
import getpass
+import simulation as sim
+import output_processing as op
import logging
# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
-# List of byte sizes for the events
-BYTE_SIZES = ['16', '32', '64', '128', '256', '512', '1024', '2048', '4096', '8192',
- '16384', '32768', '65536', '131072', '262144', '524288', '1048576', '2097152',
- '4194304', '8388608']
+# List of byte sizes for the events little one for testing
+#BYTE_SIZES = ['16', '32', '64', '128', '256', '512', '1024', '2048', '4096', '8192',
+ #'16384', '32768', '65536', '131072', '262144', '524288', '1048576', '2097152',
+ #'4194304', '8388608']
+BYTE_SIZES = ['16', '32']
# Function to print terminal output and handle errors
@@ -108,22 +111,6 @@ def create_event_list():
return e_list
-# Function to create local folders for test data and output
-def make_folders_lokal(test_path, output_path):
- os.makedirs(test_path, mode=0o777, exist_ok=True)
- os.makedirs(output_path, mode=0o777, exist_ok=True)
-
-
-# Function to create remote folders for test data and output
-def make_folders_remote(test_path, output_path, ssh_target, host):
- host_output = output_path + f'_{host}'
- test_command = f'ssh {ssh_target} "mkdir -p {test_path}"'
- output_command = f'ssh {ssh_target} "mkdir -p {host_output}"'
- subprocess.run(test_command, shell=True, check=True)
- subprocess.run(output_command, shell=True, check=True)
- return host_output
-
-
# Function to pull data from remote machines to the local machine
def pull_data(source_path, desti_path):
if args.hosts:
@@ -135,49 +122,24 @@ def pull_data(source_path, desti_path):
# Function to run the detector simulation with the given event arguments
def run_sim(e_list, d_path, sim_path, main_path):
- # Create the 'output/statistics' directory if it doesn't exist
- stat_path = os.path.join(main_path, 'output/statistics')
- os.makedirs(stat_path, mode=0o777, exist_ok=True)
- # Loop through each event and run the detector simulation
+ stat_path = op.create_output_directories(main_path)
for event in e_list:
folder_name = event[-1]
stat_file_name = 'statistics_' + folder_name
output_path = os.path.join(d_path, folder_name)
- if args.hosts:
+ if args and args.hosts:
# Remote execution on multiple machines with specified user
for host in args.hosts:
ssh_target = f'{args.user}@{host}'
- # Create remote output folders and update the event path accordingly
- remote_output = make_folders_remote(test_data_path, output_path, ssh_target, host)
- event[-1] = remote_output
- # Construct the command to run the simulation on the remote machine via SSH
- command = ['ssh', ssh_target, sim_path] + event
- result = subprocess.run(command, capture_output=True, text=True)
- output = result.stdout
- errors = result.stderr
- # Print the simulation output and errors, along with the return code if there is any error
- terminal_output(output, errors, result.returncode)
- # Write the simulation output to a statistics file
- relativ_path_stat = os.path.join(stat_path, stat_file_name + f'_{host}.dat')
- with open(relativ_path_stat, 'w') as stat_file:
- stat_file.write(output)
+ output, errors, result = sim.remote_simulation(sim_path, event, output_path, test_data_path, ssh_target, host)
+ terminal_output(output, errors, result)
+ op.write_statistics(stat_path, stat_file_name + f'_{host}', output)
logger.info(f'{folder_name}_{host} done!')
# Local execution
else:
- # Create local output folders and update the event path accordingly
- make_folders_lokal(test_data_path, output_path)
- event[-1] = output_path
- # Construct the command to run the simulation locally
- command = [sim_path] + event
- result = subprocess.run(command, capture_output=True, text=True)
- output = result.stdout
- errors = result.stderr
- # Print the simulation output and errors, along with the return code if there is any error
- terminal_output(output, errors, result.returncode)
- # Write the simulation output to a statistics file
- relativ_path_stat = os.path.join(stat_path, stat_file_name + '.dat')
- with open(relativ_path_stat, 'w') as stat_file:
- stat_file.write(output)
+ output, errors, result = sim.local_simulation(sim_path, event, output_path)
+ terminal_output(output, errors, result)
+ op.write_statistics(stat_path, stat_file_name, output)
logger.info(f'{folder_name} done!')
diff --git a/simrunner/output_processing.py b/simrunner/output_processing.py
new file mode 100644
index 0000000..f33a117
--- /dev/null
+++ b/simrunner/output_processing.py
@@ -0,0 +1,32 @@
+import os
+import subprocess
+
+
+# Function to create local folders for test data and output
+def make_folders_lokal(test_path, output_path):
+ os.makedirs(test_path, mode=0o777, exist_ok=True)
+ os.makedirs(output_path, mode=0o777, exist_ok=True)
+
+
+# Function to create a local statistics folder
+def create_output_directories(main_path):
+ stat_path = os.path.join(main_path, 'output/statistics')
+ os.makedirs(stat_path, mode=0o777, exist_ok=True)
+ return stat_path
+
+
+# Function to write statistics .dat
+def write_statistics(stat_path, stat_file_name, output):
+ relativ_path_stat = os.path.join(stat_path, stat_file_name + '.dat')
+ with open(relativ_path_stat, 'w') as stat_file:
+ stat_file.write(output)
+
+
+# Function to create remote folders for test data and output
+def make_folders_remote(test_path, output_path, ssh_target, host):
+ host_output = output_path + f'_{host}'
+ test_command = f'ssh {ssh_target} "mkdir -p {test_path}"'
+ output_command = f'ssh {ssh_target} "mkdir -p {host_output}"'
+ subprocess.run(test_command, shell=True, check=True)
+ subprocess.run(output_command, shell=True, check=True)
+ return host_output
diff --git a/simrunner/simulation.py b/simrunner/simulation.py
new file mode 100644
index 0000000..9ffad6d
--- /dev/null
+++ b/simrunner/simulation.py
@@ -0,0 +1,24 @@
+import subprocess
+import output_processing as op
+
+
+# Function to run the detector simulation remote
+def remote_simulation(sim_path, event, output_path, test_data_path, ssh_target, host):
+ # Create remote output folders and update the event path accordingly
+ remote_output = op.make_folders_remote(test_data_path, output_path, ssh_target, host)
+ event[-1] = remote_output
+ # Construct the command to run the simulation on the remote machine via SSH
+ command = ['ssh', ssh_target, sim_path] + event
+ result = subprocess.run(command, capture_output=True, text=True)
+ output = result.stdout
+ errors = result.stderr
+ return output, errors, result.returncode
+
+
+# Function to run the detector simulation local
+def local_simulation(sim_path, event, output_path):
+ command = [sim_path] + event
+ result = subprocess.run(command, capture_output=True, text=True)
+ output = result.stdout
+ errors = result.stderr
+ return output, errors, result.returncode
--
GitLab