Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
D
DetectorWriteSim
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Deploy
Releases
Container registry
Model registry
Analyze
Value stream analytics
Contributor analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
EWMSCP
DetectorWriteSim
Commits
0c334433
Commit
0c334433
authored
1 year ago
by
Michael Reuscher
Browse files
Options
Downloads
Patches
Plain Diff
Simrunner Doku Comments added
parent
8e5a1e19
Branches
Branches containing commit
No related tags found
1 merge request
!4
Python driver and data analysis part
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
simrunner/simrunner.py
+32
-5
32 additions, 5 deletions
simrunner/simrunner.py
with
32 additions
and
5 deletions
simrunner/simrunner.py
+
32
−
5
View file @
0c334433
...
@@ -4,7 +4,13 @@ import subprocess
...
@@ -4,7 +4,13 @@ import subprocess
import
argparse
import
argparse
import
getpass
import
getpass
# List of byte sizes for the events
BYTE_SIZES
=
[
'
16
'
,
'
32
'
,
'
64
'
,
'
128
'
,
'
256
'
,
'
512
'
,
'
1024
'
,
'
2048
'
,
'
4096
'
,
'
8192
'
,
'
16384
'
,
'
32768
'
,
'
65536
'
,
'
131072
'
,
'
262144
'
,
'
524288
'
,
'
1048576
'
,
'
2097152
'
,
'
4194304
'
,
'
8388608
'
]
# Function to parse command-line arguments
def
parse_arguments
():
def
parse_arguments
():
main_directory
=
os
.
path
.
dirname
(
os
.
path
.
dirname
(
os
.
path
.
abspath
(
__file__
)))
main_directory
=
os
.
path
.
dirname
(
os
.
path
.
dirname
(
os
.
path
.
abspath
(
__file__
)))
default_sim_path
=
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
"
../build/src/detsim
"
)
default_sim_path
=
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
"
../build/src/detsim
"
)
...
@@ -25,6 +31,7 @@ def parse_arguments():
...
@@ -25,6 +31,7 @@ def parse_arguments():
return
parser
.
parse_args
()
return
parser
.
parse_args
()
# Function to read the configuration from a JSON file
def
read_config
():
def
read_config
():
config_path
=
os
.
path
.
join
(
os
.
path
.
dirname
(
os
.
path
.
abspath
(
__file__
)),
'
data_config.json
'
)
config_path
=
os
.
path
.
join
(
os
.
path
.
dirname
(
os
.
path
.
abspath
(
__file__
)),
'
data_config.json
'
)
with
open
(
config_path
,
"
r
"
)
as
file
:
with
open
(
config_path
,
"
r
"
)
as
file
:
...
@@ -32,16 +39,15 @@ def read_config():
...
@@ -32,16 +39,15 @@ def read_config():
return
d_config
return
d_config
# Function to set arguments for various data configurations
def
set_arguments_various
(
da_config
):
def
set_arguments_various
(
da_config
):
data_config
=
da_config
data_config
=
da_config
e_list
=
[]
e_list
=
[]
byte_sizes
=
[
'
16
'
,
'
32
'
,
'
64
'
,
'
128
'
,
'
256
'
,
'
512
'
,
'
1024
'
,
'
2048
'
,
'
4096
'
,
'
8192
'
,
# Loop through each byte size to create corresponding event arguments
'
16384
'
,
'
32768
'
,
'
65536
'
,
'
131072
'
,
'
262144
'
,
'
524288
'
,
'
1048576
'
,
'
2097152
'
,
for
event
in
BYTE_SIZES
:
'
4194304
'
,
'
8388608
'
]
for
event
in
byte_sizes
:
event_name
=
"
Data_Set_
"
+
event
event_name
=
"
Data_Set_
"
+
event
arguments
=
[
"
-s
"
,
event
]
arguments
=
[
"
-s
"
,
event
]
# Loop through the data configuration and append key-value pairs as arguments
for
outer_key
in
data_config
.
keys
():
for
outer_key
in
data_config
.
keys
():
if
outer_key
==
'
data_config_various
'
:
if
outer_key
==
'
data_config_various
'
:
inner_dict
=
data_config
[
outer_key
]
inner_dict
=
data_config
[
outer_key
]
...
@@ -53,14 +59,17 @@ def set_arguments_various(da_config):
...
@@ -53,14 +59,17 @@ def set_arguments_various(da_config):
arguments
.
append
(
'
-o
'
)
arguments
.
append
(
'
-o
'
)
arguments
.
append
(
event_name
)
arguments
.
append
(
event_name
)
e_list
.
append
(
arguments
)
e_list
.
append
(
arguments
)
# Return the list of event arguments for various byte sizes
return
e_list
return
e_list
# Function to set arguments for long data configurations
def
set_arguments_long
(
da_config
):
def
set_arguments_long
(
da_config
):
data_config
=
da_config
data_config
=
da_config
e_list
=
[]
e_list
=
[]
event_name
=
"
Data_Set_Long
"
event_name
=
"
Data_Set_Long
"
arguments
=
[]
arguments
=
[]
# Loop through the data configuration and append key-value pairs as arguments
for
outer_key
in
data_config
.
keys
():
for
outer_key
in
data_config
.
keys
():
if
outer_key
==
'
data_config_long
'
:
if
outer_key
==
'
data_config_long
'
:
inner_dict
=
data_config
[
outer_key
]
inner_dict
=
data_config
[
outer_key
]
...
@@ -72,20 +81,26 @@ def set_arguments_long(da_config):
...
@@ -72,20 +81,26 @@ def set_arguments_long(da_config):
arguments
.
append
(
'
-o
'
)
arguments
.
append
(
'
-o
'
)
arguments
.
append
(
event_name
)
arguments
.
append
(
event_name
)
e_list
.
append
(
arguments
)
e_list
.
append
(
arguments
)
# Return the list of event arguments for long data configuration
return
e_list
return
e_list
# Function to create the list of event arguments based on the chosen function
def
create_event_list
():
def
create_event_list
():
if
args
.
function
==
'
long
'
:
if
args
.
function
==
'
long
'
:
e_list
=
set_arguments_long
(
read_config
())
e_list
=
set_arguments_long
(
read_config
())
else
:
else
:
e_list
=
set_arguments_various
(
read_config
())
e_list
=
set_arguments_various
(
read_config
())
# Return the final list of event arguments
return
e_list
return
e_list
# Function to run the detector simulation with the given event arguments
def
run_sim
(
e_list
,
d_path
,
sim_path
,
main_path
):
def
run_sim
(
e_list
,
d_path
,
sim_path
,
main_path
):
# Create the 'output/statistics' directory if it doesn't exist
stat_path
=
os
.
path
.
join
(
main_path
,
"
output/statistics
"
)
stat_path
=
os
.
path
.
join
(
main_path
,
"
output/statistics
"
)
os
.
makedirs
(
stat_path
,
mode
=
0o777
,
exist_ok
=
True
)
os
.
makedirs
(
stat_path
,
mode
=
0o777
,
exist_ok
=
True
)
# Loop through each event and run the detector simulation
for
event
in
e_list
:
for
event
in
e_list
:
folder_name
=
event
[
-
1
]
folder_name
=
event
[
-
1
]
stat_file_name
=
"
statistics_
"
+
folder_name
stat_file_name
=
"
statistics_
"
+
folder_name
...
@@ -94,32 +109,41 @@ def run_sim(e_list, d_path, sim_path, main_path):
...
@@ -94,32 +109,41 @@ def run_sim(e_list, d_path, sim_path, main_path):
# Remote execution on multiple machines with specified user
# Remote execution on multiple machines with specified user
for
host
in
args
.
hosts
:
for
host
in
args
.
hosts
:
ssh_target
=
f
'
{
args
.
user
}
@
{
host
}
'
ssh_target
=
f
'
{
args
.
user
}
@
{
host
}
'
# Create remote output folders and update the event path accordingly
remote_output
=
make_folders_remote
(
test_data_path
,
output_path
,
ssh_target
,
host
)
remote_output
=
make_folders_remote
(
test_data_path
,
output_path
,
ssh_target
,
host
)
event
[
-
1
]
=
remote_output
event
[
-
1
]
=
remote_output
# Construct the command to run the simulation on the remote machine via SSH
command
=
[
"
ssh
"
,
ssh_target
,
sim_path
]
+
event
command
=
[
"
ssh
"
,
ssh_target
,
sim_path
]
+
event
result
=
subprocess
.
run
(
command
,
capture_output
=
True
,
text
=
True
)
result
=
subprocess
.
run
(
command
,
capture_output
=
True
,
text
=
True
)
output
=
result
.
stdout
output
=
result
.
stdout
errors
=
result
.
stderr
errors
=
result
.
stderr
# Print the simulation output and errors, along with the return code if there is any error
terminal_output
(
output
,
errors
,
result
.
returncode
)
terminal_output
(
output
,
errors
,
result
.
returncode
)
# Write the simulation output to a statistics file
relativ_path_stat
=
os
.
path
.
join
(
stat_path
,
stat_file_name
+
f
"
_
{
host
}
.dat
"
)
relativ_path_stat
=
os
.
path
.
join
(
stat_path
,
stat_file_name
+
f
"
_
{
host
}
.dat
"
)
with
open
(
relativ_path_stat
,
"
w
"
)
as
stat_file
:
with
open
(
relativ_path_stat
,
"
w
"
)
as
stat_file
:
stat_file
.
write
(
output
)
stat_file
.
write
(
output
)
print
(
f
"
{
folder_name
}
_
{
host
}
done!
"
)
print
(
f
"
{
folder_name
}
_
{
host
}
done!
"
)
# Local execution
# Local execution
else
:
else
:
# Create local output folders and update the event path accordingly
make_folders_lokal
(
test_data_path
,
output_path
)
make_folders_lokal
(
test_data_path
,
output_path
)
event
[
-
1
]
=
output_path
event
[
-
1
]
=
output_path
# Construct the command to run the simulation locally
command
=
[
sim_path
]
+
event
command
=
[
sim_path
]
+
event
result
=
subprocess
.
run
(
command
,
capture_output
=
True
,
text
=
True
)
result
=
subprocess
.
run
(
command
,
capture_output
=
True
,
text
=
True
)
output
=
result
.
stdout
output
=
result
.
stdout
errors
=
result
.
stderr
errors
=
result
.
stderr
# Print the simulation output and errors, along with the return code if there is any error
terminal_output
(
output
,
errors
,
result
.
returncode
)
terminal_output
(
output
,
errors
,
result
.
returncode
)
# Write the simulation output to a statistics file
relativ_path_stat
=
os
.
path
.
join
(
stat_path
,
stat_file_name
+
"
.dat
"
)
relativ_path_stat
=
os
.
path
.
join
(
stat_path
,
stat_file_name
+
"
.dat
"
)
with
open
(
relativ_path_stat
,
"
w
"
)
as
stat_file
:
with
open
(
relativ_path_stat
,
"
w
"
)
as
stat_file
:
stat_file
.
write
(
output
)
stat_file
.
write
(
output
)
print
(
f
"
{
folder_name
}
done!
"
)
print
(
f
"
{
folder_name
}
done!
"
)
# Function to print terminal output and handle errors
def
terminal_output
(
output
,
errors
,
return_code
):
def
terminal_output
(
output
,
errors
,
return_code
):
print
(
output
)
print
(
output
)
print
(
errors
)
print
(
errors
)
...
@@ -127,11 +151,13 @@ def terminal_output(output, errors, return_code):
...
@@ -127,11 +151,13 @@ def terminal_output(output, errors, return_code):
print
(
"
return code:
"
,
return_code
)
print
(
"
return code:
"
,
return_code
)
# Function to create local folders for test data and output
def
make_folders_lokal
(
test_path
,
output_path
):
def
make_folders_lokal
(
test_path
,
output_path
):
os
.
makedirs
(
test_path
,
mode
=
0o777
,
exist_ok
=
True
)
os
.
makedirs
(
test_path
,
mode
=
0o777
,
exist_ok
=
True
)
os
.
makedirs
(
output_path
,
mode
=
0o777
,
exist_ok
=
True
)
os
.
makedirs
(
output_path
,
mode
=
0o777
,
exist_ok
=
True
)
# Function to create remote folders for test data and output
def
make_folders_remote
(
test_path
,
output_path
,
ssh_target
,
host
):
def
make_folders_remote
(
test_path
,
output_path
,
ssh_target
,
host
):
host_output
=
output_path
+
f
"
_
{
host
}
"
host_output
=
output_path
+
f
"
_
{
host
}
"
test_command
=
f
'
ssh
{
ssh_target
}
"
mkdir -p
{
test_path
}
"'
test_command
=
f
'
ssh
{
ssh_target
}
"
mkdir -p
{
test_path
}
"'
...
@@ -141,6 +167,7 @@ def make_folders_remote(test_path, output_path, ssh_target, host):
...
@@ -141,6 +167,7 @@ def make_folders_remote(test_path, output_path, ssh_target, host):
return
host_output
return
host_output
# Function to pull data from remote machines to the local machine
def
pull_data
(
source_path
,
desti_path
):
def
pull_data
(
source_path
,
desti_path
):
if
args
.
hosts
:
if
args
.
hosts
:
for
host
in
args
.
hosts
:
for
host
in
args
.
hosts
:
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment