diff --git a/simrunner/config_processing.py b/simrunner/config_processing.py index 806aff932c57956ae43aaa51668b14b7082f8663..b4379390e72d94e1e0255bdffb9cde4479a5c538 100644 --- a/simrunner/config_processing.py +++ b/simrunner/config_processing.py @@ -5,7 +5,8 @@ import os #BYTE_SIZES = ['16', '32', '64', '128', '256', '512', '1024', '2048', '4096', '8192', #'16384', '32768', '65536', '131072', '262144', '524288', '1048576', '2097152', #'4194304', '8388608', '16777216', '33554432', '67108864'] -BYTE_SIZES = ['16', '32'] +BYTE_SIZES = ['16', '32', '64', '128', '256', '512', '1024', '2048', '4096', '8192', + '16384', '32768', '65536'] # Function to read the configuration from a JSON file diff --git a/simrunner/output_processing.py b/simrunner/output_processing.py index a229e914119192fb9daa218618a2477246f21db2..02445dc894387a58d0aaf3460b70c41419f1c1dd 100644 --- a/simrunner/output_processing.py +++ b/simrunner/output_processing.py @@ -2,6 +2,8 @@ import os import subprocess from main import args from main import logger +import gzip +import tarfile # Function to create local folders for test data and output @@ -31,6 +33,27 @@ def write_statistics(main_path, stat_file_name, output): stat_file.write(output) +def extract_gzip_files(folder_path): + # Iterate through the files in the specified folder + for file_name in os.listdir(folder_path): + # Check if the file has the '.tar.gz' extension + if file_name.endswith('.tar.gz'): + tar_gz_file_path = os.path.join(folder_path, file_name) + output_sub_folder = os.path.join(folder_path) # Remove '.tar.gz' extension + + # Create the output sub_folder if it doesn't exist + os.makedirs(output_sub_folder, exist_ok=True) + + # Extract the gzip tar file + with tarfile.open(tar_gz_file_path, 'r:gz') as tar: + tar.extractall(output_sub_folder) + + # Delete the gz file + os.remove(tar_gz_file_path) + + logger.info('Extraction completed') + + # Function to pull data from remote machines to the local machine def pull_data(desti_path): source_path = os.path.join(args.data, 'output') @@ -39,9 +62,18 @@ def pull_data(desti_path): if args.hosts: for host in args.hosts: ssh_target = f'{args.user}@{host}' - scp_cmd = f'scp -r {ssh_target}:{source_path}/*{host}* {desti}' + # Komprimiere die Dateien auf dem Server mit gzip + gzip_cmd = f'ssh {ssh_target} "cd {source_path} && tar -czf {host}_files.tar.gz *{host}*"' + subprocess.run(gzip_cmd, shell=True, check=True) + + scp_cmd = f'scp {ssh_target}:{source_path}/{host}_files.tar.gz {desti}' subprocess.run(scp_cmd, shell=True, check=True) + rm_cmd = f'ssh {ssh_target} "cd {source_path} && rm -r *_files.tar.gz"' + subprocess.run(rm_cmd, shell=True, check=True) + + extract_gzip_files(desti) + def rm_data(t_data_path): if args.hosts: