output zip bevor pull and extract in output folder

simrunner/config_processing.py

@@ -5,7 +5,8 @@ import os
 #BYTE_SIZES = ['16', '32', '64', '128', '256', '512', '1024', '2048', '4096', '8192',
                   #'16384', '32768', '65536', '131072', '262144', '524288', '1048576', '2097152',
                   #'4194304', '8388608', '16777216', '33554432', '67108864']
-BYTE_SIZES = ['16', '32']
+BYTE_SIZES = ['16', '32', '64', '128', '256', '512', '1024', '2048', '4096', '8192',
+                  '16384', '32768', '65536']
 
 
 # Function to read the configuration from a JSON file

simrunner/output_processing.py

@@ -2,6 +2,8 @@ import os
 import subprocess
 from main import args
 from main import logger
+import gzip
+import tarfile
 
 
 # Function to create local folders for test data and output
@@ -31,6 +33,27 @@ def write_statistics(main_path, stat_file_name, output):
         stat_file.write(output)
 
 
+def extract_gzip_files(folder_path):
+    # Iterate through the files in the specified folder
+    for file_name in os.listdir(folder_path):
+        # Check if the file has the '.tar.gz' extension
+        if file_name.endswith('.tar.gz'):
+            tar_gz_file_path = os.path.join(folder_path, file_name)
+            output_sub_folder = os.path.join(folder_path)  # Remove '.tar.gz' extension
+
+            # Create the output sub_folder if it doesn't exist
+            os.makedirs(output_sub_folder, exist_ok=True)
+
+            # Extract the gzip tar file
+            with tarfile.open(tar_gz_file_path, 'r:gz') as tar:
+                tar.extractall(output_sub_folder)
+
+            # Delete the gz file
+            os.remove(tar_gz_file_path)
+
+    logger.info('Extraction completed')
+
+
 # Function to pull data from remote machines to the local machine
 def pull_data(desti_path):
     source_path = os.path.join(args.data, 'output')
@@ -39,9 +62,18 @@ def pull_data(desti_path):
     if args.hosts:
         for host in args.hosts:
             ssh_target = f'{args.user}@{host}'
-            scp_cmd = f'scp -r {ssh_target}:{source_path}/*{host}* {desti}'
+            # Komprimiere die Dateien auf dem Server mit gzip
+            gzip_cmd = f'ssh {ssh_target} "cd {source_path} && tar -czf {host}_files.tar.gz *{host}*"'
+            subprocess.run(gzip_cmd, shell=True, check=True)
+
+            scp_cmd = f'scp {ssh_target}:{source_path}/{host}_files.tar.gz {desti}'
             subprocess.run(scp_cmd, shell=True, check=True)
 
+            rm_cmd = f'ssh {ssh_target} "cd {source_path} && rm -r *_files.tar.gz"'
+            subprocess.run(rm_cmd, shell=True, check=True)
+
+            extract_gzip_files(desti)
+
 
 def rm_data(t_data_path):
     if args.hosts: