more improvements

.gitignore

@@ -23,3 +23,8 @@ pysbmy.egg-info/
 log.txt
 testpdf.txt
 tests/
+params/
+results/
+slurm_logs/
+slurm_scripts/
+work/

low_level.py

@@ -185,7 +185,7 @@ def print_ending_module(module:str, verbose:int=1):
         print(print_level(0, module)+f"Ending {module} module.")
 
 
-def wait_until_file_exists(filename:str, verbose:int=1, limit:int=24*3600):
+def wait_until_file_exists(filename:str, verbose:int=1, limit:int=1200):
     """
     Wait until a file exists.
     """
@@ -233,7 +233,7 @@ def progress_bar_from_logfile(filename:str, desc:str="", verbose:int=1, **kwargs
     from tqdm import tqdm
     from time import sleep
     k=0
-    limit=3600
+    limit=600
     update_interval=0.2
     sleep(2) # Wait for the process to be launched, and for the previous log file to be overwritten if necessary.
     wait_until_file_exists(filename, verbose=verbose, limit=limit)
@@ -245,12 +245,12 @@ def progress_bar_from_logfile(filename:str, desc:str="", verbose:int=1, **kwargs
         return
     
     with tqdm(desc=desc, total=total_operations, disable=(verbose==0), **kwargs) as pbar:
-        while current_operation < total_operations and k/update_interval < limit:
+        while current_operation < total_operations and k*update_interval < limit:
             sleep(update_interval)
             current_operation, total_operations = get_progress_from_logfile(filename)
             if current_operation > previous_operation:
                 pbar.update(current_operation-previous_operation)
                 previous_operation = current_operation
             k+=1
-    if k/update_interval >= limit:
+    if k*update_interval >= limit:
         print_message(f"Progress bar timed out after {limit} seconds.", 3, "low level", verbose=verbose)

monofonic.py

@@ -19,16 +19,7 @@ def main_monofonic(parsed_args):
         command_args = [path_to_monofonic_binary, monofonic_dict["config"]]
 
         if parsed_args.verbose < 2:
-            from io import BytesIO
+            subprocess.run(command_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-            from low_level import stdout_redirector, stderr_redirector
-
-            f = BytesIO()
-            g = BytesIO()
-            with stdout_redirector(f):
-                with stderr_redirector(g):
-                    subprocess.run(command_args)
-                g.close()
-            f.close()
         else:
             subprocess.run(command_args)
 
@@ -42,8 +33,8 @@ def main_monofonic(parsed_args):
         main_dict=parse_arguments_main(parsed_args)
         slurm_script = slurm_dict["scripts"]+"monofonic.sh"
 
-        if not isfile(slurm_script):
+        if not isfile(slurm_script) or parsed_args.force:
-            print_message(f"SLURM script {slurm_script} does not exist. Creating it.", 2, "monofonic", verbose=parsed_args.verbose)
+            print_message(f"SLURM script {slurm_script} does not exist (or forced). Creating it.", 2, "monofonic", verbose=parsed_args.verbose)
             create_slurm_script(
                 slurm_template=slurm_dict["monofonic_template"],
                 slurm_script=slurm_script,
@@ -60,16 +51,7 @@ def main_monofonic(parsed_args):
         command_args = ["sbatch", slurm_script]
 
         if parsed_args.verbose < 2:
-            from io import BytesIO
+            subprocess.run(command_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-            from low_level import stdout_redirector, stderr_redirector
-
-            f = BytesIO()
-            g = BytesIO()
-            with stdout_redirector(f):
-                with stderr_redirector(g):
-                    subprocess.run(command_args)
-                g.close()
-            f.close()
         else:
             subprocess.run(command_args)
 
@@ -97,6 +79,6 @@ if __name__ == "__main__":
     register_arguments_slurm(parser)
     register_arguments_card_for_ICs(parser)
     register_arguments_cosmo(parser)
-    parsed_args = parser.parse_args
+    parsed_args = parser.parse_args()
 
     main_monofonic(parsed_args)

scola.py

@@ -44,6 +44,7 @@ def main_scola(parsed_args):
     elif parsed_args.execution == "slurm":
         from slurm_submission import create_slurm_script, parse_arguments_slurm
         from args_main import parse_arguments_main
+        import os
         print_message("Running scola in slurm mode.", 1, "scola", verbose=parsed_args.verbose)
         slurm_dict=parse_arguments_slurm(parsed_args)
         main_dict=parse_arguments_main(parsed_args)
@@ -53,8 +54,8 @@ def main_scola(parsed_args):
             print_message("Submitting all boxes.", 2, "scola", verbose=parsed_args.verbose)
             slurm_script = slurm_dict["scripts"]+"scola.sh"
 
-            if not isfile(slurm_script):
+            if not isfile(slurm_script) or parsed_args.force:
-                print_message(f"SLURM script {slurm_script} does not exist. Creating it.", 2, "scola", verbose=parsed_args.verbose)
+                print_message(f"SLURM script {slurm_script} does not exist (or forced). Creating it.", 2, "scola", verbose=parsed_args.verbose)
                 create_slurm_script(
                     slurm_template=slurm_dict["scola_template"],
                     slurm_script=slurm_script,
@@ -62,6 +63,7 @@ def main_scola(parsed_args):
                     job_config_file=paramfile,
                     job_log=log_file,
                     array=(1, nboxes_tot),
+                    job_name=main_dict["simname"],
                 )
                 print_message(f"SLURM script written to {slurm_script}.", 3, "scola", verbose=parsed_args.verbose)
             else:
@@ -94,6 +96,7 @@ def main_scola(parsed_args):
                         job_config_file=paramfile,
                         job_log=log_file,
                         array=missing_tiles,
+                        job_name=main_dict["simname"],
                     )
                     print_message(f"SLURM script written to {slurm_script}.", 3, "scola", verbose=parsed_args.verbose)
                 else:
@@ -109,8 +112,8 @@ def main_scola(parsed_args):
 
                 print_message("sCOLA job submitted.", 2, "scola", verbose=parsed_args.verbose)
 
+                os.remove(slurm_script) # Remove the script after submission (because it is specific to the missing tiles)
 
-    
     else:
         raise ValueError(f"Execution mode {parsed_args.execution} not recognized.")
     

simbelmyne.py

@@ -33,14 +33,15 @@ def main_simbelmyne(parsed_args):
         main_dict=parse_arguments_main(parsed_args)
         slurm_script = slurm_dict["scripts"]+"simbelmyne_"+main_dict["simname"]+".sh"
 
-        if not isfile(slurm_script):
+        if not isfile(slurm_script) or parsed_args.force:
-            print_message(f"SLURM script {slurm_script} does not exist. Creating it.", 2, "simbelmyne", verbose=parsed_args.verbose)
+            print_message(f"SLURM script {slurm_script} does not exist (or forced). Creating it.", 2, "simbelmyne", verbose=parsed_args.verbose)
             create_slurm_script(
                 slurm_template=slurm_dict["simbelmyne_template"],
                 slurm_script=slurm_script,
                 job="simbelmyne",
                 job_config_file=paramfile,
                 job_log=log_file,
+                job_name=main_dict["simname"],
             )
             print_message(f"SLURM script written to {slurm_script}.", 3, "simbelmyne", verbose=parsed_args.verbose)
         else:

slurm_submission.py

@@ -101,6 +101,21 @@ def create_slurm_template(
 
         f.write(f"export OMP_NUM_THREADS={nthreads}\n\n")
 
+        f.write("start=`date +%s`\n")
+        f.write("%COMMAND%\n")
+        f.write("end=`date +%s`\n")
+        f.write("runtime=$((end-start))\n")
+        f.write("\n")
+        f.write("echo ''\n")
+        f.write("echo ''\n")
+        f.write("echo \"################## RUNTIME ##################\"\n")
+        f.write("echo \"Runtime was $runtime seconds\"\n")
+        #TODO: fix the two lines below
+        # f.write("echo \"Runtime per task was $((${runtime}.0/${SLURM_NTASKS}.0)) seconds\"\n") # not working yet...
+        # f.write("echo \"Runtime per total cpus was $((${runtime}.0/${($SLURM_NTASKS*$SLURM_CPUS_PER_TASK)}.0)) seconds\"\n")
+        f.write("echo \"#############################################\"\n")
+        f.write("\n")
+        f.write("exit 0\n")
 
 
 def create_slurm_script(slurm_template:str,
@@ -109,6 +124,7 @@ def create_slurm_script(slurm_template:str,
                         job_config_file:str,
                         job_log:str,
                         array:tuple|None=None,
+                        job_name:str|None=None,
                         ):
     """
     Creates a SLURM submission script based on the provided template.
@@ -131,25 +147,30 @@ def create_slurm_script(slurm_template:str,
     with open(slurm_template, "r") as f:
         template = f.readlines()
     
+    command_line = ""
+    # Add the job command
+    match job:
+        case "monofonic":
+            command_line = f"{path_to_monofonic_binary} {job_config_file} > {job_log}"
+        case "simbelmyne":
+            command_line = f"{job} {job_config_file} {job_log}"
+        case "scola":
+            command_line = f"{job} {job_config_file} {job_log} "+"-b ${SLURM_ARRAY_TASK_ID}"
+        case _:
+            raise ValueError(f"Job type {job} not recognized.")
+    
     # Create the script file
     with open(slurm_script, "w") as f:
         for line in template:
+            if job_name is not None and "--job-name" in line:
+                line = f"#SBATCH --job-name={job_name}\n"
             if array is not None and "--array" in line:
-                line = line.replace("%a-%b",f"{array[0]}-{array[1]}")
+                line = f"#SBATCH --array={array[0]}-{array[1]}\n"
             if array is not None and ("--output" in line  or "--error" in line):
                 line = line.replace("%j","%a_%A")
+            if "%COMMAND%" in line:
+                line = command_line+"\n"
             f.write(line)
-        
-        # Add the job command
-        match job:
-            case "monofonic":
-                f.write(f"{path_to_monofonic_binary} {job_config_file} > {job_log}")
-            case "simbelmyne":
-                f.write(f"{job} {job_config_file} {job_log}")
-            case "scola":
-                f.write(f"{job} {job_config_file} {job_log} "+"-b ${SLURM_ARRAY_TASK_ID}")
-            case _:
-                raise ValueError(f"Job type {job} not recognized.")