slurm submission limited to array < 800

scola.py

@@ -1,4 +1,4 @@
-limit_slurm_arrays = 512
+
 
 def main_scola(parsed_args):
     from args_main import parse_arguments_main
@@ -52,7 +52,7 @@ def main_scola(parsed_args):
         print_message("sCOLA finished.", 1, "scola", verbose=parsed_args.verbose)
     
     elif parsed_args.execution == "slurm":
-        from slurm_submission import create_slurm_script, parse_arguments_slurm
+        from slurm_submission import create_slurm_script, parse_arguments_slurm, limit_slurm_arrays
         from args_main import parse_arguments_main
         from parameters_card import parse_arguments_card
         import os
@@ -132,7 +132,7 @@ def main_scola(parsed_args):
                     subprocess.run(command_args)
 
                 print_message("sCOLA job submitted.", 2, "scola", verbose=parsed_args.verbose)
-                sleep(len(missing_tiles)*0.2) # Sleep for a bit to avoid overloading the scheduler
+                sleep((missing_tiles[1]-missing_tiles[0])*1.0) # Sleep for a bit to avoid overloading the scheduler
 
                 os.remove(slurm_script) # Remove the script after submission (because it is specific to the missing tiles)
 
@@ -243,7 +243,7 @@ def get_missing_tiles_arrays(parsed_args):
                 missing_tiles_arrays.append([b])
                 in_sequence_of_missing = True
                 len_sequence_of_missing = 1
-            if len_sequence_of_missing > limit_slurm_arrays:
+            if b%limit_slurm_arrays==limit_slurm_arrays-1:
                 missing_tiles_arrays[-1].append(b)
                 in_sequence_of_missing = False
         elif in_sequence_of_missing:
@@ -286,4 +286,4 @@ if __name__ == "__main__":
     if parsed_args.execution == "slurm":
         for b in range(1,nboxes_tot+1):
             wait_until_file_exists(f"{card_dict['OutputTilesBase']}{b}.h5", verbose=parsed_args.verbose, limit=5*60)
-    main_post_scola(parsed_args)
+    main_post_scola(parsed_args)

slurm_submission.py

@@ -2,6 +2,7 @@ from argparse import ArgumentParser
 from args_main import parse_arguments_main
 
 path_to_monofonic_binary = "/home/aubin/monofonic/build/monofonIC"
+limit_slurm_arrays=799
 
 def register_arguments_slurm(parser:ArgumentParser):
     """
@@ -133,12 +134,16 @@ def create_slurm_script(slurm_template:str,
          - simbelmyne
          - scola
      """
+    index_sub=0
     
     if array is not None and job != "scola":
         raise ValueError(f"Array job range provided for job type {job}.")
     if array is None and job == "scola":
         raise ValueError(f"Array job range not provided for job type {job}.")
-    
+    else:
+        index_sub=array[0]//limit_slurm_arrays
+        array=(array[0]%limit_slurm_arrays, array[1]%limit_slurm_arrays)
+        
     from os.path import isfile
     if not isfile(slurm_template):
         raise FileNotFoundError(f"SLURM template {slurm_template} does not exist.")
@@ -155,7 +160,7 @@ def create_slurm_script(slurm_template:str,
         case "simbelmyne":
             command_line = f"{job} {job_config_file} {job_log}"
         case "scola":
-            command_line = f"{job} {job_config_file} {job_log} "+"-b ${SLURM_ARRAY_TASK_ID}"
+            command_line = f"{job} {job_config_file} {job_log} "+f"-b $(({index_sub*limit_slurm_arrays} + $SLURM_ARRAY_TASK_ID))"
         case _:
             raise ValueError(f"Job type {job} not recognized.")
     
@@ -163,7 +168,7 @@ def create_slurm_script(slurm_template:str,
     with open(slurm_script, "w") as f:
         for line in template:
             if job_name is not None and "--job-name" in line:
-                line = f"#SBATCH --job-name={job_name}\n"
+                line = f"#SBATCH --job-name={index_sub if index_sub!=0 else ''}{job_name}\n"
             if array is not None and "--array" in line:
                 line = f"#SBATCH --array={array[0]}-{array[1]}\n"
             if array is not None and ("--output" in line  or "--error" in line):