From 748ebdd39886b968d818afb7b9617086fa55cc13 Mon Sep 17 00:00:00 2001 From: Mayeul Aubin Date: Wed, 16 Apr 2025 20:54:21 +0200 Subject: [PATCH] slurm submission limited to array < 800 --- scola.py | 10 +++++----- slurm_submission.py | 11 ++++++++--- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/scola.py b/scola.py index 09e5b12..792496d 100644 --- a/scola.py +++ b/scola.py @@ -1,4 +1,4 @@ -limit_slurm_arrays = 512 + def main_scola(parsed_args): from args_main import parse_arguments_main @@ -52,7 +52,7 @@ def main_scola(parsed_args): print_message("sCOLA finished.", 1, "scola", verbose=parsed_args.verbose) elif parsed_args.execution == "slurm": - from slurm_submission import create_slurm_script, parse_arguments_slurm + from slurm_submission import create_slurm_script, parse_arguments_slurm, limit_slurm_arrays from args_main import parse_arguments_main from parameters_card import parse_arguments_card import os @@ -132,7 +132,7 @@ def main_scola(parsed_args): subprocess.run(command_args) print_message("sCOLA job submitted.", 2, "scola", verbose=parsed_args.verbose) - sleep(len(missing_tiles)*0.2) # Sleep for a bit to avoid overloading the scheduler + sleep((missing_tiles[1]-missing_tiles[0])*1.0) # Sleep for a bit to avoid overloading the scheduler os.remove(slurm_script) # Remove the script after submission (because it is specific to the missing tiles) @@ -243,7 +243,7 @@ def get_missing_tiles_arrays(parsed_args): missing_tiles_arrays.append([b]) in_sequence_of_missing = True len_sequence_of_missing = 1 - if len_sequence_of_missing > limit_slurm_arrays: + if b%limit_slurm_arrays==limit_slurm_arrays-1: missing_tiles_arrays[-1].append(b) in_sequence_of_missing = False elif in_sequence_of_missing: @@ -286,4 +286,4 @@ if __name__ == "__main__": if parsed_args.execution == "slurm": for b in range(1,nboxes_tot+1): wait_until_file_exists(f"{card_dict['OutputTilesBase']}{b}.h5", verbose=parsed_args.verbose, limit=5*60) - main_post_scola(parsed_args) \ No newline at end of file + main_post_scola(parsed_args) diff --git a/slurm_submission.py b/slurm_submission.py index 7bcdcd2..22cea6a 100644 --- a/slurm_submission.py +++ b/slurm_submission.py @@ -2,6 +2,7 @@ from argparse import ArgumentParser from args_main import parse_arguments_main path_to_monofonic_binary = "/home/aubin/monofonic/build/monofonIC" +limit_slurm_arrays=799 def register_arguments_slurm(parser:ArgumentParser): """ @@ -133,12 +134,16 @@ def create_slurm_script(slurm_template:str, - simbelmyne - scola """ + index_sub=0 if array is not None and job != "scola": raise ValueError(f"Array job range provided for job type {job}.") if array is None and job == "scola": raise ValueError(f"Array job range not provided for job type {job}.") - + else: + index_sub=array[0]//limit_slurm_arrays + array=(array[0]%limit_slurm_arrays, array[1]%limit_slurm_arrays) + from os.path import isfile if not isfile(slurm_template): raise FileNotFoundError(f"SLURM template {slurm_template} does not exist.") @@ -155,7 +160,7 @@ def create_slurm_script(slurm_template:str, case "simbelmyne": command_line = f"{job} {job_config_file} {job_log}" case "scola": - command_line = f"{job} {job_config_file} {job_log} "+"-b ${SLURM_ARRAY_TASK_ID}" + command_line = f"{job} {job_config_file} {job_log} "+f"-b $(({index_sub*limit_slurm_arrays} + $SLURM_ARRAY_TASK_ID))" case _: raise ValueError(f"Job type {job} not recognized.") @@ -163,7 +168,7 @@ def create_slurm_script(slurm_template:str, with open(slurm_script, "w") as f: for line in template: if job_name is not None and "--job-name" in line: - line = f"#SBATCH --job-name={job_name}\n" + line = f"#SBATCH --job-name={index_sub if index_sub!=0 else ''}{job_name}\n" if array is not None and "--array" in line: line = f"#SBATCH --array={array[0]}-{array[1]}\n" if array is not None and ("--output" in line or "--error" in line):