diff --git a/.gitignore b/.gitignore index 6e4b37f..b6e6a4f 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,8 @@ pysbmy.egg-info/ log.txt testpdf.txt tests/ +params/ +results/ +slurm_logs/ +slurm_scripts/ +work/ diff --git a/low_level.py b/low_level.py index 05b5126..bd6d4f2 100644 --- a/low_level.py +++ b/low_level.py @@ -185,7 +185,7 @@ def print_ending_module(module:str, verbose:int=1): print(print_level(0, module)+f"Ending {module} module.") -def wait_until_file_exists(filename:str, verbose:int=1, limit:int=24*3600): +def wait_until_file_exists(filename:str, verbose:int=1, limit:int=1200): """ Wait until a file exists. """ @@ -233,7 +233,7 @@ def progress_bar_from_logfile(filename:str, desc:str="", verbose:int=1, **kwargs from tqdm import tqdm from time import sleep k=0 - limit=3600 + limit=600 update_interval=0.2 sleep(2) # Wait for the process to be launched, and for the previous log file to be overwritten if necessary. wait_until_file_exists(filename, verbose=verbose, limit=limit) @@ -245,12 +245,12 @@ def progress_bar_from_logfile(filename:str, desc:str="", verbose:int=1, **kwargs return with tqdm(desc=desc, total=total_operations, disable=(verbose==0), **kwargs) as pbar: - while current_operation < total_operations and k/update_interval < limit: + while current_operation < total_operations and k*update_interval < limit: sleep(update_interval) current_operation, total_operations = get_progress_from_logfile(filename) if current_operation > previous_operation: pbar.update(current_operation-previous_operation) previous_operation = current_operation k+=1 - if k/update_interval >= limit: + if k*update_interval >= limit: print_message(f"Progress bar timed out after {limit} seconds.", 3, "low level", verbose=verbose) \ No newline at end of file diff --git a/monofonic.py b/monofonic.py index 07b4c6d..6c0da8e 100644 --- a/monofonic.py +++ b/monofonic.py @@ -19,16 +19,7 @@ def main_monofonic(parsed_args): command_args = [path_to_monofonic_binary, monofonic_dict["config"]] if parsed_args.verbose < 2: - from io import BytesIO - from low_level import stdout_redirector, stderr_redirector - - f = BytesIO() - g = BytesIO() - with stdout_redirector(f): - with stderr_redirector(g): - subprocess.run(command_args) - g.close() - f.close() + subprocess.run(command_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) else: subprocess.run(command_args) @@ -42,8 +33,8 @@ def main_monofonic(parsed_args): main_dict=parse_arguments_main(parsed_args) slurm_script = slurm_dict["scripts"]+"monofonic.sh" - if not isfile(slurm_script): - print_message(f"SLURM script {slurm_script} does not exist. Creating it.", 2, "monofonic", verbose=parsed_args.verbose) + if not isfile(slurm_script) or parsed_args.force: + print_message(f"SLURM script {slurm_script} does not exist (or forced). Creating it.", 2, "monofonic", verbose=parsed_args.verbose) create_slurm_script( slurm_template=slurm_dict["monofonic_template"], slurm_script=slurm_script, @@ -60,16 +51,7 @@ def main_monofonic(parsed_args): command_args = ["sbatch", slurm_script] if parsed_args.verbose < 2: - from io import BytesIO - from low_level import stdout_redirector, stderr_redirector - - f = BytesIO() - g = BytesIO() - with stdout_redirector(f): - with stderr_redirector(g): - subprocess.run(command_args) - g.close() - f.close() + subprocess.run(command_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) else: subprocess.run(command_args) @@ -97,6 +79,6 @@ if __name__ == "__main__": register_arguments_slurm(parser) register_arguments_card_for_ICs(parser) register_arguments_cosmo(parser) - parsed_args = parser.parse_args + parsed_args = parser.parse_args() main_monofonic(parsed_args) \ No newline at end of file diff --git a/scola.py b/scola.py index 01e6d32..6114721 100644 --- a/scola.py +++ b/scola.py @@ -44,6 +44,7 @@ def main_scola(parsed_args): elif parsed_args.execution == "slurm": from slurm_submission import create_slurm_script, parse_arguments_slurm from args_main import parse_arguments_main + import os print_message("Running scola in slurm mode.", 1, "scola", verbose=parsed_args.verbose) slurm_dict=parse_arguments_slurm(parsed_args) main_dict=parse_arguments_main(parsed_args) @@ -53,8 +54,8 @@ def main_scola(parsed_args): print_message("Submitting all boxes.", 2, "scola", verbose=parsed_args.verbose) slurm_script = slurm_dict["scripts"]+"scola.sh" - if not isfile(slurm_script): - print_message(f"SLURM script {slurm_script} does not exist. Creating it.", 2, "scola", verbose=parsed_args.verbose) + if not isfile(slurm_script) or parsed_args.force: + print_message(f"SLURM script {slurm_script} does not exist (or forced). Creating it.", 2, "scola", verbose=parsed_args.verbose) create_slurm_script( slurm_template=slurm_dict["scola_template"], slurm_script=slurm_script, @@ -62,6 +63,7 @@ def main_scola(parsed_args): job_config_file=paramfile, job_log=log_file, array=(1, nboxes_tot), + job_name=main_dict["simname"], ) print_message(f"SLURM script written to {slurm_script}.", 3, "scola", verbose=parsed_args.verbose) else: @@ -94,6 +96,7 @@ def main_scola(parsed_args): job_config_file=paramfile, job_log=log_file, array=missing_tiles, + job_name=main_dict["simname"], ) print_message(f"SLURM script written to {slurm_script}.", 3, "scola", verbose=parsed_args.verbose) else: @@ -109,8 +112,8 @@ def main_scola(parsed_args): print_message("sCOLA job submitted.", 2, "scola", verbose=parsed_args.verbose) + os.remove(slurm_script) # Remove the script after submission (because it is specific to the missing tiles) - else: raise ValueError(f"Execution mode {parsed_args.execution} not recognized.") diff --git a/simbelmyne.py b/simbelmyne.py index cb3d377..417845d 100644 --- a/simbelmyne.py +++ b/simbelmyne.py @@ -33,14 +33,15 @@ def main_simbelmyne(parsed_args): main_dict=parse_arguments_main(parsed_args) slurm_script = slurm_dict["scripts"]+"simbelmyne_"+main_dict["simname"]+".sh" - if not isfile(slurm_script): - print_message(f"SLURM script {slurm_script} does not exist. Creating it.", 2, "simbelmyne", verbose=parsed_args.verbose) + if not isfile(slurm_script) or parsed_args.force: + print_message(f"SLURM script {slurm_script} does not exist (or forced). Creating it.", 2, "simbelmyne", verbose=parsed_args.verbose) create_slurm_script( slurm_template=slurm_dict["simbelmyne_template"], slurm_script=slurm_script, job="simbelmyne", job_config_file=paramfile, job_log=log_file, + job_name=main_dict["simname"], ) print_message(f"SLURM script written to {slurm_script}.", 3, "simbelmyne", verbose=parsed_args.verbose) else: diff --git a/slurm_submission.py b/slurm_submission.py index fab0025..7bcdcd2 100644 --- a/slurm_submission.py +++ b/slurm_submission.py @@ -101,6 +101,21 @@ def create_slurm_template( f.write(f"export OMP_NUM_THREADS={nthreads}\n\n") + f.write("start=`date +%s`\n") + f.write("%COMMAND%\n") + f.write("end=`date +%s`\n") + f.write("runtime=$((end-start))\n") + f.write("\n") + f.write("echo ''\n") + f.write("echo ''\n") + f.write("echo \"################## RUNTIME ##################\"\n") + f.write("echo \"Runtime was $runtime seconds\"\n") + #TODO: fix the two lines below + # f.write("echo \"Runtime per task was $((${runtime}.0/${SLURM_NTASKS}.0)) seconds\"\n") # not working yet... + # f.write("echo \"Runtime per total cpus was $((${runtime}.0/${($SLURM_NTASKS*$SLURM_CPUS_PER_TASK)}.0)) seconds\"\n") + f.write("echo \"#############################################\"\n") + f.write("\n") + f.write("exit 0\n") def create_slurm_script(slurm_template:str, @@ -109,6 +124,7 @@ def create_slurm_script(slurm_template:str, job_config_file:str, job_log:str, array:tuple|None=None, + job_name:str|None=None, ): """ Creates a SLURM submission script based on the provided template. @@ -131,25 +147,30 @@ def create_slurm_script(slurm_template:str, with open(slurm_template, "r") as f: template = f.readlines() + command_line = "" + # Add the job command + match job: + case "monofonic": + command_line = f"{path_to_monofonic_binary} {job_config_file} > {job_log}" + case "simbelmyne": + command_line = f"{job} {job_config_file} {job_log}" + case "scola": + command_line = f"{job} {job_config_file} {job_log} "+"-b ${SLURM_ARRAY_TASK_ID}" + case _: + raise ValueError(f"Job type {job} not recognized.") + # Create the script file with open(slurm_script, "w") as f: for line in template: + if job_name is not None and "--job-name" in line: + line = f"#SBATCH --job-name={job_name}\n" if array is not None and "--array" in line: - line = line.replace("%a-%b",f"{array[0]}-{array[1]}") + line = f"#SBATCH --array={array[0]}-{array[1]}\n" if array is not None and ("--output" in line or "--error" in line): line = line.replace("%j","%a_%A") + if "%COMMAND%" in line: + line = command_line+"\n" f.write(line) - - # Add the job command - match job: - case "monofonic": - f.write(f"{path_to_monofonic_binary} {job_config_file} > {job_log}") - case "simbelmyne": - f.write(f"{job} {job_config_file} {job_log}") - case "scola": - f.write(f"{job} {job_config_file} {job_log} "+"-b ${SLURM_ARRAY_TASK_ID}") - case _: - raise ValueError(f"Job type {job} not recognized.")