maubin/sbmy_control
0
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

more improvements

Browse source
This commit is contained in:
Mayeul Aubin 2025-03-06 16:22:04 +01:00
parent e488275523
commit 5fa4dd9683
6 changed files with 56 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.gitignore vendored
View file

@ -23,3 +23,8 @@ pysbmy.egg-info/
log.txt
testpdf.txt
tests/
params/
results/
slurm_logs/
slurm_scripts/
work/

8
low_level.py
View file

@ -185,7 +185,7 @@ def print_ending_module(module:str, verbose:int=1):
print(print_level(0, module)+f"Ending {module} module.")
def wait_until_file_exists(filename:str, verbose:int=1, limit:int=24*3600):
def wait_until_file_exists(filename:str, verbose:int=1, limit:int=1200):
"""
Wait until a file exists.
"""
@ -233,7 +233,7 @@ def progress_bar_from_logfile(filename:str, desc:str="", verbose:int=1, **kwargs
from tqdm import tqdm
from time import sleep
k=0
limit=3600
limit=600
update_interval=0.2
sleep(2) # Wait for the process to be launched, and for the previous log file to be overwritten if necessary.
wait_until_file_exists(filename, verbose=verbose, limit=limit)
@ -245,12 +245,12 @@ def progress_bar_from_logfile(filename:str, desc:str="", verbose:int=1, **kwargs
return
with tqdm(desc=desc, total=total_operations, disable=(verbose==0), **kwargs) as pbar:
while current_operation < total_operations and k/update_interval < limit:
while current_operation < total_operations and k*update_interval < limit:
sleep(update_interval)
current_operation, total_operations = get_progress_from_logfile(filename)
if current_operation > previous_operation:
pbar.update(current_operation-previous_operation)
previous_operation = current_operation
k+=1
if k/update_interval >= limit:
if k*update_interval >= limit:
print_message(f"Progress bar timed out after {limit} seconds.", 3, "low level", verbose=verbose)

28
monofonic.py
View file

@ -19,16 +19,7 @@ def main_monofonic(parsed_args):
command_args = [path_to_monofonic_binary, monofonic_dict["config"]]
if parsed_args.verbose < 2:
from io import BytesIO
from low_level import stdout_redirector, stderr_redirector
f = BytesIO()
g = BytesIO()
with stdout_redirector(f):
with stderr_redirector(g):
subprocess.run(command_args)
g.close()
f.close()
subprocess.run(command_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
else:
subprocess.run(command_args)
@ -42,8 +33,8 @@ def main_monofonic(parsed_args):
main_dict=parse_arguments_main(parsed_args)
slurm_script = slurm_dict["scripts"]+"monofonic.sh"
if not isfile(slurm_script):
print_message(f"SLURM script {slurm_script} does not exist. Creating it.", 2, "monofonic", verbose=parsed_args.verbose)
if not isfile(slurm_script) or parsed_args.force:
print_message(f"SLURM script {slurm_script} does not exist (or forced). Creating it.", 2, "monofonic", verbose=parsed_args.verbose)
create_slurm_script(
slurm_template=slurm_dict["monofonic_template"],
slurm_script=slurm_script,
@ -60,16 +51,7 @@ def main_monofonic(parsed_args):
command_args = ["sbatch", slurm_script]
if parsed_args.verbose < 2:
from io import BytesIO
from low_level import stdout_redirector, stderr_redirector
f = BytesIO()
g = BytesIO()
with stdout_redirector(f):
with stderr_redirector(g):
subprocess.run(command_args)
g.close()
f.close()
subprocess.run(command_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
else:
subprocess.run(command_args)
@ -97,6 +79,6 @@ if __name__ == "__main__":
register_arguments_slurm(parser)
register_arguments_card_for_ICs(parser)
register_arguments_cosmo(parser)
parsed_args = parser.parse_args
parsed_args = parser.parse_args()
main_monofonic(parsed_args)

9
scola.py
View file

@ -44,6 +44,7 @@ def main_scola(parsed_args):
elif parsed_args.execution == "slurm":
from slurm_submission import create_slurm_script, parse_arguments_slurm
from args_main import parse_arguments_main
import os
print_message("Running scola in slurm mode.", 1, "scola", verbose=parsed_args.verbose)
slurm_dict=parse_arguments_slurm(parsed_args)
main_dict=parse_arguments_main(parsed_args)
@ -53,8 +54,8 @@ def main_scola(parsed_args):
print_message("Submitting all boxes.", 2, "scola", verbose=parsed_args.verbose)
slurm_script = slurm_dict["scripts"]+"scola.sh"
if not isfile(slurm_script):
print_message(f"SLURM script {slurm_script} does not exist. Creating it.", 2, "scola", verbose=parsed_args.verbose)
if not isfile(slurm_script) or parsed_args.force:
print_message(f"SLURM script {slurm_script} does not exist (or forced). Creating it.", 2, "scola", verbose=parsed_args.verbose)
create_slurm_script(
slurm_template=slurm_dict["scola_template"],
slurm_script=slurm_script,
@ -62,6 +63,7 @@ def main_scola(parsed_args):
job_config_file=paramfile,
job_log=log_file,
array=(1, nboxes_tot),
job_name=main_dict["simname"],
)
print_message(f"SLURM script written to {slurm_script}.", 3, "scola", verbose=parsed_args.verbose)
else:
@ -94,6 +96,7 @@ def main_scola(parsed_args):
job_config_file=paramfile,
job_log=log_file,
array=missing_tiles,
job_name=main_dict["simname"],
)
print_message(f"SLURM script written to {slurm_script}.", 3, "scola", verbose=parsed_args.verbose)
else:
@ -109,8 +112,8 @@ def main_scola(parsed_args):
print_message("sCOLA job submitted.", 2, "scola", verbose=parsed_args.verbose)
os.remove(slurm_script) # Remove the script after submission (because it is specific to the missing tiles)
else:
raise ValueError(f"Execution mode {parsed_args.execution} not recognized.")

5
simbelmyne.py
View file

@ -33,14 +33,15 @@ def main_simbelmyne(parsed_args):
main_dict=parse_arguments_main(parsed_args)
slurm_script = slurm_dict["scripts"]+"simbelmyne_"+main_dict["simname"]+".sh"
if not isfile(slurm_script):
print_message(f"SLURM script {slurm_script} does not exist. Creating it.", 2, "simbelmyne", verbose=parsed_args.verbose)
if not isfile(slurm_script) or parsed_args.force:
print_message(f"SLURM script {slurm_script} does not exist (or forced). Creating it.", 2, "simbelmyne", verbose=parsed_args.verbose)
create_slurm_script(
slurm_template=slurm_dict["simbelmyne_template"],
slurm_script=slurm_script,
job="simbelmyne",
job_config_file=paramfile,
job_log=log_file,
job_name=main_dict["simname"],
)
print_message(f"SLURM script written to {slurm_script}.", 3, "simbelmyne", verbose=parsed_args.verbose)
else:

45
slurm_submission.py
View file

@ -101,6 +101,21 @@ def create_slurm_template(
f.write(f"export OMP_NUM_THREADS={nthreads}\n\n")
f.write("start=`date +%s`\n")
f.write("%COMMAND%\n")
f.write("end=`date +%s`\n")
f.write("runtime=$((end-start))\n")
f.write("\n")
f.write("echo ''\n")
f.write("echo ''\n")
f.write("echo \"################## RUNTIME ##################\"\n")
f.write("echo \"Runtime was $runtime seconds\"\n")
#TODO: fix the two lines below
# f.write("echo \"Runtime per task was $((${runtime}.0/${SLURM_NTASKS}.0)) seconds\"\n") # not working yet...
# f.write("echo \"Runtime per total cpus was $((${runtime}.0/${($SLURM_NTASKS*$SLURM_CPUS_PER_TASK)}.0)) seconds\"\n")
f.write("echo \"#############################################\"\n")
f.write("\n")
f.write("exit 0\n")
def create_slurm_script(slurm_template:str,
@ -109,6 +124,7 @@ def create_slurm_script(slurm_template:str,
job_config_file:str,
job_log:str,
array:tuple|None=None,
job_name:str|None=None,
):
"""
Creates a SLURM submission script based on the provided template.
@ -131,25 +147,30 @@ def create_slurm_script(slurm_template:str,
with open(slurm_template, "r") as f:
template = f.readlines()
command_line = ""
# Add the job command
match job:
case "monofonic":
command_line = f"{path_to_monofonic_binary} {job_config_file} > {job_log}"
case "simbelmyne":
command_line = f"{job} {job_config_file} {job_log}"
case "scola":
command_line = f"{job} {job_config_file} {job_log} "+"-b ${SLURM_ARRAY_TASK_ID}"
case _:
raise ValueError(f"Job type {job} not recognized.")
# Create the script file
with open(slurm_script, "w") as f:
for line in template:
if job_name is not None and "--job-name" in line:
line = f"#SBATCH --job-name={job_name}\n"
if array is not None and "--array" in line:
line = line.replace("%a-%b",f"{array[0]}-{array[1]}")
line = f"#SBATCH --array={array[0]}-{array[1]}\n"
if array is not None and ("--output" in line or "--error" in line):
line = line.replace("%j","%a_%A")
if "%COMMAND%" in line:
line = command_line+"\n"
f.write(line)
# Add the job command
match job:
case "monofonic":
f.write(f"{path_to_monofonic_binary} {job_config_file} > {job_log}")
case "simbelmyne":
f.write(f"{job} {job_config_file} {job_log}")
case "scola":
f.write(f"{job} {job_config_file} {job_log} "+"-b ${SLURM_ARRAY_TASK_ID}")
case _:
raise ValueError(f"Job type {job} not recognized.")