From ac5e049ad1eb9059fd61e62efabede6658b73f3f Mon Sep 17 00:00:00 2001 From: neeratyoy Date: Thu, 2 Sep 2021 22:41:25 +0200 Subject: [PATCH 01/10] Adding script to generate SLURM jobs for Dask cluster --- utils/generate_slurm_jobs.py | 132 +++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 utils/generate_slurm_jobs.py diff --git a/utils/generate_slurm_jobs.py b/utils/generate_slurm_jobs.py new file mode 100644 index 0000000..ba4c5ae --- /dev/null +++ b/utils/generate_slurm_jobs.py @@ -0,0 +1,132 @@ +import os +import argparse +from pathlib import Path + + +def scheduler_command(scheduler_file): + cmd = "dask-scheduler --scheduler-file {}" + cmd = cmd.format(scheduler_file) + cmd += "\n" + return cmd + + +def worker_command(scheduler_file, worker_name, gpu=False, gpu_per_worker=1): + cmd = "dask-worker --scheduler-file {} --name \"{}_\"$SLURM_ARRAY_TASK_ID --no-nanny" + cmd = cmd.format(scheduler_file, worker_name) + if gpu: + cmd += " --resources \"GPU={}\"".format(gpu_per_worker) + cmd += "\n" + return cmd + + +def slurm_header(args, worker=False): + cmds = list() + # adding shebang + cmds.append("#! /bin/bash") + node = args.worker_p if worker else args.scheduler_p + # adding target node + cmds.append("#SBATCH -p {}".format(node)) + if not worker: + # adding cpu request + cmds.append("#SBATCH -c {}".format(args.c)) + # adding memory request + cmds.append("#SBATCH --mem {}".format(args.mem)) + # adding timelimit + cmds.append("#SBATCH -t {}".format(args.t)) + # adding job name + suffix = "worker" if worker else "scheduler" + cmds.append("#SBATCH -J {}-{}".format(args.J, suffix)) + if args.gpu and worker: + # adding gpu request + cmds.append("#SBATCH --gres=gpu:{}".format(args.gpu_per_worker)) + # making an array job for the workers + cmds.append("#SBATCH -a 1-{}".format(args.nworkers)) + cmds.append("\n") + cmds = "\n".join(cmds) + return cmds + + +def input_arguments(): + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "--scheduler_file", + default="scheduler.json", + type=str, + help="The file name storing the Dask cluster connections" + ) + parser.add_argument( + "--scheduler_path", + default="./scheduler", + type=str, + help="The path to keep the scheduler.json like files for Dask" + ) + parser.add_argument( + "--setup_file", + default=None, + type=str, + help="The path to file that will be sourced to load environment and set path variables" + ) + parser.add_argument( + "--output_path", default="./", type=str, help="The path to dump the generated script" + ) + parser.add_argument( + "--nworkers", default=10, type=int, help="Number of workers to run" + ) + parser.add_argument( + "--worker_name", default="w", type=str, help="Dask worker name prefix" + ) + parser.add_argument( + "-c", default=2, type=int, help="CPUs per task requested" + ) + parser.add_argument( + "--gpu", default=False, action="store_true", help="If set, the workers request GPUs" + ) + parser.add_argument( + "--gpu_per_worker", default=1, type=int, help="Number of GPUs per worker" + ) + parser.add_argument( + "--scheduler_p", default=None, required=True, type=str, help="The node to submit schedulers" + ) + parser.add_argument( + "--worker_p", default=None, required=True, type=str, help="The node to submit workers" + ) + parser.add_argument( + "--mem", default=0, type=int, help="Memory requested" + ) + parser.add_argument( + "-t", default="1:00:00", type=str, help="TIMELIMIT" + ) + parser.add_argument( + "-J", default="dehb", type=str, help="Prefix to scheduler and worker job names" + ) + + args = parser.parse_args() + return args + + +if __name__ == "__main__": + args = input_arguments() + + scheduler = Path(args.scheduler_path).absolute() / args.scheduler_file + os.makedirs(Path(args.scheduler_path).absolute(), exist_ok=True) + output_path = Path(args.output_path).absolute() + os.makedirs(Path(args.output_path).absolute(), exist_ok=True) + scheduler_file = output_path / "scheduler.sh" + worker_file = output_path / "workers.sh" + setup_cmd = "source {}\n\n".format(Path(args.setup_file).absolute()) + + # generating scheduler script + cmd = slurm_header(args, worker=False) + cmd += setup_cmd + cmd += scheduler_command(scheduler) + cmd += "\n" + with open(scheduler_file, "w") as f: + f.writelines(cmd) + print("Saving scheduler job script to {}".format(scheduler_file)) + # generating worker script + cmd = slurm_header(args, worker=True) + cmd += setup_cmd + cmd += worker_command(scheduler, args.worker_name) + with open(worker_file, "w") as f: + f.writelines(cmd) + print("Saving worker job script to {}".format(worker_file)) From c49315a2b692698883e6f23ca5da71f97f4eba62 Mon Sep 17 00:00:00 2001 From: neeratyoy Date: Thu, 2 Sep 2021 22:50:11 +0200 Subject: [PATCH 02/10] Adding more dask-worker args --- utils/generate_slurm_jobs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/generate_slurm_jobs.py b/utils/generate_slurm_jobs.py index ba4c5ae..01ba544 100644 --- a/utils/generate_slurm_jobs.py +++ b/utils/generate_slurm_jobs.py @@ -12,9 +12,11 @@ def scheduler_command(scheduler_file): def worker_command(scheduler_file, worker_name, gpu=False, gpu_per_worker=1): cmd = "dask-worker --scheduler-file {} --name \"{}_\"$SLURM_ARRAY_TASK_ID --no-nanny" + extra_args = " --reconnect --nprocs 1 --nthreads 1" cmd = cmd.format(scheduler_file, worker_name) if gpu: cmd += " --resources \"GPU={}\"".format(gpu_per_worker) + cmd += extra_args cmd += "\n" return cmd From 3e998fb426169c5368f6d088043756f11b38ad30 Mon Sep 17 00:00:00 2001 From: Neeratyoy Mallik Date: Fri, 3 Sep 2021 14:17:48 +0200 Subject: [PATCH 03/10] Updating GPU arg in worker --- utils/generate_slurm_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/generate_slurm_jobs.py b/utils/generate_slurm_jobs.py index 01ba544..8265646 100644 --- a/utils/generate_slurm_jobs.py +++ b/utils/generate_slurm_jobs.py @@ -128,7 +128,7 @@ def input_arguments(): # generating worker script cmd = slurm_header(args, worker=True) cmd += setup_cmd - cmd += worker_command(scheduler, args.worker_name) + cmd += worker_command(scheduler, args.worker_name, args.gpu) with open(worker_file, "w") as f: f.writelines(cmd) print("Saving worker job script to {}".format(worker_file)) From c9324a14880a6e9c908e12b06f13abea830c55f6 Mon Sep 17 00:00:00 2001 From: neeratyoy Date: Fri, 3 Sep 2021 14:58:29 +0200 Subject: [PATCH 04/10] Fixing GPU resources for Dask worker --- utils/generate_slurm_jobs.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/utils/generate_slurm_jobs.py b/utils/generate_slurm_jobs.py index 8265646..1be427d 100644 --- a/utils/generate_slurm_jobs.py +++ b/utils/generate_slurm_jobs.py @@ -31,8 +31,6 @@ def slurm_header(args, worker=False): if not worker: # adding cpu request cmds.append("#SBATCH -c {}".format(args.c)) - # adding memory request - cmds.append("#SBATCH --mem {}".format(args.mem)) # adding timelimit cmds.append("#SBATCH -t {}".format(args.t)) # adding job name @@ -92,9 +90,6 @@ def input_arguments(): parser.add_argument( "--worker_p", default=None, required=True, type=str, help="The node to submit workers" ) - parser.add_argument( - "--mem", default=0, type=int, help="Memory requested" - ) parser.add_argument( "-t", default="1:00:00", type=str, help="TIMELIMIT" ) @@ -120,7 +115,7 @@ def input_arguments(): # generating scheduler script cmd = slurm_header(args, worker=False) cmd += setup_cmd - cmd += scheduler_command(scheduler) + cmd += scheduler_command(scheduler_file=scheduler) cmd += "\n" with open(scheduler_file, "w") as f: f.writelines(cmd) @@ -128,7 +123,12 @@ def input_arguments(): # generating worker script cmd = slurm_header(args, worker=True) cmd += setup_cmd - cmd += worker_command(scheduler, args.worker_name, args.gpu) + cmd += worker_command( + scheduler_file=scheduler, + worker_name=args.worker_name, + gpu=args.gpu, + gpu_per_worker=args.gpu_per_worker + ) with open(worker_file, "w") as f: f.writelines(cmd) print("Saving worker job script to {}".format(worker_file)) From 9c52fc40f1bef35399ed32525cca5bfaddfbae4f Mon Sep 17 00:00:00 2001 From: neeratyoy Date: Fri, 3 Sep 2021 15:21:36 +0200 Subject: [PATCH 05/10] Finalizing script --- utils/generate_slurm_jobs.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/utils/generate_slurm_jobs.py b/utils/generate_slurm_jobs.py index 1be427d..3d1c496 100644 --- a/utils/generate_slurm_jobs.py +++ b/utils/generate_slurm_jobs.py @@ -1,3 +1,19 @@ +""" +Generates 2 scripts to deploy a Dask cluster to SLURM. + +Example use: +``` +python utils/generate_slurm_jobs.py --worker_p [gpu_node] --scheduler_p [cpu_node] --gpu \ + --nworkers 4 --scheduler_path ./scheduler --output_path temp --setup_file ./setup.sh` +``` + +Generated files can be submitted: +``` +sbatch temp/scheduler.sh +sbatch temp/workers.sh +``` +""" + import os import argparse from pathlib import Path @@ -41,6 +57,10 @@ def slurm_header(args, worker=False): cmds.append("#SBATCH --gres=gpu:{}".format(args.gpu_per_worker)) # making an array job for the workers cmds.append("#SBATCH -a 1-{}".format(args.nworkers)) + log_pattern = args.slurm_dump_path / "slurm_%j-%a-%x.{}" + # adding error directory + cmds.append("#SBATCH -e {}".format(log_pattern.format("err"))) + cmds.append("#SBATCH -o {}".format(log_pattern.format("out"))) cmds.append("\n") cmds = "\n".join(cmds) return cmds @@ -69,6 +89,9 @@ def input_arguments(): parser.add_argument( "--output_path", default="./", type=str, help="The path to dump the generated script" ) + parser.add_argument( + "--slurm_dump_path", default="./slurm-logs", type=str, help="Path to dump the slurm logs" + ) parser.add_argument( "--nworkers", default=10, type=int, help="Number of workers to run" ) @@ -104,6 +127,7 @@ def input_arguments(): if __name__ == "__main__": args = input_arguments() + args.slurm_dump_path = Path(args.slurm_dump_path).absolute() scheduler = Path(args.scheduler_path).absolute() / args.scheduler_file os.makedirs(Path(args.scheduler_path).absolute(), exist_ok=True) output_path = Path(args.output_path).absolute() From 62bfba2e7d46e2559889128a0b698ba794a83771 Mon Sep 17 00:00:00 2001 From: neeratyoy Date: Fri, 3 Sep 2021 15:45:58 +0200 Subject: [PATCH 06/10] Minor str fix --- utils/generate_slurm_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/generate_slurm_jobs.py b/utils/generate_slurm_jobs.py index 3d1c496..aeaa36b 100644 --- a/utils/generate_slurm_jobs.py +++ b/utils/generate_slurm_jobs.py @@ -57,7 +57,7 @@ def slurm_header(args, worker=False): cmds.append("#SBATCH --gres=gpu:{}".format(args.gpu_per_worker)) # making an array job for the workers cmds.append("#SBATCH -a 1-{}".format(args.nworkers)) - log_pattern = args.slurm_dump_path / "slurm_%j-%a-%x.{}" + log_pattern = str(args.slurm_dump_path / "slurm_%j-%a-%x.{}") # adding error directory cmds.append("#SBATCH -e {}".format(log_pattern.format("err"))) cmds.append("#SBATCH -o {}".format(log_pattern.format("out"))) From 3c7a8708f8be117802cbf9dab12c24bbddd45883 Mon Sep 17 00:00:00 2001 From: neeratyoy Date: Fri, 3 Sep 2021 16:32:01 +0200 Subject: [PATCH 07/10] Adding instructions to use Dask on META --- utils/README.md | 41 +++++++++++++++++++++++++++++++++++++++++ utils/run_dask_setup.sh | 19 ------------------- 2 files changed, 41 insertions(+), 19 deletions(-) create mode 100644 utils/README.md delete mode 100644 utils/run_dask_setup.sh diff --git a/utils/README.md b/utils/README.md new file mode 100644 index 0000000..b791438 --- /dev/null +++ b/utils/README.md @@ -0,0 +1,41 @@ +### Scripts to setup a Dask cluster on Meta SLURM + +There are 2 distinct ways in which DEHB can be run in a distributed manner. +* Letting the DEHB process create its [own Dask cluster](../README.md#running-dehb-in-a-parallel-setting) during runtime that lives and dies with the DEHB process +* Setting up a Dask cluster that runs independently and multiple DEHB processes can connect and share the cluster + +The scripts and instructions below account for the latter case, specifically for **SLURM**: + +To create a Dask cluster with 10 workers and uses CPUs: +```bash +python utils/generate_slurm_jobs.py --worker_p [cpu_node] --scheduler_p [cpu_node] --nworkers 10 \ + --scheduler_path ./scheduler --scheduler_file scheduler_cpu.json --output_path temp --setup_file ./setup.sh +# generates 2 shell scripts +``` +```bash +sbatch temp/scheduler.sh +# sleep 2s or wait till scheduler is allocated (not mandatory) +sbatch temp/workers.sh +``` + +Alternatively, to enable GPU usage by the workers, +```bash +python utils/generate_slurm_jobs.py --worker_p [cpu_node] --scheduler_p [cpu_node] --nworkers 10 \ + --scheduler_path ./scheduler --scheduler_file scheduler_gpu.json --output_path temp \ + --setup_file ./setup.sh --gpu +# generates 2 shell scripts +sbatch temp/scheduler.sh +# sleep 2s or wait till scheduler is allocated (not mandatory) +sbatch temp/workers.sh +``` + +The above sequence of commands will have a Dask cluster running and waiting for jobs. +One or more DEHB processes can share this pool of 10 workers. +For example, running a DEHB optimization by specifiying `scheduler_file` makes that DEHB process, +connect to the Dask cluster runnning. +```bash +python examples/03_pytorch_mnist_hpo.py --min_budget 1 --max_budget 9 --verbose \ + --scheduler_file scheduler/scheduler_gpu.json --runtime 200 --seed 123 +``` +The decoupled Dask cluster remains alive even after the DEHB optimization is over. +It can be reused by other DEHB runs or processes. diff --git a/utils/run_dask_setup.sh b/utils/run_dask_setup.sh deleted file mode 100644 index e45de26..0000000 --- a/utils/run_dask_setup.sh +++ /dev/null @@ -1,19 +0,0 @@ -#! /bin/bash - -while getopts f:e:n: flag -do - case "${flag}" in - f) filename=${OPTARG};; # specified as -f - e) envname=${OPTARG};; # specified as -e - n) nworkers=${OPTARG};; # specified as -n - esac -done - -echo "Submitting Dask scheduler..." -sbatch utils/dask_scheduler.sh -f $filename -e $envname - -for ((i=1; i<=$nworkers; i++)); do - echo "Submitting worker "$i"..." - sbatch utils/dask_workers.sh -f $filename -e $envname -w worker$i - sleep 2 -done From 67ac239f8cbede87a0db9951cf0d4aa18ef806da Mon Sep 17 00:00:00 2001 From: neeratyoy Date: Fri, 22 Oct 2021 17:06:01 +0200 Subject: [PATCH 08/10] Minor example edit to suggest docstrings --- examples/00_interfacing_DEHB.ipynb | 51 ++++++++++++++++++------------ 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/examples/00_interfacing_DEHB.ipynb b/examples/00_interfacing_DEHB.ipynb index e9c4b77..00eb84a 100644 --- a/examples/00_interfacing_DEHB.ipynb +++ b/examples/00_interfacing_DEHB.ipynb @@ -172,7 +172,7 @@ "data": { "text/plain": [ "Configuration:\n", - " x0, Value: 6.3793522646424785" + " x0, Value: 3.716302229868112" ] }, "execution_count": 5, @@ -257,6 +257,15 @@ "# but can be tuned as desired" ] }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "?dehb.run" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -267,7 +276,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -275,7 +284,7 @@ "output_type": "stream", "text": [ "Configuration:\n", - " x0, Value: 4.647994905980703\n", + " x0, Value: 4.060258498267547\n", "\n" ] } @@ -294,21 +303,21 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "2021-03-22 17:53:27.292 | INFO | dehb.optimizers.dehb:reset:102 - \n", + "2021-10-22 14:45:56.117 | INFO | dehb.optimizers.dehb:reset:107 - \n", "\n", - "RESET at 03/22/21 17:53:27 CET\n", + "RESET at 10/22/21 14:45:56 CEST\n", "\n", "\n", "(Configuration:\n", - " x0, Value: 3.5500062657482494\n", - ", 0.04514887709783266)\n" + " x0, Value: 3.724555206841792\n", + ", 0.0938589687572785)\n" ] } ], @@ -329,21 +338,21 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "2021-03-22 17:53:29.658 | INFO | dehb.optimizers.dehb:reset:102 - \n", + "2021-10-22 14:45:58.567 | INFO | dehb.optimizers.dehb:reset:107 - \n", "\n", - "RESET at 03/22/21 17:53:29 CET\n", + "RESET at 10/22/21 14:45:58 CEST\n", "\n", "\n", "(Configuration:\n", - " x0, Value: 6.898789003516494\n", - ", 0.0068207743261361475)\n" + " x0, Value: 4.341818535733585\n", + ", 3.653636256717441e-05)\n" ] } ], @@ -367,7 +376,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -375,8 +384,8 @@ "output_type": "stream", "text": [ "(Configuration:\n", - " x0, Value: 6.695906943430258\n", - ", 0.015266398519239388)\n" + " x0, Value: 4.610766436763522\n", + ", 0.007774399252232556)\n" ] } ], @@ -399,7 +408,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -407,7 +416,7 @@ "output_type": "stream", "text": [ "Configuration:\n", - " x0, Value: 6.695906943430258\n", + " x0, Value: 4.610766436763522\n", "\n" ] } @@ -418,16 +427,16 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "0.015266398519239388 0.015266398519239388\n", + "0.007774399252232556 0.007774399252232556\n", "Configuration:\n", - " x0, Value: 6.695906943430258\n", + " x0, Value: 4.610766436763522\n", "\n" ] } From 760e231743c17f73f532d869e7864f604bfce5af Mon Sep 17 00:00:00 2001 From: PhMueller Date: Tue, 16 Nov 2021 13:21:57 +0100 Subject: [PATCH 09/10] Make the package installable --- dehb/__version__.py | 1 + setup.py | 48 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 dehb/__version__.py create mode 100644 setup.py diff --git a/dehb/__version__.py b/dehb/__version__.py new file mode 100644 index 0000000..e47b027 --- /dev/null +++ b/dehb/__version__.py @@ -0,0 +1 @@ +__version__ = '0.0.1dev' diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..00d920c --- /dev/null +++ b/setup.py @@ -0,0 +1,48 @@ +# -*- encoding: utf-8 -*- +import setuptools + + +def read_file(file_name): + with open(file_name, encoding='utf-8') as fh: + text = fh.read() + return text + + +setuptools.setup( + name='dehb', + author_email='{awad, mallik, fh}@cs.uni-freiburg.de', + description='Evolutionary Hyberband for Scalable, Robust and Efficient Hyperparameter Optimization', + long_description=read_file('README.md'), + long_description_content_type='text/markdown', + license='Apache-2.0', + url='https://www.automl.org/automl/', + project_urls={ + 'Documentation': 'https://github.com/automl/dehb', + 'Source Code': 'https://github.com/automl/dehb' + }, + version=read_file('dehb/__version__.py').split()[-1].strip('\''), + packages=setuptools.find_packages(exclude=[ + '*.tests', '*.tests.*', 'tests.*', 'tests'], + ), + python_requires='>3.5, <=3.9', + install_requires=read_file('./requirements.txt').split('\n'), + test_suite='pytest', + platforms=['Linux'], + classifiers=[ + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Development Status :: 3 - Alpha', + 'Natural Language :: English', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Scientific/Engineering', + 'Topic :: Software Development', + ] +) From 22d0d092c085c20619314814fd976dca82208631 Mon Sep 17 00:00:00 2001 From: PhMueller Date: Tue, 16 Nov 2021 13:26:36 +0100 Subject: [PATCH 10/10] Add a version tag. --- dehb/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dehb/__init__.py b/dehb/__init__.py index d2f68de..1dd0b47 100644 --- a/dehb/__init__.py +++ b/dehb/__init__.py @@ -1,3 +1,5 @@ from .optimizers import DE, AsyncDE from .optimizers import DEHB from .utils import SHBracketManager + +from dehb.__version__ import __version__ # noqa: F401