From 86450472bcf7c132e076b4f27df971ba0639f916 Mon Sep 17 00:00:00 2001 From: Nastaran Alipour Date: Thu, 4 Jun 2026 16:27:36 +0200 Subject: [PATCH] refactor notebooks --- CHANGELOG.md | 2 +- tutorials/1_getting_started_hpo.ipynb | 15 +- tutorials/1_getting_started_hpo.py | 13 +- tutorials/2_search_spaces.ipynb | 209 ++---------------------- tutorials/2_search_spaces.py | 83 +--------- tutorials/3_efficiency_techniques.ipynb | 148 +++++++---------- tutorials/3_efficiency_techniques.py | 61 +++---- 7 files changed, 105 insertions(+), 426 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c86b06ce..4f604b46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Added tutorial and embended existing colab tutorial in it. +- Added 3 tutorials and embended existing colab tutorial in it. ## [0.16.0] - 2026-03-25 diff --git a/tutorials/1_getting_started_hpo.ipynb b/tutorials/1_getting_started_hpo.ipynb index 08220781..029aa79b 100644 --- a/tutorials/1_getting_started_hpo.ipynb +++ b/tutorials/1_getting_started_hpo.ipynb @@ -25,7 +25,6 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "!git clone --depth 1 https://github.com/automl/neps.git /content/neps\n", "%cd /content/neps\n", "!pip install -e /content/neps" @@ -47,7 +46,6 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "import math\n", "import neps\n", "import logging" @@ -84,7 +82,7 @@ " c = 5 / math.pi\n", " r = 6\n", " s = 10\n", - " t = 1 / 8 * math.pi\n", + " t = 1 / (8 * math.pi)\n", "\n", " f = a * (x2 - b * x1**2 + c * x1 - r) ** 2 + s * (1 - t) * math.cos(x1) + s\n", " return f" @@ -99,7 +97,6 @@ }, "outputs": [], "source": [ - "# %%\n", "# Define the search space and run optimization\n", "class BraninSpace(neps.PipelineSpace):\n", " x1 = neps.Float(-5, 10)\n", @@ -113,6 +110,8 @@ "metadata": {}, "outputs": [], "source": [ + "from tutorials.utils import set_seeds\n", + "set_seeds(1)\n", "neps.run(\n", " pipeline_space=BraninSpace(),\n", " root_directory=\"branin_demo/\",\n", @@ -138,7 +137,7 @@ "id": "aeac2466", "metadata": {}, "source": [ - "Great! NePS found the minimum loss over 25 evaluations. \n", + "Great! NePS went in the direction of minimum loss over 25 evaluations. If you increase the budget to 1000 evaluations, you can get even closer to the global minimum of 0.397887.\n", "The NePS workflow always follows this pattern:\n", "1. **Define** an objective function\n", "2. **Specify** a search space\n", @@ -163,9 +162,7 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", - "from tutorials.train import training_pipeline\n", - "from tutorials.utils import set_seeds" + "from tutorials.train import training_pipeline" ] }, { @@ -296,7 +293,6 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "# View the summary files:\n", "!cat results_hpo_demo/summary/short.csv" ] @@ -371,7 +367,6 @@ "lines_to_next_cell": 2 }, "source": [ - "%% [markdown]\n", "For more details on the Tensorboard integration, see documentation [here](https://automl.github.io/neps/latest/reference/analyse/#visualizing-results).\n", "\n", "## A slightly more elaborate HPO\n", diff --git a/tutorials/1_getting_started_hpo.py b/tutorials/1_getting_started_hpo.py index 058ec415..be1a0cd4 100644 --- a/tutorials/1_getting_started_hpo.py +++ b/tutorials/1_getting_started_hpo.py @@ -18,7 +18,6 @@ # ## Installation # Requires Python 3.10+. Install NePS via: -# %% # !git clone --depth 1 https://github.com/automl/neps.git /content/neps # %cd /content/neps # !pip install -e /content/neps @@ -26,7 +25,6 @@ # ## Example 1: Synthetic Function Optimization # We start with a simple optimization problem to understand NePS basics. -# %% import math import neps import logging @@ -43,17 +41,18 @@ def branin(x1, x2): c = 5 / math.pi r = 6 s = 10 - t = 1 / 8 * math.pi + t = 1 / (8 * math.pi) f = a * (x2 - b * x1**2 + c * x1 - r) ** 2 + s * (1 - t) * math.cos(x1) + s return f -# %% # Define the search space and run optimization class BraninSpace(neps.PipelineSpace): x1 = neps.Float(-5, 10) x2 = neps.Float(0, 15) +from tutorials.utils import set_seeds +set_seeds(1) neps.run( pipeline_space=BraninSpace(), root_directory="branin_demo/", @@ -65,7 +64,7 @@ class BraninSpace(neps.PipelineSpace): # Check the optimization results: #!tail ./branin_demo/summary/best_config.txt -# Great! NePS found the minimum loss over 25 evaluations. +# Great! NePS went in the direction of minimum loss over 25 evaluations. If you increase the budget to 1000 evaluations, you can get even closer to the global minimum of 0.397887. # The NePS workflow always follows this pattern: # 1. **Define** an objective function # 2. **Specify** a search space @@ -76,9 +75,7 @@ class BraninSpace(neps.PipelineSpace): # # For the full training code, see [train.py](https://github.com/automl/neps/blob/master/tutorials/train.py). -# %% from tutorials.train import training_pipeline -from tutorials.utils import set_seeds # Test the training pipeline with a small subset training_pipeline( @@ -139,7 +136,6 @@ class HPODemoSpace(neps.PipelineSpace): !python -m neps.status results_hpo_demo/ -# %% # View the summary files: !cat results_hpo_demo/summary/short.csv @@ -158,7 +154,6 @@ class HPODemoSpace(neps.PipelineSpace): # %tensorboard --logdir /content/neps/results_hpo_demo -# %% [markdown] # For more details on the Tensorboard integration, see documentation [here](https://automl.github.io/neps/latest/reference/analyse/#visualizing-results). # # ## A slightly more elaborate HPO diff --git a/tutorials/2_search_spaces.ipynb b/tutorials/2_search_spaces.ipynb index 7f3e29aa..6d3ea4f2 100644 --- a/tutorials/2_search_spaces.ipynb +++ b/tutorials/2_search_spaces.ipynb @@ -12,7 +12,9 @@ { "cell_type": "markdown", "id": "bb16329a", - "metadata": {}, + "metadata": { + "lines_to_next_cell": 2 + }, "source": [ "## Installation and Setup" ] @@ -21,10 +23,11 @@ "cell_type": "code", "execution_count": null, "id": "8843d351", - "metadata": {}, + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [], "source": [ - "# %%\n", "!git clone --depth 1 https://github.com/automl/neps.git /content/neps\n", "%cd /content/neps\n", "!pip install -e /content/neps" @@ -37,7 +40,6 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "import neps\n", "import logging" ] @@ -67,43 +69,14 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "# Linear scale\n", - "learning_rate_linear = neps.Float(lower=0.0001, upper=0.1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ff0266cc", - "metadata": {}, - "outputs": [], - "source": [ + "learning_rate_linear = neps.Float(lower=0.0001, upper=0.1)\n", "# Log scale (useful for learning rates, regularization)\n", - "learning_rate_log = neps.Float(lower=1e-6, upper=1e-1, log=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0d6831c8", - "metadata": {}, - "outputs": [], - "source": [ + "learning_rate_log = neps.Float(lower=1e-6, upper=1e-1, log=True)\n", "# With prior value\n", "dropout_rate = neps.Float(lower=0.0, upper=0.9, prior=0.5, prior_confidence=\"medium\")" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "cac43708", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"Float parameters created successfully\")" - ] - }, { "cell_type": "markdown", "id": "7a7fff93", @@ -120,43 +93,14 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "# Basic integer\n", - "num_layers = neps.Integer(lower=1, upper=10)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fb68862b", - "metadata": {}, - "outputs": [], - "source": [ + "num_layers = neps.Integer(lower=1, upper=10)\n", "# With prior value\n", - "batch_size = neps.Integer(lower=16, upper=256, prior=64, prior_confidence=\"medium\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "02b99a46", - "metadata": {}, - "outputs": [], - "source": [ + "batch_size = neps.Integer(lower=16, upper=256, prior=64, prior_confidence=\"medium\")\n", "# With log scale\n", "hidden_units = neps.Integer(lower=32, upper=2048, log=True)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "5a491bb0", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"Integer parameters created successfully\")" - ] - }, { "cell_type": "markdown", "id": "1a0722b5", @@ -173,18 +117,8 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "# Basic categorical\n", - "optimizer = neps.Categorical(choices=[\"sgd\", \"adam\", \"adamw\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2810a892", - "metadata": {}, - "outputs": [], - "source": [ + "optimizer = neps.Categorical(choices=[\"sgd\", \"adam\", \"adamw\"])\n", "# With prior value\n", "activation = neps.Categorical(\n", " choices=[\"relu\", \"tanh\", \"sigmoid\"], \n", @@ -193,16 +127,6 @@ ")" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "eb8b7069", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"Categorical parameters created successfully\")" - ] - }, { "cell_type": "markdown", "id": "bb97569a", @@ -228,7 +152,6 @@ }, "outputs": [], "source": [ - "# %%\n", "# Simple dictionary-based search space\n", "simple_space = dict(\n", " learning_rate=neps.Float(1e-6, 1e-1, log=True),\n", @@ -255,7 +178,6 @@ }, "outputs": [], "source": [ - "# %%\n", "class MyOptimizationSpace(neps.PipelineSpace):\n", " \"\"\"Define a structured search space for neural architecture search.\"\"\"\n", " \n", @@ -306,7 +228,6 @@ }, "outputs": [], "source": [ - "# %%\n", "# Define a search space with a fidelity parameter\n", "pipeline_space = dict(\n", " learning_rate=neps.Float(1e-6, 1e-1, log=True),\n", @@ -377,7 +298,6 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "# Constructors for the conditional parts of the pipeline. In a real project these\n", "# would create layers, modules, preprocessing steps, or optimizer objects.\n", "def dense_layer(num_neurons: int, activation: str) -> dict:\n", @@ -499,9 +419,7 @@ "cell_type": "code", "execution_count": null, "id": "dbbd11e2", - "metadata": { - "lines_to_next_cell": 1 - }, + "metadata": {}, "outputs": [], "source": [ "conditional_space = ConditionalPipelineSpace()\n", @@ -527,109 +445,6 @@ "!cat conditional_search_space_example/configs/config_1/config.yaml" ] }, - { - "cell_type": "markdown", - "id": "f2c41590", - "metadata": {}, - "source": [ - "## Example: Complete Search Space" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b1ee3c16", - "metadata": { - "lines_to_next_cell": 1 - }, - "outputs": [], - "source": [ - "# %%\n", - "class ComprehensiveSpace(neps.PipelineSpace):\n", - " \"\"\"Complete example with multiple parameter types.\"\"\"\n", - " \n", - " # Continuous hyperparameters\n", - " learning_rate = neps.Float(1e-5, 1e-2, log=True, prior=1e-3, prior_confidence=\"high\")\n", - " weight_decay = neps.Float(1e-6, 1e-2, log=True, prior=1e-4, prior_confidence=\"medium\")\n", - " \n", - " # Discrete architecture choices\n", - " num_layers = neps.Integer(2, 8, prior=4, prior_confidence=\"medium\")\n", - " hidden_size = neps.Integer(64, 512, prior=256, prior_confidence=\"medium\")\n", - " \n", - " # Categorical choices\n", - " optimizer = neps.Categorical(\n", - " [\"adam\", \"adamw\", \"sgd\"],\n", - " prior=1, # adamw\n", - " prior_confidence=\"high\"\n", - " )\n", - " activation = neps.Categorical(\n", - " [\"relu\", \"gelu\", \"elu\"],\n", - " prior=0, # relu\n", - " prior_confidence=\"medium\"\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f8a41740", - "metadata": { - "lines_to_next_cell": 1 - }, - "outputs": [], - "source": [ - "# Define pipeline function\n", - "def train_model(\n", - " learning_rate: float,\n", - " weight_decay: float,\n", - " num_layers: int,\n", - " hidden_size: int,\n", - " optimizer: str,\n", - " activation: str,\n", - " **kwargs\n", - ") -> float:\n", - " \"\"\"Example training function that uses multiple parameter types.\"\"\"\n", - " \n", - " # In practice, you'd build a model with these parameters\n", - " # and return the validation loss\n", - " \n", - " # Simulated loss calculation\n", - " loss = 0.5\n", - " loss -= 0.05 * np.log10(learning_rate / 1e-3) # Learning rate matters\n", - " loss -= 0.02 * np.log10(hidden_size / 256) # Bigger network better\n", - " loss -= 0.01 if optimizer == \"adamw\" else 0 # adamw often better\n", - " loss -= 0.01 if activation == \"relu\" else 0 # relu is standard\n", - " \n", - " return max(0.1, loss)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "90958ec3", - "metadata": {}, - "outputs": [], - "source": [ - "# Run a small optimization\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1b98b7a8", - "metadata": {}, - "outputs": [], - "source": [ - "logging.basicConfig(level=logging.INFO)\n", - "neps.run(\n", - " evaluate_pipeline=train_model,\n", - " pipeline_space=ComprehensiveSpace(),\n", - " root_directory=\"search_space_example/\",\n", - " evaluations_to_spend=10, # Quick demo\n", - ")" - ] - }, { "cell_type": "markdown", "id": "59513384", diff --git a/tutorials/2_search_spaces.py b/tutorials/2_search_spaces.py index b88a8e73..96493e41 100644 --- a/tutorials/2_search_spaces.py +++ b/tutorials/2_search_spaces.py @@ -17,12 +17,12 @@ # ## Installation and Setup -# %% + # !git clone --depth 1 https://github.com/automl/neps.git /content/neps # %cd /content/neps # !pip install -e /content/neps -# %% + import neps import logging @@ -32,40 +32,28 @@ # ### 1. Float Parameters # Continuous parameters with optional log scaling. -# %% # Linear scale learning_rate_linear = neps.Float(lower=0.0001, upper=0.1) - # Log scale (useful for learning rates, regularization) learning_rate_log = neps.Float(lower=1e-6, upper=1e-1, log=True) - # With prior value dropout_rate = neps.Float(lower=0.0, upper=0.9, prior=0.5, prior_confidence="medium") -print("Float parameters created successfully") - # ### 2. Integer Parameters # Discrete integer parameters. -# %% # Basic integer num_layers = neps.Integer(lower=1, upper=10) - # With prior value batch_size = neps.Integer(lower=16, upper=256, prior=64, prior_confidence="medium") - # With log scale hidden_units = neps.Integer(lower=32, upper=2048, log=True) -print("Integer parameters created successfully") - # ### 3. Categorical Parameters # Discrete choices with optional ordering. -# %% # Basic categorical optimizer = neps.Categorical(choices=["sgd", "adam", "adamw"]) - # With prior value activation = neps.Categorical( choices=["relu", "tanh", "sigmoid"], @@ -73,13 +61,10 @@ prior_confidence="high" ) -print("Categorical parameters created successfully") - # ## Building Complex Search Spaces # ### Using a Dictionary -# %% # Simple dictionary-based search space simple_space = dict( learning_rate=neps.Float(1e-6, 1e-1, log=True), @@ -90,7 +75,6 @@ # ### Using a PipelineSpace Class # For complex search spaces, use the `PipelineSpace` class for conditioning one hyperparmater over other and better organization. -# %% class MyOptimizationSpace(neps.PipelineSpace): """Define a structured search space for neural architecture search.""" @@ -119,7 +103,6 @@ class MyOptimizationSpace(neps.PipelineSpace): # Use fidelity parameters for multi-fidelity optimization (train with different epochs, dataset sizes, etc.). -# %% # Define a search space with a fidelity parameter pipeline_space = dict( learning_rate=neps.Float(1e-6, 1e-1, log=True), @@ -164,7 +147,6 @@ class MyOptimizationSpace(neps.PipelineSpace): # The important pattern is to define a reusable search grammar and call `.resample()` # when the same grammar should be sampled independently in multiple places. -# %% # Constructors for the conditional parts of the pipeline. In a real project these # would create layers, modules, preprocessing steps, or optimizer objects. def dense_layer(num_neurons: int, activation: str) -> dict: @@ -268,67 +250,6 @@ def evaluate_conditional_pipeline(pipeline: dict) -> float: # user-facing argument names. #!cat conditional_search_space_example/configs/config_1/config.yaml -# ## Example: Complete Search Space - -# %% -class ComprehensiveSpace(neps.PipelineSpace): - """Complete example with multiple parameter types.""" - - # Continuous hyperparameters - learning_rate = neps.Float(1e-5, 1e-2, log=True, prior=1e-3, prior_confidence="high") - weight_decay = neps.Float(1e-6, 1e-2, log=True, prior=1e-4, prior_confidence="medium") - - # Discrete architecture choices - num_layers = neps.Integer(2, 8, prior=4, prior_confidence="medium") - hidden_size = neps.Integer(64, 512, prior=256, prior_confidence="medium") - - # Categorical choices - optimizer = neps.Categorical( - ["adam", "adamw", "sgd"], - prior=1, # adamw - prior_confidence="high" - ) - activation = neps.Categorical( - ["relu", "gelu", "elu"], - prior=0, # relu - prior_confidence="medium" - ) - -# Define pipeline function -def train_model( - learning_rate: float, - weight_decay: float, - num_layers: int, - hidden_size: int, - optimizer: str, - activation: str, - **kwargs -) -> float: - """Example training function that uses multiple parameter types.""" - - # In practice, you'd build a model with these parameters - # and return the validation loss - - # Simulated loss calculation - loss = 0.5 - loss -= 0.05 * np.log10(learning_rate / 1e-3) # Learning rate matters - loss -= 0.02 * np.log10(hidden_size / 256) # Bigger network better - loss -= 0.01 if optimizer == "adamw" else 0 # adamw often better - loss -= 0.01 if activation == "relu" else 0 # relu is standard - - return max(0.1, loss) - -# Run a small optimization -import numpy as np - -logging.basicConfig(level=logging.INFO) -neps.run( - evaluate_pipeline=train_model, - pipeline_space=ComprehensiveSpace(), - root_directory="search_space_example/", - evaluations_to_spend=10, # Quick demo -) - # ## Key Points # # - **Float**: For continuous parameters, use `log=True` for log-scaled distributions diff --git a/tutorials/3_efficiency_techniques.ipynb b/tutorials/3_efficiency_techniques.ipynb index 76fcd4e1..8c34b224 100644 --- a/tutorials/3_efficiency_techniques.ipynb +++ b/tutorials/3_efficiency_techniques.ipynb @@ -13,7 +13,9 @@ { "cell_type": "markdown", "id": "1fcd8576", - "metadata": {}, + "metadata": { + "lines_to_next_cell": 2 + }, "source": [ "## Installation and Setup" ] @@ -22,10 +24,11 @@ "cell_type": "code", "execution_count": null, "id": "29993327", - "metadata": {}, + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [], "source": [ - "# %%\n", "!git clone --depth 1 https://github.com/automl/neps.git /content/neps\n", "%cd /content/neps\n", "!pip install -e /content/neps" @@ -38,7 +41,6 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "import neps\n", "import logging\n", "import numpy as np\n", @@ -75,7 +77,6 @@ }, "outputs": [], "source": [ - "# %%\n", "def evaluate_pipeline(\n", " learning_rate: float,\n", " optimizer: str,\n", @@ -162,12 +163,9 @@ "cell_type": "code", "execution_count": null, "id": "d86c66f3", - "metadata": { - "lines_to_next_cell": 1 - }, + "metadata": {}, "outputs": [], "source": [ - "# %%\n", "class MultiFidelitySpace(neps.PipelineSpace):\n", " \"\"\"Search space for multi-fidelity optimization.\"\"\"\n", " learning_rate = neps.Float(1e-6, 1e-1, log=True)\n", @@ -182,7 +180,6 @@ "id": "50cf3ece", "metadata": {}, "source": [ - "%%\n", "### Run Multi-Fidelity Optimization" ] }, @@ -195,7 +192,6 @@ }, "outputs": [], "source": [ - "# %%\n", "neps.run(\n", " evaluate_pipeline=evaluate_pipeline,\n", " root_directory=\"results_multi_fidelity/\",\n", @@ -225,7 +221,6 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "!python -m neps.status results_multi_fidelity/" ] }, @@ -234,7 +229,6 @@ "id": "bd5ceb49", "metadata": {}, "source": [ - "%%\n", "## Technique 2: Incorporating Expert Priors\n", "Provide prior values and confidence levels to incorporate domain knowledge." ] @@ -256,7 +250,6 @@ }, "outputs": [], "source": [ - "# %%\n", "class ExpertPriorSpace(neps.PipelineSpace):\n", " \"\"\"Search space with expert priors incorporated.\"\"\"\n", " \n", @@ -298,7 +291,6 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "neps.run(\n", " evaluate_pipeline=evaluate_pipeline,\n", " root_directory=\"results_with_priors/\",\n", @@ -326,7 +318,6 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "!python -m neps.status results_with_priors/" ] }, @@ -354,7 +345,6 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "from neps.optimizers.algorithms import PredefinedOptimizers\n", "from neps.utils.common import extract_keyword_defaults" ] @@ -386,7 +376,6 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "for algo in [\"successive_halving\", \"asha\", \"hyperband\", \"async_hb\", \"ifbo\", \"priorband\"]:\n", " print(f\"\\n{algo} hyperparameters:\")\n", " print(extract_keyword_defaults(PredefinedOptimizers[algo]))" @@ -399,7 +388,6 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "neps.run(\n", " evaluate_pipeline=evaluate_pipeline,\n", " root_directory=\"results_custom_optimizer/\",\n", @@ -442,7 +430,6 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "# Sequential run\n", "neps.run(\n", " evaluate_pipeline=evaluate_pipeline,\n", @@ -453,13 +440,21 @@ ")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "535c121e", + "metadata": {}, + "outputs": [], + "source": [ + "!python -m neps.status results_sequential/" + ] + }, { "cell_type": "code", "execution_count": null, "id": "71880228", - "metadata": { - "lines_to_next_cell": 1 - }, + "metadata": {}, "outputs": [], "source": [ "print(\"Sequential run complete\")" @@ -478,58 +473,71 @@ "id": "10884d79", "metadata": {}, "source": [ - "To parallelize, run multiple `neps.run()` calls with the same `root_directory`:\n", - "\n", - "```bash\n", - "# In terminal, start multiple workers in parallel:\n", - "python worker.py &\n", - "python worker.py &\n", - "python worker.py &\n", - "\n", - "# They'll coordinate and sample/run new configurations without conflicts, all writing to the same results directory.\n", - "```\n", - "\n", - "In the worker script, keep `overwrite_root_directory=False` so workers attach to\n", - "the same run instead of resetting it." + "To parallelize, run multiple `neps.run()` calls with the same `root_directory`:" ] }, { "cell_type": "code", "execution_count": null, - "id": "535c121e", - "metadata": {}, + "id": "614f7b32", + "metadata": { + "lines_to_next_cell": 1 + }, "outputs": [], "source": [ - "# %%\n", - "!python -m neps.status results_sequential/" + "import multiprocessing as mp\n", + "def run_worker():\n", + " neps.run(\n", + " evaluate_pipeline=evaluate_pipeline,\n", + " root_directory=\"results_parallel/\",\n", + " pipeline_space=ExpertPriorSpace(),\n", + " evaluations_to_spend=6, # each worker's budget\n", + " overwrite_root_directory=False, # workers attach, don't reset\n", + " )\n", + "# Launch 3 workers in parallel\n", + "processes = [mp.Process(target=run_worker) for _ in range(3)]\n", + "for p in processes:\n", + " p.start()\n", + "for p in processes:\n", + " p.join()" ] }, { "cell_type": "markdown", - "id": "31c5ed06", + "id": "720386c3", "metadata": {}, "source": [ - "## Technique 5: Combining Strategies" + "# They'll coordinate and sample/run new configurations without conflicts, all writing to the same results directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "012393fb", + "metadata": {}, + "outputs": [], + "source": [ + "!python -m neps.status results_parallel/" ] }, { "cell_type": "markdown", - "id": "06505796", + "id": "31c5ed06", "metadata": {}, "source": [ - "Combine multiple techniques for maximum efficiency." + "## Technique 5: Combining Strategies" ] }, { "cell_type": "code", "execution_count": null, - "id": "89361f10", + "id": "a6f7f363", "metadata": { "lines_to_next_cell": 1 }, "outputs": [], "source": [ - "# %%\n", + "# Combine multiple techniques for maximum efficiency.\n", "class CombinedSearchSpace(neps.PipelineSpace):\n", " \"\"\"Combines multi-fidelity, priors, and complex search space.\"\"\"\n", " \n", @@ -541,7 +549,6 @@ " prior=0, # relu\n", " prior_confidence=\"high\"\n", " )\n", - " \n", " # Training with priors\n", " learning_rate = neps.Float(1e-5, 1e-2, log=True, prior=1e-3, prior_confidence=\"high\")\n", " optimizer = neps.Categorical(\n", @@ -573,7 +580,6 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "start_time = time.time()" ] }, @@ -612,7 +618,6 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "!python -m neps.status results_combined/" ] }, @@ -639,44 +644,16 @@ "metadata": {}, "outputs": [], "source": [ - "# %%\n", "import pandas as pd\n", - "import matplotlib.pyplot as plt" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "21afcbec", - "metadata": {}, - "outputs": [], - "source": [ + "import matplotlib.pyplot as plt\n", "fig, axes = plt.subplots(2, 2, figsize=(12, 8))\n", - "fig.suptitle(\"Optimization Strategies Comparison\", fontsize=14, fontweight=\"bold\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5e85a374", - "metadata": {}, - "outputs": [], - "source": [ + "fig.suptitle(\"Optimization Strategies Comparison\", fontsize=14, fontweight=\"bold\")\n", "strategies = [\n", " (\"Sequential\", \"results_sequential/summary/full.csv\"),\n", " (\"With Priors\", \"results_with_priors/summary/full.csv\"),\n", " (\"Async Hyperband\", \"results_custom_optimizer/summary/full.csv\"),\n", " (\"Combined\", \"results_combined/summary/full.csv\"),\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "35f6fb02", - "metadata": {}, - "outputs": [], - "source": [ + "]\n", "for idx, (name, path) in enumerate(strategies):\n", " ax = axes[idx // 2, idx % 2]\n", " \n", @@ -692,16 +669,7 @@ " ax.grid(True, alpha=0.3)\n", " except FileNotFoundError:\n", " ax.text(0.5, 0.5, \"Results not available\", ha='center', va='center')\n", - " ax.set_title(name)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "182e884e", - "metadata": {}, - "outputs": [], - "source": [ + " ax.set_title(name)\n", "plt.tight_layout()\n", "plt.show()" ] diff --git a/tutorials/3_efficiency_techniques.py b/tutorials/3_efficiency_techniques.py index 0cf02300..b9d4f05b 100644 --- a/tutorials/3_efficiency_techniques.py +++ b/tutorials/3_efficiency_techniques.py @@ -18,12 +18,12 @@ # ## Installation and Setup -# %% + # !git clone --depth 1 https://github.com/automl/neps.git /content/neps # %cd /content/neps # !pip install -e /content/neps -# %% + import neps import logging import numpy as np @@ -34,7 +34,6 @@ # ## The Optimization Task # Let's use a simulated deep learning task throughout this tutorial. -# %% def evaluate_pipeline( learning_rate: float, optimizer: str, @@ -104,7 +103,6 @@ def evaluate_pipeline( # ### Define Search Space with Fidelity -# %% class MultiFidelitySpace(neps.PipelineSpace): """Search space for multi-fidelity optimization.""" learning_rate = neps.Float(1e-6, 1e-1, log=True) @@ -113,10 +111,9 @@ class MultiFidelitySpace(neps.PipelineSpace): # Fidelity parameter: wrap Integer in neps.Fidelity() epochs = neps.Fidelity(neps.Integer(1, 10)) -# %% + # ### Run Multi-Fidelity Optimization -# %% neps.run( evaluate_pipeline=evaluate_pipeline, root_directory="results_multi_fidelity/", @@ -132,16 +129,13 @@ class MultiFidelitySpace(neps.PipelineSpace): # `previous_pipeline_directory` in `evaluate_pipeline` so promoted configurations can # load checkpoints from earlier fidelity rungs. -# %% # !python -m neps.status results_multi_fidelity/ -# %% # ## Technique 2: Incorporating Expert Priors # Provide prior values and confidence levels to incorporate domain knowledge. # ### Define Search Space with Priors -# %% class ExpertPriorSpace(neps.PipelineSpace): """Search space with expert priors incorporated.""" @@ -169,7 +163,6 @@ class ExpertPriorSpace(neps.PipelineSpace): # ### Run Optimization with Priors -# %% neps.run( evaluate_pipeline=evaluate_pipeline, root_directory="results_with_priors/", @@ -183,7 +176,6 @@ class ExpertPriorSpace(neps.PipelineSpace): # `optimizer="auto"`, NePS chooses the appropriate prior-aware multi-fidelity # optimizer for spaces that contain both priors and fidelities. -# %% # !python -m neps.status results_with_priors/ # ## Technique 3: Optimizer Selection @@ -191,7 +183,6 @@ class ExpertPriorSpace(neps.PipelineSpace): # NePS supports multiple search algorithms. You can let `optimizer="auto"` pick from # the search-space structure, or specify an optimizer explicitly. -# %% from neps.optimizers.algorithms import PredefinedOptimizers from neps.utils.common import extract_keyword_defaults @@ -201,12 +192,10 @@ class ExpertPriorSpace(neps.PipelineSpace): # ### Multi-Fidelity Optimizer Parameters -# %% for algo in ["successive_halving", "asha", "hyperband", "async_hb", "ifbo", "priorband"]: print(f"\n{algo} hyperparameters:") print(extract_keyword_defaults(PredefinedOptimizers[algo])) -# %% neps.run( evaluate_pipeline=evaluate_pipeline, root_directory="results_custom_optimizer/", @@ -223,7 +212,6 @@ class ExpertPriorSpace(neps.PipelineSpace): # ### Single Process Example (Sequential) -# %% # Sequential run neps.run( evaluate_pipeline=evaluate_pipeline, @@ -233,32 +221,37 @@ class ExpertPriorSpace(neps.PipelineSpace): overwrite_root_directory=True, ) +# !python -m neps.status results_sequential/ + print("Sequential run complete") # ### Parallel Execution Pattern # To parallelize, run multiple `neps.run()` calls with the same `root_directory`: -# -# ```bash -# # In terminal, start multiple workers in parallel: -# python worker.py & -# python worker.py & -# python worker.py & -# + +import multiprocessing as mp +def run_worker(): + neps.run( + evaluate_pipeline=evaluate_pipeline, + root_directory="results_parallel/", + pipeline_space=ExpertPriorSpace(), + evaluations_to_spend=6, # each worker's budget + overwrite_root_directory=False, # workers attach, don't reset + ) +# Launch 3 workers in parallel +processes = [mp.Process(target=run_worker) for _ in range(3)] +for p in processes: + p.start() +for p in processes: + p.join() + # # They'll coordinate and sample/run new configurations without conflicts, all writing to the same results directory. -# ``` -# -# In the worker script, keep `overwrite_root_directory=False` so workers attach to -# the same run instead of resetting it. -# %% -# !python -m neps.status results_sequential/ +# !python -m neps.status results_parallel/ # ## Technique 5: Combining Strategies # Combine multiple techniques for maximum efficiency. - -# %% class CombinedSearchSpace(neps.PipelineSpace): """Combines multi-fidelity, priors, and complex search space.""" @@ -270,7 +263,6 @@ class CombinedSearchSpace(neps.PipelineSpace): prior=0, # relu prior_confidence="high" ) - # Training with priors learning_rate = neps.Float(1e-5, 1e-2, log=True, prior=1e-3, prior_confidence="high") optimizer = neps.Categorical( @@ -288,7 +280,6 @@ class CombinedSearchSpace(neps.PipelineSpace): # Run optimization with combined strategies: -# %% start_time = time.time() neps.run( @@ -303,27 +294,22 @@ class CombinedSearchSpace(neps.PipelineSpace): elapsed = time.time() - start_time print(f"\nOptimization completed in {elapsed:.2f} seconds") -# %% # !python -m neps.status results_combined/ # ## Comparing Strategies # Let's visualize how the best loss improves over time for different approaches: -# %% import pandas as pd import matplotlib.pyplot as plt - fig, axes = plt.subplots(2, 2, figsize=(12, 8)) fig.suptitle("Optimization Strategies Comparison", fontsize=14, fontweight="bold") - strategies = [ ("Sequential", "results_sequential/summary/full.csv"), ("With Priors", "results_with_priors/summary/full.csv"), ("Async Hyperband", "results_custom_optimizer/summary/full.csv"), ("Combined", "results_combined/summary/full.csv"), ] - for idx, (name, path) in enumerate(strategies): ax = axes[idx // 2, idx % 2] @@ -340,7 +326,6 @@ class CombinedSearchSpace(neps.PipelineSpace): except FileNotFoundError: ax.text(0.5, 0.5, "Results not available", ha='center', va='center') ax.set_title(name) - plt.tight_layout() plt.show()