dengdifan: test for smac_facade

develop_codecov/.buildinfo

@@ -0,0 +1,4 @@
+# Sphinx build info version 1
+# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: 0970e851562abe794656b872d2e8e291
+tags: 645f666f9bcd5a90fca523b33c5a78b7

develop_codecov/_downloads/0713842f2ba18c946313bf17c241d792/restore_branin.ipynb

@@ -0,0 +1,54 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "%matplotlib inline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "\n# Restore Branin\nThis file runs SMAC and then restores the branin run with an extended computation\nbudget. This will also work for SMAC runs that have crashed and are continued.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "import logging\nlogging.basicConfig(level=logging.INFO)\n\nimport os\n\nfrom smac.facade.smac_ac_facade import SMAC4AC\nfrom smac.runhistory.runhistory import RunHistory\nfrom smac.scenario.scenario import Scenario\nfrom smac.stats.stats import Stats\nfrom smac.utils.io.traj_logging import TrajLogger\n\n\n__copyright__ = \"Copyright 2021, AutoML.org Freiburg-Hannover\"\n__license__ = \"3-clause BSD\"\n\n\nif \"__main__\" == __name__:\n\n    # Initialize scenario, using runcount_limit as budget.\n    origiginal_scenario_dict = {\n        'algo': 'python branin.py',\n        'paramfile': 'branin/configspace.pcs',\n        'run_obj': 'quality',\n        'runcount_limit': 25,\n        'deterministic': True,\n        'output_dir': 'restore_me'}\n    original_scenario = Scenario(origiginal_scenario_dict)\n\n    smac = SMAC4AC(scenario=original_scenario, run_id=1)\n    smac.optimize()\n\n    print(\"\\nBudget exhausted! Starting restoring optimization ...\\n\")\n\n    # Now the output is in the folder 'restore_me/run_1' (or whatever run_id has\n    # been passed to the SMAC-object above)\n    old_output_dir = os.path.join(original_scenario.output_dir, 'run_1')\n\n    # We could simply modify the scenario-object, stored in\n    # 'smac.solver.scenario' and start optimization again:\n\n    # smac.solver.scenario.ta_run_limit = 50\n    # smac.optimize()\n\n    # Or, to show the whole process of recovering a SMAC-run from the output\n    # directory, create a new scenario with an extended budget:\n    new_scenario = Scenario(\n        origiginal_scenario_dict,\n        cmd_options={'runcount_limit': 50,  # overwrite these args\n                     'output_dir': 'restored'})\n\n    # We load the runhistory\n    rh_path = os.path.join(old_output_dir, \"runhistory.json\")\n    runhistory = RunHistory()\n    runhistory.load_json(rh_path, new_scenario.cs)\n\n    # And the stats\n    stats_path = os.path.join(old_output_dir, \"stats.json\")\n    stats = Stats(new_scenario)\n    stats.load(stats_path)\n\n    # And the trajectory\n    traj_path = os.path.join(old_output_dir, \"traj_aclib2.json\")\n    trajectory = TrajLogger.read_traj_aclib_format(\n        fn=traj_path,\n        cs=new_scenario.cs)\n    incumbent = trajectory[-1][\"incumbent\"]\n\n    # Now we can initialize SMAC with the recovered objects and restore the\n    # state where we left off. By providing stats and a restore_incumbent, SMAC\n    # automatically detects the intention of restoring a state.\n    smac = SMAC4AC(scenario=new_scenario,\n                   runhistory=runhistory,\n                   stats=stats,\n                   restore_incumbent=incumbent,\n                   run_id=1)\n\n    # Because we changed the output_dir, we might want to copy the old\n    # trajectory-file (runhistory and stats will be complete, but trajectory is\n    # written sequentially)\n    # new_traj_path = os.path.join(new_scenario.output_dir, \"run_1\", \"traj_aclib2.json\")\n    # shutil.copy(traj_path, new_traj_path)\n\n    smac.optimize()"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.9.7"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

develop_codecov/_downloads/1cae360cf7d7a0383b1909e2f21f8b34/branin.ipynb

@@ -0,0 +1,54 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "%matplotlib inline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "\n# Branin\n\nThis file is a wrapper used by SMAC to optimize parameters on the branin function.\nTo run this example in the terminal, execute:\n\n.. code-block:: bash\n\n    cd examples/commandline\n    python ../../scripts/smac.py --scenario branin/scenario.txt\n\n\nInside the scenario, this file and also ``configspace.pcs`` is referenced and therefore used\nfor the optimization. A full call by SMAC looks like this:\n\n.. code-block:: bash\n\n    <algo>           <instance> <instance specific> <cutoff time>  <runlength> <seed> <parameters>\n    python branin.py 0          0                   9999999        0           12345  -x1 0 -x2 0\n\n\nSince SMAC processes results from the commandline, print-statements are\ncrucial. The format of the results must be the following to ensure correct usage:\n\n.. code-block:: bash\n\n    Result for SMAC: <STATUS>, <runtime>, <runlength>, <quality>, <seed>, <instance-specifics>\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "import sys\nimport numpy as np\n\n__copyright__ = \"Copyright 2021, AutoML.org Freiburg-Hannover\"\n__license__ = \"3-clause BSD\"\n\n\ndef branin(x):\n    x1 = x[0]\n    x2 = x[1]\n    a = 1.\n    b = 5.1 / (4. * np.pi ** 2)\n    c = 5. / np.pi\n    r = 6.\n    s = 10.\n    t = 1. / (8. * np.pi)\n    ret = a * (x2 - b * x1 ** 2 + c * x1 - r) ** 2 + s * (1 - t) * np.cos(x1) + s\n\n    return ret\n\n\nif __name__ == '__main__':\n    # Unused in this example:\n    # instance, instance_specific, cutoff, runlength = sys.argv[1:5]\n    seed = sys.argv[5]\n\n    # sys.argv[6] and sys.argv[8] are the names of the target algorithm\n    # parameters (here: \"-x1\", \"-x2\")\n    x = float(sys.argv[7])\n    y = float(sys.argv[9])\n    result = branin((x, y))\n\n    # This line is important so the result can be processed by SMAC:\n    print('Result for SMAC: SUCCESS, -1, -1, %f, %s' % (result, seed))"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.9.7"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

develop_codecov/_downloads/50ec7e341398627061b5a57f92b39aca/synthetic_function.ipynb

@@ -0,0 +1,54 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "%matplotlib inline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "\n# Synthetic Function with few Hyperparameters\n\nAn example of applying SMAC to optimize a synthetic function (2d rosenbrock function).\n\nWe use the SMAC4BB facade because it is designed for black-box function optimization.\nSMAC4BB uses a :term:`Gaussian Process<GP>` or a set of Gaussian Processes whose\nhyperparameters are integrated by Markov-Chain Monte-Carlo as its surrogate model.\nSMAC4BB works best on numerical hyperparameter configuration space and should not\nbe applied to the problems with large evaluation budgets (up to 1000 evaluations).\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "import logging\nlogging.basicConfig(level=logging.INFO)\n\nimport numpy as np\nfrom ConfigSpace.hyperparameters import UniformFloatHyperparameter\n\n# Import ConfigSpace and different types of parameters\nfrom smac.configspace import ConfigurationSpace\nfrom smac.facade.smac_bb_facade import SMAC4BB\nfrom smac.optimizer.acquisition import EI\n\n# Import SMAC-utilities\nfrom smac.scenario.scenario import Scenario\n\n__copyright__ = \"Copyright 2021, AutoML.org Freiburg-Hannover\"\n__license__ = \"3-clause BSD\"\n\n\ndef rosenbrock_2d(x):\n    \"\"\" The 2 dimensional Rosenbrock function as a toy model\n    The Rosenbrock function is well know in the optimization community and\n    often serves as a toy problem. It can be defined for arbitrary\n    dimensions. The minimium is always at x_i = 1 with a function value of\n    zero. All input parameters are continuous. The search domain for\n    all x's is the interval [-5, 10].\n    \"\"\"\n\n    x1 = x[\"x0\"]\n    x2 = x[\"x1\"]\n\n    val = 100. * (x2 - x1 ** 2.) ** 2. + (1 - x1) ** 2.\n    return val\n\n\nif __name__ == \"__main__\":\n    # Build Configuration Space which defines all parameters and their ranges\n    cs = ConfigurationSpace()\n    x0 = UniformFloatHyperparameter(\"x0\", -5, 10, default_value=-3)\n    x1 = UniformFloatHyperparameter(\"x1\", -5, 10, default_value=-4)\n    cs.add_hyperparameters([x0, x1])\n\n    # Scenario object\n    scenario = Scenario({\"run_obj\": \"quality\",  # we optimize quality (alternatively runtime)\n                         \"runcount-limit\": 10,  # max. number of function evaluations\n                         \"cs\": cs,  # configuration space\n                         \"deterministic\": \"true\"\n                         })\n\n    # Use 'gp' or 'gp_mcmc' here\n    model_type = 'gp'\n\n    # Example call of the function\n    # It returns: Status, Cost, Runtime, Additional Infos\n    def_value = rosenbrock_2d(cs.get_default_configuration())\n    print(\"Default Value: %.2f\" % def_value)\n\n    # Optimize, using a SMAC-object\n    print(\"Optimizing! Depending on your machine, this might take a few minutes.\")\n    smac = SMAC4BB(scenario=scenario,\n                   model_type=model_type,\n                   rng=np.random.RandomState(42),\n                   acquisition_func=EI,  # or others like PI, LCB as acquisition functions\n                   tae_runner=rosenbrock_2d)\n\n    smac.optimize()"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.9.7"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

develop_codecov/_downloads/520bfd59b36c1c7d6d9bdf5411ad2c7f/restore_branin.py

@@ -0,0 +1,91 @@
+"""
+Restore Branin
+^^^^^^^^^^^^^^
+This file runs SMAC and then restores the branin run with an extended computation
+budget. This will also work for SMAC runs that have crashed and are continued.
+
+"""
+
+import logging
+logging.basicConfig(level=logging.INFO)
+
+import os
+
+from smac.facade.smac_ac_facade import SMAC4AC
+from smac.runhistory.runhistory import RunHistory
+from smac.scenario.scenario import Scenario
+from smac.stats.stats import Stats
+from smac.utils.io.traj_logging import TrajLogger
+
+
+__copyright__ = "Copyright 2021, AutoML.org Freiburg-Hannover"
+__license__ = "3-clause BSD"
+
+
+if "__main__" == __name__:
+
+    # Initialize scenario, using runcount_limit as budget.
+    origiginal_scenario_dict = {
+        'algo': 'python branin.py',
+        'paramfile': 'branin/configspace.pcs',
+        'run_obj': 'quality',
+        'runcount_limit': 25,
+        'deterministic': True,
+        'output_dir': 'restore_me'}
+    original_scenario = Scenario(origiginal_scenario_dict)
+
+    smac = SMAC4AC(scenario=original_scenario, run_id=1)
+    smac.optimize()
+
+    print("\nBudget exhausted! Starting restoring optimization ...\n")
+
+    # Now the output is in the folder 'restore_me/run_1' (or whatever run_id has
+    # been passed to the SMAC-object above)
+    old_output_dir = os.path.join(original_scenario.output_dir, 'run_1')
+
+    # We could simply modify the scenario-object, stored in
+    # 'smac.solver.scenario' and start optimization again:
+
+    # smac.solver.scenario.ta_run_limit = 50
+    # smac.optimize()
+
+    # Or, to show the whole process of recovering a SMAC-run from the output
+    # directory, create a new scenario with an extended budget:
+    new_scenario = Scenario(
+        origiginal_scenario_dict,
+        cmd_options={'runcount_limit': 50,  # overwrite these args
+                     'output_dir': 'restored'})
+
+    # We load the runhistory
+    rh_path = os.path.join(old_output_dir, "runhistory.json")
+    runhistory = RunHistory()
+    runhistory.load_json(rh_path, new_scenario.cs)
+
+    # And the stats
+    stats_path = os.path.join(old_output_dir, "stats.json")
+    stats = Stats(new_scenario)
+    stats.load(stats_path)
+
+    # And the trajectory
+    traj_path = os.path.join(old_output_dir, "traj_aclib2.json")
+    trajectory = TrajLogger.read_traj_aclib_format(
+        fn=traj_path,
+        cs=new_scenario.cs)
+    incumbent = trajectory[-1]["incumbent"]
+
+    # Now we can initialize SMAC with the recovered objects and restore the
+    # state where we left off. By providing stats and a restore_incumbent, SMAC
+    # automatically detects the intention of restoring a state.
+    smac = SMAC4AC(scenario=new_scenario,
+                   runhistory=runhistory,
+                   stats=stats,
+                   restore_incumbent=incumbent,
+                   run_id=1)
+
+    # Because we changed the output_dir, we might want to copy the old
+    # trajectory-file (runhistory and stats will be complete, but trajectory is
+    # written sequentially)
+    # new_traj_path = os.path.join(new_scenario.output_dir, "run_1", "traj_aclib2.json")
+    # shutil.copy(traj_path, new_traj_path)
+
+    smac.optimize()