Skip to content

Commit

Permalink
Merge pull request #537 from automl/development
Browse files Browse the repository at this point in the history
Development - Create release 0.11.1
  • Loading branch information
mfeurer authored Sep 20, 2019
2 parents 81eba54 + 13f629e commit 2b27e0d
Show file tree
Hide file tree
Showing 7 changed files with 175 additions and 12 deletions.
5 changes: 4 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,12 @@ matrix:
env: TESTSUITE=run_unittests.sh PYTHON_VERSION="3.7" COVERAGE="true" DOCPUSH="true" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
# Other tests (mypy, examples, flake8...)
- os: linux
env: TESTSUITE=run_mypy.sh PYTHON_VERSION="3.6" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
env: TESTSUITE=run_flake8.sh PYTHON_VERSION="3.6" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
- os: linux
env: TESTSUITE=run_examples.sh PYTHON_VERSION="3.6" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
- os: linux
env: TESTSUITE=run_mypy.sh PYTHON_VERSION="3.6" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"

# Disable OSX building because it takes too long and hinders progress
# Set language to generic to not break travis-ci
# https://github.com/travis-ci/travis-ci/issues/2312#issuecomment-195620855
Expand Down
7 changes: 7 additions & 0 deletions changelog.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
# 0.11.1

## Changes

* Updated the default hyperparameters of the Gaussian process facade to follow recent research (#529)
* Enabled `flake8` code style checks for newly merged code (#525)

# 0.11.0

## Major changes
Expand Down
141 changes: 141 additions & 0 deletions ci_scripts/run_flake8.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
#!/bin/bash

# This script is mostly taken from https://github.com/scikit-learn/scikit-learn/blob/master/build_tools/travis/flake8_diff.sh

# This script is used in Travis to check that PRs do not add obvious
# flake8 violations. It relies on two things:
# - find common ancestor between branch and
# automl/auto-sklearn remote
# - run flake8 --diff on the diff between the branch and the common
# ancestor
#
# Additional features:
# - the line numbers in Travis match the local branch on the PR
# author machine.
# - ./build_tools/travis/flake8_diff.sh can be run locally for quick
# turn-around

set -e
# pipefail is necessary to propagate exit codes
set -o pipefail

PROJECT=automl/SMAC3
PROJECT_URL=https://github.com/$PROJECT.git

# Find the remote with the project name (upstream in most cases)
REMOTE=$(git remote -v | grep $PROJECT | cut -f1 | head -1 || echo '')

# Add a temporary remote if needed. For example this is necessary when
# Travis is configured to run in a fork. In this case 'origin' is the
# fork and not the reference repo we want to diff against.
if [[ -z "$REMOTE" ]]; then
TMP_REMOTE=tmp_reference_upstream
REMOTE=$TMP_REMOTE
git remote add $REMOTE $PROJECT_URL
fi

echo "Remotes:"
echo '--------------------------------------------------------------------------------'
git remote --verbose

# Travis does the git clone with a limited depth.
# This may not be enough to find the common ancestor with
# $REMOTE/development so we unshallow the git checkout
if [[ -a .git/shallow ]]; then
echo -e '\nTrying to unshallow the repo:'
echo '--------------------------------------------------------------------------------'
git fetch --unshallow
fi

if [[ "$TRAVIS" == "true" ]]; then
if [[ "$TRAVIS_PULL_REQUEST" == "false" ]]
then
# In main repo, using TRAVIS_COMMIT_RANGE to test the commits
# that were pushed into a branch
if [[ "$PROJECT" == "$TRAVIS_REPO_SLUG" ]]; then
if [[ -z "$TRAVIS_COMMIT_RANGE" ]]; then
echo "New branch, no commit range from Travis so passing this test by convention"
exit 0
fi
COMMIT_RANGE=$TRAVIS_COMMIT_RANGE
fi
else
# We want to fetch the code as it is in the PR branch and not
# the result of the merge into development. This way line numbers
# reported by Travis will match with the local code.
LOCAL_BRANCH_REF=travis_pr_$TRAVIS_PULL_REQUEST
# In Travis the PR target is always origin
git fetch origin pull/$TRAVIS_PULL_REQUEST/head:refs/$LOCAL_BRANCH_REF
fi
fi

# If not using the commit range from Travis we need to find the common
# ancestor between $LOCAL_BRANCH_REF and $REMOTE/development
if [[ -z "$COMMIT_RANGE" ]]; then
if [[ -z "$LOCAL_BRANCH_REF" ]]; then
LOCAL_BRANCH_REF=$(git rev-parse --abbrev-ref HEAD)
fi
echo -e "\nLast 2 commits in $LOCAL_BRANCH_REF:"
echo '--------------------------------------------------------------------------------'
git --no-pager log -2 $LOCAL_BRANCH_REF

REMOTE_DEVELOPMENT_REF="$REMOTE/development"
# Make sure that $REMOTE_DEVELOPMENT_REF is a valid reference
echo -e "\nFetching $REMOTE_DEVELOPMENT_REF"
echo '--------------------------------------------------------------------------------'
git fetch $REMOTE development:refs/remotes/$REMOTE_DEVELOPMENT_REF
LOCAL_BRANCH_SHORT_HASH=$(git rev-parse --short $LOCAL_BRANCH_REF)
REMOTE_DEVELOPMENT_SHORT_HASH=$(git rev-parse --short $REMOTE_DEVELOPMENT_REF)

COMMIT=$(git merge-base $LOCAL_BRANCH_REF $REMOTE_DEVELOPMENT_REF) || \
echo "No common ancestor found for $(git show $LOCAL_BRANCH_REF -q) and $(git show $REMOTE_DEVELOPMENT_REF -q)"

if [ -z "$COMMIT" ]; then
exit 1
fi

COMMIT_SHORT_HASH=$(git rev-parse --short $COMMIT)

echo -e "\nCommon ancestor between $LOCAL_BRANCH_REF ($LOCAL_BRANCH_SHORT_HASH)"\
"and $REMOTE_DEVELOPMENT_REF ($REMOTE_DEVELOPMENT_SHORT_HASH) is $COMMIT_SHORT_HASH:"
echo '--------------------------------------------------------------------------------'
git --no-pager show --no-patch $COMMIT_SHORT_HASH

COMMIT_RANGE="$COMMIT_SHORT_HASH..$LOCAL_BRANCH_SHORT_HASH"

if [[ -n "$TMP_REMOTE" ]]; then
git remote remove $TMP_REMOTE
fi

else
echo "Got the commit range from Travis: $COMMIT_RANGE"
fi

echo -e '\nRunning flake8 on the diff in the range' "$COMMIT_RANGE" \
"($(git rev-list $COMMIT_RANGE | wc -l) commit(s)):"
echo '--------------------------------------------------------------------------------'

# We need the following command to exit with 0 hence the echo in case
# there is no match
MODIFIED_FILES="$(git diff --name-only $COMMIT_RANGE || echo "no_match")"

check_files() {
files="$1"
shift
options="$*"
if [ -n "$files" ]; then
# Conservative approach: diff without context (--unified=0) so that code
# that was not changed does not create failures
git diff --unified=0 $COMMIT_RANGE -- $files | flake8 --diff --max-line-length=120 --show-source $options
fi
}

if [[ "$MODIFIED_FILES" == "no_match" ]]; then
echo "No file has been modified"
else

check_files "$(echo "$MODIFIED_FILES" | grep -v ^examples)"
check_files "$(echo "$MODIFIED_FILES" | grep ^examples)" \
--config ./examples/.flake8
fi
echo -e "No problem detected by flake8\n"
3 changes: 2 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,5 +37,6 @@ def get_author():
author=get_author(),
version=get_version(),
test_suite="nose.collector",
tests_require=["mock", "nose"]
tests_require=["mock", "nose"],
long_description_content_type='text/markdown',
)
2 changes: 1 addition & 1 deletion smac/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import lazy_import
from smac.utils import dependencies

__version__ = '0.11.0'
__version__ = '0.11.1'
__author__ = 'Marius Lindauer, Matthias Feurer, Katharina Eggensperger, Joshua Marben, André Biedenkapp, Aaron Klein, Stefan Falkner and Frank Hutter'


Expand Down
3 changes: 0 additions & 3 deletions smac/epm/gaussian_process_mcmc.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,6 @@ def _train(self, X: np.ndarray, y: np.ndarray, do_optimize: bool=True):

# Take the last samples from each walker
self.hypers = sampler.chain[:, -1]
print('hypers', self.hypers.mean(axis=0), np.exp(self.hypers.mean(axis=0)))
elif self.mcmc_sampler == 'nuts':
# Originally published as:
# http://www.stat.columbia.edu/~gelman/research/published/nuts.pdf
Expand Down Expand Up @@ -210,7 +209,6 @@ def _train(self, X: np.ndarray, y: np.ndarray, do_optimize: bool=True):
indices = [int(np.rint(ind)) for ind in np.linspace(start=0, stop=len(samples) - 1, num=10)]
self.hypers = samples[indices]
self.p0 = self.hypers.mean(axis=0)
print('hypers', 'log space', self.p0, 'regular space', np.exp(self.p0))
else:
raise ValueError(self.mcmc_sampler)

Expand Down Expand Up @@ -269,7 +267,6 @@ def _train(self, X: np.ndarray, y: np.ndarray, do_optimize: bool=True):
model.std_y_ = self.std_y_

self.is_trained = True
print('#LL evaluations', self._n_ll_evals)

def _ll(self, theta: np.ndarray) -> float:
"""
Expand Down
26 changes: 20 additions & 6 deletions smac/facade/smac_bo_facade.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from smac.epm.gp_base_prior import HorseshoePrior, LognormalPrior
from smac.epm.util_funcs import get_types, get_rng
from smac.initial_design.sobol_design import SobolDesign
from smac.runhistory.runhistory2epm import RunHistory2EPM4LogScaledCost
from smac.runhistory.runhistory2epm import RunHistory2EPM4Cost


__author__ = "Marius Lindauer"
Expand All @@ -20,6 +20,20 @@ class SMAC4BO(SMAC4AC):
see smac.facade.smac_Facade for API
This facade overwrites options available via the SMAC facade
Hyperparameters are chosen according to the best configuration for Gaussian process maximum likelihood found in
"Towards Assessing the Impact of Bayesian Optimization's Own Hyperparameters" by Lindauer et al., presented at the
DSO workshop 2019 (https://arxiv.org/abs/1908.06674).
Changes are:
* Instead of having an initial design of size 10*D as suggested by Jones et al. 1998 (actually, they suggested
10*D+1), we use an initial design of 8*D.
* More restrictive lower and upper bounds on the length scale for the Matern and Hamming Kernel than the ones
suggested by Klein et al. 2017 in the RoBO package. In practice, they are ``np.exp(-6.754111155189306)``
instead of ``np.exp(-10)`` for the lower bound and ``np.exp(0.0858637988771976)`` instead of
``np.exp(2)`` for the upper bound.
* The initial design is set to be a Sobol grid
* The random fraction is set to ``0.08447232371720552``, it was ``0.0`` before.
Attributes
----------
logger
Expand All @@ -40,10 +54,10 @@ def __init__(self, model_type='gp_mcmc', **kwargs):
scenario = kwargs['scenario']

kwargs['initial_design'] = kwargs.get('initial_design', SobolDesign)
kwargs['runhistory2epm'] = kwargs.get('runhistory2epm', RunHistory2EPM4LogScaledCost)
kwargs['runhistory2epm'] = kwargs.get('runhistory2epm', RunHistory2EPM4Cost)

init_kwargs = kwargs.get('initial_design_kwargs', dict())
init_kwargs['n_configs_x_params'] = init_kwargs.get('n_configs_x_params', 10)
init_kwargs['n_configs_x_params'] = init_kwargs.get('n_configs_x_params', 8)
init_kwargs['max_config_fracs'] = init_kwargs.get('max_config_fracs', 0.25)
kwargs['initial_design_kwargs'] = init_kwargs

Expand All @@ -68,15 +82,15 @@ def __init__(self, model_type='gp_mcmc', **kwargs):
if len(cont_dims) > 0:
exp_kernel = Matern(
np.ones([len(cont_dims)]),
[(np.exp(-10), np.exp(2)) for _ in range(len(cont_dims))],
[(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cont_dims))],
nu=2.5,
operate_on=cont_dims,
)

if len(cat_dims) > 0:
ham_kernel = HammingKernel(
np.ones([len(cat_dims)]),
[(np.exp(-10), np.exp(2)) for _ in range(len(cat_dims))],
[(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cat_dims))],
operate_on=cat_dims,
)

Expand Down Expand Up @@ -125,7 +139,7 @@ def __init__(self, model_type='gp_mcmc', **kwargs):

if kwargs.get('random_configuration_chooser') is None:
random_config_chooser_kwargs = kwargs.get('random_configuration_chooser_kwargs', dict())
random_config_chooser_kwargs['prob'] = random_config_chooser_kwargs.get('prob', 0.0)
random_config_chooser_kwargs['prob'] = random_config_chooser_kwargs.get('prob', 0.08447232371720552)
kwargs['random_configuration_chooser_kwargs'] = random_config_chooser_kwargs

if kwargs.get('acquisition_function_optimizer') is None:
Expand Down

0 comments on commit 2b27e0d

Please sign in to comment.