From 5cddef8466891349da92653a6f9169f023ee5598 Mon Sep 17 00:00:00 2001 From: Su Zhou Date: Thu, 27 Jul 2023 17:43:57 +0000 Subject: [PATCH] fixup! move autogluon.bench instll to setup.sh --- src/autogluon/bench/Dockerfile | 13 ++++-- .../multimodal/multimodal_benchmark.py | 4 +- .../bench/frameworks/multimodal/setup.sh | 32 +++++++++------ .../bench/frameworks/tabular/setup.sh | 41 +++++++++++-------- .../frameworks/tabular/tabular_benchmark.py | 13 +++++- src/autogluon/bench/runbenchmark.py | 3 +- 6 files changed, 68 insertions(+), 38 deletions(-) diff --git a/src/autogluon/bench/Dockerfile b/src/autogluon/bench/Dockerfile index 01debd3d..96124a13 100644 --- a/src/autogluon/bench/Dockerfile +++ b/src/autogluon/bench/Dockerfile @@ -24,16 +24,21 @@ RUN apt-get update && apt-get install -y --no-install-recommends cron unzip curl RUN chmod +x setup.sh entrypoint.sh /app/gpu_utilization.sh \ && if echo "$FRAMEWORK_PATH" | grep -q "tabular"; then \ + SETUP_COMMAND="bash setup.sh $GIT_URI $GIT_BRANCH $BENCHMARK_DIR $AMLB_FRAMEWORK"; \ if [ -n "$AMLB_USER_DIR" ]; then \ - bash setup.sh $GIT_URI $GIT_BRANCH $BENCHMARK_DIR $AMLB_FRAMEWORK $AMLB_USER_DIR; \ + SETUP_COMMAND+=" -u $AMLB_USER_DIR"; \ + fi; \ + if [ -n "$AG_BENCH_DEV_URL" ]; then \ + SETUP_COMMAND+=" -d $AG_BENCH_DEV_URL"; \ else \ - bash setup.sh $GIT_URI $GIT_BRANCH $BENCHMARK_DIR $AMLB_FRAMEWORK; \ + SETUP_COMMAND+=" -v $AG_BENCH_VERSION"; \ fi; \ + eval $SETUP_COMMAND; \ elif echo "$FRAMEWORK_PATH" | grep -q "multimodal"; then \ if [ -n "$AG_BENCH_DEV_URL" ]; then \ - bash setup.sh $GIT_URI $GIT_BRANCH $BENCHMARK_DIR --AGBENCH_DEV_URL=$AG_BENCH_DEV_URL; \ + bash setup.sh $GIT_URI $GIT_BRANCH $BENCHMARK_DIR -d $AG_BENCH_DEV_URL; \ else \ - bash setup.sh $GIT_URI $GIT_BRANCH $BENCHMARK_DIR --AG_BENCH_VER=$AG_BENCH_VERSION; \ + bash setup.sh $GIT_URI $GIT_BRANCH $BENCHMARK_DIR -v $AG_BENCH_VERSION; \ fi; \ fi \ && echo "*/5 * * * * /app/gpu_utilization.sh >> /var/log/cron.log 2>&1" > /var/spool/cron/crontabs/root \ diff --git a/src/autogluon/bench/frameworks/multimodal/multimodal_benchmark.py b/src/autogluon/bench/frameworks/multimodal/multimodal_benchmark.py index 1635222c..a2c912ac 100644 --- a/src/autogluon/bench/frameworks/multimodal/multimodal_benchmark.py +++ b/src/autogluon/bench/frameworks/multimodal/multimodal_benchmark.py @@ -44,9 +44,9 @@ def setup( setup_script_path = os.path.abspath(os.path.dirname(__file__)) + "/setup.sh" command = [setup_script_path, git_uri, git_branch, self.benchmark_dir] if agbench_dev_url is not None: - command.append(f"--AGBENCH_DEV_URL={agbench_dev_url}") + command.append(f" -d {agbench_dev_url}") else: - command.append(f"--AG_BENCH_VER={agbench_version}") + command.append(f" -v {agbench_version}") result = subprocess.run(command) if result.returncode != 0: sys.exit(1) diff --git a/src/autogluon/bench/frameworks/multimodal/setup.sh b/src/autogluon/bench/frameworks/multimodal/setup.sh index 25c6d676..574f2f5d 100755 --- a/src/autogluon/bench/frameworks/multimodal/setup.sh +++ b/src/autogluon/bench/frameworks/multimodal/setup.sh @@ -5,7 +5,16 @@ set -eo pipefail GIT_URI=$1 BRANCH=$2 DIR=$3 # from root of benchmark run -ARG=$4 # autogluon.bench dev branch uri or version +shift 3 + +while getopts "d:v:" opt; do + case $opt in + d) agbench_dev_url=$OPTARG;; + v) agbench_version=$OPTARG;; + :\?) echo "Error: invaled option -$OPTARG"; exit1;; + :) echo "Error: option -$OPTARG requires an argument"; exit1;; + esac +done if [ ! -d $DIR ]; then mkdir -p $DIR @@ -22,29 +31,28 @@ source $DIR/.venv/bin/activate python3 -m pip install --upgrade pip python3 -m pip install --upgrade setuptools wheel -if [[ "$ARG" == "--AGBENCH_DEV_URL="* ]]; then - AGBENCH_DEV_URL="${ARG#*=}" - echo "Installing autogluon.bench Dev Branch $AGBENCH_DEV_URL" - AGBENCH_URI=$(echo "$AGBENCH_DEV_URL" | cut -d '#' -f 1) - AGBENCH_BRANCH=$(echo "$AGBENCH_DEV_URL" | cut -d '#' -f 2) +# use the agbench_dev_url or agbench_version +if [ -n "$agbench_dev_url" ]; then + echo "Installing autogluon.bench Dev Branch $agbench_dev_url" + AGBENCH_URI=$(echo "$agbench_dev_url" | cut -d '#' -f 1) + AGBENCH_BRANCH=$(echo "$agbench_dev_url" | cut -d '#' -f 2) agbench_repo_name=$(basename -s .git $(echo $AGBENCH_URI)) git clone --single-branch --branch ${AGBENCH_BRANCH} ${AGBENCH_URI} $DIR/$agbench_repo_name cd $DIR/$agbench_repo_name python3 -m pip install -e . cd - -elif [[ "$ARG" == "--AG_BENCH_VER="* ]]; then - AG_BENCH_VER="${ARG#*=}" - echo "Installing autogluon.bench==$AG_BENCH_VER" - output=$(python3 -m pip install autogluon.bench==$AG_BENCH_VER 2>&1) || { +elif [ -n "$agbench_version" ]; then + echo "Installing autogluon.bench==$agbench_version" + output=$(python3 -m pip install autogluon.bench==$agbench_version 2>&1) || { err_message=$output if [[ $err_message == *"No matching distribution"* ]]; then - echo -e "ERROR: No matching distribution found for autogluon.bench==$AG_BENCH_VER\n \ + echo -e "ERROR: No matching distribution found for autogluon.bench==$agbench_version\n \ To resolve the issue, try 'agbench run --dev-branch #" fi exit 1 } else - echo "Invalid argument: $ARG" + echo "No argument provided for autogluon.bench installation. Please provide -d or -v " exit 1 fi diff --git a/src/autogluon/bench/frameworks/tabular/setup.sh b/src/autogluon/bench/frameworks/tabular/setup.sh index 5e0c7a2c..ce5e310e 100755 --- a/src/autogluon/bench/frameworks/tabular/setup.sh +++ b/src/autogluon/bench/frameworks/tabular/setup.sh @@ -6,8 +6,17 @@ GIT_URI=$1 # AMLB Git URI BRANCH=$2 # AMLB branch DIR=$3 # from root of benchmark run AMLB_FRAMEWORK=$4 # e.g. AutoGluon_dev:test -AMLB_USER_DIR=$5 # directory where AMLB customizations are located -$ARG=$6 # autogluon.bench dev branch uri or version +shift 4 + +while getopts "u:d:v:" opt; do + case $opt in + u) amlb_user_dir=$OPTARG;; + d) agbench_dev_url=$OPTARG;; + v) agbench_version=$OPTARG;; + :\?) echo "Error: invaled option -$OPTARG"; exit1;; + :) echo "Error: option -$OPTARG requires an argument"; exit1;; + esac +done if [ ! -d $DIR ]; then mkdir -p $DIR @@ -29,35 +38,33 @@ pip install -r $DIR/automlbenchmark/requirements.txt echo "Installing framework $AMLB_FRAMEWORK" amlb_args="$AMLB_FRAMEWORK -s only" -if [ -n "$AMLB_USER_DIR" ]; then - echo "using user_dir $AMLB_USER_DIR" - amlb_args+=" -u $AMLB_USER_DIR" +if [ -n "$amlb_user_dir" ]; then + echo "using user_dir $amlb_user_dir" + amlb_args+=" -u $amlb_user_dir" fi python3 $DIR/automlbenchmark/runbenchmark.py $amlb_args - -if [[ "$ARG" == "--AGBENCH_DEV_URL="* ]]; then - AGBENCH_DEV_URL="${ARG#*=}" - echo "Installing autogluon.bench Dev Branch $AGBENCH_DEV_URL" - AGBENCH_URI=$(echo "$AGBENCH_DEV_URL" | cut -d '#' -f 1) - AGBENCH_BRANCH=$(echo "$AGBENCH_DEV_URL" | cut -d '#' -f 2) +# use the agbench_dev_url or agbench_version +if [ -n "$agbench_dev_url" ]; then + echo "Installing autogluon.bench Dev Branch $agbench_dev_url" + AGBENCH_URI=$(echo "$agbench_dev_url" | cut -d '#' -f 1) + AGBENCH_BRANCH=$(echo "$agbench_dev_url" | cut -d '#' -f 2) agbench_repo_name=$(basename -s .git $(echo $AGBENCH_URI)) git clone --single-branch --branch ${AGBENCH_BRANCH} ${AGBENCH_URI} $DIR/$agbench_repo_name cd $DIR/$agbench_repo_name python3 -m pip install -e . cd - -elif [[ "$ARG" == "--AG_BENCH_VER="* ]]; then - AG_BENCH_VER="${ARG#*=}" - echo "Installing autogluon.bench==$AG_BENCH_VER" - output=$(python3 -m pip install autogluon.bench==$AG_BENCH_VER 2>&1) || { +elif [ -n "$agbench_version" ]; then + echo "Installing autogluon.bench==$agbench_version" + output=$(python3 -m pip install autogluon.bench==$agbench_version 2>&1) || { err_message=$output if [[ $err_message == *"No matching distribution"* ]]; then - echo -e "ERROR: No matching distribution found for autogluon.bench==$AG_BENCH_VER\n \ + echo -e "ERROR: No matching distribution found for autogluon.bench==$agbench_version\n \ To resolve the issue, try 'agbench run --dev-branch #" fi exit 1 } else - echo "Invalid argument: $ARG" + echo "No argument provided for autogluon.bench installation. Please provide -d or -v " exit 1 fi \ No newline at end of file diff --git a/src/autogluon/bench/frameworks/tabular/tabular_benchmark.py b/src/autogluon/bench/frameworks/tabular/tabular_benchmark.py index 691bba94..8b30b383 100644 --- a/src/autogluon/bench/frameworks/tabular/tabular_benchmark.py +++ b/src/autogluon/bench/frameworks/tabular/tabular_benchmark.py @@ -2,8 +2,8 @@ import os import subprocess import sys -from typing import List +from autogluon.bench import __version__ as agbench_version from autogluon.bench.frameworks.benchmark import Benchmark logger = logging.getLogger(__name__) @@ -14,10 +14,19 @@ def setup( self, git_uri: str = "https://github.com/openml/automlbenchmark.git", git_branch: str = "stable", + framework: str = "AutoGluon:stable", + user_dir: str = None, + agbench_dev_url: str = None, ): """Sets up the virtual environment for tabular benchmark.""" setup_script_path = os.path.abspath(os.path.dirname(__file__)) + "/setup.sh" - command = [setup_script_path, git_uri, git_branch, self.benchmark_dir] + command = [setup_script_path, git_uri, git_branch, self.benchmark_dir, framework] + if user_dir is not None: + command.append(f" -u {user_dir}") + if agbench_dev_url is not None: + command.append(f" -d {agbench_dev_url}") + else: + command.append(f" -v {agbench_version}") result = subprocess.run(command) if result.returncode != 0: logger.error(result.stderr) diff --git a/src/autogluon/bench/runbenchmark.py b/src/autogluon/bench/runbenchmark.py index 7cb72ccb..77282ce3 100644 --- a/src/autogluon/bench/runbenchmark.py +++ b/src/autogluon/bench/runbenchmark.py @@ -29,7 +29,7 @@ logger = logging.getLogger(__name__) -def get_kwargs(module: str, configs: dict, agbench_dev_url: str): +def get_kwargs(module: str, configs: dict, agbench_dev_url: str = None): """Returns a dictionary of keyword arguments to be used for setting up and running the benchmark. Args: @@ -62,6 +62,7 @@ def get_kwargs(module: str, configs: dict, agbench_dev_url: str): "git_branch": git_branch, "framework": configs["framework"], "user_dir": configs.get("amlb_user_dir"), + "agbench_dev_url": agbench_dev_url, }, "run_kwargs": { "framework": configs["framework"],