From ecc894fb5157399be20f4859d4b8f686956eb638 Mon Sep 17 00:00:00 2001 From: Lht97 Date: Tue, 24 Dec 2024 00:03:40 +0800 Subject: [PATCH] Include all the possible workflows --- .../workflows/profile_orig_cbds_bfgs_big.yml | 195 ++++++++++++++++ .../profile_orig_cbds_bfgs_small.yml | 195 ++++++++++++++++ .../workflows/profile_orig_cbds_bfo_big.yml | 209 +++++++++++++++++ .../workflows/profile_orig_cbds_bfo_small.yml | 209 +++++++++++++++++ .../profile_orig_cbds_blocks_big.yml | 198 ++++++++++++++++ .../profile_orig_cbds_blocks_small.yml | 197 ++++++++++++++++ .../workflows/profile_orig_cbds_ds_big.yml | 195 ++++++++++++++++ .../workflows/profile_orig_cbds_ds_small.yml | 195 ++++++++++++++++ .../workflows/profile_orig_cbds_lam_big.yml | 195 ++++++++++++++++ .../workflows/profile_orig_cbds_lam_small.yml | 195 ++++++++++++++++ .../profile_orig_cbds_newuoa_big.yml | 205 ++++++++++++++++ .../profile_orig_cbds_newuoa_small.yml | 205 ++++++++++++++++ .../workflows/profile_orig_cbds_nomad_big.yml | 221 ++++++++++++++++++ .../profile_orig_cbds_nomad_small.yml | 217 +++++++++++++++++ .../profile_orig_cbds_permuted_big.yml | 196 ++++++++++++++++ .../profile_orig_cbds_permuted_small.yml | 195 ++++++++++++++++ ...le_orig_cbds_randomized_orthogonal_big.yml | 197 ++++++++++++++++ ..._orig_cbds_randomized_orthogonal_small.yml | 196 ++++++++++++++++ .../profile_orig_cbds_simplex_big.yml | 195 ++++++++++++++++ .../profile_orig_cbds_simplex_small.yml | 195 ++++++++++++++++ README.md | 16 +- src/bds.m | 34 +-- src/private/get_default_constant.m | 4 + tests/profile.m | 8 +- tests/profile_optiprofiler.m | 11 + tests/tuning/plot_mat.m | 135 +++++++++++ tests/tuning/plot_parameters.m | 2 +- tests/tuning/tuning_script.m | 22 +- 28 files changed, 4204 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/profile_orig_cbds_bfgs_big.yml create mode 100644 .github/workflows/profile_orig_cbds_bfgs_small.yml create mode 100644 .github/workflows/profile_orig_cbds_bfo_big.yml create mode 100644 .github/workflows/profile_orig_cbds_bfo_small.yml create mode 100644 .github/workflows/profile_orig_cbds_blocks_big.yml create mode 100644 .github/workflows/profile_orig_cbds_blocks_small.yml create mode 100644 .github/workflows/profile_orig_cbds_ds_big.yml create mode 100644 .github/workflows/profile_orig_cbds_ds_small.yml create mode 100644 .github/workflows/profile_orig_cbds_lam_big.yml create mode 100644 .github/workflows/profile_orig_cbds_lam_small.yml create mode 100644 .github/workflows/profile_orig_cbds_newuoa_big.yml create mode 100644 .github/workflows/profile_orig_cbds_newuoa_small.yml create mode 100644 .github/workflows/profile_orig_cbds_nomad_big.yml create mode 100644 .github/workflows/profile_orig_cbds_nomad_small.yml create mode 100644 .github/workflows/profile_orig_cbds_permuted_big.yml create mode 100644 .github/workflows/profile_orig_cbds_permuted_small.yml create mode 100644 .github/workflows/profile_orig_cbds_randomized_orthogonal_big.yml create mode 100644 .github/workflows/profile_orig_cbds_randomized_orthogonal_small.yml create mode 100644 .github/workflows/profile_orig_cbds_simplex_big.yml create mode 100644 .github/workflows/profile_orig_cbds_simplex_small.yml create mode 100644 tests/tuning/plot_mat.m diff --git a/.github/workflows/profile_orig_cbds_bfgs_big.yml b/.github/workflows/profile_orig_cbds_bfgs_big.yml new file mode 100644 index 0000000..cb9ee43 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_bfgs_big.yml @@ -0,0 +1,195 @@ +name: Profile original cbds and bfgs using optiprofiler, big + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 0 */2 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS and bfgs, big + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [big] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [fminunc] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, perturbed_x0_0.001, perturbed_x0_1, perturbed_x0_10, perturbed_x0_100, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, truncated_1, truncated_2, truncated_3, truncated_4, permuted, linearly_transformed, random_nan_5, random_nan_10, random_nan_20, quantized_1, quantized_2, quantized_3, quantized_4, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_bfgs_small.yml b/.github/workflows/profile_orig_cbds_bfgs_small.yml new file mode 100644 index 0000000..67fd1e5 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_bfgs_small.yml @@ -0,0 +1,195 @@ +name: Profile original cbds and bfgs using optiprofiler, small + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 6 */2 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS and bfgs, small + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [small] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [fminunc] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, perturbed_x0_0.001, perturbed_x0_1, perturbed_x0_10, perturbed_x0_100, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, truncated_1, truncated_2, truncated_3, truncated_4, permuted, linearly_transformed, random_nan_5, random_nan_10, random_nan_20, quantized_1, quantized_2, quantized_3, quantized_4, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_bfo_big.yml b/.github/workflows/profile_orig_cbds_bfo_big.yml new file mode 100644 index 0000000..f7d42e7 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_bfo_big.yml @@ -0,0 +1,209 @@ +name: Profile original cbds and bfo using optiprofiler, big + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 0 */2 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS and bfo, big + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [big] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [bfo] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, perturbed_x0_0.001, perturbed_x0_1, perturbed_x0_10, perturbed_x0_100, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, truncated_1, truncated_2, truncated_3, truncated_4, permuted, linearly_transformed, random_nan_5, random_nan_10, random_nan_20, quantized_1, quantized_2, quantized_3, quantized_4, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Clone BFO + uses: actions/checkout@v4 + with: + repository: m01marpor/BFO + path: bfo + + - name: modify BFO + run: cd ./bfo && sed -i '/^found /{s/.*/found = 1/;n;d;}' bfo.m && cat bfo.m | grep found + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bfo + cd(fullfile(root_dir, 'bfo')); + addpath(pwd); + + % Setup bds + cd(root_dir); + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_bfo_small.yml b/.github/workflows/profile_orig_cbds_bfo_small.yml new file mode 100644 index 0000000..9a9e201 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_bfo_small.yml @@ -0,0 +1,209 @@ +name: Profile original cbds and bfo using optiprofiler, small + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 6 */2 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS and bfo, small + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [small] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [bfo] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, perturbed_x0_0.001, perturbed_x0_1, perturbed_x0_10, perturbed_x0_100, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, truncated_1, truncated_2, truncated_3, truncated_4, permuted, linearly_transformed, random_nan_5, random_nan_10, random_nan_20, quantized_1, quantized_2, quantized_3, quantized_4, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Clone BFO + uses: actions/checkout@v4 + with: + repository: m01marpor/BFO + path: bfo + + - name: modify BFO + run: cd ./bfo && sed -i '/^found /{s/.*/found = 1/;n;d;}' bfo.m && cat bfo.m | grep found + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bfo + cd(fullfile(root_dir, 'bfo')); + addpath(pwd); + + % Setup bds + cd(root_dir); + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_blocks_big.yml b/.github/workflows/profile_orig_cbds_blocks_big.yml new file mode 100644 index 0000000..b3b285a --- /dev/null +++ b/.github/workflows/profile_orig_cbds_blocks_big.yml @@ -0,0 +1,198 @@ +name: Profile original cbds with different blocks input, big + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 0 */1 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS with different blocks input, big + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [big] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor_1: [ds] + competitor_2: [cbds-half] + competitor_3: [cbds-quarter] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, linearly_transformed, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor_1 }}', '${{ matrix.competitor_2 }}', '${{ matrix.competitor_3 }}'}; + options.dim = '${{ matrix.dim }}'; + options.n_runs = 1; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_blocks_big + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_blocks_big + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_blocks_small.yml b/.github/workflows/profile_orig_cbds_blocks_small.yml new file mode 100644 index 0000000..9032283 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_blocks_small.yml @@ -0,0 +1,197 @@ +name: Profile original cbds with different blocks input, small + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 12 */1 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS with different blocks input, small + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [small] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor_1: [ds] + competitor_2: [cbds-half] + competitor_3: [cbds-quarter] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, linearly_transformed, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor_1 }}', '${{ matrix.competitor_2 }}', '${{ matrix.competitor_3 }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_blocks_small + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_blocks_small + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_ds_big.yml b/.github/workflows/profile_orig_cbds_ds_big.yml new file mode 100644 index 0000000..8fc70e5 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_ds_big.yml @@ -0,0 +1,195 @@ +name: Profile original cbds and ds using optiprofiler, big + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 0 */2 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS and DS, big + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [big] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [ds] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, perturbed_x0_0.001, perturbed_x0_1, perturbed_x0_10, perturbed_x0_100, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, truncated_1, truncated_2, truncated_3, truncated_4, permuted, linearly_transformed, random_nan_5, random_nan_10, random_nan_20, quantized_1, quantized_2, quantized_3, quantized_4, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_ds_small.yml b/.github/workflows/profile_orig_cbds_ds_small.yml new file mode 100644 index 0000000..e8fef9c --- /dev/null +++ b/.github/workflows/profile_orig_cbds_ds_small.yml @@ -0,0 +1,195 @@ +name: Profile original cbds and ds using optiprofiler, small + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 12 */2 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS and DS, small + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [small] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [ds] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, perturbed_x0_0.001, perturbed_x0_1, perturbed_x0_10, perturbed_x0_100, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, truncated_1, truncated_2, truncated_3, truncated_4, permuted, linearly_transformed, random_nan_5, random_nan_10, random_nan_20, quantized_1, quantized_2, quantized_3, quantized_4, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_lam_big.yml b/.github/workflows/profile_orig_cbds_lam_big.yml new file mode 100644 index 0000000..ae83b74 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_lam_big.yml @@ -0,0 +1,195 @@ +name: Profile original cbds and lam using optiprofiler, big + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 0 */2 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS and LAM, big + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [big] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [lam] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, perturbed_x0_0.001, perturbed_x0_1, perturbed_x0_10, perturbed_x0_100, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, truncated_1, truncated_2, truncated_3, truncated_4, permuted, linearly_transformed, random_nan_5, random_nan_10, random_nan_20, quantized_1, quantized_2, quantized_3, quantized_4, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_lam_small.yml b/.github/workflows/profile_orig_cbds_lam_small.yml new file mode 100644 index 0000000..d80a1d0 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_lam_small.yml @@ -0,0 +1,195 @@ +name: Profile original cbds and lam using optiprofiler, small + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 6 */2 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS and LAM, small + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [small] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [lam] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, perturbed_x0_0.001, perturbed_x0_1, perturbed_x0_10, perturbed_x0_100, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, truncated_1, truncated_2, truncated_3, truncated_4, permuted, linearly_transformed, random_nan_5, random_nan_10, random_nan_20, quantized_1, quantized_2, quantized_3, quantized_4, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_newuoa_big.yml b/.github/workflows/profile_orig_cbds_newuoa_big.yml new file mode 100644 index 0000000..8e0d737 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_newuoa_big.yml @@ -0,0 +1,205 @@ +name: Profile original cbds and newuoa using optiprofiler, big + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 0 */2 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS and NEWUOA, big + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [big] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [newuoa] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, perturbed_x0_0.001, perturbed_x0_1, perturbed_x0_10, perturbed_x0_100, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, truncated_1, truncated_2, truncated_3, truncated_4, permuted, linearly_transformed, random_nan_5, random_nan_10, random_nan_20, quantized_1, quantized_2, quantized_3, quantized_4, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Clone PRIMA + uses: actions/checkout@v4 + with: + repository: libprima/prima + path: prima + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup prima + cd(fullfile(root_dir, 'prima')); + setup + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_newuoa_small.yml b/.github/workflows/profile_orig_cbds_newuoa_small.yml new file mode 100644 index 0000000..98f66c4 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_newuoa_small.yml @@ -0,0 +1,205 @@ +name: Profile original cbds and newuoa using optiprofiler, small + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 6 */2 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS and NEWUOA, small + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [small] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [newuoa] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, perturbed_x0_0.001, perturbed_x0_1, perturbed_x0_10, perturbed_x0_100, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, truncated_1, truncated_2, truncated_3, truncated_4, permuted, linearly_transformed, random_nan_5, random_nan_10, random_nan_20, quantized_1, quantized_2, quantized_3, quantized_4, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Clone PRIMA + uses: actions/checkout@v4 + with: + repository: libprima/prima + path: prima + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup prima + cd(fullfile(root_dir, 'prima')); + setup + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_nomad_big.yml b/.github/workflows/profile_orig_cbds_nomad_big.yml new file mode 100644 index 0000000..dfeed33 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_nomad_big.yml @@ -0,0 +1,221 @@ +name: Profile original cbds and nomad using optiprofiler, big + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 18 */2 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS and nomad, big + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [big] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [nomad] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, perturbed_x0_0.001, perturbed_x0_1, perturbed_x0_10, perturbed_x0_100, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, truncated_1, truncated_2, truncated_3, truncated_4, permuted, linearly_transformed, random_nan_5, random_nan_10, random_nan_20, quantized_1, quantized_2, quantized_3, quantized_4, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Checkout Nomad + uses: actions/checkout@v4 + with: + repository: bbopt/nomad + path: nomad + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Set up Nomad + run: | + cd ./nomad + cmake --version + cmake -DTEST_OPENMP=OFF -DBUILD_INTERFACE_MATLAB=ON -S . -B build/release + cmake --build build/release + cmake --install build/release + find /usr/lib /lib -name "libstdc++.so.6" + + - name: Set LD_PRELOAD + run: echo "LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libstdc++.so.6" >> $GITHUB_ENV + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % Setup nomad + cd(fullfile(root_dir, 'nomad', 'build', 'release', 'lib')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options = rmfield(options, 'dim'); + options.mindim = 6; + options.maxdim = 20; + options.n_runs = 1; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_nomad_small.yml b/.github/workflows/profile_orig_cbds_nomad_small.yml new file mode 100644 index 0000000..d56297b --- /dev/null +++ b/.github/workflows/profile_orig_cbds_nomad_small.yml @@ -0,0 +1,217 @@ +name: Profile original cbds and nomad using optiprofiler, small + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 18 */2 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS and nomad, small + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [small] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [nomad] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, perturbed_x0_0.001, perturbed_x0_1, perturbed_x0_10, perturbed_x0_100, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, truncated_1, truncated_2, truncated_3, truncated_4, permuted, linearly_transformed, random_nan_5, random_nan_10, random_nan_20, quantized_1, quantized_2, quantized_3, quantized_4, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Checkout Nomad + uses: actions/checkout@v4 + with: + repository: bbopt/nomad + path: nomad + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Set up Nomad + run: | + cd ./nomad + cmake --version + cmake -DTEST_OPENMP=OFF -DBUILD_INTERFACE_MATLAB=ON -S . -B build/release + cmake --build build/release + cmake --install build/release + find /usr/lib /lib -name "libstdc++.so.6" + + - name: Set LD_PRELOAD + run: echo "LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libstdc++.so.6" >> $GITHUB_ENV + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % Setup nomad + cd(fullfile(root_dir, 'nomad', 'build', 'release', 'lib')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_permuted_big.yml b/.github/workflows/profile_orig_cbds_permuted_big.yml new file mode 100644 index 0000000..5959f98 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_permuted_big.yml @@ -0,0 +1,196 @@ +name: Profile original cbds with permuted matrix input, big + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 0 */1 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS with permuted matrix input, big + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [big] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [cbds-permuted] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, permuted, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4, permuted_noisy_1e-1, permuted_noisy_1e-2, permuted_noisy_1e-3, permuted_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options.n_runs = 1; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_permuted_big + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_permuted_big + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_permuted_small.yml b/.github/workflows/profile_orig_cbds_permuted_small.yml new file mode 100644 index 0000000..9ea8507 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_permuted_small.yml @@ -0,0 +1,195 @@ +name: Profile original cbds with permuted matrix input, small + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 12 */1 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS with permuted matrix input, small + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [small] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [cbds-permuted] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, permuted, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4, permuted_noisy_1e-1, permuted_noisy_1e-2, permuted_noisy_1e-3, permuted_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_permuted_small + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_permuted_small + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_randomized_orthogonal_big.yml b/.github/workflows/profile_orig_cbds_randomized_orthogonal_big.yml new file mode 100644 index 0000000..9d60fd7 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_randomized_orthogonal_big.yml @@ -0,0 +1,197 @@ +name: Profile original cbds with randomized orthogonal matrix input, big + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 0 */1 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS with randomized orthogonal matrix input, big + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [big] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor_1: [cbds-randomized-orthogonal] + competitor_2: [cbds-randomized-gaussian] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, linearly_transformed, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor_1 }}', '${{ matrix.competitor_2 }}'}; + options.dim = '${{ matrix.dim }}'; + options.n_runs = 1; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_randomized_orthogonal_big + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_randomized_orthogonal_big + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_randomized_orthogonal_small.yml b/.github/workflows/profile_orig_cbds_randomized_orthogonal_small.yml new file mode 100644 index 0000000..fa0f6d1 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_randomized_orthogonal_small.yml @@ -0,0 +1,196 @@ +name: Profile original cbds with randomized orthogonal matrix input, small + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 12 */1 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS with randomized orthogonal matrix input, small + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [small] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor_1: [cbds-randomized-orthogonal] + competitor_2: [cbds-randomized-gaussian] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, linearly_transformed, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor_1 }}', '${{ matrix.competitor_2 }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_randomized_orthogonal_small + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_randomized_orthogonal_small + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_simplex_big.yml b/.github/workflows/profile_orig_cbds_simplex_big.yml new file mode 100644 index 0000000..74f4e93 --- /dev/null +++ b/.github/workflows/profile_orig_cbds_simplex_big.yml @@ -0,0 +1,195 @@ +name: Profile original cbds and simplex using optiprofiler, big + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 0 */2 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS and simplex, big + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [big] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [fminsearch] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, perturbed_x0_0.001, perturbed_x0_1, perturbed_x0_10, perturbed_x0_100, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, truncated_1, truncated_2, truncated_3, truncated_4, permuted, linearly_transformed, random_nan_5, random_nan_10, random_nan_20, quantized_1, quantized_2, quantized_3, quantized_4, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/.github/workflows/profile_orig_cbds_simplex_small.yml b/.github/workflows/profile_orig_cbds_simplex_small.yml new file mode 100644 index 0000000..578e5ef --- /dev/null +++ b/.github/workflows/profile_orig_cbds_simplex_small.yml @@ -0,0 +1,195 @@ +name: Profile original cbds and simplex using optiprofiler, small + +on: + # Trigger the workflow on # push or pull request + push: + # Trigger the workflow manually + schedule: + - cron: '0 6 */2 * *' + workflow_dispatch: + +jobs: + test: + name: Profile original CBDS and simplex, small + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + matlab: [latest] + dim: [small] + with_optim_toolbox: [yes] + solver: [cbds-orig] + competitor: [fminsearch] + run_plain: [false] + summarize_log_ratio_profiles: [false] + feature: [plain, perturbed_x0_0.001, perturbed_x0_1, perturbed_x0_10, perturbed_x0_100, noisy_1e-1, noisy_1e-2, noisy_1e-3, noisy_1e-4, truncated_1, truncated_2, truncated_3, truncated_4, permuted, linearly_transformed, random_nan_5, random_nan_10, random_nan_20, quantized_1, quantized_2, quantized_3, quantized_4, rotation_noisy_1e-1, rotation_noisy_1e-2, rotation_noisy_1e-3, rotation_noisy_1e-4] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Checkout OptiProfiler + uses: actions/checkout@v4 + with: + repository: optiprofiler/optiprofiler + ref: matlab + path: optiprofiler + + - name: Install Texlive + run: sudo apt-get update && sudo apt-get install texlive + + - name: Check whether LaTeX is installed + run: pdflatex --version + + - name: Cache MATLAB + id: cache-matlab + uses: actions/cache@v4.0.1 + with: + path: ${{ runner.tool_cache }}/MATLAB + key: ${{ matrix.os }}-${{ matrix.matlab }}-${{ matrix.with_optim_toolbox }} + + - name: Set up MATLAB with optimization toolbox + if: ${{ matrix.with_optim_toolbox == 'yes' }} + uses: matlab-actions/setup-matlab@v2.1.2 + with: + release: ${{ matrix.matlab }} + products: Optimization_Toolbox + + - name: Run test + uses: matlab-actions/run-command@v2.1.1 + with: + command: | + root_dir = pwd() + ls + + % Setup optiprofiler + cd(fullfile(root_dir, 'optiprofiler')); + setup + cd(root_dir); + + % Setup bds + setup + cd(fullfile(root_dir, 'tests', 'competitors')); + addpath(pwd); + + % profile + cd(fullfile(root_dir, 'tests')); + options = struct(); + options.feature_name = '${{ matrix.feature }}'; + options.solver_names = {'${{ matrix.solver }}', '${{ matrix.competitor }}'}; + options.dim = '${{ matrix.dim }}'; + options.run_plain = ${{ matrix.run_plain }}; + options.summarize_log_ratio_profiles = ${{ matrix.summarize_log_ratio_profiles }}; + profile_optiprofiler(options); + cd(root_dir); + cd(fullfile(root_dir, 'tests', 'out')); + ls -R + cd(root_dir); + + - name: Change the summary file name + run: | + cd ./tests/out + ls -R + find . -type f -name 'summary.pdf' | while read -r file; do + # Obtain the parent directory and name + parent_dir=$(dirname "$file") + parent_name=$(basename "$parent_dir") + + # Obtain the new name(Why the fifth underline? Because the last five parts are the time stamp) + new_name=$(echo "$parent_name" | awk -F'_' '{if(NF>=5) {for(i=1;i<=NF-5;i++) printf "%s%s", $i, (i> $GITHUB_ENV + + # Merge PDF files + mkdir summary + for f in $(find . -maxdepth 3 -type f -name 'summary*.pdf'); do + if [ ! -f "summary/$(basename $f)" ]; then + cp "$f" summary/ + fi + done + cd summary + ls + bash ../merge_pdf.sh + mv merged.pdf "summary_${BASE_INFO}.pdf" + cd .. + if [ -d "summary" ]; then + mv summary "summary_${BASE_INFO}" + fi + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: merged_profiles_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/** + + - name: Upload summary files + uses: actions/upload-artifact@v3 + env: + BASE_INFO: ${{ env.BASE_INFO }} + with: + name: summary_files_${{ env.BASE_INFO }} + path: tests/testdata/downloaded_artifacts/summary_${{ env.BASE_INFO }}/** diff --git a/README.md b/README.md index 9204fa5..db58977 100644 --- a/README.md +++ b/README.md @@ -42,37 +42,51 @@ The tests are **automated** by - [Tests](https://github.com/zeroth-order-optimization/bds/actions) at [zeroth-order-optimization/bds](https://github.com/zeroth-order-optimization/bds) - [![Profile cbds and ds using optiprofiler, big](https://github.com/zeroth-order-optimization/bds/actions/workflows/profile_cbds_ds_big.yml/badge.svg)](https://github.com/zeroth-order-optimization/bds/actions/workflows/profile_cbds_ds_big.yml) + - [![Profile original cbds and ds using optiprofiler, big](https://github.com/zeroth-order-optimization/bds/actions/workflows/profile_orig_cbds_ds_big.yml/badge.svg)](https://github.com/zeroth-order-optimization/bds/actions/workflows/profile_orig_cbds_ds_big.yml) - [![Profile cbds and ds using optiprofiler, small](https://github.com/zeroth-order-optimization/bds/actions/workflows/profile_cbds_ds_small.yml/badge.svg)](https://github.com/zeroth-order-optimization/bds/actions/workflows/profile_cbds_ds_small.yml) + - [![Profile original cbds and ds using optiprofiler, small](https://github.com/zeroth-order-optimization/bds/actions/workflows/profile_cbds_ds_small.yml/badge.svg)](https://github.com/zeroth-order-optimization/bds/actions/workflows/profile_cbds_ds_small.yml) - [Tests](https://github.com/0thopt/bds/actions) at [0thopt/bds](https://github.com/0thopt/bds) - [![Profile cbds and bfo using optiprofiler, big](https://github.com/0thopt/bds/actions/workflows/profile_cbds_bfo_big.yml/badge.svg)](https://github.com/0thopt/bds/actions/workflows/profile_cbds_bfo_big.yml) + - [![Profile original cbds and bfo using optiprofiler, big](https://github.com/0thopt/bds/actions/workflows/profile_orig_cbds_bfo_big.yml/badge.svg)](https://github.com/0thopt/bds/actions/workflows/profile_orig_cbds_bfo_big.yml) - [![Profile cbds and bfo using optiprofiler, small](https://github.com/0thopt/bds/actions/workflows/profile_cbds_bfo_small.yml/badge.svg)](https://github.com/0thopt/bds/actions/workflows/profile_cbds_bfo_small.yml) + - [![Profile original cbds and bfo using optiprofiler, small](https://github.com/0thopt/bds/actions/workflows/profile_orig_cbds_bfo_small.yml/badge.svg)](https://github.com/0thopt/bds/actions/workflows/profile_orig_cbds_bfo_small.yml) - [Tests](https://github.com/bladesopt/bds/actions) at [bladesopt/bds](https://github.com/bladesopt/bds) - [![Profile cbds and bfgs using optiprofiler, big](https://github.com/bladesopt/bds/actions/workflows/profile_cbds_bfgs_big.yml/badge.svg)](https://github.com/bladesopt/bds/actions/workflows/profile_cbds_bfgs_big.yml) + - [![Profile original cbds and bfgs using optiprofiler, big](https://github.com/bladesopt/bds/actions/workflows/profile_orig_cbds_bfgs_big.yml/badge.svg)](https://github.com/bladesopt/bds/actions/workflows/profile_orig_cbds_bfgs_big.yml) - [![Profile cbds and bfgs using optiprofiler, small](https://github.com/bladesopt/bds/actions/workflows/profile_cbds_bfgs_small.yml/badge.svg)](https://github.com/bladesopt/bds/actions/workflows/profile_cbds_bfgs_small.yml) + - [![Profile original cbds and bfgs using optiprofiler, small](https://github.com/bladesopt/bds/actions/workflows/profile_orig_cbds_bfgs_small.yml/badge.svg)](https://github.com/bladesopt/bds/actions/workflows/profile_orig_cbds_bfgs_small.yml) - [Tests](https://github.com/derivative-free-optimization/bds/actions) at [derivative-free-optimization/bds](https://github.com/derivative-free-optimization/bds) - [![Profile cbds and lam using optiprofiler, big](https://github.com/derivative-free-optimization/bds/actions/workflows/profile_cbds_lam_big.yml/badge.svg)](https://github.com/derivative-free-optimization/bds/actions/workflows/profile_cbds_lam_big.yml) + - [![Profile original cbds and lam using optiprofiler, big](https://github.com/derivative-free-optimization/bds/actions/workflows/profile_orig_cbds_lam_big.yml/badge.svg)](https://github.com/derivative-free-optimization/bds/actions/workflows/profile_orig_cbds_lam_big.yml) - [![Profile cbds and lam using optiprofiler, small](https://github.com/derivative-free-optimization/bds/actions/workflows/profile_cbds_lam_small.yml/badge.svg)](https://github.com/derivative-free-optimization/bds/actions/workflows/profile_cbds_lam_small.yml) + - [![Profile original cbds and lam using optiprofiler, small](https://github.com/derivative-free-optimization/bds/actions/workflows/profile_orig_cbds_lam_small.yml/badge.svg)](https://github.com/derivative-free-optimization/bds/actions/workflows/profile_orig_cbds_lam_small.yml) - [Tests](https://github.com/dfopt/bds/actions) at [dfopt/bds](https://github.com/dfopt/bds) - [![Profile cbds and newuoa using optiprofiler, big](https://github.com/dfopt/bds/actions/workflows/profile_cbds_newuoa_big.yml/badge.svg)](https://github.com/dfopt/bds/actions/workflows/profile_cbds_newuoa_big.yml) + - [![Profile original cbds and newuoa using optiprofiler, big](https://github.com/dfopt/bds/actions/workflows/profile_orig_cbds_newuoa_big.yml/badge.svg)](https://github.com/dfopt/bds/actions/workflows/profile_orig_cbds_newuoa_big.yml) - [![Profile cbds and newuoa using optiprofiler, small](https://github.com/dfopt/bds/actions/workflows/profile_cbds_newuoa_small.yml/badge.svg)](https://github.com/dfopt/bds/actions/workflows/profile_cbds_newuoa_small.yml) + - [![Profile original cbds and newuoa using optiprofiler, small](https://github.com/dfopt/bds/actions/workflows/profile_orig_cbds_newuoa_small.yml/badge.svg)](https://github.com/dfopt/bds/actions/workflows/profile_orig_cbds_newuoa_small.yml) - [Tests](https://github.com/gradient-free-opt/bds/actions) at [gradient-free-opt/bds](https://github.com/gradient-free-opt/bds) - [![Profile cbds and simplex using optiprofiler, big](https://github.com/gradient-free-opt/bds/actions/workflows/profile_cbds_simplex_big.yml/badge.svg)](https://github.com/gradient-free-opt/bds/actions/workflows/profile_cbds_simplex_big.yml) + - [![Profile original cbds and simplex using optiprofiler, big](https://github.com/gradient-free-opt/bds/actions/workflows/profile_orig_cbds_simplex_big.yml/badge.svg)](https://github.com/gradient-free-opt/bds/actions/workflows/profile_orig_cbds_simplex_big.yml) - [![Profile cbds and simplex using optiprofiler, small](https://github.com/gradient-free-opt/bds/actions/workflows/profile_cbds_simplex_small.yml/badge.svg)](https://github.com/gradient-free-opt/bds/actions/workflows/profile_cbds_simplex_small.yml) + - [![Profile original cbds and simplex using optiprofiler, small](https://github.com/gradient-free-opt/bds/actions/workflows/profile_orig_cbds_simplex_small.yml/badge.svg)](https://github.com/gradient-free-opt/bds/actions/workflows/profile_orig_cbds_simplex_small.yml) - [Tests](https://github.com/libblades/bds/actions) at [libblades/bds](https://github.com/libblades/bds) - [![Profile cbds and nomad, small](https://github.com/libblades/bds/actions/workflows/profile_cbds_nomad_small.yml/badge.svg)](https://github.com/libblades/bds/actions/workflows/profile_cbds_nomad_small.yml) - - [![Profile cbds and nomad, big](https://github.com/libblades/bds/actions/workflows/profile_cbds_nomad_big.yml/badge.svg)](https://github.com/libblades/bds/actions/workflows/profile_cbds_nomad_big.yml) + - [![Profile original cbds and nomad, small](https://github.com/libblades/bds/actions/workflows/profile_orig_cbds_nomad_small.yml/badge.svg)](https://github.com/libblades/bds/actions/workflows/profile_orig_cbds_nomad_small.yml) + - [![Profile cbds and nomad, big](https://github.com/libblades/bds/actions/workflows/profile_cbds_nomad_big.yml/badge.svg)](https://github.com/libblades/bds/actions/workflows/profile_cbds_nomad_big.yml) + - [![Profile original cbds and nomad, big](https://github.com/libblades/bds/actions/workflows/profile_orig_cbds_nomad_big.yml/badge.svg)](https://github.com/libblades/bds/actions/workflows/profile_orig_cbds_nomad_big.yml) - [Tests](https://github.com/opt-lab/bds/actions) at [opt-lab/bds](https://github.com/opt-lab/bds) diff --git a/src/bds.m b/src/bds.m index 2311910..a4d8684 100644 --- a/src/bds.m +++ b/src/bds.m @@ -210,33 +210,35 @@ options.noisy = get_default_constant("is_noisy"); end -% Judge whether the dimension of the problem is small or big. -if numel(x0) <= 5 - expand_key = "expand_small"; - shrink_key = "shrink_small"; +% Set the value of expand and shrink according to the dimension of the problem +% and whether the problem is noisy or not, also according to the Algorithm. +if strcmpi(options.Algorithm, "ds") + expand = get_default_constant("ds_expand"); + shrink = get_default_constant("ds_shrink"); else - % Judge whether the problem is noisy or not. - if isfield(options, "is_noisy") && options.is_noisy - expand_key = "expand_big_noisy"; - shrink_key = "shrink_big_noisy"; + if numel(x0) <= 5 + expand = get_default_constant("expand_small"); + shrink = get_default_constant("shrink_small"); else - expand_key = "expand_big"; - shrink_key = "shrink_big"; + % Judge whether the problem is noisy or not. + if isfield(options, "is_noisy") && options.is_noisy + expand = get_default_constant("expand_big_noisy"); + shrink = get_default_constant("shrink_big_noisy"); + else + expand = get_default_constant("expand_big"); + shrink = get_default_constant("shrink_big"); + end end end -% Set the value of expand. +% Set the value of expand if options contains expand. if isfield(options, "expand") expand = options.expand; -else - expand = get_default_constant(expand_key); end -% Set the value of shrink. +% Set the value of shrink if options contains shrink. if isfield(options, "shrink") shrink = options.shrink; -else - shrink = get_default_constant(shrink_key); end % Set the value of reduction_factor. diff --git a/src/private/get_default_constant.m b/src/private/get_default_constant.m index 72723f6..74d81ae 100644 --- a/src/private/get_default_constant.m +++ b/src/private/get_default_constant.m @@ -9,6 +9,10 @@ constant_value = "cbds"; case {"is_noisy"} constant_value = false; + case {"ds_expand"} + constant_value = 2; + case {"ds_shrink"} + constant_value = 0.5; case {"expand_small"} constant_value = 2; case {"shrink_small"} diff --git a/tests/profile.m b/tests/profile.m index ac1f68f..9d964ff 100644 --- a/tests/profile.m +++ b/tests/profile.m @@ -48,12 +48,12 @@ % Get the parameters that the test needs. parameters = set_profile_options(parameters); - % Tell MATLAB where to find MatCUTEst. - locate_matcutest(); - % Get list of problems if isfield(parameters, "test_type") && strcmpi(parameters.test_type, "matcutest") + % Tell MATLAB where to find MatCUTEst. + locate_matcutest(); + s.type = parameters.problem_type; % Unconstrained: 'u' s.mindim = parameters.problem_mindim; % Minimum of dimension s.maxdim = parameters.problem_maxdim; % Maximum of dimension @@ -153,7 +153,7 @@ end end end - + fprintf("We will load %d problems\n\n", length(problem_names)); % Some fixed (relatively) options diff --git a/tests/profile_optiprofiler.m b/tests/profile_optiprofiler.m index ffa83b0..05a841c 100644 --- a/tests/profile_optiprofiler.m +++ b/tests/profile_optiprofiler.m @@ -151,6 +151,8 @@ function profile_optiprofiler(options) solvers{i} = @pbds_test; case 'cbds' solvers{i} = @cbds_test; + case 'cbds-orig' + solvers{i} = @cbds_orig_test; case 'cbds-noisy' solvers{i} = @(fun, x0) cbds_test_noisy(fun, x0, true); case 'cbds-half' @@ -441,6 +443,15 @@ function profile_optiprofiler(options) end +function x = cbds_orig_test(fun, x0) + + option.Algorithm = 'cbds'; + option.expand = 2; + option.shrink = 0.5; + x = bds(fun, x0, option); + +end + function x = cbds_test_noisy(fun, x0, is_noisy) option.Algorithm = 'cbds'; diff --git a/tests/tuning/plot_mat.m b/tests/tuning/plot_mat.m new file mode 100644 index 0000000..09e39e2 --- /dev/null +++ b/tests/tuning/plot_mat.m @@ -0,0 +1,135 @@ +function plot_mat(p1, p2, perfs, perfs_saved, options, parameters, solver, competitor) +% parameters: a structure with two fields; the field names are the names of the parameters; for each +% field, the value is a vector representing the values of the corresponding parameter. +% solver: a string representing the solver whose performance is to be evaluated. +% competitor: a string representing the competitor solver. +% options: a structure representing the options to be passed to the performance function. +% p1: a vector representing the values of the first parameter. +% p2: a vector representing the values of the second parameter. +% perfs: a matrix representing the differ in performance between the solver and the competitor. +% perfs_saved: a matrix representing the differ in performance between the solver and the competitor saved in the previous run. + +% Get parameter names +param_names = fieldnames(parameters); +assert(length(param_names) == 2, 'There should be two parameters.'); +param1_name = param_names{1}; +param2_name = param_names{2}; + +% We save the results in the `data_path` folder. +current_path = fileparts(mfilename("fullpath")); +% Create the folder if it does not exist. +data_path = fullfile(current_path, "tuning_data"); +if ~exist(data_path, 'dir') + mkdir(data_path); +end +% Creat a subfolder stamped with the current time for the current test. +time_str = char(datetime('now', 'Format', 'yy_MM_dd_HH_mm')); +feature_str = [char(solver), '_vs_', char(competitor), '_', num2str(options.mindim), '_', ... + num2str(options.maxdim), '_', char(options.feature), '_', char(options.test_type)]; +data_path_name = [feature_str, '_', time_str]; +data_path = fullfile(data_path, data_path_name); +mkdir(data_path); + +% Save performance data +save(fullfile(data_path, 'performance_data.mat'), 'p1', 'p2', 'perfs', 'perfs_saved'); + +% Save options into a mat file. +save(fullfile(data_path, 'options.mat'), 'options'); +% Save options into a txt file. +fileID = fopen(fullfile(data_path, 'options.txt'), 'w'); +fprintf(fileID, 'options.mindim = %d;\n', options.mindim); +fprintf(fileID, 'options.maxdim = %d;\n', options.maxdim); +fprintf(fileID, 'options.test_type = "%s";\n', options.test_type); +fprintf(fileID, 'options.tau_weights = [%s];\n', num2str(options.tau_weights)); +fprintf(fileID, 'options.feature = "%s";\n', options.feature); +fprintf(fileID, 'options.num_random = %d;\n', options.num_random); +fprintf(fileID, 'options.tau_indices = [%s];\n', num2str(options.tau_indices)); +fprintf(fileID, 'options.plot_weights = %s;\n', func2str(options.plot_weights)); +fclose(fileID); + +% Save the parameters into a mat file. +save(fullfile(data_path, 'parameters.mat'), 'parameters'); +% Save the parameters into a txt file. +fileID = fopen(fullfile(data_path, 'parameters.txt'), 'w'); +fprintf(fileID, 'parameters.%s = [%s];\n', param1_name, num2str(parameters.(param1_name))); +fprintf(fileID, 'parameters.%s = [%s];\n', param2_name, num2str(parameters.(param2_name))); +fclose(fileID); + +% Plot +FigHandle=figure('Name', ['(', param1_name, ', ', param2_name, ')', ' v.s. performance']); +hold on; + +colormap(jet); + +if isfield(options, 'log_color') && options.log_color + % Use log scale of perfs for a better usage of the color spectrum. + max_perf = max(perfs(:)); + min_perf = min(perfs(:)); + C = min_perf + (max_perf - min_perf) .* log(perfs - min_perf + 1) ./ log(max_perf - min_perf + 1); + surf(p1, p2, perfs, C, 'FaceColor','interp', 'FaceAlpha', 0.8, ... + 'EdgeColor', [0.2 0.2 0.2], 'LineWidth', 0.5); +else + surf(p1, p2, perfs, 'FaceColor','interp', 'FaceAlpha', 0.8, ... + 'EdgeColor', [0.2 0.2 0.2], 'LineWidth', 0.5); +end + +title(gca, strrep(feature_str, '_', '-')); +xlabel(param1_name); +ylabel(param2_name); + +colorbar; + +% Find the top 10 maximum values +[~, idx] = maxk(perfs(:), 10); + +markerSize = 10; % 减小圆圈大小 +labelFontSize = 10; % 减小字体大小 + +% 添加偏移 +z_offset = (max(perfs(:)) - min(perfs(:))) * 0.001; + +% 画空心圆，使用深色边缘 +h_points = plot3(p1(idx), p2(idx), perfs(idx) + z_offset, 'o', 'MarkerSize', markerSize, ... + 'MarkerFaceColor', 'none', 'MarkerEdgeColor', [0.1 0.1 0.1], 'LineWidth', 1.5); + +% 添加黑色标号 +h_text = zeros(length(idx), 1); +for i = 1:length(idx) + h_text(i) = text(p1(idx(i)), p2(idx(i)), perfs(idx(i)) + z_offset, num2str(i), ... + 'VerticalAlignment', 'middle', 'HorizontalAlignment', 'center', ... + 'Color', 'k', 'FontSize', labelFontSize, 'FontWeight', 'bold'); +end + +% 将点和文本移到最上层 +uistack(h_points, 'top'); +for i = 1:length(h_text) + uistack(h_text(i), 'top'); +end + +view(3) % 3D view +% Save fig +saveas(FigHandle, fullfile(data_path, [param1_name, '_', param2_name, '_vs_performance_3d.fig']), 'fig'); +% Use openfig to open the fig file. +% openfig('my3DPlot.fig'); +% Save eps of 3d plot +saveas(FigHandle, fullfile(data_path, [param1_name, '_', param2_name, '_vs_performance_3d.eps']), 'epsc'); +% Try converting the eps to pdf. +epsPath = fullfile(data_path, [param1_name, '_', param2_name, '_vs_performance_3d.eps']); +% One way to convert eps to pdf, without showing the output of the command. +system(('epstopdf '+epsPath+' 2> /dev/null')); + +% Save eps of 2d plot +view(2); % Top-down view +% Save fig +saveas(FigHandle, fullfile(data_path, [param1_name, '_', param2_name, '_vs_performance_2d.fig']), 'fig'); +% Save eps of 2d plot +saveas(FigHandle, fullfile(data_path, [param1_name, '_', param2_name, '_vs_performance_2d.eps']), 'epsc'); +% Try converting the eps to pdf. +epsPath = fullfile(data_path, [param1_name, '_', param2_name, '_vs_performance_2d.eps']); +% One way to convert eps to pdf, without showing the output of the command. +system(('epstopdf '+epsPath+' 2> /dev/null')); + + +fprintf('Performance data and plots saved in \n %s\n', data_path); + +end diff --git a/tests/tuning/plot_parameters.m b/tests/tuning/plot_parameters.m index 2fd3876..1586a57 100644 --- a/tests/tuning/plot_parameters.m +++ b/tests/tuning/plot_parameters.m @@ -20,7 +20,7 @@ function plot_parameters(parameters, solver, competitor, options) perfs_saved = cell(size(p1, 1), size(p1, 2)); % Get performance for each parameter combination -for ip = 1:numel(p1) +parfor ip = 1:numel(p1) % Set solver options solver_options = struct(); solver_options.(param1_name) = p1(ip); diff --git a/tests/tuning/tuning_script.m b/tests/tuning/tuning_script.m index 0a7e28e..5b97e19 100644 --- a/tests/tuning/tuning_script.m +++ b/tests/tuning/tuning_script.m @@ -1,6 +1,6 @@ parameters = struct(); -parameters.expand = [1.25:0.25:2, 2.5:0.5:5]; -parameters.shrink = 0.4:0.05:0.85; +parameters.expand = 1.25:0.25:5; +parameters.shrink = [0.2:0.1:0.5, 0.55:0.05:0.9]; fprintf('Number of parameter combinations: %d\n', length(parameters.expand)*length(parameters.shrink)); @@ -20,15 +20,15 @@ options.num_random = 1; plot_parameters(parameters, solver, competitor, options); -options.feature = "noise_1e-3_no_rotation"; -fprintf('Feature:\t %s\n', options.feature); -options.num_random = 3; -plot_parameters(parameters, solver, competitor, options); - -options.feature = "rotation_noisy_1e-3"; -fprintf('Feature:\t %s\n', options.feature); -options.num_random = 3; -plot_parameters(parameters, solver, competitor, options); +% options.feature = "noise_1e-3_no_rotation"; +% fprintf('Feature:\t %s\n', options.feature); +% options.num_random = 3; +% plot_parameters(parameters, solver, competitor, options); +% +% options.feature = "rotation_noisy_1e-3"; +% fprintf('Feature:\t %s\n', options.feature); +% options.num_random = 3; +% plot_parameters(parameters, solver, competitor, options); function w = plot_weights(x) w = 1;