From 587e2cac0a543520b3830113db9f8e2e630881ca Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Wed, 21 Jun 2023 15:42:10 +0530 Subject: [PATCH 1/6] Add benchmark script, use hyperfine --- .gitignore | 5 +++++ .gitmodules | 3 +++ Makefile | 36 ++++++++++++++++++++++++++++++++++++ README.md | 26 ++++++++++++++++++++++++++ benches/bench.sh | 8 ++++++++ vendor/pulldown-cmark | 1 + 6 files changed, 79 insertions(+) create mode 100755 benches/bench.sh create mode 160000 vendor/pulldown-cmark diff --git a/.gitignore b/.gitignore index cda7c229..573c057e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,8 @@ target comrak-* .vscode .idea +vendor/comrak +vendor/progit +benches/cmark-gfm +benches/comrak-* +benches/pulldown-cmark diff --git a/.gitmodules b/.gitmodules index 3f4aa421..0b59dcd6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "vendor/cmark-gfm"] path = vendor/cmark-gfm url = https://github.com/kivikakk/cmark-gfm.git +[submodule "vendor/pulldown-cmark"] + path = vendor/pulldown-cmark + url = https://github.com/raphlinus/pulldown-cmark.git diff --git a/Makefile b/Makefile index 9a4e2527..6153af64 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,7 @@ +ROOT:=$(shell git rev-parse --show-toplevel) +COMMIT:=$(shell git rev-parse --short HEAD) +MIN_RUNS:=25 + src/scanners.rs: src/scanners.re re2rust -W -Werror -i --no-generation-date -o $@ $< cargo fmt @@ -5,3 +9,35 @@ src/scanners.rs: src/scanners.re bench: cargo build --release (cd vendor/cmark-gfm/; make bench PROG=../../target/release/comrak) + +binaries: build-comrak-branch build-comrak-master build-cmark-gfm build-pulldown-cmark + +build-comrak-branch: + cargo build --release + cp ${ROOT}/target/release/comrak ${ROOT}/benches/comrak-${COMMIT} + +build-comrak-master: + git clone https://github.com/kivikakk/comrak.git --depth 1 --single-branch ${ROOT}/vendor/comrak || true + cd ${ROOT}/vendor/comrak && \ + cargo build --release && \ + cp ./target/release/comrak ${ROOT}/benches/comrak-main + +build-cmark-gfm: + cd ${ROOT}/vendor/cmark-gfm && \ + make && \ + cp build/src/cmark-gfm ${ROOT}/benches/cmark-gfm + +build-pulldown-cmark: + cd ${ROOT}/vendor/pulldown-cmark && \ + cargo build --release && \ + cp target/release/pulldown-cmark ${ROOT}/benches/pulldown-cmark + +bench-comrak: build-comrak-branch + git clone https://github.com/progit/progit.git ${ROOT}/vendor/progit || true > /dev/null + cd benches && \ + hyperfine --prepare 'sudo sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' --warmup 3 --min-runs ${MIN_RUNS} -L binary comrak-${COMMIT} './bench.sh ./{binary}' + +bench-all: binaries + git clone https://github.com/progit/progit.git ${ROOT}/vendor/progit || true > /dev/null + cd benches && \ + hyperfine --prepare 'sudo sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' --warmup 3 --min-runs ${MIN_RUNS} -L binary comrak-${COMMIT},comrak-main,pulldown-cmark,cmark-gfm './bench.sh ./{binary}' --export-markdown ${ROOT}/bench-output.md \ No newline at end of file diff --git a/README.md b/README.md index ea65e234..e198df91 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,32 @@ assert_eq!( \n"); ``` +## Benchmarking + +For running benchmarks, you will need to [install hyperfine](https://github.com/sharkdp/hyperfine#installation) and optionally cmake. + +If you want to just run benchmark for comrak, with current state of repo, you can simply run +```bash +make bench-comrak +``` + +This will build comrak in release mode, and run benchmark on it. You will see the time measurements as reported by hyperfine in the console. + +Makefile also provides a way to run benchmarks for comark current state (with your changes), comrak main branch, cmark-gfm and pulldown-cmark. For this you will need to install cmake. After that make sure that you have set-up the git submodules. In case you have not installed submodules when cloning, you can do it by running +```bash +git submodule update --init +``` + +After this is done, you can run +```bash +make bench-all +``` + +which will run benchmarks across all, and report the time take by each as well as relative time. + + + + ## Security As with [`cmark`](https://github.com/commonmark/cmark) and [`cmark-gfm`](https://github.com/github/cmark-gfm#security), diff --git a/benches/bench.sh b/benches/bench.sh new file mode 100755 index 00000000..cd0abec0 --- /dev/null +++ b/benches/bench.sh @@ -0,0 +1,8 @@ +#! /bin/bash + +PROG=$1 +ROOTDIR=$(git rev-parse --show-toplevel) + +for lang in ar az be ca cs de en eo es es-ni fa fi fr hi hu id it ja ko mk nl no-nb pl pt-br ro ru sr th tr uk vi zh zh-tw; do \ + cat $ROOTDIR/vendor/progit/$lang/*/*.markdown | $PROG > /dev/null +done \ No newline at end of file diff --git a/vendor/pulldown-cmark b/vendor/pulldown-cmark new file mode 160000 index 00000000..34c2bb51 --- /dev/null +++ b/vendor/pulldown-cmark @@ -0,0 +1 @@ +Subproject commit 34c2bb51a48a27c92d0ab48327c73276d6978ae6 From a798db30ba53f9ca197a2b45e1d44c5467e1aa8d Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Wed, 21 Jun 2023 16:20:38 +0530 Subject: [PATCH 2/6] Setup CI for running benchmarks and posting comments --- .github/workflows/benchmarks.yml | 36 ++++++++++++++++++++++++++++++++ README.md | 1 + 2 files changed, 37 insertions(+) create mode 100644 .github/workflows/benchmarks.yml diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000..6797a977 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,36 @@ +name: benchmarks + +on: + pull_request: + types: + - opened + - reopened + issue_comment: + types: + - created + +jobs: + run_benchmarks: + runs-on: ubuntu-latest + permissions: + pull-requests: write + # run either when pull request is opened or when comment body (only on pr) is /run-bench + if: (github.event_name == 'pull_request') || ((github.event.issue.pull_request != null) && github.event.comment.body == '/run-bench') + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: Install hyperfine + run: cargo install hyperfine + - name: Install cmake + run: sudo apt-get update && sudo apt-get install cmake -y + - name: Build Binaries + run: make binaries + - name: Run Benchmarks + run: make bench-all + - name: Post result comment + uses: mshick/add-pr-comment@v2 + with: + message-path: bench-output.md \ No newline at end of file diff --git a/README.md b/README.md index e198df91..cc5609a8 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,7 @@ make bench-all which will run benchmarks across all, and report the time take by each as well as relative time. +Apart from this, CI is also setup for running benchmarks when a pull request is first opened. It will comment the results on the pull request in a tabular format comparing the 4 versions. After that you can manually trigger this CI by commenting `/run-bench` on the PR, this will update the existing comment with new results. Note Benchmarks won't be automatically run on each push. From 645bf34d281e24afd44f8520b2e09a82827edac9 Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Tue, 27 Jun 2023 16:40:48 +0530 Subject: [PATCH 3/6] Remove prepare step from hyperfine and add date-time in the o/p --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6153af64..427d52b6 100644 --- a/Makefile +++ b/Makefile @@ -35,9 +35,11 @@ build-pulldown-cmark: bench-comrak: build-comrak-branch git clone https://github.com/progit/progit.git ${ROOT}/vendor/progit || true > /dev/null cd benches && \ - hyperfine --prepare 'sudo sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' --warmup 3 --min-runs ${MIN_RUNS} -L binary comrak-${COMMIT} './bench.sh ./{binary}' + hyperfine --warmup 3 --min-runs ${MIN_RUNS} -L binary comrak-${COMMIT} './bench.sh ./{binary}' bench-all: binaries git clone https://github.com/progit/progit.git ${ROOT}/vendor/progit || true > /dev/null cd benches && \ - hyperfine --prepare 'sudo sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' --warmup 3 --min-runs ${MIN_RUNS} -L binary comrak-${COMMIT},comrak-main,pulldown-cmark,cmark-gfm './bench.sh ./{binary}' --export-markdown ${ROOT}/bench-output.md \ No newline at end of file + hyperfine --warmup 10 --min-runs ${MIN_RUNS} -L binary comrak-${COMMIT},comrak-main,pulldown-cmark,cmark-gfm './bench.sh ./{binary}' --export-markdown ${ROOT}/bench-output.md &&\ + echo "\n\nRun on" `date -u` >> ${ROOT}/bench-output.md + From b2c3480abcf5e9576525f5558e10e3d6e2142188 Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Wed, 28 Jun 2023 12:18:24 +0530 Subject: [PATCH 4/6] Add markdown-it rs for benchmark comparison --- .gitignore | 1 + .gitmodules | 3 +++ Makefile | 9 +++++++-- vendor/markdown-it | 1 + 4 files changed, 12 insertions(+), 2 deletions(-) create mode 160000 vendor/markdown-it diff --git a/.gitignore b/.gitignore index 573c057e..9c22735a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ vendor/progit benches/cmark-gfm benches/comrak-* benches/pulldown-cmark +benches/markdown-it diff --git a/.gitmodules b/.gitmodules index 0b59dcd6..c6383db7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "vendor/pulldown-cmark"] path = vendor/pulldown-cmark url = https://github.com/raphlinus/pulldown-cmark.git +[submodule "vendor/markdown-it"] + path = vendor/markdown-it + url = https://github.com/rlidwka/markdown-it.rs.git diff --git a/Makefile b/Makefile index 427d52b6..0dfb5a25 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ bench: cargo build --release (cd vendor/cmark-gfm/; make bench PROG=../../target/release/comrak) -binaries: build-comrak-branch build-comrak-master build-cmark-gfm build-pulldown-cmark +binaries: build-comrak-branch build-comrak-master build-cmark-gfm build-pulldown-cmark build-markdown-it build-comrak-branch: cargo build --release @@ -27,6 +27,11 @@ build-cmark-gfm: make && \ cp build/src/cmark-gfm ${ROOT}/benches/cmark-gfm +build-markdown-it: + cd ${ROOT}/vendor/markdown-it && \ + cargo build --release && \ + cp target/release/markdown-it ${ROOT}/benches/markdown-it + build-pulldown-cmark: cd ${ROOT}/vendor/pulldown-cmark && \ cargo build --release && \ @@ -40,6 +45,6 @@ bench-comrak: build-comrak-branch bench-all: binaries git clone https://github.com/progit/progit.git ${ROOT}/vendor/progit || true > /dev/null cd benches && \ - hyperfine --warmup 10 --min-runs ${MIN_RUNS} -L binary comrak-${COMMIT},comrak-main,pulldown-cmark,cmark-gfm './bench.sh ./{binary}' --export-markdown ${ROOT}/bench-output.md &&\ + hyperfine --warmup 10 --min-runs ${MIN_RUNS} -L binary comrak-${COMMIT},comrak-main,pulldown-cmark,cmark-gfm,markdown-it './bench.sh ./{binary}' --export-markdown ${ROOT}/bench-output.md &&\ echo "\n\nRun on" `date -u` >> ${ROOT}/bench-output.md diff --git a/vendor/markdown-it b/vendor/markdown-it new file mode 160000 index 00000000..c2919dd0 --- /dev/null +++ b/vendor/markdown-it @@ -0,0 +1 @@ +Subproject commit c2919dd0b123f3aeb9264a6a6ec8a0d01cfbe19f From 6f9297b3547bb34bdd6a484a9d7b89cd2dbdb011 Mon Sep 17 00:00:00 2001 From: Asherah Connor Date: Mon, 10 Jul 2023 11:31:46 +1000 Subject: [PATCH 5/6] Apply suggestions from code review Co-authored-by: digitalMoksha --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cc5609a8..efca3d31 100644 --- a/README.md +++ b/README.md @@ -201,14 +201,14 @@ assert_eq!( For running benchmarks, you will need to [install hyperfine](https://github.com/sharkdp/hyperfine#installation) and optionally cmake. -If you want to just run benchmark for comrak, with current state of repo, you can simply run +If you want to just run the benchmark for `comrak`, with the current state of the repo, you can simply run ```bash make bench-comrak ``` This will build comrak in release mode, and run benchmark on it. You will see the time measurements as reported by hyperfine in the console. -Makefile also provides a way to run benchmarks for comark current state (with your changes), comrak main branch, cmark-gfm and pulldown-cmark. For this you will need to install cmake. After that make sure that you have set-up the git submodules. In case you have not installed submodules when cloning, you can do it by running +Makefile also provides a way to run benchmarks for `comrak` current state (with your changes), `comrak` main branch, [`cmark-gfm`](https://github.com/github/cmark-gfm), [`pulldown-cmark`](https://github.com/raphlinus/pulldown-cmark) and [`markdown-it.rs`](https://github.com/rlidwka/markdown-it.rs). For this you will need to install `cmake`. After that make sure that you have set-up the git submodules. In case you have not installed submodules when cloning, you can do it by running ```bash git submodule update --init ``` @@ -220,7 +220,7 @@ make bench-all which will run benchmarks across all, and report the time take by each as well as relative time. -Apart from this, CI is also setup for running benchmarks when a pull request is first opened. It will comment the results on the pull request in a tabular format comparing the 4 versions. After that you can manually trigger this CI by commenting `/run-bench` on the PR, this will update the existing comment with new results. Note Benchmarks won't be automatically run on each push. +Apart from this, CI is also setup for running benchmarks when a pull request is first opened. It will add a comment with the results on the pull request in a tabular format comparing the 5 versions. After that you can manually trigger this CI by commenting `/run-bench` on the PR, this will update the existing comment with new results. Note benchmarks won't be automatically run on each push. From e72e9a766245ab9a29069864b415ecbf7eef9a9f Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Mon, 10 Jul 2023 14:45:45 +0530 Subject: [PATCH 6/6] update proc-macro2 crate version --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d54c431f..b65fa393 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -630,9 +630,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" dependencies = [ "unicode-ident", ]