diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000..6797a977 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,36 @@ +name: benchmarks + +on: + pull_request: + types: + - opened + - reopened + issue_comment: + types: + - created + +jobs: + run_benchmarks: + runs-on: ubuntu-latest + permissions: + pull-requests: write + # run either when pull request is opened or when comment body (only on pr) is /run-bench + if: (github.event_name == 'pull_request') || ((github.event.issue.pull_request != null) && github.event.comment.body == '/run-bench') + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: Install hyperfine + run: cargo install hyperfine + - name: Install cmake + run: sudo apt-get update && sudo apt-get install cmake -y + - name: Build Binaries + run: make binaries + - name: Run Benchmarks + run: make bench-all + - name: Post result comment + uses: mshick/add-pr-comment@v2 + with: + message-path: bench-output.md \ No newline at end of file diff --git a/.gitignore b/.gitignore index cda7c229..9c22735a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,9 @@ target comrak-* .vscode .idea +vendor/comrak +vendor/progit +benches/cmark-gfm +benches/comrak-* +benches/pulldown-cmark +benches/markdown-it diff --git a/.gitmodules b/.gitmodules index 3f4aa421..c6383db7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,9 @@ [submodule "vendor/cmark-gfm"] path = vendor/cmark-gfm url = https://github.com/kivikakk/cmark-gfm.git +[submodule "vendor/pulldown-cmark"] + path = vendor/pulldown-cmark + url = https://github.com/raphlinus/pulldown-cmark.git +[submodule "vendor/markdown-it"] + path = vendor/markdown-it + url = https://github.com/rlidwka/markdown-it.rs.git diff --git a/Cargo.lock b/Cargo.lock index d54c431f..b65fa393 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -630,9 +630,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" dependencies = [ "unicode-ident", ] diff --git a/Makefile b/Makefile index 9a4e2527..0dfb5a25 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,7 @@ +ROOT:=$(shell git rev-parse --show-toplevel) +COMMIT:=$(shell git rev-parse --short HEAD) +MIN_RUNS:=25 + src/scanners.rs: src/scanners.re re2rust -W -Werror -i --no-generation-date -o $@ $< cargo fmt @@ -5,3 +9,42 @@ src/scanners.rs: src/scanners.re bench: cargo build --release (cd vendor/cmark-gfm/; make bench PROG=../../target/release/comrak) + +binaries: build-comrak-branch build-comrak-master build-cmark-gfm build-pulldown-cmark build-markdown-it + +build-comrak-branch: + cargo build --release + cp ${ROOT}/target/release/comrak ${ROOT}/benches/comrak-${COMMIT} + +build-comrak-master: + git clone https://github.com/kivikakk/comrak.git --depth 1 --single-branch ${ROOT}/vendor/comrak || true + cd ${ROOT}/vendor/comrak && \ + cargo build --release && \ + cp ./target/release/comrak ${ROOT}/benches/comrak-main + +build-cmark-gfm: + cd ${ROOT}/vendor/cmark-gfm && \ + make && \ + cp build/src/cmark-gfm ${ROOT}/benches/cmark-gfm + +build-markdown-it: + cd ${ROOT}/vendor/markdown-it && \ + cargo build --release && \ + cp target/release/markdown-it ${ROOT}/benches/markdown-it + +build-pulldown-cmark: + cd ${ROOT}/vendor/pulldown-cmark && \ + cargo build --release && \ + cp target/release/pulldown-cmark ${ROOT}/benches/pulldown-cmark + +bench-comrak: build-comrak-branch + git clone https://github.com/progit/progit.git ${ROOT}/vendor/progit || true > /dev/null + cd benches && \ + hyperfine --warmup 3 --min-runs ${MIN_RUNS} -L binary comrak-${COMMIT} './bench.sh ./{binary}' + +bench-all: binaries + git clone https://github.com/progit/progit.git ${ROOT}/vendor/progit || true > /dev/null + cd benches && \ + hyperfine --warmup 10 --min-runs ${MIN_RUNS} -L binary comrak-${COMMIT},comrak-main,pulldown-cmark,cmark-gfm,markdown-it './bench.sh ./{binary}' --export-markdown ${ROOT}/bench-output.md &&\ + echo "\n\nRun on" `date -u` >> ${ROOT}/bench-output.md + diff --git a/README.md b/README.md index ea65e234..efca3d31 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,33 @@ assert_eq!( \n"); ``` +## Benchmarking + +For running benchmarks, you will need to [install hyperfine](https://github.com/sharkdp/hyperfine#installation) and optionally cmake. + +If you want to just run the benchmark for `comrak`, with the current state of the repo, you can simply run +```bash +make bench-comrak +``` + +This will build comrak in release mode, and run benchmark on it. You will see the time measurements as reported by hyperfine in the console. + +Makefile also provides a way to run benchmarks for `comrak` current state (with your changes), `comrak` main branch, [`cmark-gfm`](https://github.com/github/cmark-gfm), [`pulldown-cmark`](https://github.com/raphlinus/pulldown-cmark) and [`markdown-it.rs`](https://github.com/rlidwka/markdown-it.rs). For this you will need to install `cmake`. After that make sure that you have set-up the git submodules. In case you have not installed submodules when cloning, you can do it by running +```bash +git submodule update --init +``` + +After this is done, you can run +```bash +make bench-all +``` + +which will run benchmarks across all, and report the time take by each as well as relative time. + +Apart from this, CI is also setup for running benchmarks when a pull request is first opened. It will add a comment with the results on the pull request in a tabular format comparing the 5 versions. After that you can manually trigger this CI by commenting `/run-bench` on the PR, this will update the existing comment with new results. Note benchmarks won't be automatically run on each push. + + + ## Security As with [`cmark`](https://github.com/commonmark/cmark) and [`cmark-gfm`](https://github.com/github/cmark-gfm#security), diff --git a/benches/bench.sh b/benches/bench.sh new file mode 100755 index 00000000..cd0abec0 --- /dev/null +++ b/benches/bench.sh @@ -0,0 +1,8 @@ +#! /bin/bash + +PROG=$1 +ROOTDIR=$(git rev-parse --show-toplevel) + +for lang in ar az be ca cs de en eo es es-ni fa fi fr hi hu id it ja ko mk nl no-nb pl pt-br ro ru sr th tr uk vi zh zh-tw; do \ + cat $ROOTDIR/vendor/progit/$lang/*/*.markdown | $PROG > /dev/null +done \ No newline at end of file diff --git a/vendor/markdown-it b/vendor/markdown-it new file mode 160000 index 00000000..c2919dd0 --- /dev/null +++ b/vendor/markdown-it @@ -0,0 +1 @@ +Subproject commit c2919dd0b123f3aeb9264a6a6ec8a0d01cfbe19f diff --git a/vendor/pulldown-cmark b/vendor/pulldown-cmark new file mode 160000 index 00000000..34c2bb51 --- /dev/null +++ b/vendor/pulldown-cmark @@ -0,0 +1 @@ +Subproject commit 34c2bb51a48a27c92d0ab48327c73276d6978ae6