From 7ad6294ffb255146d6e48d4cd866756e4764ccc6 Mon Sep 17 00:00:00 2001 From: Aaron O'Mullan Date: Sun, 1 Sep 2024 13:21:39 +0900 Subject: [PATCH 1/7] perf: cibench workflow --- .github/workflows/cibench.yml | 108 ++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 .github/workflows/cibench.yml diff --git a/.github/workflows/cibench.yml b/.github/workflows/cibench.yml new file mode 100644 index 0000000..93a09e4 --- /dev/null +++ b/.github/workflows/cibench.yml @@ -0,0 +1,108 @@ +name: CI Benchmarks +on: + pull_request: + branches: [master] + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: "-C target-cpu=native" + +jobs: + benchmark: + name: Run benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + - name: Install critcmp + run: cargo install critcmp + + - name: Run benchmarks (master) + run: | + git checkout master + cargo bench --bench parse -- --save-baseline master + + - name: Run benchmarks (PR) + run: | + git checkout ${{ github.event.pull_request.head.sha }} + cargo bench --bench parse -- --save-baseline pr + + - name: Compare benchmarks + run: | + critcmp master pr-${{ github.event.pull_request.number }} + + benchmark-simd: + name: Run SIMD benchmarks + runs-on: ubuntu-latest + strategy: + matrix: + feature: [swar, sse42, avx2] + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + - name: Install critcmp + run: cargo install critcmp + + - name: Run benchmarks (master) + run: | + git checkout master + cargo bench --bench parse --features ${{ matrix.feature }} -- --save-baseline master-${{ matrix.feature }} + + - name: Run benchmarks (PR) + run: | + git checkout ${{ github.event.pull_request.head.sha }} + cargo bench --bench parse --features ${{ matrix.feature }} -- --save-baseline pr-${{ matrix.feature }} + + - name: Compare benchmarks + run: | + critcmp master-${{ matrix.feature }} pr-${{ matrix.feature }}-${{ github.event.pull_request.number }} + + benchmark-aarch64: + name: Run aarch64/neon benchmarks + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + - name: Install critcmp + run: cargo install critcmp + + - name: Run benchmarks (master) + run: | + git checkout master + cargo bench --bench parse --features neon -- --save-baseline master-aarch64 + + - name: Run benchmarks (PR) + run: | + git checkout ${{ github.event.pull_request.head.sha }} + cargo bench --bench parse --features neon -- --save-baseline pr-aarch64 + + - name: Compare benchmarks + run: | + critcmp master-aarch64 pr-aarch64-${{ github.event.pull_request.number }} From b41a0aa885badb811a40cc8dccaa4ac19651a1a8 Mon Sep 17 00:00:00 2001 From: Aaron O'Mullan Date: Sun, 1 Sep 2024 13:34:56 +0900 Subject: [PATCH 2/7] fixes --- .github/workflows/cibench.yml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cibench.yml b/.github/workflows/cibench.yml index 93a09e4..d9cc4d9 100644 --- a/.github/workflows/cibench.yml +++ b/.github/workflows/cibench.yml @@ -34,14 +34,14 @@ jobs: - name: Run benchmarks (PR) run: | git checkout ${{ github.event.pull_request.head.sha }} - cargo bench --bench parse -- --save-baseline pr + cargo bench --bench parse -- --save-baseline pr-${{ github.event.pull_request.number }} - name: Compare benchmarks run: | critcmp master pr-${{ github.event.pull_request.number }} - benchmark-simd: - name: Run SIMD benchmarks + benchmark-x64: + name: Run x64 benchmarks runs-on: ubuntu-latest strategy: matrix: @@ -64,20 +64,23 @@ jobs: - name: Run benchmarks (master) run: | git checkout master - cargo bench --bench parse --features ${{ matrix.feature }} -- --save-baseline master-${{ matrix.feature }} + CARGO_CFG_HTTPARSE_DISABLE_SIMD=1 cargo bench --bench parse ${{ matrix.feature != 'swar' && format('--features httparse_simd,httparse_simd_target_feature_{0}', matrix.feature) || '' }} -- --save-baseline master-${{ matrix.feature }} - name: Run benchmarks (PR) run: | git checkout ${{ github.event.pull_request.head.sha }} - cargo bench --bench parse --features ${{ matrix.feature }} -- --save-baseline pr-${{ matrix.feature }} + CARGO_CFG_HTTPARSE_DISABLE_SIMD=1 cargo bench --bench parse ${{ matrix.feature != 'swar' && format('--features httparse_simd,httparse_simd_target_feature_{0}', matrix.feature) || '' }} -- --save-baseline pr-${{ matrix.feature }}-${{ github.event.pull_request.number }} - name: Compare benchmarks run: | critcmp master-${{ matrix.feature }} pr-${{ matrix.feature }}-${{ github.event.pull_request.number }} benchmark-aarch64: - name: Run aarch64/neon benchmarks + name: Run aarch64 benchmarks runs-on: macos-latest + strategy: + matrix: + feature: [swar, neon] steps: - uses: actions/checkout@v3 with: @@ -96,13 +99,13 @@ jobs: - name: Run benchmarks (master) run: | git checkout master - cargo bench --bench parse --features neon -- --save-baseline master-aarch64 + CARGO_CFG_HTTPARSE_DISABLE_SIMD=1 cargo bench --bench parse ${{ matrix.feature == 'neon' && '--features httparse_simd,httparse_simd_neon_intrinsics' || '' }} -- --save-baseline master-aarch64-${{ matrix.feature }} - name: Run benchmarks (PR) run: | git checkout ${{ github.event.pull_request.head.sha }} - cargo bench --bench parse --features neon -- --save-baseline pr-aarch64 + CARGO_CFG_HTTPARSE_DISABLE_SIMD=1 cargo bench --bench parse ${{ matrix.feature == 'neon' && '--features httparse_simd,httparse_simd_neon_intrinsics' || '' }} -- --save-baseline pr-aarch64-${{ matrix.feature }}-${{ github.event.pull_request.number }} - name: Compare benchmarks run: | - critcmp master-aarch64 pr-aarch64-${{ github.event.pull_request.number }} + critcmp master-aarch64-${{ matrix.feature }} pr-aarch64-${{ matrix.feature }}-${{ github.event.pull_request.number }} From 42844942da1c28d47621728559b2bbc7c0d6ef7e Mon Sep 17 00:00:00 2001 From: Aaron O'Mullan Date: Sun, 1 Sep 2024 13:40:29 +0900 Subject: [PATCH 3/7] features build.rs interplay --- .github/workflows/cibench.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cibench.yml b/.github/workflows/cibench.yml index d9cc4d9..284eb77 100644 --- a/.github/workflows/cibench.yml +++ b/.github/workflows/cibench.yml @@ -64,12 +64,18 @@ jobs: - name: Run benchmarks (master) run: | git checkout master - CARGO_CFG_HTTPARSE_DISABLE_SIMD=1 cargo bench --bench parse ${{ matrix.feature != 'swar' && format('--features httparse_simd,httparse_simd_target_feature_{0}', matrix.feature) || '' }} -- --save-baseline master-${{ matrix.feature }} + cargo bench --bench parse ${{ matrix.feature != 'swar' && format('--features httparse_simd') || '' }} -- --save-baseline master-${{ matrix.feature }} + env: + CARGO_CFG_HTTPARSE_DISABLE_SIMD: 1 + RUSTFLAGS: ${{ matrix.feature != 'swar' && format('-C target-feature=+{0}', matrix.feature) || '' }} - name: Run benchmarks (PR) run: | git checkout ${{ github.event.pull_request.head.sha }} - CARGO_CFG_HTTPARSE_DISABLE_SIMD=1 cargo bench --bench parse ${{ matrix.feature != 'swar' && format('--features httparse_simd,httparse_simd_target_feature_{0}', matrix.feature) || '' }} -- --save-baseline pr-${{ matrix.feature }}-${{ github.event.pull_request.number }} + cargo bench --bench parse ${{ matrix.feature != 'swar' && format('--features httparse_simd') || '' }} -- --save-baseline pr-${{ matrix.feature }}-${{ github.event.pull_request.number }} + env: + CARGO_CFG_HTTPARSE_DISABLE_SIMD: 1 + RUSTFLAGS: ${{ matrix.feature != 'swar' && format('-C target-feature=+{0}', matrix.feature) || '' }} - name: Compare benchmarks run: | @@ -99,12 +105,18 @@ jobs: - name: Run benchmarks (master) run: | git checkout master - CARGO_CFG_HTTPARSE_DISABLE_SIMD=1 cargo bench --bench parse ${{ matrix.feature == 'neon' && '--features httparse_simd,httparse_simd_neon_intrinsics' || '' }} -- --save-baseline master-aarch64-${{ matrix.feature }} + cargo bench --bench parse ${{ matrix.feature == 'neon' && '--features httparse_simd' || '' }} -- --save-baseline master-aarch64-${{ matrix.feature }} + env: + CARGO_CFG_HTTPARSE_DISABLE_SIMD: 1 + RUSTFLAGS: ${{ matrix.feature == 'neon' && '-C target-feature=+neon' || '' }} - name: Run benchmarks (PR) run: | git checkout ${{ github.event.pull_request.head.sha }} - CARGO_CFG_HTTPARSE_DISABLE_SIMD=1 cargo bench --bench parse ${{ matrix.feature == 'neon' && '--features httparse_simd,httparse_simd_neon_intrinsics' || '' }} -- --save-baseline pr-aarch64-${{ matrix.feature }}-${{ github.event.pull_request.number }} + cargo bench --bench parse ${{ matrix.feature == 'neon' && '--features httparse_simd' || '' }} -- --save-baseline pr-aarch64-${{ matrix.feature }}-${{ github.event.pull_request.number }} + env: + CARGO_CFG_HTTPARSE_DISABLE_SIMD: 1 + RUSTFLAGS: ${{ matrix.feature == 'neon' && '-C target-feature=+neon' || '' }} - name: Compare benchmarks run: | From 3567c51b2910b28f6754b56550af515d38113dc2 Mon Sep 17 00:00:00 2001 From: Aaron O'Mullan Date: Sun, 1 Sep 2024 13:44:13 +0900 Subject: [PATCH 4/7] fix --- .github/workflows/cibench.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cibench.yml b/.github/workflows/cibench.yml index 284eb77..d52f0fd 100644 --- a/.github/workflows/cibench.yml +++ b/.github/workflows/cibench.yml @@ -64,17 +64,17 @@ jobs: - name: Run benchmarks (master) run: | git checkout master - cargo bench --bench parse ${{ matrix.feature != 'swar' && format('--features httparse_simd') || '' }} -- --save-baseline master-${{ matrix.feature }} + cargo bench --bench parse -- --save-baseline master-${{ matrix.feature }} env: - CARGO_CFG_HTTPARSE_DISABLE_SIMD: 1 + CARGO_CFG_HTTPARSE_DISABLE_SIMD: ${{ matrix.feature == 'swar' && '1' || '0' }} RUSTFLAGS: ${{ matrix.feature != 'swar' && format('-C target-feature=+{0}', matrix.feature) || '' }} - name: Run benchmarks (PR) run: | git checkout ${{ github.event.pull_request.head.sha }} - cargo bench --bench parse ${{ matrix.feature != 'swar' && format('--features httparse_simd') || '' }} -- --save-baseline pr-${{ matrix.feature }}-${{ github.event.pull_request.number }} + cargo bench --bench parse -- --save-baseline pr-${{ matrix.feature }}-${{ github.event.pull_request.number }} env: - CARGO_CFG_HTTPARSE_DISABLE_SIMD: 1 + CARGO_CFG_HTTPARSE_DISABLE_SIMD: ${{ matrix.feature == 'swar' && '1' || '0' }} RUSTFLAGS: ${{ matrix.feature != 'swar' && format('-C target-feature=+{0}', matrix.feature) || '' }} - name: Compare benchmarks @@ -105,17 +105,17 @@ jobs: - name: Run benchmarks (master) run: | git checkout master - cargo bench --bench parse ${{ matrix.feature == 'neon' && '--features httparse_simd' || '' }} -- --save-baseline master-aarch64-${{ matrix.feature }} + cargo bench --bench parse -- --save-baseline master-aarch64-${{ matrix.feature }} env: - CARGO_CFG_HTTPARSE_DISABLE_SIMD: 1 + CARGO_CFG_HTTPARSE_DISABLE_SIMD: ${{ matrix.feature == 'swar' && '1' || '0' }} RUSTFLAGS: ${{ matrix.feature == 'neon' && '-C target-feature=+neon' || '' }} - name: Run benchmarks (PR) run: | git checkout ${{ github.event.pull_request.head.sha }} - cargo bench --bench parse ${{ matrix.feature == 'neon' && '--features httparse_simd' || '' }} -- --save-baseline pr-aarch64-${{ matrix.feature }}-${{ github.event.pull_request.number }} + cargo bench --bench parse -- --save-baseline pr-aarch64-${{ matrix.feature }}-${{ github.event.pull_request.number }} env: - CARGO_CFG_HTTPARSE_DISABLE_SIMD: 1 + CARGO_CFG_HTTPARSE_DISABLE_SIMD: ${{ matrix.feature == 'swar' && '1' || '0' }} RUSTFLAGS: ${{ matrix.feature == 'neon' && '-C target-feature=+neon' || '' }} - name: Compare benchmarks From 9c9462825b724fd3031fb2eb6f161419e5e9e91d Mon Sep 17 00:00:00 2001 From: Aaron O'Mullan Date: Sun, 1 Sep 2024 13:55:00 +0900 Subject: [PATCH 5/7] critcmp -t 5 (filter <5% deltas) --- .github/workflows/cibench.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cibench.yml b/.github/workflows/cibench.yml index d52f0fd..72f271e 100644 --- a/.github/workflows/cibench.yml +++ b/.github/workflows/cibench.yml @@ -38,7 +38,7 @@ jobs: - name: Compare benchmarks run: | - critcmp master pr-${{ github.event.pull_request.number }} + critcmp -t 5 master pr-${{ github.event.pull_request.number }} benchmark-x64: name: Run x64 benchmarks @@ -79,7 +79,7 @@ jobs: - name: Compare benchmarks run: | - critcmp master-${{ matrix.feature }} pr-${{ matrix.feature }}-${{ github.event.pull_request.number }} + critcmp -t 5 master-${{ matrix.feature }} pr-${{ matrix.feature }}-${{ github.event.pull_request.number }} benchmark-aarch64: name: Run aarch64 benchmarks @@ -120,4 +120,4 @@ jobs: - name: Compare benchmarks run: | - critcmp master-aarch64-${{ matrix.feature }} pr-aarch64-${{ matrix.feature }}-${{ github.event.pull_request.number }} + critcmp -t 5 master-aarch64-${{ matrix.feature }} pr-aarch64-${{ matrix.feature }}-${{ github.event.pull_request.number }} From ff3dd7b4a699ab785f39bfc3090071bd5870a776 Mon Sep 17 00:00:00 2001 From: Aaron O'Mullan Date: Sun, 1 Sep 2024 13:56:33 +0900 Subject: [PATCH 6/7] pr num before matrix feature --- .github/workflows/cibench.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cibench.yml b/.github/workflows/cibench.yml index 72f271e..d3b144c 100644 --- a/.github/workflows/cibench.yml +++ b/.github/workflows/cibench.yml @@ -72,14 +72,14 @@ jobs: - name: Run benchmarks (PR) run: | git checkout ${{ github.event.pull_request.head.sha }} - cargo bench --bench parse -- --save-baseline pr-${{ matrix.feature }}-${{ github.event.pull_request.number }} + cargo bench --bench parse -- --save-baseline pr-${{ github.event.pull_request.number }}-${{ matrix.feature }} env: CARGO_CFG_HTTPARSE_DISABLE_SIMD: ${{ matrix.feature == 'swar' && '1' || '0' }} RUSTFLAGS: ${{ matrix.feature != 'swar' && format('-C target-feature=+{0}', matrix.feature) || '' }} - name: Compare benchmarks run: | - critcmp -t 5 master-${{ matrix.feature }} pr-${{ matrix.feature }}-${{ github.event.pull_request.number }} + critcmp -t 5 master-${{ matrix.feature }} pr-${{ github.event.pull_request.number }}-${{ matrix.feature }} benchmark-aarch64: name: Run aarch64 benchmarks @@ -113,11 +113,11 @@ jobs: - name: Run benchmarks (PR) run: | git checkout ${{ github.event.pull_request.head.sha }} - cargo bench --bench parse -- --save-baseline pr-aarch64-${{ matrix.feature }}-${{ github.event.pull_request.number }} + cargo bench --bench parse -- --save-baseline pr-${{ github.event.pull_request.number }}-aarch64-${{ matrix.feature }} env: CARGO_CFG_HTTPARSE_DISABLE_SIMD: ${{ matrix.feature == 'swar' && '1' || '0' }} RUSTFLAGS: ${{ matrix.feature == 'neon' && '-C target-feature=+neon' || '' }} - name: Compare benchmarks run: | - critcmp -t 5 master-aarch64-${{ matrix.feature }} pr-aarch64-${{ matrix.feature }}-${{ github.event.pull_request.number }} + critcmp -t 5 master-aarch64-${{ matrix.feature }} pr-${{ github.event.pull_request.number }}-aarch64-${{ matrix.feature }} From 9d0c3573c2c4c57f8f0d3d9d2e01b850829dcb80 Mon Sep 17 00:00:00 2001 From: Aaron O'Mullan Date: Sun, 1 Sep 2024 13:57:38 +0900 Subject: [PATCH 7/7] short sha --- .github/workflows/cibench.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cibench.yml b/.github/workflows/cibench.yml index d3b144c..83133af 100644 --- a/.github/workflows/cibench.yml +++ b/.github/workflows/cibench.yml @@ -34,11 +34,11 @@ jobs: - name: Run benchmarks (PR) run: | git checkout ${{ github.event.pull_request.head.sha }} - cargo bench --bench parse -- --save-baseline pr-${{ github.event.pull_request.number }} + cargo bench --bench parse -- --save-baseline pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }} - name: Compare benchmarks run: | - critcmp -t 5 master pr-${{ github.event.pull_request.number }} + critcmp -t 5 master pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }} benchmark-x64: name: Run x64 benchmarks @@ -72,14 +72,14 @@ jobs: - name: Run benchmarks (PR) run: | git checkout ${{ github.event.pull_request.head.sha }} - cargo bench --bench parse -- --save-baseline pr-${{ github.event.pull_request.number }}-${{ matrix.feature }} + cargo bench --bench parse -- --save-baseline pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.feature }} env: CARGO_CFG_HTTPARSE_DISABLE_SIMD: ${{ matrix.feature == 'swar' && '1' || '0' }} RUSTFLAGS: ${{ matrix.feature != 'swar' && format('-C target-feature=+{0}', matrix.feature) || '' }} - name: Compare benchmarks run: | - critcmp -t 5 master-${{ matrix.feature }} pr-${{ github.event.pull_request.number }}-${{ matrix.feature }} + critcmp -t 5 master-${{ matrix.feature }} pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ matrix.feature }} benchmark-aarch64: name: Run aarch64 benchmarks @@ -113,11 +113,11 @@ jobs: - name: Run benchmarks (PR) run: | git checkout ${{ github.event.pull_request.head.sha }} - cargo bench --bench parse -- --save-baseline pr-${{ github.event.pull_request.number }}-aarch64-${{ matrix.feature }} + cargo bench --bench parse -- --save-baseline pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-aarch64-${{ matrix.feature }} env: CARGO_CFG_HTTPARSE_DISABLE_SIMD: ${{ matrix.feature == 'swar' && '1' || '0' }} RUSTFLAGS: ${{ matrix.feature == 'neon' && '-C target-feature=+neon' || '' }} - name: Compare benchmarks run: | - critcmp -t 5 master-aarch64-${{ matrix.feature }} pr-${{ github.event.pull_request.number }}-aarch64-${{ matrix.feature }} + critcmp -t 5 master-aarch64-${{ matrix.feature }} pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-aarch64-${{ matrix.feature }}