From 284446a0a8eca153a4fe5400584411ff7641db3a Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Tue, 28 Jul 2020 16:18:49 -0700 Subject: [PATCH 01/12] Add benchmark Github action --- .github/workflows/benchmark.yaml | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/benchmark.yaml diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 000000000..6b1548e35 --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,35 @@ +name: Benchmarks: run and upload results + +on: + push: + branches: + - test-benchmark-branch + pull_request: + branches: + - test-benchmark-branch + +jobs: + run_benchmark: + name: Run Benchmark + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: 1.45.0 + override: true + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install Python dependencies + run: pip install codespeed-client + - run: | + make bench + git clone https://github.com/wasmerio/wasmer-bench + cd wasmer-bench + + python3 send_metrics.py + From cf88df9f4f1b7f40de3b3d993e40233031a7c3ed Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Tue, 28 Jul 2020 16:21:14 -0700 Subject: [PATCH 02/12] Fix typo --- .github/workflows/benchmark.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 6b1548e35..bd6c558ec 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -1,4 +1,4 @@ -name: Benchmarks: run and upload results +name: Run Benchmarks and upload results on: push: From 6772df6497b1c9f78ddecb38e6b7f8e48bc2c4d3 Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Tue, 28 Jul 2020 16:37:40 -0700 Subject: [PATCH 03/12] Install Python toml package too --- .github/workflows/benchmark.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index bd6c558ec..db8c665b6 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -25,7 +25,9 @@ jobs: with: python-version: 3.8 - name: Install Python dependencies - run: pip install codespeed-client + run: | + pip install codespeed-client + pip install toml - run: | make bench git clone https://github.com/wasmerio/wasmer-bench From 76a887e5edfb61274af6ca7dab1fd8502d2c5ca2 Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Wed, 29 Jul 2020 14:47:25 -0700 Subject: [PATCH 04/12] Update benchmark CI step --- .github/workflows/benchmark.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index db8c665b6..8175afe6c 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -31,7 +31,6 @@ jobs: - run: | make bench git clone https://github.com/wasmerio/wasmer-bench - cd wasmer-bench - python3 send_metrics.py + python3 wasmer-bench/send_metrics.py From 72b392666f9b365a2526ecf6c135d8b3ac0e5882 Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Fri, 31 Jul 2020 12:39:46 -0700 Subject: [PATCH 05/12] Add caching to benchmark Github Action --- .github/workflows/benchmark.yaml | 43 +++++++++++++++++++++++++++++--- .github/workflows/main.yaml | 2 +- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 8175afe6c..c99dca7e8 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -9,17 +9,51 @@ on: - test-benchmark-branch jobs: + run_benchmark: - name: Run Benchmark - runs-on: ubuntu-latest + name: Benchmark on ${{ matrix.build }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + build: [linux, macos, windows] + include: + - build: linux + os: ubuntu-latest + rust: 1.45.0 + env: + CARGO_SCCACHE_VERSION: 0.2.13 + SCCACHE_AZURE_BLOB_CONTAINER: wasmerstoragesccacheblob + SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} steps: - uses: actions/checkout@v2 - - name: Install Rust + - name: Install Rust ${{ matrix.rust }} uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: 1.45.0 + toolchain: ${{ matrix.rust }} override: true + - name: Configure cargo data directory + # After this point, all cargo registry and crate data is stored in + # $GITHUB_WORKSPACE/.cargo_home. This allows us to cache only the files + # that are needed during the build process. Additionally, this works + # around a bug in the 'cache' action that causes directories outside of + # the workspace dir to be saved/restored incorrectly. + run: echo "::set-env name=CARGO_HOME::$(pwd)/.cargo_home" + - name: Cache + uses: actions/cache@master + with: + # Note: crates from the git repo always get rebuilt + # so we cache only those subdirectories of target/{debug|release} that + # contain the build output for crates that come from the registry. + path: |- + .cargo_home + target/*/.* + target/*/build + target/*/deps + key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} + restore-keys: | + ${{ matrix.os }}- - name: Install Python uses: actions/setup-python@v2 with: @@ -28,6 +62,7 @@ jobs: run: | pip install codespeed-client pip install toml + - name: Run Benchmark - run: | make bench git clone https://github.com/wasmerio/wasmer-bench diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index db38c9143..be61aff18 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -35,7 +35,7 @@ jobs: # - run: make doc-local test: - name: Test in ${{ matrix.build }} + name: Test on ${{ matrix.build }} runs-on: ${{ matrix.os }} strategy: fail-fast: false From 538f6457a81d51a1023b12526aefb441f0fc547f Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Fri, 31 Jul 2020 12:43:06 -0700 Subject: [PATCH 06/12] Fix typo in benchmark yaml --- .github/workflows/benchmark.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index c99dca7e8..787d75fb4 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -9,7 +9,6 @@ on: - test-benchmark-branch jobs: - run_benchmark: name: Benchmark on ${{ matrix.build }} runs-on: ${{ matrix.os }} @@ -63,7 +62,7 @@ jobs: pip install codespeed-client pip install toml - name: Run Benchmark - - run: | + run: | make bench git clone https://github.com/wasmerio/wasmer-bench From 1f32a5842f080540473fb66bf0188bd16c77a3b7 Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Fri, 31 Jul 2020 12:44:10 -0700 Subject: [PATCH 07/12] Fix another typo in benchmark yaml --- .github/workflows/benchmark.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 787d75fb4..811835728 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - build: [linux, macos, windows] + build: [linux] include: - build: linux os: ubuntu-latest From 35ecf3d4dea5972f67d1dc2a093b88373744aa02 Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Fri, 31 Jul 2020 15:42:20 -0700 Subject: [PATCH 08/12] Temporarily disable caching on benchmark to debug upload issue --- .github/workflows/benchmark.yaml | 66 ++++++++++++++++---------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 811835728..dce8a1f86 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -20,39 +20,39 @@ jobs: - build: linux os: ubuntu-latest rust: 1.45.0 - env: - CARGO_SCCACHE_VERSION: 0.2.13 - SCCACHE_AZURE_BLOB_CONTAINER: wasmerstoragesccacheblob - SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} - steps: - - uses: actions/checkout@v2 - - name: Install Rust ${{ matrix.rust }} - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: ${{ matrix.rust }} - override: true - - name: Configure cargo data directory - # After this point, all cargo registry and crate data is stored in - # $GITHUB_WORKSPACE/.cargo_home. This allows us to cache only the files - # that are needed during the build process. Additionally, this works - # around a bug in the 'cache' action that causes directories outside of - # the workspace dir to be saved/restored incorrectly. - run: echo "::set-env name=CARGO_HOME::$(pwd)/.cargo_home" - - name: Cache - uses: actions/cache@master - with: - # Note: crates from the git repo always get rebuilt - # so we cache only those subdirectories of target/{debug|release} that - # contain the build output for crates that come from the registry. - path: |- - .cargo_home - target/*/.* - target/*/build - target/*/deps - key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} - restore-keys: | - ${{ matrix.os }}- +# env: +# CARGO_SCCACHE_VERSION: 0.2.13 +# SCCACHE_AZURE_BLOB_CONTAINER: wasmerstoragesccacheblob +# SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} +# steps: +# - uses: actions/checkout@v2 +# - name: Install Rust ${{ matrix.rust }} +# uses: actions-rs/toolchain@v1 +# with: +# profile: minimal +# toolchain: ${{ matrix.rust }} +# override: true +# - name: Configure cargo data directory +# # After this point, all cargo registry and crate data is stored in +# # $GITHUB_WORKSPACE/.cargo_home. This allows us to cache only the files +# # that are needed during the build process. Additionally, this works +# # around a bug in the 'cache' action that causes directories outside of +# # the workspace dir to be saved/restored incorrectly. +# run: echo "::set-env name=CARGO_HOME::$(pwd)/.cargo_home" +# - name: Cache +# uses: actions/cache@master +# with: +# # Note: crates from the git repo always get rebuilt +# # so we cache only those subdirectories of target/{debug|release} that +# # contain the build output for crates that come from the registry. +# path: |- +# .cargo_home +# target/*/.* +# target/*/build +# target/*/deps +# key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} +# restore-keys: | +# ${{ matrix.os }}- - name: Install Python uses: actions/setup-python@v2 with: From ba4212c4d55b9f725728c2af9e9a7ccd646366b0 Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Fri, 31 Jul 2020 15:43:28 -0700 Subject: [PATCH 09/12] Uncomment install rust steps in benchmark yaml --- .github/workflows/benchmark.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index dce8a1f86..d36818631 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -24,14 +24,14 @@ jobs: # CARGO_SCCACHE_VERSION: 0.2.13 # SCCACHE_AZURE_BLOB_CONTAINER: wasmerstoragesccacheblob # SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} -# steps: -# - uses: actions/checkout@v2 -# - name: Install Rust ${{ matrix.rust }} -# uses: actions-rs/toolchain@v1 -# with: -# profile: minimal -# toolchain: ${{ matrix.rust }} -# override: true + steps: + - uses: actions/checkout@v2 + - name: Install Rust ${{ matrix.rust }} + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + override: true # - name: Configure cargo data directory # # After this point, all cargo registry and crate data is stored in # # $GITHUB_WORKSPACE/.cargo_home. This allows us to cache only the files From e2cf9ca001604de56461b682548b9e7741a0af2d Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Fri, 31 Jul 2020 16:04:49 -0700 Subject: [PATCH 10/12] Reenable caching on the benchmark CI step --- .github/workflows/benchmark.yaml | 50 ++++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index d36818631..811835728 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -20,10 +20,10 @@ jobs: - build: linux os: ubuntu-latest rust: 1.45.0 -# env: -# CARGO_SCCACHE_VERSION: 0.2.13 -# SCCACHE_AZURE_BLOB_CONTAINER: wasmerstoragesccacheblob -# SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} + env: + CARGO_SCCACHE_VERSION: 0.2.13 + SCCACHE_AZURE_BLOB_CONTAINER: wasmerstoragesccacheblob + SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} steps: - uses: actions/checkout@v2 - name: Install Rust ${{ matrix.rust }} @@ -32,27 +32,27 @@ jobs: profile: minimal toolchain: ${{ matrix.rust }} override: true -# - name: Configure cargo data directory -# # After this point, all cargo registry and crate data is stored in -# # $GITHUB_WORKSPACE/.cargo_home. This allows us to cache only the files -# # that are needed during the build process. Additionally, this works -# # around a bug in the 'cache' action that causes directories outside of -# # the workspace dir to be saved/restored incorrectly. -# run: echo "::set-env name=CARGO_HOME::$(pwd)/.cargo_home" -# - name: Cache -# uses: actions/cache@master -# with: -# # Note: crates from the git repo always get rebuilt -# # so we cache only those subdirectories of target/{debug|release} that -# # contain the build output for crates that come from the registry. -# path: |- -# .cargo_home -# target/*/.* -# target/*/build -# target/*/deps -# key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} -# restore-keys: | -# ${{ matrix.os }}- + - name: Configure cargo data directory + # After this point, all cargo registry and crate data is stored in + # $GITHUB_WORKSPACE/.cargo_home. This allows us to cache only the files + # that are needed during the build process. Additionally, this works + # around a bug in the 'cache' action that causes directories outside of + # the workspace dir to be saved/restored incorrectly. + run: echo "::set-env name=CARGO_HOME::$(pwd)/.cargo_home" + - name: Cache + uses: actions/cache@master + with: + # Note: crates from the git repo always get rebuilt + # so we cache only those subdirectories of target/{debug|release} that + # contain the build output for crates that come from the registry. + path: |- + .cargo_home + target/*/.* + target/*/build + target/*/deps + key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} + restore-keys: | + ${{ matrix.os }}- - name: Install Python uses: actions/setup-python@v2 with: From bab7093a56f31ff5083cf0f2ec33f1e3baf4067e Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Mon, 3 Aug 2020 16:06:38 -0700 Subject: [PATCH 11/12] Set up benchmarking to trigger on master --- .github/workflows/benchmark.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 811835728..fb7be4501 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -3,10 +3,10 @@ name: Run Benchmarks and upload results on: push: branches: - - test-benchmark-branch + - master pull_request: branches: - - test-benchmark-branch + - master jobs: run_benchmark: From 081be0a300a0b037c1aaf3dc7c4d7b69fef073d2 Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Mon, 3 Aug 2020 16:08:22 -0700 Subject: [PATCH 12/12] Only trigger benchmark on merge to master --- .github/workflows/benchmark.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index fb7be4501..6de296521 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -4,9 +4,6 @@ on: push: branches: - master - pull_request: - branches: - - master jobs: run_benchmark: