The original motivation for the matrix was to mitigate an issue some tests failing that used a vault docker container. When the tests ran in a single step, these tests would almost always fail. By splitting the tests per package, the failure rate of these tests was around 20%. GitHub was able to identify the cause. There was an issue with localhost resolution that causes requests to the vault containers to sometimes fail, resulting in the kinds of failures that were seen. Note that the same issue was not seen when accessing the test postgres database that was running in a container, since the test code connects via 127.0.0.1, while the vault container used localhost. This commit includes the recommended fix so that localhost name resolution works as expected on the GitHub hosted runners. It also removes the test matrix since it is no longer needed as a work around. In addition it seems advisable to remove the matrix given the recent discovery that some tests were not running with the matrix. While this was fixed, removing the matrix reduces the risk of any future mistakes from the test splitting. This also seems to result in a faster pipeline execution time. The overhead to determine the matrix values, wait for a runner to be available, and restore from the cache seems to add up to a longer overall execution time. Eliminating the matrix can also help with another type of failure seen. Sometimes restoring from the cache fails. By eliminating the test matrix, it will greatly reduce the number of cache restoring done by the workflow, which will reduce the chances of an issue restoring from cache. The only remaining benefit from the matrix, was that if there was a transient test failure, a re-run of only the failed jobs would be relatively quick (depending on which package had the failure), while without the matrix the entire test suite will need to be re-run. See: https://github.com/hashicorp/boundary/pull/3199 Refs:86214a85e1e04438b7f8
179 lines
6.6 KiB
YAML
179 lines
6.6 KiB
YAML
name: test
|
|
|
|
on:
|
|
- workflow_dispatch
|
|
- push
|
|
- workflow_call
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
env:
|
|
DOCKER_MIRROR: docker.mirror.hashicorp.services
|
|
|
|
jobs:
|
|
setup:
|
|
outputs:
|
|
go-version: ${{ steps.get-go-version.outputs.go-version }}
|
|
cache-go-build: ${{ steps.go-cache-paths.outputs.go-build }}
|
|
cache-go-mod: ${{ steps.go-cache-paths.outputs.go-mod }}
|
|
cache-go-bin: ${{ steps.go-cache-paths.outputs.go-bin }}
|
|
go-cache-key: ${{ steps.go-cache-key.outputs.key }}
|
|
plugin-cache-path: ${{ steps.plugin-cache-paths.outputs.path }}
|
|
plugin-cache-key: ${{ steps.plugin-cache-key.outputs.key }}
|
|
runs-on: ${{ fromJSON(vars.RUNNER) }}
|
|
steps:
|
|
- uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
|
|
with:
|
|
fetch-depth: '0'
|
|
- name: Determine Go version
|
|
id: get-go-version
|
|
# We use .go-version as our source of truth for current Go
|
|
# version, because "goenv" can react to it automatically.
|
|
run: |
|
|
echo "Building with Go $(cat .go-version)"
|
|
echo "go-version=$(cat .go-version)" >> "$GITHUB_OUTPUT"
|
|
- name: Set up Go
|
|
uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
|
|
with:
|
|
go-version: "${{ steps.get-go-version.outputs.go-version }}"
|
|
cache: false
|
|
- name: Determine go cache key
|
|
id: go-cache-key
|
|
run: |
|
|
echo "key=${{ runner.os }}-go-${{ hashFiles('**/go.sum', './Makefile', './tools/tools.go') }}" >> "$GITHUB_OUTPUT"
|
|
- name: Determine Go cache paths
|
|
id: go-cache-paths
|
|
run: |
|
|
echo "go-build=$(go env GOCACHE)" >> "$GITHUB_OUTPUT"
|
|
echo "go-mod=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
|
|
echo "go-bin=$(go env GOPATH)/bin" >> "$GITHUB_OUTPUT"
|
|
- name: Set up Go modules cache
|
|
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
|
|
with:
|
|
path: |
|
|
${{ steps.go-cache-paths.outputs.go-build }}
|
|
${{ steps.go-cache-paths.outputs.go-mod }}
|
|
${{ steps.go-cache-paths.outputs.go-bin }}
|
|
key: ${{ steps.go-cache-key.outputs.key }}
|
|
restore-keys: |
|
|
${{ runner.os }}-go
|
|
- name: Install Tools
|
|
run: |
|
|
go mod download
|
|
make tools
|
|
- name: Determine plugin cache key
|
|
id: plugin-cache-key
|
|
run: |
|
|
echo "key=${{ runner.os }}-plugins-${{ hashFiles('plugins/**/*.go', 'plugins/**/go.sum', './Makefile', './scripts/plugins.sh') }}" >> "$GITHUB_OUTPUT"
|
|
- name: Determin plugin cache path
|
|
id: plugin-cache-paths
|
|
run: |
|
|
echo "path=plugins/**/assets/*.gz" >> "$GITHUB_OUTPUT"
|
|
- name: Set up plugin cache
|
|
id: plugin-cache
|
|
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
|
|
with:
|
|
path: |
|
|
${{ steps.plugin-cache-paths.outputs.path }}
|
|
key: ${{ steps.plugin-cache-key.outputs.key }}
|
|
restore-keys: |
|
|
${{ runner.os }}-plugin
|
|
- name: Build Plugins
|
|
if: steps.plugin-cache.outputs.cache-hit != 'true'
|
|
run: |
|
|
make build-plugins
|
|
test-modules:
|
|
needs:
|
|
- setup
|
|
runs-on: ${{ fromJSON(vars.RUNNER) }}
|
|
strategy:
|
|
matrix:
|
|
module: ["api", "sdk"]
|
|
steps:
|
|
- uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
|
|
- name: Set up go
|
|
uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
|
|
with:
|
|
go-version: "${{ needs.setup.outputs.go-version }}"
|
|
cache: false
|
|
- name: Set up Go modules cache
|
|
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
|
|
with:
|
|
path: |
|
|
${{ needs.setup.outputs.cache-go-build }}
|
|
${{ needs.setup.outputs.cache-go-mod }}
|
|
${{ needs.setup.outputs.cache-go-bin }}
|
|
key: ${{ needs.setup.outputs.go-cache-key }}
|
|
restore-keys: |
|
|
${{ runner.os }}-go
|
|
fail-on-cache-miss: true
|
|
- name: Test ${{ matrix.module }} Module
|
|
run: |
|
|
make test-${{ matrix.module }}
|
|
|
|
test:
|
|
needs:
|
|
- setup
|
|
runs-on: ${{ fromJSON(vars.RUNNER_LARGE) }}
|
|
steps:
|
|
- name: ulimit
|
|
run: |
|
|
echo "Soft limits"
|
|
ulimit -Sa
|
|
echo "Hard limits"
|
|
ulimit -Ha
|
|
- uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
|
|
- name: Set up go
|
|
uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
|
|
with:
|
|
go-version: "${{ needs.setup.outputs.go-version }}"
|
|
cache: false
|
|
- name: Set up Go modules cache
|
|
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
|
|
with:
|
|
path: |
|
|
${{ needs.setup.outputs.cache-go-build }}
|
|
${{ needs.setup.outputs.cache-go-mod }}
|
|
${{ needs.setup.outputs.cache-go-bin }}
|
|
key: ${{ needs.setup.outputs.go-cache-key }}
|
|
restore-keys: |
|
|
${{ runner.os }}-go
|
|
fail-on-cache-miss: true
|
|
- name: Set up plugin cache
|
|
id: plugin-cache
|
|
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
|
|
with:
|
|
path: |
|
|
${{ needs.setup.outputs.plugin-cache-path }}
|
|
key: ${{ needs.setup.outputs.plugin-cache-key }}
|
|
restore-keys: |
|
|
${{ runner.os }}-plugin
|
|
- name: GH fix for localhost resolution
|
|
if: github.repository == 'hashicorp/boundary'
|
|
run: |
|
|
cat /etc/hosts && echo "-----------"
|
|
sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6 -localhost ip6-loopback/g' /etc/hosts
|
|
cat /etc/hosts
|
|
- name: Initialize Test Database
|
|
run: |
|
|
which pg_isready || sudo apt-get update && sudo apt-get install -y postgresql-client
|
|
make DOCKER_ARGS='-d' PG_OPTS='-c shared_buffers=256MB -c max_connections=200000' -C testing/dbtest/docker database-up
|
|
until pg_isready -h 127.0.0.1; do docker container inspect boundary-sql-tests &> /dev/null || exit 255; sleep 1; done
|
|
|
|
- name: Test
|
|
uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd # TSCCR: no entry for repository "nick-fields/retry"
|
|
env:
|
|
TEST_PACKAGE: "./..."
|
|
TESTARGS: -v
|
|
TEST_TIMEOUT: 120m
|
|
with:
|
|
max_attempts: 3
|
|
timeout_minutes: 120
|
|
retry_on: error
|
|
command: make test
|
|
- name: Cleanup
|
|
if: always()
|
|
run: |
|
|
make -C testing/dbtest/docker clean
|