diff --git a/.github/scripts/schema-diff-hide-gh-comments.sh b/.github/scripts/schema-diff-hide-gh-comments.sh new file mode 100755 index 000000000..54963406a --- /dev/null +++ b/.github/scripts/schema-diff-hide-gh-comments.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# This script finds all issue comments on a PR that match a specific prefix and +# have been posted by a specific user and minimizes them. + +die() { + # We don't ever return a non-zero status code because GitHub actions exits out + # of the workflow if any of the commands exit with a non-zero status code. In + # the case of this script, which is tailored to run in GitHub Actions, the + # next operation (post a new comment with the new db schema diff) should still + # run even when this script fails. + echo "$@" + exit 0 +} + +which jq &> /dev/null || die "jq must be installed" +which curl &> /dev/null || die "curl must be installed" + +gh_api_url=$1 +gh_gql_url=$2 +gh_token=$3 +gh_repo=$4 +gh_pr_number=$5 +gh_comment_prefix=$6 +gh_user_login=$7 + +# List all comments for the Pull Request we're working on. +echo "Listing all issue comments for PR #$gh_pr_number" +curl -fsSX GET \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $gh_token" \ + --output response.json \ + $gh_api_url/repos/$gh_repo/issues/$gh_pr_number/comments \ +|| die "Failed to list all issue comments for PR #$gh_pr_number" + +[[ $(jq 'length' response.json) -gt 0 ]] || die "No comments found for PR #$gh_pr_number, nothing to hide" + +# Use jq to find all comments we've posted before (matches against the action +# runner's user login and a partial string match on comment body). +cat response.json \ +| jq \ + --arg user_login "$gh_user_login" \ + --arg pfx "$gh_comment_prefix" \ + '.[] | + select(.user.login == $user_login) | + select(.body | startswith($pfx)) | + .node_id + ' > comment_ids.txt \ +|| die 'Failed to parse issue comments response' + +[[ $(cat comment_ids.txt | wc -l) -gt 0 ]] || die "No comments matching message prefix and github user id ($gh_user_login) found" + +# Build GitHub GraphQL queries for each comment id. Because GitHub doesn't +# return whether a comment is already hidden or not in its comment listing +# endpoint, we have to hide all of them. +while IFS= read -r node_id; do + echo "mutation { minimizeComment(input: {subjectId: $node_id, classifier: OUTDATED}) { minimizedComment { isMinimized } } }" >> graphql.txt +done <<< "$(cat comment_ids.txt)" + +# Parse it through jq to build a valid json object. +while IFS= read -r graphql; do + jq --null-input -c --arg q "$graphql" '{"query": $q}' >> hide_queries.json \ + || die "Failed to create http minimizeComment query for graphql query $graphql" +done <<< "$(cat graphql.txt)" + +# Hide Comments +echo 'Issuing GraphQL calls to GitHub to hide previous schema-diff comments' +while IFS= read -r hide_query; do + curl -fsSX POST \ + -H "Authorization: Bearer $gh_token" \ + -d "$hide_query" \ + $gh_gql_url \ + || die "Failed to issue request to minimize comment" +done <<< "$(cat hide_queries.json)" diff --git a/.github/workflows/schema-diff.yml b/.github/workflows/schema-diff.yml new file mode 100644 index 000000000..58fbd594e --- /dev/null +++ b/.github/workflows/schema-diff.yml @@ -0,0 +1,92 @@ +name: schema-diff + +on: + pull_request: + paths: + - 'internal/db/schema/migrations/**/*.sql' + - 'scripts/schema-diff.sh' + - '.github/scripts/schema-diff-hide-gh-comments.sh' + - '.github/workflows/schema-diff.yml' + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + schema-diff: + name: "Schema Diff" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + fetch-depth: '0' + - name: Generate Schema Diff + env: + PGPASSWORD: boundary + run: | + # Run schema-diff between the code in the PR and its base. + ./scripts/schema-diff.sh "origin/${GITHUB_BASE_REF}" + + gh_comment_prefix='Database schema diff between' + gh_user_login='github-actions[bot]' + + # Hide previous schema-diff comments we've posted. + echo 'Hiding previous PR comments' + ./.github/scripts/schema-diff-hide-gh-comments.sh \ + "$GITHUB_API_URL" \ + "$GITHUB_GRAPHQL_URL" \ + "${{ secrets.GITHUB_TOKEN }}" \ + "$GITHUB_REPOSITORY" \ + "${{ github.event.pull_request.number }}" \ + "$gh_comment_prefix" \ + "$gh_user_login" + + # getdiff echoes the diff in GitHub markdown syntax or "Unchanged" if + # the file is empty. + getdiff() { + if [[ -s $1 ]]; then + echo \`\`\`diff + echo "$(cat $1)" + echo \`\`\` + else + echo "Unchanged" + fi + } + + # Build heredoc with all the diffs the schema diff tool generated. + echo 'Building new GitHub schema-diff comment' + cat << EOF > github-comment.txt + $gh_comment_prefix \`${GITHUB_BASE_REF}\` and \`${GITHUB_HEAD_REF}\` @ ${{ github.event.pull_request.head.sha }} + + #### Functions + $(getdiff .schema-diff/funcs.diff) + + #### Tables + $(getdiff .schema-diff/tables.diff) + + #### Views + $(getdiff .schema-diff/views.diff) + + #### Triggers + $(getdiff .schema-diff/triggers.diff) + + #### Indexes + $(getdiff .schema-diff/indexes.diff) + + #### Post Data + $(getdiff .schema-diff/post_data.diff) + EOF + + # Parse it through jq to build a valid json object. + jq --null-input \ + --arg comment "$(cat github-comment.txt)" \ + '{"body": $comment}' > body.json + + # Post comment on PR. + echo "Posting new GitHub schema-diff comment under PR #${{ github.event.pull_request.number }}" + curl -sX POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -d @body.json \ + $GITHUB_API_URL/repos/$GITHUB_REPOSITORY/issues/${{ github.event.pull_request.number }}/comments diff --git a/.gitignore b/.gitignore index 63d63aa8d..a0b31a6eb 100644 --- a/.gitignore +++ b/.gitignore @@ -143,3 +143,5 @@ enos/ci/*/.terraform.lock.hcl !.release/ci.hcl dist out + +.schema-diff/ diff --git a/scripts/schema-diff.sh b/scripts/schema-diff.sh new file mode 100755 index 000000000..383b52c57 --- /dev/null +++ b/scripts/schema-diff.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + + +die() { + echo "$@" + exit 255 +} + +which pg_dump &> /dev/null || die "pg_dump must be installed" +which pg_restore &> /dev/null || die "pg_restore must be installed" +which pg_isready &> /dev/null || die "pg_isready must be installed" +which awk &> /dev/null || die "awk must be installed" +which git &> /dev/null || die "git must be installed" +which make &> /dev/null || die "make must be installed" + +set -e + +SQL_TEST_DB_PORT=${SQL_TEST_DB_PORT:=5432} +export SQL_TEST_DB_PORT + +DB_HOST=${DB_HOST:=127.0.0.1} + +base_branch=${1} +if [[ -z "${base_branch}" ]]; then + base_branch="main" +fi +base_commit=$(git rev-parse "${base_branch}") + +new_branch=$(git rev-parse --abbrev-ref HEAD) +new_commit=$(git rev-parse HEAD) + +tmp_dir=".schema-diff" + +extract() { + local suffix=$1 + local dump="${tmp_dir}/${suffix}.dump" + + mkdir -p \ + "${tmp_dir}/funcs_${suffix}" \ + "${tmp_dir}/tables_${suffix}" \ + "${tmp_dir}/views_${suffix}" \ + "${tmp_dir}/triggers_${suffix}" \ + "${tmp_dir}/indexes_${suffix}" + + echo "extracting function definitions from ${dump}" + while read -r f; do + fname="${f%(*}" + pg_restore -s -O -P "${f}" -f - "${dump}" | tr '[:upper:]' '[:lower:]' > "${tmp_dir}/funcs_${suffix}/${fname}.sql" + done < <(pg_restore -l "${dump}" -f - | awk '$4 == "FUNCTION" {for(i=6;i "${tmp_dir}/tables_${suffix}/${tname}.sql" + done < <(pg_restore -l "${dump}" -f - | awk '$4 == "TABLE" {for(i=6;i "${tmp_dir}/views_${suffix}/${vname}.sql" + done < <(pg_restore -l "${dump}" -f - | awk '$4 == "VIEW" {for(i=6;i "${tmp_dir}/triggers_${suffix}/${tname}.sql" + done < <(pg_restore -l "${dump}" -f - | awk '$4 == "TRIGGER" {for(i=6;i "${tmp_dir}/indexes_${suffix}/${dname}.sql" + done < <(pg_restore -l "${dump}" -f - | awk '$4 == "INDEX" {for(i=6;i "${tmp_dir}/post_data_${suffix}.sql" +} + +dump() { + local suffix=$1 + local dump="${tmp_dir}/${suffix}.dump" + + mkdir -p "${tmp_dir}" + + make -C internal/db/sqltest clean + make -C internal/db/sqltest database-up + max=120 + c=0 + until pg_isready -h "${DB_HOST}" -p "${SQL_TEST_DB_PORT}"; do + ((c+=1)) + if [[ $c -ge $max ]]; then + docker logs boundary-sql-tests + make -C internal/db/sqltest clean + die "timeout waiting for database, likely an error in a migration" + fi + sleep 1 + done + + echo "dumping to ${dump}" + pg_dump -Fc -h "${DB_HOST}" -U boundary -f "${dump}" + + make -C internal/db/sqltest clean +} + +rm -rf "${tmp_dir}" + +echo "Comparing schema between ${new_branch}@${new_commit} ${base_branch}@${base_commit}" + +dump "${new_commit}" +extract "${new_commit}" + +git checkout "${base_commit}" + +dump "${base_commit}" +extract "${base_commit}" + +if [[ "${new_branch}" == "HEAD" ]]; then + git checkout "${new_commit}" +else + git checkout "${new_branch}" +fi + +for t in "funcs" "tables" "views" "triggers" "indexes"; do + git diff --no-index "${tmp_dir}/${t}_${base_commit}" "${tmp_dir}/${t}_${new_commit}" | tee "${tmp_dir}/${t}.diff" +done + +git diff --no-index "${tmp_dir}/post_data_${base_commit}.sql" "${tmp_dir}/post_data_${new_commit}.sql" | tee "${tmp_dir}/post_data.diff"