From 498a8408624de94d58a6a5b406abb2f87c497f30 Mon Sep 17 00:00:00 2001 From: Sam Salisbury Date: Mon, 28 Sep 2020 17:15:28 +0100 Subject: [PATCH] Add packagespec CI (#479) --- .circleci/.gitattributes | 1 + .circleci/.gitignore | 1 + .circleci/Makefile | 95 ++++ .circleci/README.md | 130 ++++++ .circleci/config.yml | 578 ++++++++++++++++++++++++- .circleci/config/@build-release.yml | 489 +++++++++++++++++++++ .circleci/config/@config.yml | 4 + .circleci/config/jobs/build.yml | 17 + .circleci/config/workflows/default.yml | 2 + .circleci/pre-commit | 45 ++ .gitattributes | 2 + Makefile | 13 +- internal/ui/VERSION | 2 + packages-oss.lock/.gitattributes | 4 + packages-oss.lock/Makefile | 290 +++++++++++++ packages-oss.lock/build.mk | 146 +++++++ packages-oss.lock/config.mk | 238 ++++++++++ packages-oss.lock/layer.mk | 361 +++++++++++++++ packages-oss.lock/layers/layers.mk | 39 ++ packages-oss.lock/pkgs.yml | 322 ++++++++++++++ packages-oss.yml | 116 +++++ packagespec.mk | 82 ++++ scripts/uigen.sh | 44 +- version/version_base.go | 10 +- 24 files changed, 3002 insertions(+), 29 deletions(-) create mode 100644 .circleci/.gitattributes create mode 100644 .circleci/.gitignore create mode 100644 .circleci/Makefile create mode 100644 .circleci/README.md create mode 100644 .circleci/config/@build-release.yml create mode 100644 .circleci/config/@config.yml create mode 100644 .circleci/config/jobs/build.yml create mode 100644 .circleci/config/workflows/default.yml create mode 100755 .circleci/pre-commit create mode 100755 .gitattributes create mode 100644 internal/ui/VERSION create mode 100755 packages-oss.lock/.gitattributes create mode 100755 packages-oss.lock/Makefile create mode 100755 packages-oss.lock/build.mk create mode 100755 packages-oss.lock/config.mk create mode 100755 packages-oss.lock/layer.mk create mode 100755 packages-oss.lock/layers/layers.mk create mode 100644 packages-oss.lock/pkgs.yml create mode 100644 packages-oss.yml create mode 100755 packagespec.mk diff --git a/.circleci/.gitattributes b/.circleci/.gitattributes new file mode 100644 index 000000000..2dd06ee5f --- /dev/null +++ b/.circleci/.gitattributes @@ -0,0 +1 @@ +config.yml linguist-generated diff --git a/.circleci/.gitignore b/.circleci/.gitignore new file mode 100644 index 000000000..3018b3a68 --- /dev/null +++ b/.circleci/.gitignore @@ -0,0 +1 @@ +.tmp/ diff --git a/.circleci/Makefile b/.circleci/Makefile new file mode 100644 index 000000000..d24027f1d --- /dev/null +++ b/.circleci/Makefile @@ -0,0 +1,95 @@ +# Set SHELL to 'strict mode' without using .SHELLFLAGS for max compatibility. +# See https://fieldnotes.tech/how-to-shell-for-compatible-makefiles/ +SHELL := /usr/bin/env bash -euo pipefail -c + +# CONFIG is the name of the make target someone +# would invoke to update the main config file (config.yml). +CONFIG ?= ci-config +# VERIFY is the name of the make target someone +# would invoke to verify the config file. +VERIFY ?= ci-verify + +CIRCLECI := circleci --skip-update-check +ifeq ($(DEBUG_CIRCLECI_CLI),YES) +CIRCLECI += --debug +endif + +# For config processing, always refer to circleci.com not self-hosted circleci, +# because self-hosted does not currently support the necessary API. +CIRCLECI_CLI_HOST := https://circleci.com +export CIRCLECI_CLI_HOST + +# Set up some documentation/help message variables. +# We do not attempt to install the CircleCI CLI from this Makefile. +CCI_INSTALL_LINK := https://circleci.com/docs/2.0/local-cli/\#installation +CCI_INSTALL_MSG := Please install CircleCI CLI. See $(CCI_INSTALL_LINK) +CCI_VERSION := $(shell $(CIRCLECI) version 2> /dev/null) +ifeq ($(CCI_VERSION),) +# Attempting to use the CLI fails with installation instructions. +CIRCLECI := echo '$(CCI_INSTALL_MSG)'; exit 1; \# +endif + +SOURCE_DIR := config +SOURCE_YML := $(shell [ ! -d $(SOURCE_DIR) ] || find $(SOURCE_DIR) -name '*.yml') +CONFIG_SOURCE := Makefile $(SOURCE_YML) | $(SOURCE_DIR) +OUT := config.yml +TMP := .tmp/config-processed +CONFIG_PACKED := .tmp/config-packed + +default: help + +help: + @echo "Usage:" + @echo " make $(CONFIG): recompile config.yml from $(SOURCE_DIR)/" + @echo " make $(VERIFY): verify that config.yml is a true mapping from $(SOURCE_DIR)/" + @echo + @echo "Diagnostics:" + @[ -z "$(CCI_VERSION)" ] || echo " circleci-cli version $(CCI_VERSION)" + @[ -n "$(CCI_VERSION)" ] || echo " $(CCI_INSTALL_MSG)" + +$(SOURCE_DIR): + @echo No source directory $(SOURCE_DIR) found.; exit 1 + +# Make sure our .tmp dir exists. +$(shell [ -d .tmp ] || mkdir .tmp) + +.PHONY: $(CONFIG) +$(CONFIG): $(OUT) + +.PHONY: $(VERIFY) +$(VERIFY): config-up-to-date + @$(CIRCLECI) config validate $(OUT) + +define GENERATED_FILE_HEADER +### *** +### WARNING: DO NOT manually EDIT or MERGE this file, it is generated by 'make $(CONFIG)'. +### INSTEAD: Edit or merge the source in $(SOURCE_DIR)/ then run 'make $(CONFIG)'. +### *** +endef +export GENERATED_FILE_HEADER + +# GEN_CONFIG writes the config to a temporary file. If the whole process succeeds, +# it them moves that file to $@. This makes is an atomic operation, so if it fails +# make doesn't consider a half-baked file up to date. +define GEN_CONFIG + @$(CIRCLECI) config pack $(SOURCE_DIR) > $(CONFIG_PACKED) + @echo "$$GENERATED_FILE_HEADER" > $@.tmp || { rm -f $@; exit 1; } + @$(CIRCLECI) config process $(CONFIG_PACKED) >> $@.tmp || { rm -f $@.tmp; exit 1; } + @mv -f $@.tmp $@ +endef + +$(OUT): $(CONFIG_SOURCE) + $(GEN_CONFIG) + @echo "$@ updated" + +$(TMP): $(CONFIG_SOURCE) + $(GEN_CONFIG) + +.PHONY: config-up-to-date +config-up-to-date: $(TMP) # Note this must not depend on $(OUT)! + @if diff -w $(OUT) $<; then \ + echo "Generated $(OUT) is up to date!"; \ + else \ + echo "Generated $(OUT) is out of date, run make $(CONFIG) to update."; \ + exit 1; \ + fi diff --git a/.circleci/README.md b/.circleci/README.md new file mode 100644 index 000000000..1ec75cafa --- /dev/null +++ b/.circleci/README.md @@ -0,0 +1,130 @@ +# How to use CircleCI multi-file config + +This README and the Makefile should be in your `.circleci` directory, +in the root of your repository. +All path references in this README assume we are in this `.circleci` directory. + +The `Makefile` in this directory generates `./config.yml` in CircleCI 2.0 syntax, +from the tree rooted at `./config/`, which contains files in CircleCI 2.0 or 2.1 syntax. + + +## Quickstart + +The basic workflow is: + +- Edit source files in `./config/` +- When you are done, run `make ci-config` to update `./config.yml` +- Commit this entire `.circleci` directory, including that generated file together. +- Run `make ci-verify` to ensure the current `./config.yml` is up to date with the source. + +When merging this `.circleci` directory: + +- Do not merge the generated `./config.yml` file, instead: +- Merge the source files under `./config/`, and then +- Run `make ci-config` to re-generate the merged `./config.yml` + +And that's it, for more detail, read on! + + +## How does it work, roughly? + +CircleCI supports [generating a single config file from many], +using the `$ circleci config pack` command. +It also supports [expanding 2.1 syntax to 2.0 syntax] +using the `$ circleci config process` command. +We use these two commands, stitched together using the `Makefile` +to implement the workflow. + +[generating a single config file from many]: https://circleci.com/docs/2.0/local-cli/#packing-a-config +[expanding 2.1 syntax to 2.0 syntax]: https://circleci.com/docs/2.0/local-cli/#processing-a-config + + +## Prerequisites + +You will need the [CircleCI CLI tool] installed and working, +at least version `0.1.5607`. +You can [download this tool directly from GitHub Releases]. + +``` +$ circleci version +0.1.5607+f705856 +``` + +[CircleCI CLI tool]: https://circleci.com/docs/2.0/local-cli/ +[download this tool directly from GitHub Releases]: https://github.com/CircleCI-Public/circleci-cli/releases + + +## Updating the config source + +Before making changes, be sure to understand the layout +of the `./config/` file tree, as well as circleci 2.1 syntax. +See the [Syntax and layout] section below. + +To update the config, you should edit, add or remove files +in the `./config/` directory, +and then run `make ci-config`. +If that's successful, +you should then commit every `*.yml` file in the tree rooted in this directory. +That is: you should commit both the source under `./config/` +and the generated file `./config.yml` at the same time, in the same commit. +The included git pre-commit hook will help with this. +Do not edit the `./config.yml` file directly, as you will lose your changes +next time `make ci-config` is run. + +[Syntax and layout]: #syntax-and-layout + + +### Verifying `./config.yml` + +To check whether or not the current `./config.yml` is up to date with the source +and valid, run `$ make ci-verify`. +Note that `$ make ci-verify` should be run in CI, +in case not everyone has the git pre-commit hook set up correctly. + + +#### Example shell session + +```sh +$ make ci-config +config.yml updated +$ git add -A . # The -A makes sure to include deletions/renames etc. +$ git commit -m "ci: blah blah blah" +Changes detected in .circleci/, running 'make -C .circleci ci-verify' +--> Generated config.yml is up to date! +--> Config file at config.yml is valid. +``` + + +### Syntax and layout + +It is important to understand the layout of the config directory. +Read the documentation on [packing a config] for a full understanding +of how multiple YAML files are merged by the circleci CLI tool. + +[packing a config]: https://circleci.com/docs/2.0/local-cli/#packing-a-config + +Here is an example file tree (with comments added afterwards): + +```sh +$ tree . +. +├── Makefile +├── README.md # This file. +├── config # The source code for config.yml is rooted here. +│   ├── @config.yml # Files beginning with @ are treated specially by `circleci config pack` +│   ├── commands # Subdirectories of config become top-level keys. +│   │   └── go_test.yml # Filenames (minus .yml) become top-level keys under +│   │   └── go_build.yml # their parent (in this case "commands"). +│ │ # The contents of go_test.yml therefore are placed at: .commands.go_test: +│   └── jobs # jobs also becomes a top-level key under config... +│   ├── build.yml # ...and likewise filenames become keys under their parent. +│   └── test.yml +└── config.yml # The generated file in 2.0 syntax. +``` + +About those `@` files... Preceding a filename with `@` +indicates to `$ circleci config pack` that the contents of this YAML file +should be at the top-level, rather than underneath a key named after their filename. +This naming convention is unfortunate as it breaks autocompletion in bash, +but there we go. + diff --git a/.circleci/config.yml b/.circleci/config.yml index 43d1f91a8..e5e534f0c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,22 +1,574 @@ -version: 2.1 -orbs: - slack: circleci/slack@3.4.2 +### *** +### WARNING: DO NOT manually EDIT or MERGE this file, it is generated by 'make ci-config'. +### INSTEAD: Edit or merge the source in config/ then run 'make ci-config'. +### *** +# Orb 'circleci/slack@3.4.2' resolved to 'circleci/slack@3.4.2' +version: 2 jobs: - build: + build-common-layers: machine: - image: 'ubuntu-1604:201903-01' - working_directory: ~/boundary + image: ubuntu-1604:202007-01 + resource_class: xlarge + shell: /usr/bin/env bash -euo pipefail -c steps: - add_ssh_keys: fingerprints: - # hashicorp-ci github "machine" user, for private modules - - "c6:96:98:82:dc:04:6c:39:dd:ac:83:05:e3:15:1c:98" + - c6:96:98:82:dc:04:6c:39:dd:ac:83:05:e3:15:1c:98 + - checkout + - run: + command: make -C packages*.lock write-builder-cache-keys + name: Write builder layer cache keys + - restore_cache: + keys: + - cache006-copy-source_dde4_{{checksum ".buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3"}} + - cache006-go-modules_694e_{{checksum ".buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e"}} + - cache006-ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + - cache006-base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + name: Restore build image cache + - run: + command: | + if [ -f .buildcache/archives/03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3.tar.gz ]; then + echo "Exact match found in cache, skipping build." + circleci-agent step halt + else + echo "No exact match found, proceeding with build." + fi + name: Finish early if loaded exact match from cache. + - run: + command: | + LAYER_SPEC_ID=03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3 make -C packages*.lock load-builder-cache + name: Load whatever builder cache we have (if any) into the Docker daemon + no_output_timeout: 30m + - run: + command: |2 + + KEYFILE="$HOME/.ssh/id_rsa_c6969882dc046c39ddac8305e3151c98" + eval "$(ssh-agent -s)" + if [ -f "$KEYFILE" ]; then + ssh-add "$KEYFILE" + else + echo "==> INFO: SSH key for github.com not found" + echo " Attempts to access private repositories from within" + echo " the build will fail, e.g. for private go modules, or" + echo " attempts to directly clone private repositories." + fi + + make -f packages*.lock/layer.mk 00-base-aac1d959b4b3af1495941602b5e51fb6181baded-image + name: Build base layer + - run: + command: |2 + + KEYFILE="$HOME/.ssh/id_rsa_c6969882dc046c39ddac8305e3151c98" + eval "$(ssh-agent -s)" + if [ -f "$KEYFILE" ]; then + ssh-add "$KEYFILE" + else + echo "==> INFO: SSH key for github.com not found" + echo " Attempts to access private repositories from within" + echo " the build will fail, e.g. for private go modules, or" + echo " attempts to directly clone private repositories." + fi + + make -f packages*.lock/layer.mk 01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d-image + name: Build ui layer + - run: + command: |2 + + KEYFILE="$HOME/.ssh/id_rsa_c6969882dc046c39ddac8305e3151c98" + eval "$(ssh-agent -s)" + if [ -f "$KEYFILE" ]; then + ssh-add "$KEYFILE" + else + echo "==> INFO: SSH key for github.com not found" + echo " Attempts to access private repositories from within" + echo " the build will fail, e.g. for private go modules, or" + echo " attempts to directly clone private repositories." + fi + + make -f packages*.lock/layer.mk 02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e-image + name: Build go-modules layer + - run: + command: |2 + + KEYFILE="$HOME/.ssh/id_rsa_c6969882dc046c39ddac8305e3151c98" + eval "$(ssh-agent -s)" + if [ -f "$KEYFILE" ]; then + ssh-add "$KEYFILE" + else + echo "==> INFO: SSH key for github.com not found" + echo " Attempts to access private repositories from within" + echo " the build will fail, e.g. for private go modules, or" + echo " attempts to directly clone private repositories." + fi + + make -f packages*.lock/layer.mk 03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3-image + name: Build copy-source layer + - run: + command: make -f packages*.lock/layer.mk 03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3-save + name: Export builder image + - save_cache: + key: cache006-copy-source_dde4_{{checksum ".buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3"}} + name: Save layer copy-source to cache. + paths: + - .buildcache/archives/03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3.tar.gz + - run: + command: mv .buildcache/archives/03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3.tar.gz .buildcache/archives/02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e.tar.gz + name: Prepare layer go-modules for caching + - save_cache: + key: cache006-go-modules_694e_{{checksum ".buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e"}} + name: Save layer go-modules to cache. + paths: + - .buildcache/archives/02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e.tar.gz + - run: + command: mv .buildcache/archives/02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e.tar.gz .buildcache/archives/01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d.tar.gz + name: Prepare layer ui for caching + - save_cache: + key: cache006-ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + name: Save layer ui to cache. + paths: + - .buildcache/archives/01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d.tar.gz + - run: + command: mv .buildcache/archives/01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d.tar.gz .buildcache/archives/00-base-aac1d959b4b3af1495941602b5e51fb6181baded.tar.gz + name: Prepare layer base for caching + - save_cache: + key: cache006-base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + name: Save layer base to cache. + paths: + - .buildcache/archives/00-base-aac1d959b4b3af1495941602b5e51fb6181baded.tar.gz + environment: + - AUTO_INSTALL_TOOLS: 'YES' + - BUILDKIT_PROGRESS: plain + - PRODUCT_REVISION: '' + darwin_amd64_package: + docker: + - image: circleci/buildpack-deps + shell: /usr/bin/env bash -euo pipefail -c + environment: + - AUTO_INSTALL_TOOLS: 'YES' + - BUILDKIT_PROGRESS: plain + - PRODUCT_REVISION: '' + - PACKAGE_SPEC_ID: 3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3 + steps: + - setup_remote_docker: + version: 19.03.12 + - add_ssh_keys: + fingerprints: + - c6:96:98:82:dc:04:6c:39:dd:ac:83:05:e3:15:1c:98 + - checkout + - run: + command: make -C packages*.lock write-package-cache-key + name: Write package cache key + - restore_cache: + key: cache006-package-3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3-{{checksum ".buildcache/cache-keys/package-3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3"}} + name: Restore package cache + - run: + command: | + if ! { PKG=$(find .buildcache/packages/store -maxdepth 1 -mindepth 1 -name '*.zip' 2> /dev/null) && [ -n "$PKG" ]; }; then + echo "No package found, continuing with build." + exit 0 + fi + echo "Package already cached, skipping build." + circleci-agent step halt + name: Check the cache status. + - run: + command: make -C packages*.lock write-builder-cache-keys + name: Write builder layer cache keys + - restore_cache: + keys: + - cache006-copy-source_dde4_{{checksum ".buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3"}} + - cache006-go-modules_694e_{{checksum ".buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e"}} + - cache006-ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + - cache006-base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + name: Restore build layer cache + - run: + command: | + make -C packages*.lock load-builder-cache + name: Load whatever builder cache we have (if any) into the Docker daemon + no_output_timeout: 30m + - run: + command: |2 + + KEYFILE="$HOME/.ssh/id_rsa_c6969882dc046c39ddac8305e3151c98" + eval "$(ssh-agent -s)" + if [ -f "$KEYFILE" ]; then + ssh-add "$KEYFILE" + else + echo "==> INFO: SSH key for github.com not found" + echo " Attempts to access private repositories from within" + echo " the build will fail, e.g. for private go modules, or" + echo " attempts to directly clone private repositories." + fi + + make -C packages*.lock package + name: Compile Package + - run: + command: ls -lahR .buildcache/packages + name: List packages + - save_cache: + key: cache006-package-3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3-{{checksum ".buildcache/cache-keys/package-3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3"}} + name: Saving package cache + paths: + - .buildcache/packages/store + linux_amd64_package: + docker: + - image: circleci/buildpack-deps + shell: /usr/bin/env bash -euo pipefail -c + environment: + - AUTO_INSTALL_TOOLS: 'YES' + - BUILDKIT_PROGRESS: plain + - PRODUCT_REVISION: '' + - PACKAGE_SPEC_ID: 26ad460ac09efc9c56bdbb7a962ff92616c06c51 + steps: + - setup_remote_docker: + version: 19.03.12 + - add_ssh_keys: + fingerprints: + - c6:96:98:82:dc:04:6c:39:dd:ac:83:05:e3:15:1c:98 + - checkout + - run: + command: make -C packages*.lock write-package-cache-key + name: Write package cache key + - restore_cache: + key: cache006-package-26ad460ac09efc9c56bdbb7a962ff92616c06c51-{{checksum ".buildcache/cache-keys/package-26ad460ac09efc9c56bdbb7a962ff92616c06c51"}} + name: Restore package cache + - run: + command: | + if ! { PKG=$(find .buildcache/packages/store -maxdepth 1 -mindepth 1 -name '*.zip' 2> /dev/null) && [ -n "$PKG" ]; }; then + echo "No package found, continuing with build." + exit 0 + fi + echo "Package already cached, skipping build." + circleci-agent step halt + name: Check the cache status. + - run: + command: make -C packages*.lock write-builder-cache-keys + name: Write builder layer cache keys + - restore_cache: + keys: + - cache006-copy-source_dde4_{{checksum ".buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3"}} + - cache006-go-modules_694e_{{checksum ".buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e"}} + - cache006-ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + - cache006-base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + name: Restore build layer cache + - run: + command: | + make -C packages*.lock load-builder-cache + name: Load whatever builder cache we have (if any) into the Docker daemon + no_output_timeout: 30m + - run: + command: |2 + + KEYFILE="$HOME/.ssh/id_rsa_c6969882dc046c39ddac8305e3151c98" + eval "$(ssh-agent -s)" + if [ -f "$KEYFILE" ]; then + ssh-add "$KEYFILE" + else + echo "==> INFO: SSH key for github.com not found" + echo " Attempts to access private repositories from within" + echo " the build will fail, e.g. for private go modules, or" + echo " attempts to directly clone private repositories." + fi + + make -C packages*.lock package + name: Compile Package + - run: + command: ls -lahR .buildcache/packages + name: List packages + - save_cache: + key: cache006-package-26ad460ac09efc9c56bdbb7a962ff92616c06c51-{{checksum ".buildcache/cache-keys/package-26ad460ac09efc9c56bdbb7a962ff92616c06c51"}} + name: Saving package cache + paths: + - .buildcache/packages/store + windows_amd64_package: + docker: + - image: circleci/buildpack-deps + shell: /usr/bin/env bash -euo pipefail -c + environment: + - AUTO_INSTALL_TOOLS: 'YES' + - BUILDKIT_PROGRESS: plain + - PRODUCT_REVISION: '' + - PACKAGE_SPEC_ID: 6b5bf7794d67cf76b1b4772807af52deb1f6a808 + steps: + - setup_remote_docker: + version: 19.03.12 + - add_ssh_keys: + fingerprints: + - c6:96:98:82:dc:04:6c:39:dd:ac:83:05:e3:15:1c:98 + - checkout + - run: + command: make -C packages*.lock write-package-cache-key + name: Write package cache key + - restore_cache: + key: cache006-package-6b5bf7794d67cf76b1b4772807af52deb1f6a808-{{checksum ".buildcache/cache-keys/package-6b5bf7794d67cf76b1b4772807af52deb1f6a808"}} + name: Restore package cache + - run: + command: | + if ! { PKG=$(find .buildcache/packages/store -maxdepth 1 -mindepth 1 -name '*.zip' 2> /dev/null) && [ -n "$PKG" ]; }; then + echo "No package found, continuing with build." + exit 0 + fi + echo "Package already cached, skipping build." + circleci-agent step halt + name: Check the cache status. + - run: + command: make -C packages*.lock write-builder-cache-keys + name: Write builder layer cache keys + - restore_cache: + keys: + - cache006-copy-source_dde4_{{checksum ".buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3"}} + - cache006-go-modules_694e_{{checksum ".buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e"}} + - cache006-ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + - cache006-base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + name: Restore build layer cache + - run: + command: | + make -C packages*.lock load-builder-cache + name: Load whatever builder cache we have (if any) into the Docker daemon + no_output_timeout: 30m + - run: + command: |2 + + KEYFILE="$HOME/.ssh/id_rsa_c6969882dc046c39ddac8305e3151c98" + eval "$(ssh-agent -s)" + if [ -f "$KEYFILE" ]; then + ssh-add "$KEYFILE" + else + echo "==> INFO: SSH key for github.com not found" + echo " Attempts to access private repositories from within" + echo " the build will fail, e.g. for private go modules, or" + echo " attempts to directly clone private repositories." + fi + + make -C packages*.lock package + name: Compile Package + - run: + command: ls -lahR .buildcache/packages + name: List packages + - save_cache: + key: cache006-package-6b5bf7794d67cf76b1b4772807af52deb1f6a808-{{checksum ".buildcache/cache-keys/package-6b5bf7794d67cf76b1b4772807af52deb1f6a808"}} + name: Saving package cache + paths: + - .buildcache/packages/store + bundle-releases: + docker: + - image: circleci/buildpack-deps + shell: /usr/bin/env bash -euo pipefail -c + steps: + - checkout + - run: + command: make -C packages*.lock write-all-package-cache-keys + name: Write all package cache keys + - restore_cache: + key: cache006-package-3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3-{{checksum ".buildcache/cache-keys/package-3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3"}} + - run: + command: | + make package-meta + environment: + PACKAGE_SPEC_ID: 3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3 + name: Write package metadata for darwin_amd64_package + - restore_cache: + key: cache006-package-26ad460ac09efc9c56bdbb7a962ff92616c06c51-{{checksum ".buildcache/cache-keys/package-26ad460ac09efc9c56bdbb7a962ff92616c06c51"}} + - run: + command: | + make package-meta + environment: + PACKAGE_SPEC_ID: 26ad460ac09efc9c56bdbb7a962ff92616c06c51 + name: Write package metadata for linux_amd64_package + - restore_cache: + key: cache006-package-6b5bf7794d67cf76b1b4772807af52deb1f6a808-{{checksum ".buildcache/cache-keys/package-6b5bf7794d67cf76b1b4772807af52deb1f6a808"}} + - run: + command: | + make package-meta + environment: + PACKAGE_SPEC_ID: 6b5bf7794d67cf76b1b4772807af52deb1f6a808 + name: Write package metadata for windows_amd64_package + - run: + command: make aliases + name: Write package aliases + - run: + command: ls -lahR .buildcache + name: List Build Cache + - run: + command: cp packages*.lock/pkgs.yml lockfile-standalone.yml + - store_artifacts: + destination: lockfile-standalone.yml + path: lockfile-standalone.yml + - store_artifacts: + destination: packages-standalone + path: .buildcache/packages + - run: + command: tar -czf packages-standalone.tar.gz .buildcache/packages lockfile-standalone.yml + - store_artifacts: + destination: packages-standalone.tar.gz + path: packages-standalone.tar.gz + - run: + command: tar -czf meta-standalone.tar.gz .buildcache/packages/store/*.json lockfile-standalone.yml + - store_artifacts: + destination: meta-standalone.tar.gz + path: meta-standalone.tar.gz + environment: + - AUTO_INSTALL_TOOLS: 'YES' + - BUILDKIT_PROGRESS: plain + - PRODUCT_REVISION: '' + build: + machine: + image: ubuntu-1604:201903-01 + steps: + - add_ssh_keys: + fingerprints: + - c6:96:98:82:dc:04:6c:39:dd:ac:83:05:e3:15:1c:98 - checkout - run: - name: "Run Acceptance Tests" command: | make test-ci - - slack/status: - fail_only: true - only_for_branches: master - webhook: webhook + name: Run Acceptance Tests + - run: + command: | + echo 'export SLACK_BUILD_STATUS="fail"' >> $BASH_ENV + name: Slack - Setting Failure Condition + when: on_fail + - run: + command: | + echo 'export SLACK_BUILD_STATUS="success"' >> $BASH_ENV + name: Slack - Setting Success Condition + when: on_success + - run: + command: | + if [ ! -x /bin/bash ]; then + echo Bash not installed. + exit 1 + fi + name: Provide error if non-bash shell + - run: + command: | + current_branch_in_filter=false + + IFS="," read -ra BRANCH_FILTERS <<< "master" + + for i in "${BRANCH_FILTERS[@]}"; do + if [ "${i}" == "${CIRCLE_BRANCH}" ]; then + current_branch_in_filter=true + fi + done + + if [ "x" == "xmaster" ] || [ "$current_branch_in_filter" = true ]; then + # Provide error if no webhook is set and error. Otherwise continue + if [ -z "webhook" ]; then + echo "NO SLACK WEBHOOK SET" + echo "Please input your SLACK_WEBHOOK value either in the settings for this project, or as a parameter for this orb." + exit 1 + else + #Create Members string + if [ -n "" ]; then + IFS="," read -ra SLACK_MEMBERS <<< "" + for i in "${SLACK_MEMBERS[@]}"; do + if [ $(echo ${i} | head -c 1) == "S" ]; then + SLACK_MENTIONS="${SLACK_MENTIONS} " + elif echo ${i} | grep -E "^(here|channel|everyone)$" > /dev/null; then + SLACK_MENTIONS="${SLACK_MENTIONS} " + else + SLACK_MENTIONS="${SLACK_MENTIONS}<@${i}> " + fi + done + fi + #If successful + if [ "$SLACK_BUILD_STATUS" = "success" ]; then + #Skip if fail_only + if [ true = true ]; then + echo "The job completed successfully" + echo '"fail_only" is set to "true". No Slack notification sent.' + else + curl -X POST -H 'Content-type: application/json' \ + --data "{ \ + \"attachments\": [ \ + { \ + \"fallback\": \":tada: A $CIRCLE_JOB job has succeeded!\", \ + \"text\": \":tada: A $CIRCLE_JOB job has succeeded! $SLACK_MENTIONS\", \ + \"fields\": [ \ + { \ + \"title\": \"Project\", \ + \"value\": \"$CIRCLE_PROJECT_REPONAME\", \ + \"short\": true \ + }, \ + { \ + \"title\": \"Job Number\", \ + \"value\": \"$CIRCLE_BUILD_NUM\", \ + \"short\": true \ + } \ + ], \ + \"actions\": [ \ + { \ + \"type\": \"button\", \ + \"text\": \"Visit Job\", \ + \"url\": \"$CIRCLE_BUILD_URL\" \ + } \ + ], \ + \"color\": \"#1CBF43\" \ + } \ + ] \ + } " webhook + echo "Job completed successfully. Alert sent." + fi + else + #If Failed + curl -X POST -H 'Content-type: application/json' \ + --data "{ \ + \"attachments\": [ \ + { \ + \"fallback\": \":red_circle: A $CIRCLE_JOB job has failed!\", \ + \"text\": \":red_circle: A $CIRCLE_JOB job has failed! $SLACK_MENTIONS\", \ + \"fields\": [ \ + { \ + \"title\": \"Project\", \ + \"value\": \"$CIRCLE_PROJECT_REPONAME\", \ + \"short\": true \ + }, \ + { \ + \"title\": \"Job Number\", \ + \"value\": \"$CIRCLE_BUILD_NUM\", \ + \"short\": true \ + } \ + ], \ + \"actions\": [ \ + { \ + \"type\": \"button\", \ + \"text\": \"Visit Job\", \ + \"url\": \"$CIRCLE_BUILD_URL\" \ + } \ + ], \ + \"color\": \"#ed5c5c\" \ + } \ + ] \ + } " webhook + echo "Job failed. Alert sent." + fi + fi + else + echo "Current branch is not included in only_for_branches filter; no status alert will be sent" + fi + name: Slack - Sending Status Alert + shell: /bin/bash + when: always + working_directory: ~/boundary +workflows: + build-standalone: + jobs: + - build-common-layers + - darwin_amd64_package: + requires: + - build-common-layers + - linux_amd64_package: + requires: + - build-common-layers + - windows_amd64_package: + requires: + - build-common-layers + - bundle-releases: + requires: + - darwin_amd64_package + - linux_amd64_package + - windows_amd64_package + default: + jobs: + - build + version: 2 diff --git a/.circleci/config/@build-release.yml b/.circleci/config/@build-release.yml new file mode 100644 index 000000000..921e521ec --- /dev/null +++ b/.circleci/config/@build-release.yml @@ -0,0 +1,489 @@ +--- +# *** +# WARNING: Do not EDIT or MERGE this file, it is generated by packagespec. +# *** + +version: 2.1 + +executors: + builder: + docker: + - image: circleci/buildpack-deps + environment: + PRODUCT_REVISION: "" + AUTO_INSTALL_TOOLS: 'YES' + BUILDKIT_PROGRESS: plain + shell: /usr/bin/env bash -euo pipefail -c + builder-machine: + machine: + image: ubuntu-1604:202007-01 + resource_class: xlarge + environment: + PRODUCT_REVISION: "" + AUTO_INSTALL_TOOLS: 'YES' + BUILDKIT_PROGRESS: plain + shell: /usr/bin/env bash -euo pipefail -c + +workflows: + version: 2 + build-standalone: + jobs: + - build-common-layers + - darwin_amd64_package: { requires: [ build-common-layers ] } + - linux_amd64_package: { requires: [ build-common-layers ] } + - windows_amd64_package: { requires: [ build-common-layers ] } + - bundle-releases: + requires: + - darwin_amd64_package + - linux_amd64_package + - windows_amd64_package +jobs: + build-common-layers: + executor: builder-machine + steps: + + - add_ssh_keys: + fingerprints: + # hashicorp-ci github user. + - "c6:96:98:82:dc:04:6c:39:dd:ac:83:05:e3:15:1c:98" + + - checkout + - write-build-layer-cache-keys + - restore_cache: + name: Restore build image cache + keys: + - cache006-copy-source_dde4_{{checksum ".buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3"}} + - cache006-go-modules_694e_{{checksum ".buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e"}} + - cache006-ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + - cache006-base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + - run: + name: Finish early if loaded exact match from cache. + command: | + if [ -f .buildcache/archives/03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3.tar.gz ]; then + echo "Exact match found in cache, skipping build." + circleci-agent step halt + else + echo "No exact match found, proceeding with build." + fi + - run: + name: Load whatever builder cache we have (if any) into the Docker daemon + no_output_timeout: 30m + command: | + LAYER_SPEC_ID=03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3 make -C packages*.lock load-builder-cache + - run: + name: Build base layer + command: | + + KEYFILE="$HOME/.ssh/id_rsa_c6969882dc046c39ddac8305e3151c98" + eval "$(ssh-agent -s)" + if [ -f "$KEYFILE" ]; then + ssh-add "$KEYFILE" + else + echo "==> INFO: SSH key for github.com not found" + echo " Attempts to access private repositories from within" + echo " the build will fail, e.g. for private go modules, or" + echo " attempts to directly clone private repositories." + fi + + make -f packages*.lock/layer.mk 00-base-aac1d959b4b3af1495941602b5e51fb6181baded-image + - run: + name: Build ui layer + command: | + + KEYFILE="$HOME/.ssh/id_rsa_c6969882dc046c39ddac8305e3151c98" + eval "$(ssh-agent -s)" + if [ -f "$KEYFILE" ]; then + ssh-add "$KEYFILE" + else + echo "==> INFO: SSH key for github.com not found" + echo " Attempts to access private repositories from within" + echo " the build will fail, e.g. for private go modules, or" + echo " attempts to directly clone private repositories." + fi + + make -f packages*.lock/layer.mk 01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d-image + - run: + name: Build go-modules layer + command: | + + KEYFILE="$HOME/.ssh/id_rsa_c6969882dc046c39ddac8305e3151c98" + eval "$(ssh-agent -s)" + if [ -f "$KEYFILE" ]; then + ssh-add "$KEYFILE" + else + echo "==> INFO: SSH key for github.com not found" + echo " Attempts to access private repositories from within" + echo " the build will fail, e.g. for private go modules, or" + echo " attempts to directly clone private repositories." + fi + + make -f packages*.lock/layer.mk 02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e-image + - run: + name: Build copy-source layer + command: | + + KEYFILE="$HOME/.ssh/id_rsa_c6969882dc046c39ddac8305e3151c98" + eval "$(ssh-agent -s)" + if [ -f "$KEYFILE" ]; then + ssh-add "$KEYFILE" + else + echo "==> INFO: SSH key for github.com not found" + echo " Attempts to access private repositories from within" + echo " the build will fail, e.g. for private go modules, or" + echo " attempts to directly clone private repositories." + fi + + make -f packages*.lock/layer.mk 03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3-image + + - run: + name: Export builder image + command: make -f packages*.lock/layer.mk 03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3-save + - save_cache: + name: Save layer copy-source to cache. + key: cache006-copy-source_dde4_{{checksum ".buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3"}} + paths: + - .buildcache/archives/03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3.tar.gz + - run: + name: Prepare layer go-modules for caching + command: mv .buildcache/archives/03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3.tar.gz .buildcache/archives/02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e.tar.gz + - save_cache: + name: Save layer go-modules to cache. + key: cache006-go-modules_694e_{{checksum ".buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e"}} + paths: + - .buildcache/archives/02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e.tar.gz + - run: + name: Prepare layer ui for caching + command: mv .buildcache/archives/02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e.tar.gz .buildcache/archives/01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d.tar.gz + - save_cache: + name: Save layer ui to cache. + key: cache006-ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + paths: + - .buildcache/archives/01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d.tar.gz + - run: + name: Prepare layer base for caching + command: mv .buildcache/archives/01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d.tar.gz .buildcache/archives/00-base-aac1d959b4b3af1495941602b5e51fb6181baded.tar.gz + - save_cache: + name: Save layer base to cache. + key: cache006-base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + paths: + - .buildcache/archives/00-base-aac1d959b4b3af1495941602b5e51fb6181baded.tar.gz + darwin_amd64_package: + executor: builder + environment: + - PACKAGE_SPEC_ID: 3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3 + steps: + + - setup_remote_docker: + version: 19.03.12 + + - add_ssh_keys: + fingerprints: + # hashicorp-ci github user. + - "c6:96:98:82:dc:04:6c:39:dd:ac:83:05:e3:15:1c:98" + + + - checkout + + # Restore the package cache first, we might not need to rebuild. + - write-package-cache-key + - restore_cache: + name: Restore package cache + key: 'cache006-package-3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3-{{checksum ".buildcache/cache-keys/package-3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3"}}' + - run: + name: Check the cache status. + command: | + if ! { PKG=$(find .buildcache/packages/store -maxdepth 1 -mindepth 1 -name '*.zip' 2> /dev/null) && [ -n "$PKG" ]; }; then + echo "No package found, continuing with build." + exit 0 + fi + echo "Package already cached, skipping build." + circleci-agent step halt + + # We need to rebuild, so load the builder cache. + - write-build-layer-cache-keys + - restore_cache: + name: Restore build layer cache + keys: + - cache006-copy-source_dde4_{{checksum ".buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3"}} + - cache006-go-modules_694e_{{checksum ".buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e"}} + - cache006-ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + - cache006-base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + - run: + name: Load whatever builder cache we have (if any) into the Docker daemon + no_output_timeout: 30m + command: | + make -C packages*.lock load-builder-cache + + - run: + name: Compile Package + command: | + + KEYFILE="$HOME/.ssh/id_rsa_c6969882dc046c39ddac8305e3151c98" + eval "$(ssh-agent -s)" + if [ -f "$KEYFILE" ]; then + ssh-add "$KEYFILE" + else + echo "==> INFO: SSH key for github.com not found" + echo " Attempts to access private repositories from within" + echo " the build will fail, e.g. for private go modules, or" + echo " attempts to directly clone private repositories." + fi + + make -C packages*.lock package + + - run: + name: List packages + command: ls -lahR .buildcache/packages + - save_cache: + name: Saving package cache + key: 'cache006-package-3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3-{{checksum ".buildcache/cache-keys/package-3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3"}}' + paths: + - .buildcache/packages/store + # Save builder image cache if necessary. + # The range should only iterate over a single layer. + + linux_amd64_package: + executor: builder + environment: + - PACKAGE_SPEC_ID: 26ad460ac09efc9c56bdbb7a962ff92616c06c51 + steps: + + - setup_remote_docker: + version: 19.03.12 + + - add_ssh_keys: + fingerprints: + # hashicorp-ci github user. + - "c6:96:98:82:dc:04:6c:39:dd:ac:83:05:e3:15:1c:98" + + + - checkout + + # Restore the package cache first, we might not need to rebuild. + - write-package-cache-key + - restore_cache: + name: Restore package cache + key: 'cache006-package-26ad460ac09efc9c56bdbb7a962ff92616c06c51-{{checksum ".buildcache/cache-keys/package-26ad460ac09efc9c56bdbb7a962ff92616c06c51"}}' + - run: + name: Check the cache status. + command: | + if ! { PKG=$(find .buildcache/packages/store -maxdepth 1 -mindepth 1 -name '*.zip' 2> /dev/null) && [ -n "$PKG" ]; }; then + echo "No package found, continuing with build." + exit 0 + fi + echo "Package already cached, skipping build." + circleci-agent step halt + + # We need to rebuild, so load the builder cache. + - write-build-layer-cache-keys + - restore_cache: + name: Restore build layer cache + keys: + - cache006-copy-source_dde4_{{checksum ".buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3"}} + - cache006-go-modules_694e_{{checksum ".buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e"}} + - cache006-ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + - cache006-base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + - run: + name: Load whatever builder cache we have (if any) into the Docker daemon + no_output_timeout: 30m + command: | + make -C packages*.lock load-builder-cache + + - run: + name: Compile Package + command: | + + KEYFILE="$HOME/.ssh/id_rsa_c6969882dc046c39ddac8305e3151c98" + eval "$(ssh-agent -s)" + if [ -f "$KEYFILE" ]; then + ssh-add "$KEYFILE" + else + echo "==> INFO: SSH key for github.com not found" + echo " Attempts to access private repositories from within" + echo " the build will fail, e.g. for private go modules, or" + echo " attempts to directly clone private repositories." + fi + + make -C packages*.lock package + + - run: + name: List packages + command: ls -lahR .buildcache/packages + - save_cache: + name: Saving package cache + key: 'cache006-package-26ad460ac09efc9c56bdbb7a962ff92616c06c51-{{checksum ".buildcache/cache-keys/package-26ad460ac09efc9c56bdbb7a962ff92616c06c51"}}' + paths: + - .buildcache/packages/store + # Save builder image cache if necessary. + # The range should only iterate over a single layer. + + windows_amd64_package: + executor: builder + environment: + - PACKAGE_SPEC_ID: 6b5bf7794d67cf76b1b4772807af52deb1f6a808 + steps: + + - setup_remote_docker: + version: 19.03.12 + + - add_ssh_keys: + fingerprints: + # hashicorp-ci github user. + - "c6:96:98:82:dc:04:6c:39:dd:ac:83:05:e3:15:1c:98" + + + - checkout + + # Restore the package cache first, we might not need to rebuild. + - write-package-cache-key + - restore_cache: + name: Restore package cache + key: 'cache006-package-6b5bf7794d67cf76b1b4772807af52deb1f6a808-{{checksum ".buildcache/cache-keys/package-6b5bf7794d67cf76b1b4772807af52deb1f6a808"}}' + - run: + name: Check the cache status. + command: | + if ! { PKG=$(find .buildcache/packages/store -maxdepth 1 -mindepth 1 -name '*.zip' 2> /dev/null) && [ -n "$PKG" ]; }; then + echo "No package found, continuing with build." + exit 0 + fi + echo "Package already cached, skipping build." + circleci-agent step halt + + # We need to rebuild, so load the builder cache. + - write-build-layer-cache-keys + - restore_cache: + name: Restore build layer cache + keys: + - cache006-copy-source_dde4_{{checksum ".buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3"}} + - cache006-go-modules_694e_{{checksum ".buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e"}} + - cache006-ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + - cache006-base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + - run: + name: Load whatever builder cache we have (if any) into the Docker daemon + no_output_timeout: 30m + command: | + make -C packages*.lock load-builder-cache + + - run: + name: Compile Package + command: | + + KEYFILE="$HOME/.ssh/id_rsa_c6969882dc046c39ddac8305e3151c98" + eval "$(ssh-agent -s)" + if [ -f "$KEYFILE" ]; then + ssh-add "$KEYFILE" + else + echo "==> INFO: SSH key for github.com not found" + echo " Attempts to access private repositories from within" + echo " the build will fail, e.g. for private go modules, or" + echo " attempts to directly clone private repositories." + fi + + make -C packages*.lock package + + - run: + name: List packages + command: ls -lahR .buildcache/packages + - save_cache: + name: Saving package cache + key: 'cache006-package-6b5bf7794d67cf76b1b4772807af52deb1f6a808-{{checksum ".buildcache/cache-keys/package-6b5bf7794d67cf76b1b4772807af52deb1f6a808"}}' + paths: + - .buildcache/packages/store + # Save builder image cache if necessary. + # The range should only iterate over a single layer. + + + bundle-releases: + executor: builder + steps: + - checkout + - write-all-package-cache-keys + - load-darwin_amd64_package + - run: + environment: + PACKAGE_SPEC_ID: 3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3 + name: Write package metadata for darwin_amd64_package + command: | + make package-meta + - load-linux_amd64_package + - run: + environment: + PACKAGE_SPEC_ID: 26ad460ac09efc9c56bdbb7a962ff92616c06c51 + name: Write package metadata for linux_amd64_package + command: | + make package-meta + - load-windows_amd64_package + - run: + environment: + PACKAGE_SPEC_ID: 6b5bf7794d67cf76b1b4772807af52deb1f6a808 + name: Write package metadata for windows_amd64_package + command: | + make package-meta + - run: + name: Write package aliases + command: + make aliases + - run: + name: List Build Cache + command: ls -lahR .buildcache + + # Give the lockfile a name store_artifacts can work with. + - run: cp packages*.lock/pkgs.yml lockfile-standalone.yml + + # Surface the lock file as an artifact. + - store_artifacts: + path: lockfile-standalone.yml + destination: lockfile-standalone.yml + + # Surface the package store directory as an artifact. + # This makes each zipped package separately downloadable. + - store_artifacts: + path: .buildcache/packages + destination: packages-standalone + + # Surface a tarball of the whole package store plus lockfile as an artifact. + - run: tar -czf packages-standalone.tar.gz .buildcache/packages lockfile-standalone.yml + - store_artifacts: + path: packages-standalone.tar.gz + destination: packages-standalone.tar.gz + + # Surface a tarball of just the metadata files. + - run: tar -czf meta-standalone.tar.gz .buildcache/packages/store/*.json lockfile-standalone.yml + - store_artifacts: + path: meta-standalone.tar.gz + destination: meta-standalone.tar.gz + +commands: + load-darwin_amd64_package: + steps: + - restore_cache: + key: 'cache006-package-3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3-{{checksum ".buildcache/cache-keys/package-3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3"}}' + + load-linux_amd64_package: + steps: + - restore_cache: + key: 'cache006-package-26ad460ac09efc9c56bdbb7a962ff92616c06c51-{{checksum ".buildcache/cache-keys/package-26ad460ac09efc9c56bdbb7a962ff92616c06c51"}}' + + load-windows_amd64_package: + steps: + - restore_cache: + key: 'cache006-package-6b5bf7794d67cf76b1b4772807af52deb1f6a808-{{checksum ".buildcache/cache-keys/package-6b5bf7794d67cf76b1b4772807af52deb1f6a808"}}' + + + write-build-layer-cache-keys: + steps: + - run: + name: Write builder layer cache keys + command: make -C packages*.lock write-builder-cache-keys + + write-package-cache-key: + steps: + - run: + name: Write package cache key + command: make -C packages*.lock write-package-cache-key + + write-all-package-cache-keys: + steps: + - run: + name: Write all package cache keys + command: make -C packages*.lock write-all-package-cache-keys diff --git a/.circleci/config/@config.yml b/.circleci/config/@config.yml new file mode 100644 index 000000000..ad4c23829 --- /dev/null +++ b/.circleci/config/@config.yml @@ -0,0 +1,4 @@ +version: 2.1 + +orbs: + slack: circleci/slack@3.4.2 diff --git a/.circleci/config/jobs/build.yml b/.circleci/config/jobs/build.yml new file mode 100644 index 000000000..e80b0175b --- /dev/null +++ b/.circleci/config/jobs/build.yml @@ -0,0 +1,17 @@ +machine: + image: 'ubuntu-1604:201903-01' +working_directory: ~/boundary +steps: +- add_ssh_keys: + fingerprints: + # hashicorp-ci github "machine" user, for private modules + - "c6:96:98:82:dc:04:6c:39:dd:ac:83:05:e3:15:1c:98" +- checkout +- run: + name: "Run Acceptance Tests" + command: | + make test-ci +- slack/status: + fail_only: true + only_for_branches: master + webhook: webhook diff --git a/.circleci/config/workflows/default.yml b/.circleci/config/workflows/default.yml new file mode 100644 index 000000000..eaa188f53 --- /dev/null +++ b/.circleci/config/workflows/default.yml @@ -0,0 +1,2 @@ +jobs: + - build diff --git a/.circleci/pre-commit b/.circleci/pre-commit new file mode 100755 index 000000000..b7c63e898 --- /dev/null +++ b/.circleci/pre-commit @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Check .circleci/config.yml is up to date and valid, and that all changes are +# included together in this commit. + +# Fail early if we accidentally used '.yaml' instead of '.yml' +if ! git diff --name-only --cached --exit-code -- '.circleci/***.yaml'; then + echo "ERROR: File(s) with .yaml extension detected. Please rename them .yml instead." + exit 1 +fi + +# Succeed early if no changes to yml files in .circleci/ are currently staged. +# make ci-verify is slow so we really don't want to run it unnecessarily. +if git diff --name-only --cached --exit-code -- '.circleci/***.yml'; then + exit 0 +fi + +# Make sure to add no explicit output before this line, as it would just be noise +# for those making non-circleci changes. +echo "==> Verifying config changes in .circleci/" +echo "--> OK: All files are .yml not .yaml" + +# Ensure commit includes _all_ files in .circleci/ +# So not only are the files up to date, but we are also committing them in one go. +if ! git diff --name-only --exit-code -- '.circleci/***.yml'; then + echo "ERROR: Some .yml diffs in .circleci/ are staged, others not." + echo "Please commit the entire .circleci/ directory together, or omit it altogether." + exit 1 +fi + +# Even untracked yml or yaml files will get baked into the output. +# This is a loose check on _any_ untracked files in .circleci/ for simplicity. +if [ -n "$(git ls-files --others --exclude-standard '.circleci/')" ]; then + echo "ERROR: You have untracked files in .circleci/ please add or delete them." + exit 1 +fi + +echo "--> OK: All .yml files in .circleci are staged." +if ! make -C .circleci ci-verify; then + echo "ERROR: make ci-verify failed" + exit 1 +fi +echo "--> OK: make ci-verify succeeded." diff --git a/.gitattributes b/.gitattributes new file mode 100755 index 000000000..266c085dc --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +/packagespec.mk linguist-generated +/packagespec.mk linguist-generated \ No newline at end of file diff --git a/Makefile b/Makefile index 5c3d27031..ae490317e 100644 --- a/Makefile +++ b/Makefile @@ -116,6 +116,17 @@ test-ci: install-go install-go: ./ci/goinstall.sh -.PHONY: api tools gen migrations proto website +.PHONY: api tools gen migrations proto website ci-config ci-verify .NOTPARALLEL: + +ci-config: + @$(MAKE) -C .circleci ci-config + +ci-verify: + @$(MAKE) -C .circleci ci-verify + +PACKAGESPEC_CIRCLECI_CONFIG := .circleci/config/@build-release.yml +PACKAGESPEC_HOOK_POST_CI_CONFIG := $(MAKE) ci-config + +-include packagespec.mk diff --git a/internal/ui/VERSION b/internal/ui/VERSION new file mode 100644 index 000000000..0d1c73586 --- /dev/null +++ b/internal/ui/VERSION @@ -0,0 +1,2 @@ +ae36654bc2afababef5b3ed2c5cacedebaf65348 Merge pull request #229 from hashicorp/feature-loading-bar +# Above commit is used for production builds. diff --git a/packages-oss.lock/.gitattributes b/packages-oss.lock/.gitattributes new file mode 100755 index 000000000..6fff647da --- /dev/null +++ b/packages-oss.lock/.gitattributes @@ -0,0 +1,4 @@ +# *** +# WARNING: Do not EDIT or MERGE this file, it is generated by packagespec. +# *** +* linguist-generated \ No newline at end of file diff --git a/packages-oss.lock/Makefile b/packages-oss.lock/Makefile new file mode 100755 index 000000000..78c2d88d7 --- /dev/null +++ b/packages-oss.lock/Makefile @@ -0,0 +1,290 @@ +# *** +# WARNING: Do not EDIT or MERGE this file, it is generated by packagespec. +# *** +include $(shell git rev-parse --show-toplevel)/packages*.lock/config.mk + +.PHONY: packages commands build package write-builder-cache-keys \ + write-all-package-cache-keys build-all + +GOOS ?= $(shell go env GOOS 2>/dev/null || echo linux) +GOARCH ?= $(shell go env GOARCH 2>/dev/null || echo amd64) + +DEFAULT_PACKAGE_YQ := [ .packages[] | select(.inputs.GOOS=="$(GOOS)" and .inputs.GOARCH=="$(GOARCH)") ][0] +QUERY_DEFAULT_PACKAGESPEC = $(call QUERY_LOCK,$(DEFAULT_PACKAGE_YQ) | $(1)) + +# MK is shorthand for changing to repo root and selecting a make target +# from a file in this directory. All *.mk files in this directory assume +# the current working directory is the repo root. This Makefile exists +# to invoke those mk files correctly. +MK := $(MAKE) -C $(REPO_ROOT) -f $(LOCKDIR)/ + +# configure load-builder-cache target to load the most specific builder layer cache +# available as an archive in the build cache. + +ifneq ($(PACKAGE_SPEC_ID),) + +PACKAGE_CACHE_KEY_FILE := $(shell $(call QUERY_PACKAGESPEC,.meta.builtin.PACKAGE_CACHE_KEY_FILE)) + +# Loading the best available archive for a specific package build. + +BUILD_LAYER_ARCHIVES := $(shell $(call QUERY_PACKAGESPEC,.meta.builtin.BUILD_LAYERS[].archive)) +BEST_BUILD_LAYER_ARCHIVE := $(shell cd $(REPO_ROOT) && for F in $(BUILD_LAYER_ARCHIVES); do \ + if [ -f $$F ]; then echo $$F; exit 0; fi; done) +ifeq ($(BEST_BUILD_LAYER_ARCHIVE),) +load-builder-cache: + @echo "No build layer archives found in build cache. Looked for: $(BUILD_LAYER_ARCHIVES)" +else +BEST_BUILD_LAYER_NAME := $(shell $(call QUERY_PACKAGESPEC,.meta.builtin.BUILD_LAYERS[] \ + | select(.archive=="$(BEST_BUILD_LAYER_ARCHIVE)") | .name)) +BEST_BUILD_LAYER_LOAD_TARGET := $(BEST_BUILD_LAYER_NAME)-load + +load-builder-cache: + @$(MK)layer.mk $(BEST_BUILD_LAYER_LOAD_TARGET) + +endif + +else ifneq ($(LAYER_SPEC_ID),) + +# Loading the best avilable archive for a specific layer build. + +BUILD_LAYER_ARCHIVES := $(shell $(call QUERY_LOCK,.layers[] | select(.name=="$(LAYER_SPEC_ID)") \ + | .meta.builtin.LAYER_LIST[].archive)) +BEST_BUILD_LAYER_ARCHIVE := $(shell cd $(REPO_ROOT) && for F in $(BUILD_LAYER_ARCHIVES); do \ + if [ -f $$F ]; then echo $$F; exit 0; fi; done) +ifeq ($(BEST_BUILD_LAYER_ARCHIVE),) + +load-builder-cache: + @echo "No build layer archives found in build cache. Looked for: $(BUILD_LAYER_ARCHIVES)" + +else +BEST_BUILD_LAYER_NAME := $(shell $(call QUERY_LOCK,.layers[] | select(.name=="$(LAYER_SPEC_ID)") \ + | .meta.builtin.LAYER_LIST[] | select(.archive=="$(BEST_BUILD_LAYER_ARCHIVE)") | .name)) +BEST_BUILD_LAYER_LOAD_TARGET := $(BEST_BUILD_LAYER_NAME)-load + +load-builder-cache: + @$(MK)layer.mk $(BEST_BUILD_LAYER_LOAD_TARGET) + +endif + +else + +load-builder-cache: + @echo "You must set PACKAGE_SPEC_ID or LAYER_SPEC_ID so we know which caches to look for." + +endif + +commands: + @$(MAKE) -f packages.mk commands + +ifeq ($(DIRTY_FILES),) +DIRTY_SOURCE_WARNING := +else +DIRTY_SOURCE_WARNING = echo "==> SOURCE TREE IS DIRTY; $(1)" +endif + +# build is a convenience target for local builds, do not use in CI. +# Instead, use `make package` specifying PACKAGE_SPEC_ID. +build: + @$(call DIRTY_SOURCE_WARNING,PERFORMING DIRTY BUILD) + @echo "==> Building default package for GOOS=$(GOOS) GOARCH=$(GOARCH)" + @ALIASES=$$($(call QUERY_DEFAULT_PACKAGESPEC,.aliases[] | "alias type:\(.type) path:\(.path)") | column -t); \ + echo "$$ALIASES" + @PACKAGE_SPEC_ID="$$($(call QUERY_DEFAULT_PACKAGESPEC,.packagespecid) | head -n1)"; \ + COMMAND="PACKAGE_SOURCE_ID=$$PACKAGE_SOURCE_ID PACKAGE_SPEC_ID=$$PACKAGE_SPEC_ID $(MK)build.mk package"; \ + echo "$$COMMAND"; \ + $(SHELL) "$$COMMAND" + +# package-contents is a convenience target for local builds, do not use in CI. +package-contents: + @$(call DIRTY_SOURCE_WARNING,GETTING CONTENTS OF DIRTY BUILD) + @echo "==> Getting contents of default package for GOOS=$(GOOS) GOARCH=$(GOARCH)" + @ALIASES=$$($(call QUERY_DEFAULT_PACKAGESPEC,.aliases[] | "alias type:\(.type) path:\(.path)") | column -t); \ + echo "$$ALIASES" + @PACKAGE_SPEC_ID="$$($(call QUERY_DEFAULT_PACKAGESPEC,.packagespecid) | head -n1)"; \ + COMMAND="PACKAGE_SOURCE_ID=$$PACKAGE_SOURCE_ID PACKAGE_SPEC_ID=$$PACKAGE_SPEC_ID $(MK)build.mk package-contents"; \ + echo "$$COMMAND"; \ + $(SHELL) "$$COMMAND" + +# copy-package-contents is a convenience target for local builds, do not use in CI. +copy-package-contents: + @$(call DIRTY_SOURCE_WARNING,COPYING CONTENTS OF DIRTY BUILD) + @echo "==> Getting contents of default package for GOOS=$(GOOS) GOARCH=$(GOARCH)" + @ALIASES=$$($(call QUERY_DEFAULT_PACKAGESPEC,.aliases[] | "alias type:\(.type) path:\(.path)") | column -t); \ + echo "$$ALIASES" + @PACKAGE_SPEC_ID="$$($(call QUERY_DEFAULT_PACKAGESPEC,.packagespecid) | head -n1)"; \ + COMMAND="PACKAGE_SOURCE_ID=$$PACKAGE_SOURCE_ID PACKAGE_SPEC_ID=$$PACKAGE_SPEC_ID $(MK)build.mk copy-package-contents"; \ + echo "$$COMMAND"; \ + $(SHELL) "$$COMMAND" + +# meta is a convenience target for local builds, do not use in CI. +# Instead, use `make package-meta` specifying PACKAGE_SPEC_ID. +meta: + @$(call DIRTY_SOURCE_WARNING,WRITING METADATA FOR DIRTY BUILD) + @echo "==> Writing metacdata for default package (GOOS=$(GOOS) GOARCH=$(GOARCH))" + @ALIASES=$$($(call QUERY_DEFAULT_PACKAGESPEC,.aliases[] | "alias type:\(.type) path:\(.path)") | column -t); \ + echo "$$ALIASES" + @PACKAGE_SPEC_ID="$$($(call QUERY_DEFAULT_PACKAGESPEC,.packagespecid) | head -n1)"; \ + COMMAND="PACKAGE_SOURCE_ID=$$PACKAGE_SOURCE_ID PACKAGE_SPEC_ID=$$PACKAGE_SPEC_ID $(MK)build.mk package-meta"; \ + echo "$$COMMAND"; \ + $(SHELL) "$$COMMAND" + +# build-all is a convenience target to sequentially build each package. +# It is mostly useful in the tutorial, do not use this in CI as it is much slower +# than building packages in parallel. +build-all: + @PACKAGE_SPEC_IDS="$$($(call QUERY_LOCK,.packages[] | .packagespecid))"; \ + COUNT=$$(echo $$PACKAGE_SPEC_IDS | wc -w | xargs); \ + echo "==> Building all $$COUNT packages sequentially."; \ + for PACKAGE_SPEC_ID in $$PACKAGE_SPEC_IDS; do \ + COMMAND="PACKAGE_SOURCE_ID=$$PACKAGE_SOURCE_ID PACKAGE_SPEC_ID=$$PACKAGE_SPEC_ID $(MK)build.mk package"; \ + echo "$$COMMAND"; \ + $(SHELL) "$$COMMAND"; \ + done + +# package expects PACKAGE_SPEC_ID to already be set, use this in CI. +package: + @$(call DIRTY_SOURCE_WARNING,BUILDING DIRTY PACKAGE) + @echo "==> Building package spec $(PACKAGE_SPEC_ID)" + @ALIASES=$$($(call QUERY_PACKAGESPEC,.aliases[] | "alias type:\(.type) path:\(.path)") | column -t); \ + echo "$$ALIASES" + @COMMAND="PACKAGE_SOURCE_ID=$$PACKAGE_SOURCE_ID PACKAGE_SPEC_ID=$$PACKAGE_SPEC_ID $(MK)build.mk package"; \ + echo "$$COMMAND"; \ + $(SHELL) "$$COMMAND" + +# package-meta expects PACKAGE_SPEC_ID to already be set, use this in CI. +package-meta: + @$(call DIRTY_SOURCE_WARNING,WRITING DIRTY METADATA FOR DIRTY PACKAGE) + @echo "==> Writing metadata for package $(PACKAGE_SPEC_ID)" + @ALIASES=$$($(call QUERY_PACKAGESPEC,.aliases[] | "alias type:\(.type) path:\(.path)") | column -t); \ + echo "$$ALIASES" + @COMMAND="PACKAGE_SOURCE_ID=$$PACKAGE_SOURCE_ID PACKAGE_SPEC_ID=$$PACKAGE_SPEC_ID $(MK)build.mk package-meta"; \ + echo "$$COMMAND"; \ + $(SHELL) "$$COMMAND" + +# aliases writes all alias symlinks for packages in the package store that +# match the current LOCKFILE and PRODUCT_REVISION. It does not cause a new build. +# If the package store contains no matchin binaries, then this does nothing. +aliases: + @echo "==> Writing alias symlinks for existing packages in the store."; \ + cd $(REPO_ROOT); \ + PACKAGE_SPEC_IDS="$$($(call QUERY_LOCK,.packages[] | .packagespecid))"; \ + for PACKAGE_SPEC_ID in $$PACKAGE_SPEC_IDS; do \ + PACKAGE_FILE="$$PACKAGE_SOURCE_ID-$$PACKAGE_SPEC_ID.zip"; \ + PACKAGE="$(CACHE_ROOT)/packages/store/$$PACKAGE_FILE"; \ + [ -f $$PACKAGE ] || continue; \ + ALIASES=$$($(call QUERY_PACKAGESPEC_BY_ID,'$$PACKAGE_SPEC_ID',.aliases[] | "$(CACHE_ROOT)/packages/by-alias/\(.type)/\(.path)")); \ + for A in $$ALIASES; do \ + mkdir -p $$(dirname $$A); \ + $(LN) -rfs $$PACKAGE $$A; \ + echo "==> Alias written: $$A -> $$PACKAGE"; \ + done; \ + done + + +write-builder-cache-keys: + @echo "==> Writing build layer cache keys" + @$(MK)layer.mk write-cache-keys + +write-package-cache-key: + @if [ -z "$(PACKAGE_CACHE_KEY_FILE)" ]; then echo "Must set PACKAGE_SPEC_ID"; exit 1; fi + @$(WRITE_PACKAGE_CACHE_KEY) + @echo "==> Package cache key written: $(PACKAGE_CACHE_KEY_FILE)" + +# WRITE_PACKAGE_CACHE_KEY writes the package cache key for PACKAGE_SPEC_ID. +# We reference this as an environment variable, so you can override it in a +# recipe rather than relying on the global setting. +define WRITE_PACKAGE_CACHE_KEY +( \ + cd $(REPO_ROOT); \ + KEY="PACKAGE_SPEC_ID=$$PACKAGE_SPEC_ID"$$'\n'"PACKAGE_SOURCE_ID=$(PACKAGE_SOURCE_ID)"; \ + FILE=$$(yq -r ".packages[] | select(.packagespecid==\"$$PACKAGE_SPEC_ID\") \ + | .meta.builtin.PACKAGE_CACHE_KEY_FILE" < $(LOCK)); \ + echo "$$FILE"; \ + echo "$$KEY"; \ + mkdir -p $$(dirname $$FILE); \ + echo "$$KEY" > "$$FILE";\ +) +endef + +write-all-package-cache-keys: + @IDS="$$($(call QUERY_LOCK,.packages[].packagespecid))"; \ + for PACKAGE_SPEC_ID in $$IDS; do \ + $(WRITE_PACKAGE_CACHE_KEY); \ + done; \ + echo "==> All package cache keys written" + +clean-builder-images: + @IMAGES=$$(docker images --format '{{.Repository}}:{{.Tag}}' | grep '^$(BUILDER_IMAGE_PREFIX)' || true); \ + if [ -z "$$IMAGES" ]; then exit 0; fi; \ + docker rmi -f $$IMAGES + +clean: + @cd $(REPO_ROOT); rm -rf $(CACHE_ROOT) + +clean-all: clean clean-builder-images + +clean-all-prune: clean-all + docker container prune + docker image prune + +RELEASER_DIR := $(REPO_ROOT)/.packagespec/release + +# REQUIRE_EXPORT requires a set of make variables to be nonempty, +# exits 1 if any are not, and exports each one otherwise. +# To be used in recipe bodies. +define REQUIRE_EXPORT +$(foreach VAR,$(1),[ -n "$($(VAR))" ] || { echo "Must set $(VAR)"; exit 1; }; export $(VAR)='$($(VAR))';) +endef + +# EXPORT exports each named variable, if it exists. +define EXPORT +$(foreach VAR,$(1),export $(VAR)='$($(VAR))';) +endef + +# INVOKE_RELEASER_TARGET invokes the named target (first arg) in the releaser +# repository, first calling REQUIRE_EXPORT on all the named variables (second arg). +define INVOKE_RELEASER_TARGET + $(call REQUIRE_EXPORT,\ + PRODUCT_REPO_LOCAL PRODUCT_REPO PRODUCT_PATH \ + PRODUCT_CIRCLECI_SLUG PRODUCT_CIRCLECI_HOST RELEASE_SYSTEM_BRANCH \ + PRODUCT_RELEASE_REPO SPEC LOCKDIR \ + ) \ + ( cd $(REPO_ROOT) && packagespec load -asset=PREP_TIME -lockdir "$(LOCKDIR)"; ); \ + ( cd $(REPO_ROOT) && packagespec load -asset=WORK_DIR -lockdir "$(LOCKDIR)"; ); \ + $(MAKE) -C $(RELEASER_DIR) $(1) +endef + +# RELEASE_TARGETS are targets in the release repo we pass control to +# to perform release actions. +# Note: The release repo is only available to HashiCorp employees. +RELEASE_TARGETS := build-ci stage-config stage + +# We always rev-parse the PRODUCT_REVISION to obtain the full SHA. This is required +# for downstream processes which use it to determine part of the package name. +$(RELEASE_TARGETS): PRODUCT_REVISION := $(shell git rev-parse $${PRODUCT_REVISION:-HEAD}) +$(RELEASE_TARGETS): PRODUCT_VERSION ?= 0.0.0-$(USER)-snapshot +$(RELEASE_TARGETS): RELEASE_SYSTEM_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) +$(RELEASE_TARGETS): + @\ + $(call REQUIRE_EXPORT,PRODUCT_REVISION PRODUCT_VERSION) \ + $(call INVOKE_RELEASER_TARGET,$@) + +# QUERY_TARGETS are targets in the release repo that perform queries, and are therefore +# not necessarily bound to a specific PRODUCT_VERSION or PRODUCT_REVISION. +QUERY_TARGETS := list-staged-builds watch-ci + +$(QUERY_TARGETS): RELEASE_SYSTEM_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) +$(QUERY_TARGETS): + @\ + $(call EXPORT,PRODUCT_REVISION PRODUCT_VERSION) \ + $(call INVOKE_RELEASER_TARGET,$@) + +# BUNDLE_TARGETS are targets acting on specific staged bundles, identified by +# their BUNDLE_ID. +BUNDLE_TARGETS := publish-config publish inspect-staged-build + +$(BUNDLE_TARGETS): RELEASE_SYSTEM_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) +$(BUNDLE_TARGETS): + @\ + $(call REQUIRE_EXPORT,BUNDLE_ID) \ + $(call INVOKE_RELEASER_TARGET,$@) diff --git a/packages-oss.lock/build.mk b/packages-oss.lock/build.mk new file mode 100755 index 000000000..5419197aa --- /dev/null +++ b/packages-oss.lock/build.mk @@ -0,0 +1,146 @@ +# *** +# WARNING: Do not EDIT or MERGE this file, it is generated by packagespec. +# *** +# build.mk builds the packages defined in packages.lock, first building all necessary +# builder images. +# +# NOTE: This file should always run as though it were in the repo root, so all paths +# are relative to the repo root. + +# Include config.mk relative to repo root. +include $(shell git rev-parse --show-toplevel)/packages*.lock/config.mk + +ifeq ($(PACKAGE_SPEC_ID),) +$(error You must set PACKAGE_SPEC_ID; 'make build' does this for you.) +endif + +ifneq ($(PRODUCT_VERSION),) +$(error You cannot set PRODUCT_VERSION for local builds, did you mean PRODUCT_REVISION?) +endif + +# PACKAGES_ROOT holds the package store, as well as other package aliases. +PACKAGES_ROOT := $(CACHE_ROOT)/packages +# PACKAGE_STORE is where we store all the package files themselves +# addressed by their input hashes. +PACKAGE_STORE := $(PACKAGES_ROOT)/store +# BY_ALIAS is where we store alias symlinks to the store. +BY_ALIAS := $(PACKAGES_ROOT)/by-alias + +# Include the layers driver. +include $(LOCKDIR)/layer.mk + +# GET_IMAGE_MARKER_FILE gets the name of the Docker image marker file +# for the named build layer. +GET_IMAGE_MARKER_FILE = $($(1)_IMAGE) +# GET_IMAGE_NAME gets the Docker image name of the build layer. +GET_IMAGE_NAME = $($(1)_IMAGE_NAME) + +# Determine the top-level build layer. +BUILD_LAYER_NAME := $(shell $(call QUERY_PACKAGESPEC,.meta.builtin.BUILD_LAYERS[0].name)) +BUILD_LAYER_IMAGE = $(call GET_IMAGE_MARKER_FILE,$(BUILD_LAYER_NAME)) +BUILD_LAYER_IMAGE_NAME = $(call GET_IMAGE_NAME,$(BUILD_LAYER_NAME)) + +BUILD_COMMAND := $(shell $(call QUERY_PACKAGESPEC,.["build-command"])) +BUILD_ENV := $(shell $(call QUERY_PACKAGESPEC,.inputs | to_entries[] | "\(.key)=\(.value)")) +ALIASES := $(shell $(call QUERY_PACKAGESPEC,.aliases[] | "\(.type)/\(.path)")) +ALIASES := $(addprefix $(BY_ALIAS)/,$(ALIASES)) + +ifeq ($(BUILD_COMMAND),) +$(error Unable to find build command for package spec ID $(PACKAGE_SPEC_ID)) +endif +ifeq ($(BUILD_ENV),) +$(error Unable to find build inputs for package spec ID $(PACKAGE_SPEC_ID)) +endif + +# Configure paths and filenames. +OUTPUT_DIR := $(PACKAGE_STORE) +_ := $(shell mkdir -p $(OUTPUT_DIR)) +# PACKAGE_NAME is the input-addressed name of the package. +PACKAGE_NAME := $(PACKAGE_SOURCE_ID)-$(PACKAGE_SPEC_ID) +PACKAGE_ZIP_NAME := $(PACKAGE_NAME).zip +PACKAGE := $(OUTPUT_DIR)/$(PACKAGE_ZIP_NAME) +# PACKAGE_CONTENTS is used when a built package needs to be unzipped to examine +# its contents. It is a path to a directory where these contents will be unzipped +# to. This is not needed to produce builds, but is useful for post-build tasks +# when the package contents need to be checked. +PACKAGE_CONTENTS := $(PACKAGE)_contents +META_JSON_NAME := $(PACKAGE_ZIP_NAME).meta.json +META := $(OUTPUT_DIR)/$(META_JSON_NAME) + +# In the container, place the output dir at root. This makes 'docker cp' easier. +CONTAINER_OUTPUT_DIR := /$(OUTPUT_DIR) + +FULL_BUILD_COMMAND := export $(BUILD_ENV) && mkdir -p $(CONTAINER_OUTPUT_DIR) && $(BUILD_COMMAND) + +### Docker run command configuration. + +DOCKER_SHELL := /bin/bash -euo pipefail -c + +DOCKER_RUN_ENV_FLAGS := \ + -e PACKAGE_SOURCE_ID=$(PACKAGE_SOURCE_ID) \ + -e OUTPUT_DIR=$(CONTAINER_OUTPUT_DIR) \ + -e PACKAGE_ZIP_NAME=$(PACKAGE_ZIP_NAME) + +BUILD_CONTAINER_NAME := build-$(PACKAGE_SPEC_ID)-$(PACKAGE_SOURCE_ID) +DOCKER_RUN_FLAGS := $(DOCKER_RUN_ENV_FLAGS) --name $(BUILD_CONTAINER_NAME) +# DOCKER_RUN_COMMAND ties everything together to build the final package as a +# single docker run invocation. +DOCKER_RUN_COMMAND = docker run $(DOCKER_RUN_FLAGS) $(BUILD_LAYER_IMAGE_NAME) $(DOCKER_SHELL) '$(FULL_BUILD_COMMAND)' +# DOCKER_CP_COMMAND copies the built artefact from the build container. +DOCKER_CP_COMMAND = docker cp $(BUILD_CONTAINER_NAME):$(CONTAINER_OUTPUT_DIR)/$(PACKAGE_ZIP_NAME) $(PACKAGE) + +# package builds the package according to the set PACKAGE_SPEC_ID and PRODUCT_REVISION. +.PHONY: package +package: $(ALIASES) + @echo $(PACKAGE) + +# package-contents builds the package according to PACKAGE_SPEC_ID and PRODUCT_REVISION, +# and then extracts the zip file into an adjacent directory. +.PHONY: package-contents +package-contents: $(PACKAGE_CONTENTS) + @echo "$(PACKAGE_CONTENTS)/" + +# copy-package-contents allows copying the contents of a package to a specific +# directory. You must set PACKAGE_CONTENTS_DEST_DIR to this directory. +# This is useful for implementing a top-level make target that places your +# build artifacts in a well-known location. +.PHONY: copy-package-contents +copy-package-contents: $(PACKAGE_CONTENTS) + @[ -n "$(PACKAGE_CONTENTS_DEST_DIR)" ] || { \ + echo "==> ERROR: Must set PACKAGE_CONTENTS_DEST_DIR"; \ + exit 1; \ + }; \ + mkdir -p "$(PACKAGE_CONTENTS_DEST_DIR)"; \ + cp -r "$(PACKAGE_CONTENTS)"/* "$(PACKAGE_CONTENTS_DEST_DIR)" + +.PHONY: package-meta +package-meta: $(META) + @echo $(META) + +$(META): $(LOCK) + @$(call QUERY_PACKAGESPEC,.) > $@ + +# PACKAGE builds the package. +$(PACKAGE): $(BUILD_LAYER_IMAGE) + @mkdir -p $$(dirname $@) + @echo "==> Building package: $@" + @echo "PACKAGE_SOURCE_ID: $(PACKAGE_SOURCE_ID)" + @echo "PACKAGE_SPEC_ID: $(PACKAGE_SPEC_ID)" + @# Print alias info. + @$(call QUERY_PACKAGESPEC,.aliases[] | "alias type:\(.type) path:\(.path)") | column -t + @docker rm -f $(BUILD_CONTAINER_NAME) > /dev/null 2>&1 || true # Speculative cleanup. + $(DOCKER_RUN_COMMAND) + $(DOCKER_CP_COMMAND) + @docker rm -f $(BUILD_CONTAINER_NAME) + +$(PACKAGE_CONTENTS): $(PACKAGE) + @mkdir -p "$@" && unzip "$<" -d "$@" + +# ALIASES writes the package alias links. +# ALIASES must be phony to ensure they are updated to point to the +# latest builds. +.PHONY: $(ALIASES) +$(ALIASES): $(PACKAGE) + @mkdir -p $(dir $@) + @$(LN) -rfs $(PACKAGE) $@ + @echo "==> Package alias written: $@" diff --git a/packages-oss.lock/config.mk b/packages-oss.lock/config.mk new file mode 100755 index 000000000..4c422814b --- /dev/null +++ b/packages-oss.lock/config.mk @@ -0,0 +1,238 @@ +# *** +# WARNING: Do not EDIT or MERGE this file, it is generated by packagespec. +# *** +# config.mk contains constants and derived configuration that applies to +# building both layers and final packages. + +# Only include the config once. This means we can include it in the header +# of each makefile, to allow calling them individually and when they call +# each other. +ifneq ($(CONFIG_INCLUDED),YES) +CONFIG_INCLUDED := YES + +# Set SHELL to strict mode, in a way compatible with both old and new GNU make. +SHELL := /usr/bin/env bash -euo pipefail -c + +REPO_ROOT := $(shell git rev-parse --show-toplevel) + +# Set AUTO_INSTALL_TOOLS to YES in CI to have any missing required tools installed +# automatically. +AUTO_INSTALL_TOOLS ?= NO + +# CACHE_ROOT is the build cache directory. +CACHE_ROOT ?= .buildcache + +CACHE_GITIGNORE := $(CACHE_ROOT)/.gitignore +$(shell [ -f $(CACHE_GITIGNORE) ] || { mkdir -p $(CACHE_ROOT); echo '*' > $(CACHE_GITIGNORE); }) + +# SPEC is the human-managed description of which packages we are able to build. +SPEC_FILE_PATTERN := packages*.yml +SPEC := $(shell cd $(REPO_ROOT); find . -mindepth 1 -maxdepth 1 -name '$(SPEC_FILE_PATTERN)') +ifneq ($(words $(SPEC)),1) +$(error Found $(words $(SPEC)) $(SPEC_FILE_PATTERN) files, need exactly 1: $(SPEC)) +endif + +SPEC_FILENAME := $(notdir $(SPEC)) +SPEC_MODIFIER := $(SPEC_FILENAME:packages%.yml=%) + +# LOCKDIR contains the lockfile and layer files. +LOCKDIR := packages$(SPEC_MODIFIER).lock + +# BUILDER_IMAGE_PREFIX is used in generating layers' docker image names. +BUILDER_IMAGE_PREFIX := build-layer + +# LOCK is the generated fully-expanded rendition of SPEC, for use in generating CI +# pipelines and other things. +LOCK := $(LOCKDIR)/pkgs.yml + +### Utilities and constants +GIT_EXCLUDE_PREFIX := :(exclude) +# SUM generates the sha1sum of its input. +SUM := sha1sum | cut -d' ' -f1 +# QUOTE_LIST wraps a list of space-separated strings in quotes. +QUOTE := $(shell echo "'") +QUOTE_LIST = $(addprefix $(QUOTE),$(addsuffix $(QUOTE),$(1))) +GIT_EXCLUDE_LIST = $(call QUOTE_LIST,$(addprefix $(GIT_EXCLUDE_PREFIX),$(1))) +### End utilities and constants. + +# ALWAYS_EXCLUDE_SOURCE prevents source from these directories from taking +# part in the SOURCE_ID, or from being sent to the builder image layers. +# This is important for allowing the head of master to build other commits +# where this build system has not been vendored. +# +# Source in LOCKDIR is encoded as PACKAGE_SPEC_ID and included in paths +# and cache keys. Source in .circleci/ should not do much more than call +# code in the release/ directory, SPEC is the source of LOCKDIR. +ALWAYS_EXCLUDE_SOURCE := $(SPEC) $(LOCKDIR)/ ./packagespec.mk ./.circleci/ +# ALWAYS_EXCLUD_SOURCE_GIT is git path filter parlance for the above. +ALWAYS_EXCLUDE_SOURCE_GIT := $(call GIT_EXCLUDE_LIST,$(ALWAYS_EXCLUDE_SOURCE)) + +YQ_PACKAGE_BY_ID = .packages[] | select(.packagespecid == "$(1)") + +# YQ_PACKAGE_PATH is a yq query fragment to select the package PACKAGE_SPEC_ID. +# This may be invalid, check that PACKAGE_SPEC_ID is not empty before use. +YQ_PACKAGE_PATH := $(call YQ_PACKAGE_BY_ID,$(PACKAGE_SPEC_ID)) + +YQ_PACKAGE_PATH_BY_ID = $(call YQ_PACKAGE_BY_ID,$(1)) + +# QUERY_LOCK is a macro to query the lock file. +QUERY_LOCK = cd $(REPO_ROOT); yq -r '$(1)' < $(LOCK) + +QUERY_SPEC = cd $(REPO_ROOT); yq -r '$(1)' < $(SPEC) + +# QUERY_PACKAGESPEC queries the package according to the current PACKAGE_SPEC_ID. +QUERY_PACKAGESPEC = $(call QUERY_LOCK,$(YQ_PACKAGE_PATH) | $(1)) +QUERY_PACKAGESPEC_BY_ID = $(call QUERY_LOCK,$(call YQ_PACKAGE_PATH_BY_ID,$(1)) | $(2)) + +# GIT_COMMIT_OR_TAG_REF returns the git commit or tag ref SHA that the passed +# commit-ish points to (that can be a commit, tag or branch ref). +# +# Note we used to suffix the passed commit-ish with '^{}' in order to traverse tags down +# to individual commits, in case the commit-ish is an annotated tag. However this +# makes build output confusing in case a tag ref is used rather than a commit ref. +# Therefore we now allow building tag refs, even though this means sometimes we might +# be building the same source with two different source IDs, and potentially wasting +# some potential cache hits. The tradeoff in terms of ease of use seems worth it for +# now, but this could be revisited later. +# The original of the line below was: +define GIT_COMMIT_OR_TAG_REF +git rev-parse --verify '$(1)' +endef + +ifeq ($(PACKAGE_SOURCE_ID),) +# Even though layers may have different Git revisions, based on the latest +# revision of their source, we always want to +# honour either HEAD or the specified PRODUCT_REVISION for compiling the +# final binaries, as this revision is the one picked by a human to form +# the release, and may be baked into the binaries produced. +ifeq ($(PRODUCT_REVISION),) +# If PRODUCT_REVISION is empty (the default) we are concerned with building the +# current work tree, regardless of whether it is dirty or not. For local builds +# this is more convenient and more likely expected behaviour than having to commit +# just to perform a new build. +# +# Determine the PACKAGE_SOURCE_ID. +# +# Dirty package builds should never be cached because their PACKAGE_SOURCE_ID +# is not unique to the code, it just reflects the last commit ID in the git log +# prefixed with dirty_. +GIT_REF := HEAD +ALLOW_DIRTY ?= YES +PRODUCT_REVISION_NICE_NAME := +DIRTY_FILES := $(shell cd $(REPO_ROOT) && git ls-files -o -m --exclude-standard -- $(ALWAYS_EXCLUDE_SOURCE_GIT) | xargs) +ifneq ($(DIRTY_FILES),) +DIRTY := dirty_$(shell cd $(REPO_ROOT) && cat $(DIRTY_FILES) | $(SUM) || echo FAIL)_ +ifeq ($(DIRTY),FAIL_) +$(error Failed to determine dirty files sha1sum) +endif +endif +PACKAGE_SOURCE_ID := $(DIRTY)$(shell $(call GIT_COMMIT_OR_TAG_REF,$(GIT_REF))) +else + +# PRODUCT_REVISION is non-empty so treat it as a git commit ref and pull files +# directly from git rather than the work tree. +GIT_REF := $(PRODUCT_REVISION) +ALLOW_DIRTY := NO +PRODUCT_REVISION_NICE_NAME := $(PRODUCT_REVISION) +PACKAGE_SOURCE_ID := $(shell if COMMIT=$$($(call GIT_COMMIT_OR_TAG_REF,$(PRODUCT_REVISION))); then echo $$COMMIT; else echo FAILED; fi) + +ifeq ($(PACKAGE_SOURCE_ID),FAILED) +$(error Unable to find git ref "$(PRODUCT_REVISION)", do you need to 'git fetch' it?) +endif + +endif +endif + +export PRODUCT_REVISION GIT_REF ALLOW_DIRTY PACKAGE_SOURCE_ID + +# REQ_TOOLS detects availability of a set of tools, and optionally auto-installs them. +define REQ_TOOLS +GROUP_NAME := $(1) +INSTALL_TOOL := $(2) +INSTALL_COMMAND := $(3) +TOOLS := $(4) +TOOL_INSTALL_LOG := $(REPO_ROOT)/$(CACHE_ROOT)/tool-install-$$(GROUP_NAME).log +_ := $$(shell mkdir -p $$(dir $$(TOOL_INSTALL_LOG))) +INSTALL_TOOL_AVAILABLE := $$(shell command -v $$(INSTALL_TOOL) > /dev/null 2>&1 && echo YES) +ATTEMPT_AUTO_INSTALL := NO +ifeq ($$(INSTALL_TOOL_AVAILABLE),YES) +ifeq ($$(AUTO_INSTALL_TOOLS),YES) +ATTEMPT_AUTO_INSTALL := YES +endif +endif +MISSING_PACKAGES := $$(shell \ + for T in $$(TOOLS); do \ + BIN=$$$$(echo $$$$T | cut -d':' -f1); \ + if ! command -v $$$$BIN > /dev/null 2>&1; then \ + echo $$$$T | cut -d':' -f2; \ + fi; \ + done | sort | uniq) +ifneq ($$(MISSING_PACKAGES),) +ifneq ($$(ATTEMPT_AUTO_INSTALL),YES) +$$(error You are missing required tools, please run '$$(INSTALL_COMMAND) $$(MISSING_PACKAGES)'.) +else +RESULT := $$(shell $$(INSTALL_COMMAND) $$(MISSING_PACKAGES) && echo OK > $$(TOOL_INSTALL_LOG)) +ifneq ($$(shell cat $$(TOOL_INSTALL_LOG)),OK) +$$(info Failed to auto-install packages with command $$(INSTALL_COMMAND) $$(MISSING_PACKAGES)) +$$(error $$(shell cat $$(TOOL_INSTALL_LOG))) +else +$$(info $$(TOOL_INSTALL_LOG)) +$$(info Installed $$(GROUP_NAME) tools successfully.) +endif +endif +endif +endef + +ifeq ($(shell uname),Darwin) +# On Mac, try to install things with homebrew. +BREW_TOOLS := gln:coreutils gtouch:coreutils gstat:coreutils \ + gtar:gnu-tar gfind:findutils jq:jq yq:python-yq +$(eval $(call REQ_TOOLS,brew,brew,brew install,$(BREW_TOOLS))) +else +# If not mac, try to install using apt. +APT_TOOLS := pip3:python3-pip jq:jq column:bsdmainutils +$(eval $(call REQ_TOOLS,apt,apt-get,sudo apt-get update && sudo apt-get install -y,$(APT_TOOLS))) +PIP_TOOLS := yq:yq +$(eval $(call REQ_TOOLS,pip,pip3,pip3 install,$(PIP_TOOLS))) + +endif + +# We rely on GNU touch, tar and ln. +# On macOS, we assume they are installed as gtouch, gtar, gln by homebrew. +ifeq ($(shell uname),Darwin) +TOUCH := gtouch +TAR := gtar +LN := gln +STAT := gstat +FIND := gfind +else +TOUCH := touch +TAR := tar +LN := ln +STAT := stat +FIND := find +endif + +# Read config from the spec. + +# PRODUCT_REPO is the official Git repo for this project. +PRODUCT_REPO := $(shell $(call QUERY_SPEC,.config["product-repo"])) + +# PRODUCT_REPO_LOCAL is the local clone of this git repo. +PRODUCT_REPO_LOCAL := $(REPO_ROOT) + +# RELEASE_REPO is the release repository for this project. +PRODUCT_RELEASE_REPO := $(shell $(call QUERY_SPEC,.config["release-repo"])) + +# PRODUCT_PATH must be unique for every repo. +# A golang-style package path is ideal. +PRODUCT_PATH := $(shell $(call QUERY_SPEC,.config["product-id"])) + +# PRODUCT_CIRCLECI_SLUG is the slug of this repo's CircleCI project. +PRODUCT_CIRCLECI_SLUG := $(shell $(call QUERY_SPEC,.config["circleci-project-slug"])) + +# PRODUCT_CIRCLECI_HOST is the host configured to build this repo. +PRODUCT_CIRCLECI_HOST := $(shell $(call QUERY_SPEC,.config["circleci-host"])) + +# End including config once only. +endif diff --git a/packages-oss.lock/layer.mk b/packages-oss.lock/layer.mk new file mode 100755 index 000000000..0cc69d8eb --- /dev/null +++ b/packages-oss.lock/layer.mk @@ -0,0 +1,361 @@ +# *** +# WARNING: Do not EDIT or MERGE this file, it is generated by packagespec. +# *** +# layer.mk contains the machinery to incrementally build the builder image +# as separate layers, so each can be cached both locally and in CI. This serves +# both to speed up builds by avoiding unnecessary repetition of work already done, +# as well as to ehnance the reliability of builds by downloading external +# dependencies only once per build when necessary. +# +# The build layers themselves can be individually exported as tarballs (by calling +# make -save) for later inspection, for sharing, or for implementing +# on-host caching without recourse to external docker registries. +# +# To use this file, include it in another makefile, and from there you must eval +# calls to the LAYER macro with this syntax: +# +# $(eval $(call LAYER,,,,,)) +# +# Each layer assumes the existence of a Dockerfile named .Dockerfile in +# packages.lock/layers. +# It uses the to set a Docker build arg called BASE_IMAGE to the +# resultant docker image ref of the named parent layer. You should use this BASE_IMAGE +# in the FROM line in your image. +# +# There must also be a base image which has no parent, and that Dockerfile should +# use a FROM line from an explicit docker image, e.g. debian:buster. +# +# Each image is provided only the source code identified by , minus +# any source code matched by . Source code is any files which are +# present and not ignored by Git. This includes cached files, modified files and new, +# untracked files. The Dockerfile belonging to this layer is ALWAYS included in the +# source, so you don't need to manually specify that. +# +# The set of source code identified by a single image layer is used to produce its +# SOURCE_ID. The SOURCE_ID, when all the files are tracked by Git and are not modified +# equals the latest Git commit SHA that affected any of those files or directories. +# When there are any new or modified files, we take a SHA 256 sum of the latest Git +# commit affecting those files concatenated with the output of git diff and the contents +# of any untracked files, and prefix this with "dirty_". The SOURCE_ID is used as part of +# the cache key for that layer. +# +# Because different combinations of source-include and source-exclude may have been +# modified by the same commit, they may share the same source ID. Therefore, we also +# calculate the LAYER_ID which takes into account not only the current layer's source +# ID, but also its source include/exclude and the ID of its base layer. Thus any change +# in any of the inputs of any base layer invalidates the cache of all subsequent layers. + +include $(shell git rev-parse --show-toplevel)/packages*.lock/config.mk + +.SECONDARY: + +_ := $(shell mkdir -p $(CACHE_ROOT)/source-archives) + +### END BUILDER IMAGE LAYERS + +## LAYER + +# The LAYER macro defines all the targets for each image defined above. +# +# The phony targets are the ones we typically run ourselves or in CI, they are: +# +# -debug : dump debug info for this image layer +# -image : build the image for this image layer +# -save : save the docker image for this layer as a tar.gz +# -load : load this image from the saved tar.gz + +define LAYER +LAYERS += $(1) +$(1)_NAME := $(1) +$(1)_TYPE := $(2) +$(1)_BASE := $(3) +$(1)_SOURCE_INCLUDE := $(4) +$(1)_SOURCE_EXCLUDE := $(sort $(5) $(ALWAYS_EXCLUDE_SOURCE)) +$(1)_CACHE_KEY_FILE := $(REPO_ROOT)/$(6) +$(1)_IMAGE_ARCHIVE := $(REPO_ROOT)/$(7) + +$(1)_CACHE_ROOT := $(CACHE_ROOT)/layers/$$($(1)_NAME) + +ifneq ($$($(1)_BASE),) +$(1)_BASE_CACHE_ROOT := $(CACHE_ROOT)/layers/$$($(1)_BASE) +$(1)_BASE_ID_FILE := $$($(1)_BASE_CACHE_ROOT)/current-layer-id +$(1)_BASE_LAYER_ID := $$(shell cat $$($(1)_BASE_ID_FILE)) +$(1)_BASE_CACHE := $$($(1)_BASE_CACHE_ROOT)/$$($(1)_BASE_LAYER_ID) +$(1)_BASE_IMAGE := $$($(1)_BASE_CACHE)/image.marker +$(1)_BASE_IMAGE_NAME = $$(shell cat $$($(1)_BASE_IMAGE)) +endif + +# If no source is included, set source ID to none. +# Note that we include the checksum of the generated Dockerfile as part of cache IDs +# so we still invalidate the cache appropriately. +ifeq ($$($(1)_SOURCE_INCLUDE),) + +$(1)_SOURCE_CMD := echo "" +$(1)_SOURCE_ID := packagespec-only-$$($(1)_NAME) +$(1)_SOURCE_ID_NICE_NAME := + +else + +$(1)_SOURCE_GIT = $$(call QUOTE_LIST,$$($(1)_SOURCE_INCLUDE)) $$(call GIT_EXCLUDE_LIST,$$($(1)_SOURCE_EXCLUDE)) +$(1)_SOURCE_COMMIT := $$(shell git rev-list -n1 $(GIT_REF) -- $$($(1)_SOURCE_GIT)) + +# If we allow dirty builds, generate the source ID as a function of the +# source in in the current work tree. Where the source all happens to match a Git commit, +# that commit's SHA will be the source ID. +ifeq ($(ALLOW_DIRTY),YES) + +$(1)_SOURCE_CMD := { { \ + git ls-files -- $$($(1)_SOURCE_GIT); \ + git ls-files -m --exclude-standard -- $$($(1)_SOURCE_GIT); \ + } | sort | uniq; } + +$(1)_SOURCE_MODIFIED := $$(shell git ls-files -m -- $$($(1)_SOURCE_GIT) | xargs) +$(1)_SOURCE_NEW := $$(shell git ls-files -o --exclude-standard -- $$($(1)_SOURCE_GIT) | xargs) +$(1)_SOURCE_DIRTY_LIST := $$(shell echo "$$($(1)_SOURCE_MODIFIED) $$($(1)_SOURCE_NEW)" | xargs) +$(1)_SOURCE_DIRTY_SUM := $$(shell [ -z "$$($(1)_SOURCE_DIRTY_LIST)" ] || cat $$($(1)_SOURCE_DIRTY_LIST) | $(SUM)) + +$(1)_SOURCE_ID := $$(shell if [ -z "$$($(1)_SOURCE_DIRTY_LIST)" ]; then \ + echo "$$($(1)_SOURCE_COMMIT)"; \ + else \ + echo -n "dirty_$$($(1)_SOURCE_DIRTY_SUM)"; \ + fi) + +$(1)_ID_PREFIX := $$(shell [ -z "$$($(1)_SOURCE_DIRTY_LIST)" ] || echo "dirty_") + +$(1)_SOURCE_ID_NICE_NAME := $$($(1)_SOURCE_ID) + +# No dirty builds allowed, so the SOURCE_ID is the git commit SHA, +# and we list files using git ls-tree. +else + +$(1)_SOURCE_ID := $$($(1)_SOURCE_COMMIT) +$(1)_SOURCE_ID_NICE_NAME := $$($(1)_SOURCE_ID) +$(1)_SOURCE_CMD := git ls-tree -r --name-only $(GIT_REF) -- $$($(1)_SOURCE_GIT) + +endif +endif + +# LAYER_ID_CONTENTS dictates all the fields that can cause cache invalidation +# to propagate from the current layer to all dependent layers. +define $(1)_LAYER_ID_CONTENTS +BASE_LAYER_ID=$$($(1)_BASE_LAYER_ID); +LAYER_NAME=$$($(1)_NAME); +SOURCE_ID=$$($(1)_SOURCE_ID); +SOURCE_INCLUDE=$$($(1)_SOURCE_INCLUDE); +SOURCE_EXCLUDE=$$($(1)_SOURCE_EXCLUDE); +endef + +$(1)_LAYER_ID_CONTENTS_FILE := $$($(1)_CACHE_ROOT)/current-layer-id-contents +$(1)_LAYER_ID_FILE := $$($(1)_CACHE_ROOT)/current-layer-id +$(1)_DOCKERFILE := $$($(1)_CACHE_ROOT)/Dockerfile + +# Create cache root dir and write LAYER_ID_FILE_CONTENTS file. +_ := $$(shell \ + mkdir -p $$($(1)_CACHE_ROOT); \ + echo "$$($(1)_LAYER_ID_CONTENTS)" > $$($(1)_LAYER_ID_CONTENTS_FILE); \ +) + +$(1)_LAYER_ID := $$($(1)_ID_PREFIX)$$(shell cat $$($(1)_LAYER_ID_CONTENTS_FILE) | $(SUM)) +$(1)_SOURCE_ARCHIVE := $(CACHE_ROOT)/source-archives/$$($(1)_TYPE)-$$($(1)_LAYER_ID).tar +$(1)_IMAGE_NAME := $(BUILDER_IMAGE_PREFIX)-$$($(1)_NAME):$$($(1)_LAYER_ID) +$(1)_CACHE := $(CACHE_ROOT)/layers/$$($(1)_NAME)/$$($(1)_LAYER_ID) + + +ifeq ($(DEBUG),YES) +$$(info ===== LAYER DEBUG INFO ($(1)) ) +$$(info SOURCE_GIT=$$($(1)_SOURCE_GIT)) +$$(info SOURCE_COMMIT=$$($(1)_SOURCE_COMMIT)) +$$(info SOURCE_MODIFIED=$$($(1)_SOURCE_MODIFIED)) +$$(info SOURCE_NEW=$$($(1)_SOURCE_NEW)) +$$(info SOURCE_DIRTY_LIST=$$($(1)_SOURCE_DIRTY_LIST)) +$$(info SOURCE_DIRTY_SUM=$$($(1)_SOURCE_DIRTY_SUM)) +$$(info SOURCE_ID=$$($(1)_SOURCE_ID)) +$$(info LAYER_ID=$$($(1)_LAYER_ID)) +$$(info SOURCE_LIST=$$(shell $$($(1)_SOURCE_CMD))) +$$(info =====) +endif + + +# Create cache dir and write Layer ID file. +_ := $$(shell \ + mkdir -p $$($(1)_CACHE); \ + echo $$($(1)_LAYER_ID) > $$($(1)_LAYER_ID_FILE); \ +) + +$(1)_PHONY_TARGET_NAMES := debug id image save load + +$(1)_PHONY_TARGETS := $$(addprefix $$($(1)_NAME)-,$$($(1)_PHONY_TARGET_NAMES)) + +.PHONY: $$($(1)_PHONY_TARGETS) + +# File targets. +$(1)_IMAGE := $$($(1)_CACHE)/image.marker +$(1)_LAYER_REFS := $$($(1)_CACHE)/image.layer_refs +$(1)_IMAGE_TIMESTAMP := $$($(1)_CACHE)/image.created_time + +$(1)_TARGETS = $$($(1)_PHONY_TARGETS) + +# UPDATE_MARKER_FILE ensures the image marker file has the same timestamp as the +# docker image creation date it represents. This enables make to only rebuild it when +# it has really changed, especially after loading the image from an archive. +# It also writes a list of all the layers in this docker image's history, for use +# when saving layers out to archives for use in pre-populating Docker build caches. +define $(1)_UPDATE_MARKER_FILE + export MARKER=$$($(1)_IMAGE); \ + export LAYER_REFS=$$($(1)_LAYER_REFS); \ + export IMAGE=$$($(1)_IMAGE_NAME); \ + export IMAGE_CREATED; \ + if ! { IMAGE_CREATED="$$$$(docker inspect -f '{{.Created}}' $$$$IMAGE 2>/dev/null)"; }; then \ + if [ -f "$$$$MARKER" ]; then \ + echo "==> Removing stale marker file for $$$$IMAGE" 1>&2; \ + rm -f $$$$MARKER; \ + fi; \ + exit 0; \ + fi; \ + if [ ! -f "$$$$MARKER" ]; then \ + echo "==> Writing marker file for $$$$IMAGE (created $$$$IMAGE_CREATED)" 1>&2; \ + fi; \ + echo $$$$IMAGE > $$$$MARKER; \ + $(TOUCH) -m -d $$$$IMAGE_CREATED $$$$MARKER; \ + echo "$$$$IMAGE" > $$$$LAYER_REFS; \ + docker history --no-trunc -q $$$$IMAGE | grep -Fv '' >> $$$$LAYER_REFS; +endef + +## PHONY targets +$(1)-debug: + @echo "==> Debug info: $$($(1)_NAME) depends on $$($(1)_BASE)" + @echo "$(1)_TARGETS = $$($(1)_TARGETS)" + @echo "$(1)_SOURCE_CMD = $$($(1)_SOURCE_CMD)" + @echo "$(1)_CACHE = $$($(1)_CACHE)" + @echo "$(1)_DOCKERFILE = $$($(1)_DOCKERFILE)" + @echo "$(1)_SOURCE_COMMIT = $$($(1)_SOURCE_COMMIT)" + @echo "$(1)_SOURCE_ID = $$($(1)_SOURCE_ID)" + @echo "$(1)_SOURCE_MODIFIED = $$($(1)_SOURCE_MODIFIED)" + @echo "$(1)_SOURCE_NEW = $$($(1)_SOURCE_NEW)" + @echo "$(1)_IMAGE = $$($(1)_IMAGE)" + @echo "$(1)_IMAGE_TIMESTAMP = $$($(1)_IMAGE_TIMESTAMP)" + @echo "$(1)_IMAGE_ARCHIVE = $$($(1)_IMAGE_ARCHIVE)" + @echo "$(1)_BASE_IMAGE = $$($(1)_BASE_IMAGE)" + @echo + +$(1)-id: + @echo $(1)-$$($(1)_SOURCE_ID) + +$(1)-write-cache-key: + @mkdir -p $$(dir $$($(1)_CACHE_KEY_FILE)); \ + cp $$($(1)_LAYER_ID_CONTENTS_FILE) $$($(1)_CACHE_KEY_FILE); \ + echo "==> Cache key for $(1) written to $$($(1)_CACHE_KEY_FILE)"; \ + cat $$($(1)_CACHE_KEY_FILE) + +$(1)-image: $$($(1)_IMAGE) + @cat $$< + +$(1)-layer-refs: $$($(1)_LAYER_REFS) + @echo $$< + +$(1)-save: $$($(1)_IMAGE_ARCHIVE) + @echo $$< + +$(1)-load: + @\ + ARCHIVE=$$($(1)_IMAGE_ARCHIVE); \ + IMAGE=$$($(1)_IMAGE_NAME); \ + MARKER=$$($(1)_IMAGE); \ + rm -f $$$$MARKER; \ + echo "==> Loading $$$$IMAGE image from $$$$ARCHIVE"; \ + docker load < $$$$ARCHIVE + @$$(call $(1)_UPDATE_MARKER_FILE) + +## END PHONY targets + +# Set the BASE_IMAGE build arg to reference the appropriate base image, +# unless there is no referenced base image. +$(1)_DOCKER_BUILD_ARGS = $$(shell [ -z "$$($(1)_BASE)" ] || echo --build-arg BASE_IMAGE=$$$$(cat $$($(1)_BASE_IMAGE))) + +$(1)_SOURCE_ARCHIVE_WITH_DOCKERFILE := $$($(1)_CACHE)/source-archive.tar + +$$($(1)_DOCKERFILE): + @mkdir -p "$$(dir $$(@))" + @$$(call QUERY_LOCK,.layers[] | select(.name=="$$($(1)_NAME)").dockerfile) > "$$@" + +# Build the docker image. +# +# For dirty builds, tar up a source archive from the local filesystem. +# We --ignore-failed-read so that deleted files that are not +# committed do not cause problems. This should be OK for dirty builds. +# +# For non-dirty builds, ask Git directly for a source archive. +# +# We explicitly set the TAR format to ustar because this seems more compatible +# with Docker than any other format. In future we should change this to POSIX +# once Docker supports that properly, because ustar only supports filenames +# < 256 chars which could eventually be an issue. +TAR_FORMAT := --format=ustar +export DOCKER_BUILDKIT=1 +$(1)_FULL_DOCKER_BUILD_COMMAND = docker build --ssh=default -t $$($(1)_IMAGE_NAME) $$($(1)_DOCKER_BUILD_ARGS) \ + -f $$($(1)_DOCKERFILE) - < $$($(1)_SOURCE_ARCHIVE_WITH_DOCKERFILE) + +$$($(1)_IMAGE): $$($(1)_BASE_IMAGE) $$($(1)_DOCKERFILE) + @$$(call $(1)_UPDATE_MARKER_FILE) + @if [ -f "$$@" ]; then exit 0; fi; \ + echo "==> Building Docker image $$($(1)_IMAGE_NAME)"; \ + echo " Layer name : $$($(1)_NAME)"; \ + echo " Layer source ID : $$($(1)_SOURCE_ID_NICE_NAME)"; \ + echo " For product revision : $(PRODUCT_REVISION_NICE_NAME)"; \ + echo " For package source ID : $(PACKAGE_SOURCE_ID)"; \ + if [ ! -f "$$($(1)_SOURCE_ARCHIVE)" ]; then \ + if [ "$(ALLOW_DIRTY)" = "YES" ]; then \ + echo "==> Building source archive from working directory: $$($(1)_SOURCE_ARCHIVE)" 1>&2; \ + $$($(1)_SOURCE_CMD) | $(TAR) --create $(TAR_FORMAT) --file $$($(1)_SOURCE_ARCHIVE) --ignore-failed-read -T -; \ + else \ + echo "==> Building source archive from git: $$($(1)_SOURCE_ARCHIVE)" 1>&2; \ + git archive --format=tar $(GIT_REF) $$($(1)_SOURCE_GIT) > $$($(1)_SOURCE_ARCHIVE); \ + fi; \ + fi; \ + if [ ! -f "$$($(1)_SOURCE_ARCHIVE_WITH_DOCKERFILE)" ]; then \ + echo "==> Appending Dockerfile to source archive: $$($(1)_SOURCE_ARCHIVE_WITH_DOCKERFILE)" 1>&2; \ + cp $$($(1)_SOURCE_ARCHIVE) $$($(1)_SOURCE_ARCHIVE_WITH_DOCKERFILE); \ + $(TAR) --append $(TAR_FORMAT) $$($(1)_DOCKERFILE) --file $$($(1)_SOURCE_ARCHIVE_WITH_DOCKERFILE); \ + fi; \ + echo $$($(1)_FULL_DOCKER_BUILD_COMMAND); \ + $$($(1)_FULL_DOCKER_BUILD_COMMAND); \ + $$(call $(1)_UPDATE_MARKER_FILE) + +# Save the docker image as a tar.gz. +$$($(1)_IMAGE_ARCHIVE): | $$($(1)_IMAGE) + @mkdir -p $$(dir $$@); \ + IMAGE=$$$$(cat $$($(1)_IMAGE)); \ + echo "==> Saving $(1) image to $$@"; \ + docker save $$$$IMAGE \ + $$$$(docker history -q --no-trunc $$$$IMAGE | grep -v missing) \ + | gzip > $$@ + +$$($(1)_LAYER_REFS): + @echo "$$($(1)_IMAGE_NAME)" > $$@ + @docker history --no-trunc -q $$($(1)_IMAGE_NAME) | grep -Fv '' >> $$@ + +endef + +### END LAYER + +# Include the generated instructions to build each layer. +include $(LOCKDIR)/layers/layers.mk + +# Eagerly update the docker image marker files. +_ := $(foreach L,$(LAYERS),$(shell $(call $(L)_UPDATE_MARKER_FILE))) + +# DOCKER_LAYER_LIST is used to dump the name of every docker ref in use +# by all of the current builder images. By running 'docker save' against +# this list, we end up with a tarball that can pre-populate the docker +# cache to avoid unnecessary rebuilds. +DOCKER_LAYER_LIST := $(CACHE_ROOT)/docker-layer-list + +write-cache-keys: $(addsuffix -write-cache-key,$(LAYERS)) + @echo "==> All cache keys written." + +build-all-layers: $(addsuffix -image,$(LAYERS)) + @echo "==> All builder layers built." + +.PHONY: debug +debug: $(addsuffix -debug,$(LAYERS)) + diff --git a/packages-oss.lock/layers/layers.mk b/packages-oss.lock/layers/layers.mk new file mode 100755 index 000000000..01153d66f --- /dev/null +++ b/packages-oss.lock/layers/layers.mk @@ -0,0 +1,39 @@ +# *** +# WARNING: Do not EDIT or MERGE this file, it is generated by packagespec. +# *** + +LAYER_00-base-aac1d959b4b3af1495941602b5e51fb6181baded_ID := 00-base-aac1d959b4b3af1495941602b5e51fb6181baded +LAYER_00-base-aac1d959b4b3af1495941602b5e51fb6181baded_TYPE := base +LAYER_00-base-aac1d959b4b3af1495941602b5e51fb6181baded_BASE_LAYER := +LAYER_00-base-aac1d959b4b3af1495941602b5e51fb6181baded_SOURCE_INCLUDE := +LAYER_00-base-aac1d959b4b3af1495941602b5e51fb6181baded_SOURCE_EXCLUDE := +LAYER_00-base-aac1d959b4b3af1495941602b5e51fb6181baded_CACHE_KEY_FILE := .buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded +LAYER_00-base-aac1d959b4b3af1495941602b5e51fb6181baded_ARCHIVE_FILE := .buildcache/archives/00-base-aac1d959b4b3af1495941602b5e51fb6181baded.tar.gz +$(eval $(call LAYER,$(LAYER_00-base-aac1d959b4b3af1495941602b5e51fb6181baded_ID),$(LAYER_00-base-aac1d959b4b3af1495941602b5e51fb6181baded_TYPE),$(LAYER_00-base-aac1d959b4b3af1495941602b5e51fb6181baded_BASE_LAYER),$(LAYER_00-base-aac1d959b4b3af1495941602b5e51fb6181baded_SOURCE_INCLUDE),$(LAYER_00-base-aac1d959b4b3af1495941602b5e51fb6181baded_SOURCE_EXCLUDE),$(LAYER_00-base-aac1d959b4b3af1495941602b5e51fb6181baded_CACHE_KEY_FILE),$(LAYER_00-base-aac1d959b4b3af1495941602b5e51fb6181baded_ARCHIVE_FILE))) + +LAYER_01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d_ID := 01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d +LAYER_01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d_TYPE := ui +LAYER_01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d_BASE_LAYER := 00-base-aac1d959b4b3af1495941602b5e51fb6181baded +LAYER_01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d_SOURCE_INCLUDE := internal/ui/VERSION +LAYER_01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d_SOURCE_EXCLUDE := +LAYER_01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d_CACHE_KEY_FILE := .buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d +LAYER_01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d_ARCHIVE_FILE := .buildcache/archives/01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d.tar.gz +$(eval $(call LAYER,$(LAYER_01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d_ID),$(LAYER_01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d_TYPE),$(LAYER_01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d_BASE_LAYER),$(LAYER_01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d_SOURCE_INCLUDE),$(LAYER_01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d_SOURCE_EXCLUDE),$(LAYER_01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d_CACHE_KEY_FILE),$(LAYER_01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d_ARCHIVE_FILE))) + +LAYER_02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e_ID := 02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e +LAYER_02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e_TYPE := go-modules +LAYER_02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e_BASE_LAYER := 01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d +LAYER_02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e_SOURCE_INCLUDE := go.mod go.sum +LAYER_02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e_SOURCE_EXCLUDE := +LAYER_02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e_CACHE_KEY_FILE := .buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e +LAYER_02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e_ARCHIVE_FILE := .buildcache/archives/02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e.tar.gz +$(eval $(call LAYER,$(LAYER_02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e_ID),$(LAYER_02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e_TYPE),$(LAYER_02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e_BASE_LAYER),$(LAYER_02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e_SOURCE_INCLUDE),$(LAYER_02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e_SOURCE_EXCLUDE),$(LAYER_02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e_CACHE_KEY_FILE),$(LAYER_02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e_ARCHIVE_FILE))) + +LAYER_03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3_ID := 03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3 +LAYER_03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3_TYPE := copy-source +LAYER_03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3_BASE_LAYER := 02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e +LAYER_03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3_SOURCE_INCLUDE := *.go +LAYER_03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3_SOURCE_EXCLUDE := +LAYER_03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3_CACHE_KEY_FILE := .buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3 +LAYER_03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3_ARCHIVE_FILE := .buildcache/archives/03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3.tar.gz +$(eval $(call LAYER,$(LAYER_03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3_ID),$(LAYER_03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3_TYPE),$(LAYER_03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3_BASE_LAYER),$(LAYER_03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3_SOURCE_INCLUDE),$(LAYER_03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3_SOURCE_EXCLUDE),$(LAYER_03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3_CACHE_KEY_FILE),$(LAYER_03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3_ARCHIVE_FILE))) diff --git a/packages-oss.lock/pkgs.yml b/packages-oss.lock/pkgs.yml new file mode 100644 index 000000000..57a3b2e20 --- /dev/null +++ b/packages-oss.lock/pkgs.yml @@ -0,0 +1,322 @@ +# *** +# WARNING: Do not EDIT or MERGE this file, it is generated by packagespec. +# *** +lockid: 748714d3a36e1911 +packagespec-version: 0.1.3 +cache-version: 8 +packages: +- packagespecid: 3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3 + inputs: + BINARY_NAME: boundary + BUILD_TAGS: ui + CGO_ENABLED: "0" + GOARCH: amd64 + GOOS: darwin + PRODUCT_NAME: boundary + PRODUCT_VERSION: 0.1.0-beta.2 + PRODUCT_VERSION_MMP: 0.1.0 + PRODUCT_VERSION_PRE: beta.2 + meta: + BUILD_JOB_NAME: darwin_amd64_package + BUNDLE_NAME: boundary_0.1.0-beta.2 + POST_PROCESSORS: apple-notarize av-scan + builtin: + BUILD_LAYERS: + - type: copy-source + name: 03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3 + archive: .buildcache/archives/03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3.tar.gz + - type: go-modules + name: 02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e + archive: .buildcache/archives/02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e.tar.gz + - type: ui + name: 01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d + archive: .buildcache/archives/01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d.tar.gz + - type: base + name: 00-base-aac1d959b4b3af1495941602b5e51fb6181baded + archive: .buildcache/archives/00-base-aac1d959b4b3af1495941602b5e51fb6181baded.tar.gz + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3 + circleci: + BUILDER_CACHE_KEY_PREFIX_LIST: + - copy-source_dde4_{{checksum ".buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3"}} + - go-modules_694e_{{checksum ".buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e"}} + - ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + - base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + PACKAGE_CACHE_KEY: package-3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3-{{checksum + ".buildcache/cache-keys/package-3f81e804b31cb70e63a9c834931d9a0a4c2e2bb3"}} + build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; + go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID + -X $VERSION_PKG_PATH.Version=0.1.0 -X $VERSION_PKG_PATH.VersionPrerelease=beta.2" + -o $OUTPUT_DIR/boundary ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME + boundary + aliases: + - type: local + path: boundary_0.1.0-beta.2_darwin_amd64.zip + - type: github-releases + path: boundary_0.1.0-beta.2_darwin_amd64.zip +- packagespecid: 26ad460ac09efc9c56bdbb7a962ff92616c06c51 + inputs: + BINARY_NAME: boundary + BUILD_TAGS: ui + CGO_ENABLED: "0" + GOARCH: amd64 + GOOS: linux + PRODUCT_NAME: boundary + PRODUCT_VERSION: 0.1.0-beta.2 + PRODUCT_VERSION_MMP: 0.1.0 + PRODUCT_VERSION_PRE: beta.2 + meta: + BUILD_JOB_NAME: linux_amd64_package + BUNDLE_NAME: boundary_0.1.0-beta.2 + POST_PROCESSORS: av-scan + builtin: + BUILD_LAYERS: + - type: copy-source + name: 03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3 + archive: .buildcache/archives/03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3.tar.gz + - type: go-modules + name: 02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e + archive: .buildcache/archives/02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e.tar.gz + - type: ui + name: 01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d + archive: .buildcache/archives/01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d.tar.gz + - type: base + name: 00-base-aac1d959b4b3af1495941602b5e51fb6181baded + archive: .buildcache/archives/00-base-aac1d959b4b3af1495941602b5e51fb6181baded.tar.gz + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-26ad460ac09efc9c56bdbb7a962ff92616c06c51 + circleci: + BUILDER_CACHE_KEY_PREFIX_LIST: + - copy-source_dde4_{{checksum ".buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3"}} + - go-modules_694e_{{checksum ".buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e"}} + - ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + - base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + PACKAGE_CACHE_KEY: package-26ad460ac09efc9c56bdbb7a962ff92616c06c51-{{checksum + ".buildcache/cache-keys/package-26ad460ac09efc9c56bdbb7a962ff92616c06c51"}} + build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; + go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID + -X $VERSION_PKG_PATH.Version=0.1.0 -X $VERSION_PKG_PATH.VersionPrerelease=beta.2" + -o $OUTPUT_DIR/boundary ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME + boundary + aliases: + - type: local + path: boundary_0.1.0-beta.2_linux_amd64.zip + - type: github-releases + path: boundary_0.1.0-beta.2_linux_amd64.zip +- packagespecid: 6b5bf7794d67cf76b1b4772807af52deb1f6a808 + inputs: + BINARY_NAME: boundary.exe + BUILD_TAGS: ui + CGO_ENABLED: "0" + GOARCH: amd64 + GOOS: windows + PRODUCT_NAME: boundary + PRODUCT_VERSION: 0.1.0-beta.2 + PRODUCT_VERSION_MMP: 0.1.0 + PRODUCT_VERSION_PRE: beta.2 + meta: + BUILD_JOB_NAME: windows_amd64_package + BUNDLE_NAME: boundary_0.1.0-beta.2 + POST_PROCESSORS: microsoft-notarize av-scan + builtin: + BUILD_LAYERS: + - type: copy-source + name: 03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3 + archive: .buildcache/archives/03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3.tar.gz + - type: go-modules + name: 02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e + archive: .buildcache/archives/02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e.tar.gz + - type: ui + name: 01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d + archive: .buildcache/archives/01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d.tar.gz + - type: base + name: 00-base-aac1d959b4b3af1495941602b5e51fb6181baded + archive: .buildcache/archives/00-base-aac1d959b4b3af1495941602b5e51fb6181baded.tar.gz + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-6b5bf7794d67cf76b1b4772807af52deb1f6a808 + circleci: + BUILDER_CACHE_KEY_PREFIX_LIST: + - copy-source_dde4_{{checksum ".buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3"}} + - go-modules_694e_{{checksum ".buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e"}} + - ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + - base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + PACKAGE_CACHE_KEY: package-6b5bf7794d67cf76b1b4772807af52deb1f6a808-{{checksum + ".buildcache/cache-keys/package-6b5bf7794d67cf76b1b4772807af52deb1f6a808"}} + build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; + go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID + -X $VERSION_PKG_PATH.Version=0.1.0 -X $VERSION_PKG_PATH.VersionPrerelease=beta.2" + -o $OUTPUT_DIR/boundary.exe ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME + boundary.exe + aliases: + - type: local + path: boundary_0.1.0-beta.2_windows_amd64.zip + - type: github-releases + path: boundary_0.1.0-beta.2_windows_amd64.zip +base-image: golang@sha256:29452ef08bd0535f550c47b6034522c82ac10517b49c5f3b7a00035ecee0089e +layers: +- depth: 0 + final: false + type: base + id: aac1d959b4b3af1495941602b5e51fb6181baded + name: 00-base-aac1d959b4b3af1495941602b5e51fb6181baded + parentname: "" + parenttype: "" + parentid: "" + cachekeyfile: .buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded + sourceinclude: "" + sourceexclude: "" + archivefile: .buildcache/archives/00-base-aac1d959b4b3af1495941602b5e51fb6181baded.tar.gz + dockerfile: | + # syntax=docker/dockerfile:1.1.7-experimental + FROM golang@sha256:29452ef08bd0535f550c47b6034522c82ac10517b49c5f3b7a00035ecee0089e + COPY . ./ + ENV \ + GOPATH= \ + GOBIN=/usr/local/bin \ + GO111MODULE=on \ + GOPRIVATE=github.com/hashicorp/* \ + GOCACHE=/go/cache \ + CGO_ENABLED=0 + RUN --mount=type=ssh \ + go env && mkdir -p "$GOCACHE" \ + && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg \ + | apt-key add - \ + && echo "deb https://dl.yarnpkg.com/debian/ stable main" \ + | tee /etc/apt/sources.list.d/yarn.list \ + && apt-get update \ + && apt-get install -yq yarn zip \ + && mkdir -m 700 /root/.ssh \ + && touch -m 600 /root/.ssh/known_hosts \ + && ssh-keyscan github.com > /root/.ssh/known_hosts \ + && git config --global url."git@github.com:".insteadOf https://github.com/ \ + && go get github.com/go-bindata/go-bindata/v3/go-bindata \ + && go-bindata --version + meta: + builtin: + LAYER_LIST: + - type: base + name: 00-base-aac1d959b4b3af1495941602b5e51fb6181baded + archive: .buildcache/archives/00-base-aac1d959b4b3af1495941602b5e51fb6181baded.tar.gz + circleci: + CACHE_KEY_PREFIX_LIST: + - base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + childcount: 1 +- depth: 1 + final: false + type: ui + id: 5f82d1f0cbf979ee051aa1ba0f7105483fa6947d + name: 01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d + parentname: 00-base-aac1d959b4b3af1495941602b5e51fb6181baded + parenttype: base + parentid: aac1d959b4b3af1495941602b5e51fb6181baded + cachekeyfile: .buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d + sourceinclude: internal/ui/VERSION + sourceexclude: "" + archivefile: .buildcache/archives/01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d.tar.gz + dockerfile: | + # syntax=docker/dockerfile:1.1.7-experimental + ARG BASE_IMAGE + FROM $BASE_IMAGE + COPY . ./ + RUN --mount=type=ssh \ + SHA="$(head -n1 internal/ui/VERSION | cut -d ' ' -f1)" \ + && DIR=internal/ui/source/boundary-ui \ + && mkdir -p "$(dirname "$DIR")" \ + && git clone https://github.com/hashicorp/boundary-ui "$DIR" \ + && ( cd "$DIR" \ + && git fetch origin "$SHA" \ + && git checkout "$SHA" \ + && yarn install \ + && yarn build:ui:core; ) \ + && TARGET=internal/ui/assets.go \ + && UIDIR="$DIR/ui/core/dist" \ + && go-bindata -fs -o "$TARGET" -pkg ui \ + -prefix "$UIDIR" "$UIDIR" "$UIDIR/assets" \ + && rm -rf "$DIR" + meta: + builtin: + LAYER_LIST: + - type: ui + name: 01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d + archive: .buildcache/archives/01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d.tar.gz + - type: base + name: 00-base-aac1d959b4b3af1495941602b5e51fb6181baded + archive: .buildcache/archives/00-base-aac1d959b4b3af1495941602b5e51fb6181baded.tar.gz + circleci: + CACHE_KEY_PREFIX_LIST: + - ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + - base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + childcount: 1 +- depth: 2 + final: false + type: go-modules + id: 694e8500194e3b187b35bd9a40229f3bd215710e + name: 02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e + parentname: 01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d + parenttype: ui + parentid: 5f82d1f0cbf979ee051aa1ba0f7105483fa6947d + cachekeyfile: .buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e + sourceinclude: go.mod go.sum + sourceexclude: "" + archivefile: .buildcache/archives/02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e.tar.gz + dockerfile: | + # syntax=docker/dockerfile:1.1.7-experimental + ARG BASE_IMAGE + FROM $BASE_IMAGE + COPY . ./ + RUN --mount=type=ssh go mod download -x + meta: + builtin: + LAYER_LIST: + - type: go-modules + name: 02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e + archive: .buildcache/archives/02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e.tar.gz + - type: ui + name: 01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d + archive: .buildcache/archives/01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d.tar.gz + - type: base + name: 00-base-aac1d959b4b3af1495941602b5e51fb6181baded + archive: .buildcache/archives/00-base-aac1d959b4b3af1495941602b5e51fb6181baded.tar.gz + circleci: + CACHE_KEY_PREFIX_LIST: + - go-modules_694e_{{checksum ".buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e"}} + - ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + - base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + childcount: 1 +- depth: 3 + final: true + type: copy-source + id: dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3 + name: 03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3 + parentname: 02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e + parenttype: go-modules + parentid: 694e8500194e3b187b35bd9a40229f3bd215710e + cachekeyfile: .buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3 + sourceinclude: '*.go' + sourceexclude: "" + archivefile: .buildcache/archives/03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3.tar.gz + dockerfile: | + # syntax=docker/dockerfile:1.1.7-experimental + ARG BASE_IMAGE + FROM $BASE_IMAGE + COPY . ./ + RUN go build -v -tags 'ui' -o /dev/null ./cmd/boundary + meta: + builtin: + LAYER_LIST: + - type: copy-source + name: 03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3 + archive: .buildcache/archives/03-copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3.tar.gz + - type: go-modules + name: 02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e + archive: .buildcache/archives/02-go-modules-694e8500194e3b187b35bd9a40229f3bd215710e.tar.gz + - type: ui + name: 01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d + archive: .buildcache/archives/01-ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d.tar.gz + - type: base + name: 00-base-aac1d959b4b3af1495941602b5e51fb6181baded + archive: .buildcache/archives/00-base-aac1d959b4b3af1495941602b5e51fb6181baded.tar.gz + circleci: + CACHE_KEY_PREFIX_LIST: + - copy-source_dde4_{{checksum ".buildcache/cache-keys/copy-source-dde49480ad145a6d7ce0f55c0d23e6f50c04d8d3"}} + - go-modules_694e_{{checksum ".buildcache/cache-keys/go-modules-694e8500194e3b187b35bd9a40229f3bd215710e"}} + - ui_5f82_{{checksum ".buildcache/cache-keys/ui-5f82d1f0cbf979ee051aa1ba0f7105483fa6947d"}} + - base_aac1_{{checksum ".buildcache/cache-keys/base-aac1d959b4b3af1495941602b5e51fb6181baded"}} + childcount: 0 diff --git a/packages-oss.yml b/packages-oss.yml new file mode 100644 index 000000000..910673217 --- /dev/null +++ b/packages-oss.yml @@ -0,0 +1,116 @@ +--- +config: + product-repo: git@github.com:hashicorp/boundary.git + release-repo: git@github.com:hashicorp/boundary-release.git + product-id: github.com/hashicorp/boundary + circleci-project-slug: gh/hashicorp/boundary + circleci-host: circleci.com + +inputs: + defaults: + PRODUCT_NAME: boundary + PRODUCT_VERSION: 0.1.0-beta.2 + BUILD_TAGS: ui + CGO_ENABLED: 0 + + templates: + + BINARY_NAME: '{{.PRODUCT_NAME}}{{if eq .GOOS "windows"}}.exe{{end}}' + + PRODUCT_VERSION_MMP: >- + {{with .PRODUCT_VERSION | strings.SplitN "-" 2}}{{index . 0}}{{end}} + + PRODUCT_VERSION_PRE: >- + {{with .PRODUCT_VERSION | strings.SplitN "-" 2}}{{if gt (len .) 1}}{{index . 1}}{{else}}"''"{{end}}{{end}} + +packages: + - inputs: { GOOS: darwin, GOARCH: amd64 } + - inputs: { GOOS: linux, GOARCH: amd64 } + - inputs: { GOOS: windows, GOARCH: amd64 } + +meta: + defaults: + # No default metadata. + templates: + BUILD_JOB_NAME: >- + {{.GOOS}}_{{.GOARCH}}_package + POST_PROCESSORS: ' + {{- if and (eq .GOOS "darwin") (eq .GOARCH "amd64")}}apple-notarize {{end -}} + {{- if and (eq .GOOS "windows") (eq .GOARCH "amd64")}}microsoft-notarize {{end -}} + av-scan' + BUNDLE_NAME: "boundary_{{.PRODUCT_VERSION}}" + +package-aliases: + - type: local + template: >- + {{.BUNDLE_NAME}}_{{.GOOS}}_{{.GOARCH}}.zip + - type: github-releases + template: >- + {{.BUNDLE_NAME}}_{{.GOOS}}_{{.GOARCH}}.zip +# - type: public-hc-releases +# template: >- +# boundary/{{.BUNDLE_NAME}}/{{.BUNDLE_NAME}}_{{.GOOS}}_{{.GOARCH}}.zip + +# golang@sha256:29452ef08bd0535f550c47b6034522c82ac10517b49c5f3b7a00035ecee0089e = golang:1.15.2-buster +base-image: "golang@sha256:29452ef08bd0535f550c47b6034522c82ac10517b49c5f3b7a00035ecee0089e" + +layers: + - name: base + dockerfile: | + ENV \ + GOPATH= \ + GOBIN=/usr/local/bin \ + GO111MODULE=on \ + GOPRIVATE=github.com/hashicorp/* \ + GOCACHE=/go/cache \ + CGO_ENABLED=0 + RUN --mount=type=ssh \ + go env && mkdir -p "$GOCACHE" \ + && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg \ + | apt-key add - \ + && echo "deb https://dl.yarnpkg.com/debian/ stable main" \ + | tee /etc/apt/sources.list.d/yarn.list \ + && apt-get update \ + && apt-get install -yq yarn zip \ + && mkdir -m 700 /root/.ssh \ + && touch -m 600 /root/.ssh/known_hosts \ + && ssh-keyscan github.com > /root/.ssh/known_hosts \ + && git config --global url."git@github.com:".insteadOf https://github.com/ \ + && go get github.com/go-bindata/go-bindata/v3/go-bindata \ + && go-bindata --version + - name: ui + source-include: internal/ui/VERSION + dockerfile: | + RUN --mount=type=ssh \ + SHA="$(head -n1 internal/ui/VERSION | cut -d ' ' -f1)" \ + && DIR=internal/ui/source/boundary-ui \ + && mkdir -p "$(dirname "$DIR")" \ + && git clone https://github.com/hashicorp/boundary-ui "$DIR" \ + && ( cd "$DIR" \ + && git fetch origin "$SHA" \ + && git checkout "$SHA" \ + && yarn install \ + && yarn build:ui:core; ) \ + && TARGET=internal/ui/assets.go \ + && UIDIR="$DIR/ui/core/dist" \ + && go-bindata -fs -o "$TARGET" -pkg ui \ + -prefix "$UIDIR" "$UIDIR" "$UIDIR/assets" \ + && rm -rf "$DIR" + - name: go-modules + source-include: go.mod go.sum + dockerfile: | + RUN --mount=type=ssh go mod download -x + - name: copy-source + source-include: "*.go" + dockerfile: | + RUN go build -v -tags '{{.BUILD_TAGS}}' -o /dev/null ./cmd/boundary + +build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; + unset GOPATH; + go build -v -tags '{{.BUILD_TAGS}}' + -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID + -X $VERSION_PKG_PATH.Version={{.PRODUCT_VERSION_MMP}} + -X $VERSION_PKG_PATH.VersionPrerelease={{.PRODUCT_VERSION_PRE}}" + -o $OUTPUT_DIR/{{.BINARY_NAME}} + ./cmd/boundary + && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME {{.BINARY_NAME}} diff --git a/packagespec.mk b/packagespec.mk new file mode 100755 index 000000000..f1b1fe086 --- /dev/null +++ b/packagespec.mk @@ -0,0 +1,82 @@ +# *** +# WARNING: Do not EDIT or MERGE this file, it is generated by packagespec. +# *** +# packagespec.mk should be included at the end of your main Makefile, +# it provides hooks into packagespec targets, so you can run them +# from the root of your product repository. +# +# All packagespec-generated make targets assume they are invoked by +# targets in this file, which provides the necessary context for those +# other targets. Therefore, this file is not just for conveninence but +# is currently necessary to the correct functioning of Packagespec. + +# Since this file is included in other Makefiles, which may or may not want +# to use bash with these options, we explicitly set the shell for specific +# targets, in this file, rather than setting the global SHELL variable. +PACKAGESPEC_SHELL := /usr/bin/env bash -euo pipefail -c + +# The RUN macro is used in place of the shell builtin in this file, so that +# we can use the PACKAGESPEC_SHELL rather than the default from the Makefile +# that includes this one. +RUN = $(shell $(PACKAGESPEC_SHELL) '$1') + +# This can be overridden by the calling Makefile to write config to a different path. +PACKAGESPEC_CIRCLECI_CONFIG ?= .circleci/config.yml +PACKAGESPEC_HOOK_POST_CI_CONFIG ?= echo > /dev/null + +SPEC_FILE_PATTERN := packages*.yml +# SPEC is the human-managed description of which packages we are able to build. +SPEC := $(call RUN,find . -mindepth 1 -maxdepth 1 -name '$(SPEC_FILE_PATTERN)') +ifneq ($(words $(SPEC)),1) +$(error Found $(words $(SPEC)) $(SPEC_FILE_PATTERN) files, need exactly 1: $(SPEC)) +endif +SPEC_FILENAME := $(notdir $(SPEC)) +SPEC_MODIFIER := $(SPEC_FILENAME:packages%.yml=%) +# LOCKDIR contains the lockfile and layer files. +LOCKDIR := packages$(SPEC_MODIFIER).lock +LOCKFILE := $(LOCKDIR)/pkgs.yml + +export PACKAGE_SPEC_ID LAYER_SPEC_ID PRODUCT_REVISION PRODUCT_VERSION + +# PASSTHROUGH_TARGETS are convenience aliases for targets defined in $(LOCKDIR)/Makefile +PASSTHROUGH_TARGETS := \ + build package-contents copy-package-contents build-all \ + aliases meta package package-meta \ + build-ci watch-ci \ + stage-config stage \ + list-staged-builds \ + publish-config publish + +.PHONY: $(PASSTHROUGH_TARGETS) + +LOCAL_TARGETS := packages packagespec-circleci-config $(PACKAGESPEC_CIRCLECI_CONFIG) + +# Set the shell for all packagespec targets. +$(PASSTHROUGH_TARGETS) $(LOCAL_TARGETS): SHELL := $(PACKAGESPEC_SHELL) + +$(PASSTHROUGH_TARGETS): + @PRODUCT_REPO_ROOT="$(call RUN,git rev-parse --show-toplevel)" $(MAKE) -C $(LOCKDIR) $@ + +# packages regenerates $(LOCKDIR) from $(SPEC) using packagespec. This is only for +# internal HashiCorp use, as it has dependencies not available externally. +.PHONY: packages +packages: + @command -v packagespec > /dev/null 2>&1 || { \ + echo "Please install packagespec."; \ + echo "Note: packagespec is only available to HashiCorp employees at present."; \ + exit 1; \ + } + @packagespec lock -specfile $(SPEC) -lockdir $(LOCKDIR) + @$(MAKE) $(PACKAGESPEC_CIRCLECI_CONFIG) + +CIRCLECI_PRIMARY_TPL := .packagespec/templates/circleci-primary.yml.tpl + +$(PACKAGESPEC_CIRCLECI_CONFIG): $(LOCKFILE) $(CIRCLECI_PRIMARY_TPL) + @\ + echo "==> Updating $(PACKAGESPEC_CIRCLECI_CONFIG)..."; \ + mkdir -p "$(dir $@)"; \ + cat $< | gomplate -f $(CIRCLECI_PRIMARY_TPL) -d 'lock-file=stdin://?type=application/yaml' > $@; \ + $(PACKAGESPEC_HOOK_POST_CI_CONFIG) + +# This target is needed by packagespec, do not remove. +packagespec-circleci-config: $(PACKAGESPEC_CIRCLECI_CONFIG) diff --git a/scripts/uigen.sh b/scripts/uigen.sh index ec60022c5..cef6046b5 100755 --- a/scripts/uigen.sh +++ b/scripts/uigen.sh @@ -2,21 +2,41 @@ set -e -origdir=$(pwd) -tempdir=$(mktemp -d update-ui-assets.XXXXXX) -ui_commitish=${UI_COMMITISH:-develop} +ui_commitish="${UI_COMMITISH:-develop}" -cd $tempdir -git clone https://github.com/hashicorp/boundary-ui -cd boundary-ui -git fetch origin ${ui_commitish} -git checkout ${ui_commitish} +targetdir="internal/ui" +shafile="${targetdir}/VERSION" +shafileabs="$(pwd)/${shafile}" +tempdir="${targetdir}/source" +uirepodir="${tempdir}/boundary-ui" +mkdir -p "${tempdir}" +echo "*" > "${tempdir}/.gitignore" -docker-compose -f docker-compose-embedding.yml run build +if ! [ -d "${uirepodir}/.git" ]; then + git clone https://github.com/hashicorp/boundary-ui "${uirepodir}" +fi -cd $origdir +( + cd "${uirepodir}" + git reset --hard + git fetch origin "${ui_commitish}" + git checkout "${ui_commitish}" + if ! docker-compose -f docker-compose-embedding.yml run build; then + echo "==> UI build failed." + exit 1 + fi + git log -n1 --pretty=oneline > "${shafileabs}" + echo "# Above commit is used for production builds." >> "${shafileabs}" +) -go-bindata -fs -o internal/ui/assets.go -pkg ui -prefix "${origdir}/${tempdir}/boundary-ui/ui/core/dist" "${origdir}/${tempdir}/boundary-ui/ui/core/dist" "${origdir}/${tempdir}/boundary-ui/ui/core/dist/assets" +uidir="${uirepodir}/ui/core/dist" +target="${targetdir}/assets.go" -rm -rf $tempdir +go-bindata -fs -o "${target}.tmp" -pkg ui -prefix "${uidir}" "${uidir}" "${uidir}/assets" + +printf "// +build ui\n" > "${target}" +cat "${target}.tmp" >> "${target}" +rm "${target}.tmp" + +rm -rf "$tempdir" diff --git a/version/version_base.go b/version/version_base.go index 4bf9169e5..3f02b94d1 100644 --- a/version/version_base.go +++ b/version/version_base.go @@ -8,7 +8,11 @@ var ( // Whether cgo is enabled or not; set at build time CgoEnabled bool - Version = "0.1.0" - VersionPrerelease = "beta1" - VersionMetadata = "" + // Version is set at compile time when using 'make build' or building in CI. + Version = "" + + // VersionPrerelease is also set at compile time, similarly to Version. + VersionPrerelease = "" + + VersionMetadata = "" )