From 8caaf3e0760308bb6bf45dc5624cd2f82e993653 Mon Sep 17 00:00:00 2001 From: Timothy Messier Date: Fri, 10 Sep 2021 09:31:27 -0400 Subject: [PATCH 1/4] refactor(schema): Add support for migration editions This refactors the schema package and subpackages to support editions. An Edition is a collection of sql migrations. A given boundary binary can have multiple migration editions. The `schema.Manager` is updated to apply the necessary migrations in the correct order, leveraging an Edition's priority to determine the order in which the separate Editions are applied. The sql migration files are now nested under a directory for the edition. This also switches from the generated code to using embed, which eliminates the need for the `make migrations` target and the generated code. As such this commit removes the target and generated code. See: https://pkg.go.dev/embed --- Makefile | 7 +- internal/cmd/base/dev.go | 4 +- internal/cmd/commands/database/funcs.go | 14 +- internal/cmd/commands/database/funcs_test.go | 28 +- internal/cmd/commands/server/server.go | 22 +- internal/db/package_registration.go | 6 + internal/db/schema/doc.go | 22 + internal/db/schema/edition.go | 68 + internal/db/schema/edition_test.go | 161 + internal/db/schema/export_test.go | 18 + .../db/schema/internal/edition/edition.go | 136 + .../schema/internal/edition/edition_test.go | 264 + .../major/01_initial.up.sql | 5 + .../0/01-initial.up.sql | 5 + .../0/minor_initial.up.sql | 5 + .../no-major-version/01_initial.up.sql | 5 + .../invalid/no-minor-version/0/initial.up.sql | 5 + .../edition/testdata/one/0/01_initial.up.sql | 5 + .../testdata/three/0/01_initial.up.sql | 5 + .../testdata/three/1/01_add_table.up.sql | 3 + .../testdata/two/0/01_add_table.up.sql | 5 + .../edition/testdata/two/0/02_initial.up.sql | 3 + internal/db/schema/internal/log/doc.go | 3 + internal/db/schema/internal/log/entry.go | 12 + internal/db/schema/internal/log/options.go | 29 + internal/db/schema/internal/postgres/doc.go | 3 + .../internal/postgres/ensure_version_test.go | 111 + .../internal/postgres/error_processing.go | 66 + .../postgres/error_processing_test.go | 115 + .../db/schema/internal/postgres/lock_test.go | 152 + .../internal/postgres/migration_log_test.go | 203 + .../db/schema/internal/postgres/postgres.go | 457 + .../schema/internal/postgres/postgres_test.go | 63 + internal/db/schema/internal/postgres/query.go | 139 + .../db/schema/internal/postgres/state_test.go | 56 + .../schema/internal/postgres/testing_test.go | 35 + .../db/schema/internal/provider/provider.go | 100 + .../schema/internal/provider/provider_test.go | 159 + internal/db/schema/manager.go | 208 +- internal/db/schema/manager_test.go | 397 +- internal/db/schema/migration_log.go | 49 + internal/db/schema/migrations/README.md | 27 - internal/db/schema/migrations/base.go | 66 + .../01_boundary_schema_version.up.sql | 6 + .../base/postgres/02_log_migration.up.sql | 34 + .../db/schema/migrations/generate/Makefile | 9 - .../db/schema/migrations/generate/generate.go | 145 - .../db/schema/migrations/generate/main.go | 5 - internal/db/schema/migrations/oss/oss.go | 17 + internal/db/schema/migrations/oss/oss_test.go | 87 + .../postgres/0/01_domain_types.up.sql | 0 .../{ => oss}/postgres/0/02_oplog.up.sql | 0 .../{ => oss}/postgres/0/03_db.up.sql | 0 .../{ => oss}/postgres/0/06_iam.up.sql | 0 .../{ => oss}/postgres/0/07_auth.up.sql | 0 .../{ => oss}/postgres/0/08_servers.up.sql | 0 .../{ => oss}/postgres/0/11_auth_token.up.sql | 0 .../postgres/0/12_auth_password.up.sql | 0 .../postgres/0/13_auth_password_argon.up.sql | 0 .../postgres/0/14_auth_password_views.up.sql | 0 .../{ => oss}/postgres/0/20_host.up.sql | 0 .../postgres/0/22_static_host.up.sql | 0 .../{ => oss}/postgres/0/30_keys.up.sql | 0 .../{ => oss}/postgres/0/31_keys.up.sql | 0 .../{ => oss}/postgres/0/40_targets.up.sql | 0 .../{ => oss}/postgres/0/41_targets.up.sql | 0 .../{ => oss}/postgres/0/50_session.up.sql | 0 .../{ => oss}/postgres/0/51_connection.up.sql | 0 .../postgres/0/60_wh_domain_types.up.sql | 0 .../postgres/0/62_wh_datetime.up.sql | 0 .../0/65_wh_session_dimensions.up.sql | 0 .../0/66_wh_session_dimensions.up.sql | 0 .../postgres/0/68_wh_session_facts.up.sql | 0 .../postgres/0/69_wh_session_facts.up.sql | 0 .../1/01_server_tags_migrations.up.sql | 0 .../{ => oss}/postgres/1/02_domains.up.sql | 0 .../{ => oss}/postgres/1/03_kms.up.sql | 0 .../postgres/10/01_job_migrations.up.sql | 0 .../{ => oss}/postgres/10/02_sentinels.up.sql | 0 .../postgres/10/03_credential.up.sql | 0 .../postgres/10/04_vault_credential.up.sql | 0 .../{ => oss}/postgres/10/05_target.up.sql | 0 .../{ => oss}/postgres/10/06_session.up.sql | 0 .../{ => oss}/postgres/10/07_sentinels.up.sql | 0 .../postgres/11/01_server_type_enum.up.sql | 0 .../postgres/12/01_timestamp_sub_funcs.up.sql | 0 .../postgres/13/01_oidc_add_claims.up.sql | 0 .../14/01_wh_user_dimension_oidc.up.sql | 0 .../15/01_wh_rename_key_columns.up.sql | 0 .../15/02_wh_rename_host_dimension_org.up.sql | 0 .../16/01_wh_credential_dimension.up.sql | 0 .../16/02_wh_credential_dimension.up.sql | 0 .../16/03_wh_credential_dimension.up.sql | 0 .../16/04_wh_credential_dimension.up.sql | 0 .../16/05_wh_credential_dimension.up.sql | 0 .../migrations/oss/postgres/2/01_log.up.sql | 54 + .../{ => oss}/postgres/2/03_oidc_enm.up.sql | 0 .../{ => oss}/postgres/2/04_oidc.up.sql | 0 .../{ => oss}/postgres/2/05_authtoken.up.sql | 0 .../{ => oss}/postgres/2/06_iam.up.sql | 0 .../{ => oss}/postgres/2/07_iam.up.sql | 4 +- .../{ => oss}/postgres/2/10_auth.up.sql | 0 .../{ => oss}/postgres/2/15_oidc.up.sql | 0 .../{ => oss}/postgres/2/20_pass.up.sql | 0 .../{ => oss}/postgres/3/01_auth.up.sql | 0 .../{ => oss}/postgres/3/02_iam.up.sql | 0 .../{ => oss}/postgres/4/01_iam.up.sql | 0 .../{ => oss}/postgres/5/01_oidc.up.sql | 0 .../{ => oss}/postgres/6/01_oidc.up.sql | 0 .../{ => oss}/postgres/7/01_functions.up.sql | 0 .../{ => oss}/postgres/7/02_plugin.up.sql | 0 .../{ => oss}/postgres/7/03_job.up.sql | 0 .../{ => oss}/postgres/8/01_connection.up.sql | 0 .../postgres/8/08_connection_test.go | 0 .../postgres/9/01_managed_groups.up.sql | 0 .../postgres/9/02_oidc_managed_group.up.sql | 0 .../9/03_oidc_managed_group_member.up.sql | 0 ...4_oidc_managed_group_principal_role.up.sql | 0 .../postgres/9/managed_group_test.go | 0 .../postgres_11_01_test.go} | 54 +- .../postgres_12_01_test.go} | 32 +- .../postgres_14_01_test.go} | 50 +- .../postgres_16_05_test.go} | 83 +- .../postgres_2_07_test.go} | 60 +- .../postgres_2_10_test.go} | 4 +- .../migrations/postgres/2/01_log.up.sql | 50 - internal/db/schema/options.go | 12 +- internal/db/schema/options_test.go | 13 +- internal/db/schema/postgres/postgres.go | 477 - internal/db/schema/postgres/postgres_test.go | 462 - internal/db/schema/postgres/testing.go | 140 - internal/db/schema/postgres_migration.gen.go | 9119 ----------------- internal/db/schema/schema.go | 13 +- internal/db/schema/schema_test.go | 102 +- internal/db/schema/state.go | 106 +- internal/db/schema/state_test.go | 14 - internal/db/schema/statement.go | 50 - internal/db/schema/statement_test.go | 42 - .../schema/testdata/one/0/01_initial.up.sql | 6 + .../schema/testdata/three/0/01_initial.up.sql | 6 + .../schema/testdata/two/0/01_initial.up.sql | 5 + internal/db/schema/testing.go | 54 +- internal/db/testing.go | 2 +- internal/oplog/testing.go | 4 +- 144 files changed, 3469 insertions(+), 11098 deletions(-) create mode 100644 internal/db/package_registration.go create mode 100644 internal/db/schema/doc.go create mode 100644 internal/db/schema/edition.go create mode 100644 internal/db/schema/edition_test.go create mode 100644 internal/db/schema/export_test.go create mode 100644 internal/db/schema/internal/edition/edition.go create mode 100644 internal/db/schema/internal/edition/edition_test.go create mode 100644 internal/db/schema/internal/edition/testdata/invalid/major-version-not-int/major/01_initial.up.sql create mode 100644 internal/db/schema/internal/edition/testdata/invalid/minor-version-invalid-separator/0/01-initial.up.sql create mode 100644 internal/db/schema/internal/edition/testdata/invalid/minor-version-not-int/0/minor_initial.up.sql create mode 100644 internal/db/schema/internal/edition/testdata/invalid/no-major-version/01_initial.up.sql create mode 100644 internal/db/schema/internal/edition/testdata/invalid/no-minor-version/0/initial.up.sql create mode 100644 internal/db/schema/internal/edition/testdata/one/0/01_initial.up.sql create mode 100644 internal/db/schema/internal/edition/testdata/three/0/01_initial.up.sql create mode 100644 internal/db/schema/internal/edition/testdata/three/1/01_add_table.up.sql create mode 100644 internal/db/schema/internal/edition/testdata/two/0/01_add_table.up.sql create mode 100644 internal/db/schema/internal/edition/testdata/two/0/02_initial.up.sql create mode 100644 internal/db/schema/internal/log/doc.go create mode 100644 internal/db/schema/internal/log/entry.go create mode 100644 internal/db/schema/internal/log/options.go create mode 100644 internal/db/schema/internal/postgres/doc.go create mode 100644 internal/db/schema/internal/postgres/ensure_version_test.go create mode 100644 internal/db/schema/internal/postgres/error_processing.go create mode 100644 internal/db/schema/internal/postgres/error_processing_test.go create mode 100644 internal/db/schema/internal/postgres/lock_test.go create mode 100644 internal/db/schema/internal/postgres/migration_log_test.go create mode 100644 internal/db/schema/internal/postgres/postgres.go create mode 100644 internal/db/schema/internal/postgres/postgres_test.go create mode 100644 internal/db/schema/internal/postgres/query.go create mode 100644 internal/db/schema/internal/postgres/state_test.go create mode 100644 internal/db/schema/internal/postgres/testing_test.go create mode 100644 internal/db/schema/internal/provider/provider.go create mode 100644 internal/db/schema/internal/provider/provider_test.go create mode 100644 internal/db/schema/migration_log.go delete mode 100644 internal/db/schema/migrations/README.md create mode 100644 internal/db/schema/migrations/base.go create mode 100644 internal/db/schema/migrations/base/postgres/01_boundary_schema_version.up.sql create mode 100644 internal/db/schema/migrations/base/postgres/02_log_migration.up.sql delete mode 100644 internal/db/schema/migrations/generate/Makefile delete mode 100644 internal/db/schema/migrations/generate/generate.go delete mode 100644 internal/db/schema/migrations/generate/main.go create mode 100644 internal/db/schema/migrations/oss/oss.go create mode 100644 internal/db/schema/migrations/oss/oss_test.go rename internal/db/schema/migrations/{ => oss}/postgres/0/01_domain_types.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/02_oplog.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/03_db.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/06_iam.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/07_auth.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/08_servers.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/11_auth_token.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/12_auth_password.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/13_auth_password_argon.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/14_auth_password_views.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/20_host.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/22_static_host.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/30_keys.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/31_keys.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/40_targets.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/41_targets.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/50_session.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/51_connection.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/60_wh_domain_types.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/62_wh_datetime.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/65_wh_session_dimensions.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/66_wh_session_dimensions.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/68_wh_session_facts.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/0/69_wh_session_facts.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/1/01_server_tags_migrations.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/1/02_domains.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/1/03_kms.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/10/01_job_migrations.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/10/02_sentinels.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/10/03_credential.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/10/04_vault_credential.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/10/05_target.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/10/06_session.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/10/07_sentinels.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/11/01_server_type_enum.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/12/01_timestamp_sub_funcs.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/13/01_oidc_add_claims.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/14/01_wh_user_dimension_oidc.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/15/01_wh_rename_key_columns.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/15/02_wh_rename_host_dimension_org.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/16/01_wh_credential_dimension.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/16/02_wh_credential_dimension.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/16/03_wh_credential_dimension.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/16/04_wh_credential_dimension.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/16/05_wh_credential_dimension.up.sql (100%) create mode 100644 internal/db/schema/migrations/oss/postgres/2/01_log.up.sql rename internal/db/schema/migrations/{ => oss}/postgres/2/03_oidc_enm.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/2/04_oidc.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/2/05_authtoken.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/2/06_iam.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/2/07_iam.up.sql (96%) rename internal/db/schema/migrations/{ => oss}/postgres/2/10_auth.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/2/15_oidc.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/2/20_pass.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/3/01_auth.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/3/02_iam.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/4/01_iam.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/5/01_oidc.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/6/01_oidc.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/7/01_functions.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/7/02_plugin.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/7/03_job.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/8/01_connection.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/8/08_connection_test.go (100%) rename internal/db/schema/migrations/{ => oss}/postgres/9/01_managed_groups.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/9/02_oidc_managed_group.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/9/03_oidc_managed_group_member.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/9/04_oidc_managed_group_principal_role.up.sql (100%) rename internal/db/schema/migrations/{ => oss}/postgres/9/managed_group_test.go (100%) rename internal/db/schema/migrations/{postgres/11/01_server_type_enum_test.go => oss/postgres_11_01_test.go} (78%) rename internal/db/schema/migrations/{postgres/12/01_timestamp_sub_funcs_test.go => oss/postgres_12_01_test.go} (65%) rename internal/db/schema/migrations/{postgres/14/warehouse_user_dim_test.go => oss/postgres_14_01_test.go} (80%) rename internal/db/schema/migrations/{postgres/16/warehouse_credential_dim_test.go => oss/postgres_16_05_test.go} (72%) rename internal/db/schema/migrations/{postgres/2/07_iam_test.go => oss/postgres_2_07_test.go} (71%) rename internal/db/schema/migrations/{postgres/2/10_auth_test.go => oss/postgres_2_10_test.go} (98%) delete mode 100644 internal/db/schema/migrations/postgres/2/01_log.up.sql delete mode 100644 internal/db/schema/postgres/postgres.go delete mode 100644 internal/db/schema/postgres/postgres_test.go delete mode 100644 internal/db/schema/postgres/testing.go delete mode 100644 internal/db/schema/postgres_migration.gen.go delete mode 100644 internal/db/schema/state_test.go delete mode 100644 internal/db/schema/statement.go delete mode 100644 internal/db/schema/statement_test.go create mode 100644 internal/db/schema/testdata/one/0/01_initial.up.sql create mode 100644 internal/db/schema/testdata/three/0/01_initial.up.sql create mode 100644 internal/db/schema/testdata/two/0/01_initial.up.sql diff --git a/Makefile b/Makefile index 14187654c..26d5dd77c 100644 --- a/Makefile +++ b/Makefile @@ -77,10 +77,7 @@ endif perms-table: @go run internal/website/permstable/permstable.go -gen: cleangen proto api cli migrations perms-table fmt - -migrations: - $(MAKE) --environment-overrides -C internal/db/schema/migrations/generate migrations +gen: cleangen proto api cli perms-table fmt ### oplog requires protoc-gen-go v1.20.0 or later # GO111MODULE=on go get -u github.com/golang/protobuf/protoc-gen-go@v1.40 @@ -221,7 +218,7 @@ docker-publish: docker push $(IMAGE_TAG) docker push hashicorp/boundary:latest -.PHONY: api cli tools gen migrations proto website ci-config ci-verify set-ui-version docker docker-build docker-build-dev docker-publish test-database-up test-database-down +.PHONY: api cli tools gen proto website ci-config ci-verify set-ui-version docker docker-build docker-build-dev docker-publish test-database-up test-database-down .NOTPARALLEL: diff --git a/internal/cmd/base/dev.go b/internal/cmd/base/dev.go index 58185f21e..589081f01 100644 --- a/internal/cmd/base/dev.go +++ b/internal/cmd/base/dev.go @@ -65,7 +65,7 @@ func (b *Server) CreateDevDatabase(ctx context.Context, opt ...Option) error { } // Let migrate store manage the dirty bit since dev DBs should be ephemeral anyways. - _, err := schema.MigrateStore(ctx, dialect, url) + _, err := schema.MigrateStore(ctx, schema.Dialect(dialect), url) if err != nil { err = fmt.Errorf("unable to initialize dev database with dialect %s: %w", dialect, err) if c != nil { @@ -78,7 +78,7 @@ func (b *Server) CreateDevDatabase(ctx context.Context, opt ...Option) error { b.DatabaseUrl = url default: // Let migrate store manage the dirty bit since dev DBs should be ephemeral anyways. - if _, err := schema.MigrateStore(ctx, dialect, b.DatabaseUrl); err != nil { + if _, err := schema.MigrateStore(ctx, schema.Dialect(dialect), b.DatabaseUrl); err != nil { err = fmt.Errorf("error initializing store: %w", err) if c != nil { err = multierror.Append(err, c()) diff --git a/internal/cmd/commands/database/funcs.go b/internal/cmd/commands/database/funcs.go index 9438b8725..105c5ea1e 100644 --- a/internal/cmd/commands/database/funcs.go +++ b/internal/cmd/commands/database/funcs.go @@ -29,7 +29,7 @@ func migrateDatabase(ctx context.Context, ui cli.Ui, dialect, u string, initiali ui.Error(fmt.Sprintf("Unable to connect to the database at %q", u)) return noop, 2 } - man, err := schema.NewManager(ctx, dialect, dBase) + man, err := schema.NewManager(ctx, schema.Dialect(dialect), dBase) if err != nil { if errors.Match(errors.T(errors.MigrationLock), err) { ui.Error("Unable to capture a lock on the database.") @@ -53,26 +53,22 @@ func migrateDatabase(ctx context.Context, ui cli.Ui, dialect, u string, initiali ui.Error(fmt.Errorf("Error getting database state: %w", err).Error()) return unlock, 2 } - if initialized && !st.InitializationStarted { + if initialized && !st.Initialized { ui.Output(base.WrapAtLength("Database has not been initialized. Please use 'boundary database init' to initialize the boundary database.")) return unlock, -1 } - if !initialized && st.InitializationStarted { + if !initialized && st.Initialized { ui.Output(base.WrapAtLength("Database has already been initialized. Please use 'boundary database migrate' for any upgrade needs.")) return unlock, -1 } - if st.Dirty { - ui.Error(base.WrapAtLength("Database is in a bad state. Please revert back to the last known good state.")) - return unlock, 2 - } - if err := man.RollForward(ctx); err != nil { + if err := man.ApplyMigrations(ctx); err != nil { ui.Error(fmt.Errorf("Error running database migrations: %w", err).Error()) return unlock, 2 } if base.Format(ui) == "table" { ui.Info("Migrations successfully run.") } - migrationLogs, err := schema.GetMigrationLog(ctx, dBase) + migrationLogs, err := man.GetMigrationLog(ctx) if err != nil { ui.Error(fmt.Errorf("Error retrieving database migration logs: %w", err).Error()) return unlock, 2 diff --git a/internal/cmd/commands/database/funcs_test.go b/internal/cmd/commands/database/funcs_test.go index 6ad2dd1b5..f4b5d2fcf 100644 --- a/internal/cmd/commands/database/funcs_test.go +++ b/internal/cmd/commands/database/funcs_test.go @@ -53,13 +53,11 @@ func TestMigrateDatabase(t *testing.T) { require.NoError(t, err) earlyMigrationVersion := 2000 - oState := schema.TestCloneMigrationStates(t) - nState := schema.TestCreatePartialMigrationState(oState["postgres"], earlyMigrationVersion) - oState["postgres"] = nState - - man, err := schema.NewManager(ctx, dialect, dBase, schema.WithMigrationStates(oState)) + man, err := schema.NewManager(ctx, schema.Dialect(dialect), dBase, schema.WithEditions( + schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": earlyMigrationVersion}), + )) require.NoError(t, err) - require.NoError(t, man.RollForward(ctx)) + require.NoError(t, man.ApplyMigrations(ctx)) return u }, expectedCode: 0, @@ -79,13 +77,11 @@ func TestMigrateDatabase(t *testing.T) { require.NoError(t, err) earlyMigrationVersion := 2000 - oState := schema.TestCloneMigrationStates(t) - nState := schema.TestCreatePartialMigrationState(oState["postgres"], earlyMigrationVersion) - oState["postgres"] = nState - - man, err := schema.NewManager(ctx, dialect, dBase, schema.WithMigrationStates(oState)) + man, err := schema.NewManager(ctx, schema.Dialect(dialect), dBase, schema.WithEditions( + schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": earlyMigrationVersion}), + )) require.NoError(t, err) - require.NoError(t, man.RollForward(ctx)) + require.NoError(t, man.ApplyMigrations(ctx)) return u }, expectedCode: -1, @@ -157,7 +153,7 @@ func TestMigrateDatabase(t *testing.T) { dBase, err := common.SqlOpen(dialect, u) require.NoError(t, err) - man, err := schema.NewManager(ctx, dialect, dBase) + man, err := schema.NewManager(ctx, schema.Dialect(dialect), dBase) require.NoError(t, err) // This is an advisory lock on the DB which is released when the DB session ends. err = man.ExclusiveLock(ctx) @@ -180,7 +176,7 @@ func TestMigrateDatabase(t *testing.T) { dBase, err := common.SqlOpen(dialect, u) require.NoError(t, err) - man, err := schema.NewManager(ctx, dialect, dBase) + man, err := schema.NewManager(ctx, schema.Dialect(dialect), dBase) require.NoError(t, err) // This is an advisory lock on the DB which is released when the DB session ends. err = man.ExclusiveLock(ctx) @@ -220,9 +216,9 @@ func TestVerifyOplogIsEmpty(t *testing.T) { dBase, err := common.SqlOpen(dialect, u) require.NoError(t, err) - man, err := schema.NewManager(ctx, dialect, dBase) + man, err := schema.NewManager(ctx, schema.Dialect(dialect), dBase) require.NoError(t, err) - require.NoError(t, man.RollForward(ctx)) + require.NoError(t, man.ApplyMigrations(ctx)) cmd := InitCommand{Command: base.NewCommand(cli.NewMockUi())} cmd.srv = base.NewServer(&base.Command{UI: cmd.UI}) diff --git a/internal/cmd/commands/server/server.go b/internal/cmd/commands/server/server.go index a2851a79d..f21adf2c1 100644 --- a/internal/cmd/commands/server/server.go +++ b/internal/cmd/commands/server/server.go @@ -425,24 +425,22 @@ func (c *Command) Run(args []string) int { c.UI.Error(fmt.Errorf("Error checking schema state: %w", err).Error()) return base.CommandCliError } - if !ckState.InitializationStarted { + if !ckState.Initialized { c.UI.Error(base.WrapAtLength("The database has not been initialized. Please run 'boundary database init'.")) return base.CommandCliError } - if ckState.Dirty { - c.UI.Error(base.WrapAtLength("Database is in a bad state. Please revert the database into the last known good state.")) - return base.CommandCliError - } - if ckState.BinarySchemaVersion > ckState.DatabaseSchemaVersion { + if !ckState.MigrationsApplied() { + for _, e := range ckState.Editions { + if e.DatabaseSchemaState == schema.Ahead { + c.UI.Error(base.WrapAtLength(fmt.Sprintf("Newer schema version (%s %d) "+ + "than this binary expects. Please use a newer version of the boundary "+ + "binary.", e.Name, e.DatabaseSchemaVersion))) + return base.CommandCliError + } + } c.UI.Error(base.WrapAtLength("Database schema must be updated to use this version. Run 'boundary database migrate' to update the database. NOTE: Boundary does not currently support live migration; ensure all controllers are shut down before running the migration command.")) return base.CommandCliError } - if ckState.BinarySchemaVersion < ckState.DatabaseSchemaVersion { - c.UI.Error(base.WrapAtLength(fmt.Sprintf("Newer schema version (%d) "+ - "than this binary expects. Please use a newer version of the boundary "+ - "binary.", ckState.DatabaseSchemaVersion))) - return base.CommandCliError - } if err := c.verifyKmsSetup(); err != nil { c.UI.Error(base.WrapAtLength("Database is in a bad state. Please revert the database into the last known good state.")) return base.CommandCliError diff --git a/internal/db/package_registration.go b/internal/db/package_registration.go new file mode 100644 index 000000000..3e1eee7e1 --- /dev/null +++ b/internal/db/package_registration.go @@ -0,0 +1,6 @@ +package db + +import ( + // import for init side-effects to include migrations + _ "github.com/hashicorp/boundary/internal/db/schema/migrations/oss" +) diff --git a/internal/db/schema/doc.go b/internal/db/schema/doc.go new file mode 100644 index 000000000..ef67b11b4 --- /dev/null +++ b/internal/db/schema/doc.go @@ -0,0 +1,22 @@ +// Package schema is used to apply sql migrations to modify the state of a +// database instance. It also provides functionality to report on the state of +// a database instance and how it compares with the migration editions in the +// running binary. +// +// This package requires that the migration editions are first registered, +// prior to creating a schema.Manager. This is generally done in an init +// function of another package, i.e.: +// +// //go:embed postgres +// var postgres embed.FS +// +// func init() { +// schema.RegisterEdition("oss", schema.Postgres, postgres, 0) +// } +// +// Then a manager can be created and used to apply the migrations: +// +// m := schema.NewManager(ctx, schema.Postgres, db) +// m.ApplyMigrations() +// +package schema diff --git a/internal/db/schema/edition.go b/internal/db/schema/edition.go new file mode 100644 index 000000000..1aa0899ef --- /dev/null +++ b/internal/db/schema/edition.go @@ -0,0 +1,68 @@ +package schema + +import ( + "embed" + "fmt" + "sync" + + "github.com/hashicorp/boundary/internal/db/schema/internal/edition" +) + +// Dialect is same as edition.Dialect +type Dialect = edition.Dialect + +// Supported dialects. +const ( + Postgres Dialect = "postgres" +) + +var supportedDialects = map[Dialect]struct{}{ + Postgres: struct{}{}, +} + +// type dialects map[Dialect]edition.Editions + +type dialects struct { + sync.Mutex + + m map[Dialect]edition.Editions +} + +var editions = dialects{ + m: make(map[Dialect]edition.Editions), +} + +// RegisterEdition registers an edition for use by the Manager. It will panic if: +// - An unsupported dialect is provided. +// - The same (dialect, name) is registered. +// - The same (dialect, priority) is registered. +func RegisterEdition(name string, dialect Dialect, fs embed.FS, priority int) { + editions.Lock() + defer editions.Unlock() + + if _, ok := supportedDialects[dialect]; !ok { + panic(fmt.Sprintf("unsupported dialect: %s", dialect)) + } + var e edition.Editions + var ok bool + + e, ok = editions.m[dialect] + if !ok { + e = make(edition.Editions, 0) + } + + for _, ee := range e { + if ee.Name == name { + panic(fmt.Sprintf("edition %s with dialect %s already registered", name, dialect)) + } + + if ee.Priority == priority { + panic(fmt.Sprintf("edition %s with dialect %s and priority %d has same priority as edition %s", name, dialect, priority, ee.Name)) + } + } + + e = append(e, edition.New(name, dialect, fs, priority)) + e.Sort() + + editions.m[dialect] = e +} diff --git a/internal/db/schema/edition_test.go b/internal/db/schema/edition_test.go new file mode 100644 index 000000000..79a1dfd44 --- /dev/null +++ b/internal/db/schema/edition_test.go @@ -0,0 +1,161 @@ +package schema_test + +import ( + "embed" + "testing" + + "github.com/hashicorp/boundary/internal/db/schema" + "github.com/stretchr/testify/assert" +) + +type testEdition struct { + name string + dialect schema.Dialect + fs embed.FS + priority int +} + +var ( + //go:embed testdata/one + one embed.FS + + //go:embed testdata/two + two embed.FS + + //go:embed testdata/three + three embed.FS +) + +func TestRegisterEditionPanics(t *testing.T) { + tests := []struct { + name string + editions []testEdition + }{ + { + "unsupportedDialect", + []testEdition{ + { + "one", + schema.Dialect("mongodb"), + one, + 0, + }, + }, + }, + { + "duplicateName", + []testEdition{ + { + "one", + schema.Postgres, + one, + 0, + }, + { + "one", + schema.Postgres, + one, + 1, + }, + }, + }, + { + "duplicatePriority", + []testEdition{ + { + "one", + schema.Postgres, + one, + 0, + }, + { + "two", + schema.Postgres, + two, + 0, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Panics(t, func() { + for _, e := range tt.editions { + schema.RegisterEdition(e.name, e.dialect, e.fs, e.priority) + } + }, tt.name) + }) + } +} + +func TestRegisterEdition(t *testing.T) { + tests := []struct { + name string + editions []testEdition + }{ + { + "singleEdition", + []testEdition{ + { + "one", + schema.Postgres, + one, + 0, + }, + }, + }, + { + "twoEditions", + []testEdition{ + { + "one", + schema.Postgres, + one, + 0, + }, + { + "two", + schema.Postgres, + one, + 1, + }, + }, + }, + { + "threeEditions", + []testEdition{ + { + "one", + schema.Postgres, + one, + 0, + }, + { + "two", + schema.Postgres, + one, + 1, + }, + { + "three", + schema.Postgres, + one, + 3, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.NotPanics(t, func() { + schema.TestClearEditions(t) + for _, e := range tt.editions { + schema.RegisterEdition(e.name, e.dialect, e.fs, e.priority) + } + }, tt.name) + }) + } + schema.TestClearEditions(t) +} diff --git a/internal/db/schema/export_test.go b/internal/db/schema/export_test.go new file mode 100644 index 000000000..47aaec2c2 --- /dev/null +++ b/internal/db/schema/export_test.go @@ -0,0 +1,18 @@ +package schema + +import ( + "testing" + + "github.com/hashicorp/boundary/internal/db/schema/internal/edition" +) + +const NilVersion = nilVersion + +// TestClearEditions is a test helper to reset the editions map. +func TestClearEditions(t *testing.T) { + t.Helper() + editions.Lock() + defer editions.Unlock() + + editions.m = make(map[edition.Dialect]edition.Editions) +} diff --git a/internal/db/schema/internal/edition/edition.go b/internal/db/schema/internal/edition/edition.go new file mode 100644 index 000000000..ba84d5c9f --- /dev/null +++ b/internal/db/schema/internal/edition/edition.go @@ -0,0 +1,136 @@ +// Package edition provides internal structs for the schema package for +// defining and organizing database migration editions. +package edition + +import ( + "embed" + "fmt" + "io/fs" + "path/filepath" + "sort" + "strconv" + "strings" +) + +// Dialect is a specific SQL language variant. This generally is the same as +// a specific SQL server implementation. +type Dialect string + +func (d Dialect) String() string { + return string(d) +} + +// Edition is a collection of sql statements along with a version number. +// It is used to apply changes to the database instance. +type Edition struct { + // The identifier of an edition. The Name and Dialect pair should be unique. + Name string + // The specific SQL language variant that the Migrtions are for. The Name + // and Dialect pair should be unique. + Dialect Dialect + + // The latest version of the schema for this edition that this binary supports. + LatestVersion int + + // The set of migrations that should be applied to a database to reach the latest version. + // This is a map of schema versions to sql. + Migrations map[int][]byte + + // Priority is used to determine the order that multiple Editions should be applied. + Priority int +} + +// Editions is a collection of Edition +type Editions []Edition + +// Sort orders the Editions by priority. +func (e Editions) Sort() { + sort.SliceStable(e, func(i, j int) bool { + return e[i].Priority < e[j].Priority + }) +} + +// New creates an Edition with the provided parameters. The embed.FS m will be +// walked to extract the sql statements. The priority is used to determine +// when this Edition's migrations are applied relative to other Editions. A +// lower number indicates a higher priority. New will panic if the structure +// of the embed.FS is not correct. The files must be structured as follows: +// +// / +// _.up.sql +// +// Where majorVersion and minorVersion are integers. There can be any number of +// leading directories prior to the major versions. For example a directory +// structure like the following is correct: +// +// migrations/oss/postgres/ +// 0/ +// 01_initial.up.sql +// 1/ +// 01_add_columns.up.sql +// 02_rename_table.up.sql +// 2/ +// 01_add_new_table.up.sql +// 02_refactor_views.up.sql +func New(name string, dialect Dialect, m embed.FS, priority int) Edition { + var largestSchemaVersion int + migrations := make(map[int][]byte) + + fs.WalkDir(m, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + panic(fmt.Sprintf("unable to process migration files: %s", err)) + } + + if d.IsDir() { + return nil + } + + if !strings.HasSuffix(path, ".up.sql") { + return nil + } + + dir, file := filepath.Split(path) + verMajorDir := filepath.Base(dir) + + verMajor, err := strconv.Atoi(verMajorDir) + if err != nil { + panic(fmt.Sprintf("migration file does not have valid major version directory: %s", path)) + } + + verMinor, err := strconv.Atoi(strings.SplitN(file, "_", 2)[0]) + if err != nil { + panic(fmt.Sprintf("migration file does not have valid minor version prefix: %s", path)) + } + + fullV := (verMajor * 1000) + verMinor + if fullV > largestSchemaVersion { + largestSchemaVersion = fullV + } + + cbts, err := m.ReadFile(path) + if err != nil { + panic(fmt.Sprintf("unable to read migration file: %s", path)) + } + + contents := strings.TrimSpace(string(cbts)) + if strings.ToLower(contents[:len("begin;")]) == "begin;" { + contents = contents[len("begin;"):] + } + if strings.ToLower(contents[len(contents)-len("commit;"):]) == "commit;" { + contents = contents[:len(contents)-len("commit;")] + } + contents = strings.TrimSpace(contents) + + migrations[fullV] = []byte(contents) + + return nil + }) + + return Edition{ + Name: name, + Dialect: dialect, + LatestVersion: largestSchemaVersion, + Migrations: migrations, + Priority: priority, + } +} diff --git a/internal/db/schema/internal/edition/edition_test.go b/internal/db/schema/internal/edition/edition_test.go new file mode 100644 index 000000000..d0e6810f4 --- /dev/null +++ b/internal/db/schema/internal/edition/edition_test.go @@ -0,0 +1,264 @@ +package edition_test + +import ( + "embed" + "testing" + + "github.com/hashicorp/boundary/internal/db/schema/internal/edition" + "github.com/stretchr/testify/assert" +) + +// valid embed.FS +var ( + //go:embed testdata/one + one embed.FS + //go:embed testdata/two + two embed.FS + //go:embed testdata/three + three embed.FS +) + +func TestNew(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + priority int + fs embed.FS + expectedVersion int + expectedMigrationCount int + }{ + { + "one", + 0, + one, + 1, + 1, + }, + { + "two", + 1, + two, + 2, + 2, + }, + { + "three", + 3, + three, + 1001, + 2, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := edition.New(tt.name, edition.Dialect("postgres"), tt.fs, tt.priority) + assert.Equal(t, e.Name, tt.name, "Name") + assert.Equal(t, e.Dialect, edition.Dialect("postgres"), "Dialect") + assert.Equal(t, e.LatestVersion, tt.expectedVersion, "Version") + assert.Equal(t, e.Priority, tt.priority, "Priority") + assert.Equal(t, len(e.Migrations), tt.expectedMigrationCount, "Number of migrations") + }) + } +} + +// invalid embed.FS +var ( + //go:embed testdata/invalid/major-version-not-int + majorVersionNotInt embed.FS + //go:embed testdata/invalid/minor-version-invalid-separator + minorVersionInvalidSeparator embed.FS + //go:embed testdata/invalid/minor-version-not-int + minorVersionNotInt embed.FS + //go:embed testdata/invalid/no-minor-version + noMinorVersion embed.FS + //go:embed testdata/invalid/no-major-version + noMajorVersion embed.FS +) + +func TestNewPanics(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + fs embed.FS + }{ + { + "majorVersionNotInt", + majorVersionNotInt, + }, + { + "minorVersionInvalidSeparator", + minorVersionInvalidSeparator, + }, + { + "minorVersionNotInt", + minorVersionNotInt, + }, + { + "noMinorVersion", + noMinorVersion, + }, + { + "noMajorVersion", + noMajorVersion, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Panics(t, func() { + edition.New(tt.name, edition.Dialect("postgres"), tt.fs, 0) + }, tt.name) + }) + } +} + +func TestSort(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + editions edition.Editions + sorted edition.Editions + }{ + { + "unsortedConsecutive", + edition.Editions{ + { + Name: "two", + Priority: 2, + }, + { + Name: "zero", + Priority: 0, + }, + { + Name: "one", + Priority: 1, + }, + }, + edition.Editions{ + { + Name: "zero", + Priority: 0, + }, + { + Name: "one", + Priority: 1, + }, + { + Name: "two", + Priority: 2, + }, + }, + }, + { + "unsorted", + edition.Editions{ + { + Name: "three", + Priority: 3, + }, + { + Name: "zero", + Priority: 0, + }, + { + Name: "one", + Priority: 1, + }, + }, + edition.Editions{ + { + Name: "zero", + Priority: 0, + }, + { + Name: "one", + Priority: 1, + }, + { + Name: "three", + Priority: 3, + }, + }, + }, + { + "equalPriority", + edition.Editions{ + { + Name: "three-also", + Priority: 3, + }, + { + Name: "three", + Priority: 3, + }, + { + Name: "zero", + Priority: 0, + }, + { + Name: "one", + Priority: 1, + }, + }, + edition.Editions{ + { + Name: "zero", + Priority: 0, + }, + { + Name: "one", + Priority: 1, + }, + { + Name: "three-also", + Priority: 3, + }, + { + Name: "three", + Priority: 3, + }, + }, + }, + { + "alreadySorted", + edition.Editions{ + { + Name: "zero", + Priority: 0, + }, + { + Name: "one", + Priority: 1, + }, + { + Name: "two", + Priority: 2, + }, + }, + edition.Editions{ + { + Name: "zero", + Priority: 0, + }, + { + Name: "one", + Priority: 1, + }, + { + Name: "two", + Priority: 2, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.editions.Sort() + assert.Equal(t, tt.editions, tt.sorted) + }) + } +} diff --git a/internal/db/schema/internal/edition/testdata/invalid/major-version-not-int/major/01_initial.up.sql b/internal/db/schema/internal/edition/testdata/invalid/major-version-not-int/major/01_initial.up.sql new file mode 100644 index 000000000..cda8bcced --- /dev/null +++ b/internal/db/schema/internal/edition/testdata/invalid/major-version-not-int/major/01_initial.up.sql @@ -0,0 +1,5 @@ +begin; + create table one ( + id bigint primary key + ); +commit; diff --git a/internal/db/schema/internal/edition/testdata/invalid/minor-version-invalid-separator/0/01-initial.up.sql b/internal/db/schema/internal/edition/testdata/invalid/minor-version-invalid-separator/0/01-initial.up.sql new file mode 100644 index 000000000..cda8bcced --- /dev/null +++ b/internal/db/schema/internal/edition/testdata/invalid/minor-version-invalid-separator/0/01-initial.up.sql @@ -0,0 +1,5 @@ +begin; + create table one ( + id bigint primary key + ); +commit; diff --git a/internal/db/schema/internal/edition/testdata/invalid/minor-version-not-int/0/minor_initial.up.sql b/internal/db/schema/internal/edition/testdata/invalid/minor-version-not-int/0/minor_initial.up.sql new file mode 100644 index 000000000..cda8bcced --- /dev/null +++ b/internal/db/schema/internal/edition/testdata/invalid/minor-version-not-int/0/minor_initial.up.sql @@ -0,0 +1,5 @@ +begin; + create table one ( + id bigint primary key + ); +commit; diff --git a/internal/db/schema/internal/edition/testdata/invalid/no-major-version/01_initial.up.sql b/internal/db/schema/internal/edition/testdata/invalid/no-major-version/01_initial.up.sql new file mode 100644 index 000000000..cda8bcced --- /dev/null +++ b/internal/db/schema/internal/edition/testdata/invalid/no-major-version/01_initial.up.sql @@ -0,0 +1,5 @@ +begin; + create table one ( + id bigint primary key + ); +commit; diff --git a/internal/db/schema/internal/edition/testdata/invalid/no-minor-version/0/initial.up.sql b/internal/db/schema/internal/edition/testdata/invalid/no-minor-version/0/initial.up.sql new file mode 100644 index 000000000..cda8bcced --- /dev/null +++ b/internal/db/schema/internal/edition/testdata/invalid/no-minor-version/0/initial.up.sql @@ -0,0 +1,5 @@ +begin; + create table one ( + id bigint primary key + ); +commit; diff --git a/internal/db/schema/internal/edition/testdata/one/0/01_initial.up.sql b/internal/db/schema/internal/edition/testdata/one/0/01_initial.up.sql new file mode 100644 index 000000000..cda8bcced --- /dev/null +++ b/internal/db/schema/internal/edition/testdata/one/0/01_initial.up.sql @@ -0,0 +1,5 @@ +begin; + create table one ( + id bigint primary key + ); +commit; diff --git a/internal/db/schema/internal/edition/testdata/three/0/01_initial.up.sql b/internal/db/schema/internal/edition/testdata/three/0/01_initial.up.sql new file mode 100644 index 000000000..3ea8d2cc1 --- /dev/null +++ b/internal/db/schema/internal/edition/testdata/three/0/01_initial.up.sql @@ -0,0 +1,5 @@ +begin; + create table three ( + id bigint primary key + ); +commit; diff --git a/internal/db/schema/internal/edition/testdata/three/1/01_add_table.up.sql b/internal/db/schema/internal/edition/testdata/three/1/01_add_table.up.sql new file mode 100644 index 000000000..8bc37b80c --- /dev/null +++ b/internal/db/schema/internal/edition/testdata/three/1/01_add_table.up.sql @@ -0,0 +1,3 @@ +begin; + alter table tree add column name text; +commit; diff --git a/internal/db/schema/internal/edition/testdata/two/0/01_add_table.up.sql b/internal/db/schema/internal/edition/testdata/two/0/01_add_table.up.sql new file mode 100644 index 000000000..f30983711 --- /dev/null +++ b/internal/db/schema/internal/edition/testdata/two/0/01_add_table.up.sql @@ -0,0 +1,5 @@ +begin; + create table two ( + id bigint primary key + ); +commit; diff --git a/internal/db/schema/internal/edition/testdata/two/0/02_initial.up.sql b/internal/db/schema/internal/edition/testdata/two/0/02_initial.up.sql new file mode 100644 index 000000000..fb283e1ec --- /dev/null +++ b/internal/db/schema/internal/edition/testdata/two/0/02_initial.up.sql @@ -0,0 +1,3 @@ +begin; + alter table two add column name text; +commit; diff --git a/internal/db/schema/internal/log/doc.go b/internal/db/schema/internal/log/doc.go new file mode 100644 index 000000000..22035e0ef --- /dev/null +++ b/internal/db/schema/internal/log/doc.go @@ -0,0 +1,3 @@ +// Package log provides internal structs and options for the schema package for +// tracking logs generated when applying migrations. +package log diff --git a/internal/db/schema/internal/log/entry.go b/internal/db/schema/internal/log/entry.go new file mode 100644 index 000000000..eecfa58ee --- /dev/null +++ b/internal/db/schema/internal/log/entry.go @@ -0,0 +1,12 @@ +package log + +import "time" + +// Entry represents a log entry generated during migrations. +type Entry struct { + Id int + MigrationEdition string + MigrationVersion int + CreateTime time.Time + Entry string +} diff --git a/internal/db/schema/internal/log/options.go b/internal/db/schema/internal/log/options.go new file mode 100644 index 000000000..4005e352a --- /dev/null +++ b/internal/db/schema/internal/log/options.go @@ -0,0 +1,29 @@ +package log + +// GetOpts - iterate the inbound Options and return a struct. +func GetOpts(opt ...Option) Options { + opts := getDefaultOptions() + for _, o := range opt { + o(&opts) + } + return opts +} + +// Option - how Options are passed as arguments. +type Option func(*Options) + +// Options = how options are represented +type Options struct { + WithDeleteLog bool +} + +func getDefaultOptions() Options { + return Options{} +} + +// WithDeleteLog provides an option to specify the deletion of log entries. +func WithDeleteLog(del bool) Option { + return func(o *Options) { + o.WithDeleteLog = del + } +} diff --git a/internal/db/schema/internal/postgres/doc.go b/internal/db/schema/internal/postgres/doc.go new file mode 100644 index 000000000..e4c595570 --- /dev/null +++ b/internal/db/schema/internal/postgres/doc.go @@ -0,0 +1,3 @@ +// Package postgres provides an implementation of the schema.driver interface +// for a PostgreSQL database. +package postgres diff --git a/internal/db/schema/internal/postgres/ensure_version_test.go b/internal/db/schema/internal/postgres/ensure_version_test.go new file mode 100644 index 000000000..80d7ec4c2 --- /dev/null +++ b/internal/db/schema/internal/postgres/ensure_version_test.go @@ -0,0 +1,111 @@ +package postgres_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEnsureVersion(t *testing.T) { + ctx := context.Background() + p, _, _ := setup(ctx, t) + + err := p.StartRun(ctx) + require.NoError(t, err) + + err = p.EnsureVersionTable(ctx) + require.NoError(t, err) + + err = p.CommitRun(ctx) + require.NoError(t, err) +} + +func TestEnsureVersion_NoTxn(t *testing.T) { + ctx := context.Background() + p, _, _ := setup(ctx, t) + err := p.EnsureVersionTable(ctx) + require.Error(t, err) +} + +func TestEnsureVersion_UpdateForEditionSupport(t *testing.T) { + ctx := context.Background() + p, db, _ := setup(ctx, t) + + _, err := db.ExecContext(ctx, ` + create table boundary_schema_version ( + version bigint primary key, + dirty boolean not null + );`) + require.NoError(t, err) + _, err = db.ExecContext(ctx, ` + insert into boundary_schema_version + (version, dirty) + values + (1001, false);`) + require.NoError(t, err) + + err = p.StartRun(ctx) + require.NoError(t, err) + + err = p.EnsureVersionTable(ctx) + require.NoError(t, err) + + err = p.CommitRun(ctx) + require.NoError(t, err) + + var edition string + var version int + + err = db.QueryRowContext( + ctx, + `select edition, version from boundary_schema_version`, + ).Scan(&edition, &version) + + require.NoError(t, err) + + assert.Equal(t, edition, "oss") + assert.Equal(t, version, 1001) +} + +func TestEnsureVersion_RenameOldSchemaTable(t *testing.T) { + ctx := context.Background() + p, db, _ := setup(ctx, t) + + _, err := db.ExecContext(ctx, ` + create table schema_migrations ( + version bigint primary key, + dirty boolean not null + );`) + require.NoError(t, err) + require.NoError(t, err) + _, err = db.ExecContext(ctx, ` + insert into schema_migrations + (version, dirty) + values + (1001, false);`) + require.NoError(t, err) + + err = p.StartRun(ctx) + require.NoError(t, err) + + err = p.EnsureVersionTable(ctx) + require.NoError(t, err) + + err = p.CommitRun(ctx) + require.NoError(t, err) + + var edition string + var version int + + err = db.QueryRowContext( + ctx, + `select edition, version from boundary_schema_version`, + ).Scan(&edition, &version) + + require.NoError(t, err) + + assert.Equal(t, edition, "oss") + assert.Equal(t, version, 1001) +} diff --git a/internal/db/schema/internal/postgres/error_processing.go b/internal/db/schema/internal/postgres/error_processing.go new file mode 100644 index 000000000..67465d4a9 --- /dev/null +++ b/internal/db/schema/internal/postgres/error_processing.go @@ -0,0 +1,66 @@ +// The MIT License (MIT) +// +// Original Work +// Copyright (c) 2016 Matthias Kadenbach +// https://github.com/mattes/migrate +// +// Modified Work +// Copyright (c) 2018 Dale Hui +// https://github.com/golang-migrate/migrate +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package postgres + +import "strings" + +const newLine = '\n' + +func runesCount(input []rune, target rune) int { + var count int + for _, r := range input { + if r == target { + count++ + } + } + return count +} + +func runesLastIndex(input []rune, target rune) int { + for i := len(input) - 1; i >= 0; i-- { + if input[i] == target { + return i + } + } + return -1 +} + +func computeLineFromPos(s string, pos int) (line uint, col uint, ok bool) { + // replace crlf with lf + s = strings.Replace(s, "\r\n", "\n", -1) + // pg docs: pos uses index 1 for the first character, and positions are measured in characters not bytes + runes := []rune(s) + if pos > len(runes) { + return 0, 0, false + } + sel := runes[:pos] + line = uint(runesCount(sel, newLine) + 1) + col = uint(pos - 1 - runesLastIndex(sel, newLine)) + return line, col, true +} diff --git a/internal/db/schema/internal/postgres/error_processing_test.go b/internal/db/schema/internal/postgres/error_processing_test.go new file mode 100644 index 000000000..d60b65133 --- /dev/null +++ b/internal/db/schema/internal/postgres/error_processing_test.go @@ -0,0 +1,115 @@ +// The MIT License (MIT) +// +// Original Work +// Copyright (c) 2016 Matthias Kadenbach +// https://github.com/mattes/migrate +// +// Modified Work +// Copyright (c) 2018 Dale Hui +// https://github.com/golang-migrate/migrate +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package postgres + +import ( + "strconv" + "strings" + "testing" +) + +func Test_computeLineFromPos(t *testing.T) { + testcases := []struct { + pos int + wantLine uint + wantCol uint + input string + wantOk bool + }{ + { + 15, 2, 6, "SELECT *\nFROM foo", true, // foo table does not exists + }, + { + 16, 3, 6, "SELECT *\n\nFROM foo", true, // foo table does not exists, empty line + }, + { + 25, 3, 7, "SELECT *\nFROM foo\nWHERE x", true, // x column error + }, + { + 27, 5, 7, "SELECT *\n\nFROM foo\n\nWHERE x", true, // x column error, empty lines + }, + { + 10, 2, 1, "SELECT *\nFROMM foo", true, // FROMM typo + }, + { + 11, 3, 1, "SELECT *\n\nFROMM foo", true, // FROMM typo, empty line + }, + { + 17, 2, 8, "SELECT *\nFROM foo", true, // last character + }, + { + 18, 0, 0, "SELECT *\nFROM foo", false, // invalid position + }, + } + for i, tc := range testcases { + t.Run("tc"+strconv.Itoa(i), func(t *testing.T) { + run := func(crlf bool, nonASCII bool) { + var name string + if crlf { + name = "crlf" + } else { + name = "lf" + } + if nonASCII { + name += "-nonascii" + } else { + name += "-ascii" + } + t.Run(name, func(t *testing.T) { + input := tc.input + if crlf { + input = strings.Replace(input, "\n", "\r\n", -1) + } + if nonASCII { + input = strings.Replace(input, "FROM", "FRÖM", -1) + } + gotLine, gotCol, gotOK := computeLineFromPos(input, tc.pos) + + if tc.wantOk { + t.Logf("pos %d, want %d:%d, %#v", tc.pos, tc.wantLine, tc.wantCol, input) + } + + if gotOK != tc.wantOk { + t.Fatalf("expected ok %v but got %v", tc.wantOk, gotOK) + } + if gotLine != tc.wantLine { + t.Fatalf("expected line %d but got %d", tc.wantLine, gotLine) + } + if gotCol != tc.wantCol { + t.Fatalf("expected col %d but got %d", tc.wantCol, gotCol) + } + }) + } + run(false, false) + run(true, false) + run(false, true) + run(true, true) + }) + } +} diff --git a/internal/db/schema/internal/postgres/lock_test.go b/internal/db/schema/internal/postgres/lock_test.go new file mode 100644 index 000000000..e6b9f5d7e --- /dev/null +++ b/internal/db/schema/internal/postgres/lock_test.go @@ -0,0 +1,152 @@ +package postgres_test + +import ( + "context" + "testing" + "time" + + "github.com/hashicorp/boundary/internal/db/common" + "github.com/hashicorp/boundary/internal/db/schema/internal/postgres" + "github.com/hashicorp/boundary/testing/dbtest" + "github.com/stretchr/testify/require" +) + +// Tests that TrySharedLock: +// - can get a lock as the only connection +// - a second connection can also get the shared lock +// - a second connection cannot get an exclusive lock +func TestTrySharedLock(t *testing.T) { + t.Parallel() + ctx := context.Background() + p, _, u := setup(ctx, t) + + err := p.TrySharedLock(ctx) + require.NoError(t, err) + + d2, err := common.SqlOpen(dbtest.Postgres, u) + p2, err := postgres.New(ctx, d2) + require.NoError(t, err) + + err = p2.TrySharedLock(ctx) + require.NoError(t, err) + + err = p2.TryLock(ctx) + require.Error(t, err) +} + +// Test that TryLock gets an exclusive lock and: +// - a separate connection cannot get the lock via TryLock +// - a separate connection cannot get the lock via TrySharedLock +// - a separate connection cannot get the lock via Lock +func TestTryLock(t *testing.T) { + t.Parallel() + ctx := context.Background() + p, _, u := setup(ctx, t) + + err := p.TryLock(ctx) + require.NoError(t, err) + + d2, err := common.SqlOpen(dbtest.Postgres, u) + p2, err := postgres.New(ctx, d2) + require.NoError(t, err) + + err = p2.TrySharedLock(ctx) + require.Error(t, err) + + err = p2.TryLock(ctx) + require.Error(t, err) + + ctx2, _ := context.WithTimeout(ctx, time.Millisecond) + err = p2.Lock(ctx2) + require.Error(t, err) +} + +// Test that Lock gets an exclusive lock and: +// - a separate connection cannot get the lock via TryLock +// - a separate connection cannot get the lock via TrySharedLock +// - a separate connection cannot get the lock via Lock +func TestLock(t *testing.T) { + t.Parallel() + ctx := context.Background() + p, _, u := setup(ctx, t) + + err := p.Lock(ctx) + require.NoError(t, err) + + d2, err := common.SqlOpen(dbtest.Postgres, u) + p2, err := postgres.New(ctx, d2) + require.NoError(t, err) + + err = p2.TrySharedLock(ctx) + require.Error(t, err) + + err = p2.TryLock(ctx) + require.Error(t, err) + + ctx2, _ := context.WithTimeout(ctx, time.Millisecond) + err = p2.Lock(ctx2) + require.Error(t, err) +} + +// Test the Unlock: +// - can unlock a lock granted via TryLock +// - can unlock a lock granted via Lock +func TestUnlock(t *testing.T) { + t.Parallel() + ctx := context.Background() + p, _, u := setup(ctx, t) + + err := p.Lock(ctx) + require.NoError(t, err) + + d2, err := common.SqlOpen(dbtest.Postgres, u) + p2, err := postgres.New(ctx, d2) + require.NoError(t, err) + + // cannot get lock since p has it. + err = p2.TryLock(ctx) + require.Error(t, err) + + // clear lock on p + err = p.Unlock(ctx) + require.NoError(t, err) + + // p2 can now get lock + err = p2.TryLock(ctx) + require.NoError(t, err) + + // p can not get the lock now + err = p.TryLock(ctx) + require.Error(t, err) + + // clear lock on p2 + err = p2.Unlock(ctx) + + // p can now get lock + err = p.TryLock(ctx) + require.NoError(t, err) +} + +// Test that UnlockShared +// - can unlock a lock granted by TrySharedLock +func TestUnlockShared(t *testing.T) { + t.Parallel() + ctx := context.Background() + p, _, u := setup(ctx, t) + + err := p.TrySharedLock(ctx) + require.NoError(t, err) + + d2, err := common.SqlOpen(dbtest.Postgres, u) + p2, err := postgres.New(ctx, d2) + require.NoError(t, err) + + err = p2.TryLock(ctx) + require.Error(t, err) + + err = p.UnlockShared(ctx) + require.NoError(t, err) + + err = p2.TryLock(ctx) + require.NoError(t, err) +} diff --git a/internal/db/schema/internal/postgres/migration_log_test.go b/internal/db/schema/internal/postgres/migration_log_test.go new file mode 100644 index 000000000..01b178381 --- /dev/null +++ b/internal/db/schema/internal/postgres/migration_log_test.go @@ -0,0 +1,203 @@ +package postgres_test + +import ( + "context" + "testing" + + "github.com/hashicorp/boundary/internal/db/schema/internal/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEnsureMigrationLog(t *testing.T) { + ctx := context.Background() + p, _, _ := setup(ctx, t) + + err := p.StartRun(ctx) + require.NoError(t, err) + + err = p.EnsureVersionTable(ctx) + require.NoError(t, err) + + err = p.EnsureMigrationLogTable(ctx) + require.NoError(t, err) + + err = p.CommitRun(ctx) + require.NoError(t, err) +} + +func TestEnsureMigrationLog_NoTxn(t *testing.T) { + ctx := context.Background() + p, _, _ := setup(ctx, t) + err := p.EnsureMigrationLogTable(ctx) + require.Error(t, err) +} + +// Test that a database that was created with a log_migration table +// prior to edition support, can be updated to support editions, +// and any existing log entries can be updated. +// +// The scenario involves a case where: +// - The database was initialized with the original boundary_schema_version table +// - The original migrations where run, creating the log_migration table and some entries in the table. +func TestEnsureMigrationLog_UpdateForEditionSupport(t *testing.T) { + ctx := context.Background() + p, db, _ := setup(ctx, t) + + statements := []string{ + // original boundary_schema_version table + `create table boundary_schema_version ( + version bigint primary key, + dirty boolean not null);`, + // populate the table with a version + `insert into boundary_schema_version + (version, dirty) + values + (1001, false);`, + // need to create some things from other migrations that the original tables used + `create domain wt_timestamp as + timestamp with time zone + default current_timestamp; + + create or replace function + default_create_time() + returns trigger + as $$ + begin + if new.create_time is distinct from now() then + raise warning 'create_time cannot be set to %', new.create_time; + new.create_time = now(); + end if; + return new; + end; + $$ language plpgsql; + + create or replace function + immutable_columns() + returns trigger + as $$ + declare + col_name text; + new_value text; + old_value text; + begin + foreach col_name in array tg_argv loop + execute format('SELECT $1.%I', col_name) into new_value using new; + execute format('SELECT $1.%I', col_name) into old_value using old; + if new_value is distinct from old_value then + raise exception 'immutable column: %.%', tg_table_name, col_name using + errcode = '23601', + schema = tg_table_schema, + table = tg_table_name, + column = col_name; + end if; + end loop; + return new; + end; + $$ language plpgsql; + `, + // create original log_migration table, triggers + `create table log_migration( + id bigint generated always as identity primary key, + migration_version bigint not null, -- cannot declare FK since the table is truncated during runtime + create_time wt_timestamp, + entry text not null + ); + comment on table log_migration is + 'log_migration entries are logging output from databaes migrations'; + + create trigger + default_create_time_column + before + insert on log_migration + for each row execute procedure default_create_time(); + + create trigger + immutable_columns + before + update on log_migration + for each row execute procedure immutable_columns('id', 'migration_version', 'create_time', 'entry'); + + create or replace function + log_migration_version() + returns trigger + as $$ + declare current_version bigint; + begin + select max(version) from boundary_schema_version into current_version; + new.migration_version = current_version; + return new; + end; + $$ language plpgsql; + comment on function log_migration_version() is + 'log_migration_version will set the log_migration entries to the current migration version'; + + create trigger + migration_version_column + before + insert on log_migration + for each row execute procedure log_migration_version(); + `, + // insert some logs + `insert into log_migration (entry) + values + ('log entry');`, + } + + for _, s := range statements { + _, err := db.ExecContext(ctx, s) + require.NoError(t, err) + } + + err := p.StartRun(ctx) + require.NoError(t, err) + + err = p.EnsureVersionTable(ctx) + require.NoError(t, err) + + err = p.EnsureMigrationLogTable(ctx) + require.NoError(t, err) + + err = p.CommitRun(ctx) + require.NoError(t, err) + + logs, err := p.GetMigrationLog(ctx, log.WithDeleteLog(true)) + require.NoError(t, err) + assert.Equal(t, 1, len(logs)) + assert.NotEqual(t, logs[0].Id, 0) + assert.False(t, logs[0].CreateTime.IsZero()) + assert.Equal(t, "oss", logs[0].MigrationEdition) + assert.Equal(t, 1001, logs[0].MigrationVersion) + assert.Equal(t, "log entry", logs[0].Entry) + + // now ensure the updated table and trigger still works + statements = []string{ + // update the version, to make it easy to filter on log_migrations + `update boundary_schema_version set version = 1002;`, + // insert a new log entry + `insert into log_migration (entry, edition) + values + ('new log entry', 'oss');`, + } + for _, s := range statements { + _, err := db.ExecContext(ctx, s) + require.NoError(t, err) + } + + logs, err = p.GetMigrationLog(ctx, log.WithDeleteLog(true)) + require.NoError(t, err) + assert.Equal(t, 1, len(logs)) + assert.NotEqual(t, logs[0].Id, 0) + assert.False(t, logs[0].CreateTime.IsZero()) + assert.Equal(t, "oss", logs[0].MigrationEdition) + assert.Equal(t, 1002, logs[0].MigrationVersion) + assert.Equal(t, "new log entry", logs[0].Entry) + + // ensure it is idempotent + err = p.StartRun(ctx) + require.NoError(t, err) + err = p.EnsureMigrationLogTable(ctx) + require.NoError(t, err) + err = p.CommitRun(ctx) + require.NoError(t, err) +} diff --git a/internal/db/schema/internal/postgres/postgres.go b/internal/db/schema/internal/postgres/postgres.go new file mode 100644 index 000000000..e67ec74e8 --- /dev/null +++ b/internal/db/schema/internal/postgres/postgres.go @@ -0,0 +1,457 @@ +// The MIT License (MIT) +// +// Original Work +// Copyright (c) 2016 Matthias Kadenbach +// https://github.com/mattes/migrate +// +// Modified Work +// Copyright (c) 2018 Dale Hui +// https://github.com/golang-migrate/migrate +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package postgres + +import ( + "context" + "database/sql" + stderrors "errors" + "fmt" + "io" + "io/ioutil" + + "github.com/hashicorp/boundary/internal/db/schema/internal/log" + "github.com/hashicorp/boundary/internal/db/schema/migrations" + "github.com/hashicorp/boundary/internal/errors" + "github.com/hashicorp/go-multierror" + "github.com/jackc/pgconn" +) + +// schemaAccessLockId is a Lock key used to ensure a single boundary binary is operating +// on a postgres server at a time. The value has no meaning and was picked randomly. +const schemaAccessLockId int64 = 3865661975 + +// nilVersion is used to identify when a migration version has not be set. +const nilVersion = -1 + +// Postgres is a driver usable by a boundary schema.Manager. +// This struct is not thread safe. +type Postgres struct { + // Locking and unlocking need to use the same connection + conn *sql.Conn + + tx *sql.Tx +} + +// New creates a Postgres with the provided sql.DB verified as connectable +func New(ctx context.Context, db *sql.DB) (*Postgres, error) { + const op = "postgres.New" + if err := db.PingContext(ctx); err != nil { + return nil, errors.Wrap(ctx, err, op) + } + conn, err := db.Conn(ctx) + if err != nil { + return nil, errors.Wrap(ctx, err, op) + } + + px := &Postgres{ + conn: conn, + } + return px, nil +} + +// TrySharedLock attempts to capture a shared lock. If it is not successful it returns an error. +// https://www.postgresql.org/docs/11/static/explicit-locking.html#ADVISORY-LOCKS +func (p *Postgres) TrySharedLock(ctx context.Context) error { + const op = "postgres.(Postgres).TrySharedLock" + + r := p.conn.QueryRowContext(ctx, trySharedLock, schemaAccessLockId) + if r.Err() != nil { + return errors.Wrap(ctx, r.Err(), op) + } + var gotLock bool + if err := r.Scan(&gotLock); err != nil { + return errors.Wrap(ctx, err, op) + } + if !gotLock { + return errors.New(ctx, errors.MigrationLock, op, "Lock failed") + } + return nil +} + +// TryLock attempts to capture an exclusive lock. If it is not successful it returns an error. +// https://www.postgresql.org/docs/11/static/explicit-locking.html#ADVISORY-LOCKS +func (p *Postgres) TryLock(ctx context.Context) error { + const op = "postgres.(Postgres).TryLock" + + r := p.conn.QueryRowContext(ctx, tryLock, schemaAccessLockId) + if r.Err() != nil { + return errors.Wrap(ctx, r.Err(), op) + } + var gotLock bool + if err := r.Scan(&gotLock); err != nil { + return errors.Wrap(ctx, err, op) + } + if !gotLock { + return errors.New(ctx, errors.MigrationLock, op, "Lock failed") + } + return nil +} + +// Lock calls pg_advisory_lock with the provided context and returns an error +// if we were unable to get the lock before the context cancels. +func (p *Postgres) Lock(ctx context.Context) error { + const op = "postgres.(Postgres).Lock" + + if _, err := p.conn.ExecContext(ctx, lock, schemaAccessLockId); err != nil { + return errors.Wrap(ctx, err, op) + } + return nil +} + +// Unlock calls pg_advisory_unlock and returns an error if we were unable to +// release the lock before the context cancels. +func (p *Postgres) Unlock(ctx context.Context) error { + const op = "postgres.(Postgres).Unlock" + + if _, err := p.conn.ExecContext(ctx, unlock, schemaAccessLockId); err != nil { + return errors.Wrap(ctx, err, op) + } + return nil +} + +// UnlockShared calls pg_advisory_unlock_shared and returns an error if we were unable to +// release the lock before the context cancels. +func (p *Postgres) UnlockShared(ctx context.Context) error { + const op = "postgres.(Postgres).UnlockShared" + + if _, err := p.conn.ExecContext(ctx, unlockSharedLock, schemaAccessLockId); err != nil { + return errors.Wrap(ctx, err, op) + } + return nil +} + +// StartRun begins a transaction internal to the driver. +func (p *Postgres) StartRun(ctx context.Context) error { + tx, err := p.conn.BeginTx(ctx, nil) + if err != nil { + return err + } + p.tx = tx + return nil +} + +// CommitRun commits a transaction, if there is an error it should rollback the transaction. +func (p *Postgres) CommitRun(ctx context.Context) error { + const op = "postgres.(Postgres).CommitRun" + defer func() { + p.tx = nil + }() + if p.tx == nil { + return errors.New(ctx, errors.MigrationIntegrity, op, "no pending transaction") + } + if err := p.tx.Commit(); err != nil { + if errRollback := p.tx.Rollback(); errRollback != nil && errRollback != sql.ErrTxDone { + err = multierror.Append(err, errRollback) + } + return errors.Wrap(ctx, err, op) + } + return nil +} + +// Run will apply a migration. The io.Reader should provide the SQL +// statements to execute, and the int is the version for that set of +// statements. This should always be wrapped by StartRun and CommitRun. +func (p *Postgres) Run(ctx context.Context, migration io.Reader, version int, edition string) error { + const op = "postgres.(Postgres).Run" + + if p.tx == nil { + return errors.New(ctx, errors.MigrationIntegrity, op, "no pending transaction") + } + + migr, err := ioutil.ReadAll(migration) + if err != nil { + return errors.Wrap(ctx, err, op) + } + // Run migration + query := string(migr) + + // set the version first, so logs will be associated with this new version. + // if there's an error, it will get rollback + if err := p.setVersion(ctx, version, edition); err != nil { + return errors.Wrap(ctx, err, op) + } + + if _, err := p.conn.ExecContext(ctx, query); err != nil { + if pgErr, ok := err.(*pgconn.PgError); ok { + var line uint + var col uint + var lineColOK bool + if pgErr.Position != 0 { + line, col, lineColOK = computeLineFromPos(query, int(pgErr.Position)) + } + message := "migration failed" + if lineColOK { + message = fmt.Sprintf("%s (column %d)", message, col) + } + if pgErr.Detail != "" { + message = fmt.Sprintf("%s, %s", message, pgErr.Detail) + } + message = fmt.Sprintf("%s, on line %v: %s", message, line, migr) + return errors.Wrap(ctx, err, op, errors.WithMsg(message)) + } + return errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("migration failed: %s", migr))) + } + + return nil +} + +var errOldMigrationTable = stderrors.New("old schema migration table") + +func (p *Postgres) schemaInitialized(ctx context.Context) (bool, error) { + const op = "postgres.(Postgres).CurrentState" + var initialized bool + + tableNames := make([]string, 0, 2) + tableResult, err := p.conn.QueryContext(ctx, tablesExist) + if err != nil { + return false, errors.Wrap(ctx, err, op) + } + defer tableResult.Close() + for tableResult.Next() { + var tableName string + if err := tableResult.Scan(&tableName); err != nil { + return false, errors.Wrap(ctx, err, op) + } + tableNames = append(tableNames, tableName) + } + + if len(tableNames) <= 0 { + // No version table found + return initialized, nil + } + + initialized = true + + if len(tableNames) > 1 { + return initialized, errors.New(ctx, errors.MigrationIntegrity, op, "both old and new migration tables exist") + } + + if tableNames[0] != schemaVersionTable { + return initialized, errOldMigrationTable + } + + return initialized, nil +} + +// CurrentState returns the state of the given edition. +// ver is the current migration version number as recorded in the database. +// A version of -1 indicates no version is set. +// initialized will be true if the schema was previously initialized. +func (p *Postgres) CurrentState(ctx context.Context, edition string) (version int, initialized bool, err error) { + const op = "postgres.(Postgres).CurrentState" + + version = nilVersion + initialized, err = p.schemaInitialized(ctx) + switch { + case err == errOldMigrationTable: + // only oss edition had the old migration table + if edition != "oss" { + return nilVersion, initialized, nil + } + err = p.conn.QueryRowContext(ctx, selectOldVersion).Scan(&version) + switch { + case err == sql.ErrNoRows: + // no version recorded + return nilVersion, initialized, nil + case err != nil: + return nilVersion, initialized, errors.Wrap(ctx, err, op) + default: + return version, initialized, nil + } + case err != nil: + return nilVersion, initialized, err + default: + // continue + } + + if !initialized { + return nilVersion, initialized, nil + } + + err = p.conn.QueryRowContext(ctx, selectVersion, edition).Scan(&version) + switch { + case err == sql.ErrNoRows: + // no version recorded + return nilVersion, initialized, nil + case err != nil: + // try to query for an edition in the pre-edition version table + if edition == "oss" { + preEditionErr := p.conn.QueryRowContext(ctx, selectPreEditionVersion).Scan(&version) + switch { + case preEditionErr == sql.ErrNoRows: + return nilVersion, initialized, nil + case preEditionErr != nil: + // return the original error + return nilVersion, initialized, errors.Wrap(ctx, err, op) + default: + return version, initialized, nil + } + } + return nilVersion, initialized, errors.Wrap(ctx, err, op) + default: + return version, initialized, nil + } +} + +// EnsureVersionTable ensures that the table used to record the schema versions for each edition +// exists and is in the correct state. +func (p *Postgres) EnsureVersionTable(ctx context.Context) error { + const op = "postgres.(Postgres).EnsureVersionTable" + + if p.tx == nil { + return errors.New(ctx, errors.MigrationIntegrity, op, "no pending transaction") + } + + exists := false + if err := p.tx.QueryRowContext(ctx, tableExists).Scan(&exists); err != nil { + return errors.Wrap(ctx, err, op) + } + + if !exists { + oldExists := false + if err := p.tx.QueryRowContext(ctx, oldTableExists).Scan(&oldExists); err != nil { + return errors.Wrap(ctx, err, op) + } + if !oldExists { + // create table for the first time it its correct state + if _, err := p.tx.ExecContext(ctx, migrations.Base("postgres").CreateSchemaVersion); err != nil { + return errors.Wrap(ctx, err, op) + } + return nil + } + + // else convert old table to new table + if _, err := p.tx.ExecContext(ctx, updateTableName); err != nil { + return errors.Wrap(ctx, err, op) + } + } + + // table exists, update it to ensure it has the correct columns and constraints + alterations := []string{ + dropDirtyColumn, + addEditionColumn, + setVersionNotNull, + } + + for _, a := range alterations { + if _, err := p.tx.ExecContext(ctx, a); err != nil { + return errors.Wrap(ctx, err, op) + } + } + return nil +} + +// setVersion sets the version number, and whether the database is in a dirty state. +// A version value of -1 indicates no version is set. +func (p *Postgres) setVersion(ctx context.Context, version int, edition string) error { + const op = "postgres.(Postgres).setVersion" + + if p.tx == nil { + return errors.New(ctx, errors.MigrationIntegrity, op, "no pending transaction") + } + + if _, err := p.tx.ExecContext(ctx, upsertVersion, edition, version); err != nil { + return errors.Wrap(ctx, err, op) + } + + return nil +} + +// EnsureMigrationLogTable ensures that the table used to record migration lgos +// exists and is in the correct state. +func (p *Postgres) EnsureMigrationLogTable(ctx context.Context) error { + const op = "postgres.(Postgres).EnsureMigrationLogTable" + + if p.tx == nil { + return errors.New(ctx, errors.MigrationIntegrity, op, "no pending transaction") + } + + exists := false + if err := p.tx.QueryRowContext(ctx, migrationLogTableExists).Scan(&exists); err != nil { + return errors.Wrap(ctx, err, op) + } + + if !exists { + if _, err := p.tx.ExecContext(ctx, migrations.Base("postgres").CreateLogMigration); err != nil { + return errors.Wrap(ctx, err, op) + } + return nil + } + + // table exists, update it to ensure it has the correct columns and triggers + alterations := []string{ + migrationLogAlterColumns, + migrationLogDropTriggers, + migrationLogAddEditionColumn, + migrationLogReplaceVersionTrigger, + } + for _, a := range alterations { + if _, err := p.tx.ExecContext(ctx, a); err != nil { + return errors.Wrap(ctx, err, op) + } + } + return nil +} + +// GetMigrationLog will retrieve the migration logs from the db for the last +// migration. +// The WithDeleteLog option is supported and will remove all log entries, +// after reading the entries, when provided. +func (p *Postgres) GetMigrationLog(ctx context.Context, opt ...log.Option) ([]*log.Entry, error) { + const op = "postgres.(Postgres).GetMigrationLog" + + rows, err := p.conn.QueryContext(ctx, getMigrationLogs) + if err != nil { + return nil, errors.Wrap(ctx, err, op) + } + defer rows.Close() + + var entries []*log.Entry + for rows.Next() { + e := &log.Entry{} + + if err := rows.Scan(&e.Id, &e.CreateTime, &e.MigrationVersion, &e.MigrationEdition, &e.Entry); err != nil { + return nil, errors.Wrap(ctx, err, op) + } + entries = append(entries, e) + } + if rows.Err() != nil { + return nil, errors.Wrap(ctx, err, op) + } + opts := log.GetOpts(opt...) + if opts.WithDeleteLog { + // this truncate could change to a delete if FKs are needed in the future + _, err = p.conn.ExecContext(ctx, truncateMigrationLogs) + if err != nil { + return nil, errors.Wrap(ctx, err, op) + } + } + return entries, nil +} diff --git a/internal/db/schema/internal/postgres/postgres_test.go b/internal/db/schema/internal/postgres/postgres_test.go new file mode 100644 index 000000000..c9c82984a --- /dev/null +++ b/internal/db/schema/internal/postgres/postgres_test.go @@ -0,0 +1,63 @@ +package postgres_test + +import ( + "bytes" + "context" + "fmt" + "testing" + + "github.com/hashicorp/boundary/internal/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRun(t *testing.T) { + ctx := context.Background() + p, _, _ := setup(ctx, t) + + statements := bytes.NewReader([]byte(` +create table foo ( + id bigint primary key, + bar text +); +`)) + + err := p.StartRun(ctx) + require.NoError(t, err) + + err = p.EnsureVersionTable(ctx) + require.NoError(t, err) + + err = p.EnsureMigrationLogTable(ctx) + require.NoError(t, err) + + err = p.Run(ctx, statements, 1001, "oss") + require.NoError(t, err) + + err = p.CommitRun(ctx) + require.NoError(t, err) + + v, i, err := p.CurrentState(ctx, "oss") + require.NoError(t, err) + assert.True(t, i) + assert.Equal(t, v, 1001) +} + +func TestRun_NoTxn(t *testing.T) { + ctx := context.Background() + p, _, _ := setup(ctx, t) + + statements := bytes.NewReader([]byte(` +create table foo ( + id bigint primary key, + bar text +); +`)) + + err := p.Run(ctx, statements, 1001, "oss") + require.EqualError( + t, + err, + fmt.Sprintf("postgres.(Postgres).Run: no pending transaction: integrity violation: error #%d", errors.MigrationIntegrity), + ) +} diff --git a/internal/db/schema/internal/postgres/query.go b/internal/db/schema/internal/postgres/query.go new file mode 100644 index 000000000..1a3ae8a1e --- /dev/null +++ b/internal/db/schema/internal/postgres/query.go @@ -0,0 +1,139 @@ +package postgres + +// Queries for interacting with locks +const ( + trySharedLock = `select pg_try_advisory_lock_shared($1)` + tryLock = `select pg_try_advisory_lock($1)` + lock = `select pg_advisory_lock($1)` + unlock = `select pg_advisory_unlock($1)` + unlockSharedLock = `select pg_advisory_unlock_shared($1)` +) + +// Queries for interacting with the schema version table +const ( + schemaVersionTable = `boundary_schema_version` + upsertVersion = ` +insert into boundary_schema_version + (edition, version) +values + ($1, $2) +on conflict (edition) +do update set + version = $2 +;` + + selectVersion = ` +select version + from boundary_schema_version + where edition = $1 +;` + + selectOldVersion = ` +select version from schema_migrations; +` + + selectPreEditionVersion = ` +select max(version) from boundary_schema_version; +` + + tablesExist = ` +select table_name from information_schema.tables + where table_schema = (select current_schema()) + and table_name in ('schema_migrations', 'boundary_schema_version') +;` + + tableExists = ` +select exists ( + select 1 from information_schema.tables + where table_schema = (select current_schema()) + and table_name = 'boundary_schema_version' +);` + + oldTableExists = ` +select exists ( + select 1 from information_schema.tables + where table_schema = (select current_schema()) + and table_name = 'schema_migrations' +);` + + updateTableName = ` +alter table if exists schema_migrations + rename to boundary_schema_version +;` + + dropDirtyColumn = ` +alter table boundary_schema_version + drop column if exists dirty +;` + + addEditionColumn = ` +alter table boundary_schema_version + add column if not exists + edition text default 'oss'; +alter table boundary_schema_version + alter column edition drop default; +alter table boundary_schema_version drop constraint if exists schema_migrations_pkey; +alter table boundary_schema_version drop constraint if exists boundary_schema_version_pkey; +alter table boundary_schema_version add primary key (edition); +` + setVersionNotNull = ` +alter table boundary_schema_version + alter column version set not null; +` +) + +// Quries for interacting with migration logs table +const ( + migrationLogTableExists = ` +select exists ( + select 1 from information_schema.tables + where table_schema = (select current_schema()) + and table_name = 'log_migration' +);` + + getMigrationLogs = ` +select + id, create_time, migration_version, edition, entry + from log_migration +where + migration_version in ( + select max(version) from boundary_schema_version + );` + + truncateMigrationLogs = `truncate log_migration;` + + migrationLogAlterColumns = ` +alter table log_migration + alter column create_time type timestamp with time zone; +alter table log_migration + alter column create_time set default current_timestamp; +` + migrationLogDropTriggers = ` +drop trigger if exists default_create_time_column on log_migration; +drop trigger if exists immutable_columns on log_migration; +` + migrationLogAddEditionColumn = ` +alter table log_migration + add column if not exists + edition text default 'oss'; +alter table log_migration + alter column edition drop default; +` + migrationLogReplaceVersionTrigger = ` +create or replace function + log_migration_version() + returns trigger +as $$ + declare current_version bigint; + begin + select max(version) + from boundary_schema_version into current_version + where edition = new.edition; + new.migration_version = current_version; + return new; + end; +$$ language plpgsql; +comment on function log_migration_version() is +'log_migration_version will set the log_migration entries to the current migration version'; +` +) diff --git a/internal/db/schema/internal/postgres/state_test.go b/internal/db/schema/internal/postgres/state_test.go new file mode 100644 index 000000000..1936921d7 --- /dev/null +++ b/internal/db/schema/internal/postgres/state_test.go @@ -0,0 +1,56 @@ +package postgres_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCurrentStatePreEdition(t *testing.T) { + ctx := context.Background() + p, db, _ := setup(ctx, t) + + _, err := db.ExecContext(ctx, ` + create table boundary_schema_version ( + version bigint primary key, + dirty boolean not null + );`) + require.NoError(t, err) + _, err = db.ExecContext(ctx, ` + insert into boundary_schema_version + (version, dirty) + values + (1001, false);`) + require.NoError(t, err) + + version, initialized, err := p.CurrentState(ctx, "oss") + require.NoError(t, err) + assert.Equal(t, version, 1001) + assert.True(t, initialized) +} + +func TestCurrentStateOldSchemaTable(t *testing.T) { + ctx := context.Background() + p, db, _ := setup(ctx, t) + + _, err := db.ExecContext(ctx, ` + create table schema_migrations ( + version bigint primary key, + dirty boolean not null + );`) + require.NoError(t, err) + require.NoError(t, err) + _, err = db.ExecContext(ctx, ` + insert into schema_migrations + (version, dirty) + values + (1001, false);`) + require.NoError(t, err) + + version, initialized, err := p.CurrentState(ctx, "oss") + require.NoError(t, err) + assert.Equal(t, version, 1001) + assert.True(t, initialized) +} diff --git a/internal/db/schema/internal/postgres/testing_test.go b/internal/db/schema/internal/postgres/testing_test.go new file mode 100644 index 000000000..4fb5f2b37 --- /dev/null +++ b/internal/db/schema/internal/postgres/testing_test.go @@ -0,0 +1,35 @@ +package postgres_test + +import ( + "context" + "database/sql" + "testing" + + "github.com/hashicorp/boundary/internal/db/common" + "github.com/hashicorp/boundary/internal/db/schema/internal/postgres" + "github.com/hashicorp/boundary/testing/dbtest" + "github.com/stretchr/testify/require" +) + +// setup is a helper function for tests. +// It creates a new database using Template1, establishes an connection, +// and returns a *postgres.Postgrs. +// It also returns the underlying sql.DB and connection url so tests +// can establish additional connections if necessary. +func setup(ctx context.Context, t *testing.T) (*postgres.Postgres, *sql.DB, string) { + t.Helper() + + c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1)) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, c()) + }) + d, err := common.SqlOpen(dbtest.Postgres, u) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, d.Close()) + }) + p, err := postgres.New(ctx, d) + + return p, d, u +} diff --git a/internal/db/schema/internal/provider/provider.go b/internal/db/schema/internal/provider/provider.go new file mode 100644 index 000000000..02e4262a6 --- /dev/null +++ b/internal/db/schema/internal/provider/provider.go @@ -0,0 +1,100 @@ +// Package provider provides an iterator for iterating over all of the +// migration statements that need to be applied. It will provide the statements +// in the correct order based on the Edition priority and migration version. +package provider + +import ( + "sort" + + "github.com/hashicorp/boundary/internal/db/schema/internal/edition" +) + +const nilVersion = -1 + +type migration struct { + version int + edition string + statements []byte +} + +// Provider provides the migrations to the schema.Manager in the correct order. +type Provider struct { + pos int + migrations []migration +} + +// DatabaseState is a map of edition names to versions. +type DatabaseState map[string]int + +// New creates a Provider. The given DatabaseState is compared against the editions +// to determine which migrations need to be applied and the migrations are then ordered +// based on the Edition priority and by the migration version. +func New(dbState DatabaseState, editions edition.Editions) *Provider { + m := &Provider{ + pos: -1, + } + + // ensure editions in priority order + editions.Sort() + + allMigrations := make([]migration, 0) + + for _, e := range editions { + dbVer, ok := dbState[e.Name] + if !ok { + dbVer = nilVersion + } + + migrations := make([]migration, 0, len(e.Migrations)) + for ver, statements := range e.Migrations { + if ver > dbVer { + migrations = append(migrations, migration{ + version: ver, + edition: e.Name, + statements: statements, + }) + } + } + + sort.SliceStable(migrations, func(i, j int) bool { + return migrations[i].version < migrations[j].version + }) + + allMigrations = append(allMigrations, migrations...) + } + + m.migrations = allMigrations + + return m +} + +// Next proceeds to the next migration. It returns true on success or false +// if there are no more migrations. +func (p *Provider) Next() bool { + p.pos++ + return len(p.migrations) > p.pos +} + +// Version returns the version for the current migration. +func (p *Provider) Version() int { + if p.pos < 0 || p.pos >= len(p.migrations) { + return -1 + } + return p.migrations[p.pos].version +} + +// Edition returns the edition name for the current migration. +func (p *Provider) Edition() string { + if p.pos < 0 || p.pos >= len(p.migrations) { + return "" + } + return p.migrations[p.pos].edition +} + +// Statements returns the sql statements name for the current migration. +func (p *Provider) Statements() []byte { + if p.pos < 0 || p.pos >= len(p.migrations) { + return nil + } + return p.migrations[p.pos].statements +} diff --git a/internal/db/schema/internal/provider/provider_test.go b/internal/db/schema/internal/provider/provider_test.go new file mode 100644 index 000000000..a55b02a67 --- /dev/null +++ b/internal/db/schema/internal/provider/provider_test.go @@ -0,0 +1,159 @@ +package provider_test + +import ( + "testing" + + "github.com/hashicorp/boundary/internal/db/schema/internal/edition" + "github.com/hashicorp/boundary/internal/db/schema/internal/provider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type expectedMigration struct { + version int + edition string + statements []byte +} + +type expectedMigrations []expectedMigration + +func TestProvider(t *testing.T) { + tests := []struct { + name string + editions edition.Editions + dbState provider.DatabaseState + expected expectedMigrations + }{ + { + "oneEditionNoneApplied", + edition.Editions{ + edition.Edition{ + Name: "one", + LatestVersion: 2, + Migrations: map[int][]byte{ + 1: []byte(`migration one`), + 2: []byte(`migration two`), + }, + Priority: 0, + }, + }, + provider.DatabaseState{"one": -1}, + expectedMigrations{ + {1, "one", []byte(`migration one`)}, + {2, "one", []byte(`migration two`)}, + }, + }, + { + "twoEditionsNoneApplied", + edition.Editions{ + edition.Edition{ + Name: "one", + LatestVersion: 2, + Migrations: map[int][]byte{ + 1: []byte(`migration one`), + 2: []byte(`migration two`), + }, + Priority: 0, + }, + edition.Edition{ + Name: "two", + LatestVersion: 1, + Migrations: map[int][]byte{ + 1: []byte(`migration one`), + }, + Priority: 0, + }, + }, + provider.DatabaseState{ + "one": -1, + }, + expectedMigrations{ + {1, "one", []byte(`migration one`)}, + {2, "one", []byte(`migration two`)}, + {1, "two", []byte(`migration one`)}, + }, + }, + { + "twoEditionsOnePartial", + edition.Editions{ + edition.Edition{ + Name: "one", + LatestVersion: 2, + Migrations: map[int][]byte{ + 1: []byte(`migration one`), + 2: []byte(`migration two`), + }, + Priority: 0, + }, + edition.Edition{ + Name: "two", + LatestVersion: 1, + Migrations: map[int][]byte{ + 1: []byte(`migration one`), + }, + Priority: 0, + }, + }, + provider.DatabaseState{ + "one": 1, + "two": -1, + }, + expectedMigrations{ + {2, "one", []byte(`migration two`)}, + {1, "two", []byte(`migration one`)}, + }, + }, + { + "twoEditionsBothPartial", + edition.Editions{ + edition.Edition{ + Name: "one", + LatestVersion: 2, + Migrations: map[int][]byte{ + 1: []byte(`migration one`), + 2: []byte(`migration two`), + }, + Priority: 0, + }, + edition.Edition{ + Name: "two", + LatestVersion: 1, + Migrations: map[int][]byte{ + 1: []byte(`migration one`), + 2: []byte(`migration two`), + }, + Priority: 0, + }, + }, + provider.DatabaseState{ + "one": 1, + "two": 1, + }, + expectedMigrations{ + {2, "one", []byte(`migration two`)}, + {2, "two", []byte(`migration two`)}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := provider.New(tt.dbState, tt.editions) + + for _, expected := range tt.expected { + next := p.Next() + require.True(t, next) + + assert.Equal(t, p.Version(), expected.version, tt.name) + assert.Equal(t, p.Edition(), expected.edition, tt.name) + assert.Equal(t, p.Statements(), expected.statements, tt.name) + } + + assert.False(t, p.Next(), tt.name) + assert.Equal(t, p.Version(), -1, tt.name) + assert.Equal(t, p.Edition(), "", tt.name) + assert.Nil(t, p.Statements(), tt.name) + }) + } + +} diff --git a/internal/db/schema/manager.go b/internal/db/schema/manager.go index 6cc1c313b..ed2054b82 100644 --- a/internal/db/schema/manager.go +++ b/internal/db/schema/manager.go @@ -6,11 +6,12 @@ import ( "database/sql" "fmt" "io" - "time" - "github.com/hashicorp/boundary/internal/db/schema/postgres" + "github.com/hashicorp/boundary/internal/db/schema/internal/edition" + "github.com/hashicorp/boundary/internal/db/schema/internal/log" + "github.com/hashicorp/boundary/internal/db/schema/internal/postgres" + "github.com/hashicorp/boundary/internal/db/schema/internal/provider" "github.com/hashicorp/boundary/internal/errors" - "github.com/hashicorp/go-multierror" ) // driver provides functionality to a database. @@ -20,43 +21,59 @@ type driver interface { Lock(context.Context) error Unlock(context.Context) error UnlockShared(context.Context) error - // Either starts a transactioon internal to the driver or sets a dirty - // bit so if the Run fails the CurrentState reflects it. + // StartRun begins a transaction internal to the driver. StartRun(context.Context) error - // Either commits the transaction or clears the dirty bit. - CommitRun() error - // Performs the mutation on the driver. This should always be - // wrapped by StartRun and CommitRun. The driver must properly - // handle the transaction or dirty bit in case of error when - // executing Run. - Run(context.Context, io.Reader, int) error + // CommitRun commits a transaction, if there is an error it should rollback the transaction. + CommitRun(context.Context) error + // Run will apply a migration. The io.Reader should provide the SQL + // statements to execute, and the int is the version for that set of + // statements. This should always be wrapped by StartRun and CommitRun. + Run(ctx context.Context, migration io.Reader, version int, edition string) error + // CurrentState returns the state of the given edition. + // ver is the current migration version number as recorded in the database. // A version of -1 indicates no version is set. - CurrentState(context.Context) (ver int, everRan bool, dirty bool, err error) + // initialized will be true if the schema was previously initialized. + CurrentState(ctx context.Context, edition string) (version int, initialized bool, err error) + // EnsureVersionTable ensures that the table used to record the schema versions for each edition + // exists and is in the correct state. EnsureVersionTable(ctx context.Context) error + // EnsureMigrationLogTable ensures that the table used to record migration lgos + // exists and is in the correct state. + EnsureMigrationLogTable(ctx context.Context) error + // GetMigrationLog will retrieve the migration logs from the db for the last + // migration. + // The WithDeleteLog option is supported and will remove all log entries, + // after reading the entries, when provided. + GetMigrationLog(ctx context.Context, opt ...log.Option) ([]*log.Entry, error) } // Manager provides a way to run operations and retrieve information regarding // the underlying boundary database schema. // Manager is not thread safe. type Manager struct { - db *sql.DB - driver driver - dialect string - migrationStates map[string]migrationState + db *sql.DB + driver driver + dialect string + editions edition.Editions } // NewManager creates a new schema manager. An error is returned // if the provided dialect is unrecognized or if the passed in db is unreachable. -func NewManager(ctx context.Context, dialect string, db *sql.DB, opt ...Option) (*Manager, error) { +func NewManager(ctx context.Context, dialect Dialect, db *sql.DB, opt ...Option) (*Manager, error) { const op = "schema.NewManager" - dbM := Manager{db: db, dialect: dialect} + + editions.Lock() + defer editions.Unlock() + + dbM := Manager{db: db, dialect: string(dialect)} opts := getOpts(opt...) - if opts.withMigrationStates != nil { - dbM.migrationStates = opts.withMigrationStates + if opts.withEditions != nil { + dbM.editions = opts.withEditions } else { - // intentionally set it to the reference, so changes to the global var - // will be reflected in the manager instance - dbM.migrationStates = migrationStates + dbM.editions = make(edition.Editions, 0, len(editions.m[edition.Dialect(dialect)])) + for _, e := range editions.m[edition.Dialect(dialect)] { + dbM.editions = append(dbM.editions, e) + } } switch dialect { case "postgres": @@ -71,32 +88,25 @@ func NewManager(ctx context.Context, dialect string, db *sql.DB, opt ...Option) return &dbM, nil } -// State contains information regarding the current state of a boundary database's schema. -type State struct { - // InitializationStarted indicates if the current database has been initialized previously. - InitializationStarted bool - // Dirty is set to true if the database failed in a previous migration/initialization. - Dirty bool - // DatabaseSchemaVersion is the schema version that is currently running in the database. - DatabaseSchemaVersion int - // BinarySchemaVersion is the schema version which this boundary binary supports. - BinarySchemaVersion int -} - // CurrentState provides the state of the boundary schema contained in the backing database. func (b *Manager) CurrentState(ctx context.Context) (*State, error) { const op = "schema.(Manager).CurrentState" - dbS := State{ - BinarySchemaVersion: BinarySchemaVersion(b.dialect), + var dbS State + + for _, e := range b.editions { + v, initialized, err := b.driver.CurrentState(ctx, e.Name) + if err != nil { + return nil, errors.Wrap(ctx, err, op) + } + dbS.Initialized = initialized || dbS.Initialized + dbS.Editions = append(dbS.Editions, EditionState{ + Name: e.Name, + DatabaseSchemaVersion: v, + BinarySchemaVersion: e.LatestVersion, + DatabaseSchemaState: compareVersions(v, e.LatestVersion), + }) } - v, initialized, dirty, err := b.driver.CurrentState(ctx) - if err != nil { - return nil, errors.Wrap(ctx, err, op) - } - dbS.InitializationStarted = initialized - dbS.DatabaseSchemaVersion = v - dbS.Dirty = dirty return &dbS, nil } @@ -143,11 +153,11 @@ func (b *Manager) ExclusiveUnlock(ctx context.Context) error { return nil } -// RollForward updates the database schema to match the latest version known by +// ApplyMigrations updates the database schema to match the latest version known by // the boundary binary. An error is not returned if the database is already at // the most recent version. -func (b *Manager) RollForward(ctx context.Context) error { - const op = "schema.(Manager).RollForward" +func (b *Manager) ApplyMigrations(ctx context.Context) error { + const op = "schema.(Manager).ApplyMigrations" // Capturing a lock that this session to the db already possesses is okay. if err := b.driver.Lock(ctx); err != nil { @@ -161,101 +171,57 @@ func (b *Manager) RollForward(ctx context.Context) error { } }() - curVersion, _, dirty, err := b.driver.CurrentState(ctx) + state, err := b.CurrentState(ctx) if err != nil { return errors.Wrap(ctx, err, op) } - if dirty { - return errors.New(ctx, errors.NotSpecificIntegrity, op, fmt.Sprintf("schema is dirty with version %d", curVersion)) - } - - if err = b.runMigrations(ctx, newStatementProvider(b.dialect, curVersion, WithMigrationStates(b.migrationStates))); err != nil { + if err = b.runMigrations(ctx, provider.New(state.databaseState(), b.editions)); err != nil { return errors.Wrap(ctx, err, op) } return nil } -type rollbacker interface { - Rollback() error -} - // runMigrations passes migration queries to a database driver and manages // the version and dirty bit. Cancellation or deadline/timeout is managed // through the passed in context. -func (b *Manager) runMigrations(ctx context.Context, qp *statementProvider) error { +func (b *Manager) runMigrations(ctx context.Context, p *provider.Provider) (err error) { const op = "schema.(Manager).runMigrations" - if err := b.driver.StartRun(ctx); err != nil { - return errors.Wrap(ctx, err, op) + if startErr := b.driver.StartRun(ctx); startErr != nil { + err = errors.Wrap(ctx, startErr, op) + return err } - if err := b.driver.EnsureVersionTable(ctx); err != nil { - return errors.Wrap(ctx, err, op) + + defer func() { + if commitErr := b.driver.CommitRun(ctx); commitErr != nil { + err = errors.Wrap(ctx, commitErr, op) + } + }() + + if ensureErr := b.driver.EnsureVersionTable(ctx); ensureErr != nil { + err = errors.Wrap(ctx, ensureErr, op) + return err } - for qp.Next() { + + if ensureErr := b.driver.EnsureMigrationLogTable(ctx); ensureErr != nil { + err = errors.Wrap(ctx, ensureErr, op) + return err + } + + for p.Next() { select { case <-ctx.Done(): - err := ctx.Err() - if d, ok := b.driver.(rollbacker); ok { - if rbErr := d.Rollback(); rbErr != nil { - err = multierror.Append(err, rbErr) - } - } - return errors.Wrap(ctx, err, op) + err = errors.Wrap(ctx, ctx.Err(), op) + return err default: // context is not done yet. Continue on to the next query to execute. } - if err := b.driver.Run(ctx, bytes.NewReader(qp.ReadUp()), qp.Version()); err != nil { - return errors.Wrap(ctx, err, op) + if runErr := b.driver.Run(ctx, bytes.NewReader(p.Statements()), p.Version(), p.Edition()); err != nil { + err = errors.Wrap(ctx, runErr, op) + return err } } - if err := b.driver.CommitRun(); err != nil { - return errors.Wrap(ctx, err, op) - } + return nil } - -// LogEntry represents a log entry generated during migrations. -type LogEntry struct { - Id int - MigrationVersion string - CreateTime time.Time - Entry string -} - -// GetMigrationLog will retrieve the migration logs from the db for the last -// migration. Once it's read the entries, it will delete them from the database. -// The WithDeleteLog option is supported and will remove all log entries when provided. -func GetMigrationLog(ctx context.Context, d *sql.DB, opt ...Option) ([]LogEntry, error) { - const op = "schema.GetMigrationLog" - const sql = "select id, create_time, migration_version, entry from log_migration where migration_version in (select max(version) from boundary_schema_version)" - if d == nil { - return nil, errors.New(ctx, errors.InvalidParameter, op, "missing sql db") - } - rows, err := d.QueryContext(ctx, sql) - if err != nil { - return nil, errors.Wrap(ctx, err, op) - } - defer rows.Close() - - var entries []LogEntry - for rows.Next() { - var e LogEntry - if err := rows.Scan(&e.Id, &e.CreateTime, &e.MigrationVersion, &e.Entry); err != nil { - return nil, errors.Wrap(ctx, err, op) - } - entries = append(entries, e) - } - if rows.Err() != nil { - return nil, errors.Wrap(ctx, err, op) - } - opts := getOpts(opt...) - if opts.withDeleteLog { - // this truncate could change to a delete if FKs are needed in the future - _, err = d.ExecContext(ctx, "truncate log_migration") - if err != nil { - return nil, errors.Wrap(ctx, err, op) - } - } - return entries, nil -} diff --git a/internal/db/schema/manager_test.go b/internal/db/schema/manager_test.go index ea58b0fb1..1b685c015 100644 --- a/internal/db/schema/manager_test.go +++ b/internal/db/schema/manager_test.go @@ -1,14 +1,14 @@ -package schema +package schema_test import ( "context" - "database/sql" "sort" - "strings" "testing" "github.com/hashicorp/boundary/internal/db/common" - "github.com/hashicorp/boundary/internal/db/schema/postgres" + "github.com/hashicorp/boundary/internal/db/schema" + "github.com/hashicorp/boundary/internal/db/schema/internal/edition" + "github.com/hashicorp/boundary/internal/errors" "github.com/hashicorp/boundary/testing/dbtest" "github.com/stretchr/testify/assert" @@ -27,13 +27,13 @@ func TestNewManager(t *testing.T) { require.NoError(t, err) ctx := context.Background() - _, err = NewManager(ctx, dialect, d) + _, err = schema.NewManager(ctx, schema.Dialect(dialect), d) require.NoError(t, err) - _, err = NewManager(ctx, "unknown", d) + _, err = schema.NewManager(ctx, schema.Dialect("unknown"), d) assert.True(t, errors.Match(errors.T(errors.InvalidParameter), err)) d.Close() - _, err = NewManager(ctx, dialect, d) + _, err = schema.NewManager(ctx, schema.Dialect(dialect), d) assert.True(t, errors.Match(errors.T(errors.Op("schema.NewManager")), err)) } @@ -51,32 +51,232 @@ func TestCurrentState(t *testing.T) { d, err := common.SqlOpen(dialect, u) require.NoError(t, err) - m, err := NewManager(ctx, dialect, d) + m, err := schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions( + edition.Editions{ + { + Name: "oss", + Dialect: schema.Postgres, + LatestVersion: 2, + Migrations: map[int][]byte{ + 2: []byte(`select 1`), + }, + Priority: 0, + }, + }, + )) require.NoError(t, err) - want := &State{ - BinarySchemaVersion: BinarySchemaVersion(dialect), - DatabaseSchemaVersion: nilVersion, + want := &schema.State{ + Editions: []schema.EditionState{ + { + Name: "oss", + BinarySchemaVersion: 2, + DatabaseSchemaVersion: schema.NilVersion, + DatabaseSchemaState: schema.Behind, + }, + }, } s, err := m.CurrentState(ctx) require.NoError(t, err) assert.Equal(t, want, s) - testDriver, err := postgres.New(ctx, d) - require.NoError(t, err) - require.NoError(t, testDriver.EnsureVersionTable(ctx)) - require.NoError(t, testDriver.Run(ctx, strings.NewReader("select 1"), 2)) + assert.NoError(t, m.ApplyMigrations(ctx)) - want = &State{ - InitializationStarted: true, - BinarySchemaVersion: BinarySchemaVersion(dialect), - DatabaseSchemaVersion: 2, + want = &schema.State{ + Initialized: true, + Editions: []schema.EditionState{ + { + Name: "oss", + BinarySchemaVersion: 2, + DatabaseSchemaVersion: 2, + DatabaseSchemaState: schema.Equal, + }, + }, } s, err = m.CurrentState(ctx) require.NoError(t, err) assert.Equal(t, want, s) + assert.True(t, s.MigrationsApplied()) } -func TestRollForward(t *testing.T) { +func TestApplyMigration(t *testing.T) { + tests := []struct { + name string + editions edition.Editions + expectErr bool + state *schema.State + }{ + { + "oneEdition", + edition.Editions{ + edition.New("one", schema.Postgres, one, 0), + }, + false, + &schema.State{ + Initialized: true, + Editions: []schema.EditionState{ + { + Name: "one", + BinarySchemaVersion: 1, + DatabaseSchemaVersion: 1, + DatabaseSchemaState: schema.Equal, + }, + }, + }, + }, + { + "twoEditions", + edition.Editions{ + edition.New("one", schema.Postgres, one, 0), + edition.New("two", schema.Postgres, two, 1), + }, + false, + &schema.State{ + Initialized: true, + Editions: []schema.EditionState{ + { + Name: "one", + BinarySchemaVersion: 1, + DatabaseSchemaVersion: 1, + DatabaseSchemaState: schema.Equal, + }, + { + Name: "two", + BinarySchemaVersion: 1, + DatabaseSchemaVersion: 1, + DatabaseSchemaState: schema.Equal, + }, + }, + }, + }, + { + "twoEditionsIncorrectPriority", + edition.Editions{ + edition.New("one", schema.Postgres, one, 1), + edition.New("two", schema.Postgres, two, 0), + }, + true, + &schema.State{ + Initialized: false, + Editions: []schema.EditionState{ + { + Name: "one", + BinarySchemaVersion: 1, + DatabaseSchemaVersion: schema.NilVersion, + DatabaseSchemaState: schema.Behind, + }, + { + Name: "two", + BinarySchemaVersion: 1, + DatabaseSchemaVersion: schema.NilVersion, + DatabaseSchemaState: schema.Behind, + }, + }, + }, + }, + { + "threeEditions", + edition.Editions{ + edition.New("one", schema.Postgres, one, 0), + edition.New("two", schema.Postgres, two, 1), + edition.New("three", schema.Postgres, three, 2), + }, + false, + &schema.State{ + Initialized: true, + Editions: []schema.EditionState{ + { + Name: "one", + BinarySchemaVersion: 1, + DatabaseSchemaVersion: 1, + DatabaseSchemaState: schema.Equal, + }, + { + Name: "two", + BinarySchemaVersion: 1, + DatabaseSchemaVersion: 1, + DatabaseSchemaState: schema.Equal, + }, + { + Name: "three", + BinarySchemaVersion: 1, + DatabaseSchemaVersion: 1, + DatabaseSchemaState: schema.Equal, + }, + }, + }, + }, + { + "threeEditionsIncorrectPriority", + edition.Editions{ + edition.New("one", schema.Postgres, one, 0), + edition.New("two", schema.Postgres, two, 2), + edition.New("three", schema.Postgres, three, 1), + }, + true, + &schema.State{ + Initialized: false, + Editions: []schema.EditionState{ + { + Name: "one", + BinarySchemaVersion: 1, + DatabaseSchemaVersion: schema.NilVersion, + DatabaseSchemaState: schema.Behind, + }, + { + Name: "two", + BinarySchemaVersion: 1, + DatabaseSchemaVersion: schema.NilVersion, + DatabaseSchemaState: schema.Behind, + }, + { + Name: "three", + BinarySchemaVersion: 1, + DatabaseSchemaVersion: schema.NilVersion, + DatabaseSchemaState: schema.Behind, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dialect := dbtest.Postgres + + c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1)) + t.Cleanup(func() { + if err := c(); err != nil { + t.Fatalf("Got error at cleanup: %v", err) + } + }) + require.NoError(t, err) + ctx := context.Background() + d, err := common.SqlOpen(dialect, u) + require.NoError(t, err) + + m, err := schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions(tt.editions)) + require.NoError(t, err) + if tt.expectErr { + assert.Error(t, m.ApplyMigrations(ctx)) + } else { + assert.NoError(t, m.ApplyMigrations(ctx)) + } + + s, err := m.CurrentState(ctx) + require.NoError(t, err) + assert.Equal(t, tt.state.Initialized, s.Initialized) + assert.ElementsMatch(t, tt.state.Editions, s.Editions) + + if tt.expectErr { + assert.False(t, s.MigrationsApplied()) + } else { + assert.True(t, s.MigrationsApplied()) + } + }) + } +} + +func TestApplyMigration_canceledContext(t *testing.T) { dialect := dbtest.Postgres c, u, _, err := dbtest.StartUsingTemplate(dialect) @@ -88,87 +288,16 @@ func TestRollForward(t *testing.T) { require.NoError(t, err) ctx := context.Background() - m, err := NewManager(ctx, dialect, d) - require.NoError(t, err) - assert.NoError(t, m.RollForward(ctx)) - - // Now set to dirty at an early version - _, err = postgres.New(ctx, d) - require.NoError(t, err) - // TODO: Extract out a way to mock the db to test failing rollforwards. - _, err = d.ExecContext(ctx, "TRUNCATE boundary_schema_version; INSERT INTO boundary_schema_version (version, dirty) VALUES (2, true)") - require.NoError(t, err) - assert.Error(t, m.RollForward(ctx)) -} - -func TestRollForward_NotFromFresh(t *testing.T) { - dialect := dbtest.Postgres - oState := migrationStates[dialect] - - nState := createPartialMigrationState(oState, 8) - migrationStates[dialect] = nState - - c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1)) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, c()) - }) - d, err := common.SqlOpen(dialect, u) + m, err := schema.NewManager(ctx, schema.Dialect(dialect), d) require.NoError(t, err) - // Initialize the DB with only a portion of the current sql scripts. - ctx := context.Background() - m, err := NewManager(ctx, dialect, d) - require.NoError(t, err) - assert.NoError(t, m.RollForward(ctx)) - - state, err := m.CurrentState(ctx) - require.NoError(t, err) - assert.Equal(t, nState.binarySchemaVersion, state.DatabaseSchemaVersion) - assert.False(t, state.Dirty) - - // Restore the full set of sql scripts and roll the rest of the way forward. - migrationStates[dialect] = oState - - newM, err := NewManager(ctx, dialect, d) - require.NoError(t, err) - assert.NoError(t, newM.RollForward(ctx)) - state, err = newM.CurrentState(ctx) - require.NoError(t, err) - assert.Equal(t, oState.binarySchemaVersion, state.DatabaseSchemaVersion) - assert.False(t, state.Dirty) -} - -func TestRunMigration_canceledContext(t *testing.T) { - dialect := dbtest.Postgres - - c, u, _, err := dbtest.StartUsingTemplate(dialect) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, c()) - }) - d, err := common.SqlOpen(dialect, u) - require.NoError(t, err) - - ctx := context.Background() - m, err := NewManager(ctx, dialect, d) - require.NoError(t, err) - - // TODO: Find a way to test different parts of the runMigrations loop. ctx, cancel := context.WithCancel(ctx) cancel() - assert.Error(t, m.runMigrations(ctx, newStatementProvider(dialect, 0))) + assert.Error(t, m.ApplyMigrations(ctx)) } -func TestRollForward_BadSQL(t *testing.T) { +func TestApplyMigrations_BadSQL(t *testing.T) { dialect := dbtest.Postgres - oState := migrationStates[dialect] - defer func() { migrationStates[dialect] = oState }() - - nState := createPartialMigrationState(oState, 8) - nState.binarySchemaVersion = 10 - nState.upMigrations[10] = []byte("SELECT 1 FROM NonExistantTable;") - migrationStates[dialect] = nState c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1)) require.NoError(t, err) @@ -180,14 +309,37 @@ func TestRollForward_BadSQL(t *testing.T) { // Initialize the DB with only a portion of the current sql scripts. ctx := context.Background() - m, err := NewManager(ctx, dialect, d) + m, err := schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions( + edition.Editions{ + { + Name: "oss", + Dialect: schema.Postgres, + LatestVersion: 1, + Migrations: map[int][]byte{ + 1: []byte(`select 1 from nonexistanttable;`), + }, + Priority: 0, + }, + }, + )) require.NoError(t, err) - assert.Error(t, m.RollForward(ctx)) + assert.Error(t, m.ApplyMigrations(ctx)) state, err := m.CurrentState(ctx) require.NoError(t, err) - assert.Equal(t, nilVersion, state.DatabaseSchemaVersion) - assert.False(t, state.Dirty) + want := &schema.State{ + Initialized: false, + Editions: []schema.EditionState{ + { + Name: "oss", + BinarySchemaVersion: 1, + DatabaseSchemaVersion: schema.NilVersion, + DatabaseSchemaState: schema.Behind, + }, + }, + } + assert.Equal(t, want, state) + assert.False(t, state.MigrationsApplied()) } func TestManager_ExclusiveLock(t *testing.T) { @@ -201,12 +353,12 @@ func TestManager_ExclusiveLock(t *testing.T) { }) d1, err := common.SqlOpen(dialect, u) require.NoError(t, err) - m1, err := NewManager(ctx, dialect, d1) + m1, err := schema.NewManager(ctx, schema.Dialect(dialect), d1) require.NoError(t, err) d2, err := common.SqlOpen(dialect, u) require.NoError(t, err) - m2, err := NewManager(ctx, dialect, d2) + m2, err := schema.NewManager(ctx, schema.Dialect(dialect), d2) require.NoError(t, err) assert.NoError(t, m1.ExclusiveLock(ctx)) @@ -224,14 +376,14 @@ func TestManager_SharedLock(t *testing.T) { t.Cleanup(func() { require.NoError(t, c()) }) - d1, err := common.SqlOpen("postgres", u) + d1, err := common.SqlOpen(dialect, u) require.NoError(t, err) - m1, err := NewManager(ctx, "postgres", d1) + m1, err := schema.NewManager(ctx, schema.Dialect(dialect), d1) require.NoError(t, err) - d2, err := common.SqlOpen("postgres", u) + d2, err := common.SqlOpen(dialect, u) require.NoError(t, err) - m2, err := NewManager(ctx, "postgres", d2) + m2, err := schema.NewManager(ctx, schema.Dialect(dialect), d2) require.NoError(t, err) assert.NoError(t, m1.SharedLock(ctx)) @@ -253,22 +405,21 @@ func Test_GetMigrationLog(t *testing.T) { t.Cleanup(func() { require.NoError(t, c()) }) - d, err := common.SqlOpen("postgres", u) + d, err := common.SqlOpen(dialect, u) require.NoError(t, err) - m, err := NewManager(ctx, "postgres", d) + m, err := schema.NewManager(ctx, schema.Dialect(dialect), d) require.NoError(t, err) - require.NoError(t, m.RollForward(ctx)) + require.NoError(t, m.ApplyMigrations(ctx)) - const insert = `insert into log_migration(entry) values ($1)` + const insert = `insert into log_migration(entry, edition) values ($1, $2)` createEntries := func(entries ...string) { for _, e := range entries { - _, err := d.Exec(insert, e) + _, err := d.Exec(insert, e, "oss") require.NoError(t, err) } } tests := []struct { name string - d *sql.DB setup func() withDeleteLog bool wantEntries []string @@ -276,21 +427,15 @@ func Test_GetMigrationLog(t *testing.T) { }{ { name: "simple", - d: d, setup: func() { createEntries("alice", "eve", "bob") }, wantEntries: []string{"alice", "eve", "bob"}, }, { name: "with-delete-log", - d: d, setup: func() { createEntries("alice", "eve", "bob") }, withDeleteLog: true, wantEntries: []string{"alice", "eve", "bob"}, }, - { - name: "missing-sql-DB", - wantErrMatch: errors.T(errors.InvalidParameter), - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -303,7 +448,7 @@ func Test_GetMigrationLog(t *testing.T) { if tt.setup != nil { tt.setup() } - gotLog, err := GetMigrationLog(ctx, tt.d, WithDeleteLog(tt.withDeleteLog)) + gotLog, err := m.GetMigrationLog(ctx, schema.WithDeleteLog(tt.withDeleteLog)) if tt.wantErrMatch != nil { require.Error(err) assert.Truef(errors.Match(tt.wantErrMatch, err), "expected error with code: %s and got err: %q", tt.wantErrMatch.Code, err) @@ -330,21 +475,3 @@ func Test_GetMigrationLog(t *testing.T) { }) } } - -// Creates a new migrationState only with the versions <= the provided maxVer -func createPartialMigrationState(om migrationState, maxVer int) migrationState { - nState := migrationState{ - upMigrations: make(map[int][]byte), - } - for k := range om.upMigrations { - if k > maxVer { - // Don't store any versions past our test version. - continue - } - nState.upMigrations[k] = om.upMigrations[k] - if nState.binarySchemaVersion < k { - nState.binarySchemaVersion = k - } - } - return nState -} diff --git a/internal/db/schema/migration_log.go b/internal/db/schema/migration_log.go new file mode 100644 index 000000000..c067b7bbb --- /dev/null +++ b/internal/db/schema/migration_log.go @@ -0,0 +1,49 @@ +package schema + +import ( + "context" + "time" + + "github.com/hashicorp/boundary/internal/db/schema/internal/log" + "github.com/hashicorp/boundary/internal/errors" +) + +// LogEntry represents a log entry generated during migrations. +type LogEntry struct { + Id int + MigrationVersion int + MigrationEdition string + CreateTime time.Time + Entry string +} + +// GetMigrationLog will retrieve the migration logs from the db for the last +// migration. Once it's read the entries, it will delete them from the database. +// The WithDeleteLog option is supported and will remove all log entries when provided. +func (b *Manager) GetMigrationLog(ctx context.Context, opt ...Option) ([]LogEntry, error) { + const op = "schema.GetMigrationLog" + + var logOpts []log.Option + opts := getOpts(opt...) + if opts.withDeleteLog { + logOpts = append(logOpts, log.WithDeleteLog(opts.withDeleteLog)) + } + + entries, err := b.driver.GetMigrationLog(ctx, logOpts...) + if err != nil { + return nil, errors.Wrap(ctx, err, op) + } + + logEntries := make([]LogEntry, 0, len(entries)) + for _, e := range entries { + logEntries = append(logEntries, LogEntry{ + Id: e.Id, + MigrationVersion: e.MigrationVersion, + MigrationEdition: e.MigrationEdition, + CreateTime: e.CreateTime, + Entry: e.Entry, + }) + } + + return logEntries, nil +} diff --git a/internal/db/schema/migrations/README.md b/internal/db/schema/migrations/README.md deleted file mode 100644 index 4167cdc46..000000000 --- a/internal/db/schema/migrations/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# migrations package -This package handles the generation of the database schema in a format that can -be compiled into the boundary binary. - -## Organization - -* `./generate`: contains the makefile, code, and templates needed to generate the schema info. -* `./postgres`: contains the versioned schema folders. The contents of these folders, except - for `dev` should not be modified. - -## Usage -To regenerate the schema information into the format the boundary binary uses run -`make migrations` or `make gen` to recreate all generated code. - -The content of the folders under `./postgres` are compiled into the -boundary binary and when the `boundary database init` or `boundary database migrate` -commands are executed they are applied in order of their version. - -The `./postgres/dev` directory contains schema files that are under development and -are not included in a release yet and so it is the only directory where additions and -modifications are allowed. When a boundary binary is built when this directory is not -empty a special flag is required to run the `boundary database init` command to indicate -the user is aware that this is a development release and running this command can -result in a completely broken schema and dataloss. - -When a new release is made the contents of the `dev` directory are moved into a new -versioned directory. \ No newline at end of file diff --git a/internal/db/schema/migrations/base.go b/internal/db/schema/migrations/base.go new file mode 100644 index 000000000..98bd6ae95 --- /dev/null +++ b/internal/db/schema/migrations/base.go @@ -0,0 +1,66 @@ +// Package migrations contains the base sql statements needed to bootstrap the +// migration process. These statements setup the tables necessary for tracking +// the state of migrations. +package migrations + +import ( + "embed" + "path/filepath" + "strings" +) + +// postgres contains the sql for creating the base tables used to manage +// migrations. +//go:embed base/postgres +var postgres embed.FS + +// Base contains the sql migrtaions needed to create the initial based tables used +// to manage other migrations. +// Unlike the other migrations these create statements represent the "current" state of the tables +// and are only used on a fresh install. +type base struct { + CreateSchemaVersion string + CreateLogMigration string +} + +var baseByDialect map[string]base + +func stripBeginCommit(c []byte) string { + contents := strings.TrimSpace(string(c)) + if strings.ToLower(contents[:len("begin;")]) == "begin;" { + contents = contents[len("begin;"):] + } + if strings.ToLower(contents[len(contents)-len("commit;"):]) == "commit;" { + contents = contents[:len(contents)-len("commit;")] + } + contents = strings.TrimSpace(contents) + return contents +} + +func newBase(root string, m embed.FS) base { + schemaVersion, err := m.ReadFile(filepath.Join(root, "01_boundary_schema_version.up.sql")) + if err != nil { + panic("missing base migration file 01_boundary_schema_version.up.sql") + } + + logMigration, err := m.ReadFile(filepath.Join(root, "02_log_migration.up.sql")) + if err != nil { + panic("missing base migration file 02_log_migration.up.sql") + } + + return base{ + CreateSchemaVersion: stripBeginCommit(schemaVersion), + CreateLogMigration: stripBeginCommit(logMigration), + } +} + +func init() { + baseByDialect = map[string]base{ + "postgres": newBase("base/postgres", postgres), + } +} + +// Base returns the base create statements for the given dialect. +func Base(dialect string) base { + return baseByDialect[dialect] +} diff --git a/internal/db/schema/migrations/base/postgres/01_boundary_schema_version.up.sql b/internal/db/schema/migrations/base/postgres/01_boundary_schema_version.up.sql new file mode 100644 index 000000000..e8d689995 --- /dev/null +++ b/internal/db/schema/migrations/base/postgres/01_boundary_schema_version.up.sql @@ -0,0 +1,6 @@ +begin; +create table boundary_schema_version ( + edition text primary key, + version bigint not null +); +commit; diff --git a/internal/db/schema/migrations/base/postgres/02_log_migration.up.sql b/internal/db/schema/migrations/base/postgres/02_log_migration.up.sql new file mode 100644 index 000000000..6df6e007b --- /dev/null +++ b/internal/db/schema/migrations/base/postgres/02_log_migration.up.sql @@ -0,0 +1,34 @@ +begin; +create table log_migration( + id bigint generated always as identity primary key, + migration_version bigint not null, -- cannot declare FK since the table is truncated during runtime + edition text not null, -- cannot declare FK since the table is truncated during runtime + create_time timestamp with time zone default current_timestamp, + entry text not null +); +comment on table log_migration is +'log_migration entries are logging output from database migrations'; + +create or replace function + log_migration_version() + returns trigger +as $$ + declare current_version bigint; + begin + select max(version) + from boundary_schema_version into current_version + where edition = new.edition; + new.migration_version = current_version; + return new; + end; +$$ language plpgsql; +comment on function log_migration_version() is +'log_migration_version will set the log_migration entries to the current migration version'; + +create trigger + version_column +before + insert on log_migration + for each row execute procedure log_migration_version(); + +commit; diff --git a/internal/db/schema/migrations/generate/Makefile b/internal/db/schema/migrations/generate/Makefile deleted file mode 100644 index d067e6c6f..000000000 --- a/internal/db/schema/migrations/generate/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -# Be sure to place this BEFORE `include` directives, if any. -THIS_FILE := $(lastword $(MAKEFILE_LIST)) - -migrations: - go run . - goimports -w ${GEN_BASEPATH}/internal/db/schema - gofumpt -w ${GEN_BASEPATH}/internal/db/schema - -.PHONY: migrations diff --git a/internal/db/schema/migrations/generate/generate.go b/internal/db/schema/migrations/generate/generate.go deleted file mode 100644 index 616cfcaec..000000000 --- a/internal/db/schema/migrations/generate/generate.go +++ /dev/null @@ -1,145 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "sort" - "strconv" - "strings" - "text/template" -) - -// generate looks for migration sql in a directory for the given dialect and -// applies the templates below to the contents of the files, building up a -// migrations map for the dialect -func generate(dialect string) { - baseDir := os.Getenv("GEN_BASEPATH") + "/internal/db/schema" - srcDir := baseDir + "/migrations" - dir, err := os.Open(fmt.Sprintf("%s/%s", srcDir, dialect)) - if err != nil { - fmt.Printf("error opening dir with dialect %s: %v\n", dialect, err) - os.Exit(1) - } - versions, err := dir.Readdirnames(0) - if err != nil { - fmt.Printf("error reading dir names with dialect %s: %v\n", dialect, err) - os.Exit(1) - } - sort.Strings(versions) - - type ContentValues struct { - Name string - Content string - } - var upContents []ContentValues - - var largestSchemaVersion int - for _, ver := range versions { - if strings.HasPrefix(ver, ".") { - continue - } - - var verVal int - verVal, err := strconv.Atoi(ver) - if err != nil { - fmt.Printf("error reading major schema version directory %q. Must be a number: %v\n", ver, err) - os.Exit(1) - } - - dir, err := os.Open(fmt.Sprintf("%s/%s/%s", srcDir, dialect, ver)) - if err != nil { - fmt.Printf("error opening dir with dialect %s: %v\n", dialect, err) - os.Exit(1) - } - names, err := dir.Readdirnames(0) - if err != nil { - fmt.Printf("error reading dir names with dialect %s: %v\n", dialect, err) - os.Exit(1) - } - - sort.Strings(names) - for _, name := range names { - if !strings.HasSuffix(name, ".sql") { - continue - } - - nameParts := strings.SplitN(name, "_", 2) - if len(nameParts) != 2 { - continue - } - - v, err := strconv.Atoi(nameParts[0]) - if err != nil { - fmt.Printf("Unable to get file version from %q\n", name) - continue - } - - fullV := (verVal * 1000) + v - if fullV > largestSchemaVersion { - largestSchemaVersion = fullV - } - - cbts, err := ioutil.ReadFile(fmt.Sprintf("%s/%s/%s/%s", srcDir, dialect, ver, name)) - if err != nil { - fmt.Printf("error opening file %s with dialect %s: %v", name, dialect, err) - os.Exit(1) - } - - contents := strings.TrimSpace(string(cbts)) - if strings.ToLower(contents[:len("begin;")]) == "begin;" { - contents = contents[len("begin;"):] - } - if strings.ToLower(contents[len(contents)-len("commit;"):]) == "commit;" { - contents = contents[:len(contents)-len("commit;")] - } - contents = strings.TrimSpace(contents) - - cv := ContentValues{ - Name: fmt.Sprint(fullV), - Content: contents, - } - switch { - case strings.Contains(nameParts[1], ".up."): - upContents = append(upContents, cv) - } - } - } - - outBuf := bytes.NewBuffer(nil) - if err := migrationsTemplate.Execute(outBuf, struct { - Type string - UpValues []ContentValues - BinarySchemaVersion int - }{ - Type: dialect, - UpValues: upContents, - BinarySchemaVersion: largestSchemaVersion, - }); err != nil { - fmt.Printf("error executing migrations value template for dialect %s: %s", dialect, err) - os.Exit(1) - } - - outFile := fmt.Sprintf("%s/%s_migration.gen.go", baseDir, dialect) - if err := ioutil.WriteFile(outFile, outBuf.Bytes(), 0o644); err != nil { - fmt.Printf("error writing file %q: %v\n", outFile, err) - os.Exit(1) - } -} - -var migrationsTemplate = template.Must(template.Must(template.New("Content").Parse( - `{{ .Name }}: []byte(` + "`\n{{ .Content }}\n`" + `), -`)).New("MainPage").Parse(`package schema - -// Code generated by "make migrations"; DO NOT EDIT. - -func init() { - migrationStates["{{ .Type }}"] = migrationState{ - binarySchemaVersion: {{ .BinarySchemaVersion }}, - upMigrations: map[int][]byte{ - {{range .UpValues }}{{ template "Content" . }}{{end}} - }, - } -} -`)) diff --git a/internal/db/schema/migrations/generate/main.go b/internal/db/schema/migrations/generate/main.go deleted file mode 100644 index e560da4a0..000000000 --- a/internal/db/schema/migrations/generate/main.go +++ /dev/null @@ -1,5 +0,0 @@ -package main - -func main() { - generate("postgres") -} diff --git a/internal/db/schema/migrations/oss/oss.go b/internal/db/schema/migrations/oss/oss.go new file mode 100644 index 000000000..7a9536c47 --- /dev/null +++ b/internal/db/schema/migrations/oss/oss.go @@ -0,0 +1,17 @@ +// Package oss is used to embed the sql statements for the oss edition and +// registering the edition for the schema.Manager. +package oss + +import ( + "embed" + + "github.com/hashicorp/boundary/internal/db/schema" +) + +// postgres contains the migrations sql files for postgres oss edition +//go:embed postgres +var postgres embed.FS + +func init() { + schema.RegisterEdition("oss", schema.Postgres, postgres, 0) +} diff --git a/internal/db/schema/migrations/oss/oss_test.go b/internal/db/schema/migrations/oss/oss_test.go new file mode 100644 index 000000000..bc877888b --- /dev/null +++ b/internal/db/schema/migrations/oss/oss_test.go @@ -0,0 +1,87 @@ +package oss_test + +import ( + "context" + "testing" + + "github.com/hashicorp/boundary/internal/db/common" + "github.com/hashicorp/boundary/internal/db/schema" + "github.com/hashicorp/boundary/testing/dbtest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + // import for init side-effects to include migrations + _ "github.com/hashicorp/boundary/internal/db/schema/migrations/oss" +) + +func TestApplyMigrations(t *testing.T) { + dialect := dbtest.Postgres + + c, u, _, err := dbtest.StartUsingTemplate(dialect) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, c()) + }) + d, err := common.SqlOpen(dialect, u) + require.NoError(t, err) + + ctx := context.Background() + m, err := schema.NewManager(ctx, schema.Dialect(dialect), d) + require.NoError(t, err) + assert.NoError(t, m.ApplyMigrations(ctx)) +} + +func TestApplyMigrations_NotFromFresh(t *testing.T) { + dialect := dbtest.Postgres + + c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1)) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, c()) + }) + d, err := common.SqlOpen(dialect, u) + require.NoError(t, err) + + // Initialize the DB with only a portion of the current sql scripts. + ctx := context.Background() + m, err := schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions( + schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": 1}), + )) + require.NoError(t, err) + assert.NoError(t, m.ApplyMigrations(ctx)) + + state, err := m.CurrentState(ctx) + require.NoError(t, err) + want := &schema.State{ + Initialized: true, + Editions: []schema.EditionState{ + { + Name: "oss", + BinarySchemaVersion: 1, + DatabaseSchemaVersion: 1, + DatabaseSchemaState: schema.Equal, + }, + }, + } + assert.Equal(t, want, state) + + newM, err := schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions( + schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": 3}), + )) + require.NoError(t, err) + assert.NoError(t, newM.ApplyMigrations(ctx)) + state, err = newM.CurrentState(ctx) + require.NoError(t, err) + want = &schema.State{ + Initialized: true, + Editions: []schema.EditionState{ + { + Name: "oss", + BinarySchemaVersion: 3, + DatabaseSchemaVersion: 3, + DatabaseSchemaState: schema.Equal, + }, + }, + } + assert.Equal(t, want, state) +} diff --git a/internal/db/schema/migrations/postgres/0/01_domain_types.up.sql b/internal/db/schema/migrations/oss/postgres/0/01_domain_types.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/01_domain_types.up.sql rename to internal/db/schema/migrations/oss/postgres/0/01_domain_types.up.sql diff --git a/internal/db/schema/migrations/postgres/0/02_oplog.up.sql b/internal/db/schema/migrations/oss/postgres/0/02_oplog.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/02_oplog.up.sql rename to internal/db/schema/migrations/oss/postgres/0/02_oplog.up.sql diff --git a/internal/db/schema/migrations/postgres/0/03_db.up.sql b/internal/db/schema/migrations/oss/postgres/0/03_db.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/03_db.up.sql rename to internal/db/schema/migrations/oss/postgres/0/03_db.up.sql diff --git a/internal/db/schema/migrations/postgres/0/06_iam.up.sql b/internal/db/schema/migrations/oss/postgres/0/06_iam.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/06_iam.up.sql rename to internal/db/schema/migrations/oss/postgres/0/06_iam.up.sql diff --git a/internal/db/schema/migrations/postgres/0/07_auth.up.sql b/internal/db/schema/migrations/oss/postgres/0/07_auth.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/07_auth.up.sql rename to internal/db/schema/migrations/oss/postgres/0/07_auth.up.sql diff --git a/internal/db/schema/migrations/postgres/0/08_servers.up.sql b/internal/db/schema/migrations/oss/postgres/0/08_servers.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/08_servers.up.sql rename to internal/db/schema/migrations/oss/postgres/0/08_servers.up.sql diff --git a/internal/db/schema/migrations/postgres/0/11_auth_token.up.sql b/internal/db/schema/migrations/oss/postgres/0/11_auth_token.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/11_auth_token.up.sql rename to internal/db/schema/migrations/oss/postgres/0/11_auth_token.up.sql diff --git a/internal/db/schema/migrations/postgres/0/12_auth_password.up.sql b/internal/db/schema/migrations/oss/postgres/0/12_auth_password.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/12_auth_password.up.sql rename to internal/db/schema/migrations/oss/postgres/0/12_auth_password.up.sql diff --git a/internal/db/schema/migrations/postgres/0/13_auth_password_argon.up.sql b/internal/db/schema/migrations/oss/postgres/0/13_auth_password_argon.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/13_auth_password_argon.up.sql rename to internal/db/schema/migrations/oss/postgres/0/13_auth_password_argon.up.sql diff --git a/internal/db/schema/migrations/postgres/0/14_auth_password_views.up.sql b/internal/db/schema/migrations/oss/postgres/0/14_auth_password_views.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/14_auth_password_views.up.sql rename to internal/db/schema/migrations/oss/postgres/0/14_auth_password_views.up.sql diff --git a/internal/db/schema/migrations/postgres/0/20_host.up.sql b/internal/db/schema/migrations/oss/postgres/0/20_host.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/20_host.up.sql rename to internal/db/schema/migrations/oss/postgres/0/20_host.up.sql diff --git a/internal/db/schema/migrations/postgres/0/22_static_host.up.sql b/internal/db/schema/migrations/oss/postgres/0/22_static_host.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/22_static_host.up.sql rename to internal/db/schema/migrations/oss/postgres/0/22_static_host.up.sql diff --git a/internal/db/schema/migrations/postgres/0/30_keys.up.sql b/internal/db/schema/migrations/oss/postgres/0/30_keys.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/30_keys.up.sql rename to internal/db/schema/migrations/oss/postgres/0/30_keys.up.sql diff --git a/internal/db/schema/migrations/postgres/0/31_keys.up.sql b/internal/db/schema/migrations/oss/postgres/0/31_keys.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/31_keys.up.sql rename to internal/db/schema/migrations/oss/postgres/0/31_keys.up.sql diff --git a/internal/db/schema/migrations/postgres/0/40_targets.up.sql b/internal/db/schema/migrations/oss/postgres/0/40_targets.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/40_targets.up.sql rename to internal/db/schema/migrations/oss/postgres/0/40_targets.up.sql diff --git a/internal/db/schema/migrations/postgres/0/41_targets.up.sql b/internal/db/schema/migrations/oss/postgres/0/41_targets.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/41_targets.up.sql rename to internal/db/schema/migrations/oss/postgres/0/41_targets.up.sql diff --git a/internal/db/schema/migrations/postgres/0/50_session.up.sql b/internal/db/schema/migrations/oss/postgres/0/50_session.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/50_session.up.sql rename to internal/db/schema/migrations/oss/postgres/0/50_session.up.sql diff --git a/internal/db/schema/migrations/postgres/0/51_connection.up.sql b/internal/db/schema/migrations/oss/postgres/0/51_connection.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/51_connection.up.sql rename to internal/db/schema/migrations/oss/postgres/0/51_connection.up.sql diff --git a/internal/db/schema/migrations/postgres/0/60_wh_domain_types.up.sql b/internal/db/schema/migrations/oss/postgres/0/60_wh_domain_types.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/60_wh_domain_types.up.sql rename to internal/db/schema/migrations/oss/postgres/0/60_wh_domain_types.up.sql diff --git a/internal/db/schema/migrations/postgres/0/62_wh_datetime.up.sql b/internal/db/schema/migrations/oss/postgres/0/62_wh_datetime.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/62_wh_datetime.up.sql rename to internal/db/schema/migrations/oss/postgres/0/62_wh_datetime.up.sql diff --git a/internal/db/schema/migrations/postgres/0/65_wh_session_dimensions.up.sql b/internal/db/schema/migrations/oss/postgres/0/65_wh_session_dimensions.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/65_wh_session_dimensions.up.sql rename to internal/db/schema/migrations/oss/postgres/0/65_wh_session_dimensions.up.sql diff --git a/internal/db/schema/migrations/postgres/0/66_wh_session_dimensions.up.sql b/internal/db/schema/migrations/oss/postgres/0/66_wh_session_dimensions.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/66_wh_session_dimensions.up.sql rename to internal/db/schema/migrations/oss/postgres/0/66_wh_session_dimensions.up.sql diff --git a/internal/db/schema/migrations/postgres/0/68_wh_session_facts.up.sql b/internal/db/schema/migrations/oss/postgres/0/68_wh_session_facts.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/68_wh_session_facts.up.sql rename to internal/db/schema/migrations/oss/postgres/0/68_wh_session_facts.up.sql diff --git a/internal/db/schema/migrations/postgres/0/69_wh_session_facts.up.sql b/internal/db/schema/migrations/oss/postgres/0/69_wh_session_facts.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/0/69_wh_session_facts.up.sql rename to internal/db/schema/migrations/oss/postgres/0/69_wh_session_facts.up.sql diff --git a/internal/db/schema/migrations/postgres/1/01_server_tags_migrations.up.sql b/internal/db/schema/migrations/oss/postgres/1/01_server_tags_migrations.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/1/01_server_tags_migrations.up.sql rename to internal/db/schema/migrations/oss/postgres/1/01_server_tags_migrations.up.sql diff --git a/internal/db/schema/migrations/postgres/1/02_domains.up.sql b/internal/db/schema/migrations/oss/postgres/1/02_domains.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/1/02_domains.up.sql rename to internal/db/schema/migrations/oss/postgres/1/02_domains.up.sql diff --git a/internal/db/schema/migrations/postgres/1/03_kms.up.sql b/internal/db/schema/migrations/oss/postgres/1/03_kms.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/1/03_kms.up.sql rename to internal/db/schema/migrations/oss/postgres/1/03_kms.up.sql diff --git a/internal/db/schema/migrations/postgres/10/01_job_migrations.up.sql b/internal/db/schema/migrations/oss/postgres/10/01_job_migrations.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/10/01_job_migrations.up.sql rename to internal/db/schema/migrations/oss/postgres/10/01_job_migrations.up.sql diff --git a/internal/db/schema/migrations/postgres/10/02_sentinels.up.sql b/internal/db/schema/migrations/oss/postgres/10/02_sentinels.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/10/02_sentinels.up.sql rename to internal/db/schema/migrations/oss/postgres/10/02_sentinels.up.sql diff --git a/internal/db/schema/migrations/postgres/10/03_credential.up.sql b/internal/db/schema/migrations/oss/postgres/10/03_credential.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/10/03_credential.up.sql rename to internal/db/schema/migrations/oss/postgres/10/03_credential.up.sql diff --git a/internal/db/schema/migrations/postgres/10/04_vault_credential.up.sql b/internal/db/schema/migrations/oss/postgres/10/04_vault_credential.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/10/04_vault_credential.up.sql rename to internal/db/schema/migrations/oss/postgres/10/04_vault_credential.up.sql diff --git a/internal/db/schema/migrations/postgres/10/05_target.up.sql b/internal/db/schema/migrations/oss/postgres/10/05_target.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/10/05_target.up.sql rename to internal/db/schema/migrations/oss/postgres/10/05_target.up.sql diff --git a/internal/db/schema/migrations/postgres/10/06_session.up.sql b/internal/db/schema/migrations/oss/postgres/10/06_session.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/10/06_session.up.sql rename to internal/db/schema/migrations/oss/postgres/10/06_session.up.sql diff --git a/internal/db/schema/migrations/postgres/10/07_sentinels.up.sql b/internal/db/schema/migrations/oss/postgres/10/07_sentinels.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/10/07_sentinels.up.sql rename to internal/db/schema/migrations/oss/postgres/10/07_sentinels.up.sql diff --git a/internal/db/schema/migrations/postgres/11/01_server_type_enum.up.sql b/internal/db/schema/migrations/oss/postgres/11/01_server_type_enum.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/11/01_server_type_enum.up.sql rename to internal/db/schema/migrations/oss/postgres/11/01_server_type_enum.up.sql diff --git a/internal/db/schema/migrations/postgres/12/01_timestamp_sub_funcs.up.sql b/internal/db/schema/migrations/oss/postgres/12/01_timestamp_sub_funcs.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/12/01_timestamp_sub_funcs.up.sql rename to internal/db/schema/migrations/oss/postgres/12/01_timestamp_sub_funcs.up.sql diff --git a/internal/db/schema/migrations/postgres/13/01_oidc_add_claims.up.sql b/internal/db/schema/migrations/oss/postgres/13/01_oidc_add_claims.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/13/01_oidc_add_claims.up.sql rename to internal/db/schema/migrations/oss/postgres/13/01_oidc_add_claims.up.sql diff --git a/internal/db/schema/migrations/postgres/14/01_wh_user_dimension_oidc.up.sql b/internal/db/schema/migrations/oss/postgres/14/01_wh_user_dimension_oidc.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/14/01_wh_user_dimension_oidc.up.sql rename to internal/db/schema/migrations/oss/postgres/14/01_wh_user_dimension_oidc.up.sql diff --git a/internal/db/schema/migrations/postgres/15/01_wh_rename_key_columns.up.sql b/internal/db/schema/migrations/oss/postgres/15/01_wh_rename_key_columns.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/15/01_wh_rename_key_columns.up.sql rename to internal/db/schema/migrations/oss/postgres/15/01_wh_rename_key_columns.up.sql diff --git a/internal/db/schema/migrations/postgres/15/02_wh_rename_host_dimension_org.up.sql b/internal/db/schema/migrations/oss/postgres/15/02_wh_rename_host_dimension_org.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/15/02_wh_rename_host_dimension_org.up.sql rename to internal/db/schema/migrations/oss/postgres/15/02_wh_rename_host_dimension_org.up.sql diff --git a/internal/db/schema/migrations/postgres/16/01_wh_credential_dimension.up.sql b/internal/db/schema/migrations/oss/postgres/16/01_wh_credential_dimension.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/16/01_wh_credential_dimension.up.sql rename to internal/db/schema/migrations/oss/postgres/16/01_wh_credential_dimension.up.sql diff --git a/internal/db/schema/migrations/postgres/16/02_wh_credential_dimension.up.sql b/internal/db/schema/migrations/oss/postgres/16/02_wh_credential_dimension.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/16/02_wh_credential_dimension.up.sql rename to internal/db/schema/migrations/oss/postgres/16/02_wh_credential_dimension.up.sql diff --git a/internal/db/schema/migrations/postgres/16/03_wh_credential_dimension.up.sql b/internal/db/schema/migrations/oss/postgres/16/03_wh_credential_dimension.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/16/03_wh_credential_dimension.up.sql rename to internal/db/schema/migrations/oss/postgres/16/03_wh_credential_dimension.up.sql diff --git a/internal/db/schema/migrations/postgres/16/04_wh_credential_dimension.up.sql b/internal/db/schema/migrations/oss/postgres/16/04_wh_credential_dimension.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/16/04_wh_credential_dimension.up.sql rename to internal/db/schema/migrations/oss/postgres/16/04_wh_credential_dimension.up.sql diff --git a/internal/db/schema/migrations/postgres/16/05_wh_credential_dimension.up.sql b/internal/db/schema/migrations/oss/postgres/16/05_wh_credential_dimension.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/16/05_wh_credential_dimension.up.sql rename to internal/db/schema/migrations/oss/postgres/16/05_wh_credential_dimension.up.sql diff --git a/internal/db/schema/migrations/oss/postgres/2/01_log.up.sql b/internal/db/schema/migrations/oss/postgres/2/01_log.up.sql new file mode 100644 index 000000000..a0d7c9d5b --- /dev/null +++ b/internal/db/schema/migrations/oss/postgres/2/01_log.up.sql @@ -0,0 +1,54 @@ +begin; + +-- management of the log_migation table is moved into +-- the schema.Manager prior to migrations running +-- since it interacts with the boundary_schema_version table + +-- log_migration entries represent logs generated during migrations +-- create table log_migration( +-- id bigint generated always as identity primary key, +-- migration_version bigint not null, -- cannot declare FK since the table is truncated during runtime +-- create_time wt_timestamp, +-- entry text not null +-- ); +-- comment on table log_migration is +-- 'log_migration entries are logging output from databaes migrations'; +-- +-- -- log_migration triggers +-- create trigger +-- default_create_time_column +-- before +-- insert on log_migration +-- for each row execute procedure default_create_time(); +-- +-- create trigger +-- immutable_columns +-- before +-- update on log_migration +-- for each row execute procedure immutable_columns('id', 'migration_version', 'create_time', 'entry'); +-- +-- +-- -- log_migration_version() defines a function to be used in a "before update" +-- -- trigger for log_migrations entries. Its intent: set the log_migration +-- -- version column to the current migration version. +-- create or replace function +-- log_migration_version() +-- returns trigger +-- as $$ +-- declare current_version bigint; +-- begin +-- select max(version) from boundary_schema_version into current_version; +-- new.migration_version = current_version; +-- return new; +-- end; +-- $$ language plpgsql; +-- comment on function log_migration_version() is +-- 'log_migration_version will set the log_migration entries to the current migration version'; +-- +-- create trigger +-- migration_version_column +-- before +-- insert on log_migration +-- for each row execute procedure log_migration_version(); +-- +commit; diff --git a/internal/db/schema/migrations/postgres/2/03_oidc_enm.up.sql b/internal/db/schema/migrations/oss/postgres/2/03_oidc_enm.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/2/03_oidc_enm.up.sql rename to internal/db/schema/migrations/oss/postgres/2/03_oidc_enm.up.sql diff --git a/internal/db/schema/migrations/postgres/2/04_oidc.up.sql b/internal/db/schema/migrations/oss/postgres/2/04_oidc.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/2/04_oidc.up.sql rename to internal/db/schema/migrations/oss/postgres/2/04_oidc.up.sql diff --git a/internal/db/schema/migrations/postgres/2/05_authtoken.up.sql b/internal/db/schema/migrations/oss/postgres/2/05_authtoken.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/2/05_authtoken.up.sql rename to internal/db/schema/migrations/oss/postgres/2/05_authtoken.up.sql diff --git a/internal/db/schema/migrations/postgres/2/06_iam.up.sql b/internal/db/schema/migrations/oss/postgres/2/06_iam.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/2/06_iam.up.sql rename to internal/db/schema/migrations/oss/postgres/2/06_iam.up.sql diff --git a/internal/db/schema/migrations/postgres/2/07_iam.up.sql b/internal/db/schema/migrations/oss/postgres/2/07_iam.up.sql similarity index 96% rename from internal/db/schema/migrations/postgres/2/07_iam.up.sql rename to internal/db/schema/migrations/oss/postgres/2/07_iam.up.sql index e94d73f7e..bff54b24c 100644 --- a/internal/db/schema/migrations/postgres/2/07_iam.up.sql +++ b/internal/db/schema/migrations/oss/postgres/2/07_iam.up.sql @@ -50,7 +50,7 @@ with many_authmethod (scope_id, authmethod_cnt) as ( am.scope_id = many.scope_id and many.cnt > 1 ) -insert into log_migration(entry) +insert into log_migration(entry, edition) select distinct concat( 'unable to set primary_auth_method for ', @@ -58,7 +58,7 @@ select ' there were ', m.authmethod_cnt, ' password auth methods for that scope.' - ) as entry + ) as entry, 'oss' from iam_scope s, many_authmethod m diff --git a/internal/db/schema/migrations/postgres/2/10_auth.up.sql b/internal/db/schema/migrations/oss/postgres/2/10_auth.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/2/10_auth.up.sql rename to internal/db/schema/migrations/oss/postgres/2/10_auth.up.sql diff --git a/internal/db/schema/migrations/postgres/2/15_oidc.up.sql b/internal/db/schema/migrations/oss/postgres/2/15_oidc.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/2/15_oidc.up.sql rename to internal/db/schema/migrations/oss/postgres/2/15_oidc.up.sql diff --git a/internal/db/schema/migrations/postgres/2/20_pass.up.sql b/internal/db/schema/migrations/oss/postgres/2/20_pass.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/2/20_pass.up.sql rename to internal/db/schema/migrations/oss/postgres/2/20_pass.up.sql diff --git a/internal/db/schema/migrations/postgres/3/01_auth.up.sql b/internal/db/schema/migrations/oss/postgres/3/01_auth.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/3/01_auth.up.sql rename to internal/db/schema/migrations/oss/postgres/3/01_auth.up.sql diff --git a/internal/db/schema/migrations/postgres/3/02_iam.up.sql b/internal/db/schema/migrations/oss/postgres/3/02_iam.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/3/02_iam.up.sql rename to internal/db/schema/migrations/oss/postgres/3/02_iam.up.sql diff --git a/internal/db/schema/migrations/postgres/4/01_iam.up.sql b/internal/db/schema/migrations/oss/postgres/4/01_iam.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/4/01_iam.up.sql rename to internal/db/schema/migrations/oss/postgres/4/01_iam.up.sql diff --git a/internal/db/schema/migrations/postgres/5/01_oidc.up.sql b/internal/db/schema/migrations/oss/postgres/5/01_oidc.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/5/01_oidc.up.sql rename to internal/db/schema/migrations/oss/postgres/5/01_oidc.up.sql diff --git a/internal/db/schema/migrations/postgres/6/01_oidc.up.sql b/internal/db/schema/migrations/oss/postgres/6/01_oidc.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/6/01_oidc.up.sql rename to internal/db/schema/migrations/oss/postgres/6/01_oidc.up.sql diff --git a/internal/db/schema/migrations/postgres/7/01_functions.up.sql b/internal/db/schema/migrations/oss/postgres/7/01_functions.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/7/01_functions.up.sql rename to internal/db/schema/migrations/oss/postgres/7/01_functions.up.sql diff --git a/internal/db/schema/migrations/postgres/7/02_plugin.up.sql b/internal/db/schema/migrations/oss/postgres/7/02_plugin.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/7/02_plugin.up.sql rename to internal/db/schema/migrations/oss/postgres/7/02_plugin.up.sql diff --git a/internal/db/schema/migrations/postgres/7/03_job.up.sql b/internal/db/schema/migrations/oss/postgres/7/03_job.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/7/03_job.up.sql rename to internal/db/schema/migrations/oss/postgres/7/03_job.up.sql diff --git a/internal/db/schema/migrations/postgres/8/01_connection.up.sql b/internal/db/schema/migrations/oss/postgres/8/01_connection.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/8/01_connection.up.sql rename to internal/db/schema/migrations/oss/postgres/8/01_connection.up.sql diff --git a/internal/db/schema/migrations/postgres/8/08_connection_test.go b/internal/db/schema/migrations/oss/postgres/8/08_connection_test.go similarity index 100% rename from internal/db/schema/migrations/postgres/8/08_connection_test.go rename to internal/db/schema/migrations/oss/postgres/8/08_connection_test.go diff --git a/internal/db/schema/migrations/postgres/9/01_managed_groups.up.sql b/internal/db/schema/migrations/oss/postgres/9/01_managed_groups.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/9/01_managed_groups.up.sql rename to internal/db/schema/migrations/oss/postgres/9/01_managed_groups.up.sql diff --git a/internal/db/schema/migrations/postgres/9/02_oidc_managed_group.up.sql b/internal/db/schema/migrations/oss/postgres/9/02_oidc_managed_group.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/9/02_oidc_managed_group.up.sql rename to internal/db/schema/migrations/oss/postgres/9/02_oidc_managed_group.up.sql diff --git a/internal/db/schema/migrations/postgres/9/03_oidc_managed_group_member.up.sql b/internal/db/schema/migrations/oss/postgres/9/03_oidc_managed_group_member.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/9/03_oidc_managed_group_member.up.sql rename to internal/db/schema/migrations/oss/postgres/9/03_oidc_managed_group_member.up.sql diff --git a/internal/db/schema/migrations/postgres/9/04_oidc_managed_group_principal_role.up.sql b/internal/db/schema/migrations/oss/postgres/9/04_oidc_managed_group_principal_role.up.sql similarity index 100% rename from internal/db/schema/migrations/postgres/9/04_oidc_managed_group_principal_role.up.sql rename to internal/db/schema/migrations/oss/postgres/9/04_oidc_managed_group_principal_role.up.sql diff --git a/internal/db/schema/migrations/postgres/9/managed_group_test.go b/internal/db/schema/migrations/oss/postgres/9/managed_group_test.go similarity index 100% rename from internal/db/schema/migrations/postgres/9/managed_group_test.go rename to internal/db/schema/migrations/oss/postgres/9/managed_group_test.go diff --git a/internal/db/schema/migrations/postgres/11/01_server_type_enum_test.go b/internal/db/schema/migrations/oss/postgres_11_01_test.go similarity index 78% rename from internal/db/schema/migrations/postgres/11/01_server_type_enum_test.go rename to internal/db/schema/migrations/oss/postgres_11_01_test.go index 4f3c675b2..5ea3d60c1 100644 --- a/internal/db/schema/migrations/postgres/11/01_server_type_enum_test.go +++ b/internal/db/schema/migrations/oss/postgres_11_01_test.go @@ -1,4 +1,4 @@ -package migration +package oss_test import ( "context" @@ -36,6 +36,9 @@ type testServer struct { // 3) running the enum migration // // 4) asserting some bits about the state of the db. +// +// Tests migration: +// migrations/oss/11/01_server_type_enum.up.sql func Test_ServerEnumChanges(t *testing.T) { t.Parallel() require := require.New(t) @@ -53,38 +56,53 @@ func Test_ServerEnumChanges(t *testing.T) { require.NoError(err) // migration to the prior migration (before the one we want to test) - oState := schema.TestCloneMigrationStates(t) - nState := schema.TestCreatePartialMigrationState(oState["postgres"], priorMigration) - oState["postgres"] = nState - - m, err := schema.NewManager(ctx, dialect, d, schema.WithMigrationStates(oState)) + m, err := schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions( + schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": priorMigration}), + )) require.NoError(err) - require.NoError(m.RollForward(ctx)) + require.NoError(m.ApplyMigrations(ctx)) state, err := m.CurrentState(ctx) require.NoError(err) - require.Equal(priorMigration, state.DatabaseSchemaVersion) - require.False(state.Dirty) + want := &schema.State{ + Initialized: true, + Editions: []schema.EditionState{ + { + Name: "oss", + BinarySchemaVersion: priorMigration, + DatabaseSchemaVersion: priorMigration, + DatabaseSchemaState: schema.Equal, + }, + }, + } + require.Equal(want, state) // Seed the data expectedController := insertServer(ctx, t, d, "test-controller", "controller", "127.0.0.1") expectedWorker := insertServer(ctx, t, d, "test-worker", "worker", "127.0.0.1") // now we're ready for the migration we want to test. - oState = schema.TestCloneMigrationStates(t) - nState = schema.TestCreatePartialMigrationState(oState["postgres"], serverEnumMigration) - oState["postgres"] = nState - - m, err = schema.NewManager(ctx, dialect, d, schema.WithMigrationStates(oState)) + m, err = schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions( + schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": serverEnumMigration}), + )) require.NoError(err) - require.NoError(m.RollForward(ctx)) + require.NoError(m.ApplyMigrations(ctx)) state, err = m.CurrentState(ctx) require.NoError(err) - require.Equal(serverEnumMigration, state.DatabaseSchemaVersion) - require.False(state.Dirty) - t.Log("current migration version: ", state.DatabaseSchemaVersion) + want = &schema.State{ + Initialized: true, + Editions: []schema.EditionState{ + { + Name: "oss", + BinarySchemaVersion: serverEnumMigration, + DatabaseSchemaVersion: serverEnumMigration, + DatabaseSchemaState: schema.Equal, + }, + }, + } + require.Equal(want, state) // Assert servers actualController := selectServer(ctx, t, d, "test-controller") diff --git a/internal/db/schema/migrations/postgres/12/01_timestamp_sub_funcs_test.go b/internal/db/schema/migrations/oss/postgres_12_01_test.go similarity index 65% rename from internal/db/schema/migrations/postgres/12/01_timestamp_sub_funcs_test.go rename to internal/db/schema/migrations/oss/postgres_12_01_test.go index 7519b73ab..f6cf43a0c 100644 --- a/internal/db/schema/migrations/postgres/12/01_timestamp_sub_funcs_test.go +++ b/internal/db/schema/migrations/oss/postgres_12_01_test.go @@ -1,4 +1,4 @@ -package migration +package oss_test import ( "context" @@ -12,8 +12,8 @@ import ( "github.com/stretchr/testify/require" ) -const targetMigration = 12001 - +// Tests migration: +// migrations/oss/12/01_timestamp_sub_funcs.up.sql func TestWtSubSeconds(t *testing.T) { t.Parallel() require := require.New(t) @@ -34,6 +34,7 @@ func TestWtSubSeconds(t *testing.T) { func testSetupDb(ctx context.Context, t *testing.T) *sql.DB { t.Helper() require := require.New(t) + const targetMigration = 12001 dialect := dbtest.Postgres @@ -45,18 +46,25 @@ func testSetupDb(ctx context.Context, t *testing.T) *sql.DB { d, err := common.SqlOpen(dialect, u) require.NoError(err) - oState := schema.TestCloneMigrationStates(t) - nState := schema.TestCreatePartialMigrationState(oState["postgres"], targetMigration) - oState["postgres"] = nState - - m, err := schema.NewManager(ctx, dialect, d, schema.WithMigrationStates(oState)) + m, err := schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions( + schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": targetMigration}), + )) require.NoError(err) - require.NoError(m.RollForward(ctx)) + require.NoError(m.ApplyMigrations(ctx)) state, err := m.CurrentState(ctx) require.NoError(err) - require.Equal(targetMigration, state.DatabaseSchemaVersion) - require.False(state.Dirty) - + want := &schema.State{ + Initialized: true, + Editions: []schema.EditionState{ + { + Name: "oss", + BinarySchemaVersion: targetMigration, + DatabaseSchemaVersion: targetMigration, + DatabaseSchemaState: schema.Equal, + }, + }, + } + require.Equal(want, state) return d } diff --git a/internal/db/schema/migrations/postgres/14/warehouse_user_dim_test.go b/internal/db/schema/migrations/oss/postgres_14_01_test.go similarity index 80% rename from internal/db/schema/migrations/postgres/14/warehouse_user_dim_test.go rename to internal/db/schema/migrations/oss/postgres_14_01_test.go index 749190bbe..3483a078a 100644 --- a/internal/db/schema/migrations/postgres/14/warehouse_user_dim_test.go +++ b/internal/db/schema/migrations/oss/postgres_14_01_test.go @@ -1,4 +1,4 @@ -package migration +package oss_test import ( "context" @@ -40,18 +40,26 @@ func TestMigrations_UserDimension(t *testing.T) { require.NoError(err) // migration to the prior migration (before the one we want to test) - oState := schema.TestCloneMigrationStates(t) - nState := schema.TestCreatePartialMigrationState(oState["postgres"], priorMigration) - oState["postgres"] = nState - - m, err := schema.NewManager(ctx, dialect, d, schema.WithMigrationStates(oState)) + m, err := schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions( + schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": priorMigration}), + )) require.NoError(err) - assert.NoError(m.RollForward(ctx)) + assert.NoError(m.ApplyMigrations(ctx)) state, err := m.CurrentState(ctx) require.NoError(err) - assert.Equal(priorMigration, state.DatabaseSchemaVersion) - assert.False(state.Dirty) + want := &schema.State{ + Initialized: true, + Editions: []schema.EditionState{ + { + Name: "oss", + BinarySchemaVersion: priorMigration, + DatabaseSchemaVersion: priorMigration, + DatabaseSchemaState: schema.Equal, + }, + }, + } + require.Equal(want, state) // okay, now we can seed the database with test data dbType, err := db.StringToDbType(dialect) @@ -124,18 +132,26 @@ func TestMigrations_UserDimension(t *testing.T) { } // now we're ready for the migration we want to test. - oState = schema.TestCloneMigrationStates(t) - nState = schema.TestCreatePartialMigrationState(oState["postgres"], currentMigration) - oState["postgres"] = nState - - m, err = schema.NewManager(ctx, dialect, d, schema.WithMigrationStates(oState)) + m, err = schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions( + schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": currentMigration}), + )) require.NoError(err) - assert.NoError(m.RollForward(ctx)) + assert.NoError(m.ApplyMigrations(ctx)) state, err = m.CurrentState(ctx) require.NoError(err) - assert.Equal(currentMigration, state.DatabaseSchemaVersion) - assert.False(state.Dirty) + want = &schema.State{ + Initialized: true, + Editions: []schema.EditionState{ + { + Name: "oss", + BinarySchemaVersion: currentMigration, + DatabaseSchemaVersion: currentMigration, + DatabaseSchemaState: schema.Equal, + }, + }, + } + require.Equal(want, state) } func testOidcAuthToken(t *testing.T, conn *db.DB, kms *kms.Kms, wrapper wrapping.Wrapper, scopeId string) *authtoken.AuthToken { diff --git a/internal/db/schema/migrations/postgres/16/warehouse_credential_dim_test.go b/internal/db/schema/migrations/oss/postgres_16_05_test.go similarity index 72% rename from internal/db/schema/migrations/postgres/16/warehouse_credential_dim_test.go rename to internal/db/schema/migrations/oss/postgres_16_05_test.go index 2f82503f0..f2d960738 100644 --- a/internal/db/schema/migrations/postgres/16/warehouse_credential_dim_test.go +++ b/internal/db/schema/migrations/oss/postgres_16_05_test.go @@ -1,15 +1,14 @@ -package migration +package oss_test import ( "context" - "database/sql" "testing" - "github.com/hashicorp/boundary/internal/auth/oidc" "github.com/hashicorp/boundary/internal/authtoken" "github.com/hashicorp/boundary/internal/credential" "github.com/hashicorp/boundary/internal/credential/vault" "github.com/hashicorp/boundary/internal/db" + "github.com/hashicorp/boundary/internal/db/common" "github.com/hashicorp/boundary/internal/db/schema" "github.com/hashicorp/boundary/internal/host/static" "github.com/hashicorp/boundary/internal/iam" @@ -38,22 +37,30 @@ func TestMigrations_CredentialDimension(t *testing.T) { t.Cleanup(func() { require.NoError(c()) }) - d, err := sql.Open(dialect, u) + d, err := common.SqlOpen(dialect, u) require.NoError(err) // migration to the prior migration (before the one we want to test) - oState := schema.TestCloneMigrationStates(t) - nState := schema.TestCreatePartialMigrationState(oState["postgres"], priorMigration) - oState["postgres"] = nState - - m, err := schema.NewManager(ctx, dialect, d, schema.WithMigrationStates(oState)) + m, err := schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions( + schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": priorMigration}), + )) require.NoError(err) - assert.NoError(m.RollForward(ctx)) + assert.NoError(m.ApplyMigrations(ctx)) state, err := m.CurrentState(ctx) require.NoError(err) - assert.Equal(priorMigration, state.DatabaseSchemaVersion) - assert.False(state.Dirty) + want := &schema.State{ + Initialized: true, + Editions: []schema.EditionState{ + { + Name: "oss", + BinarySchemaVersion: priorMigration, + DatabaseSchemaVersion: priorMigration, + DatabaseSchemaState: schema.Equal, + }, + }, + } + require.Equal(want, state) dbType, err := db.StringToDbType(dialect) require.NoError(err) @@ -129,45 +136,25 @@ func TestMigrations_CredentialDimension(t *testing.T) { } // now we're ready for the migration we want to test. - oState = schema.TestCloneMigrationStates(t) - nState = schema.TestCreatePartialMigrationState(oState["postgres"], currentMigration) - oState["postgres"] = nState - - m, err = schema.NewManager(ctx, dialect, d, schema.WithMigrationStates(oState)) + m, err = schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions( + schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": currentMigration}), + )) require.NoError(err) - assert.NoError(m.RollForward(ctx)) + assert.NoError(m.ApplyMigrations(ctx)) state, err = m.CurrentState(ctx) - require.NoError(err) - assert.Equal(currentMigration, state.DatabaseSchemaVersion) - assert.False(state.Dirty) -} - -func testOidcAuthToken(t *testing.T, conn *db.DB, kms *kms.Kms, wrapper wrapping.Wrapper, scopeId string) *authtoken.AuthToken { - t.Helper() - - authMethod := oidc.TestAuthMethod( - t, conn, wrapper, scopeId, oidc.ActivePrivateState, - "alice-rp", "fido", - oidc.WithIssuer(oidc.TestConvertToUrls(t, "https://www.alice.com")[0]), - oidc.WithSigningAlgs(oidc.RS256), - oidc.WithApiUrl(oidc.TestConvertToUrls(t, "https://www.alice.com/callback")[0]), - ) - acct := oidc.TestAccount(t, conn, authMethod, "test-subject") - - ctx := context.Background() - rw := db.New(conn) - iamRepo, err := iam.NewRepository(rw, rw, kms) - require.NoError(t, err) - - u := iam.TestUser(t, iamRepo, scopeId, iam.WithAccountIds(acct.PublicId)) - - repo, err := authtoken.NewRepository(rw, rw, kms) - require.NoError(t, err) - - at, err := repo.CreateAuthToken(ctx, u, acct.GetPublicId()) - require.NoError(t, err) - return at + want = &schema.State{ + Initialized: true, + Editions: []schema.EditionState{ + { + Name: "oss", + BinarySchemaVersion: currentMigration, + DatabaseSchemaVersion: currentMigration, + DatabaseSchemaState: schema.Equal, + }, + }, + } + require.Equal(want, state) } func testSessionCredentialParams(t *testing.T, conn *db.DB, kms *kms.Kms, wrapper wrapping.Wrapper, tar *target.TcpTarget) []*session.DynamicCredential { diff --git a/internal/db/schema/migrations/postgres/2/07_iam_test.go b/internal/db/schema/migrations/oss/postgres_2_07_test.go similarity index 71% rename from internal/db/schema/migrations/postgres/2/07_iam_test.go rename to internal/db/schema/migrations/oss/postgres_2_07_test.go index 04e76927b..4e8273148 100644 --- a/internal/db/schema/migrations/postgres/2/07_iam_test.go +++ b/internal/db/schema/migrations/oss/postgres_2_07_test.go @@ -1,4 +1,4 @@ -package migration +package oss_test import ( "context" @@ -15,6 +15,8 @@ import ( "github.com/stretchr/testify/require" ) +// Tests migration: +// migrations/oss/2/07_iam.up.sql func Test_PrimaryAuthMethodChanges(t *testing.T) { t.Parallel() const priorMigration = 2006 @@ -42,18 +44,26 @@ func Test_PrimaryAuthMethodChanges(t *testing.T) { require.NoError(err) // migration to the prior migration (before the one we want to test) - oState := schema.TestCloneMigrationStates(t) - nState := schema.TestCreatePartialMigrationState(oState["postgres"], priorMigration) - oState["postgres"] = nState - - m, err := schema.NewManager(ctx, dialect, d, schema.WithMigrationStates(oState)) + m, err := schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions( + schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": priorMigration}), + )) require.NoError(err) - assert.NoError(m.RollForward(ctx)) + assert.NoError(m.ApplyMigrations(ctx)) state, err := m.CurrentState(ctx) require.NoError(err) - assert.Equal(priorMigration, state.DatabaseSchemaVersion) - assert.False(state.Dirty) + want := &schema.State{ + Initialized: true, + Editions: []schema.EditionState{ + { + Name: "oss", + BinarySchemaVersion: priorMigration, + DatabaseSchemaVersion: priorMigration, + DatabaseSchemaState: schema.Equal, + }, + }, + } + assert.Equal(want, state) // okay, now we can seed the database with test data dbType, err := db.StringToDbType(dialect) @@ -73,25 +83,29 @@ func Test_PrimaryAuthMethodChanges(t *testing.T) { org4AuthMethods := password.TestAuthMethods(t, conn, org4.PublicId, 1) // now we're ready for the migration we want to test. - oState = schema.TestCloneMigrationStates(t) - nState = schema.TestCreatePartialMigrationState(oState["postgres"], primaryAuthMethodMigration) - oState["postgres"] = nState - - m, err = schema.NewManager(ctx, dialect, d, schema.WithMigrationStates(oState)) + m, err = schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions( + schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": primaryAuthMethodMigration}), + )) require.NoError(err) - assert.NoError(m.RollForward(ctx)) + assert.NoError(m.ApplyMigrations(ctx)) state, err = m.CurrentState(ctx) require.NoError(err) - assert.Equal(primaryAuthMethodMigration, state.DatabaseSchemaVersion) - assert.False(state.Dirty) - - t.Log("current migration version: ", state.DatabaseSchemaVersion) - entries, err := schema.GetMigrationLog(ctx, d) - require.NoError(err) - for _, e := range entries { - t.Logf("%+v", e) + want = &schema.State{ + Initialized: true, + Editions: []schema.EditionState{ + { + Name: "oss", + BinarySchemaVersion: primaryAuthMethodMigration, + DatabaseSchemaVersion: primaryAuthMethodMigration, + DatabaseSchemaState: schema.Equal, + }, + }, } + assert.Equal(want, state) + + entries, err := m.GetMigrationLog(ctx) + require.NoError(err) assert.Equalf(2, len(entries), "expected 2 scopes without a primary auth method and got: %d", len(entries)) scope1, err := iamRepo.LookupScope(ctx, org.PublicId) diff --git a/internal/db/schema/migrations/postgres/2/10_auth_test.go b/internal/db/schema/migrations/oss/postgres_2_10_test.go similarity index 98% rename from internal/db/schema/migrations/postgres/2/10_auth_test.go rename to internal/db/schema/migrations/oss/postgres_2_10_test.go index 9fe69827b..058ef31b5 100644 --- a/internal/db/schema/migrations/postgres/2/10_auth_test.go +++ b/internal/db/schema/migrations/oss/postgres_2_10_test.go @@ -1,4 +1,4 @@ -package migration +package oss_test import ( "context" @@ -14,6 +14,8 @@ import ( "github.com/stretchr/testify/require" ) +// Tests migration: +// migrations/oss/2/10_auth.up.sql func Test_AuthMethodSubtypes(t *testing.T) { t.Parallel() assert, require := assert.New(t), require.New(t) diff --git a/internal/db/schema/migrations/postgres/2/01_log.up.sql b/internal/db/schema/migrations/postgres/2/01_log.up.sql deleted file mode 100644 index 0cd2b745a..000000000 --- a/internal/db/schema/migrations/postgres/2/01_log.up.sql +++ /dev/null @@ -1,50 +0,0 @@ -begin; - --- log_migration entries represent logs generated during migrations -create table log_migration( - id bigint generated always as identity primary key, - migration_version bigint not null, -- cannot declare FK since the table is truncated during runtime - create_time wt_timestamp, - entry text not null -); -comment on table log_migration is -'log_migration entries are logging output from databaes migrations'; - --- log_migration triggers -create trigger - default_create_time_column -before -insert on log_migration - for each row execute procedure default_create_time(); - -create trigger - immutable_columns -before -update on log_migration - for each row execute procedure immutable_columns('id', 'migration_version', 'create_time', 'entry'); - - --- log_migration_version() defines a function to be used in a "before update" --- trigger for log_migrations entries. Its intent: set the log_migration --- version column to the current migration version. -create or replace function - log_migration_version() - returns trigger -as $$ - declare current_version bigint; - begin - select max(version) from boundary_schema_version into current_version; - new.migration_version = current_version; - return new; - end; -$$ language plpgsql; -comment on function log_migration_version() is -'log_migration_version will set the log_migration entries to the current migration version'; - -create trigger - migration_version_column -before -insert on log_migration - for each row execute procedure log_migration_version(); - - commit; diff --git a/internal/db/schema/options.go b/internal/db/schema/options.go index 386365192..a68a04160 100644 --- a/internal/db/schema/options.go +++ b/internal/db/schema/options.go @@ -1,5 +1,7 @@ package schema +import "github.com/hashicorp/boundary/internal/db/schema/internal/edition" + // getOpts - iterate the inbound Options and return a struct. func getOpts(opt ...Option) options { opts := getDefaultOptions() @@ -14,18 +16,18 @@ type Option func(*options) // options = how options are represented type options struct { - withMigrationStates map[string]migrationState - withDeleteLog bool + withEditions edition.Editions + withDeleteLog bool } func getDefaultOptions() options { return options{} } -// WithMigrationStates provides an optional migration states. -func WithMigrationStates(states map[string]migrationState) Option { +// WithEditions provides an optional migration states. +func WithEditions(editions edition.Editions) Option { return func(o *options) { - o.withMigrationStates = states + o.withEditions = editions } } diff --git a/internal/db/schema/options_test.go b/internal/db/schema/options_test.go index 7f9d46b2a..a4edda292 100644 --- a/internal/db/schema/options_test.go +++ b/internal/db/schema/options_test.go @@ -9,15 +9,12 @@ import ( // Test_GetOpts provides unit tests for GetOpts and all the options func Test_GetOpts(t *testing.T) { t.Parallel() - t.Run("WithMigrationStates", func(t *testing.T) { + + t.Run("WithEditions", func(t *testing.T) { assert := assert.New(t) - oState := TestCloneMigrationStates(t) - nState := TestCreatePartialMigrationState(oState["postgres"], 8) - oState["postgres"] = nState - opts := getOpts(WithMigrationStates(oState)) - testOpts := getDefaultOptions() - testOpts.withMigrationStates = oState - assert.Equal(opts, testOpts) + editions := TestCreatePartialEditions(Postgres, PartialEditions{"oss": 1}) + opts := getOpts(WithEditions(editions)) + assert.Equal(opts.withEditions, editions) }) t.Run("WithDeleteLog", func(t *testing.T) { assert := assert.New(t) diff --git a/internal/db/schema/postgres/postgres.go b/internal/db/schema/postgres/postgres.go deleted file mode 100644 index 8a038abca..000000000 --- a/internal/db/schema/postgres/postgres.go +++ /dev/null @@ -1,477 +0,0 @@ -// The MIT License (MIT) -// -// Original Work -// Copyright (c) 2016 Matthias Kadenbach -// https://github.com/mattes/migrate -// -// Modified Work -// Copyright (c) 2018 Dale Hui -// https://github.com/golang-migrate/migrate -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package postgres - -import ( - "context" - "database/sql" - "fmt" - "io" - "io/ioutil" - "strings" - - "github.com/hashicorp/boundary/internal/errors" - "github.com/hashicorp/go-multierror" - "github.com/jackc/pgconn" -) - -// schemaAccessLockId is a Lock key used to ensure a single boundary binary is operating -// on a postgres server at a time. The value has no meaning and was picked randomly. -const ( - schemaAccessLockId int64 = 3865661975 - nilVersion = -1 -) - -var defaultMigrationsTable = "boundary_schema_version" - -// Postgres is a driver usable by a boundary schema manager. -// This struct is not thread safe. -type Postgres struct { - // Locking and unlocking need to use the same connection - conn *sql.Conn - db *sql.DB - - tx *sql.Tx -} - -// New returns a postgres pointer with the provided db verified as -// connectable and a version table being initialized. -func New(ctx context.Context, instance *sql.DB) (*Postgres, error) { - const op = "postgres.New" - if err := instance.PingContext(ctx); err != nil { - return nil, errors.Wrap(ctx, err, op) - } - conn, err := instance.Conn(ctx) - if err != nil { - return nil, errors.Wrap(ctx, err, op) - } - - px := &Postgres{ - conn: conn, - db: instance, - } - return px, nil -} - -// TrySharedLock attempts to capture a shared lock. If it is not successful it returns an error. -// https://www.postgresql.org/docs/9.6/static/explicit-locking.html#ADVISORY-LOCKS -func (p *Postgres) TrySharedLock(ctx context.Context) error { - const op = "postgres.(Postgres).TrySharedLock" - const query = "select pg_try_advisory_lock_shared($1)" - r := p.conn.QueryRowContext(ctx, query, schemaAccessLockId) - if r.Err() != nil { - return errors.Wrap(ctx, r.Err(), op) - } - var gotLock bool - if err := r.Scan(&gotLock); err != nil { - return errors.Wrap(ctx, err, op) - } - if !gotLock { - return errors.New(ctx, errors.MigrationLock, op, "Lock failed") - } - return nil -} - -// TryLock attempts to capture an exclusive lock. If it is not successful it returns an error. -// https://www.postgresql.org/docs/9.6/static/explicit-locking.html#ADVISORY-LOCKS -func (p *Postgres) TryLock(ctx context.Context) error { - const op = "postgres.(Postgres).TryLock" - const query = "select pg_try_advisory_lock($1)" - r := p.conn.QueryRowContext(ctx, query, schemaAccessLockId) - if r.Err() != nil { - return errors.Wrap(ctx, r.Err(), op) - } - var gotLock bool - if err := r.Scan(&gotLock); err != nil { - return errors.Wrap(ctx, err, op) - } - if !gotLock { - return errors.New(ctx, errors.MigrationLock, op, "Lock failed") - } - return nil -} - -// Lock calls pg_advisory_lock with the provided context and returns an error -// if we were unable to get the lock before the context cancels. -func (p *Postgres) Lock(ctx context.Context) error { - const op = "postgres.(Postgres).Lock" - const query = "select pg_advisory_lock($1)" - if _, err := p.conn.ExecContext(ctx, query, schemaAccessLockId); err != nil { - return errors.Wrap(ctx, err, op) - } - return nil -} - -// Unlock calls pg_advisory_unlock and returns an error if we were unable to -// release the lock before the context cancels. -func (p *Postgres) Unlock(ctx context.Context) error { - const op = "postgres.(Postgres).Unlock" - const query = `select pg_advisory_unlock($1)` - if _, err := p.conn.ExecContext(ctx, query, schemaAccessLockId); err != nil { - return errors.Wrap(ctx, err, op) - } - return nil -} - -// UnlockShared calls pg_advisory_unlock_shared and returns an error if we were unable to -// release the lock before the context cancels. -func (p *Postgres) UnlockShared(ctx context.Context) error { - const op = "postgres.(Postgres).UnlockShared" - query := `select pg_advisory_unlock_shared($1)` - if _, err := p.conn.ExecContext(ctx, query, schemaAccessLockId); err != nil { - return errors.Wrap(ctx, err, op) - } - return nil -} - -// Rollback rolls back the outstanding transaction. -// Calling Rollback when there is not an outstanding transaction is an error. -func (p *Postgres) Rollback() error { - const op = "postgres.(Postgres).Rollback" - defer func() { - p.tx = nil - }() - if p.tx == nil { - return errors.NewDeprecated(errors.MigrationIntegrity, op, "no pending transaction") - } - if err := p.tx.Rollback(); err != nil { - return errors.WrapDeprecated(err, op) - } - return nil -} - -// StartRun starts a transaction that all subsequent calls to Run will use. -func (p *Postgres) StartRun(ctx context.Context) error { - tx, err := p.conn.BeginTx(ctx, nil) - if err != nil { - return err - } - p.tx = tx - return nil -} - -// CommitRun commits the pending transaction if there is one -func (p *Postgres) CommitRun() error { - const op = "postgres.(Postgres).CommitRun" - defer func() { - p.tx = nil - }() - if p.tx == nil { - return errors.NewDeprecated(errors.MigrationIntegrity, op, "no pending transaction") - } - if err := p.tx.Commit(); err != nil { - if errRollback := p.tx.Rollback(); errRollback != nil { - err = multierror.Append(err, errRollback) - } - return errors.WrapDeprecated(err, op) - } - return nil -} - -type execContexter interface { - ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) - QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row -} - -// Run executes the sql provided in the passed in io.Reader. The contents of the reader must -// fit in memory as the full content is read into a string before being passed to the -// backing database. EnsureVersionTable should be ran prior to this call. -func (p *Postgres) Run(ctx context.Context, migration io.Reader, version int) error { - const op = "postgres.(Postgres).Run" - migr, err := ioutil.ReadAll(migration) - if err != nil { - return errors.Wrap(ctx, err, op) - } - // Run migration - query := string(migr) - - var extr execContexter = p.conn - rollback := func() error { return nil } - if p.tx != nil { - extr = p.tx - rollback = func() error { - defer func() { p.tx = nil }() - return p.tx.Rollback() - } - } - - // set the version first, so logs will be associated with this new version. - // if there's an error, it will get rollback - if err := p.setVersion(ctx, version, false); err != nil { - return errors.Wrap(ctx, err, op) - } - - if _, err := extr.ExecContext(ctx, query); err != nil { - if rollbackErr := rollback(); rollbackErr != nil { - err = multierror.Append(err, rollbackErr) - } - if pgErr, ok := err.(*pgconn.PgError); ok { - var line uint - var col uint - var lineColOK bool - if pgErr.Position != 0 { - line, col, lineColOK = computeLineFromPos(query, int(pgErr.Position)) - } - message := "migration failed" - if lineColOK { - message = fmt.Sprintf("%s (column %d)", message, col) - } - if pgErr.Detail != "" { - message = fmt.Sprintf("%s, %s", message, pgErr.Detail) - } - message = fmt.Sprintf("%s, on line %v: %s", message, line, migr) - return errors.Wrap(ctx, err, op, errors.WithMsg(message)) - } - return errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("migration failed: %s", migr))) - } - - return nil -} - -func computeLineFromPos(s string, pos int) (line uint, col uint, ok bool) { - // replace crlf with lf - s = strings.Replace(s, "\r\n", "\n", -1) - // pg docs: pos uses index 1 for the first character, and positions are measured in characters not bytes - runes := []rune(s) - if pos > len(runes) { - return 0, 0, false - } - sel := runes[:pos] - line = uint(runesCount(sel, newLine) + 1) - col = uint(pos - 1 - runesLastIndex(sel, newLine)) - return line, col, true -} - -const newLine = '\n' - -func runesCount(input []rune, target rune) int { - var count int - for _, r := range input { - if r == target { - count++ - } - } - return count -} - -func runesLastIndex(input []rune, target rune) int { - for i := len(input) - 1; i >= 0; i-- { - if input[i] == target { - return i - } - } - return -1 -} - -// setVersion sets the version number, and whether the database is in a dirty state. -// A version value of -1 indicates no version is set. -func (p *Postgres) setVersion(ctx context.Context, version int, dirty bool) error { - const op = "postgres.(Postgres).SetVersion" - tx := p.tx - var err error - if tx == nil { - tx, err = p.conn.BeginTx(ctx, nil) - if err != nil { - return err - } - } - rollback := func() error { - defer func() { p.tx = nil }() - return tx.Rollback() - } - query := `truncate ` + quoteIdentifier(defaultMigrationsTable) - if _, err := tx.ExecContext(ctx, query); err != nil { - if errRollback := rollback(); errRollback != nil { - err = multierror.Append(err, errRollback) - } - return errors.Wrap(ctx, err, op) - } - - // Also re-write the schema Version for nil dirty versions to prevent - // empty schema Version for failed down migration on the first migration - // See: https://github.com/golang-migrate/migrate/issues/330 - if version >= 0 || (version == nilVersion && dirty) { - query = `insert into ` + quoteIdentifier(defaultMigrationsTable) + - ` (version, dirty) values ($1, $2)` - if _, err := tx.ExecContext(ctx, query, version, dirty); err != nil { - if errRollback := rollback(); errRollback != nil { - err = multierror.Append(err, errRollback) - } - return errors.Wrap(ctx, err, op) - } - } - - if p.tx == nil { - if err := tx.Commit(); err != nil { - return errors.Wrap(ctx, err, op) - } - } - return nil -} - -// CurrentState returns the version, if the database was ever initialized -// previously, if it is currently in a dirty state, and any error. A version -// value of -1 indicates no version is set. -func (p *Postgres) CurrentState(ctx context.Context) (version int, previouslyRan, dirty bool, err error) { - const op = "postgres.(Postgres).CurrentState" - - version = nilVersion - previouslyRan, dirty = false, false - - tableQuery := `select table_name from information_schema.tables where table_schema=(select current_schema()) and table_name in ('schema_migrations', '` + defaultMigrationsTable + `')` - tableResult, err := p.conn.QueryContext(ctx, tableQuery) - if err != nil { - return nilVersion, previouslyRan, dirty, errors.Wrap(ctx, err, op) - } - defer tableResult.Close() - if !tableResult.Next() { - // No version table found - return nilVersion, previouslyRan, dirty, nil - } - - tableName := defaultMigrationsTable - if err := tableResult.Scan(&tableName); err != nil { - return nilVersion, previouslyRan, dirty, errors.Wrap(ctx, err, op) - } - previouslyRan = true - if tableResult.Next() { - return nilVersion, previouslyRan, dirty, errors.New(ctx, errors.MigrationIntegrity, op, "both old and new migration tables exist") - } - - query := `select version, dirty from ` + quoteIdentifier(tableName) - results, err := p.conn.QueryContext(ctx, query) - if err != nil { - return nilVersion, previouslyRan, dirty, errors.Wrap(ctx, err, op) - } - defer results.Close() - if !results.Next() { - // no version recorded - return nilVersion, previouslyRan, dirty, nil - } - if err := results.Scan(&version, &dirty); err != nil { - return nilVersion, previouslyRan, dirty, errors.Wrap(ctx, err, op) - } - if results.Next() { - return nilVersion, previouslyRan, dirty, errors.New(ctx, errors.MigrationIntegrity, op, "to many versions in version table") - } - return version, previouslyRan, dirty, nil -} - -func (p *Postgres) drop(ctx context.Context) (err error) { - const op = "postgres.(Postgres).drop" - // select all tables in current schema - query := `select table_name from information_schema.tables where table_schema=(select current_schema()) and table_type='BASE TABLE'` - tables, err := p.conn.QueryContext(ctx, query) - if err != nil { - return errors.Wrap(ctx, err, op) - } - defer func() { - if errClose := tables.Close(); errClose != nil { - err = multierror.Append(err, errClose) - err = errors.Wrap(ctx, err, op) - } - }() - - // delete one table after another - tableNames := make([]string, 0) - for tables.Next() { - var tableName string - if err := tables.Scan(&tableName); err != nil { - return errors.Wrap(ctx, err, op) - } - if len(tableName) > 0 { - tableNames = append(tableNames, tableName) - } - } - if err := tables.Err(); err != nil { - return errors.Wrap(ctx, err, op) - } - - if len(tableNames) > 0 { - // delete one by one ... - for _, t := range tableNames { - query = `drop table if exists ` + quoteIdentifier(t) + ` cascade` - if _, err := p.conn.ExecContext(ctx, query); err != nil { - return errors.Wrap(ctx, err, op) - } - } - } - - return nil -} - -// EnsureVersionTable checks if versions table exists and, if not, creates it. -func (p *Postgres) EnsureVersionTable(ctx context.Context) (err error) { - const op = "postgres.(Postgres).EnsureVersionTable" - - var extr execContexter = p.conn - rollback := func() error { return nil } - if p.tx != nil { - extr = p.tx - rollback = func() error { - defer func() { p.tx = nil }() - return p.tx.Rollback() - } - } - - query := `select exists (select 1 from information_schema.tables where table_schema=(select current_schema()) and table_name = '` + defaultMigrationsTable + `');` - exists := false - if err := extr.QueryRowContext(ctx, query).Scan(&exists); err != nil { - if wpErr := rollback(); wpErr != nil { - err = multierror.Append(err, wpErr) - } - return errors.Wrap(ctx, err, op) - } - if exists { - return nil - } - - updateQuery := `alter table if exists schema_migrations rename to ` + defaultMigrationsTable + `;` - if _, err = extr.ExecContext(ctx, updateQuery); err != nil { - if wpErr := rollback(); wpErr != nil { - err = multierror.Append(err, wpErr) - } - return errors.Wrap(ctx, err, op) - } - - createStmt := `create table if not exists ` + quoteIdentifier(defaultMigrationsTable) + ` (version bigint primary key, dirty boolean not null)` - if _, err = extr.ExecContext(ctx, createStmt); err != nil { - if wpErr := rollback(); wpErr != nil { - err = multierror.Append(err, wpErr) - } - return errors.Wrap(ctx, err, op) - } - - return nil -} - -func quoteIdentifier(input string) (output string) { - output = `"` + strings.Replace(input, `"`, `""`, -1) + `"` - return -} diff --git a/internal/db/schema/postgres/postgres_test.go b/internal/db/schema/postgres/postgres_test.go deleted file mode 100644 index 185e9f3f9..000000000 --- a/internal/db/schema/postgres/postgres_test.go +++ /dev/null @@ -1,462 +0,0 @@ -// The MIT License (MIT) -// -// Original Work -// Copyright (c) 2016 Matthias Kadenbach -// https://github.com/mattes/migrate -// -// Modified Work -// Copyright (c) 2018 Dale Hui -// https://github.com/golang-migrate/migrate -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package postgres - -// errors: https://pkg.go.dev/github.com/jackc/pgconn#PgError - -import ( - "bytes" - "context" - "fmt" - "strconv" - "strings" - "testing" - - "github.com/hashicorp/boundary/testing/dbtest" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestDbStuff(t *testing.T) { - t.Parallel() - ctx := context.Background() - c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1)) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, c()) - }) - d, err := open(t, ctx, u) - require.NoError(t, err) - - test(t, d, []byte("SELECT 1")) -} - -func TestCurrentState_NoVersionTable(t *testing.T) { - t.Parallel() - ctx := context.Background() - c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1)) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, c()) - }) - d, err := open(t, ctx, u) - require.NoError(t, err) - - // Drop the version table so calls to CurrentState don't rely on that - require.NoError(t, d.drop(ctx)) - v, alreadyRan, dirt, err := d.CurrentState(ctx) - assert.NoError(t, err) - assert.False(t, alreadyRan) - assert.Equal(t, v, nilVersion) - assert.False(t, dirt) -} - -func TestCurrentState_ToManyTables(t *testing.T) { - t.Parallel() - ctx := context.Background() - c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1)) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, c()) - }) - d, err := open(t, ctx, u) - require.NoError(t, err) - - // Create the most recent table - require.NoError(t, d.EnsureVersionTable(ctx)) - - // Create the legacy version of the table. - oldTableCreate := `create table if not exists schema_migrations (version bigint primary key, dirty boolean not null)` - _, err = d.conn.ExecContext(ctx, oldTableCreate) - require.NoError(t, err) - v, alreadyRan, dirt, err := d.CurrentState(ctx) - assert.Error(t, err) - assert.True(t, alreadyRan) - assert.Equal(t, v, nilVersion) - assert.False(t, dirt) -} - -func TestMultiStatement(t *testing.T) { - t.Parallel() - ctx := context.Background() - c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1)) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, c()) - }) - d, err := open(t, ctx, u) - require.NoError(t, err) - - if err := d.EnsureVersionTable(ctx); err != nil { - t.Fatalf("expected err to be nil, got %v", err) - } - - if err := d.Run(ctx, strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);"), 2); err != nil { - t.Fatalf("expected err to be nil, got %v", err) - } - - // make sure second table exists - var exists bool - if err := d.conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil { - t.Fatal(err) - } - if !exists { - t.Fatalf("expected table bar to exist") - } -} - -func TestTransaction(t *testing.T) { - t.Parallel() - ctx := context.Background() - c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1)) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, c()) - }) - d, err := open(t, ctx, u) - require.NoError(t, err) - - v, alreadyRan, dirty, err := d.CurrentState(ctx) - assert.NoError(t, err) - assert.False(t, alreadyRan) - assert.False(t, dirty) - assert.Equal(t, -1, v) - - // Fail the initial setup of the db. - assert.NoError(t, d.StartRun(ctx)) - assert.NoError(t, d.EnsureVersionTable(ctx)) - assert.Error(t, d.Run(ctx, strings.NewReader("SELECT 1 from nonExistantTable"), 3)) - assert.Error(t, d.CommitRun()) - - v, alreadyRan, dirty, err = d.CurrentState(ctx) - assert.NoError(t, err) - assert.False(t, alreadyRan) - assert.False(t, dirty) - assert.Equal(t, -1, v) - - assert.NoError(t, d.StartRun(ctx)) - assert.NoError(t, d.EnsureVersionTable(ctx)) - assert.NoError(t, d.Run(ctx, strings.NewReader("CREATE TABLE foo (foo text);"), 2)) - assert.NoError(t, d.Run(ctx, strings.NewReader("SELECT 1;"), 3)) - assert.NoError(t, d.CommitRun()) - - v, alreadyRan, dirty, err = d.CurrentState(ctx) - assert.NoError(t, err) - assert.True(t, alreadyRan) - assert.False(t, dirty) - assert.Equal(t, 3, v) - - assert.NoError(t, d.StartRun(ctx)) - assert.NoError(t, d.Run(ctx, strings.NewReader("CREATE TABLE bar (bar text);"), 20)) - assert.Error(t, d.Run(ctx, strings.NewReader("SELECT 1 FROM NonExistingTable"), 30)) - assert.Error(t, d.CommitRun()) - - v, alreadyRan, dirty, err = d.CurrentState(ctx) - assert.NoError(t, err) - assert.True(t, alreadyRan) - assert.False(t, dirty) - assert.Equal(t, 3, v) -} - -func TestWithSchema(t *testing.T) { - t.Parallel() - ctx := context.Background() - c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1)) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, c()) - }) - d, err := open(t, ctx, u) - require.NoError(t, err) - - require.NoError(t, d.EnsureVersionTable(ctx)) - - // create foobar schema - require.NoError(t, d.Run(ctx, strings.NewReader("CREATE SCHEMA foobar AUTHORIZATION postgres"), 1)) - - // re-connect using that schema - d2, err := open(t, ctx, fmt.Sprintf("%s&search_path=foobar", u)) - require.NoError(t, err) - defer func() { - if err := d2.close(t); err != nil { - t.Fatal(err) - } - }() - - version, alreadyRan, _, err := d2.CurrentState(ctx) - require.NoError(t, err) - require.Equal(t, nilVersion, version) - assert.False(t, alreadyRan) - - // now update CurrentState and compare - require.NoError(t, d2.EnsureVersionTable(ctx)) - require.NoError(t, d2.setVersion(ctx, 2, false)) - version, alreadyRan, _, err = d2.CurrentState(ctx) - require.NoError(t, err) - require.Equal(t, 2, version) - assert.True(t, alreadyRan) - - // meanwhile, the public schema still has the other CurrentState - version, alreadyRan, _, err = d.CurrentState(ctx) - require.NoError(t, err) - require.Equal(t, 1, version) - assert.True(t, alreadyRan) -} - -func TestPostgres_Lock(t *testing.T) { - t.Parallel() - ctx := context.Background() - c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1)) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, c()) - }) - d, err := open(t, ctx, u) - require.NoError(t, err) - - test(t, d, []byte("SELECT 1")) - - err = d.Lock(ctx) - if err != nil { - t.Fatal(err) - } - - err = d.Unlock(ctx) - if err != nil { - t.Fatal(err) - } - - err = d.Lock(ctx) - if err != nil { - t.Fatal(err) - } - - err = d.Unlock(ctx) - if err != nil { - t.Fatal(err) - } - - // make sure we call call Unlock in an idempotent manner. - err = d.Unlock(ctx) - if err != nil { - t.Fatal(err) - } -} - -func TestEnsureTable_Fresh(t *testing.T) { - t.Parallel() - ctx := context.Background() - c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1)) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, c()) - }) - d, err := open(t, ctx, u) - require.NoError(t, err) - - tableCreated := false - query := "SELECT exists (SELECT 1 FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_name = '" + defaultMigrationsTable + "')" - assert.NoError(t, d.db.QueryRowContext(ctx, query).Scan(&tableCreated)) - assert.False(t, tableCreated) - - assert.NoError(t, d.EnsureVersionTable(ctx)) - assert.NoError(t, d.db.QueryRowContext(ctx, query).Scan(&tableCreated)) - assert.True(t, tableCreated) -} - -func TestEnsureTable_ExistingTable(t *testing.T) { - t.Parallel() - ctx := context.Background() - c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1)) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, c()) - }) - d, err := open(t, ctx, u) - require.NoError(t, err) - - assert.NoError(t, d.EnsureVersionTable(ctx)) - - oldTableCreate := `CREATE TABLE IF NOT EXISTS schema_migrations (version bigint primary key, dirty boolean not null)` - _, err = d.db.ExecContext(ctx, oldTableCreate) - assert.NoError(t, err) - - assert.NoError(t, d.EnsureVersionTable(ctx)) -} - -func TestEnsureTable_OldTable(t *testing.T) { - t.Parallel() - ctx := context.Background() - c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1)) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, c()) - }) - d, err := open(t, ctx, u) - require.NoError(t, err) - - oldTableCreate := `CREATE TABLE IF NOT EXISTS schema_migrations (version bigint primary key, dirty boolean not null)` - _, err = d.db.ExecContext(ctx, oldTableCreate) - assert.NoError(t, err) - - tableExists := false - oldTableCheck := "SELECT exists (SELECT 1 FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_name = 'schema_migrations')" - assert.NoError(t, d.db.QueryRowContext(ctx, oldTableCheck).Scan(&tableExists)) - assert.True(t, tableExists) - - query := "SELECT exists (SELECT 1 FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_name = '" + defaultMigrationsTable + "')" - assert.NoError(t, d.db.QueryRowContext(ctx, query).Scan(&tableExists)) - assert.False(t, tableExists) - - assert.NoError(t, d.EnsureVersionTable(ctx)) - - assert.NoError(t, d.db.QueryRowContext(ctx, oldTableCheck).Scan(&tableExists)) - assert.False(t, tableExists) - assert.NoError(t, d.db.QueryRowContext(ctx, query).Scan(&tableExists)) - assert.True(t, tableExists) -} - -func TestRollback(t *testing.T) { - t.Parallel() - ctx := context.Background() - c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1)) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, c()) - }) - d, err := open(t, ctx, u) - require.NoError(t, err) - - assert.NoError(t, d.StartRun(ctx)) - assert.NoError(t, d.EnsureVersionTable(ctx)) - assert.NoError(t, d.Run(ctx, bytes.NewReader([]byte("create table if not exists foo (foo text)")), 2)) - var exists bool - query := "select exists (select 1 from information_schema.tables where table_name = 'foo' and table_schema = (select current_schema()))" - assert.NoError(t, d.conn.QueryRowContext(context.Background(), query).Scan(&exists)) - assert.True(t, exists) - assert.NoError(t, d.Rollback()) - - assert.NoError(t, d.conn.QueryRowContext(context.Background(), query).Scan(&exists)) - assert.False(t, exists) -} - -func TestRun_Error(t *testing.T) { - t.Parallel() - ctx := context.Background() - c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1)) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, c()) - }) - d, err := open(t, ctx, u) - require.NoError(t, err) - - err = d.Run(ctx, bytes.NewReader([]byte("SELECT *\nFROM foo")), 2) - assert.Error(t, err) -} - -func Test_computeLineFromPos(t *testing.T) { - testcases := []struct { - pos int - wantLine uint - wantCol uint - input string - wantOk bool - }{ - { - 15, 2, 6, "SELECT *\nFROM foo", true, // foo table does not exists - }, - { - 16, 3, 6, "SELECT *\n\nFROM foo", true, // foo table does not exists, empty line - }, - { - 25, 3, 7, "SELECT *\nFROM foo\nWHERE x", true, // x column error - }, - { - 27, 5, 7, "SELECT *\n\nFROM foo\n\nWHERE x", true, // x column error, empty lines - }, - { - 10, 2, 1, "SELECT *\nFROMM foo", true, // FROMM typo - }, - { - 11, 3, 1, "SELECT *\n\nFROMM foo", true, // FROMM typo, empty line - }, - { - 17, 2, 8, "SELECT *\nFROM foo", true, // last character - }, - { - 18, 0, 0, "SELECT *\nFROM foo", false, // invalid position - }, - } - for i, tc := range testcases { - t.Run("tc"+strconv.Itoa(i), func(t *testing.T) { - run := func(crlf bool, nonASCII bool) { - var name string - if crlf { - name = "crlf" - } else { - name = "lf" - } - if nonASCII { - name += "-nonascii" - } else { - name += "-ascii" - } - t.Run(name, func(t *testing.T) { - input := tc.input - if crlf { - input = strings.Replace(input, "\n", "\r\n", -1) - } - if nonASCII { - input = strings.Replace(input, "FROM", "FRÖM", -1) - } - gotLine, gotCol, gotOK := computeLineFromPos(input, tc.pos) - - if tc.wantOk { - t.Logf("pos %d, want %d:%d, %#v", tc.pos, tc.wantLine, tc.wantCol, input) - } - - if gotOK != tc.wantOk { - t.Fatalf("expected ok %v but got %v", tc.wantOk, gotOK) - } - if gotLine != tc.wantLine { - t.Fatalf("expected line %d but got %d", tc.wantLine, gotLine) - } - if gotCol != tc.wantCol { - t.Fatalf("expected col %d but got %d", tc.wantCol, gotCol) - } - }) - } - run(false, false) - run(true, false) - run(false, true) - run(true, true) - }) - } -} diff --git a/internal/db/schema/postgres/testing.go b/internal/db/schema/postgres/testing.go deleted file mode 100644 index 854495a36..000000000 --- a/internal/db/schema/postgres/testing.go +++ /dev/null @@ -1,140 +0,0 @@ -// The MIT License (MIT) -// -// Original Work -// Copyright (c) 2016 Matthias Kadenbach -// https://github.com/mattes/migrate -// -// Modified Work -// Copyright (c) 2018 Dale Hui -// https://github.com/golang-migrate/migrate -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package postgres - -import ( - "bytes" - "context" - "testing" - "time" - - "github.com/golang-migrate/migrate/v4/database" - "github.com/hashicorp/boundary/internal/db/common" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// test runs tests against database implementations. -func test(t *testing.T, d *Postgres, migration []byte) { - if migration == nil { - t.Fatal("test must provide migration reader") - } - ctx := context.Background() - - v, alreadyRan, dirty, err := d.CurrentState(ctx) - require.NoError(t, err) - assert.False(t, alreadyRan) - assert.False(t, dirty) - assert.Equal(t, nilVersion, v) - - testLockAndUnlock(t, d) - assert.NoError(t, d.EnsureVersionTable(ctx)) - assert.NoError(t, d.Run(ctx, bytes.NewReader(migration), 1)) - - testSetVersion(t, d) // also tests CurrentState() - // drop breaks the driver, so test it last. - assert.NoError(t, d.drop(ctx)) -} - -func testLockAndUnlock(t *testing.T, d *Postgres) { - ctx := context.Background() - - ctx, _ = context.WithTimeout(ctx, 15*time.Second) - - // locking twice is ok, no error - require.NoError(t, d.Lock(ctx)) - assert.NoError(t, d.Lock(ctx)) - - // Unlock - assert.NoError(t, d.Unlock(ctx)) - - // try to Lock - assert.NoError(t, d.Lock(ctx)) - assert.NoError(t, d.Unlock(ctx)) -} - -func testSetVersion(t *testing.T, d *Postgres) { - ctx := context.Background() - require.NoError(t, d.EnsureVersionTable(ctx)) - // nolint:maligned - testCases := []struct { - name string - version int - dirty bool - expectedErr error - expectedReadErr error - expectedVersion int - expectedDirty bool - }{ - {name: "set 1 dirty", version: 1, dirty: true, expectedErr: nil, expectedReadErr: nil, expectedVersion: 1, expectedDirty: true}, - {name: "re-set 1 dirty", version: 1, dirty: true, expectedErr: nil, expectedReadErr: nil, expectedVersion: 1, expectedDirty: true}, - {name: "set 2 clean", version: 2, dirty: false, expectedErr: nil, expectedReadErr: nil, expectedVersion: 2, expectedDirty: false}, - {name: "re-set 2 clean", version: 2, dirty: false, expectedErr: nil, expectedReadErr: nil, expectedVersion: 2, expectedDirty: false}, - {name: "last migration dirty", version: database.NilVersion, dirty: true, expectedErr: nil, expectedReadErr: nil, expectedVersion: database.NilVersion, expectedDirty: true}, - {name: "last migration clean", version: database.NilVersion, dirty: false, expectedErr: nil, expectedReadErr: nil, expectedVersion: database.NilVersion, expectedDirty: false}, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // d.EnsureVersionTable(ctx) - err := d.setVersion(ctx, tc.version, tc.dirty) - if err != tc.expectedErr { - t.Fatal("Got unexpected error:", err, "!=", tc.expectedErr) - } - v, _, dirty, readErr := d.CurrentState(ctx) - if readErr != tc.expectedReadErr { - t.Fatal("Got unexpected error:", readErr, "!=", tc.expectedReadErr) - } - if v != tc.expectedVersion { - t.Error("Got unexpected Version:", v, "!=", tc.expectedVersion) - } - if dirty != tc.expectedDirty { - t.Error("Got unexpected dirty value:", dirty, "!=", tc.dirty) - } - }) - } -} - -func open(t *testing.T, ctx context.Context, u string) (*Postgres, error) { - t.Helper() - db, err := common.SqlOpen("postgres", u) - require.NoError(t, err) - - px, err := New(ctx, db) - require.NoError(t, err) - - return px, nil -} - -func (p *Postgres) close(t *testing.T) error { - t.Helper() - require.NoError(t, p.conn.Close()) - require.NoError(t, p.db.Close()) - return nil -} diff --git a/internal/db/schema/postgres_migration.gen.go b/internal/db/schema/postgres_migration.gen.go deleted file mode 100644 index 7ced41ff4..000000000 --- a/internal/db/schema/postgres_migration.gen.go +++ /dev/null @@ -1,9119 +0,0 @@ -package schema - -// Code generated by "make migrations"; DO NOT EDIT. - -func init() { - migrationStates["postgres"] = migrationState{ - binarySchemaVersion: 16005, - upMigrations: map[int][]byte{ - 1: []byte(` -create domain wt_public_id as text -check( - length(trim(value)) > 10 -); -comment on domain wt_public_id is -'Random ID generated with github.com/hashicorp/go-secure-stdlib/base62'; - -create domain wt_private_id as text -not null -check( - length(trim(value)) > 10 -); -comment on domain wt_private_id is -'Random ID generated with github.com/hashicorp/go-secure-stdlib/base62'; - -create domain wt_scope_id as text -check( - length(trim(value)) > 10 or value = 'global' -); -comment on domain wt_scope_id is -'"global" or random ID generated with github.com/hashicorp/go-secure-stdlib/base62'; - -create domain wt_user_id as text -not null -check( - length(trim(value)) > 10 or value = 'u_anon' or value = 'u_auth' or value = 'u_recovery' -); -comment on domain wt_scope_id is -'"u_anon", "u_auth", or random ID generated with github.com/hashicorp/go-secure-stdlib/base62'; - -create domain wt_role_id as text -not null -check( - length(trim(value)) > 10 -); -comment on domain wt_scope_id is -'Random ID generated with github.com/hashicorp/go-secure-stdlib/base62'; - -create domain wt_timestamp as - timestamp with time zone - default current_timestamp; -comment on domain wt_timestamp is -'Standard timestamp for all create_time and update_time columns'; - -create or replace function - update_time_column() - returns trigger -as $$ -begin - if row(new.*) is distinct from row(old.*) then - new.update_time = now(); - return new; - else - return old; - end if; -end; -$$ language plpgsql; - -comment on function - update_time_column() -is - 'function used in before update triggers to properly set update_time columns'; - -create or replace function - default_create_time() - returns trigger -as $$ -begin - if new.create_time is distinct from now() then - raise warning 'create_time cannot be set to %', new.create_time; - new.create_time = now(); - end if; - return new; -end; -$$ language plpgsql; - -comment on function - default_create_time() -is - 'function used in before insert triggers to set create_time column to now'; - - -create domain wt_version as bigint - default 1 - not null - check( - value > 0 - ); -comment on domain wt_version is -'standard column for row version'; - --- update_version_column() will increment the version column whenever row data --- is updated and should only be used in an update after trigger. This function --- will overwrite any explicit updates to the version column. The function --- accepts an optional parameter of 'private_id' for the tables primary key. -create or replace function - update_version_column() - returns trigger -as $$ -begin - if pg_trigger_depth() = 1 then - if row(new.*) is distinct from row(old.*) then - if tg_nargs = 0 then - execute format('update %I set version = $1 where public_id = $2', tg_relid::regclass) using old.version+1, new.public_id; - new.version = old.version + 1; - return new; - end if; - if tg_argv[0] = 'private_id' then - execute format('update %I set version = $1 where private_id = $2', tg_relid::regclass) using old.version+1, new.private_id; - new.version = old.version + 1; - return new; - end if; - end if; - end if; - return new; -end; -$$ language plpgsql; - -comment on function - update_version_column() -is - 'function used in after update triggers to properly set version columns'; - --- immutable_columns() will make the column names immutable which are passed as --- parameters when the trigger is created. It raises error code 23601 which is a --- class 23 integrity constraint violation: immutable column -create or replace function - immutable_columns() - returns trigger -as $$ -declare - col_name text; - new_value text; - old_value text; -begin - foreach col_name in array tg_argv loop - execute format('SELECT $1.%I', col_name) into new_value using new; - execute format('SELECT $1.%I', col_name) into old_value using old; - if new_value is distinct from old_value then - raise exception 'immutable column: %.%', tg_table_name, col_name using - errcode = '23601', - schema = tg_table_schema, - table = tg_table_name, - column = col_name; - end if; - end loop; - return new; -end; -$$ language plpgsql; - -comment on function - immutable_columns() -is - 'function used in before update triggers to make columns immutable'; -`), - 2: []byte(` --- TODO (jimlambrt 7/2020) remove update_time -create table if not exists oplog_entry ( - id bigint generated always as identity primary key, - create_time wt_timestamp, - update_time wt_timestamp, - version text not null, - aggregate_name text not null, - "data" bytea not null -); - -create trigger - update_time_column -before -update on oplog_entry - for each row execute procedure update_time_column(); - -create trigger - default_create_time_column -before -insert on oplog_entry - for each row execute procedure default_create_time(); - --- oplog_entry is immutable. -create trigger - immutable_columns -before -update on oplog_entry - for each row execute procedure immutable_columns('id','update_time','create_time','version','aggregate_name', 'data'); - -create table if not exists oplog_ticket ( - id bigint generated always as identity primary key, - create_time wt_timestamp, - update_time wt_timestamp, - "name" text not null unique, - "version" bigint not null -); - -create trigger - update_time_column -before -update on oplog_ticket - for each row execute procedure update_time_column(); - -create trigger - default_create_time_column -before -insert on oplog_ticket - for each row execute procedure default_create_time(); - --- oplog_ticket: only allow updates to: version and update_time -create trigger - immutable_columns -before -update on oplog_ticket - for each row execute procedure immutable_columns('id','create_time','name'); - --- TODO (jimlambrt 7/2020) remove update_time -create table if not exists oplog_metadata ( - id bigint generated always as identity primary key, - create_time wt_timestamp, - update_time wt_timestamp, - entry_id bigint not null references oplog_entry(id) on delete cascade on update cascade, - "key" text not null, - value text null -); - -create trigger - update_time_column -before -update on oplog_metadata - for each row execute procedure update_time_column(); - -create trigger - default_create_time_column -before -insert on oplog_metadata - for each row execute procedure default_create_time(); - - -- oplog_metadata is immutable -create trigger - immutable_columns -before -update on oplog_metadata - for each row execute procedure immutable_columns('id','create_time','update_time','entry_id','key','value'); - -create index if not exists idx_oplog_metatadata_key on oplog_metadata(key); - -create index if not exists idx_oplog_metatadata_value on oplog_metadata(value); - -insert into oplog_ticket (name, version) -values - ('auth_token', 1), - ('default', 1), - ('iam_scope', 1), - ('iam_user', 1), - ('iam_group', 1), - ('iam_group_member', 1), - ('iam_role', 1), - ('iam_role_grant', 1), - ('iam_group_role', 1), - ('iam_user_role', 1), - ('db_test_user', 1), - ('db_test_car', 1), - ('db_test_rental', 1), - ('db_test_scooter', 1), - ('auth_account', 1), - ('iam_principal_role', 1); -`), - 3: []byte(` --- create test tables used in the unit tests for the internal/db package --- these tables (db_test_user, db_test_car, db_test_rental, db_test_scooter) are --- not part of the boundary domain model... they are simply used for testing --- the internal/db package -create table if not exists db_test_user ( - id bigint generated always as identity primary key, - create_time wt_timestamp, - update_time wt_timestamp, - public_id text not null unique, - name text unique, - phone_number text, - email text, - version wt_version -); - -create trigger - update_time_column -before -update on db_test_user - for each row execute procedure update_time_column(); - --- define the immutable fields for db_test_user -create trigger - immutable_columns -before -update on db_test_user - for each row execute procedure immutable_columns('create_time'); - -create trigger - default_create_time_column -before -insert on db_test_user - for each row execute procedure default_create_time(); - -create trigger - update_version_column -after update on db_test_user - for each row execute procedure update_version_column(); - -create table if not exists db_test_car ( - id bigint generated always as identity primary key, - create_time wt_timestamp, - update_time wt_timestamp, - public_id text not null unique, - name text unique, - model text, - mpg smallint -); - -create trigger - update_time_column -before -update on db_test_car - for each row execute procedure update_time_column(); - --- define the immutable fields for db_test_car -create trigger - immutable_columns -before -update on db_test_car - for each row execute procedure immutable_columns('create_time'); - -create trigger - default_create_time_column -before -insert on db_test_car - for each row execute procedure default_create_time(); - -create table if not exists db_test_rental ( - id bigint generated always as identity primary key, - create_time wt_timestamp, - update_time wt_timestamp, - public_id text not null unique, - name text unique, - user_id bigint not null references db_test_user(id), - car_id bigint not null references db_test_car(id) -); - -create trigger - update_time_column -before -update on db_test_rental - for each row execute procedure update_time_column(); - --- define the immutable fields for db_test_rental -create trigger - immutable_columns -before -update on db_test_rental - for each row execute procedure immutable_columns('create_time'); - -create trigger - default_create_time_column -before -insert on db_test_rental - for each row execute procedure default_create_time(); - - -create table if not exists db_test_scooter ( - id bigint generated always as identity primary key, - create_time wt_timestamp, - update_time wt_timestamp, - private_id text not null unique, - name text unique, - model text, - mpg smallint -); - -create trigger - update_time_column -before -update on db_test_scooter - for each row execute procedure update_time_column(); - - --- define the immutable fields for db_test_scooter -create trigger - immutable_columns -before -update on db_test_scooter - for each row execute procedure immutable_columns('create_time'); - -create trigger - default_create_time_column -before -insert on db_test_scooter - for each row execute procedure default_create_time(); -`), - 6: []byte(` -create table iam_scope_type_enm ( - string text not null primary key - constraint only_predefined_scope_types_allowed - check(string in ('unknown', 'global', 'org', 'project')) -); - -insert into iam_scope_type_enm (string) -values - ('unknown'), - ('global'), - ('org'), - ('project'); - - -- define the immutable fields of iam_scope_type_enm -create trigger - immutable_columns -before -update on iam_scope_type_enm - for each row execute procedure immutable_columns('string'); - -create table iam_scope ( - public_id wt_scope_id primary key, - create_time wt_timestamp, - update_time wt_timestamp, - name text, - type text not null - references iam_scope_type_enm(string) - constraint only_known_scope_types_allowed - check( - ( - type = 'global' - and parent_id is null - ) - or ( - type = 'org' - and parent_id = 'global' - ) - or ( - type = 'project' - and parent_id is not null - and parent_id != 'global' - ) - ), - description text, - parent_id text references iam_scope(public_id) on delete cascade on update cascade, - - -- version allows optimistic locking of the role when modifying the role - -- itself and when modifying dependent items like principal roles. - version wt_version - ); - -create table iam_scope_global ( - scope_id wt_scope_id primary key - references iam_scope(public_id) - on delete cascade - on update cascade - constraint only_one_global_scope_allowed - check( - scope_id = 'global' - ), - name text unique -); - -create table iam_scope_org ( - scope_id wt_scope_id primary key - references iam_scope(public_id) - on delete cascade - on update cascade, - parent_id wt_scope_id - not null - references iam_scope_global(scope_id) - on delete cascade - on update cascade, - name text, - unique (parent_id, name) -); - -create table iam_scope_project ( - scope_id wt_scope_id - not null - references iam_scope(public_id) - on delete cascade - on update cascade, - parent_id wt_public_id not null references iam_scope_org(scope_id) on delete cascade on update cascade, - name text, - unique(parent_id, name), - unique(scope_id), - primary key(scope_id, parent_id) - ); - -create or replace function - iam_sub_scopes_func() - returns trigger -as $$ -declare parent_type int; -begin - if new.type = 'global' then - insert into iam_scope_global (scope_id, name) - values - (new.public_id, new.name); - return new; - end if; - if new.type = 'org' then - insert into iam_scope_org (scope_id, parent_id, name) - values - (new.public_id, new.parent_id, new.name); - return new; - end if; - if new.type = 'project' then - insert into iam_scope_project (scope_id, parent_id, name) - values - (new.public_id, new.parent_id, new.name); - return new; - end if; - raise exception 'unknown scope type'; -end; -$$ language plpgsql; - -create trigger - iam_scope_insert -after -insert on iam_scope - for each row execute procedure iam_sub_scopes_func(); - -create or replace function - disallow_global_scope_deletion() - returns trigger -as $$ -begin - if old.type = 'global' then - raise exception 'deletion of global scope not allowed'; - end if; - return old; -end; -$$ language plpgsql; - -create trigger - iam_scope_disallow_global_deletion -before -delete on iam_scope - for each row execute procedure disallow_global_scope_deletion(); - -create trigger - update_time_column -before update on iam_scope - for each row execute procedure update_time_column(); - -create trigger - default_create_time_column -before -insert on iam_scope - for each row execute procedure default_create_time(); - -create trigger - update_version_column -after update on iam_scope - for each row execute procedure update_version_column(); - - -- define the immutable fields for iam_scope -create trigger - immutable_columns -before -update on iam_scope - for each row execute procedure immutable_columns('public_id', 'create_time', 'type', 'parent_id'); - - -- define the immutable fields of iam_scope_global -create trigger - immutable_columns -before -update on iam_scope_global - for each row execute procedure immutable_columns('scope_id'); - - -- define the immutable fields of iam_scope_org -create trigger - immutable_columns -before -update on iam_scope_org - for each row execute procedure immutable_columns('scope_id'); - - -- define the immutable fields of iam_scope_project -create trigger - immutable_columns -before -update on iam_scope_project - for each row execute procedure immutable_columns('scope_id'); - - --- iam_sub_names will allow us to enforce the different name constraints for --- orgs and projects via a before update trigger on the iam_scope --- table. -create or replace function - iam_sub_names() - returns trigger -as $$ -begin - if new.name != old.name then - if new.type = 'global' then - update iam_scope_global set name = new.name where scope_id = old.public_id; - return new; - end if; - if new.type = 'org' then - update iam_scope_org set name = new.name where scope_id = old.public_id; - return new; - end if; - if new.type = 'project' then - update iam_scope_project set name = new.name where scope_id = old.public_id; - return new; - end if; - raise exception 'unknown scope type'; - end if; - return new; -end; -$$ language plpgsql; - -create trigger - iam_sub_names -before -update on iam_scope - for each row execute procedure iam_sub_names(); - -insert into iam_scope (public_id, name, type, description) - values ('global', 'global', 'global', 'Global Scope'); - - -create table iam_user ( - public_id wt_user_id - primary key, - create_time wt_timestamp, - update_time wt_timestamp, - name text, - description text, - scope_id wt_scope_id - not null - references iam_scope(public_id) - on delete cascade - on update cascade, - unique(name, scope_id), - version wt_version, - - -- The order of columns is important for performance. See: - -- https://dba.stackexchange.com/questions/58970/enforcing-constraints-two-tables-away/58972#58972 - -- https://dba.stackexchange.com/questions/27481/is-a-composite-index-also-good-for-queries-on-the-first-field - unique(scope_id, public_id) - ); - -create or replace function - user_scope_id_valid() - returns trigger -as $$ -begin - perform from iam_scope where public_id = new.scope_id and type in ('global', 'org'); - if not found then - raise exception 'invalid scope type for user creation'; - end if; - return new; -end; -$$ language plpgsql; - -create or replace function - grant_scope_id_valid() - returns trigger -as $$ -declare parent_scope_id text; -declare role_scope_type text; -begin - -- There is a not-null constraint so ensure that if the value passed in is - -- empty we simply set to the scope ID - if new.grant_scope_id = '' or new.grant_scope_id is null then - new.grant_scope_id = new.scope_id; - end if; - -- If the scopes match, it's allowed - if new.grant_scope_id = new.scope_id then - return new; - end if; - -- Fetch the type of scope - select isc.type from iam_scope isc where isc.public_id = new.scope_id into role_scope_type; - -- Always allowed - if role_scope_type = 'global' then - return new; - end if; - -- Never allowed; the case where it's set to the same scope ID as the project - -- itself is covered above - if role_scope_type = 'project' then - raise exception 'invalid to set grant_scope_id to non-same scope_id when role scope type is project'; - end if; - if role_scope_type = 'org' then - -- Look up the parent scope ID for the scope ID given - select isc.parent_id from iam_scope isc where isc.public_id = new.grant_scope_id into parent_scope_id; - -- Allow iff the grant scope ID's parent matches the role's scope ID; that - -- is, match if the role belongs to a direct child scope of this - -- org - if parent_scope_id = new.scope_id then - return new; - end if; - raise exception 'grant_scope_id is not a child project of the role scope'; - end if; - raise exception 'unknown scope type'; -end; -$$ language plpgsql; - -create or replace function - disallow_iam_predefined_user_deletion() - returns trigger -as $$ -begin - if old.public_id = 'u_anon' then - raise exception 'deletion of anonymous user not allowed'; - end if; - if old.public_id = 'u_auth' then - raise exception 'deletion of authenticated user not allowed'; - end if; - if old.public_id = 'u_recovery' then - raise exception 'deletion of recovery user not allowed'; - end if; - return old; -end; -$$ language plpgsql; - -create trigger - update_version_column -after update on iam_user - for each row execute procedure update_version_column(); - -create trigger - ensure_user_scope_id_valid -before -insert or update on iam_user - for each row execute procedure user_scope_id_valid(); - -create trigger - update_time_column -before update on iam_user - for each row execute procedure update_time_column(); - -create trigger - default_create_time_column -before -insert on iam_user - for each row execute procedure default_create_time(); - -create trigger - iam_user_disallow_predefined_user_deletion -before -delete on iam_user - for each row execute procedure disallow_iam_predefined_user_deletion(); - --- TODO: Do we want to disallow changing the name or description? -insert into iam_user (public_id, name, description, scope_id) - values ('u_anon', 'anonymous', 'The anonymous user matches any request, whether authenticated or not', 'global'); - -insert into iam_user (public_id, name, description, scope_id) - values ('u_auth', 'authenticated', 'The authenticated user matches any user that has a valid token', 'global'); - -insert into iam_user (public_id, name, description, scope_id) - values ('u_recovery', 'recovery', 'The recovery user is used for any request that was performed with the recovery KMS workflow', 'global'); - - -- define the immutable fields for iam_user -create trigger - immutable_columns -before -update on iam_user - for each row execute procedure immutable_columns('public_id', 'create_time', 'scope_id'); - -create table iam_role ( - public_id wt_role_id primary key, - create_time wt_timestamp, - update_time wt_timestamp, - name text, - description text, - scope_id wt_scope_id - not null - references iam_scope(public_id) - on delete cascade - on update cascade, - grant_scope_id wt_scope_id - not null - references iam_scope(public_id) - on delete cascade - on update cascade, - unique(name, scope_id), - version wt_version, - - -- add unique index so a composite fk can be declared. - unique(scope_id, public_id) - ); - - -- Grants are immutable, which is enforced via the trigger below - create table iam_role_grant ( - create_time wt_timestamp, - role_id wt_role_id -- pk - references iam_role(public_id) - on delete cascade - on update cascade, - canonical_grant text -- pk - constraint canonical_grant_must_not_be_empty - check( - length(trim(canonical_grant)) > 0 - ), - raw_grant text not null - constraint raw_grant_must_not_be_empty - check( - length(trim(raw_grant)) > 0 - ), - primary key(role_id, canonical_grant) - ); - --- iam_immutable_role_grant() ensures that grants assigned to roles are immutable. -create or replace function - iam_immutable_role_grant() - returns trigger -as $$ -begin - raise exception 'role grants are immutable'; -end; -$$ language plpgsql; - -create trigger immutable_role_grant -before -update on iam_role_grant - for each row execute procedure iam_immutable_role_grant(); - -create trigger - default_create_time_column -before -insert on iam_role_grant - for each row execute procedure default_create_time(); - -create trigger - update_version_column -after update on iam_role - for each row execute procedure update_version_column(); - -create trigger - update_time_column -before update on iam_role - for each row execute procedure update_time_column(); - -create trigger - default_create_time_column -before -insert on iam_role - for each row execute procedure default_create_time(); - -create trigger - ensure_grant_scope_id_valid -before -insert or update on iam_role - for each row execute procedure grant_scope_id_valid(); - --- define the immutable fields for iam_role (started trigger name with "a_" so --- it will run first) -create trigger - a_immutable_columns -before -update on iam_role - for each row execute procedure immutable_columns('public_id', 'create_time', 'scope_id'); - -create or replace function - recovery_user_not_allowed() - returns trigger -as $$ -declare - new_value text; -begin - execute format('SELECT $1.%I', tg_argv[0]) into new_value using new; - if new_value = 'u_recovery' then - raise exception '"u_recovery" not allowed here"'; - end if; - return new; -end; -$$ language plpgsql; - -create table iam_group ( - public_id wt_public_id - primary key, - create_time wt_timestamp, - update_time wt_timestamp, - name text, - description text, - scope_id wt_scope_id - not null - references iam_scope(public_id) - on delete cascade - on update cascade, - unique(name, scope_id), - -- version allows optimistic locking of the group when modifying the group - -- itself and when modifying dependent items like group members. - version wt_version, - - -- add unique index so a composite fk can be declared. - unique(scope_id, public_id) - ); - -create trigger - update_version_column -after update on iam_group - for each row execute procedure update_version_column(); - -create trigger - update_time_column -before update on iam_group - for each row execute procedure update_time_column(); - -create trigger - default_create_time_column -before -insert on iam_group - for each row execute procedure default_create_time(); - --- define the immutable fields for iam_group -create trigger - immutable_columns -before -update on iam_group - for each row execute procedure immutable_columns('public_id', 'create_time', 'scope_id'); - --- iam_user_role contains roles that have been assigned to users. Users can be --- from any scope. The rows in this table must be immutable after insert, which --- will be ensured with a before update trigger using --- iam_immutable_role_principal(). -create table iam_user_role ( - create_time wt_timestamp, - role_id wt_role_id - references iam_role(public_id) - on delete cascade - on update cascade, - principal_id wt_user_id - references iam_user(public_id) - on delete cascade - on update cascade, - primary key (role_id, principal_id) - ); - --- iam_group_role contains roles that have been assigned to groups. --- Groups can be from any scope. The rows in this table must be immutable after --- insert, which will be ensured with a before update trigger using --- iam_immutable_role_principal(). -create table iam_group_role ( - create_time wt_timestamp, - role_id wt_role_id - references iam_role(public_id) - on delete cascade - on update cascade, - principal_id wt_public_id - references iam_group(public_id) - on delete cascade - on update cascade, - primary key (role_id, principal_id) - ); - --- get_scoped_principal_id is used by the iam_principle_role view as a convient --- way to create : to reference principals from --- other scopes than the role's scope. -create or replace function get_scoped_principal_id(role_scope text, principal_scope text, principal_id text) returns text -as $$ -begin - if role_scope = principal_scope then - return principal_id; - end if; - return principal_scope || ':' || principal_id; -end; -$$ language plpgsql; - --- iam_principle_role provides a consolidated view all principal roles assigned --- (user and group roles). --- REPLACED in 9/04_oidc_managed_group_principal_role -create view iam_principal_role as -select - ur.create_time, - ur.principal_id, - ur.role_id, - u.scope_id as principal_scope_id, - r.scope_id as role_scope_id, - get_scoped_principal_id(r.scope_id, u.scope_id, ur.principal_id) as scoped_principal_id, - 'user' as type -from - iam_user_role ur, - iam_role r, - iam_user u -where - ur.role_id = r.public_id and - u.public_id = ur.principal_id -union -select - gr.create_time, - gr.principal_id, - gr.role_id, - g.scope_id as principal_scope_id, - r.scope_id as role_scope_id, - get_scoped_principal_id(r.scope_id, g.scope_id, gr.principal_id) as scoped_principal_id, - 'group' as type -from - iam_group_role gr, - iam_role r, - iam_group g -where - gr.role_id = r.public_id and - g.public_id = gr.principal_id; - --- iam_immutable_role_principal() ensures that roles assigned to principals are immutable. -create or replace function - iam_immutable_role_principal() - returns trigger -as $$ -begin - raise exception 'roles are immutable'; -end; -$$ language plpgsql; - -create trigger immutable_role_principal -before -update on iam_user_role - for each row execute procedure iam_immutable_role_principal(); - -create trigger - recovery_user_not_allowed_user_role -before -insert on iam_user_role - for each row execute procedure recovery_user_not_allowed('principal_id'); - -create trigger - default_create_time_column -before -insert on iam_user_role - for each row execute procedure default_create_time(); - -create trigger immutable_role_principal -before -update on iam_group_role - for each row execute procedure iam_immutable_role_principal(); - -create trigger - default_create_time_column -before -insert on iam_group_role - for each row execute procedure default_create_time(); - --- iam_group_member_user is an association table that represents groups with --- associated users. -create table iam_group_member_user ( - create_time wt_timestamp, - group_id wt_public_id references iam_group(public_id) on delete cascade on update cascade, - member_id wt_user_id references iam_user(public_id) on delete cascade on update cascade, - primary key (group_id, member_id) -); - --- iam_immutable_group_member() ensures that group members are immutable. -create or replace function - iam_immutable_group_member() - returns trigger -as $$ -begin - raise exception 'group members are immutable'; -end; -$$ language plpgsql; - -create trigger - default_create_time_column -before -insert on iam_group_member_user - for each row execute procedure default_create_time(); - -create trigger iam_immutable_group_member -before -update on iam_group_member_user - for each row execute procedure iam_immutable_group_member(); - -create trigger - recovery_user_not_allowed_group_member -before -insert on iam_group_member_user - for each row execute procedure recovery_user_not_allowed('member_id'); - --- get_scoped_member_id is used by the iam_group_member view as a convient --- way to create : to reference members from --- other scopes than the group's scope. -create or replace function get_scoped_member_id(group_scope text, member_scope text, member_id text) returns text -as $$ -begin - if group_scope = member_scope then - return member_id; - end if; - return member_scope || ':' || member_id; -end; -$$ language plpgsql; - --- iam_group_member provides a consolidated view of group members. -create view iam_group_member as -select - gm.create_time, - gm.group_id, - gm.member_id, - u.scope_id as member_scope_id, - g.scope_id as group_scope_id, - get_scoped_member_id(g.scope_id, u.scope_id, gm.member_id) as scoped_member_id, - 'user' as type -from - iam_group_member_user gm, - iam_user u, - iam_group g -where - gm.member_id = u.public_id and - gm.group_id = g.public_id; -`), - 7: []byte(` -/* - - ┌────────────────┐ ┌────────────────┐ - │ iam_scope │ │ auth_method │ - ├────────────────┤ ├────────────────┤ - │ public_id (pk) │ ╱│ public_id (pk) │ - │ │┼┼───────────○─│ scope_id (fk) │ - │ │ ╲│ │ - └────────────────┘ └────────────────┘ - ┼ ┼ - ┼ ┼ - │ │ - │ │ ▲fk1 - │ │ - ○ ○ - ╱│╲ ╱│╲ - ┌────────────────┐ ┌──────────────────────────┐ - │ iam_user │ │ auth_account │ - ├────────────────┤ ├──────────────────────────┤ - │ public_id (pk) │ │ public_id (pk) │ - │ scope_id (fk) │ ◀fk2 │ scope_id (fk1) │ - │ │┼○──────○┼│ auth_method_id (fk1) │ - │ │ │ iam_user_scope_id (fk2) │ - └────────────────┘ │ iam_user_id (fk2) │ - └──────────────────────────┘ - - An iam_scope can have 0 to many iam_users. - An iam_scope can have 0 to many auth_methods. - - An iam_user belongs to 1 iam_scope. - An auth_method belongs to 1 iam_scope. - - An iam_user can have 0 or 1 auth_account. - An auth_account belongs to 0 or 1 iam_user. - - An auth_method can have 0 to many auth_accounts. - An auth_account belongs to 1 auth_account. - - An auth_account can only be associated with an iam_user in the same scope of - the auth_account's auth_method. Including scope_id in fk1 and fk2 ensures this - restriction is not violated. - - Design influenced by: - https://dba.stackexchange.com/questions/58970/enforcing-constraints-two-tables-away/58972#58972 - -*/ - - -- base table for auth methods - create table auth_method ( - public_id wt_public_id - primary key, - scope_id wt_scope_id - not null - references iam_scope(public_id) - on delete cascade - on update cascade, - - -- The order of columns is important for performance. See: - -- https://dba.stackexchange.com/questions/58970/enforcing-constraints-two-tables-away/58972#58972 - -- https://dba.stackexchange.com/questions/27481/is-a-composite-index-also-good-for-queries-on-the-first-field - unique(scope_id, public_id) - ); - - - -- base table for auth accounts - create table auth_account ( - public_id wt_public_id - primary key, - auth_method_id wt_public_id - not null, - scope_id wt_scope_id - not null, - iam_user_id wt_public_id, - -- The auth_account can only be assigned to an iam_user in the same scope as - -- the auth_method the auth_account belongs to. A separate column for - -- iam_user's scope id is needed because using the scope_id column in the - -- foreign key constraint causes an error when the iam_user is deleted but - -- the auth_account still exists. This is a valid scenario since the - -- lifetime of the auth_account is tied to the auth_method not the iam_user. - iam_user_scope_id wt_scope_id, - constraint user_and_auth_account_in_same_scope - check( - (iam_user_id is null and iam_user_scope_id is null) - or - (iam_user_id is not null and (iam_user_scope_id = scope_id)) - ), - -- including scope_id in fk1 and fk2 ensures the scope_id of the owning - -- auth_method and the scope_id of the owning iam_user are the same - foreign key (scope_id, auth_method_id) -- fk1 - references auth_method (scope_id, public_id) - on delete cascade - on update cascade, - foreign key (iam_user_scope_id, iam_user_id) -- fk2 - references iam_user (scope_id, public_id) - on delete set null - on update cascade, - unique(scope_id, auth_method_id, public_id) - ); - - create or replace function - insert_auth_method_subtype() - returns trigger - as $$ - begin - insert into auth_method - (public_id, scope_id) - values - (new.public_id, new.scope_id); - return new; - end; - $$ language plpgsql; - - create or replace function - insert_auth_account_subtype() - returns trigger - as $$ - begin - - select auth_method.scope_id - into new.scope_id - from auth_method - where auth_method.public_id = new.auth_method_id; - - insert into auth_account - (public_id, auth_method_id, scope_id) - values - (new.public_id, new.auth_method_id, new.scope_id); - - return new; - - end; - $$ language plpgsql; - - -- update_iam_user_auth_account is a before update trigger on the auth_account - -- table. If the new.iam_user_id column is different from the old.iam_user_id - -- column, update_iam_user_auth_account retrieves the scope id of the iam user - -- and sets new.iam_user_scope_id to that value. If the new.iam_user_id column - -- is null and the old.iam_user_id column is not null, - -- update_iam_user_auth_account sets the iam_user_scope_id to null. - create or replace function - update_iam_user_auth_account() - returns trigger - as $$ - begin - if new.iam_user_id is distinct from old.iam_user_id then - if new.iam_user_id is null then - new.iam_user_scope_id = null; - else - select iam_user.scope_id into new.iam_user_scope_id - from iam_user - where iam_user.public_id = new.iam_user_id; - end if; - end if; - return new; - end; - $$ language plpgsql; - - - create trigger update_iam_user_auth_account - before update of iam_user_id on auth_account - for each row - execute procedure update_iam_user_auth_account(); -`), - 8: []byte(` --- For now at least the IDs will be the same as the name, because this allows us --- to not have to persist some generated ID to worker and controller nodes. --- Eventually we may want them to diverge, so we have both here for now. - -create table server ( - private_id text, - type text, - name text not null unique - constraint server_name_must_not_be_empty - check(length(trim(name)) > 0), - description text, - address text, - create_time wt_timestamp, - update_time wt_timestamp, - primary key (private_id, type) - ); - -create trigger - immutable_columns -before -update on server - for each row execute procedure immutable_columns('create_time'); - -create trigger - default_create_time_column -before -insert on server - for each row execute procedure default_create_time(); - -create table recovery_nonces ( - nonce text - primary key, - create_time wt_timestamp - ); - -create trigger - default_create_time_column -before -insert on recovery_nonces - for each row execute procedure default_create_time(); - -create trigger - immutable_columns -before -update on recovery_nonces - for each row execute procedure immutable_columns('nonce', 'create_time'); -`), - 11: []byte(` --- an auth token belongs to 1 and only 1 auth account - -- an auth account can have 0 to many auth tokens - create table auth_token ( - public_id wt_public_id primary key, - token bytea not null unique, - -- TODO: Make key_id a foreign key once we have DEKs - key_id text not null - constraint key_id_must_not_be_empty - check(length(trim(key_id)) > 0), - auth_account_id wt_public_id not null - references auth_account(public_id) - on delete cascade - on update cascade, - create_time wt_timestamp, - update_time wt_timestamp, - -- This column is not updated every time this auth token is accessed. - -- It is updated after X minutes from the last time it was updated on - -- a per row basis. - approximate_last_access_time wt_timestamp - constraint last_access_time_must_not_be_after_expiration_time - check( - approximate_last_access_time <= expiration_time - ), - expiration_time wt_timestamp - constraint create_time_must_not_be_after_expiration_time - check( - create_time <= expiration_time - ) - ); - - create view auth_token_account as - select at.public_id, - at.token, - at.auth_account_id, - at.create_time, - at.update_time, - at.approximate_last_access_time, - at.expiration_time, - aa.scope_id, - aa.iam_user_id, - aa.auth_method_id - from auth_token as at - inner join auth_account as aa - on at.auth_account_id = aa.public_id; - - create or replace function - update_last_access_time() - returns trigger - as $$ - begin - if new.approximate_last_access_time is distinct from old.approximate_last_access_time then - new.approximate_last_access_time = now(); - end if; - return new; - end; - $$ language plpgsql; - - comment on function - update_last_access_time() - is - 'function used in before update triggers to properly set last_access_time columns'; - - create or replace function - immutable_auth_token_columns() - returns trigger - as $$ - begin - if new.auth_account_id is distinct from old.auth_account_id then - raise exception 'auth_account_id is read-only'; - end if; - if new.token is distinct from old.token then - raise exception 'token is read-only'; - end if; - return new; - end; - $$ language plpgsql; - - comment on function - immutable_auth_token_columns() - is - 'function used in before update triggers to make specific columns immutable'; - - -- This allows the expiration to be calculated on the server side and still hold the constraint that - -- the expiration time cant be before the creation time of the auth token. - create or replace function - expire_time_not_older_than_token() - returns trigger - as $$ - begin - if new.expiration_time < new.create_time then - new.expiration_time = new.create_time; - end if; - return new; - end; - $$ language plpgsql; - - comment on function - expire_time_not_older_than_token() - is - 'function used in before insert triggers to ensure expiration time is not older than create time'; - - create trigger - default_create_time_column - before insert on auth_token - for each row execute procedure default_create_time(); - - create trigger - expire_time_not_older_than_token - before insert on auth_token - for each row execute procedure expire_time_not_older_than_token(); - - create trigger - update_time_column - before update on auth_token - for each row execute procedure update_time_column(); - - create trigger - update_last_access_time - before update on auth_token - for each row execute procedure update_last_access_time(); - - - create trigger - immutable_auth_token_columns - before update on auth_token - for each row execute procedure immutable_auth_token_columns(); - - create trigger - immutable_columns - before - update on auth_token - for each row execute procedure immutable_columns('public_id', 'auth_account_id', 'create_time'); -`), - 12: []byte(` -/* - - ┌────────────────┐ ┌──────────────────────┐ ┌────────────────────────────┐ - │ auth_method │ │ auth_password_method │ │ auth_password_conf │ - ├────────────────┤ ├──────────────────────┤ ├────────────────────────────┤ - │ public_id (pk) │ │ public_id (pk,fk) │ ╱│ private_id (pk,fk) │ - │ scope_id (fk) │┼┼─────────────○┼│ scope_id (fk) │┼┼─────────○─│ password_method_id (fk) │ - │ │ │ ... │ ╲│ │ - └────────────────┘ └──────────────────────┘ └────────────────────────────┘ - ┼ ┼ ┼ - ┼ ┼ ┼ - │ │ │ - │ ▲fk1 │ ▲fk1 │ ▲fk1 - │ │ │ - ○ ○ ○ - ╱│╲ ╱│╲ ╱│╲ - ┌──────────────────────────┐ ┌──────────────────────────┐ ┌───────────────────────────────┐ - │ auth_account │ │ auth_password_account │ │ auth_password_credential │ - ├──────────────────────────┤ ├──────────────────────────┤ ├───────────────────────────────┤ - │ public_id (pk) │ │ public_id (pk,fk2) │ │ private_id (pk) │ - │ scope_id (fk1) │ ◀fk2 │ scope_id (fk1,fk2) │ ◀fk2 │ password_method_id (fk1,fk2) │ - │ auth_method_id (fk1) │┼┼──────○┼│ auth_method_id (fk1,fk2) │┼┼──────○┼│ password_conf_id (fk1) │ - │ iam_user_scope_id (fk2) │ │ ... │ │ password_account_id (fk2) │ - │ iam_user_id (fk2) │ └──────────────────────────┘ └───────────────────────────────┘ - └──────────────────────────┘ - - An auth_method is a base type. An auth_password_method is an auth_method - subtype. For every row in auth_password_method there is one row in auth_method - with the same public_id and scope_id. - - Similarly, an auth_account is a base type. An auth_password_account is an - auth_account subtype. For every row in auth_password_account there is one row - in auth_account with the same public_id, scope_id, and auth_method_id. - - Both auth_password_conf and auth_password_credential are base types. Each - password key derivation function will require a auth_password_conf and - auth_password_credential table. - - An auth_method can have 0 or 1 auth_password_method. - An auth_account can have 0 or 1 auth_password_account. - - An auth_password_method belongs to 1 auth_method. - An auth_password_method can have 0 to many auth_password_accounts. - An auth_password_method can have 0 to many auth_password_confs. - - An auth_password_account belongs to 1 auth_account. - An auth_password_account belongs to 1 auth_password_method. - An auth_password_account can have 0 or 1 auth_password_credential. - - An auth_password_conf belongs to 1 auth_password_method. - An auth_password_conf can have 0 to many auth_password_credentials. - - An auth_password_credential belongs to 1 auth_password_account. - An auth_password_credential belongs to 1 auth_password_conf. - -*/ - - create table auth_password_method ( - public_id wt_public_id - primary key, - scope_id wt_scope_id - not null, - password_conf_id wt_private_id, -- FK to auth_password_conf added below - name text, - description text, - create_time wt_timestamp, - update_time wt_timestamp, - min_login_name_length int - not null - default 3, - min_password_length int - not null - default 8, - version wt_version, - foreign key (scope_id, public_id) - references auth_method (scope_id, public_id) - on delete cascade - on update cascade, - unique(scope_id, name), - unique(scope_id, public_id) - ); - - create trigger - update_version_column - after update on auth_password_method - for each row execute procedure update_version_column(); - - create trigger - insert_auth_method_subtype - before insert on auth_password_method - for each row execute procedure insert_auth_method_subtype(); - - create table auth_password_account ( - public_id wt_public_id - primary key, - auth_method_id wt_public_id - not null, - -- NOTE(mgaffney): The scope_id type is not wt_scope_id because the domain - -- check is executed before the insert trigger which retrieves the scope_id - -- causing an insert to fail. - scope_id text not null, - name text, - description text, - create_time wt_timestamp, - update_time wt_timestamp, - login_name text not null - constraint login_name_must_be_lowercase - check(lower(trim(login_name)) = login_name) - constraint login_name_must_not_be_empty - check(length(trim(login_name)) > 0), - version wt_version, - foreign key (scope_id, auth_method_id) - references auth_password_method (scope_id, public_id) - on delete cascade - on update cascade, - foreign key (scope_id, auth_method_id, public_id) - references auth_account (scope_id, auth_method_id, public_id) - on delete cascade - on update cascade, - unique(auth_method_id, name), - unique(auth_method_id, login_name), - unique(auth_method_id, public_id) - ); - - create trigger - update_version_column - after update on auth_password_account - for each row execute procedure update_version_column(); - - create trigger - insert_auth_account_subtype - before insert on auth_password_account - for each row execute procedure insert_auth_account_subtype(); - - create table auth_password_conf ( - private_id wt_private_id - primary key, - password_method_id wt_public_id - not null - references auth_password_method (public_id) - on delete cascade - on update cascade - deferrable initially deferred, - unique(password_method_id, private_id) - ); - - alter table auth_password_method - add constraint current_conf_fkey - foreign key (public_id, password_conf_id) - references auth_password_conf (password_method_id, private_id) - on delete cascade - on update cascade - deferrable initially deferred; - - -- insert_auth_password_conf_subtype() is a trigger function for subtypes of - -- auth_password_conf - create or replace function - insert_auth_password_conf_subtype() - returns trigger - as $$ - begin - insert into auth_password_conf - (private_id, password_method_id) - values - (new.private_id, new.password_method_id); - return new; - end; - $$ language plpgsql; - - create table auth_password_credential ( - private_id wt_private_id - primary key, - password_account_id wt_public_id - not null - unique, - password_conf_id wt_private_id, - password_method_id wt_public_id - not null, - foreign key (password_method_id, password_conf_id) - references auth_password_conf (password_method_id, private_id) - on delete cascade - on update cascade, - foreign key (password_method_id, password_account_id) - references auth_password_account (auth_method_id, public_id) - on delete cascade - on update cascade, - unique(password_method_id, password_conf_id, password_account_id) - ); - - -- insert_auth_password_credential_subtype() is a trigger function for - -- subtypes of auth_password_credential - create or replace function - insert_auth_password_credential_subtype() - returns trigger - as $$ - begin - - select auth_password_account.auth_method_id - into new.password_method_id - from auth_password_account - where auth_password_account.public_id = new.password_account_id; - - insert into auth_password_credential - (private_id, password_account_id, password_conf_id, password_method_id) - values - (new.private_id, new.password_account_id, new.password_conf_id, new.password_method_id); - return new; - end; - $$ language plpgsql; - - -- update_auth_password_credential_subtype() is an after update trigger - -- function for subtypes of auth_password_credential - create or replace function - update_auth_password_credential_subtype() - returns trigger - as $$ - begin - /* - The configuration id of a credential is updated when a credential is - rehashed during authentication. - */ - if new.password_conf_id is distinct from old.password_conf_id then - update auth_password_credential - set password_conf_id = new.password_conf_id - where private_id = new.private_id; - end if; - return null; -- result is ignored since this is an after trigger - end; - $$ language plpgsql; - - -- delete_auth_password_credential_subtype() is an after delete trigger - -- function for subtypes of auth_password_credential - create or replace function - delete_auth_password_credential_subtype() - returns trigger - as $$ - begin - delete - from auth_password_credential - where private_id = old.private_id; - return null; -- result is ignored since this is an after trigger - end; - $$ language plpgsql; - - -- - -- triggers for time columns - -- - - create trigger - update_time_column - before - update on auth_password_method - for each row execute procedure update_time_column(); - - create trigger - immutable_columns - before - update on auth_password_method - for each row execute procedure immutable_columns('create_time'); - - create trigger - default_create_time_column - before - insert on auth_password_method - for each row execute procedure default_create_time(); - - create trigger - update_time_column - before - update on auth_password_account - for each row execute procedure update_time_column(); - - create trigger - immutable_columns - before - update on auth_password_account - for each row execute procedure immutable_columns('create_time'); - - create trigger - default_create_time_column - before - insert on auth_password_account - for each row execute procedure default_create_time(); - - -- The tickets for oplog are the subtypes not the base types because no updates - -- are done to any values in the base types. - insert into oplog_ticket - (name, version) - values - ('auth_password_method', 1), - ('auth_password_account', 1), - ('auth_password_credential', 1); -`), - 13: []byte(` -create table auth_password_argon2_conf ( - private_id wt_private_id primary key - references auth_password_conf (private_id) - on delete cascade - on update cascade, - password_method_id wt_public_id not null, - create_time wt_timestamp, - iterations int not null default 3 - constraint iterations_must_be_greater_than_0 - check(iterations > 0), - memory int not null default 65536 - constraint memory_must_be_greater_than_0 - check(memory > 0), - threads int not null default 1 - constraint threads_must_be_greater_than_0 - check(threads > 0), - -- salt_length unit is bytes - salt_length int not null default 32 - -- minimum of 16 bytes (128 bits) - constraint salt_must_be_at_least_16_bytes - check(salt_length >= 16), - -- key_length unit is bytes - key_length int not null default 32 - -- minimum of 16 bytes (128 bits) - constraint key_length_must_be_at_least_16_bytes - check(key_length >= 16), - unique(password_method_id, iterations, memory, threads, salt_length, key_length), - unique (password_method_id, private_id), - foreign key (password_method_id, private_id) - references auth_password_conf (password_method_id, private_id) - on delete cascade - on update cascade - deferrable initially deferred - ); - - create or replace function - read_only_auth_password_argon2_conf() - returns trigger - as $$ - begin - raise exception 'auth_password_argon2_conf is read-only'; - end; - $$ language plpgsql; - - create trigger - read_only_auth_password_argon2_conf - before - update on auth_password_argon2_conf - for each row execute procedure read_only_auth_password_argon2_conf(); - - create trigger - insert_auth_password_conf_subtype - before insert on auth_password_argon2_conf - for each row execute procedure insert_auth_password_conf_subtype(); - - create table auth_password_argon2_cred ( - private_id wt_private_id primary key - references auth_password_credential (private_id) - on delete cascade - on update cascade, - password_account_id wt_public_id not null, - password_conf_id wt_private_id, - -- NOTE(mgaffney): The password_method_id type is not wt_public_id because - -- the domain check is executed before the insert trigger which retrieves - -- the password_method_id causing an insert to fail. - password_method_id text not null, - create_time wt_timestamp, - update_time wt_timestamp, - salt bytea not null -- cannot be changed unless derived_key is changed too - constraint salt_must_not_be_empty - check(length(salt) > 0), - derived_key bytea not null - constraint derived_key_must_not_be_empty - check(length(derived_key) > 0), - -- TODO: Make key_id a foreign key once we have DEKs - key_id text not null - constraint key_id_must_not_be_empty - check(length(trim(key_id)) > 0), - foreign key (password_method_id, password_conf_id) - references auth_password_argon2_conf (password_method_id, private_id) - on delete cascade - on update cascade, - foreign key (password_method_id, password_conf_id, password_account_id) - references auth_password_credential (password_method_id, password_conf_id, password_account_id) - on delete cascade - on update cascade - deferrable initially deferred - ); - - create trigger - insert_auth_password_credential_subtype - before insert on auth_password_argon2_cred - for each row execute procedure insert_auth_password_credential_subtype(); - - create trigger - update_auth_password_credential_subtype - after update on auth_password_argon2_cred - for each row execute procedure update_auth_password_credential_subtype(); - - create trigger - delete_auth_password_credential_subtype - after delete on auth_password_argon2_cred - for each row execute procedure delete_auth_password_credential_subtype(); - - -- - -- triggers for time columns - -- - - create trigger - immutable_columns - before - update on auth_password_argon2_conf - for each row execute procedure immutable_columns('create_time'); - - create trigger - default_create_time_column - before - insert on auth_password_argon2_conf - for each row execute procedure default_create_time(); - - create trigger - update_time_column - before - update on auth_password_argon2_cred - for each row execute procedure update_time_column(); - - create trigger - immutable_columns - before - update on auth_password_argon2_cred - for each row execute procedure immutable_columns('create_time'); - - create trigger - default_create_time_column - before - insert on auth_password_argon2_cred - for each row execute procedure default_create_time(); - - -- The tickets for oplog are the subtypes not the base types because no updates - -- are done to any values in the base types. - insert into oplog_ticket - (name, version) - values - ('auth_password_argon2_conf', 1), - ('auth_password_argon2_cred', 1); -`), - 14: []byte(` --- auth_password_conf_union is a union of the configuration settings - -- of all supported key derivation functions. - -- It will be updated as new key derivation functions are supported. - create or replace view auth_password_conf_union as - -- Do not change the order of the columns when adding new configurations. - -- Union with new tables appending new columns as needed. - select c.password_method_id, c.private_id as password_conf_id, c.private_id, - 'argon2' as conf_type, - c.iterations, c.memory, c.threads, c.salt_length, c.key_length - from auth_password_argon2_conf c; - - -- auth_password_current_conf provides a view of the current password - -- configuration for each password auth method. - -- The view will be updated as new key derivation functions are supported - -- but the query to create the view should not need to be updated. - create or replace view auth_password_current_conf as - -- Rerun this query whenever auth_password_conf_union is updated. - select pm.min_login_name_length, pm.min_password_length, c.* - from auth_password_method pm - inner join auth_password_conf_union c - on pm.password_conf_id = c.password_conf_id; -`), - 20: []byte(` -/* - - ┌─────────────────┐ - │ host │ - ├─────────────────┤ - │ public_id (pk) │ - │ catalog_id (fk) │ - │ │ - └─────────────────┘ - ╲│╱ - ○ - │ - ┼ - ┼ - ┌─────────────────┐ ┌─────────────────┐ - │ iam_scope │ │ host_catalog │ - ├─────────────────┤ ├─────────────────┤ - │ public_id (pk) │ ╱│ public_id (pk) │ - │ │┼┼──────○─│ scope_id (fk) │ - │ │ ╲│ │ - └─────────────────┘ └─────────────────┘ - ┼ - ┼ - │ - ○ - ╱│╲ - ┌─────────────────┐ - │ host_set │ - ├─────────────────┤ - │ public_id (pk) │ - │ catalog_id (fk) │ - │ │ - └─────────────────┘ - -*/ - - -- host_catalog - create table host_catalog ( - public_id wt_public_id - primary key, - scope_id wt_scope_id - not null - references iam_scope (public_id) - on delete cascade - on update cascade, - - -- The order of columns is important for performance. See: - -- https://dba.stackexchange.com/questions/58970/enforcing-constraints-two-tables-away/58972#58972 - -- https://dba.stackexchange.com/questions/27481/is-a-composite-index-also-good-for-queries-on-the-first-field - unique(scope_id, public_id) - ); - - create trigger immutable_columns before update on host_catalog - for each row execute procedure immutable_columns('public_id', 'scope_id'); - - -- insert_host_catalog_subtype() is a before insert trigger - -- function for subtypes of host_catalog - create or replace function insert_host_catalog_subtype() - returns trigger - as $$ - begin - insert into host_catalog - (public_id, scope_id) - values - (new.public_id, new.scope_id); - return new; - end; - $$ language plpgsql; - - -- delete_host_catalog_subtype() is an after delete trigger - -- function for subtypes of host_catalog - create or replace function delete_host_catalog_subtype() - returns trigger - as $$ - begin - delete from host_catalog - where public_id = old.public_id; - return null; -- result is ignored since this is an after trigger - end; - $$ language plpgsql; - - -- host - create table host ( - public_id wt_public_id primary key, - catalog_id wt_public_id not null - references host_catalog (public_id) - on delete cascade - on update cascade, - unique(catalog_id, public_id) - ); - - create trigger immutable_columns before update on host - for each row execute procedure immutable_columns('public_id', 'catalog_id'); - - -- insert_host_subtype() is a before insert trigger - -- function for subtypes of host - create or replace function insert_host_subtype() - returns trigger - as $$ - begin - insert into host - (public_id, catalog_id) - values - (new.public_id, new.catalog_id); - return new; - end; - $$ language plpgsql; - - -- delete_host_subtype() is an after delete trigger - -- function for subtypes of host - create or replace function delete_host_subtype() - returns trigger - as $$ - begin - delete from host - where public_id = old.public_id; - return null; -- result is ignored since this is an after trigger - end; - $$ language plpgsql; - - -- host_set - create table host_set ( - public_id wt_public_id primary key, - catalog_id wt_public_id not null - references host_catalog (public_id) - on delete cascade - on update cascade, - unique(catalog_id, public_id) - ); - - create trigger immutable_columns before update on host_set - for each row execute procedure immutable_columns('public_id', 'catalog_id'); - - -- insert_host_set_subtype() is a before insert trigger - -- function for subtypes of host_set - create or replace function insert_host_set_subtype() - returns trigger - as $$ - begin - insert into host_set - (public_id, catalog_id) - values - (new.public_id, new.catalog_id); - return new; - end; - $$ language plpgsql; - - -- delete_host_set_subtype() is an after delete trigger - -- function for subtypes of host_set - create or replace function delete_host_set_subtype() - returns trigger - as $$ - begin - delete from host_set - where public_id = old.public_id; - return null; -- result is ignored since this is an after trigger - end; - $$ language plpgsql; - - insert into oplog_ticket (name, version) - values - ('host_catalog', 1), - ('host', 1), - ('host_set', 1); -`), - 22: []byte(` -/* - - ┌─────────────────┐ ┌─────────────────────┐ - │ host │ │ static_host │ - ├─────────────────┤ ├─────────────────────┤ - │ public_id (pk) │ │ public_id (pk) │ - │ catalog_id (fk) │┼┼──────○┼│ catalog_id (fk) │┼┼─────────────────────┐ - │ │ │ address │ ◀fk1 │ - └─────────────────┘ └─────────────────────┘ │ - ╲│╱ ╲│╱ │ - ○ ○ │ - │ │ │ - ┼ ┼ ○ - ┼ ┼ ╱│╲ - ┌─────────────────┐ ┌─────────────────────┐ ┌────────────────────────┐ - │ host_catalog │ │ static_host_catalog │ │ static_host_set_member │ - ├─────────────────┤ ├─────────────────────┤ ├────────────────────────┤ - │ public_id (pk) │ │ public_id (pk) │ │ host_id (pk,fk1) │ - │ scope_id (fk) │┼┼──────○┼│ scope_id (fk) │ │ set_id (pk,fk2) │ - │ │ │ │ │ catalog_id (fk1,fk2) │ - └─────────────────┘ └─────────────────────┘ └────────────────────────┘ - ┼ ┼ ╲│╱ - ┼ ┼ ○ - │ │ │ - ○ ○ │ - ╱│╲ ╱│╲ │ - ┌─────────────────┐ ┌─────────────────────┐ │ - │ host_set │ │ static_host_set │ │ - ├─────────────────┤ ├─────────────────────┤ │ - │ public_id (pk) │ │ public_id (pk) │ ◀fk2 │ - │ catalog_id (fk) │┼┼──────○┼│ catalog_id (fk) │┼┼─────────────────────┘ - │ │ │ │ - └─────────────────┘ └─────────────────────┘ - -*/ - - create table static_host_catalog ( - public_id wt_public_id - primary key, - scope_id wt_scope_id - not null - references iam_scope (public_id) - on delete cascade - on update cascade, - name text, - description text, - create_time wt_timestamp, - update_time wt_timestamp, - version wt_version, - foreign key (scope_id, public_id) - references host_catalog (scope_id, public_id) - on delete cascade - on update cascade, - unique(scope_id, name) - ); - - create trigger update_version_column after update on static_host_catalog - for each row execute procedure update_version_column(); - - create trigger update_time_column before update on static_host_catalog - for each row execute procedure update_time_column(); - - create trigger default_create_time_column before insert on static_host_catalog - for each row execute procedure default_create_time(); - - create trigger immutable_columns before update on static_host_catalog - for each row execute procedure immutable_columns('public_id', 'scope_id','create_time'); - - create trigger insert_host_catalog_subtype before insert on static_host_catalog - for each row execute procedure insert_host_catalog_subtype(); - - create trigger delete_host_catalog_subtype after delete on static_host_catalog - for each row execute procedure delete_host_catalog_subtype(); - - create table static_host ( - public_id wt_public_id primary key, - catalog_id wt_public_id not null - references static_host_catalog (public_id) - on delete cascade - on update cascade, - name text, - description text, - address text not null - constraint address_must_be_more_than_2_characters - check(length(trim(address)) > 2) - constraint address_must_be_less_than_256_characters - check(length(trim(address)) < 256), - create_time wt_timestamp, - update_time wt_timestamp, - version wt_version, - unique(catalog_id, name), - - foreign key (catalog_id, public_id) - references host (catalog_id, public_id) - on delete cascade - on update cascade, - - -- The order of columns is important for performance. See: - -- https://dba.stackexchange.com/questions/58970/enforcing-constraints-two-tables-away/58972#58972 - -- https://dba.stackexchange.com/questions/27481/is-a-composite-index-also-good-for-queries-on-the-first-field - unique(catalog_id, public_id) - ); - - create trigger update_version_column after update on static_host - for each row execute procedure update_version_column(); - - create trigger update_time_column before update on static_host - for each row execute procedure update_time_column(); - - create trigger default_create_time_column before insert on static_host - for each row execute procedure default_create_time(); - - create trigger immutable_columns before update on static_host - for each row execute procedure immutable_columns('public_id', 'catalog_id','create_time'); - - create trigger insert_host_subtype before insert on static_host - for each row execute procedure insert_host_subtype(); - - create trigger delete_host_subtype after delete on static_host - for each row execute procedure delete_host_subtype(); - - create table static_host_set ( - public_id wt_public_id primary key, - catalog_id wt_public_id not null - references static_host_catalog (public_id) - on delete cascade - on update cascade, - name text, - description text, - create_time wt_timestamp, - update_time wt_timestamp, - version wt_version, - unique(catalog_id, name), - foreign key (catalog_id, public_id) - references host_set (catalog_id, public_id) - on delete cascade - on update cascade, - unique(catalog_id, public_id) - ); - - create trigger update_version_column after update on static_host_set - for each row execute procedure update_version_column(); - - create trigger update_time_column before update on static_host_set - for each row execute procedure update_time_column(); - - create trigger default_create_time_column before insert on static_host_set - for each row execute procedure default_create_time(); - - create trigger immutable_columns before update on static_host_set - for each row execute procedure immutable_columns('public_id', 'catalog_id','create_time'); - - create trigger insert_host_set_subtype before insert on static_host_set - for each row execute procedure insert_host_set_subtype(); - - create trigger delete_host_set_subtype after delete on static_host_set - for each row execute procedure delete_host_set_subtype(); - - create table static_host_set_member ( - host_id wt_public_id not null, - set_id wt_public_id not null, - catalog_id wt_public_id not null, - primary key(host_id, set_id), - foreign key (catalog_id, host_id) -- fk1 - references static_host (catalog_id, public_id) - on delete cascade - on update cascade, - foreign key (catalog_id, set_id) -- fk2 - references static_host_set (catalog_id, public_id) - on delete cascade - on update cascade - ); - - create trigger immutable_columns before update on static_host_set_member - for each row execute procedure immutable_columns('host_id', 'set_id', 'catalog_id'); - - create or replace function insert_static_host_set_member() - returns trigger - as $$ - begin - select static_host_set.catalog_id - into new.catalog_id - from static_host_set - where static_host_set.public_id = new.set_id; - return new; - end; - $$ language plpgsql; - - create trigger insert_static_host_set_member before insert on static_host_set_member - for each row execute procedure insert_static_host_set_member(); - - insert into oplog_ticket (name, version) - values - ('static_host_catalog', 1), - ('static_host', 1), - ('static_host_set', 1), - ('static_host_set_member', 1); -`), - 30: []byte(` --- kms_version_column() will increment the version column whenever row data --- is inserted and should only be used in an before insert trigger. This --- function will overwrite any explicit values to the version column. -create or replace function - kms_version_column() - returns trigger -as $$ -declare - _key_id text; - _max bigint; -begin - execute format('SELECT $1.%I', tg_argv[0]) into _key_id using new; - execute format('select max(version) + 1 from %I where %I = $1', tg_relid::regclass, tg_argv[0]) using _key_id into _max; - if _max is null then - _max = 1; - end if; - new.version = _max; - return new; -end; -$$ language plpgsql; - -comment on function - kms_version_column() -is - 'function used in before insert triggers to properly set version columns for kms_* tables with a version column'; -`), - 31: []byte(` -/* - ┌────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ - ├────────────────────────────────────────────────────────────────────────────────────────────┐ ○ - ├────────────────────────────────────────────────────────────────┐ ○ ┼ - ├────────────────────────────────────┐ ○ ┼ ┌────────────────────────┐ - │ ○ ┼ ┌────────────────────────┐ │ kms_token_key │ - ┼ ┼ ┌────────────────────────┐ │ kms_session_key │ ├────────────────────────┤ -┌────────────────────────┐ ┌────────────────────────┐ │ kms_oplog_key │ ├────────────────────────┤ │private_id │ -│ kms_root_key │ │ kms_database_key │ ├────────────────────────┤ │private_id │ │root_key_id │ -├────────────────────────┤ ├────────────────────────┤ │private_id │ │root_key_id │ │ │ -│private_id │ │private_id │ │root_key_id │ │ │ │ │ -│scope_id │ │root_key_id │ │ │ │ │ │ │ -│ │ │ │ │ │ │ │ └────────────────────────┘ -└────────────────────────┘ └────────────────────────┘ └────────────────────────┘ └────────────────────────┘ ┼ - ┼ ┼ ┼ ┼ │ - │ │ │ │ │ - │ │ │ │ │ - │ │ │ │ │ - │ │ │ │ │ - ┼ ┼ ┼ ┼ │ - ╱│╲ ╱│╲ ╱│╲ ╱│╲ ┼ -┌────────────────────────┐ ┌────────────────────────┐ ┌────────────────────────┐ ┌────────────────────────┐ ╱│╲ -│ kms_root_key_version │ │kms_database_key_version│ │ kms_oplog_key_version │ │kms_session_key_version │ ┌────────────────────────┐ -├────────────────────────┤ ├────────────────────────┤ ├────────────────────────┤ ├────────────────────────┤ │ kms_token_key_version │ -│private_id │ │private_id │ │private_id │ │private_id │ ├────────────────────────┤ -│root_key_id │ │database_key_id │ │oplog_key_id │ │session_key_id │ │private_id │ -│key │ │root_key_id │ │root_key_id │ │root_key_id │ │token_key_id │ -│version │ │key │ │key │ │key │ │root_key_id │ -│ │ │version │ │version │ │version │ │key │ -└────────────────────────┘ └────────────────────────┘ │ │ │ │ │version │ - ┼ ┼ └────────────────────────┘ │ │ │ │ - │ ○ ┼ └────────────────────────┘ │ │ - ├────────────────────────────────────┘ ○ ┼ └────────────────────────┘ - ├────────────────────────────────────────────────────────────────┘ ○ ┼ - ├────────────────────────────────────────────────────────────────────────────────────────────┘ ○ - └────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ -*/ - -create table kms_root_key ( - private_id wt_private_id primary key, - scope_id wt_scope_id not null unique -- there can only be one root key for a scope. - references iam_scope(public_id) - on delete cascade - on update cascade, - create_time wt_timestamp -); - - -- define the immutable fields for kms_root_key (all of them) -create trigger - immutable_columns -before -update on kms_root_key - for each row execute procedure immutable_columns('private_id', 'scope_id', 'create_time'); - -create trigger - default_create_time_column -before -insert on kms_root_key - for each row execute procedure default_create_time(); - -create table kms_root_key_version ( - private_id wt_private_id primary key, - root_key_id wt_private_id not null - references kms_root_key(private_id) - on delete cascade - on update cascade, - version wt_version, - key bytea not null, - create_time wt_timestamp, - unique(root_key_id, version) -); - - -- define the immutable fields for kms_root_key_version (all of them) -create trigger - immutable_columns -before -update on kms_root_key_version - for each row execute procedure immutable_columns('private_id', 'root_key_id', 'version', 'key', 'create_time'); - -create trigger - default_create_time_column -before -insert on kms_root_key_version - for each row execute procedure default_create_time(); - - -create trigger - kms_version_column -before insert on kms_root_key_version - for each row execute procedure kms_version_column('root_key_id'); - -create table kms_database_key ( - private_id wt_private_id primary key, - root_key_id wt_private_id not null unique -- there can be only one database dek per root key - references kms_root_key(private_id) - on delete cascade - on update cascade, - create_time wt_timestamp -); - - -- define the immutable fields for kms_database_key (all of them) -create trigger - immutable_columns -before -update on kms_database_key - for each row execute procedure immutable_columns('private_id', 'root_key_id', 'create_time'); - -create trigger - default_create_time_column -before -insert on kms_database_key - for each row execute procedure default_create_time(); - -create table kms_database_key_version ( - private_id wt_private_id primary key, - database_key_id wt_private_id not null - references kms_database_key(private_id) - on delete cascade - on update cascade, - root_key_version_id wt_private_id not null - references kms_root_key_version(private_id) - on delete cascade - on update cascade, - version wt_version, - key bytea not null, - create_time wt_timestamp, - unique(database_key_id, version) -); - - -- define the immutable fields for kms_database_key_version (all of them) -create trigger - immutable_columns -before -update on kms_database_key_version - for each row execute procedure immutable_columns('private_id', 'database_key_id', 'root_key_version_id', 'version', 'key', 'create_time'); - -create trigger - default_create_time_column -before -insert on kms_database_key_version - for each row execute procedure default_create_time(); - -create trigger - kms_version_column -before insert on kms_database_key_version - for each row execute procedure kms_version_column('database_key_id'); - -create table kms_oplog_key ( - private_id wt_private_id primary key, - root_key_id wt_private_id not null unique -- there can be only one oplog dek per root key - references kms_root_key(private_id) - on delete cascade - on update cascade, - create_time wt_timestamp -); - - -- define the immutable fields for kms_oplog_key (all of them) -create trigger - immutable_columns -before -update on kms_oplog_key - for each row execute procedure immutable_columns('private_id', 'root_key_id', 'create_time'); - -create trigger - default_create_time_column -before -insert on kms_oplog_key - for each row execute procedure default_create_time(); - -create table kms_oplog_key_version ( - private_id wt_private_id primary key, - oplog_key_id wt_private_id not null - references kms_oplog_key(private_id) - on delete cascade - on update cascade, - root_key_version_id wt_private_id not null - references kms_root_key_version(private_id) - on delete cascade - on update cascade, - version wt_version, - key bytea not null, - create_time wt_timestamp, - unique(oplog_key_id, version) -); - - -- define the immutable fields for kms_oplog_key_version (all of them) -create trigger - immutable_columns -before -update on kms_oplog_key_version - for each row execute procedure immutable_columns('private_id', 'oplog_key_id', 'root_key_version_id', 'version', 'key', 'create_time'); - -create trigger - default_create_time_column -before -insert on kms_oplog_key_version - for each row execute procedure default_create_time(); - -create trigger - kms_version_column -before insert on kms_oplog_key_version - for each row execute procedure kms_version_column('oplog_key_id'); - -create table kms_session_key ( - private_id wt_private_id primary key, - root_key_id wt_private_id not null unique -- there can be only one session dek per root key - references kms_root_key(private_id) - on delete cascade - on update cascade, - create_time wt_timestamp -); - - -- define the immutable fields for kms_oplog_key (all of them) -create trigger - immutable_columns -before -update on kms_session_key - for each row execute procedure immutable_columns('private_id', 'root_key_id', 'create_time'); - -create trigger - default_create_time_column -before -insert on kms_session_key - for each row execute procedure default_create_time(); - -create table kms_session_key_version ( - private_id wt_private_id primary key, - session_key_id wt_private_id not null - references kms_session_key(private_id) - on delete cascade - on update cascade, - root_key_version_id wt_private_id not null - references kms_root_key_version(private_id) - on delete cascade - on update cascade, - version wt_version, - key bytea not null, - create_time wt_timestamp, - unique(session_key_id, version) -); - - - -- define the immutable fields for kms_session_key_version (all of them) -create trigger - immutable_columns -before -update on kms_session_key_version - for each row execute procedure immutable_columns('private_id', 'session_key_id', 'root_key_version_id', 'version', 'key', 'create_time'); - -create trigger - default_create_time_column -before -insert on kms_session_key_version - for each row execute procedure default_create_time(); - -create trigger - kms_version_column -before insert on kms_session_key_version - for each row execute procedure kms_version_column('session_key_id'); - -create table kms_token_key ( - private_id wt_private_id primary key, - root_key_id wt_private_id not null unique -- there can be only one token dek per root key - references kms_root_key(private_id) - on delete cascade - on update cascade, - create_time wt_timestamp -); - - -- define the immutable fields for kms_token_key (all of them) -create trigger - immutable_columns -before -update on kms_token_key - for each row execute procedure immutable_columns('private_id', 'root_key_id', 'create_time'); - -create trigger - default_create_time_column -before -insert on kms_token_key - for each row execute procedure default_create_time(); - -create table kms_token_key_version ( - private_id wt_private_id primary key, - token_key_id wt_private_id not null - references kms_token_key(private_id) - on delete cascade - on update cascade, - root_key_version_id wt_private_id not null - references kms_root_key_version(private_id) - on delete cascade - on update cascade, - version wt_version, - key bytea not null, - create_time wt_timestamp, - unique(token_key_id, version) -); - - -- define the immutable fields for kms_token_key_version (all of them) -create trigger - immutable_columns -before -update on kms_token_key_version - for each row execute procedure immutable_columns('private_id', 'token_key_id', 'root_key_version_id', 'version', 'key', 'create_time'); - -create trigger - default_create_time_column -before -insert on kms_token_key_version - for each row execute procedure default_create_time(); - -create trigger - kms_version_column -before insert on kms_token_key_version - for each row execute procedure kms_version_column('token_key_id'); - - insert into oplog_ticket - (name, version) - values - ('kms_root_key', 1), - ('kms_root_key_version', 1); -`), - 40: []byte(` --- insert_target_subtype() is a before insert trigger --- function for subtypes of target -create or replace function - insert_target_subtype() - returns trigger -as $$ -begin - insert into target - (public_id, scope_id) - values - (new.public_id, new.scope_id); - return new; -end; -$$ language plpgsql; - --- delete_target_subtype() is an after delete trigger --- function for subtypes of host -create or replace function delete_target_subtype() - returns trigger -as $$ -begin - delete from target - where - public_id = old.public_id; - return null; -- result is ignored since this is an after trigger -end; -$$ language plpgsql; - --- target_scope_valid() is a before insert trigger function for target -create or replace function - target_scope_valid() - returns trigger -as $$ -declare scope_type text; -begin - -- Fetch the type of scope - select isc.type from iam_scope isc where isc.public_id = new.scope_id into scope_type; - if scope_type = 'project' then - return new; - end if; - raise exception 'invalid target scope type % (must be project)', scope_type; -end; -$$ language plpgsql; - --- target_host_set_scope_valid() is a before insert trigger function for target_host_set -create or replace function - target_host_set_scope_valid() - returns trigger -as $$ -begin - perform from - host_catalog hc, - host_set hs, - target t, - iam_scope s - where - hc.public_id = hs.catalog_id and - hc.scope_id = t.scope_id and - t.public_id = new.target_id; -if not found then - raise exception 'target scope and host set scope are not equal'; -end if; -return new; -end; -$$ language plpgsql; -`), - 41: []byte(` -/* - ┌─────────────────┐ -┌─────────────────┐ │ target_tcp │ -│ target │ ├─────────────────┤ -├─────────────────┤ │public_id │ -│public_id │┼─────────────○┼│scope_id │ -│scope_id │ │default_port │ -│ │ │name (not null) │ -└─────────────────┘ │description │ - ┼ └─────────────────┘ - │ - ○ - ╱│╲ -┌─────────────────┐ -│ target_host_set │ -├─────────────────┤ -│target_id │ -│host_set_id │ -│ │ -└─────────────────┘ - ╲│╱ - ○ - │ - │ - ┼ -┌─────────────────┐ -│ host_set │ -├─────────────────┤ -│public_id │ -│catalog_id │ -│ │ -└─────────────────┘ - -*/ - -create table target ( - public_id wt_public_id primary key, - scope_id wt_scope_id not null - references iam_scope(public_id) - on delete cascade - on update cascade, - create_time wt_timestamp -); - -create trigger - immutable_columns -before -update on target - for each row execute procedure immutable_columns('public_id', 'scope_id', 'create_time'); - -create trigger - default_create_time_column -before -insert on target - for each row execute procedure default_create_time(); - -create trigger - target_scope_valid -before insert on target - for each row execute procedure target_scope_valid(); - - -create table target_host_set( - target_id wt_public_id - references target(public_id) - on delete cascade - on update cascade, - host_set_id wt_public_id - references host_set(public_id) - on delete cascade - on update cascade, - primary key(target_id, host_set_id), - create_time wt_timestamp -); - -create trigger - immutable_columns -before -update on target_host_set - for each row execute procedure immutable_columns('target_id', 'host_set_id', 'create_time'); - -create trigger - target_host_set_scope_valid -before -insert on target_host_set - for each row execute procedure target_host_set_scope_valid(); - -create table target_tcp ( - public_id wt_public_id primary key - references target(public_id) - on delete cascade - on update cascade, - scope_id wt_scope_id not null - references iam_scope(public_id) - on delete cascade - on update cascade, - name text not null, -- name is not optional for a target subtype - description text, - default_port int, -- default_port can be null - -- max duration of the session in seconds. - -- default is 8 hours - session_max_seconds int not null default 28800 - constraint session_max_seconds_must_be_greater_than_0 - check(session_max_seconds > 0), - -- limit on number of session connections allowed. -1 equals no limit - session_connection_limit int not null default 1 - constraint session_connection_limit_must_be_greater_than_0_or_negative_1 - check(session_connection_limit > 0 or session_connection_limit = -1), - create_time wt_timestamp, - update_time wt_timestamp, - version wt_version, - unique(scope_id, name) -- name must be unique within a scope -); - -create trigger - insert_target_subtype -before insert on target_tcp - for each row execute procedure insert_target_subtype(); - -create trigger - delete_target_subtype -after delete on target_tcp - for each row execute procedure delete_target_subtype(); - - -- define the immutable fields for target -create trigger - immutable_columns -before -update on target_tcp - for each row execute procedure immutable_columns('public_id', 'scope_id', 'create_time'); - -create trigger - update_version_column -after update on target_tcp - for each row execute procedure update_version_column(); - -create trigger - update_time_column -before update on target_tcp - for each row execute procedure update_time_column(); - -create trigger - default_create_time_column -before -insert on target_tcp - for each row execute procedure default_create_time(); - -create trigger - target_scope_valid -before insert on target_tcp - for each row execute procedure target_scope_valid(); - - --- target_all_subtypes is a union of all target subtypes --- NOTE: this is replaced in 100 to add worker_filter -create view target_all_subtypes -as -select - public_id, - scope_id, - name, - description, - default_port, - session_max_seconds, - session_connection_limit, - version, - create_time, - update_time, - 'tcp' as type - from target_tcp; - -create view target_set -as -select - hs.public_id, - hs.catalog_id, - ths.target_id -from - target_host_set ths, - host_set hs -where - hs.public_id = ths.host_set_id; - -insert into oplog_ticket - (name, version) -values - ('target_tcp', 1); -`), - 50: []byte(` -/* - - ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ - │ iam_scope_org │ │ iam_user │ │ auth_token │ - ├─────────────────┤ ├─────────────────┤ ├─────────────────┤ - │ public_id (pk) │ │ public_id (pk) │ │ public_id (pk) │ - │ │ │ │ │ │ - └─────────────────┘ └─────────────────┘ └─────────────────┘ - ▲fk7 ┼ ▲fk1 ┼ ▲fk6 ┼ - ┼ ┼ ┼ - ├─────────────────────────────────┴─────────────────────┘ - │ - ○ - ╱│╲ - ┌──────────────────────────┐ ┌─────────────────┐ ┌─────────────────┐ - │ session │╲ fk3▶ │ server │ │ target │ - ├──────────────────────────┤─○──────○┼├─────────────────┤ ├─────────────────┤ - │ public_id (pk) │╱ │ private_id (pk) │ │ public_id (pk) │ - │ user_id (fk1) │ │ type (pk) │ │ │ - │ host_id (fk2) │ └─────────────────┘ └─────────────────┘ - │ server_id (fk3) │ ▲fk4 ┼ - │ server_type (fk3) │╲ ┼ - │ target_id (fk4) │─○─────────────────┬─────────────────────┤ - │ host_set_id (fk5) │╱ ┼ ┼ - │ auth_token_id (fk6) │ ▼fk5 ┼ ▼fk2 ┼ - │ scope_id (fk7) │ ┌─────────────────┐ ┌─────────────────┐ - │ termination_reason (fk8) │ │ host_set │ │ host │ - └──────────────────────────┘ ├─────────────────┤ ├─────────────────┤ - ▲fk1 ┼ ╲│╱ │ public_id (pk) │ │ public_id (pk) │ - ┼ ○ │ │ │ │ - │ │ └─────────────────┘ └─────────────────┘ - │ │ - └─┐ │ - │ │ ┌───────────────────────────────┐ - │ │ │session_termination_reason_enm │ - │ │ fk8▶ ├───────────────────────────────┤ - ┼ └──────────○┼│ name │ - ╱│╲ └───────────────────────────────┘ - ┌──────────────────────────────────────────┐ - │ session_state │ - ├──────────────────────────────────────────┤┼○┐ - │ session_id (pk,fk1,fk2,unq1,unq2) │ │◀fk2 - │ state (fk3) │ │ - │ previous_end_time (fk2,unq1) │┼○┘ - │ start_time (pk) │ - │ end_time (unq2) │ - └──────────────────────────────────────────┘ - ╲│╱ - ○ - │ - │ - ┼ - ▼fk3 ┼ - ┌───────────────────────────────┐ - │ session_state_enm │ - ├───────────────────────────────┤ - │ name │ - └───────────────────────────────┘ - -*/ - - create table session_termination_reason_enm ( - name text primary key - constraint only_predefined_session_termination_reasons_allowed - check ( - name in ( - 'unknown', - 'timed out', - 'closed by end-user', - 'terminated', - 'network error', - 'system error', - 'connection limit', - 'canceled' - ) - ) - ); - - insert into session_termination_reason_enm (name) - values - ('unknown'), - ('timed out'), - ('closed by end-user'), - ('terminated'), - ('network error'), - ('system error'), - ('connection limit'), - ('canceled'); - --- Note: here, and in the session_connection table, we should add a trigger --- ensuring that if server_id goes to null, we mark connections as closed. See --- https://hashicorp.atlassian.net/browse/ICU-1495 - create table session ( - public_id wt_public_id primary key, - -- the user of the session - user_id text -- fk1 - -- not using the wt_user_id domain type because it is marked 'not null' - references iam_user (public_id) - on delete set null - on update cascade, - -- the host the user is connected to via this session - host_id wt_public_id -- fk2 - references host (public_id) - on delete set null - on update cascade, - -- the worker proxying the connection between the user and the host - server_id text, -- fk3 - server_type text,-- fk3 - foreign key (server_id, server_type) - references server (private_id, type) - on delete set null - on update cascade, - -- the target the host was chosen from and the user was authorized to - -- connect to - target_id wt_public_id -- fk4 - references target (public_id) - on delete set null - on update cascade, - -- the host set the host was chosen from and the user was authorized to - -- connect to via the target - host_set_id wt_public_id -- fk5 - references host_set (public_id) - on delete set null - on update cascade, - -- the auth token of the user when this session was created - auth_token_id wt_public_id -- fk6 - references auth_token (public_id) - on delete set null - on update cascade, - -- the project which owns this session - scope_id wt_scope_id -- fk7 - references iam_scope_project (scope_id) - on delete set null - on update cascade, - -- Certificate to use when connecting (or if using custom certs, to - -- serve as the "login"). Raw DER bytes. - certificate bytea not null, - -- after this time the connection will be expired, e.g. forcefully terminated - expiration_time wt_timestamp, -- maybe null - -- limit on number of session connections allowed. default of 0 equals no limit - connection_limit int not null default 1 - constraint connection_limit_must_be_greater_than_0_or_negative_1 - check(connection_limit > 0 or connection_limit = -1), - -- trust of first use token - tofu_token bytea, -- will be null when session is first created - -- the reason this session ended (null until terminated) - -- TODO: Make key_id a foreign key once we have DEKs - key_id text, -- will be null on insert - -- references kms_database_key_version(private_id) - -- on delete restrict - -- on update cascade, - termination_reason text -- fk8 - references session_termination_reason_enm (name) - on delete restrict - on update cascade, - version wt_version, - create_time wt_timestamp, - update_time wt_timestamp, - endpoint text -- not part of the warehouse, used to send info to the worker - ); - - -- Replaced in 100 to add worker_filter - create trigger - immutable_columns - before - update on session - for each row execute procedure immutable_columns('public_id', 'certificate', 'expiration_time', 'connection_limit', 'create_time', 'endpoint'); - - -- session table has some cascades of FK to null, so we need to be careful - -- which columns trigger an update of the version column - create trigger - update_version_column - after update of version, termination_reason, key_id, tofu_token, server_id, server_type on session - for each row execute procedure update_version_column(); - - create trigger - update_time_column - before update on session - for each row execute procedure update_time_column(); - - create trigger - default_create_time_column - before - insert on session - for each row execute procedure default_create_time(); - - create or replace function - insert_session() - returns trigger - as $$ - begin - case - when new.user_id is null then - raise exception 'user_id is null'; - when new.host_id is null then - raise exception 'host_id is null'; - when new.target_id is null then - raise exception 'target_id is null'; - when new.host_set_id is null then - raise exception 'host_set_id is null'; - when new.auth_token_id is null then - raise exception 'auth_token_id is null'; - when new.scope_id is null then - raise exception 'scope_id is null'; - when new.endpoint is null then - raise exception 'endpoint is null'; - else - end case; - return new; - end; - $$ language plpgsql; - - create trigger - insert_session - before insert on session - for each row execute procedure insert_session(); - - create or replace function - insert_new_session_state() - returns trigger - as $$ - begin - insert into session_state (session_id, state) - values - (new.public_id, 'pending'); - return new; - end; - $$ language plpgsql; - - create trigger - insert_new_session_state - after insert on session - for each row execute procedure insert_new_session_state(); - - -- update_connection_state_on_closed_reason() is used in an update insert trigger on the - -- session_connection table. it will valiadate that all the session's - -- connections are closed, and then insert a state of "closed" in - -- session_connection_state for the closed session connection. - create or replace function - update_session_state_on_termination_reason() - returns trigger - as $$ - begin - if new.termination_reason is not null then - perform from - session - where - public_id = new.public_id and - public_id not in ( - select session_id - from session_connection - where - public_id in ( - select connection_id - from session_connection_state - where - state != 'closed' and - end_time is null - ) - ); - if not found then - raise 'session %s has open connections', new.public_id; - end if; - - -- check to see if there's a terminated state already, before inserting a - -- new one. - perform from - session_state ss - where - ss.session_id = new.public_id and - ss.state = 'terminated'; - if found then - return new; - end if; - - insert into session_state (session_id, state) - values - (new.public_id, 'terminated'); - end if; - return new; - end; - $$ language plpgsql; - - - create trigger - update_session_state_on_termination_reason - after update of termination_reason on session - for each row execute procedure update_session_state_on_termination_reason(); - - - -- cancel_session will insert a cancel state for the session, if there's isn't - -- a canceled state already. It's used by cancel_session_with_null_fk. - create or replace function - cancel_session(in sessionId text) returns void - as $$ - declare - rows_affected numeric; - begin - insert into session_state(session_id, state) - select - sessionId::text, 'canceling' - from - session s - where - s.public_id = sessionId::text and - s.public_id not in ( - select - session_id - from - session_state - where - session_id = sessionId::text and - state = 'canceling' - ) limit 1; - get diagnostics rows_affected = row_count; - if rows_affected > 1 then - raise exception 'cancel session: more than one row affected: %', rows_affected; - end if; - end; - $$ language plpgsql; - - -- cancel_session_with_null_fk is intended to be a before update trigger that - -- sets the session's state to cancel if a FK is set to null. - create or replace function - cancel_session_with_null_fk() - returns trigger - as $$ - begin - case - when new.user_id is null then - perform cancel_session(new.public_id); - when new.host_id is null then - perform cancel_session(new.public_id); - when new.target_id is null then - perform cancel_session(new.public_id); - when new.host_set_id is null then - perform cancel_session(new.public_id); - when new.auth_token_id is null then - perform cancel_session(new.public_id); - when new.scope_id is null then - perform cancel_session(new.public_id); - end case; - return new; - end; - $$ language plpgsql; - - create trigger - cancel_session_with_null_fk - before update of user_id, host_id, target_id, host_set_id, auth_token_id, scope_id on session - for each row execute procedure cancel_session_with_null_fk(); - - create table session_state_enm ( - name text primary key - constraint only_predefined_session_states_allowed - check ( - name in ('pending', 'active', 'canceling', 'terminated') - ) - ); - - insert into session_state_enm (name) - values - ('pending'), - ('active'), - ('canceling'), - ('terminated'); - -/* - - - ┌────────────────┐ - start │ │ - . │ Canceling │ - (●) ┌────▶│ │─────┐ - ' │ │ │ │ - │ │ └────────────────┘ │ - │ │ │ - ▼ │ ▼ - ┌────────────────┐ ┌────────────────┐ ┌────────────────┐ - │ │ │ │ │ │ - │ Pending │ │ Active │ │ Terminated │ - │ │──────────▶│ │──────────▶│ │ - │ │ │ │ │ │ - └────────────────┘ └────────────────┘ └────────────────┘ - │ │ - │ │ - │ │ - │ │ - └──────────────────────▶ ◉ End ◀────────────────────────┘ - -*/ - - -- Design influenced by: - -- Joe Celko's SQL for Smarties: Advanced SQL Programming, 5th edition - -- Chapter 12, p270 - create table session_state ( - session_id wt_public_id not null -- fk1, fk2 - references session (public_id) - on delete cascade - on update cascade, - state text not null -- fk3 - references session_state_enm(name) - on delete restrict - on update cascade, - previous_end_time timestamp with time zone, -- fk2 -- null means first state - start_time timestamp with time zone default current_timestamp not null, - constraint previous_end_time_and_start_time_in_sequence - check (previous_end_time <= start_time), - end_time timestamp with time zone, -- null means unfinished current state - constraint start_and_end_times_in_sequence - check (start_time <= end_time), - constraint end_times_in_sequence - check (previous_end_time <> end_time), - primary key (session_id, start_time), - unique (session_id, previous_end_time), -- null means first state - unique (session_id, end_time), -- one null current state - foreign key (session_id, previous_end_time) -- self-reference - references session_state (session_id, end_time) - ); - - - create trigger - immutable_columns - before - update on session_state - for each row execute procedure immutable_columns('session_id', 'state', 'start_time', 'previous_end_time'); - - - create or replace function - insert_session_state() - returns trigger - as $$ - begin - - update session_state - set end_time = now() - where session_id = new.session_id - and end_time is null; - - if not found then - new.previous_end_time = null; - new.start_time = now(); - new.end_time = null; - return new; - end if; - - new.previous_end_time = now(); - new.start_time = now(); - new.end_time = null; - return new; - - end; - $$ language plpgsql; - - - create trigger insert_session_state before insert on session_state - for each row execute procedure insert_session_state(); - - -- Replaced in 100 to add worker_filter - create view session_with_state as - select - s.public_id, - s.user_id, - s.host_id, - s.server_id, - s.server_type, - s.target_id, - s.host_set_id, - s.auth_token_id, - s.scope_id, - s.certificate, - s.expiration_time, - s.connection_limit, - s.tofu_token, - s.key_id, - s.termination_reason, - s.version, - s.create_time, - s.update_time, - s.endpoint, - ss.state, - ss.previous_end_time, - ss.start_time, - ss.end_time - from - session s, - session_state ss - where - s.public_id = ss.session_id; -`), - 51: []byte(` -/* - - ┌────────────────┐ - │ session │ - ├────────────────┤ - │ public_id (pk) │ - │ │ - │ │ - └────────────────┘ - ▲fk1 ┼ - ┼ - │ - │ - ┼ - ╱│╲ ┌──────────────────────────────────────┐ - ┌───────────────────────┐ │ session_connection_closed_reason_enm │ - │ session_connection │╲ fk2▶ ├──────────────────────────────────────┤ - ├───────────────────────┤─○───○┼│ name │ - │ public_id (pk) │╱ └──────────────────────────────────────┘ - │ session_id (fk1) │ - │ closed_reason (fk2) │ - └───────────────────────┘ - ▲fk1 ┼ - ┼ - │ - │ - ┼ - ╱│╲ - ┌──────────────────────────────────────────┐ - │ session_connection_state │ - ├──────────────────────────────────────────┤┼○┐ - │ connection_id (pk,fk1,fk2,unq1,unq2) │ │◀fk2 - │ state (fk3) │ │ - │ previous_end_time (fk2,unq1) │┼○┘ - │ start_time (pk) │ - │ end_time (unq2) │ - └──────────────────────────────────────────┘ - ╲│╱ - ○ - │ - │ - ┼ - ▼fk3 ┼ - ┌───────────────────────────────┐ - │ session_connection_state_enm │ - ├───────────────────────────────┤ - │ name │ - └───────────────────────────────┘ - -*/ - - create table session_connection_closed_reason_enm ( - name text primary key - constraint only_predefined_session_connection_closed_reasons_allowed - check ( - name in ( - 'unknown', - 'timed out', - 'closed by end-user', - 'canceled', - 'network error', - 'system error' - ) - ) - ); - - insert into session_connection_closed_reason_enm (name) - values - ('unknown'), - ('timed out'), - ('closed by end-user'), - ('canceled'), - ('network error'), - ('system error'); - - -- A session connection is one connection proxied by a worker from a client to - -- a endpoint for a session. The client initiates the connection to the worker - -- and the worker initiates the connection to the endpoint. - -- A session can have zero or more session connections. - -- Note: Updated to add server_id, server_type in 801 - create table session_connection ( - public_id wt_public_id primary key, - session_id wt_public_id not null - references session (public_id) - on delete cascade - on update cascade, - -- the client_tcp_address is the network address of the client which initiated - -- the connection to a worker - client_tcp_address inet, -- maybe null on insert - -- the client_tcp_port is the network port at the address of the client the - -- worker proxied a connection for the user - client_tcp_port integer -- maybe null on insert - constraint client_tcp_port_must_be_greater_than_0 - check(client_tcp_port > 0) - constraint client_tcp_port_must_less_than_or_equal_to_65535 - check(client_tcp_port <= 65535), - -- the endpoint_tcp_address is the network address of the endpoint which the - -- worker initiated the connection to, for the user - endpoint_tcp_address inet, -- maybe be null on insert - -- the endpoint_tcp_port is the network port at the address of the endpoint the - -- worker proxied a connection to, for the user - endpoint_tcp_port integer -- maybe null on insert - constraint endpoint_tcp_port_must_be_greater_than_0 - check(endpoint_tcp_port > 0) - constraint endpoint_tcp_port_must_less_than_or_equal_to_65535 - check(endpoint_tcp_port <= 65535), - -- the total number of bytes received by the worker from the client and sent - -- to the endpoint for this connection - bytes_up bigint -- can be null - constraint bytes_up_must_be_null_or_a_non_negative_number - check ( - bytes_up is null - or - bytes_up >= 0 - ), - -- the total number of bytes received by the worker from the endpoint and sent - -- to the client for this connection - bytes_down bigint -- can be null - constraint bytes_down_must_be_null_or_a_non_negative_number - check ( - bytes_down is null - or - bytes_down >= 0 - ), - closed_reason text - references session_connection_closed_reason_enm (name) - on delete restrict - on update cascade, - version wt_version, - create_time wt_timestamp, - update_time wt_timestamp - ); - - create trigger - immutable_columns - before - update on session_connection - for each row execute procedure immutable_columns('public_id', 'session_id', 'create_time'); - - create trigger - update_version_column - after update on session_connection - for each row execute procedure update_version_column(); - - create trigger - update_time_column - before update on session_connection - for each row execute procedure update_time_column(); - - create trigger - default_create_time_column - before - insert on session_connection - for each row execute procedure default_create_time(); - - -- insert_new_connection_state() is used in an after insert trigger on the - -- session_connection table. it will insert a state of "authorized" in - -- session_connection_state for the new session connection. - create or replace function - insert_new_connection_state() - returns trigger - as $$ - begin - insert into session_connection_state (connection_id, state) - values - (new.public_id, 'authorized'); - return new; - end; - $$ language plpgsql; - - create trigger - insert_new_connection_state - after insert on session_connection - for each row execute procedure insert_new_connection_state(); - - -- update_connection_state_on_closed_reason() is used in an update trigger on the - -- session_connection table. it will insert a state of "closed" in - -- session_connection_state for the closed session connection. - create or replace function - update_connection_state_on_closed_reason() - returns trigger - as $$ - begin - if new.closed_reason is not null then - -- check to see if there's a closed state already, before inserting a - -- new one. - perform from - session_connection_state cs - where - cs.connection_id = new.public_id and - cs.state = 'closed'; - if not found then - insert into session_connection_state (connection_id, state) - values - (new.public_id, 'closed'); - end if; - -- whenever we close a connection, we want to terminate the session if - -- possible. - perform terminate_session_if_possible(new.session_id); - end if; - return new; - end; - $$ language plpgsql; - - create trigger - update_connection_state_on_closed_reason - after update of closed_reason on session_connection - for each row execute procedure update_connection_state_on_closed_reason(); - - create table session_connection_state_enm ( - name text primary key - constraint only_predefined_session_connection_states_allowed - check ( - name in ('authorized', 'connected', 'closed') - ) - ); - - insert into session_connection_state_enm (name) - values - ('authorized'), - ('connected'), - ('closed'); - - - create table session_connection_state ( - connection_id wt_public_id not null - references session_connection (public_id) - on delete cascade - on update cascade, - state text not null - references session_connection_state_enm(name) - on delete restrict - on update cascade, - previous_end_time timestamp with time zone, -- fk2 -- null means first state - start_time timestamp with time zone default current_timestamp not null, - constraint previous_end_time_and_start_time_in_sequence - check (previous_end_time <= start_time), - end_time timestamp with time zone, -- null means unfinished current state - constraint start_and_end_times_in_sequence - check (start_time <= end_time), - constraint end_times_in_sequence - check (previous_end_time <> end_time), - primary key (connection_id, start_time), - unique (connection_id, previous_end_time), -- null means first state - unique (connection_id, end_time), -- one null current state - foreign key (connection_id, previous_end_time) -- self-reference - references session_connection_state (connection_id, end_time) - ); - - create trigger - immutable_columns - before - update on session_connection_state - for each row execute procedure immutable_columns('connection_id', 'state', 'start_time', 'previous_end_time'); - - create or replace function - insert_session_connection_state() - returns trigger - as $$ - begin - - update session_connection_state - set end_time = now() - where connection_id = new.connection_id - and end_time is null; - - if not found then - new.previous_end_time = null; - new.start_time = now(); - new.end_time = null; - return new; - end if; - - new.previous_end_time = now(); - new.start_time = now(); - new.end_time = null; - return new; - - end; - $$ language plpgsql; - - create trigger insert_session_connection_state before insert on session_connection_state - for each row execute procedure insert_session_connection_state(); - --- terminate_session_if_possible takes a session id and terminates the session --- if the following conditions are met: --- * the session is expired and all its connections are closed. --- * the session is canceling and all its connections are closed --- * the session has exhausted its connection limit and all its connections --- are closed. --- --- Note: this function should align closely with the domain function --- TerminateCompletedSessions -create or replace function - terminate_session_if_possible(terminate_session_id text) - returns void - as $$ - begin - -- is terminate_session_id in a canceling state - with canceling_session(session_id) as - ( - select - session_id - from - session_state ss - where - ss.session_id = terminate_session_id and - ss.state = 'canceling' and - ss.end_time is null - ) - update session us - set termination_reason = - case - -- timed out sessions - when now() > us.expiration_time then 'timed out' - -- canceling sessions - when us.public_id in( - select - session_id - from - canceling_session cs - where - us.public_id = cs.session_id - ) then 'canceled' - -- default: session connection limit reached. - else 'connection limit' - end - where - -- limit update to just the terminating_session_id - us.public_id = terminate_session_id and - termination_reason is null and - -- session expired or connection limit reached - ( - -- expired sessions... - now() > us.expiration_time or - -- connection limit reached... - ( - -- handle unlimited connections... - connection_limit != -1 and - ( - select count (*) - from session_connection sc - where - sc.session_id = us.public_id - ) >= connection_limit - ) or - -- canceled sessions - us.public_id in ( - select - session_id - from - canceling_session cs - where - us.public_id = cs.session_id - ) - ) and - -- make sure there are no existing connections - us.public_id not in ( - select - session_id - from - session_connection - where public_id in ( - select - connection_id - from - session_connection_state - where - state != 'closed' and - end_time is null - ) - ); - end; - $$ language plpgsql; -`), - 60: []byte(` -create extension if not exists "pgcrypto"; - - create domain wh_inet_port as integer - check( - value > 0 - and - value <= 65535 - ); - comment on domain wh_inet_port is - 'An ordinal number between 1 and 65535 representing a network port'; - - create domain wh_bytes_transmitted as bigint - check( - value >= 0 - ); - comment on domain wh_bytes_transmitted is - 'A non-negative integer representing the number of bytes transmitted'; - - -- wh_dim_id generates and returns a random ID which should be considered as - -- universally unique. - create or replace function wh_dim_id() - returns text - as $$ - select encode(digest(gen_random_bytes(16), 'sha256'), 'base64'); - $$ language sql; - - create domain wh_dim_id as text - check( - length(trim(value)) > 0 - ); - comment on domain wh_dim_id is - 'Random ID generated with pgcrypto'; - - create domain wh_public_id as text - check( - value = 'None' - or - length(trim(value)) > 10 - ); - comment on domain wh_public_id is - 'Equivalent to wt_public_id but also allows the value to be ''None'''; - - create domain wh_timestamp as timestamp with time zone not null; - comment on domain wh_timestamp is - 'Timestamp used in warehouse tables'; - - create domain wh_dim_text as text not null - check( - length(trim(value)) > 0 - ); - comment on domain wh_dim_text is - 'Text fields in dimension tables are always not null and always not empty strings'; - - -- wh_date_id returns the wh_date_dimension id for ts. - create or replace function wh_date_id(ts wh_timestamp) - returns integer - as $$ - select to_char(ts, 'YYYYMMDD')::integer; - $$ language sql; - - -- wh_time_id returns the wh_time_of_day_dimension id for ts. - create or replace function wh_time_id(ts wh_timestamp) - returns integer - as $$ - select to_char(ts, 'SSSS')::integer; - $$ language sql; - - -- wh_date_id returns the wh_date_dimension id for current_timestamp. - create or replace function wh_current_date_id() - returns integer - as $$ - select wh_date_id(current_timestamp); - $$ language sql; - - -- wh_time_id returns the wh_time_of_day_dimension id for current_timestamp. - create or replace function wh_current_time_id() - returns integer - as $$ - select wh_time_id(current_timestamp); - $$ language sql; -`), - 62: []byte(` -create table wh_date_dimension ( - id integer primary key, - date date not null, - calendar_quarter wh_dim_text, - calendar_month wh_dim_text, - calendar_year smallint not null, - day_of_week wh_dim_text, - day_of_week_number smallint not null, - day_of_week_number_iso smallint not null, - day_of_week_number_zero_based smallint not null, - day_number_in_calendar_month smallint not null, - day_number_in_calendar_year smallint not null, - weekday_indicator wh_dim_text - ); - - insert into wh_date_dimension ( - id, date, - calendar_quarter, calendar_month, calendar_year, - day_of_week, day_of_week_number, day_of_week_number_iso, day_of_week_number_zero_based, - day_number_in_calendar_month, day_number_in_calendar_year, - weekday_indicator - ) values ( - -1, 'infinity', - 'None', 'None', -1, - 'None', -1, -1, -1, - -1, -1, - 'None' - ); - - insert - into wh_date_dimension - select to_char(t.day, 'YYYYMMDD')::integer as id, - t.day::date as date, - 'Q' || to_char(t.day, 'Q') as calendar_quarter, - to_char(t.day, 'Month') as calendar_month, - extract(year from t.day) as calendar_year, - to_char(t.day, 'Day') as day_of_week, - to_char(t.day, 'D')::int as day_of_week_number, - extract(isodow from t.day) as day_of_week_number_iso, - extract(dow from t.day) as day_of_week_number_zero_based, - extract(day from t.day) as day_number_in_calendar_month, - extract(doy from t.day) as day_number_in_calendar_year, - case extract(isodow from t.day) - when 6 then 'Weekend' - when 7 then 'Weekend' - else 'Weekday' - end as weekday_indicator - from generate_series( - date_trunc('day', timestamp '2019-10-09'), - date_trunc('day', timestamp '2019-10-09' + interval '50 years'), - interval '1 day' - ) as t(day); - - create table wh_time_of_day_dimension ( - id integer primary key, - time_no_zone time not null, - time_at_utc timetz not null, - hour_of_day smallint not null, - minute_of_hour smallint not null, - second_of_minute smallint not null, - display_time_24 wh_dim_text, - display_time_12 wh_dim_text, - meridiem_indicator wh_dim_text - ); - - set timezone = 'UTC'; - - insert into wh_time_of_day_dimension ( - id, time_no_zone, time_at_utc, - hour_of_day, minute_of_hour, second_of_minute, - display_time_24, display_time_12, meridiem_indicator - ) values ( - -1, 'allballs', 'allballs', - -1, -1, -1, - 'None', 'None', 'None' - ); - - insert - into wh_time_of_day_dimension - select to_char(t.second, 'SSSS')::integer as id, - t.second::time as time_no_zone, - t.second::time as time_at_utc, - extract(hour from t.second) as hour_of_day, - extract(minute from t.second) as minute_of_hour, - extract(second from t.second) as second_of_minute, - to_char(t.second, 'HH24:MI:SS') as display_time_24, - to_char(t.second, 'HH12:MI:SS AM') as display_time_12, - to_char(t.second, 'PM') as meridiem_indicator - from generate_series( - date_trunc('day', current_timestamp), - date_trunc('day', current_timestamp) + interval '24 hours' - interval '1 second', - interval '1 second' - ) as t(second); -`), - 65: []byte(` -create table wh_host_dimension ( - -- random id generated using encode(digest(gen_random_bytes(16), 'sha256'), 'base64') - -- this is done to prevent conflicts with rows in other clusters - -- which enables warehouse data from multiple clusters to be loaded into a - -- single database instance - id wh_dim_id primary key default wh_dim_id(), - - host_id wh_public_id not null, - host_type wh_dim_text, - host_name wh_dim_text, - host_description wh_dim_text, - host_address wh_dim_text, - - host_set_id wh_public_id not null, - host_set_type wh_dim_text, - host_set_name wh_dim_text, - host_set_description wh_dim_text, - - host_catalog_id wh_public_id not null, - host_catalog_type wh_dim_text, - host_catalog_name wh_dim_text, - host_catalog_description wh_dim_text, - - target_id wh_public_id not null, - target_type wh_dim_text, - target_name wh_dim_text, - target_description wh_dim_text, - target_default_port_number integer not null, - target_session_max_seconds integer not null, - target_session_connection_limit integer not null, - - project_id wt_scope_id not null, - project_name wh_dim_text, - project_description wh_dim_text, - - host_organization_id wt_scope_id not null, - host_organization_name wh_dim_text, - host_organization_description wh_dim_text, - - current_row_indicator wh_dim_text, - row_effective_time wh_timestamp, - row_expiration_time wh_timestamp - ); - - -- https://www.postgresql.org/docs/current/indexes-partial.html - create unique index wh_host_dim_current_constraint - on wh_host_dimension (target_id, host_set_id, host_id) - where current_row_indicator = 'Current'; - - -- The whx_host_dimension_source and whx_host_dimension_target views are used - -- by an insert trigger to determine if the current row for the dimension has - -- changed and new one needs to be inserted. The first column in the target - -- view must be the current warehouse id and all remaining columns must match - -- the columns in the source view. - - -- The whx_host_dimension_source view shows the current values in the - -- operational tables of the host dimension. - create view whx_host_dimension_source as - select -- id is the first column in the target view - h.public_id as host_id, - 'static host' as host_type, - coalesce(h.name, 'None') as host_name, - coalesce(h.description, 'None') as host_description, - coalesce(h.address, 'Unknown') as host_address, - s.public_id as host_set_id, - 'static host set' as host_set_type, - coalesce(s.name, 'None') as host_set_name, - coalesce(s.description, 'None') as host_set_description, - c.public_id as host_catalog_id, - 'static host catalog' as host_catalog_type, - coalesce(c.name, 'None') as host_catalog_name, - coalesce(c.description, 'None') as host_catalog_description, - t.public_id as target_id, - 'tcp target' as target_type, - coalesce(t.name, 'None') as target_name, - coalesce(t.description, 'None') as target_description, - coalesce(t.default_port, 0) as target_default_port_number, - t.session_max_seconds as target_session_max_seconds, - t.session_connection_limit as target_session_connection_limit, - p.public_id as project_id, - coalesce(p.name, 'None') as project_name, - coalesce(p.description, 'None') as project_description, - o.public_id as host_organization_id, - coalesce(o.name, 'None') as host_organization_name, - coalesce(o.description, 'None') as host_organization_description - from static_host as h, - static_host_catalog as c, - static_host_set_member as m, - static_host_set as s, - target_host_set as ts, - target_tcp as t, - iam_scope as p, - iam_scope as o - where h.catalog_id = c.public_id - and h.public_id = m.host_id - and s.public_id = m.set_id - and t.public_id = ts.target_id - and s.public_id = ts.host_set_id - and p.public_id = t.scope_id - and p.type = 'project' - and o.public_id = p.parent_id - and o.type = 'org' - ; - - -- The whx_host_dimension_target view shows the rows in the wh_host_dimension - -- table marked as 'Current'. - create view whx_host_dimension_target as - select id, - host_id, - host_type, - host_name, - host_description, - host_address, - host_set_id, - host_set_type, - host_set_name, - host_set_description, - host_catalog_id, - host_catalog_type, - host_catalog_name, - host_catalog_description, - target_id, - target_type, - target_name, - target_description, - target_default_port_number, - target_session_max_seconds, - target_session_connection_limit, - project_id, - project_name, - project_description, - host_organization_id, - host_organization_name, - host_organization_description - from wh_host_dimension - where current_row_indicator = 'Current' - ; - - create table wh_user_dimension ( - -- random id generated using encode(digest(gen_random_bytes(16), 'sha256'), 'base64') - -- this is done to prevent conflicts with rows in other clusters - -- which enables warehouse data from multiple clusters to be loaded into a - -- single database instance - id wh_dim_id primary key default wh_dim_id(), - - user_id wt_user_id not null, - user_name wh_dim_text, - user_description wh_dim_text, - - auth_account_id wh_public_id not null, - auth_account_type wh_dim_text, - auth_account_name wh_dim_text, - auth_account_description wh_dim_text, - - auth_method_id wh_public_id not null, - auth_method_type wh_dim_text, - auth_method_name wh_dim_text, - auth_method_description wh_dim_text, - - user_organization_id wt_scope_id not null, - user_organization_name wh_dim_text, - user_organization_description wh_dim_text, - - current_row_indicator wh_dim_text, - row_effective_time wh_timestamp, - row_expiration_time wh_timestamp - ); - - -- The whx_user_dimension_source and whx_user_dimension_target views are used - -- by an insert trigger to determine if the current row for the dimension has - -- changed and new one needs to be inserted. The first column in the target - -- view must be the current warehouse id and all remaining columns must match - -- the columns in the source view. - - -- The whx_user_dimension_source view shows the current values in the - -- operational tables of the user dimension. - create view whx_user_dimension_source as - select -- id is the first column in the target view - u.public_id as user_id, - coalesce(u.name, 'None') as user_name, - coalesce(u.description, 'None') as user_description, - coalesce(aa.public_id, 'None') as auth_account_id, - case when aa.public_id is null then 'None' - else 'password auth account' - end as auth_account_type, - coalesce(apa.name, 'None') as auth_account_name, - coalesce(apa.description, 'None') as auth_account_description, - coalesce(am.public_id, 'None') as auth_method_id, - case when am.public_id is null then 'None' - else 'password auth method' - end as auth_method_type, - coalesce(apm.name, 'None') as auth_method_name, - coalesce(apm.description, 'None') as auth_method_description, - org.public_id as user_organization_id, - coalesce(org.name, 'None') as user_organization_name, - coalesce(org.description, 'None') as user_organization_description - from iam_user as u - left join auth_account as aa on u.public_id = aa.iam_user_id - left join auth_method as am on aa.auth_method_id = am.public_id - left join auth_password_account as apa on aa.public_id = apa.public_id - left join auth_password_method as apm on am.public_id = apm.public_id - join iam_scope as org on u.scope_id = org.public_id - ; - - -- The whx_user_dimension_target view shows the rows in the wh_user_dimension - -- table marked as 'Current'. - create view whx_user_dimension_target as - select id, - user_id, - user_name, - user_description, - auth_account_id, - auth_account_type, - auth_account_name, - auth_account_description, - auth_method_id, - auth_method_type, - auth_method_name, - auth_method_description, - user_organization_id, - user_organization_name, - user_organization_description - from wh_user_dimension - where current_row_indicator = 'Current' - ; -`), - 66: []byte(` --- wh_upsert_host returns the wh_host_dimension id for p_host_id, - -- p_host_set_id, and p_target_id. wh_upsert_host compares the current values - -- in the wh_host_dimension with the current values in the operational tables - -- for the provide parameters. If the values between the operational tables - -- and the wh_host_dimension differ, a new row is inserted in the - -- wh_host_dimension to match the current values in the operational tables and - -- the new id is returned. If the values do not differ, the current id is - -- returned. - create or replace function wh_upsert_host(p_host_id wt_public_id, p_host_set_id wt_public_id, p_target_id wt_public_id) - returns wh_dim_id - as $$ - declare - src whx_host_dimension_target%rowtype; - target whx_host_dimension_target%rowtype; - new_row wh_host_dimension%rowtype; - begin - select * into target - from whx_host_dimension_target as t - where t.host_id = p_host_id - and t.host_set_id = p_host_set_id - and t.target_id = p_target_id; - - select target.id, t.* into src - from whx_host_dimension_source as t - where t.host_id = p_host_id - and t.host_set_id = p_host_set_id - and t.target_id = p_target_id; - - if src is distinct from target then - - -- expire the current row - update wh_host_dimension - set current_row_indicator = 'Expired', - row_expiration_time = current_timestamp - where host_id = p_host_id - and host_set_id = p_host_set_id - and target_id = p_target_id - and current_row_indicator = 'Current'; - - -- insert a new row - insert into wh_host_dimension ( - host_id, host_type, host_name, host_description, host_address, - host_set_id, host_set_type, host_set_name, host_set_description, - host_catalog_id, host_catalog_type, host_catalog_name, host_catalog_description, - target_id, target_type, target_name, target_description, - target_default_port_number, target_session_max_seconds, target_session_connection_limit, - project_id, project_name, project_description, - host_organization_id, host_organization_name, host_organization_description, - current_row_indicator, row_effective_time, row_expiration_time - ) - select host_id, host_type, host_name, host_description, host_address, - host_set_id, host_set_type, host_set_name, host_set_description, - host_catalog_id, host_catalog_type, host_catalog_name, host_catalog_description, - target_id, target_type, target_name, target_description, - target_default_port_number, target_session_max_seconds, target_session_connection_limit, - project_id, project_name, project_description, - host_organization_id, host_organization_name, host_organization_description, - 'Current', current_timestamp, 'infinity'::timestamptz - from whx_host_dimension_source - where host_id = p_host_id - and host_set_id = p_host_set_id - and target_id = p_target_id - returning * into new_row; - - return new_row.id; - end if; - return target.id; - - end; - $$ language plpgsql; - - -- wh_upsert_user returns the wh_user_dimension id for p_user_id and - -- p_auth_token_id. wh_upsert_user compares the current values in the - -- wh_user_dimension with the current values in the operational tables for the - -- provide parameters. If the values between the operational tables and the - -- wh_user_dimension differ, a new row is inserted in the wh_user_dimension to - -- match the current values in the operational tables and the new id is - -- returned. If the values do not differ, the current id is returned. - create or replace function wh_upsert_user(p_user_id wt_user_id, p_auth_token_id wt_public_id) - returns wh_dim_id - as $$ - declare - src whx_user_dimension_target%rowtype; - target whx_user_dimension_target%rowtype; - new_row wh_user_dimension%rowtype; - acct_id wt_public_id; - begin - select auth_account_id into strict acct_id - from auth_token - where public_id = p_auth_token_id; - - select * into target - from whx_user_dimension_target as t - where t.user_id = p_user_id - and t.auth_account_id = acct_id; - - select target.id, t.* into src - from whx_user_dimension_source as t - where t.user_id = p_user_id - and t.auth_account_id = acct_id; - - if src is distinct from target then - - -- expire the current row - update wh_user_dimension - set current_row_indicator = 'Expired', - row_expiration_time = current_timestamp - where user_id = p_user_id - and auth_account_id = acct_id - and current_row_indicator = 'Current'; - - -- insert a new row - insert into wh_user_dimension ( - user_id, user_name, user_description, - auth_account_id, auth_account_type, auth_account_name, auth_account_description, - auth_method_id, auth_method_type, auth_method_name, auth_method_description, - user_organization_id, user_organization_name, user_organization_description, - current_row_indicator, row_effective_time, row_expiration_time - ) - select user_id, user_name, user_description, - auth_account_id, auth_account_type, auth_account_name, auth_account_description, - auth_method_id, auth_method_type, auth_method_name, auth_method_description, - user_organization_id, user_organization_name, user_organization_description, - 'Current', current_timestamp, 'infinity'::timestamptz - from whx_user_dimension_source - where user_id = p_user_id - and auth_account_id = acct_id - returning * into new_row; - - return new_row.id; - end if; - return target.id; - - end; - $$ language plpgsql; -`), - 68: []byte(` --- Column names for numeric fields that are not a measurement end in id or - -- number. This naming convention enables automatic field type detection in - -- certain data analysis tools. - -- https://help.tableau.com/current/pro/desktop/en-us/data_clean_adm.htm - - -- The wh_session_accumulating_fact table is an accumulating snapshot. - -- The table wh_session_accumulating_fact is an accumulating fact table. - -- The grain of the fact table is one row per session. - create table wh_session_accumulating_fact ( - session_id wt_public_id primary key, - -- auth token id is a degenerate dimension - auth_token_id wt_public_id not null, - - -- foreign keys to the dimension tables - host_id wh_dim_id not null - references wh_host_dimension (id) - on delete restrict - on update cascade, - user_id wh_dim_id not null - references wh_user_dimension (id) - on delete restrict - on update cascade, - - -- TODO(mgaffney) 09/2020: add dimension and foreign key for the session - -- termination reason - - -- date and time foreign keys - session_pending_date_id integer not null - references wh_date_dimension (id) - on delete restrict - on update cascade, - session_pending_time_id integer not null - references wh_time_of_day_dimension (id) - on delete restrict - on update cascade, - session_pending_time wh_timestamp, - - session_active_date_id integer default -1 not null - references wh_date_dimension (id) - on delete restrict - on update cascade, - session_active_time_id integer default -1 not null - references wh_time_of_day_dimension (id) - on delete restrict - on update cascade, - session_active_time wh_timestamp default 'infinity'::timestamptz, - - session_canceling_date_id integer default -1 not null - references wh_date_dimension (id) - on delete restrict - on update cascade, - session_canceling_time_id integer default -1 not null - references wh_time_of_day_dimension (id) - on delete restrict - on update cascade, - session_canceling_time wh_timestamp default 'infinity'::timestamptz, - - session_terminated_date_id integer default -1 not null - references wh_date_dimension (id) - on delete restrict - on update cascade, - session_terminated_time_id integer default -1 not null - references wh_time_of_day_dimension (id) - on delete restrict - on update cascade, - session_terminated_time wh_timestamp default 'infinity'::timestamptz, - - -- TODO(mgaffney) 09/2020: add columns for session expiration - - -- TODO(mgaffney) 09/2020: add connection limit. This may need a dimension - -- table and foreign key column to represent unlimited connections. - - -- The total number of connections made during the session. - total_connection_count bigint, -- will be null until the first connection is created - - -- The total number of bytes received by workers from the client and sent - -- to the endpoint for this session. - -- total_bytes_up is a fully additive measurement. - total_bytes_up wh_bytes_transmitted, -- will be null until the first connection is closed - -- The total number of bytes received by workers from the endpoint and sent - -- to the client for this session. - -- total_bytes_down is a fully additive measurement. - total_bytes_down wh_bytes_transmitted -- will be null until the first connection is closed - ); - - -- The wh_session_connection_accumulating_fact table is an accumulating fact table. - -- The grain of the fact table is one row per session connection. - create table wh_session_connection_accumulating_fact ( - connection_id wt_public_id primary key, - -- session_id is a degenerate dimension - session_id wt_public_id not null - references wh_session_accumulating_fact (session_id) - on delete cascade - on update cascade, - - -- foreign keys to the dimension tables - host_id wh_dim_id not null - references wh_host_dimension (id) - on delete restrict - on update cascade, - user_id wh_dim_id not null - references wh_user_dimension (id) - on delete restrict - on update cascade, - - -- TODO(mgaffney) 09/2020: add dimension and foreign key for the connection - -- closed reason - - -- date and time foreign keys and timestamps - connection_authorized_date_id integer not null - references wh_date_dimension (id) - on delete restrict - on update cascade, - connection_authorized_time_id integer not null - references wh_time_of_day_dimension (id) - on delete restrict - on update cascade, - connection_authorized_time wh_timestamp, - - connection_connected_date_id integer default -1 not null - references wh_date_dimension (id) - on delete restrict - on update cascade, - connection_connected_time_id integer default -1 not null - references wh_time_of_day_dimension (id) - on delete restrict - on update cascade, - connection_connected_time wh_timestamp default 'infinity'::timestamptz, - - connection_closed_date_id integer default -1 not null - references wh_date_dimension (id) - on delete restrict - on update cascade, - connection_closed_time_id integer default -1 not null - references wh_time_of_day_dimension (id) - on delete restrict - on update cascade, - connection_closed_time wh_timestamp default 'infinity'::timestamptz, - - -- TODO(mgaffney) 09/2020: add a connection_duration_in_seconds column - - -- The client address and port are degenerate dimensions - client_tcp_address inet, -- can be null - client_tcp_port_number wh_inet_port, -- can be null - - -- The endpoint address and port are degenerate dimensions - endpoint_tcp_address inet, -- can be null - endpoint_tcp_port_number wh_inet_port, -- can be null - - -- the connection_count must always be 1 - -- this is a common pattern in data warehouse models - -- See The Data Warehouse Toolkit, Third Edition - -- by Ralph Kimball and Margy Ross for more information - connection_count smallint default 1 not null - constraint connection_count_must_be_1 - check(connection_count = 1), - - -- The total number of bytes received by the worker from the client and sent - -- to the endpoint for this connection. - -- bytes_up is a fully additive measurement. - bytes_up wh_bytes_transmitted, -- can be null - -- The total number of bytes received by the worker from the endpoint and sent - -- to the client for this connection. - -- bytes_down is a fully additive measurement. - bytes_down wh_bytes_transmitted -- can be null - ); - - -- TODO(mgaffney) 09/2020: Research and test if the comment fields are used by - -- data analysis tools. - comment on table wh_session_connection_accumulating_fact is - 'The Wh Session Connection Accumulating Fact table is an accumulating fact table. ' - 'The grain of the fact table is one row per session connection.'; - comment on column wh_session_connection_accumulating_fact.bytes_up is - 'Bytes Up is the total number of bytes received by the worker from the ' - 'client and sent to the endpoint for this connection. Bytes Up is a fully ' - 'additive measurement.'; - comment on column wh_session_connection_accumulating_fact.bytes_down is - 'Bytes Down is the total number of bytes received by the worker from the ' - 'endpoint and sent to the client for this connection. Bytes Down is a fully ' - 'additive measurement.'; - - create index on wh_session_connection_accumulating_fact(session_id); -`), - 69: []byte(` --- wh_rollup_connections calculates the aggregate values from - -- wh_session_connection_accumulating_fact for p_session_id and updates - -- wh_session_accumulating_fact for p_session_id with those values. - create or replace function wh_rollup_connections(p_session_id wt_public_id) - returns void - as $$ - declare - session_row wh_session_accumulating_fact%rowtype; - begin - with - session_totals (session_id, total_connection_count, total_bytes_up, total_bytes_down) as ( - select session_id, - sum(connection_count), - sum(bytes_up), - sum(bytes_down) - from wh_session_connection_accumulating_fact - where session_id = p_session_id - group by session_id - ) - update wh_session_accumulating_fact - set total_connection_count = session_totals.total_connection_count, - total_bytes_up = session_totals.total_bytes_up, - total_bytes_down = session_totals.total_bytes_down - from session_totals - where wh_session_accumulating_fact.session_id = session_totals.session_id - returning wh_session_accumulating_fact.* into strict session_row; - end; - $$ language plpgsql; - - -- - -- Session triggers - -- - - -- wh_insert_session returns an after insert trigger for the session table - -- which inserts a row in wh_session_accumulating_fact for the new session. - -- wh_insert_session also calls the wh_upsert_host and wh_upsert_user - -- functions which can result in new rows in wh_host_dimension and - -- wh_user_dimension respectively. - create or replace function wh_insert_session() - returns trigger - as $$ - declare - new_row wh_session_accumulating_fact%rowtype; - begin - with - pending_timestamp (date_dim_id, time_dim_id, ts) as ( - select wh_date_id(start_time), wh_time_id(start_time), start_time - from session_state - where session_id = new.public_id - and state = 'pending' - ) - insert into wh_session_accumulating_fact ( - session_id, - auth_token_id, - host_id, - user_id, - session_pending_date_id, - session_pending_time_id, - session_pending_time - ) - select new.public_id, - new.auth_token_id, - wh_upsert_host(new.host_id, new.host_set_id, new.target_id), - wh_upsert_user(new.user_id, new.auth_token_id), - pending_timestamp.date_dim_id, - pending_timestamp.time_dim_id, - pending_timestamp.ts - from pending_timestamp - returning * into strict new_row; - return null; - end; - $$ language plpgsql; - - create trigger wh_insert_session - after insert on session - for each row - execute function wh_insert_session(); - - -- - -- Session Connection triggers - -- - - -- wh_insert_session_connection returns an after insert trigger for the - -- session_connection table which inserts a row in - -- wh_session_connection_accumulating_fact for the new session connection. - -- wh_insert_session_connection also calls wh_rollup_connections which can - -- result in updates to wh_session_accumulating_fact. - create or replace function wh_insert_session_connection() - returns trigger - as $$ - declare - new_row wh_session_connection_accumulating_fact%rowtype; - begin - with - authorized_timestamp (date_dim_id, time_dim_id, ts) as ( - select wh_date_id(start_time), wh_time_id(start_time), start_time - from session_connection_state - where connection_id = new.public_id - and state = 'authorized' - ), - session_dimension (host_dim_id, user_dim_id) as ( - select host_id, user_id - from wh_session_accumulating_fact - where session_id = new.session_id - ) - insert into wh_session_connection_accumulating_fact ( - connection_id, - session_id, - host_id, - user_id, - connection_authorized_date_id, - connection_authorized_time_id, - connection_authorized_time, - client_tcp_address, - client_tcp_port_number, - endpoint_tcp_address, - endpoint_tcp_port_number, - bytes_up, - bytes_down - ) - select new.public_id, - new.session_id, - session_dimension.host_dim_id, - session_dimension.user_dim_id, - authorized_timestamp.date_dim_id, - authorized_timestamp.time_dim_id, - authorized_timestamp.ts, - new.client_tcp_address, - new.client_tcp_port, - new.endpoint_tcp_address, - new.endpoint_tcp_port, - new.bytes_up, - new.bytes_down - from authorized_timestamp, - session_dimension - returning * into strict new_row; - perform wh_rollup_connections(new.session_id); - return null; - end; - $$ language plpgsql; - - create trigger wh_insert_session_connection - after insert on session_connection - for each row - execute function wh_insert_session_connection(); - - -- wh_update_session_connection returns an after update trigger for the - -- session_connection table which updates a row in - -- wh_session_connection_accumulating_fact for the session connection. - -- wh_update_session_connection also calls wh_rollup_connections which can - -- result in updates to wh_session_accumulating_fact. - create or replace function wh_update_session_connection() - returns trigger - as $$ - declare - updated_row wh_session_connection_accumulating_fact%rowtype; - begin - update wh_session_connection_accumulating_fact - set client_tcp_address = new.client_tcp_address, - client_tcp_port_number = new.client_tcp_port, - endpoint_tcp_address = new.endpoint_tcp_address, - endpoint_tcp_port_number = new.endpoint_tcp_port, - bytes_up = new.bytes_up, - bytes_down = new.bytes_down - where connection_id = new.public_id - returning * into strict updated_row; - perform wh_rollup_connections(new.session_id); - return null; - end; - $$ language plpgsql; - - create trigger wh_update_session_connection - after update on session_connection - for each row - execute function wh_update_session_connection(); - - -- - -- Session State trigger - -- - - -- wh_insert_session_state returns an after insert trigger for the - -- session_state table which updates wh_session_accumulating_fact. - create or replace function wh_insert_session_state() - returns trigger - as $$ - declare - date_col text; - time_col text; - ts_col text; - q text; - session_row wh_session_accumulating_fact%rowtype; - begin - if new.state = 'pending' then - -- The pending state is the first state which is handled by the - -- wh_insert_session trigger. The update statement in this trigger will - -- fail for the pending state because the row for the session has not yet - -- been inserted into the wh_session_accumulating_fact table. - return null; - end if; - - date_col = 'session_' || new.state || '_date_id'; - time_col = 'session_' || new.state || '_time_id'; - ts_col = 'session_' || new.state || '_time'; - - q = format('update wh_session_accumulating_fact - set (%I, %I, %I) = (select wh_date_id(%L), wh_time_id(%L), %L::timestamptz) - where session_id = %L - returning *', - date_col, time_col, ts_col, - new.start_time, new.start_time, new.start_time, - new.session_id); - execute q into strict session_row; - - return null; - end; - $$ language plpgsql; - - create trigger wh_insert_session_state - after insert on session_state - for each row - execute function wh_insert_session_state(); - - -- - -- Session Connection State trigger - -- - - -- wh_insert_session_connection_state returns an after insert trigger for the - -- session_connection_state table which updates - -- wh_session_connection_accumulating_fact. - create or replace function wh_insert_session_connection_state() - returns trigger - as $$ - declare - date_col text; - time_col text; - ts_col text; - q text; - connection_row wh_session_connection_accumulating_fact%rowtype; - begin - if new.state = 'authorized' then - -- The authorized state is the first state which is handled by the - -- wh_insert_session_connection trigger. The update statement in this - -- trigger will fail for the authorized state because the row for the - -- session connection has not yet been inserted into the - -- wh_session_connection_accumulating_fact table. - return null; - end if; - - date_col = 'connection_' || new.state || '_date_id'; - time_col = 'connection_' || new.state || '_time_id'; - ts_col = 'connection_' || new.state || '_time'; - - q = format('update wh_session_connection_accumulating_fact - set (%I, %I, %I) = (select wh_date_id(%L), wh_time_id(%L), %L::timestamptz) - where connection_id = %L - returning *', - date_col, time_col, ts_col, - new.start_time, new.start_time, new.start_time, - new.connection_id); - execute q into strict connection_row; - - return null; - end; - $$ language plpgsql; - - create trigger wh_insert_session_connection_state - after insert on session_connection_state - for each row - execute function wh_insert_session_connection_state(); -`), - 1001: []byte(` --- This series of expressions fixes the primary key on the server table --- PG 12+ -alter table session - drop constraint if exists session_server_id_server_type_fkey; --- PG 11 -alter table session - drop constraint if exists session_server_id_fkey; -alter table server - drop constraint server_pkey; -alter table server - drop column name; -alter table server - add primary key (private_id); -alter table server - add constraint server_id_must_not_be_empty - check(length(trim(private_id)) > 0); -alter table session - add constraint session_server_id_fkey - foreign key (server_id) - references server(private_id) - on delete set null - on update cascade; - -create domain wt_bexprfilter as text -check( - value is null - or - ( - length(trim(value)) > 0 - and - length(trim(value)) <= 2048 - ) -); -comment on domain wt_bexprfilter is - 'Text field with constraints for go-bexpr filters'; - --- Add the worker filter to the target_tcp table and session table -alter table target_tcp - add column worker_filter wt_bexprfilter; -alter table session - add column worker_filter wt_bexprfilter; - --- Replace the immutable columns trigger from 50 to add worker_filter -drop trigger immutable_columns on session; -create trigger immutable_columns - before update on session - for each row execute procedure immutable_columns('public_id', 'certificate', 'expiration_time', 'connection_limit', 'create_time', 'endpoint', 'worker_filter'); - --- Replaces the view created in 41 to include worker_filter -drop view target_all_subtypes; -create view target_all_subtypes -as -select - public_id, - scope_id, - name, - description, - default_port, - session_max_seconds, - session_connection_limit, - version, - create_time, - update_time, - worker_filter, - 'tcp' as type -from target_tcp; - --- Replaces the view created in 50 to include worker_filter -drop view session_with_state; -create view session_with_state as - select - s.public_id, - s.user_id, - s.host_id, - s.server_id, - s.server_type, - s.target_id, - s.host_set_id, - s.auth_token_id, - s.scope_id, - s.certificate, - s.expiration_time, - s.connection_limit, - s.tofu_token, - s.key_id, - s.termination_reason, - s.version, - s.create_time, - s.update_time, - s.endpoint, - s.worker_filter, - ss.state, - ss.previous_end_time, - ss.start_time, - ss.end_time - from - session s, - session_state ss - where - s.public_id = ss.session_id; - -create domain wt_tagpair as text -check( - value is not null - and - length(trim(value)) > 0 - and - length(trim(value)) <= 512 - and - lower(trim(value)) = value -); -comment on domain wt_tagpair is - 'Text field with constraints for key/value pairs'; - -create table server_tag ( - server_id text - references server(private_id) - on delete cascade - on update cascade, - key wt_tagpair, - value wt_tagpair, - primary key(server_id, key, value) -); -`), - 1002: []byte(` --- wt_email defines a type for email which must be less than 320 chars and only --- contain lower case values. The type is defined to allow nulls and not be --- unique, which can be overriden as needed when used in tables. -create domain wt_email as text - constraint wt_email_too_short - check (length(trim(value)) > 0) - constraint wt_email_too_long - check (length(trim(value)) < 320); -comment on domain wt_email is -'standard column for email addresses'; - --- wt_full_name defines a type for a person's full name which must be less than --- 512 chars. The type is defined to allow nulls and not be unique, which can --- be overriden as needed when used in tables. -create domain wt_full_name text - constraint wt_full_name_too_short - check (length(trim(value)) > 0) - constraint wt_full_name_too_long - check(length(trim(value)) <= 512); -- gotta pick some upper limit. -comment on domain wt_full_name is -'standard column for the full name of a person'; - --- wt_url defines a type for URLs which must be longer that 3 chars and --- less than 4k chars. It's defined to allow nulls, which can be overridden as --- needed when used in tables. -create domain wt_url as text - constraint wt_url_too_short - check (length(trim(value)) > 3) - constraint wt_url_too_long - check (length(trim(value)) < 4000) - constraint wt_url_invalid_protocol - check (value ~ 'https?:\/\/*'); -comment on domain wt_url is -'standard column for URLs'; - --- wt_name defines a type for resource names that must be less than 128 chars. --- It's defined to allow nulls. -create domain wt_name as text - constraint wt_name_too_short - check (length(trim(value)) > 0) - constraint wt_name_too_long - check (length(trim(value)) < 128); -comment on domain wt_name is -'standard column for resource names'; - --- wt_description defines a type for resource descriptions that must be less --- than 1024 chars. It's defined to allow nulls. -create domain wt_description as text - constraint wt_description_too_short - check (length(trim(value)) > 0) - constraint wt_description_too_long - check (length(trim(value)) < 1024); -comment on domain wt_description is -'standard column for resource descriptions'; -`), - 1003: []byte(` --- kms_oidc_key entries are DEKs for encrypting oidc entries. -create table kms_oidc_key ( - private_id wt_private_id primary key, - root_key_id wt_private_id not null unique -- there can be only one oidc dek per root key - references kms_root_key(private_id) - on delete cascade - on update cascade, - create_time wt_timestamp -); - - -- define the immutable fields for kms_oidc_key (all of them) -create trigger - immutable_columns -before -update on kms_oidc_key - for each row execute procedure immutable_columns('private_id', 'root_key_id', 'create_time'); - --- define the value of kms_oidc_key's create_time -create trigger - default_create_time_column -before -insert on kms_oidc_key - for each row execute procedure default_create_time(); - --- kms_oidc_key_version entries are version of DEK keys used to encrypt oidc --- entries. -create table kms_oidc_key_version ( - private_id wt_private_id primary key, - oidc_key_id wt_private_id not null - references kms_oidc_key(private_id) - on delete cascade - on update cascade, - root_key_version_id wt_private_id not null - references kms_root_key_version(private_id) - on delete cascade - on update cascade, - version wt_version, - key bytea not null, - create_time wt_timestamp, - unique(oidc_key_id, version) -); - - -- define the immutable fields for kms_oidc_key_version (all of them) -create trigger - immutable_columns -before -update on kms_oidc_key_version - for each row execute procedure immutable_columns('private_id', 'oidc_key_id', 'root_key_version_id', 'version', 'key', 'create_time'); - --- define the value of kms_oidc_key_version's create_time -create trigger - default_create_time_column -before -insert on kms_oidc_key_version - for each row execute procedure default_create_time(); - --- define the value of kms_oidc_key_version's version column -create trigger - kms_version_column -before insert on kms_oidc_key_version - for each row execute procedure kms_version_column('oidc_key_id'); -`), - 10001: []byte(` -alter table job_run - alter column server_id type text; -alter table job_run - add constraint server_id_must_not_be_empty - check(length(trim(server_id)) > 0); -`), - 10002: []byte(` -create function wt_is_sentinel(string text) - returns bool - as $$ - select length(trim(leading u&'\fffe ' from string)) > 0 AND starts_with(string, u&'\fffe'); - $$ language sql - immutable - returns null on null input; - comment on function wt_is_sentinel is - 'wt_is_sentinel returns true if string is a sentinel value'; - - create domain wt_sentinel as text - constraint wt_sentinel_not_valid - check( - wt_is_sentinel(value) - or - length(trim(u&'\fffe ' from value)) > 0 - ); - comment on domain wt_sentinel is - 'A non-empty string with a Unicode prefix of U+FFFE to indicate it is a sentinel value'; - - create function wt_to_sentinel(string text) - returns text - as $$ - select concat(u&'\fffe', trim(ltrim(string, u&'\fffe '))); - $$ language sql - immutable - returns null on null input; - comment on function wt_to_sentinel is - 'wt_to_sentinel takes string and returns it as a wt_sentinel'; -`), - 10003: []byte(` --- credential_store - create table credential_store ( - public_id wt_public_id primary key, - scope_id wt_scope_id not null - constraint iam_scope_fkey - references iam_scope (public_id) - on delete cascade - on update cascade, - - -- The order of columns is important for performance. See: - -- https://dba.stackexchange.com/questions/58970/enforcing-constraints-two-tables-away/58972#58972 - -- https://dba.stackexchange.com/questions/27481/is-a-composite-index-also-good-for-queries-on-the-first-field - constraint credential_store_scope_id_public_id_uq - unique(scope_id, public_id) - ); - comment on table credential_store is - 'credential_store is a base table for the credential store type. ' - 'Each row is owned by a single scope and maps 1-to-1 to a row in one of the credential store subtype tables.'; - - create trigger immutable_columns before update on credential_store - for each row execute procedure immutable_columns('public_id', 'scope_id'); - - -- insert_credential_store_subtype() is a before insert trigger - -- function for subtypes of credential_store - create function insert_credential_store_subtype() - returns trigger - as $$ - begin - insert into credential_store - (public_id, scope_id) - values - (new.public_id, new.scope_id); - return new; - end; - $$ language plpgsql; - - -- delete_credential_store_subtype() is an after delete trigger - -- function for subtypes of credential_store - create function delete_credential_store_subtype() - returns trigger - as $$ - begin - delete from credential_store - where public_id = old.public_id; - return null; -- result is ignored since this is an after trigger - end; - $$ language plpgsql; - - - -- credential_library - create table credential_library ( - public_id wt_public_id primary key, - store_id wt_public_id not null - constraint credential_store_fkey - references credential_store (public_id) - on delete cascade - on update cascade, - constraint credential_library_store_id_public_id_uq - unique(store_id, public_id) - ); - comment on table credential_library is - 'credential_library is a base table for the credential library type and a child table of credential_store. ' - 'Each row maps 1-to-1 to a row in one of the credential library subtype tables.'; - - create trigger immutable_columns before update on credential_library - for each row execute procedure immutable_columns('public_id', 'store_id'); - - -- insert_credential_library_subtype() is a before insert trigger - -- function for subtypes of credential_library - create function insert_credential_library_subtype() - returns trigger - as $$ - begin - insert into credential_library - (public_id, store_id) - values - (new.public_id, new.store_id); - return new; - end; - $$ language plpgsql; - - -- delete_credential_library_subtype() is an after delete trigger - -- function for subtypes of credential_library - create function delete_credential_library_subtype() - returns trigger - as $$ - begin - delete from credential_library - where public_id = old.public_id; - return null; -- result is ignored since this is an after trigger - end; - $$ language plpgsql; - - -- credential - create table credential ( - public_id wt_public_id primary key - ); - comment on table credential is - 'credential is a base table for the credential type. '; - - create trigger immutable_columns before update on credential - for each row execute procedure immutable_columns('public_id'); - - -- insert_credential_subtype() is a before insert trigger - -- function for subtypes of credential - create function insert_credential_subtype() - returns trigger - as $$ - begin - insert into credential - (public_id) - values - (new.public_id); - return new; - end; - $$ language plpgsql; - - -- delete_credential_subtype() is an after delete trigger - -- function for subtypes of credential - create function delete_credential_subtype() - returns trigger - as $$ - begin - delete from credential - where public_id = old.public_id; - return null; -- result is ignored since this is an after trigger - end; - $$ language plpgsql; - - -- credential_static - create table credential_static ( - public_id wt_public_id primary key - constraint credential_fkey - references credential (public_id) - on delete cascade - on update cascade, - store_id wt_public_id not null - constraint credential_store_fkey - references credential_store (public_id) - on delete cascade - on update cascade, - constraint credential_static_store_id_public_id_uq - unique(store_id, public_id) - ); - comment on table credential_static is - 'credential_static is a base table for the credential static type. ' - 'It is a credential subtype and a child table of credential_store. '; - - create trigger immutable_columns before update on credential_static - for each row execute procedure immutable_columns('public_id', 'store_id'); - - create trigger insert_credential_subtype before insert on credential_static - for each row execute procedure insert_credential_subtype(); - - create trigger delete_credential_subtype after delete on credential_static - for each row execute procedure delete_credential_subtype(); - - -- insert_credential_static_subtype() is a before insert trigger - -- function for subtypes of credential_static - create function insert_credential_static_subtype() - returns trigger - as $$ - begin - insert into credential_static - (public_id, store_id) - values - (new.public_id, new.store_id); - return new; - end; - $$ language plpgsql; - - -- delete_credential_static_subtype() is an after delete trigger - -- function for subtypes of credential_static - create function delete_credential_static_subtype() - returns trigger - as $$ - begin - delete from credential_static - where public_id = old.public_id; - return null; -- result is ignored since this is an after trigger - end; - $$ language plpgsql; - - -- credential_dynamic - create table credential_dynamic ( - public_id wt_public_id primary key - constraint credential_fkey - references credential (public_id) - on delete cascade - on update cascade, - library_id wt_public_id not null - constraint credential_library_fkey - references credential_library (public_id) - on delete cascade - on update cascade, - constraint credential_dynamic_library_id_public_id_uq - unique(library_id, public_id) - ); - comment on table credential_dynamic is - 'credential_dynamic is a base table for the credential dynamic type. ' - 'It is a credential subtype and a child table of credential_library. '; - - create trigger immutable_columns before update on credential_dynamic - for each row execute procedure immutable_columns('public_id', 'library_id'); - - create trigger insert_credential_subtype before insert on credential_dynamic - for each row execute procedure insert_credential_subtype(); - - create trigger delete_credential_subtype after delete on credential_dynamic - for each row execute procedure delete_credential_subtype(); - - -- insert_credential_dynamic_subtype() is a before insert trigger - -- function for subtypes of credential_dynamic - create function insert_credential_dynamic_subtype() - returns trigger - as $$ - begin - insert into credential_dynamic - (public_id, library_id) - values - (new.public_id, new.library_id); - return new; - end; - $$ language plpgsql; - - -- delete_credential_dynamic_subtype() is an after delete trigger - -- function for subtypes of credential_dynamic - create function delete_credential_dynamic_subtype() - returns trigger - as $$ - begin - delete from credential_dynamic - where public_id = old.public_id; - return null; -- result is ignored since this is an after trigger - end; - $$ language plpgsql; - - create table credential_purpose_enm ( - name text primary key - constraint only_predefined_credential_purposes_allowed - check ( - name in ( - 'application', - 'ingress', - 'egress' - ) - ) - ); - comment on table credential_purpose_enm is - 'credential_purpose_enm is an enumeration table for credential purposes. ' - 'It contains rows for representing the application, egress, and ingress credential purposes.'; - - insert into credential_purpose_enm (name) - values - ('application'), - ('ingress'), - ('egress'); -`), - 10004: []byte(` -create table credential_vault_store ( - public_id wt_public_id primary key, - scope_id wt_scope_id not null - constraint iam_scope_fkey - references iam_scope (public_id) - on delete cascade - on update cascade, - name wt_name, - description wt_description, - create_time wt_timestamp, - update_time wt_timestamp, - -- delete_time is set to indicate the row has been soft deleted - delete_time timestamp with time zone, - version wt_version, - vault_address wt_url not null, - -- the remaining text columns can be null but if they are not null, they - -- cannot contain an empty string - namespace text - constraint namespace_must_not_be_empty - check(length(trim(namespace)) > 0), - ca_cert bytea -- PEM encoded certificate bundle - constraint ca_cert_must_not_be_empty - check(length(ca_cert) > 0), - tls_server_name text - constraint tls_server_name_must_not_be_empty - check(length(trim(tls_server_name)) > 0), - tls_skip_verify boolean default false not null, - constraint credential_store_fkey - foreign key (scope_id, public_id) - references credential_store (scope_id, public_id) - on delete cascade - on update cascade, - constraint credential_vault_store_scope_id_name_uq - unique(scope_id, name) - ); - comment on table credential_vault_store is - 'credential_vault_store is a table where each row is a resource that represents a vault credential store. ' - 'It is a credential_store subtype.'; - - create trigger update_version_column after update on credential_vault_store - for each row execute procedure update_version_column(); - - create trigger update_time_column before update on credential_vault_store - for each row execute procedure update_time_column(); - - create trigger default_create_time_column before insert on credential_vault_store - for each row execute procedure default_create_time(); - - create trigger immutable_columns before update on credential_vault_store - for each row execute procedure immutable_columns('public_id', 'scope_id','create_time'); - - create trigger insert_credential_store_subtype before insert on credential_vault_store - for each row execute procedure insert_credential_store_subtype(); - - create trigger delete_credential_store_subtype after delete on credential_vault_store - for each row execute procedure delete_credential_store_subtype(); - - -- before_soft_delete_credential_vault_store is a before update trigger for - -- the credential_vault_store table that makes the delete_time column a - -- set once column. Once the delete_time column is set to a value other than - -- null, it cannot be changed. If the current delete_time of a row is not null - -- and an update contains a value for delete_time different from the current - -- value, this trigger will raise an error with error code 23602 which is a - -- class 23 integrity constraint violation: set_once_violation. - create function before_soft_delete_credential_vault_store() - returns trigger - as $$ - begin - if new.delete_time is distinct from old.delete_time then - if old.delete_time is not null then - raise exception 'set_once_violation: %.%', tg_table_name, 'delete_time' using - errcode = '23602', - schema = tg_table_schema, - table = tg_table_name, - column = 'delete_time'; - end if; - end if; - return new; - end; - $$ language plpgsql; - - create trigger before_soft_delete_credential_vault_store before update on credential_vault_store - for each row execute procedure before_soft_delete_credential_vault_store(); - - -- after_soft_delete_credential_vault_store is an after update trigger for the - -- credential_vault_store table that performs cleanup actions when a - -- credential store is soft deleted. A credential store is considered "soft - -- deleted" if the delete_time for the row is updated from null to not null. - -- - -- When a credential store is soft deleted, this trigger: - -- * marks any active Vault tokens owned by the credential store for revocation - -- * deletes any credential library owned by the credential store - create function after_soft_delete_credential_vault_store() - returns trigger - as $$ - begin - if new.delete_time is distinct from old.delete_time then - if old.delete_time is null then - - -- mark current and maintaining tokens as revoke - update credential_vault_token - set status = 'revoke' - where store_id = new.public_id - and status in ('current', 'maintaining'); - - -- delete the store's libraries - delete - from credential_vault_library - where store_id = new.public_id; - - end if; - end if; - return null; - end; - $$ language plpgsql; - - create trigger after_soft_delete_credential_vault_store after update on credential_vault_store - for each row execute procedure after_soft_delete_credential_vault_store(); - - create table credential_vault_token_status_enm ( - name text primary key - constraint only_predefined_token_statuses_allowed - check ( - name in ( - 'current', - 'maintaining', - 'revoke', - 'revoked', - 'expired' - ) - ) - ); - comment on table credential_vault_token_status_enm is - 'credential_vault_token_status_enm is an enumeration table for the status of vault tokens. ' - 'It contains rows for representing the current, maintaining, revoke, revoked, and expired statuses.'; - - insert into credential_vault_token_status_enm (name) - values - ('current'), - ('maintaining'), - ('revoke'), - ('revoked'), - ('expired'); - - create table credential_vault_token ( - token_hmac bytea primary key, -- hmac-sha256(token, key(blake2b-256(token_accessor)) - token bytea not null, -- encrypted value - store_id wt_public_id not null - constraint credential_vault_store_fkey - references credential_vault_store (public_id) - on delete cascade - on update cascade, - create_time wt_timestamp, - update_time wt_timestamp, - last_renewal_time timestamp with time zone not null, - expiration_time timestamp with time zone not null - constraint last_renewal_time_must_be_before_expiration_time - check(last_renewal_time < expiration_time), - key_id text not null - constraint kms_database_key_version_fkey - references kms_database_key_version (private_id) - on delete restrict - on update cascade, - status text not null - constraint credential_vault_token_status_enm_fkey - references credential_vault_token_status_enm (name) - on delete restrict - on update cascade - ); - comment on table credential_vault_token is - 'credential_vault_token is a table where each row contains a Vault token for one Vault credential store. ' - 'A credential store can have only one vault token with the status of current'; - comment on column credential_vault_token.token_hmac is - 'token_hmac contains the hmac-sha256 value of the token. ' - 'The hmac key is the blake2b-256 value of the token accessor.'; - - -- https://www.postgresql.org/docs/current/indexes-partial.html - create unique index credential_vault_token_current_status_constraint - on credential_vault_token (store_id) - where status = 'current'; - - create index credential_vault_token_expiration_time_ix - on credential_vault_token(expiration_time); - comment on index credential_vault_token_expiration_time_ix is - 'the credential_vault_token_expiration_time_ix is used by the token renewal job'; - - create trigger update_time_column before update on credential_vault_token - for each row execute procedure update_time_column(); - - create trigger default_create_time_column before insert on credential_vault_token - for each row execute procedure default_create_time(); - - create trigger immutable_columns before update on credential_vault_token - for each row execute procedure immutable_columns('token_hmac', 'token', 'store_id','create_time'); - - -- insert_credential_vault_token() is a before insert trigger - -- function for credential_vault_token that changes the status of the current - -- token to 'maintaining' - create function insert_credential_vault_token() - returns trigger - as $$ - begin - update credential_vault_token - set status = 'maintaining' - where store_id = new.store_id - and status = 'current'; - return new; - end; - $$ language plpgsql; - - create trigger insert_credential_vault_token before insert on credential_vault_token - for each row execute procedure insert_credential_vault_token(); - - create table credential_vault_client_certificate ( - store_id wt_public_id primary key - constraint credential_vault_store_fkey - references credential_vault_store (public_id) - on delete cascade - on update cascade, - certificate bytea not null -- PEM encoded certificate - constraint certificate_must_not_be_empty - check(length(certificate) > 0), - certificate_key bytea not null -- encrypted PEM encoded private key for certificate - constraint certificate_key_must_not_be_empty - check(length(certificate_key) > 0), - certificate_key_hmac bytea not null - constraint certificate_key_hmac_must_not_be_empty - check(length(certificate_key_hmac) > 0), - key_id text not null - constraint kms_database_key_version_fkey - references kms_database_key_version (private_id) - on delete restrict - on update cascade - ); - comment on table credential_vault_client_certificate is - 'credential_vault_client_certificate is a table where each row contains a client certificate that a credential_vault_store uses for mTLS when connecting to Vault. ' - 'A credential_vault_store can have 0 or 1 client certificates.'; - - create trigger immutable_columns before update on credential_vault_client_certificate - for each row execute procedure immutable_columns('store_id'); - - create table credential_vault_http_method_enm ( - name text primary key - constraint only_predefined_http_methods_allowed - check ( - name in ( - 'GET', - 'POST' - ) - ) - ); - comment on table credential_vault_http_method_enm is - 'credential_vault_http_method_enm is an enumeration table for the http method used by a vault library when communicating with vault. ' - 'It contains rows for representing the HTTP GET and the HTTP POST methods.'; - - insert into credential_vault_http_method_enm (name) - values - ('GET'), - ('POST'); - - create table credential_vault_library ( - public_id wt_public_id primary key, - store_id wt_public_id not null - constraint credential_vault_store_fkey - references credential_vault_store (public_id) - on delete cascade - on update cascade, - name wt_name, - description wt_description, - create_time wt_timestamp, - update_time wt_timestamp, - version wt_version, - vault_path text not null - constraint vault_path_must_not_be_empty - check(length(trim(vault_path)) > 0), - http_method text not null - constraint credential_vault_http_method_enm_fkey - references credential_vault_http_method_enm (name) - on delete restrict - on update cascade, - http_request_body bytea - constraint http_request_body_only_allowed_with_post_method - check( - http_request_body is null - or - ( - http_method = 'POST' - and - length(http_request_body) > 0 - ) - ), - constraint credential_vault_library_store_id_name_uq - unique(store_id, name), - constraint credential_library_fkey - foreign key (store_id, public_id) - references credential_library (store_id, public_id) - on delete cascade - on update cascade, - constraint credential_vault_library_store_id_public_id_uq - unique(store_id, public_id) - ); - comment on table credential_vault_library is - 'credential_vault_library is a table where each row is a resource that represents a vault credential library. ' - 'It is a credential_library subtype and a child table of credential_vault_store.'; - - create trigger update_version_column after update on credential_vault_library - for each row execute procedure update_version_column(); - - create trigger update_time_column before update on credential_vault_library - for each row execute procedure update_time_column(); - - create trigger default_create_time_column before insert on credential_vault_library - for each row execute procedure default_create_time(); - - create trigger immutable_columns before update on credential_vault_library - for each row execute procedure immutable_columns('public_id', 'store_id','create_time'); - - create trigger insert_credential_library_subtype before insert on credential_vault_library - for each row execute procedure insert_credential_library_subtype(); - - create trigger delete_credential_library_subtype after delete on credential_vault_library - for each row execute procedure delete_credential_library_subtype(); - - - -- before_insert_credential_vault_library is a before insert trigger for - -- the credential_vault_library table that prevents a library from being - -- inserted for a soft deleted credential store. - create function before_insert_credential_vault_library() - returns trigger - as $$ - declare - delete_time_val timestamp with time zone; - begin - select delete_time into delete_time_val - from credential_vault_store - where public_id = new.store_id; - - if delete_time_val is not null then - raise exception 'foreign_key_violation: %.%', tg_table_name, 'store_id' using - errcode = '23503', - schema = tg_table_schema, - table = tg_table_name, - column = 'store_id'; - end if; - return new; - end; - $$ language plpgsql; - - create trigger before_insert_credential_vault_library before insert on credential_vault_library - for each row execute procedure before_insert_credential_vault_library(); - - create table credential_vault_credential_status_enm ( - name text primary key - constraint only_predefined_credential_statuses_allowed - check ( - name in ( - 'active', - 'revoke', - 'revoked', - 'expired', - 'unknown' - ) - ) - ); - comment on table credential_vault_credential_status_enm is - 'credential_vault_credential_status_enm is an enumeration table for the status of vault credentials. ' - 'It contains rows for representing the active, revoke, revoked, expired, and unknown statuses.'; - - insert into credential_vault_credential_status_enm (name) - values - ('active'), - ('revoke'), - ('revoked'), - ('expired'), - ('unknown'); - - create table credential_vault_credential ( - public_id wt_public_id primary key, - library_id wt_public_id - constraint credential_vault_library_fkey - references credential_vault_library (public_id) - on delete set null - on update cascade, - session_id wt_public_id - constraint session_fkey - references session (public_id) - on delete set null - on update cascade, - token_hmac bytea not null - constraint credential_vault_token_fkey - references credential_vault_token (token_hmac) - on delete cascade - on update cascade, - create_time wt_timestamp, - update_time wt_timestamp, - version wt_version, - external_id wt_sentinel not null, - last_renewal_time timestamp with time zone not null, - expiration_time timestamp with time zone not null - constraint last_renewal_time_must_be_before_expiration_time - check(last_renewal_time < expiration_time), - is_renewable boolean not null, - status text not null - constraint credential_vault_credential_status_enm_fkey - references credential_vault_credential_status_enm (name) - on delete restrict - on update cascade, - constraint credential_dynamic_fkey - foreign key (library_id, public_id) - references credential_dynamic (library_id, public_id) - on delete cascade - on update cascade, - constraint credential_vault_credential_library_id_public_id_uq - unique(library_id, public_id) - ); - comment on table credential_vault_credential is - 'credential_vault_credential is a table where each row contains the lease information for a single Vault secret retrieved from a vault credential library for a session.'; - - create trigger update_version_column after update on credential_vault_credential - for each row execute procedure update_version_column(); - - create trigger update_time_column before update on credential_vault_credential - for each row execute procedure update_time_column(); - - -- update_credential_status_column() is a before update trigger function for - -- credential_vault_credential that changes the status of the credential to 'revoke' if - -- the session_id is updated to null - create function update_credential_status_column() - returns trigger - as $$ - begin - if new.session_id is distinct from old.session_id then - if new.session_id is null and old.status = 'active' then - new.status = 'revoke'; - end if; - end if; - return new; - end; - $$ language plpgsql; - - create trigger update_credential_status_column before update on credential_vault_credential - for each row execute procedure update_credential_status_column(); - - -- not_null_columns() will make the column names not null which are passed as - -- parameters when the trigger is created. It raises error code 23502 which is a - -- class 23 integrity constraint violation: not null column - create function not_null_columns() - returns trigger - as $$ - declare - col_name text; - new_value text; - begin - foreach col_name in array tg_argv loop - execute format('SELECT $1.%I', col_name) into new_value using new; - if new_value is null then - raise exception 'not null column: %.%', tg_table_name, col_name using - errcode = '23502', - schema = tg_table_schema, - table = tg_table_name, - column = col_name; - end if; - end loop; - return new; - end; - $$ language plpgsql; - - comment on function not_null_columns() is - 'function used in before insert triggers to make columns not null on insert, but are allowed be updated to null'; - - create trigger not_null_columns before insert on credential_vault_credential - for each row execute procedure not_null_columns('library_id', 'session_id'); - - create trigger default_create_time_column before insert on credential_vault_credential - for each row execute procedure default_create_time(); - - create trigger immutable_columns before update on credential_vault_credential - for each row execute procedure immutable_columns('external_id', 'create_time'); - - create trigger insert_credential_dynamic_subtype before insert on credential_vault_credential - for each row execute procedure insert_credential_dynamic_subtype(); - - create trigger delete_credential_dynamic_subtype after delete on credential_vault_credential - for each row execute procedure delete_credential_dynamic_subtype(); - - create index credential_vault_credential_expiration_time_ix - on credential_vault_credential(expiration_time); - comment on index credential_vault_credential_expiration_time_ix is - 'the credential_vault_credential_expiration_time_ix is used by the credential renewal job'; - - insert into oplog_ticket (name, version) - values - ('credential_vault_store', 1), - ('credential_vault_library', 1), - ('credential_vault_credential', 1) ; - - create view credential_vault_store_private as - with - active_tokens as ( - select token_hmac, - token, -- encrypted - store_id, - create_time, - update_time, - last_renewal_time, - expiration_time, - -- renewal time is the midpoint between the last renewal time and the expiration time - last_renewal_time + (expiration_time - last_renewal_time) / 2 as renewal_time, - key_id, - status - from credential_vault_token - where status in ('current', 'maintaining', 'revoke') - ) - select store.public_id as public_id, - store.scope_id as scope_id, - store.name as name, - store.description as description, - store.create_time as create_time, - store.update_time as update_time, - store.delete_time as delete_time, - store.version as version, - store.vault_address as vault_address, - store.namespace as namespace, - store.ca_cert as ca_cert, - store.tls_server_name as tls_server_name, - store.tls_skip_verify as tls_skip_verify, - store.public_id as store_id, - token.token_hmac as token_hmac, - token.token as ct_token, -- encrypted - token.create_time as token_create_time, - token.update_time as token_update_time, - token.last_renewal_time as token_last_renewal_time, - token.expiration_time as token_expiration_time, - token.renewal_time as token_renewal_time, - token.key_id as token_key_id, - token.status as token_status, - cert.certificate as client_cert, - cert.certificate_key as ct_client_key, -- encrypted - cert.certificate_key_hmac as client_cert_key_hmac, - cert.key_id as client_key_id - from credential_vault_store store - left join active_tokens token - on store.public_id = token.store_id - left join credential_vault_client_certificate cert - on store.public_id = cert.store_id; - comment on view credential_vault_store_private is - 'credential_vault_store_private is a view where each row contains a credential store and the credential store''s data needed to connect to Vault. ' - 'The view returns a separate row for each current, maintaining and revoke token; maintaining tokens should only be used for token/credential renewal and revocation. ' - 'Each row may contain encrypted data. This view should not be used to retrieve data which will be returned external to boundary.'; - - create view credential_vault_store_public as - select public_id, - scope_id, - name, - description, - create_time, - update_time, - version, - vault_address, - namespace, - ca_cert, - tls_server_name, - tls_skip_verify, - token_hmac, - token_create_time, - token_update_time, - token_last_renewal_time, - token_expiration_time, - client_cert, - client_cert_key_hmac - from credential_vault_store_private - where token_status = 'current' - and delete_time is null; - comment on view credential_vault_store_public is - 'credential_vault_store_public is a view where each row contains a credential store. ' - 'No encrypted data is returned. This view can be used to retrieve data which will be returned external to boundary.'; - - create view credential_vault_library_private as - select library.public_id as public_id, - library.store_id as store_id, - library.name as name, - library.description as description, - library.create_time as create_time, - library.update_time as update_time, - library.version as version, - library.vault_path as vault_path, - library.http_method as http_method, - library.http_request_body as http_request_body, - store.scope_id as scope_id, - store.vault_address as vault_address, - store.namespace as namespace, - store.ca_cert as ca_cert, - store.tls_server_name as tls_server_name, - store.tls_skip_verify as tls_skip_verify, - store.token_hmac as token_hmac, - store.ct_token as ct_token, -- encrypted - store.token_key_id as token_key_id, - store.client_cert as client_cert, - store.ct_client_key as ct_client_key, -- encrypted - store.client_key_id as client_key_id - from credential_vault_library library - join credential_vault_store_private store - on library.store_id = store.public_id - and store.token_status = 'current'; - comment on view credential_vault_library_private is - 'credential_vault_library_private is a view where each row contains a credential library and the credential library''s data needed to connect to Vault. ' - 'Each row may contain encrypted data. This view should not be used to retrieve data which will be returned external to boundary.'; - - create view credential_vault_credential_private as - select credential.public_id as public_id, - credential.library_id as library_id, - credential.session_id as session_id, - credential.create_time as create_time, - credential.update_time as update_time, - credential.version as version, - credential.external_id as external_id, - credential.last_renewal_time as last_renewal_time, - credential.expiration_time as expiration_time, - credential.is_renewable as is_renewable, - credential.status as status, - credential.last_renewal_time + (credential.expiration_time - credential.last_renewal_time) / 2 as renewal_time, - token.token_hmac as token_hmac, - token.token as ct_token, -- encrypted - token.create_time as token_create_time, - token.update_time as token_update_time, - token.last_renewal_time as token_last_renewal_time, - token.expiration_time as token_expiration_time, - token.key_id as token_key_id, - token.status as token_status, - store.scope_id as scope_id, - store.vault_address as vault_address, - store.namespace as namespace, - store.ca_cert as ca_cert, - store.tls_server_name as tls_server_name, - store.tls_skip_verify as tls_skip_verify, - cert.certificate as client_cert, - cert.certificate_key as ct_client_key, -- encrypted - cert.certificate_key_hmac as client_cert_key_hmac, - cert.key_id as client_key_id - from credential_vault_credential credential - join credential_vault_token token - on credential.token_hmac = token.token_hmac - join credential_vault_store store - on token.store_id = store.public_id - left join credential_vault_client_certificate cert - on store.public_id = cert.store_id - where credential.expiration_time != 'infinity'::date; - comment on view credential_vault_credential_private is - 'credential_vault_credential_private is a view where each row contains a credential, ' - 'the vault token used to issue the credential, and the credential store data needed to connect to Vault. ' - 'Each row may contain encrypted data. This view should not be used to retrieve data which will be returned external to boundary.'; -`), - 10005: []byte(` -create table target_credential_library ( - target_id wt_public_id not null - constraint target_fkey - references target (public_id) - on delete cascade - on update cascade, - credential_library_id wt_public_id not null - constraint credential_library_fkey - references credential_library (public_id) - on delete cascade - on update cascade, - credential_purpose text not null - constraint credential_purpose_enm_fkey - references credential_purpose_enm (name) - on delete restrict - on update cascade, - create_time wt_timestamp, - primary key(target_id, credential_library_id, credential_purpose) - ); - comment on table target_credential_library is - 'target_credential_library is a join table between the target and credential_library tables. ' - 'It also contains the credential purpose the relationship represents.'; - - create trigger default_create_time_column before insert on target_credential_library - for each row execute procedure default_create_time(); - - create trigger immutable_columns before update on target_credential_library - for each row execute procedure immutable_columns('target_id', 'credential_library_id', 'credential_purpose', 'create_time'); - - -- target_library provides the store id along with the other data stored in - -- target_credential_library - create view target_library - as - select - tcl.target_id, - tcl.credential_library_id, - tcl.credential_purpose, - cl.store_id - from - target_credential_library tcl, - credential_library cl - where - cl.public_id = tcl.credential_library_id; -`), - 10006: []byte(` -create table session_credential_dynamic ( - session_id wt_public_id not null - constraint session_fkey - references session (public_id) - on delete cascade - on update cascade, - library_id wt_public_id not null - constraint credential_library_fkey - references credential_library (public_id) - on delete cascade - on update cascade, - credential_id wt_public_id - constraint credential_dynamic_fkey - references credential_dynamic (public_id) - on delete cascade - on update cascade, - credential_purpose text not null - constraint credential_purpose_fkey - references credential_purpose_enm (name) - on delete restrict - on update cascade, - primary key(session_id, library_id, credential_purpose), - create_time wt_timestamp, - constraint session_credential_dynamic_credential_id_uq - unique(credential_id) - ); - comment on table session_credential_dynamic is - 'session_credential_dynamic is a join table between the session and dynamic credential tables. ' - 'It also contains the credential purpose the relationship represents.'; - - create trigger default_create_time_column before insert on session_credential_dynamic - for each row execute procedure default_create_time(); - - create trigger immutable_columns before update on session_credential_dynamic - for each row execute procedure immutable_columns('session_id', 'library_id', 'credential_purpose', 'create_time'); - - -- revoke_credentials revokes any active credentials for a session when the - -- session enters the canceling or terminated states. - create function revoke_credentials() - returns trigger - as $$ - begin - if new.state in ('canceling', 'terminated') then - update credential_vault_credential - set status = 'revoke' - where session_id = new.session_id - and status = 'active'; - end if; - return new; - end; - $$ language plpgsql; - create trigger revoke_credentials after insert on session_state - for each row execute procedure revoke_credentials(); -`), - 10007: []byte(` -update credential_vault_credential - set external_id = concat(external_id, u&'\ffff') - where wt_is_sentinel(external_id) - and not starts_with(reverse(external_id), u&'\ffff'); - -alter domain wt_sentinel - drop constraint wt_sentinel_not_valid; - -drop function wt_is_sentinel; - -create function wt_is_sentinel(string text) - returns bool -as $$ -select starts_with(string, u&'\fffe') and starts_with(reverse(string), u&'\ffff'); -$$ language sql - immutable - returns null on null input; -comment on function wt_is_sentinel is - 'wt_is_sentinel returns true if string is a sentinel value'; - -alter domain wt_sentinel - add constraint wt_sentinel_not_valid - check( - wt_is_sentinel(value) - or - length(trim(trailing u&'\ffff' from trim(leading u&'\fffe ' from value))) > 0 - ); - -comment on domain wt_sentinel is - 'A non-empty string with a Unicode prefix of U+FFFE and suffix of U+FFFF to indicate it is a sentinel value'; - -drop function wt_to_sentinel; -- wt_to_sentinel is not needed, dropping and not re-creating -`), - 11001: []byte(` -create table server_type_enm ( - name text primary key - constraint only_predefined_server_types_allowed - check ( - name in ( - 'controller', - 'worker' - ) - ) -); -comment on table server_type_enm is - 'server_type_enm is an enumeration table for server types. ' - 'It contains rows for representing servers as either a controller or worker.'; - -insert into server_type_enm (name) values - ('controller'), - ('worker'); - -alter table server - add constraint server_type_enm_fkey - foreign key (type) references server_type_enm(name) - on update cascade - on delete restrict; -`), - 12001: []byte(` -create function wt_sub_seconds(sec integer, ts timestamp with time zone) - returns timestamp with time zone - as $$ - select ts - sec * '1 second'::interval; - $$ language sql - stable - returns null on null input; - comment on function wt_add_seconds is - 'wt_sub_seconds returns ts - sec.'; - - create function wt_sub_seconds_from_now(sec integer) - returns timestamp with time zone - as $$ - select wt_sub_seconds(sec, current_timestamp); - $$ language sql - stable - returns null on null input; - comment on function wt_add_seconds_to_now is - 'wt_sub_seconds_from_now returns current_timestamp - sec.'; -`), - 13001: []byte(` -alter table auth_oidc_account - add column token_claims text - constraint token_claims_must_not_be_empty - check( - length(trim(token_claims)) > 0 - ); -alter table auth_oidc_account - add column userinfo_claims text - constraint userinfo_claims_must_not_be_empty - check( - length(trim(userinfo_claims)) > 0 - ); -`), - 14001: []byte(` -alter table wh_user_dimension - add column auth_method_external_id text, - add column auth_account_external_id text, - add column auth_account_full_name text, - add column auth_account_email text - ; - - update wh_user_dimension - set auth_method_type = - case when auth_method_id like 'ampw_%' then 'password auth method' - when auth_method_id like 'amoidc_%' then 'oidc auth method' - else 'Unknown' end, - auth_account_type = - case when auth_account_id like 'acctpw_%' then 'password auth account' - when auth_account_id like 'acctoidc_%' then 'oidc auth account' - else 'Unknown' end, - auth_method_external_id = - case when auth_method_id like 'ampw_%' then 'Not Applicable' - else 'Unknown' end, - auth_account_external_id = - case when auth_method_id like 'ampw_%' then 'Not Applicable' - else 'Unknown' end, - auth_account_full_name = - case when auth_method_id like 'ampw_%' then 'Not Applicable' - else 'Unknown' end, - auth_account_email = - case when auth_method_id like 'ampw_%' then 'Not Applicable' - else 'Unknown' end; - - alter table wh_user_dimension - alter column auth_method_external_id type wh_dim_text, - alter column auth_account_external_id type wh_dim_text, - alter column auth_account_full_name type wh_dim_text, - alter column auth_account_email type wh_dim_text - ; - - drop view whx_user_dimension_source; - create view whx_user_dimension_source as - select -- id is the first column in the target view - u.public_id as user_id, - coalesce(u.name, 'None') as user_name, - coalesce(u.description, 'None') as user_description, - coalesce(aa.public_id, 'None') as auth_account_id, - case - when apa.public_id is not null then 'password auth account' - when aoa.public_id is not null then 'oidc auth account' - else 'None' - end as auth_account_type, - case - when apa.public_id is not null then coalesce(apa.name, 'None') - when aoa.public_id is not null then coalesce(aoa.name, 'None') - else 'None' - end as auth_account_name, - case - when apa.public_id is not null then coalesce(apa.description, 'None') - when aoa.public_id is not null then coalesce(aoa.description, 'None') - else 'None' - end as auth_account_description, - case - when apa.public_id is not null then 'Not Applicable' - when aoa.public_id is null then 'None' - else aoa.subject - end as auth_account_external_id, - case - when apa.public_id is not null then 'Not Applicable' - when aoa.public_id is not null - and aoa.full_name is not null then aoa.full_name - else 'None' - end as auth_account_full_name, - case - when apa.public_id is not null then 'Not Applicable' - when aoa.public_id is not null - and aoa.email is not null then aoa.email - else 'None' - end as auth_account_email, - coalesce(am.public_id, 'None') as auth_method_id, - case - when apa.public_id is not null then 'password auth method' - when aoa.public_id is not null then 'oidc auth method' - else 'None' - end as auth_method_type, - case - when apm.public_id is not null then coalesce(apm.name, 'None') - when aom.public_id is not null then coalesce(aom.name, 'None') - else 'None' - end as auth_method_name, - case - when apm.public_id is not null then coalesce(apm.description, 'None') - when aom.public_id is not null then coalesce(aom.description, 'None') - else 'None' - end as auth_method_description, - case - when apa.public_id is not null then 'Not Applicable' - when aom.public_id is null then 'None' - else aom.issuer - end as auth_method_external_id, - org.public_id as user_organization_id, - coalesce(org.name, 'None') as user_organization_name, - coalesce(org.description, 'None') as user_organization_description - from iam_user as u - left join auth_account as aa on u.public_id = aa.iam_user_id - left join auth_method as am on aa.auth_method_id = am.public_id - left join auth_password_account as apa on aa.public_id = apa.public_id - left join auth_password_method as apm on am.public_id = apm.public_id - left join auth_oidc_account as aoa on aa.public_id = aoa.public_id - left join auth_oidc_method as aom on am.public_id = aom.public_id - join iam_scope as org on u.scope_id = org.public_id - ; - - drop view whx_user_dimension_target; - create view whx_user_dimension_target as - select id, - user_id, - user_name, - user_description, - auth_account_id, - auth_account_type, - auth_account_name, - auth_account_description, - auth_account_external_id, - auth_account_full_name, - auth_account_email, - auth_method_id, - auth_method_type, - auth_method_name, - auth_method_description, - auth_method_external_id, - user_organization_id, - user_organization_name, - user_organization_description - from wh_user_dimension - where current_row_indicator = 'Current' - ; - - drop function wh_upsert_user; - create function wh_upsert_user(p_user_id wt_user_id, p_auth_token_id wt_public_id) - returns wh_dim_id - as $$ - declare - src whx_user_dimension_target%rowtype; - target whx_user_dimension_target%rowtype; - new_row wh_user_dimension%rowtype; - acct_id wt_public_id; - begin - select auth_account_id into strict acct_id - from auth_token - where public_id = p_auth_token_id; - - select * into target - from whx_user_dimension_target as t - where t.user_id = p_user_id - and t.auth_account_id = acct_id; - - select target.id, t.* into src - from whx_user_dimension_source as t - where t.user_id = p_user_id - and t.auth_account_id = acct_id; - - if src is distinct from target then - - -- expire the current row - update wh_user_dimension - set current_row_indicator = 'Expired', - row_expiration_time = current_timestamp - where user_id = p_user_id - and auth_account_id = acct_id - and current_row_indicator = 'Current'; - - -- insert a new row - insert into wh_user_dimension ( - user_id, user_name, user_description, - auth_account_id, auth_account_type, auth_account_name, auth_account_description, - auth_account_external_id, auth_account_full_name, auth_account_email, - auth_method_id, auth_method_type, auth_method_name, auth_method_description, - auth_method_external_id, - user_organization_id, user_organization_name, user_organization_description, - current_row_indicator, row_effective_time, row_expiration_time - ) - select user_id, user_name, user_description, - auth_account_id, auth_account_type, auth_account_name, auth_account_description, - auth_account_external_id, auth_account_full_name, auth_account_email, - auth_method_id, auth_method_type, auth_method_name, auth_method_description, - auth_method_external_id, - user_organization_id, user_organization_name, user_organization_description, - 'Current', current_timestamp, 'infinity'::timestamptz - from whx_user_dimension_source - where user_id = p_user_id - and auth_account_id = acct_id - returning * into new_row; - - return new_row.id; - end if; - return target.id; - - end; - $$ language plpgsql; -`), - 15001: []byte(` -alter table wh_date_dimension - rename column id to key; - - alter table wh_time_of_day_dimension - rename column id to key; - - alter table wh_host_dimension - rename column id to key; - - alter table wh_user_dimension - rename column id to key; - - alter table wh_session_accumulating_fact - rename column host_id to host_key; - alter table wh_session_accumulating_fact - rename column user_id to user_key; - alter table wh_session_accumulating_fact - rename column session_pending_date_id to session_pending_date_key; - alter table wh_session_accumulating_fact - rename column session_pending_time_id to session_pending_time_key; - alter table wh_session_accumulating_fact - rename column session_active_date_id to session_active_date_key; - alter table wh_session_accumulating_fact - rename column session_active_time_id to session_active_time_key; - alter table wh_session_accumulating_fact - rename column session_canceling_date_id to session_canceling_date_key; - alter table wh_session_accumulating_fact - rename column session_canceling_time_id to session_canceling_time_key; - alter table wh_session_accumulating_fact - rename column session_terminated_date_id to session_terminated_date_key; - alter table wh_session_accumulating_fact - rename column session_terminated_time_id to session_terminated_time_key; - - alter table wh_session_connection_accumulating_fact - rename column host_id to host_key; - alter table wh_session_connection_accumulating_fact - rename column user_id to user_key; - alter table wh_session_connection_accumulating_fact - rename column connection_authorized_date_id to connection_authorized_date_key; - alter table wh_session_connection_accumulating_fact - rename column connection_authorized_time_id to connection_authorized_time_key; - alter table wh_session_connection_accumulating_fact - rename column connection_connected_date_id to connection_connected_date_key; - alter table wh_session_connection_accumulating_fact - rename column connection_connected_time_id to connection_connected_time_key; - alter table wh_session_connection_accumulating_fact - rename column connection_closed_date_id to connection_closed_date_key; - alter table wh_session_connection_accumulating_fact - rename column connection_closed_time_id to connection_closed_time_key; - - -- rename function from internal/db/schema/migrations/postgres/0/60_wh_domain_types.up.sql - alter function wh_dim_id rename to wh_dim_key; - - -- rename domain from internal/db/schema/migrations/postgres/0/60_wh_domain_types.up.sql - alter domain wh_dim_id rename to wh_dim_key; - - -- removes unused function from internal/db/schema/migrations/postgres/0/60_wh_domain_types.up.sql - drop function wh_current_date_id; - - -- removes unused function from internal/db/schema/migrations/postgres/0/60_wh_domain_types.up.sql - drop function wh_current_time_id; - - -- rename function from internal/db/schema/migrations/postgres/0/60_wh_domain_types.up.sql - alter function wh_date_id rename to wh_date_key; - - -- rename function from internal/db/schema/migrations/postgres/0/60_wh_domain_types.up.sql - alter function wh_time_id rename to wh_time_key; - - -- replaces view from 14/01_wh_user_dimension_oidc.up.sql - drop view whx_user_dimension_target; - create view whx_user_dimension_target as - select key, - user_id, - user_name, - user_description, - auth_account_id, - auth_account_type, - auth_account_name, - auth_account_description, - auth_account_external_id, - auth_account_full_name, - auth_account_email, - auth_method_id, - auth_method_type, - auth_method_name, - auth_method_description, - auth_method_external_id, - user_organization_id, - user_organization_name, - user_organization_description - from wh_user_dimension - where current_row_indicator = 'Current' - ; - - -- replaces function from 14/01_wh_user_dimension_oidc.up.sql - drop function wh_upsert_user; - create function wh_upsert_user(p_user_id wt_user_id, p_auth_token_id wt_public_id) - returns wh_dim_key - as $$ - declare - src whx_user_dimension_target%rowtype; - target whx_user_dimension_target%rowtype; - new_row wh_user_dimension%rowtype; - acct_id wt_public_id; - begin - select auth_account_id into strict acct_id - from auth_token - where public_id = p_auth_token_id; - - select * into target - from whx_user_dimension_target as t - where t.user_id = p_user_id - and t.auth_account_id = acct_id; - - select target.key, t.* into src - from whx_user_dimension_source as t - where t.user_id = p_user_id - and t.auth_account_id = acct_id; - - if src is distinct from target then - - -- expire the current row - update wh_user_dimension - set current_row_indicator = 'Expired', - row_expiration_time = current_timestamp - where user_id = p_user_id - and auth_account_id = acct_id - and current_row_indicator = 'Current'; - - -- insert a new row - insert into wh_user_dimension ( - user_id, user_name, user_description, - auth_account_id, auth_account_type, auth_account_name, auth_account_description, - auth_account_external_id, auth_account_full_name, auth_account_email, - auth_method_id, auth_method_type, auth_method_name, auth_method_description, - auth_method_external_id, - user_organization_id, user_organization_name, user_organization_description, - current_row_indicator, row_effective_time, row_expiration_time - ) - select user_id, user_name, user_description, - auth_account_id, auth_account_type, auth_account_name, auth_account_description, - auth_account_external_id, auth_account_full_name, auth_account_email, - auth_method_id, auth_method_type, auth_method_name, auth_method_description, - auth_method_external_id, - user_organization_id, user_organization_name, user_organization_description, - 'Current', current_timestamp, 'infinity'::timestamptz - from whx_user_dimension_source - where user_id = p_user_id - and auth_account_id = acct_id - returning * into new_row; - - return new_row.key; - end if; - return target.key; - - end; - $$ language plpgsql; - - -- replaces view from 0/65_wh_session_dimensions.up.sql - drop view whx_host_dimension_target; - create view whx_host_dimension_target as - select key, - host_id, - host_type, - host_name, - host_description, - host_address, - host_set_id, - host_set_type, - host_set_name, - host_set_description, - host_catalog_id, - host_catalog_type, - host_catalog_name, - host_catalog_description, - target_id, - target_type, - target_name, - target_description, - target_default_port_number, - target_session_max_seconds, - target_session_connection_limit, - project_id, - project_name, - project_description, - host_organization_id, - host_organization_name, - host_organization_description - from wh_host_dimension - where current_row_indicator = 'Current' - ; - - -- replaces function from 0/66_wh_session_dimensions.up.sql - drop function wh_upsert_host; - create function wh_upsert_host(p_host_id wt_public_id, p_host_set_id wt_public_id, p_target_id wt_public_id) - returns wh_dim_key - as $$ - declare - src whx_host_dimension_target%rowtype; - target whx_host_dimension_target%rowtype; - new_row wh_host_dimension%rowtype; - begin - select * into target - from whx_host_dimension_target as t - where t.host_id = p_host_id - and t.host_set_id = p_host_set_id - and t.target_id = p_target_id; - - select target.key, t.* into src - from whx_host_dimension_source as t - where t.host_id = p_host_id - and t.host_set_id = p_host_set_id - and t.target_id = p_target_id; - - if src is distinct from target then - - -- expire the current row - update wh_host_dimension - set current_row_indicator = 'Expired', - row_expiration_time = current_timestamp - where host_id = p_host_id - and host_set_id = p_host_set_id - and target_id = p_target_id - and current_row_indicator = 'Current'; - - -- insert a new row - insert into wh_host_dimension ( - host_id, host_type, host_name, host_description, host_address, - host_set_id, host_set_type, host_set_name, host_set_description, - host_catalog_id, host_catalog_type, host_catalog_name, host_catalog_description, - target_id, target_type, target_name, target_description, - target_default_port_number, target_session_max_seconds, target_session_connection_limit, - project_id, project_name, project_description, - host_organization_id, host_organization_name, host_organization_description, - current_row_indicator, row_effective_time, row_expiration_time - ) - select host_id, host_type, host_name, host_description, host_address, - host_set_id, host_set_type, host_set_name, host_set_description, - host_catalog_id, host_catalog_type, host_catalog_name, host_catalog_description, - target_id, target_type, target_name, target_description, - target_default_port_number, target_session_max_seconds, target_session_connection_limit, - project_id, project_name, project_description, - host_organization_id, host_organization_name, host_organization_description, - 'Current', current_timestamp, 'infinity'::timestamptz - from whx_host_dimension_source - where host_id = p_host_id - and host_set_id = p_host_set_id - and target_id = p_target_id - returning * into new_row; - - return new_row.key; - end if; - return target.key; - - end; - $$ language plpgsql; - - -- replaces function from 0/69_wh_session_facts.up.sql - drop trigger wh_insert_session on session; - drop function wh_insert_session; - create function wh_insert_session() - returns trigger - as $$ - declare - new_row wh_session_accumulating_fact%rowtype; - begin - with - pending_timestamp (date_dim_key, time_dim_key, ts) as ( - select wh_date_key(start_time), wh_time_key(start_time), start_time - from session_state - where session_id = new.public_id - and state = 'pending' - ) - insert into wh_session_accumulating_fact ( - session_id, - auth_token_id, - host_key, - user_key, - session_pending_date_key, - session_pending_time_key, - session_pending_time - ) - select new.public_id, - new.auth_token_id, - wh_upsert_host(new.host_id, new.host_set_id, new.target_id), - wh_upsert_user(new.user_id, new.auth_token_id), - pending_timestamp.date_dim_key, - pending_timestamp.time_dim_key, - pending_timestamp.ts - from pending_timestamp - returning * into strict new_row; - return null; - end; - $$ language plpgsql; - create trigger wh_insert_session - after insert on session - for each row - execute function wh_insert_session(); - - -- replaces function from 0/69_wh_session_facts.up.sql - drop trigger wh_insert_session_connection on session_connection; - drop function wh_insert_session_connection; - create function wh_insert_session_connection() - returns trigger - as $$ - declare - new_row wh_session_connection_accumulating_fact%rowtype; - begin - with - authorized_timestamp (date_dim_key, time_dim_key, ts) as ( - select wh_date_key(start_time), wh_time_key(start_time), start_time - from session_connection_state - where connection_id = new.public_id - and state = 'authorized' - ), - session_dimension (host_dim_key, user_dim_key) as ( - select host_key, user_key - from wh_session_accumulating_fact - where session_id = new.session_id - ) - insert into wh_session_connection_accumulating_fact ( - connection_id, - session_id, - host_key, - user_key, - connection_authorized_date_key, - connection_authorized_time_key, - connection_authorized_time, - client_tcp_address, - client_tcp_port_number, - endpoint_tcp_address, - endpoint_tcp_port_number, - bytes_up, - bytes_down - ) - select new.public_id, - new.session_id, - session_dimension.host_dim_key, - session_dimension.user_dim_key, - authorized_timestamp.date_dim_key, - authorized_timestamp.time_dim_key, - authorized_timestamp.ts, - new.client_tcp_address, - new.client_tcp_port, - new.endpoint_tcp_address, - new.endpoint_tcp_port, - new.bytes_up, - new.bytes_down - from authorized_timestamp, - session_dimension - returning * into strict new_row; - perform wh_rollup_connections(new.session_id); - return null; - end; - $$ language plpgsql; - - create trigger wh_insert_session_connection - after insert on session_connection - for each row - execute function wh_insert_session_connection(); - - -- replaces function from 0/69_wh_session_facts.up.sql - drop trigger wh_insert_session_state on session_state; - drop function wh_insert_session_state; - create function wh_insert_session_state() - returns trigger - as $$ - declare - date_col text; - time_col text; - ts_col text; - q text; - session_row wh_session_accumulating_fact%rowtype; - begin - if new.state = 'pending' then - -- The pending state is the first state which is handled by the - -- wh_insert_session trigger. The update statement in this trigger will - -- fail for the pending state because the row for the session has not yet - -- been inserted into the wh_session_accumulating_fact table. - return null; - end if; - - date_col = 'session_' || new.state || '_date_key'; - time_col = 'session_' || new.state || '_time_key'; - ts_col = 'session_' || new.state || '_time'; - - q = format('update wh_session_accumulating_fact - set (%I, %I, %I) = (select wh_date_key(%L), wh_time_key(%L), %L::timestamptz) - where session_id = %L - returning *', - date_col, time_col, ts_col, - new.start_time, new.start_time, new.start_time, - new.session_id); - execute q into strict session_row; - - return null; - end; - $$ language plpgsql; - - create trigger wh_insert_session_state - after insert on session_state - for each row - execute function wh_insert_session_state(); - - -- replaces function from 0/69_wh_session_facts.up.sql - drop trigger wh_insert_session_connection_state on session_connection_state; - drop function wh_insert_session_connection_state; - create function wh_insert_session_connection_state() - returns trigger - as $$ - declare - date_col text; - time_col text; - ts_col text; - q text; - connection_row wh_session_connection_accumulating_fact%rowtype; - begin - if new.state = 'authorized' then - -- The authorized state is the first state which is handled by the - -- wh_insert_session_connection trigger. The update statement in this - -- trigger will fail for the authorized state because the row for the - -- session connection has not yet been inserted into the - -- wh_session_connection_accumulating_fact table. - return null; - end if; - - date_col = 'connection_' || new.state || '_date_key'; - time_col = 'connection_' || new.state || '_time_key'; - ts_col = 'connection_' || new.state || '_time'; - - q = format('update wh_session_connection_accumulating_fact - set (%I, %I, %I) = (select wh_date_key(%L), wh_time_key(%L), %L::timestamptz) - where connection_id = %L - returning *', - date_col, time_col, ts_col, - new.start_time, new.start_time, new.start_time, - new.connection_id); - execute q into strict connection_row; - - return null; - end; - $$ language plpgsql; - - create trigger wh_insert_session_connection_state - after insert on session_connection_state - for each row - execute function wh_insert_session_connection_state(); -`), - 15002: []byte(` -alter table wh_host_dimension - rename column host_organization_id to organization_id; - alter table wh_host_dimension - rename column host_organization_name to organization_name; - alter table wh_host_dimension - rename column host_organization_description to organization_description; - - -- replaces view from 0/65_wh_session_dimensions.up.sql - drop view whx_host_dimension_source; - create view whx_host_dimension_source as - select -- id is the first column in the target view - h.public_id as host_id, - 'static host' as host_type, - coalesce(h.name, 'None') as host_name, - coalesce(h.description, 'None') as host_description, - coalesce(h.address, 'Unknown') as host_address, - s.public_id as host_set_id, - 'static host set' as host_set_type, - coalesce(s.name, 'None') as host_set_name, - coalesce(s.description, 'None') as host_set_description, - c.public_id as host_catalog_id, - 'static host catalog' as host_catalog_type, - coalesce(c.name, 'None') as host_catalog_name, - coalesce(c.description, 'None') as host_catalog_description, - t.public_id as target_id, - 'tcp target' as target_type, - coalesce(t.name, 'None') as target_name, - coalesce(t.description, 'None') as target_description, - coalesce(t.default_port, 0) as target_default_port_number, - t.session_max_seconds as target_session_max_seconds, - t.session_connection_limit as target_session_connection_limit, - p.public_id as project_id, - coalesce(p.name, 'None') as project_name, - coalesce(p.description, 'None') as project_description, - o.public_id as organization_id, - coalesce(o.name, 'None') as organization_name, - coalesce(o.description, 'None') as organization_description - from static_host as h, - static_host_catalog as c, - static_host_set_member as m, - static_host_set as s, - target_host_set as ts, - target_tcp as t, - iam_scope as p, - iam_scope as o - where h.catalog_id = c.public_id - and h.public_id = m.host_id - and s.public_id = m.set_id - and t.public_id = ts.target_id - and s.public_id = ts.host_set_id - and p.public_id = t.scope_id - and p.type = 'project' - and o.public_id = p.parent_id - and o.type = 'org' - ; - - -- replaces view from 15/01_wh_rename_key_columns.up.sql - drop view whx_host_dimension_target; - create view whx_host_dimension_target as - select key, - host_id, - host_type, - host_name, - host_description, - host_address, - host_set_id, - host_set_type, - host_set_name, - host_set_description, - host_catalog_id, - host_catalog_type, - host_catalog_name, - host_catalog_description, - target_id, - target_type, - target_name, - target_description, - target_default_port_number, - target_session_max_seconds, - target_session_connection_limit, - project_id, - project_name, - project_description, - organization_id, - organization_name, - organization_description - from wh_host_dimension - where current_row_indicator = 'Current' - ; - - -- replaces function from 15/01_wh_rename_key_columns.up.sql - drop function wh_upsert_host; - create function wh_upsert_host(p_host_id wt_public_id, p_host_set_id wt_public_id, p_target_id wt_public_id) - returns wh_dim_key - as $$ - declare - src whx_host_dimension_target%rowtype; - target whx_host_dimension_target%rowtype; - new_row wh_host_dimension%rowtype; - begin - select * into target - from whx_host_dimension_target as t - where t.host_id = p_host_id - and t.host_set_id = p_host_set_id - and t.target_id = p_target_id; - - select target.key, t.* into src - from whx_host_dimension_source as t - where t.host_id = p_host_id - and t.host_set_id = p_host_set_id - and t.target_id = p_target_id; - - if src is distinct from target then - - -- expire the current row - update wh_host_dimension - set current_row_indicator = 'Expired', - row_expiration_time = current_timestamp - where host_id = p_host_id - and host_set_id = p_host_set_id - and target_id = p_target_id - and current_row_indicator = 'Current'; - - -- insert a new row - insert into wh_host_dimension ( - host_id, host_type, host_name, host_description, host_address, - host_set_id, host_set_type, host_set_name, host_set_description, - host_catalog_id, host_catalog_type, host_catalog_name, host_catalog_description, - target_id, target_type, target_name, target_description, - target_default_port_number, target_session_max_seconds, target_session_connection_limit, - project_id, project_name, project_description, - organization_id, organization_name, organization_description, - current_row_indicator, row_effective_time, row_expiration_time - ) - select host_id, host_type, host_name, host_description, host_address, - host_set_id, host_set_type, host_set_name, host_set_description, - host_catalog_id, host_catalog_type, host_catalog_name, host_catalog_description, - target_id, target_type, target_name, target_description, - target_default_port_number, target_session_max_seconds, target_session_connection_limit, - project_id, project_name, project_description, - organization_id, organization_name, organization_description, - 'Current', current_timestamp, 'infinity'::timestamptz - from whx_host_dimension_source - where host_id = p_host_id - and host_set_id = p_host_set_id - and target_id = p_target_id - returning * into new_row; - - return new_row.key; - end if; - return target.key; - - end; - $$ language plpgsql; -`), - 16001: []byte(` --- replaces check from internal/db/schema/migrations/postgres/0/60_wh_domain_types.up.sql - alter domain wh_public_id drop constraint wh_public_id_check; - alter domain wh_public_id add constraint wh_public_id_check - check( - value = 'None' - or - value = 'Unknown' - or - length(trim(value)) > 10 - ); - - create table wh_credential_dimension ( - -- random id generated using encode(digest(gen_random_bytes(16), 'sha256'), 'base64') - -- this is done to prevent conflicts with rows in other clusters - -- which enables warehouse data from multiple clusters to be loaded into a - -- single database instance - key wh_dim_key primary key default wh_dim_key(), - - credential_purpose wh_dim_text, - credential_library_id wh_public_id not null, - credential_library_type wh_dim_text, - credential_library_name wh_dim_text, - credential_library_description wh_dim_text, - credential_library_vault_path wh_dim_text, - credential_library_vault_http_method wh_dim_text, - credential_library_vault_http_request_body wh_dim_text, - - credential_store_id wh_public_id not null, - credential_store_type wh_dim_text, - credential_store_name wh_dim_text, - credential_store_description wh_dim_text, - credential_store_vault_namespace wh_dim_text, - credential_store_vault_address wh_dim_text, - - target_id wh_public_id not null, - target_type wh_dim_text, - target_name wh_dim_text, - target_description wh_dim_text, - target_default_port_number integer not null, - target_session_max_seconds integer not null, - target_session_connection_limit integer not null, - - project_id wt_scope_id not null, - project_name wh_dim_text, - project_description wh_dim_text, - - organization_id wt_scope_id not null, - organization_name wh_dim_text, - organization_description wh_dim_text, - - current_row_indicator wh_dim_text, - row_effective_time wh_timestamp, - row_expiration_time wh_timestamp - ); - - -- https://www.postgresql.org/docs/current/indexes-partial.html - create unique index wh_credential_dim_current_constraint - on wh_credential_dimension (credential_library_id, credential_store_id, target_id, credential_purpose) - where current_row_indicator = 'Current'; - - -- One part of a bridge table to associated the set of wh_credential_dimension with a fact table. - -- The other part of the bridge is wh_credential_group_membership. - create table wh_credential_group ( - -- random id generated using encode(digest(gen_random_bytes(16), 'sha256'), 'base64') - -- this is done to prevent conflicts with rows in other clusters - -- which enables warehouse data from multiple clusters to be loaded into a - -- single database instance - key wh_dim_key primary key default wh_dim_key() - ); - - -- The second part of the bridge table. The other part is wh_credential_group. - create table wh_credential_group_membership ( - credential_group_key wh_dim_key not null - references wh_credential_group (key) - on delete restrict - on update cascade, - credential_key wh_dim_key not null - references wh_credential_dimension (key) - on delete restrict - on update cascade - ); - - -- Add "no credentials" and "Unknown" group an dimension. - -- When a session has no credentials "no credentials" is used as the "None" value. - -- "Unknown" is used for existing data prior to the credential_dimension existing. - insert into wh_credential_group - (key) - values - ('no credentials'), - ('Unknown'); - insert into wh_credential_dimension ( - key, - credential_purpose, - credential_library_id, credential_library_type, credential_library_name, credential_library_description, credential_library_vault_path, credential_library_vault_http_method, credential_library_vault_http_request_body, - credential_store_id, credential_store_type, credential_store_name, credential_store_description, credential_store_vault_namespace, credential_store_vault_address, - target_id, target_type, target_name, target_description, target_default_port_number, target_session_max_seconds, target_session_connection_limit, - project_id, project_name, project_description, - organization_id, organization_name, organization_description, - current_row_indicator, row_effective_time, row_expiration_time - ) - values - ( - 'no credential', - 'None', - 'None', 'None', 'None', 'None', 'None', 'None', 'None', - 'None', 'None', 'None', 'None', 'None', 'None', - 'None', 'None', 'None', 'None', -1, -1, -1, - '00000000000', 'None', 'None', - '00000000000', 'None', 'None', - 'Current', now(), 'infinity'::timestamptz - ), - ( - 'Unknown', - 'Unknown', - 'Unknown', 'Unknown', 'Unknown', 'Unknown', 'Unknown', 'Unknown', 'Unknown', - 'Unknown', 'Unknown', 'Unknown', 'Unknown', 'Unknown', 'Unknown', - 'Unknown', 'Unknown', 'Unknown', 'Unknown', -1, -1, -1, - '00000000000', 'Unknown', 'Unknown', - '00000000000', 'Unknown', 'Unknown', - 'Current', now(), 'infinity'::timestamptz - ); - insert into wh_credential_group_membership - (credential_group_key, credential_key) - values - ('no credentials', 'no credential'), - ('Unknown', 'Unknown'); -`), - 16002: []byte(` --- The whx_credential_dimension_source and whx_credential_dimension_target views are used - -- by an insert trigger to determine if the current row for the dimension has - -- changed and a new one needs to be inserted. The first column in the target view - -- must be the current warehouse id and all remaining columns must match the columns - -- in the source view. - - -- The whx_credential_dimension_source view shows the current values in the - -- operational tables of the credential dimension. - create view whx_credential_dimension_source as - select -- id is the first column in the target view - s.public_id as session_id, - coalesce(scd.credential_purpose, 'None') as credential_purpose, - cl.public_id as credential_library_id, - case - when vcl is null then 'None' - else 'vault credential library' - end as credential_library_type, - coalesce(vcl.name, 'None') as credential_library_name, - coalesce(vcl.description, 'None') as credential_library_description, - coalesce(vcl.vault_path, 'None') as credential_library_vault_path, - coalesce(vcl.http_method, 'None') as credential_library_vault_http_method, - coalesce(vcl.http_request_body, 'None') as credential_library_vault_http_request_body, - cs.public_id as credential_store_id, - case - when vcs is null then 'None' - else 'vault credential store' - end as credential_store_type, - coalesce(vcs.name, 'None') as credential_store_name, - coalesce(vcs.description, 'None') as credential_store_description, - coalesce(vcs.namespace, 'None') as credential_store_vault_namespace, - coalesce(vcs.vault_address, 'None') as credential_store_vault_address, - t.public_id as target_id, - 'tcp target' as target_type, - coalesce(tt.name, 'None') as target_name, - coalesce(tt.description, 'None') as target_description, - coalesce(tt.default_port, 0) as target_default_port_number, - tt.session_max_seconds as target_session_max_seconds, - tt.session_connection_limit as target_session_connection_limit, - p.public_id as project_id, - coalesce(p.name, 'None') as project_name, - coalesce(p.description, 'None') as project_description, - o.public_id as organization_id, - coalesce(o.name, 'None') as organization_name, - coalesce(o.description, 'None') as organization_description - from session_credential_dynamic as scd, - session as s, - credential_library as cl, - credential_store as cs, - credential_vault_library as vcl, - credential_vault_store as vcs, - target as t, - target_tcp as tt, - iam_scope as p, - iam_scope as o - where scd.library_id = cl.public_id - and cl.store_id = cs.public_id - and vcl.public_id = cl.public_id - and vcs.public_id = cs.public_id - and s.public_id = scd.session_id - and s.target_id = t.public_id - and t.public_id = tt.public_id - and p.public_id = t.scope_id - and p.type = 'project' - and o.public_id = p.parent_id - and o.type = 'org'; - - create view whx_credential_dimension_target as - select key, - credential_purpose, - credential_library_id, - credential_library_type, - credential_library_name, - credential_library_description, - credential_library_vault_path, - credential_library_vault_http_method, - credential_library_vault_http_request_body, - credential_store_id, - credential_store_type, - credential_store_name, - credential_store_description, - credential_store_vault_namespace, - credential_store_vault_address, - target_id, - target_type, - target_name, - target_description, - target_default_port_number, - target_session_max_seconds, - target_session_connection_limit, - project_id, - project_name, - project_description, - organization_id, - organization_name, - organization_description - from wh_credential_dimension - where current_row_indicator = 'Current' - ; -`), - 16003: []byte(` --- wh_upsert_credential_dimension compares the current vaules in the wh_credential_dimension - -- with the current values in the operational tables for the given parameters. IF the values - -- between operational tables and the wh_credential_dimension differ, a new row is inserted in - -- the wh_credential_dimension to match the current values in the operational tables. - create function wh_upsert_credential_dimension(p_session_id wt_public_id, p_library_id wt_public_id, p_credential_purpose wh_dim_text) - returns wh_dim_key - as $$ - declare - src whx_credential_dimension_target%rowtype; - target whx_credential_dimension_target%rowtype; - new_row wh_credential_dimension%rowtype; - t_id wt_public_id; - begin - select s.target_id into strict t_id - from session as s - where s.public_id = p_session_id; - - select * into target - from whx_credential_dimension_target as t - where t.credential_library_id = p_library_id - and t.target_id = t_id - and t.credential_purpose = p_credential_purpose; - - select - target.key, t.credential_purpose, - t.credential_library_id, t.credential_library_type, t.credential_library_name, t.credential_library_description, t.credential_library_vault_path, t.credential_library_vault_http_method, t.credential_library_vault_http_request_body, - t.credential_store_id, t.credential_store_type, t.credential_store_name, t.credential_store_description, t.credential_store_vault_namespace, t.credential_store_vault_address, - t.target_id, t.target_type, t.target_name, t.target_description, t.target_default_port_number, t.target_session_max_seconds, t.target_session_connection_limit, - t.project_id, t.project_name, t.project_description, - t.organization_id, t.organization_name, t.organization_description - into src - from whx_credential_dimension_source as t - where t.credential_library_id = p_library_id - and t.session_id = p_session_id - and t.target_id = t_id - and t.credential_purpose = p_credential_purpose; - - if src is distinct from target then - update wh_credential_dimension - set current_row_indicator = 'Expired', - row_expiration_time = current_timestamp - where credential_library_id = p_library_id - and target_id = t_id - and credential_purpose = p_credential_purpose - and current_row_indicator = 'Current'; - - insert into wh_credential_dimension ( - credential_purpose, - credential_library_id, credential_library_type, credential_library_name, credential_library_description, credential_library_vault_path, credential_library_vault_http_method, credential_library_vault_http_request_body, - credential_store_id, credential_store_type, credential_store_name, credential_store_description, credential_store_vault_namespace, credential_store_vault_address, - target_id, target_type, target_name, target_description, target_default_port_number, target_session_max_seconds, target_session_connection_limit, - project_id, project_name, project_description, - organization_id, organization_name, organization_description, - current_row_indicator, row_effective_time, row_expiration_time - ) - select credential_purpose, - credential_library_id, credential_library_type, credential_library_name, credential_library_description, credential_library_vault_path, credential_library_vault_http_method, credential_library_vault_http_request_body, - credential_store_id, credential_store_type, credential_store_name, credential_store_description, credential_store_vault_namespace, credential_store_vault_address, - target_id, target_type, target_name, target_description, target_default_port_number, target_session_max_seconds, target_session_connection_limit, - project_id, project_name, project_description, - organization_id, organization_name, organization_description, - 'Current', current_timestamp, 'infinity'::timestamptz - from whx_credential_dimension_source - where credential_library_id = p_library_id - and session_id = p_session_id - and target_id = t_id - and credential_purpose = p_credential_purpose - returning * into new_row; - - return new_row.key; - end if; - - return target.key; - end - $$ language plpgsql; - - -- Run wh_upsert_credential_dimension for session_credential_dynamic row that is inserted. - create function wh_insert_session_credential_dynamic() - returns trigger - as $$ - begin - perform wh_upsert_credential_dimension(new.session_id, new.library_id, new.credential_purpose); - return null; - end; - $$ language plpgsql; - create trigger wh_insert_session_credential_dynamic - after insert on session_credential_dynamic - for each row - execute function wh_insert_session_credential_dynamic(); - - -- wh_upsert_credentail_group determines if a new wh_credential_group needs to be - -- created due to changes to the coresponding wh_credential_dimensions. It then - -- updates the wh_session_accumulating_fact to associate it with the correct wh_credential_group. - create function wh_upsert_credentail_group() - returns trigger - as $$ - declare - cg_key wh_dim_key; - t_id wt_public_id; - s_id wt_public_id; - c_key wh_dim_key; - begin - select distinct scd.session_id into strict s_id - from new_table as scd; - - select distinct s.target_id into strict t_id - from new_table as scd - left join session as s on s.public_id = scd.session_id; - - -- based on query written by Michele Gaffney - with - credential_list (key) as ( - select key - from wh_credential_dimension - where target_id = t_id - and credential_library_id in (select credential_library_id from new_table) - ) - select distinct credential_group_key into cg_key - from wh_credential_group_membership a - where a.credential_key in (select key from credential_list) - and (select count(key) from credential_list) = - ( - select count(b.credential_key) - from wh_credential_group_membership b - where a.credential_key = b.credential_key - and b.credential_key in (select key from credential_list) - ) - and not exists - ( - select 1 - from wh_credential_group_membership b - where a.credential_key = b.credential_key - and b.credential_key not in (select key from credential_list) - ) - ; - if cg_key is null then - insert into wh_credential_group default values returning key into cg_key; - for c_key in - select key - from wh_credential_dimension - where target_id = t_id - and credential_library_id in (select credential_library_id from new_table) - loop - insert into wh_credential_group_membership - (credential_group_key, credential_key) - values - (cg_key, c_key); - end loop; - end if; - - update wh_session_connection_accumulating_fact - set credential_group_key = cg_key - where session_id = s_id; - - return null; - end; - $$ language plpgsql; - - -- Run wh_upsert_credentail_group on statement. This assumes that all relevant - -- session_credential_dynamic rows are inserted as a single statement and that - -- the wh_insert_session_credential_dynamic trigger ran for each row and updated - -- the wh_credential_dimensions. Then this statement trigger can run to update the - -- bridge tables and wh_session_accumulating_fact. - create trigger wh_insert_stmt_session_credential_dynamic - after insert on session_credential_dynamic - referencing new table as new_table - for each statement - execute function wh_upsert_credentail_group(); -`), - 16004: []byte(` -alter table wh_session_accumulating_fact - add column credential_group_key wh_dim_key not null - default 'Unknown' - references wh_credential_group (key) - on delete restrict - on update cascade; - alter table wh_session_accumulating_fact - alter column credential_group_key drop default; - - -- replaces function from 15/01_wh_rename_key_columns.up.sql - drop trigger wh_insert_session on session; - drop function wh_insert_session; - create function wh_insert_session() - returns trigger - as $$ - declare - new_row wh_session_accumulating_fact%rowtype; - begin - with - pending_timestamp (date_dim_key, time_dim_key, ts) as ( - select wh_date_key(start_time), wh_time_key(start_time), start_time - from session_state - where session_id = new.public_id - and state = 'pending' - ) - insert into wh_session_accumulating_fact ( - session_id, - auth_token_id, - host_key, - user_key, - credential_group_key, - session_pending_date_key, - session_pending_time_key, - session_pending_time - ) - select new.public_id, - new.auth_token_id, - wh_upsert_host(new.host_id, new.host_set_id, new.target_id), - wh_upsert_user(new.user_id, new.auth_token_id), - 'no credentials', -- will be updated by wh_upsert_credentail_group - pending_timestamp.date_dim_key, - pending_timestamp.time_dim_key, - pending_timestamp.ts - from pending_timestamp - returning * into strict new_row; - return null; - end; - $$ language plpgsql; - create trigger wh_insert_session - after insert on session - for each row - execute function wh_insert_session(); -`), - 16005: []byte(` -alter table wh_session_connection_accumulating_fact - add column credential_group_key wh_dim_key not null - default 'Unknown' - references wh_credential_group (key) - on delete restrict - on update cascade; - alter table wh_session_connection_accumulating_fact - alter column credential_group_key drop default; - - -- replaces function from 15/01_wh_rename_key_columns.up.sql - drop trigger wh_insert_session_connection on session_connection; - drop function wh_insert_session_connection; - create function wh_insert_session_connection() - returns trigger - as $$ - declare - new_row wh_session_connection_accumulating_fact%rowtype; - begin - with - authorized_timestamp (date_dim_key, time_dim_key, ts) as ( - select wh_date_key(start_time), wh_time_key(start_time), start_time - from session_connection_state - where connection_id = new.public_id - and state = 'authorized' - ), - session_dimension (host_dim_key, user_dim_key, credential_group_dim_key) as ( - select host_key, user_key, credential_group_key - from wh_session_accumulating_fact - where session_id = new.session_id - ) - insert into wh_session_connection_accumulating_fact ( - connection_id, - session_id, - host_key, - user_key, - credential_group_key, - connection_authorized_date_key, - connection_authorized_time_key, - connection_authorized_time, - client_tcp_address, - client_tcp_port_number, - endpoint_tcp_address, - endpoint_tcp_port_number, - bytes_up, - bytes_down - ) - select new.public_id, - new.session_id, - session_dimension.host_dim_key, - session_dimension.user_dim_key, - session_dimension.credential_group_dim_key, - authorized_timestamp.date_dim_key, - authorized_timestamp.time_dim_key, - authorized_timestamp.ts, - new.client_tcp_address, - new.client_tcp_port, - new.endpoint_tcp_address, - new.endpoint_tcp_port, - new.bytes_up, - new.bytes_down - from authorized_timestamp, - session_dimension - returning * into strict new_row; - perform wh_rollup_connections(new.session_id); - return null; - end; - $$ language plpgsql; - - create trigger wh_insert_session_connection - after insert on session_connection - for each row - execute function wh_insert_session_connection(); -`), - 2001: []byte(` --- log_migration entries represent logs generated during migrations -create table log_migration( - id bigint generated always as identity primary key, - migration_version bigint not null, -- cannot declare FK since the table is truncated during runtime - create_time wt_timestamp, - entry text not null -); -comment on table log_migration is -'log_migration entries are logging output from databaes migrations'; - --- log_migration triggers -create trigger - default_create_time_column -before -insert on log_migration - for each row execute procedure default_create_time(); - -create trigger - immutable_columns -before -update on log_migration - for each row execute procedure immutable_columns('id', 'migration_version', 'create_time', 'entry'); - - --- log_migration_version() defines a function to be used in a "before update" --- trigger for log_migrations entries. Its intent: set the log_migration --- version column to the current migration version. -create or replace function - log_migration_version() - returns trigger -as $$ - declare current_version bigint; - begin - select max(version) from boundary_schema_version into current_version; - new.migration_version = current_version; - return new; - end; -$$ language plpgsql; -comment on function log_migration_version() is -'log_migration_version will set the log_migration entries to the current migration version'; - -create trigger - migration_version_column -before -insert on log_migration - for each row execute procedure log_migration_version(); -`), - 2003: []byte(` --- auth_oidc_method_state_enum entries define the possible oidc auth method --- states. -create table auth_oidc_method_state_enm ( - name text primary key - constraint name_only_predefined_oidc_method_states_allowed - check ( - name in ('inactive', 'active-private', 'active-public') - ) -); - --- populate the values of auth_oidc_method_state_enm -insert into auth_oidc_method_state_enm(name) - values - ('inactive'), - ('active-private'), - ('active-public'); - - -- define the immutable fields for auth_oidc_method_state_enm (all of them) -create trigger - immutable_columns -before -update on auth_oidc_method_state_enm - for each row execute procedure immutable_columns('name'); - - --- auth_oidc_signing_alg entries define the supported oidc auth method --- signing algorithms. -create table auth_oidc_signing_alg_enm ( - name text primary key - constraint only_predefined_auth_oidc_signing_algs_allowed - check ( - name in ( - 'RS256', - 'RS384', - 'RS512', - 'ES256', - 'ES384', - 'ES512', - 'PS256', - 'PS384', - 'PS512', - 'EdDSA') - ) -); - --- populate the values of auth_oidc_signing_alg -insert into auth_oidc_signing_alg_enm (name) - values - ('RS256'), - ('RS384'), - ('RS512'), - ('ES256'), - ('ES384'), - ('ES512'), - ('PS256'), - ('PS384'), - ('PS512'), - ('EdDSA') - ; - - -- define the immutable fields for auth_oidc_signing_alg (all of them) -create trigger - immutable_columns -before -update on auth_oidc_signing_alg_enm - for each row execute procedure immutable_columns('name'); -`), - 2004: []byte(` --- auth_oidc_method entries are the current oidc auth methods configured for --- existing scopes. -create table auth_oidc_method ( - public_id wt_public_id - primary key, - scope_id wt_scope_id - not null, - name wt_name, - description wt_description, - create_time wt_timestamp, - update_time wt_timestamp, - version wt_version, - state text not null - constraint auth_oidc_method_state_enm_fkey - references auth_oidc_method_state_enm(name) - on delete restrict - on update cascade, - disable_discovered_config_validation bool not null default false, - api_url wt_url, -- an address prefix at which the boundary api is reachable. - issuer wt_url, - client_id text -- oidc client identifier issued by the oidc provider. - constraint client_id_not_empty - check(length(trim(client_id)) > 0), - client_secret bytea, -- encrypted oidc client secret issued by the oidc provider. - client_secret_hmac text - constraint client_secret_hmac_not_empty - check(length(trim(client_secret_hmac)) > 0), - key_id wt_private_id not null -- key used to encrypt entries via wrapping wrapper. - constraint kms_database_key_version_fkey - references kms_database_key_version(private_id) - on delete restrict - on update cascade, - constraint key_id_not_empty - check(length(trim(key_id)) > 0), - max_age int -- the allowable elapsed time in secs since the last time the user was authenticated. A value -1 basically forces the IdP to re-authenticate the End-User. Zero is not a valid value. - constraint max_age_not_equal_zero - check(max_age != 0) - constraint max_age_not_less_then_negative_one - check(max_age >= -1), - constraint auth_method_fkey - foreign key (scope_id, public_id) - references auth_method (scope_id, public_id) - on delete cascade - on update cascade, - constraint auth_oidc_method_scope_id_name_uq - unique(scope_id, name), - constraint auth_oidc_method_scope_id_public_id_uq - unique(scope_id, public_id), - constraint auth_oidc_method_scope_id_issuer_client_id_unique - unique(scope_id, issuer, client_id) -- a client_id must be unique for a provider within a scope. -); -comment on table auth_oidc_method is -'auth_oidc_method entries are the current oidc auth methods configured for existing scopes.'; - --- auth_oidc_signing_alg entries are the signing algorithms allowed for an oidc --- auth method. There must be at least one allowed alg for each oidc auth method. -create table auth_oidc_signing_alg ( - create_time wt_timestamp, - oidc_method_id wt_public_id - constraint auth_oidc_method_fkey - references auth_oidc_method(public_id) - on delete cascade - on update cascade, - signing_alg_name text - constraint auth_oidc_signing_alg_enm_fkey - references auth_oidc_signing_alg_enm(name) - on delete restrict - on update cascade, - primary key(oidc_method_id, signing_alg_name) -); -comment on table auth_oidc_signing_alg is -'auth_oidc_signing_alg entries are the signing algorithms allowed for an oidc auth method. There must be at least one allowed alg for each oidc auth method'; - --- auth_oidc_aud_claim entries are the audience claims for a specific oidc auth --- method. There can be 0 or more for each parent oidc auth method. If an auth --- method has any aud claims, an ID token must contain one of them to be valid. -create table auth_oidc_aud_claim ( - create_time wt_timestamp, - oidc_method_id wt_public_id - constraint auth_oidc_method_fkey - references auth_oidc_method(public_id) - on delete cascade - on update cascade, - aud_claim text not null - constraint aud_claim_must_not_be_empty - check(length(trim(aud_claim)) > 0) - constraint aud_claim_must_be_less_than_1024_chars - check(length(trim(aud_claim)) < 1024), - primary key(oidc_method_id, aud_claim) -); -comment on table auth_oidc_aud_claim is -'auth_oidc_aud_claim entries are the audience claims for a specific oidc auth method. There can be 0 or more for each parent oidc auth method. If an auth method has any aud claims, an ID token must contain one of them to be valid.'; - - --- auth_oidc_certificate entries are optional PEM encoded x509 certificates. --- Each entry is a single certificate. An oidc auth method may have 0 or more --- of these optional x509s. If an auth method has any cert entries, they are --- used as trust anchors when connecting to the auth method's oidc provider --- (instead of the host system's cert chain). -create table auth_oidc_certificate ( - create_time wt_timestamp, - oidc_method_id wt_public_id - constraint auth_oidc_method_fkey - references auth_oidc_method(public_id) - on delete cascade - on update cascade, - certificate bytea not null, - primary key(oidc_method_id, certificate) -); -comment on table auth_oidc_certificate is -'auth_oidc_certificate entries are optional PEM encoded x509 certificates. Each entry is a single certificate. An oidc auth method may have 0 or more of these optional x509s. If an auth method has any cert entries, they are used as trust anchors when connecting to the auth methods oidc provider (instead of the host system cert chain)'; - - --- auth_oidc_account entries are subtypes of auth_account and represent an --- oidc account. -create table auth_oidc_account ( - public_id wt_public_id - primary key, - auth_method_id wt_public_id - not null, - -- NOTE(mgaffney): The scope_id type is not wt_scope_id because the domain - -- check is executed before the insert trigger which retrieves the scope_id - -- causing an insert to fail. - scope_id text not null, - name wt_name, - description wt_description, - create_time wt_timestamp, - update_time wt_timestamp, - version wt_version, - issuer wt_url not null, -- case-sensitive URL that maps to an id_token's iss claim, - subject text not null -- case-sensitive string that maps to an id_token's sub claim - constraint subject_must_not_be_empty - check ( - length(trim(subject)) > 0 - ) - constraint subject_must_be_less_than_256_chars - check( - length(trim(subject)) <= 255 -- length limit per OIDC spec - ), - full_name wt_full_name, -- may be null and maps to an id_token's name claim - email wt_email, -- may be null and maps to the id_token's email claim - constraint auth_oidc_method_fkey - foreign key (scope_id, auth_method_id) - references auth_oidc_method (scope_id, public_id) - on delete cascade - on update cascade, - constraint auth_account_fkey - foreign key (scope_id, auth_method_id, public_id) - references auth_account (scope_id, auth_method_id, public_id) - on delete cascade - on update cascade, - constraint auth_oidc_account_auth_method_id_name_uq - unique(auth_method_id, name), - -- ############################################################### - -- any change to this constraints name must be aligned with the - -- acctUpsertQuery const in internal/auth/oidc/query.go - -- ############################################################### - constraint auth_oidc_account_auth_method_id_issuer_subject_uq - unique(auth_method_id, issuer, subject), -- subject must be unique for a provider within specific auth method - constraint auth_oidc_account_auth_method_id_public_id_uq - unique(auth_method_id, public_id) -); -comment on table auth_oidc_method is -'auth_oidc_account entries are subtypes of auth_account and represent an oidc account.'; - --- auth_oidc_method column triggers -create trigger - insert_auth_method_subtype -before insert on auth_oidc_method - for each row execute procedure insert_auth_method_subtype(); - -create trigger - update_time_column -before -update on auth_oidc_method - for each row execute procedure update_time_column(); - -create trigger - immutable_columns -before -update on auth_oidc_method - for each row execute procedure immutable_columns('public_id', 'scope_id', 'create_time'); - -create trigger - default_create_time_column -before -insert on auth_oidc_method - for each row execute procedure default_create_time(); - -create trigger - update_version_column -after update on auth_oidc_method - for each row execute procedure update_version_column(); - --- active_auth_oidc_method_must_be_complete() defines a function to be used in --- a "before update" trigger for auth_oidc_method entries. Its intent: prevent --- incomplete oidc methods from transitioning out of the "inactive" state. -create or replace function - active_auth_oidc_method_must_be_complete() - returns trigger -as $$ - begin - -- validate signing alg - if old.state = 'inactive' and new.state != 'inactive' then - perform - from - auth_oidc_method am - join auth_oidc_signing_alg alg on am.public_id = alg.oidc_method_id - where - new.public_id = am.public_id; - if not found then - raise exception 'an incomplete oidc auth method must remain inactive'; - end if; - -- validate issuer - case - when new.issuer != old.issuer then - if length(trim(new.issuer)) = 0 then - raise exception 'empty issuer: an incomplete oidc auth method must remain inactive'; - end if; - when new.issuer = old.issuer then - if length(trim(old.issuer)) = 0 then - raise exception 'empty issuer: an incomplete oidc auth method must remain inactive'; - end if; - else - end case; - -- validate client_id - case - when new.client_id != old.client_id then - if length(trim(new.client_id)) = 0 then - raise exception 'empty client_id: an incomplete oidc auth method must remain inactive'; - end if; - when new.client_id = old.client_id then - if length(trim(old.client_id)) = 0 then - raise exception 'empty client_id: an incomplete oidc auth method must remain inactive'; - end if; - else - end case; - -- validate client_secret - case - when new.client_secret != old.client_secret then - if length(new.client_secret) = 0 then - raise exception 'empty client_secret: an incomplete oidc auth method must remain inactive'; - end if; - when new.client_secret = old.client_secret then - if length(old.client_secret) = 0 then - raise exception 'empty client_secret: an incomplete oidc auth method must remain inactive'; - end if; - else - end case; - - - end if; - return new; - end; -$$ language plpgsql; -comment on function active_auth_oidc_method_must_be_complete() is -'active_auth_oidc_method_must_be_complete() will raise an error if the oidc auth method is not complete'; - -create trigger - update_active_auth_oidc_method_must_be_complete -before -update on auth_oidc_method - for each row execute procedure active_auth_oidc_method_must_be_complete(); - --- new_auth_oidc_method_must_be_inactive() defines a function to be used in --- a "before insert" trigger for auth_oidc_method entries. Its intent: --- only allow "inactive" auth methods to be inserted. Why? there's no way --- you can insert an entry that's anything but incomplete, since we have a --- chicken/egg problem: you need the auth method id to create the required --- signing algs value objects. -create or replace function - new_auth_oidc_method_must_be_inactive() - returns trigger -as $$ - begin - if new.state != 'inactive' then - raise exception 'an incomplete oidc method must be inactive'; - end if; - end; -$$ language plpgsql; -comment on function new_auth_oidc_method_must_be_inactive() is -'new_auth_oidc_method_must_be_inactive ensures that new incomplete oidc auth methods must remain inactive'; - -create trigger - new_auth_oidc_method_must_be_inactive -before -insert on auth_oidc_method - for each row execute procedure active_auth_oidc_method_must_be_complete(); - --- auth_oidc_account column triggers -create trigger - update_time_column -before -update on auth_oidc_account - for each row execute procedure update_time_column(); - -create trigger - immutable_columns -before -update on auth_oidc_account - for each row execute procedure immutable_columns('public_id', 'auth_method_id', 'scope_id', 'create_time', 'issuer', 'subject'); - -create trigger - default_create_time_column -before -insert on auth_oidc_account - for each row execute procedure default_create_time(); - -create trigger - update_version_column -after update on auth_oidc_account - for each row execute procedure update_version_column(); - - --- insert_auth_oidc_account_subtype is intended as a before insert --- trigger on auth_oidc_account. Its purpose is to insert a base --- auth_account for new oidc accounts. It's a bit different than the --- standard trigger for this, because it will have conflicting PKs --- and we just want to "do nothing" on those conflicts, deferring the --- raising on an error to insert into the auth_oidc_account table. --- this is all necessary because of we're using predictable public ids --- for oidc accounts. -create or replace function - insert_auth_oidc_account_subtype() - returns trigger -as $$ -begin - select auth_method.scope_id - into new.scope_id - from auth_method - where auth_method.public_id = new.auth_method_id; - - insert into auth_account - (public_id, auth_method_id, scope_id) - values - (new.public_id, new.auth_method_id, new.scope_id) - on conflict do nothing; - - return new; -end; - $$ language plpgsql; - -create trigger - insert_auth_oidc_account_subtype -before insert on auth_oidc_account - for each row execute procedure insert_auth_oidc_account_subtype(); - --- triggers for auth_oidc_method children tables: auth_oidc_aud_claim, --- auth_oidc_certificate, auth_oidc_signing_alg - - --- on_delete_active_auth_oidc_method_must_be_complete() defines a function --- to be used in an "after delete" trigger for auth_oidc_signing_alg --- Its intent: prevent deletes that would result in an "active" oidc --- auth method which is incomplete. -create or replace function - on_delete_active_auth_oidc_method_must_be_complete() - returns trigger -as $$ -declare am_state text; -declare alg_cnt int; - begin - select - am.state, - count(alg.oidc_method_id) as alg_cnt - from - auth_oidc_method am - left outer join auth_oidc_signing_alg alg on am.public_id = alg.oidc_method_id - where - new.oidc_method_id = am.public_id - group by am.public_id - into am_state, alg_cnt; - - if not found then - return new; -- auth method was deleted, so we're done - end if; - - if am_state != inactive then - case - when alg_cnt = 0 then - raise exception 'delete would have resulted in an incomplete active oidc auth method with no signing algorithms'; - end case; - end if; - - return new; - end; -$$ language plpgsql; -comment on function on_delete_active_auth_oidc_method_must_be_complete() is -'on_delete_active_auth_oidc_method_must_be_complete() will raise an error if the oidc auth method is not complete after a delete on algs'; - -create trigger - default_create_time_column -before -insert on auth_oidc_aud_claim - for each row execute procedure default_create_time(); - -create trigger - default_create_time_column -before -insert on auth_oidc_certificate - for each row execute procedure default_create_time(); - -create trigger - default_create_time_column -before -insert on auth_oidc_signing_alg - for each row execute procedure default_create_time(); - -create trigger - on_delete_active_auth_oidc_method_must_be_complete -after -delete on auth_oidc_signing_alg - for each row execute procedure on_delete_active_auth_oidc_method_must_be_complete(); - -insert into oplog_ticket (name, version) -values - ('auth_oidc_method', 1), -- auth method is the root aggregate itself and all of its value objects. - ('auth_oidc_account', 1); -`), - 2005: []byte(` --- auth_token_status_enm entries define the possible auth token --- states. -create table auth_token_status_enm ( - name text primary key - constraint name_only_predefined_auth_token_states_allowed - check ( - name in ('auth token pending','token issued', 'authentication failed', 'system error') - ) -); - --- populate the values of auth_token_status_enm -insert into auth_token_status_enm(name) - values - ('auth token pending'), - ('token issued'), - ('authentication failed'), - ('system error'); - - --- add the state column with a default to the auth_token table. -alter table auth_token -add column status text -not null -default 'token issued' -- safest default -references auth_token_status_enm(name) - on update cascade - on delete restrict; - - -create or replace view auth_token_account as - select at.public_id, - at.token, - at.auth_account_id, - at.create_time, - at.update_time, - at.approximate_last_access_time, - at.expiration_time, - aa.scope_id, - aa.iam_user_id, - aa.auth_method_id, - at.status - from auth_token as at - inner join auth_account as aa - on at.auth_account_id = aa.public_id; -`), - 2006: []byte(` --- add the primary_auth_method_id which determines which auth_method is --- designated as for "account info" in the user's scope. It also determines --- which auth method is allowed to auto viviify users. -alter table iam_scope -add column primary_auth_method_id wt_public_id -- allowed to be null and is mutable of course. -constraint auth_method_fkey -references auth_method(public_id) - on update cascade - on delete set null; - --- establish a compond fk, but there's no cascading of deletes or updates, since --- we only want to cascade changes to the primary_auth_method_id portion of --- the compond fk and that is handled in a separate fk declaration. -alter table iam_scope -add constraint auth_method - foreign key (public_id, primary_auth_method_id) - references auth_method(scope_id, public_id); - --- iam_user_acct_info provides account info for users by determining which --- auth_method is designated as for "account info" in the user's scope via the --- scope's primary_auth_method_id. Every sub-type of auth_account must be --- added to this view's union. -create view iam_acct_info as -select - aa.iam_user_id, - oa.subject as login_name, - oa.full_name as full_name, - oa.email as email -from - iam_scope s, - auth_account aa, - auth_oidc_account oa -where - aa.public_id = oa.public_id and - aa.auth_method_id = s.primary_auth_method_id -union -select - aa.iam_user_id, - pa.login_name, - '' as full_name, - '' as email -from - iam_scope s, - auth_account aa, - auth_password_account pa -where - aa.public_id = pa.public_id and - aa.auth_method_id = s.primary_auth_method_id; - --- iam_user_acct_info provides a simple way to retrieve entries that include --- both the iam_user fields with an outer join to the user's account info. -create view iam_user_acct_info as -select - u.public_id, - u.scope_id, - u.name, - u.description, - u.create_time, - u.update_time, - u.version, - i.login_name, - i.full_name, - i.email -from - iam_user u -left outer join iam_acct_info i on u.public_id = i.iam_user_id; -`), - 2007: []byte(` --- the intent of this update statement: set the primary auth method for scopes --- that only have a single auth_password_method, since currently there are only --- auth_password_methods in boundary. Before this release all --- auth_password_methods were "basically" primary auth methods and would create --- an iam_user on first login. -with single_authmethod (scope_id, public_id) as ( - select - am.scope_id, - am.public_id - from - auth_password_method am, - (select - scope_id, - count(public_id) as cnt - from - auth_password_method - group by scope_id) as singles - where - am.scope_id = singles.scope_id and - singles.cnt = 1 -) -update - iam_scope -set - primary_auth_method_id = p.public_id -from - single_authmethod p -where p.scope_id = iam_scope.public_id; - - --- the intent of the insert with select statement: log the scopes that have more --- than 1 auth method and therefore cannot have their primary auth method --- automatically set for them. -with many_authmethod (scope_id, authmethod_cnt) as ( - select - am.scope_id, - many.cnt - from - auth_password_method am, - (select - scope_id, - count(public_id) as cnt - from - auth_password_method - group by scope_id) as many - where - am.scope_id = many.scope_id and - many.cnt > 1 -) -insert into log_migration(entry) -select - distinct concat( - 'unable to set primary_auth_method for ', - public_id, - ' there were ', - m.authmethod_cnt, - ' password auth methods for that scope.' - ) as entry -from - iam_scope s, - many_authmethod m -where - s.primary_auth_method_id is null and - s.public_id = m.scope_id; -`), - 2010: []byte(` --- By adding the name column to the base auth method type, the database can --- ensure that auth method names are unique across all sub types. -alter table auth_method - add column name wt_name; - -alter table auth_method - add constraint auth_method_scope_id_name_uq - unique (scope_id, name); - - --- the intent of this statement is to update the base type's name with the --- existing password auth method names. -update auth_method -set name = pw.name -from - auth_password_method pw -where - auth_method.public_id = pw.public_id and - pw.name is not null; - --- insert_auth_method_subtype() is a replacement of the function definition in --- migration 07_auth.up.sql This new definition also inserts the sub type's name --- into the base type. The name column must be on the base type, so the database --- can ensure that auth method names are unique across all sub types. -create or replace function - insert_auth_method_subtype() - returns trigger -as $$ -begin - insert into auth_method - (public_id, scope_id, name) - values - (new.public_id, new.scope_id, new.name); - return new; -end; -$$ language plpgsql; -comment on function insert_auth_method_subtype() is -'insert_auth_method_subtype() inserts sub type name into the base type auth method table'; - --- update_auth_method_subtype() is a new function intended to be used in "before --- update" triggers for all auth method sub types. It's purpose is to ensure --- that the name column is syncronized between the sub and base auth method --- types. The name column must be on the base type, so the database can ensure --- that auth method names are unique across all sub types. -create or replace function - update_auth_method_subtype() - returns trigger -as $$ -begin - update auth_method set name = new.name where public_id = new.public_id and new.name != name; - return new; -end; -$$ language plpgsql; -comment on function update_auth_method_subtype() is -'update_auth_method_subtype() will update base auth method type name column with new values from sub type'; - -create trigger - update_auth_method_subtype -before update on auth_oidc_method - for each row execute procedure update_auth_method_subtype(); - - -create trigger - update_auth_method_subtype -before update on auth_password_method - for each row execute procedure update_auth_method_subtype(); - - --- delete_auth_method_subtype() is an after trigger function for subytypes of --- auth_method -create or replace function - delete_auth_method_subtype() - returns trigger -as $$ -begin - delete from auth_method - where public_id = old.public_id; - return null; -- results are ignore since this is an after trigger. -end; -$$ language plpgsql; -comment on function delete_auth_method_subtype is -'delete_auth_method_subtype() is an after trigger function for subytypes of auth_method'; - -create trigger - delete_auth_method_subtype -after delete on auth_oidc_method - for each row execute procedure delete_auth_method_subtype(); - - -create trigger - delete_auth_method_subtype -after delete on auth_password_method - for each row execute procedure delete_auth_method_subtype(); -`), - 2015: []byte(` --- oidc_auth_method_with_value_obj is useful for reading an oidc auth method --- with its associated value objects (algs, auds, certs) as columns --- with | delimited values. The use of the postgres string_agg(...) to --- aggregate the value objects into a column works because we are only pulling --- in one column from the associated tables and that value is part of the --- primary key and unique. This view will make things like recursive listing of --- oidc auth methods fairly straightforward to implement for the oidc repo. --- The view also includes an is_primary_auth_method bool -create view oidc_auth_method_with_value_obj as -select - case when s.primary_auth_method_id is not null then - true - else false end - as is_primary_auth_method, - am.public_id, - am.scope_id, - am.name, - am.description, - am.create_time, - am.update_time, - am.version, - am.state, - am.api_url, - am.disable_discovered_config_validation, - am.issuer, - am.client_id, - am.client_secret, - am.client_secret_hmac, - am.key_id, - am.max_age, - -- the string_agg(..) column will be null if there are no associated value objects - string_agg(distinct alg.signing_alg_name, '|') as algs, - string_agg(distinct aud.aud_claim, '|') as auds, - string_agg(distinct cert.certificate, '|') as certs -from - auth_oidc_method am - left outer join iam_scope s on am.public_id = s.primary_auth_method_id - left outer join auth_oidc_signing_alg alg on am.public_id = alg.oidc_method_id - left outer join auth_oidc_aud_claim aud on am.public_id = aud.oidc_method_id - left outer join auth_oidc_certificate cert on am.public_id = cert.oidc_method_id -group by am.public_id, is_primary_auth_method; -- there can be only one public_id + is_primary_auth_method, so group by isn't a problem. -comment on view oidc_auth_method_with_value_obj is -'oidc auth method with its associated value objects (algs, auds, certs) as columns with | delimited values'; -`), - 2020: []byte(` --- auth_password_method_with_is_primary is useful for reading a password auth --- method with a bool to determine if it's the scope's primary auth method. -create view auth_password_method_with_is_primary as -select - case when s.primary_auth_method_id is not null then - true - else false end - as is_primary_auth_method, - am.public_id, - am.scope_id, - am.password_conf_id, - am.name, - am.description, - am.create_time, - am.update_time, - am.version, - am.min_login_name_length, - am.min_password_length -from - auth_password_method am - left outer join iam_scope s on am.public_id = s.primary_auth_method_id; -comment on view auth_password_method_with_is_primary is -'password auth method with an is_primary_auth_method bool'; -`), - 3001: []byte(` --- this constraint is intended to ensure that a user cannot have more than one --- account per auth_method. --- --- If this constraint causes the migrations to fail the operartor can run the --- following query to get a list of user ids which have more that one account --- within the same auth method. At that point, the operator will need to pick --- which account to keep. ----------------------------------------------------------------------------- --- with too_many_accounts(iam_user_id, acct_count) as ( --- select --- iam_user_id, --- count(*) as acct_count --- from --- auth_account --- group by auth_method_id, iam_user_id --- ) --- select --- * --- from --- too_many_accounts --- where --- acct_count > 1; -alter table auth_account - add constraint auth_account_auth_method_id_public_id_uq - unique(auth_method_id, iam_user_id); -`), - 3002: []byte(` -drop view iam_user_acct_info; -drop view iam_acct_info; - --- iam_user_acct_info provides account info for users by determining which --- auth_method is designated as for "account info" in the user's scope via the --- scope's primary_auth_method_id. Every sub-type of auth_account must be --- added to this view's union. -create view iam_acct_info as -select - aa.iam_user_id, - oa.subject as login_name, - oa.public_id as primary_account_id, - oa.full_name as full_name, - oa.email as email -from - iam_scope s, - auth_account aa, - auth_oidc_account oa -where - aa.public_id = oa.public_id and - aa.auth_method_id = s.primary_auth_method_id -union -select - aa.iam_user_id, - pa.public_id as primary_account_id, - pa.login_name, - '' as full_name, - '' as email -from - iam_scope s, - auth_account aa, - auth_password_account pa -where - aa.public_id = pa.public_id and - aa.auth_method_id = s.primary_auth_method_id; - - --- iam_user_acct_info provides a simple way to retrieve entries that include --- both the iam_user fields with an outer join to the user's account info. -create view iam_user_acct_info as -select - u.public_id, - u.scope_id, - u.name, - u.description, - u.create_time, - u.update_time, - u.version, - i.primary_account_id, - i.login_name, - i.full_name, - i.email -from - iam_user u -left outer join iam_acct_info i on u.public_id = i.iam_user_id; -`), - 4001: []byte(` --- fix ordering of fields in iam_acct_info for auth_password_account select --- portion of union. requires recreating both views because of deps. - -drop view iam_user_acct_info; -drop view iam_acct_info; - -create view iam_acct_info as -select - aa.iam_user_id, - oa.subject as login_name, - oa.public_id as primary_account_id, - oa.full_name as full_name, - oa.email as email -from - iam_scope s, - auth_account aa, - auth_oidc_account oa -where - aa.public_id = oa.public_id and - aa.auth_method_id = s.primary_auth_method_id -union -select - aa.iam_user_id, - pa.login_name, - pa.public_id as primary_account_id, - '' as full_name, - '' as email -from - iam_scope s, - auth_account aa, - auth_password_account pa -where - aa.public_id = pa.public_id and - aa.auth_method_id = s.primary_auth_method_id; - - -create view iam_user_acct_info as -select - u.public_id, - u.scope_id, - u.name, - u.description, - u.create_time, - u.update_time, - u.version, - i.primary_account_id, - i.login_name, - i.full_name, - i.email -from - iam_user u -left outer join iam_acct_info i on u.public_id = i.iam_user_id; -`), - 5001: []byte(` --- auth_oidc_scope entries are the optional scopes for a specific oidc auth --- method. There can be 0 or more for each parent oidc auth method. If an auth --- method has any scopes, they will be added to provider requests along with the --- default of "openid". -create table auth_oidc_scope ( - create_time wt_timestamp, - oidc_method_id wt_public_id - constraint auth_oidc_method_fkey - references auth_oidc_method(public_id) - on delete cascade - on update cascade, - scope text not null - constraint scope_must_not_be_empty - check(length(trim(scope)) > 0) - constraint scope_must_be_less_than_1024_chars - check(length(trim(scope)) < 1024) - constraint scope_must_not_be_openid -- the default scope is not allowed, since it's redundant - check(lower(trim(scope)) != 'openid'), - primary key(oidc_method_id, scope) -); -comment on table auth_oidc_scope is -'auth_oidc_scope entries are the optional scopes for a specific oidc auth method. There can be 0 or more for each parent oidc auth method. If an auth method has any scopes, they will be added to provider requests along with the openid default.'; - - -create trigger - default_create_time_column -before -insert on auth_oidc_scope - for each row execute procedure default_create_time(); - - --- we will drop the oidc_auth_method_with_value_obj view, so we can recreate it --- and add the oidc claim's scopes to the returned set. -drop view oidc_auth_method_with_value_obj; - --- oidc_auth_method_with_value_obj is useful for reading an oidc auth method --- with its associated value objects (algs, auds, certs) as columns --- with | delimited values. The use of the postgres string_agg(...) to --- aggregate the value objects into a column works because we are only pulling --- in one column from the associated tables and that value is part of the --- primary key and unique. This view will make things like recursive listing of --- oidc auth methods fairly straightforward to implement for the oidc repo. --- The view also includes an is_primary_auth_method bool -create view oidc_auth_method_with_value_obj as -select - case when s.primary_auth_method_id is not null then - true - else false end - as is_primary_auth_method, - am.public_id, - am.scope_id, - am.name, - am.description, - am.create_time, - am.update_time, - am.version, - am.state, - am.api_url, - am.disable_discovered_config_validation, - am.issuer, - am.client_id, - am.client_secret, - am.client_secret_hmac, - am.key_id, - am.max_age, - -- the string_agg(..) column will be null if there are no associated value objects - string_agg(distinct alg.signing_alg_name, '|') as algs, - string_agg(distinct aud.aud_claim, '|') as auds, - string_agg(distinct cert.certificate, '|') as certs, - string_agg(distinct cs.scope, '|') as claims_scopes -from - auth_oidc_method am - left outer join iam_scope s on am.public_id = s.primary_auth_method_id - left outer join auth_oidc_signing_alg alg on am.public_id = alg.oidc_method_id - left outer join auth_oidc_aud_claim aud on am.public_id = aud.oidc_method_id - left outer join auth_oidc_certificate cert on am.public_id = cert.oidc_method_id - left outer join auth_oidc_scope cs on am.public_id = cs.oidc_method_id -group by am.public_id, is_primary_auth_method; -- there can be only one public_id + is_primary_auth_method, so group by isn't a problem. -comment on view oidc_auth_method_with_value_obj is -'oidc auth method with its associated value objects (algs, auds, certs, scopes) as columns with | delimited values'; -`), - 6001: []byte(` --- auth_oidc_account_claim_map entries are the optional claim maps from custom --- claims to the standard claims of sub, name and email. There can be 0 or more --- for each parent oidc auth method. -create table auth_oidc_account_claim_map ( - create_time wt_timestamp, - oidc_method_id wt_public_id - constraint auth_oidc_method_fkey - references auth_oidc_method(public_id) - on delete cascade - on update cascade, - from_claim text not null - constraint from_claim_must_not_be_empty - check(length(trim(from_claim)) > 0) - constraint from_claim_must_be_less_than_1024_chars - check(length(trim(from_claim)) < 1024), - to_claim text not null - constraint to_claim_valid_values - check (to_claim in ('sub', 'name', 'email')), -- intentionally case-sensitive matching - primary key(oidc_method_id, to_claim) -); -comment on table auth_oidc_account_claim_map is -'auth_oidc_account_claim_map entries are the optional claim maps from custom claims to the standard claims of sub, name and email. There can be 0 or more for each parent oidc auth method.'; - - -create trigger - default_create_time_column -before -insert on auth_oidc_account_claim_map - for each row execute procedure default_create_time(); - - -create trigger - immutable_columns -before -update on auth_oidc_account_claim_map - for each row execute procedure immutable_columns('oidc_method_id', 'from_claim', 'to_claim', 'create_time'); - - --- we will drop the oidc_auth_method_with_value_obj view, so we can recreate it --- and add the oidc claim's scopes to the returned set. -drop view oidc_auth_method_with_value_obj; - --- oidc_auth_method_with_value_obj is useful for reading an oidc auth method --- with its associated value objects (algs, auds, certs, claims scopes and --- account claim maps) as columns with | delimited values. The use of the --- postgres string_agg(...) to aggregate the value objects into a column works --- because we are only pulling in one column from the associated tables and that --- value is part of the primary key and unique. This view will make things like --- recursive listing of oidc auth methods fairly straightforward to implement --- for the oidc repo. The view also includes an is_primary_auth_method bool -create view oidc_auth_method_with_value_obj as -select - case when s.primary_auth_method_id is not null then - true - else false end - as is_primary_auth_method, - am.public_id, - am.scope_id, - am.name, - am.description, - am.create_time, - am.update_time, - am.version, - am.state, - am.api_url, - am.disable_discovered_config_validation, - am.issuer, - am.client_id, - am.client_secret, - am.client_secret_hmac, - am.key_id, - am.max_age, - -- the string_agg(..) column will be null if there are no associated value objects - string_agg(distinct alg.signing_alg_name, '|') as algs, - string_agg(distinct aud.aud_claim, '|') as auds, - string_agg(distinct cert.certificate, '|') as certs, - string_agg(distinct cs.scope, '|') as claims_scopes, - string_agg(distinct concat_ws('=', acm.from_claim, acm.to_claim), '|') as account_claim_maps -from - auth_oidc_method am - left outer join iam_scope s on am.public_id = s.primary_auth_method_id - left outer join auth_oidc_signing_alg alg on am.public_id = alg.oidc_method_id - left outer join auth_oidc_aud_claim aud on am.public_id = aud.oidc_method_id - left outer join auth_oidc_certificate cert on am.public_id = cert.oidc_method_id - left outer join auth_oidc_scope cs on am.public_id = cs.oidc_method_id - left outer join auth_oidc_account_claim_map acm on am.public_id = acm.oidc_method_id -group by am.public_id, is_primary_auth_method; -- there can be only one public_id + is_primary_auth_method, so group by isn't a problem. -comment on view oidc_auth_method_with_value_obj is -'oidc auth method with its associated value objects (algs, auds, certs, scopes) as columns with | delimited values'; -`), - 7001: []byte(` -create function wt_add_seconds(sec integer, ts timestamp with time zone) - returns timestamp with time zone - as $$ - select ts + sec * '1 second'::interval; - $$ language sql - stable - returns null on null input; - comment on function wt_add_seconds is - 'wt_add_seconds returns ts + sec.'; - - create function wt_add_seconds_to_now(sec integer) - returns timestamp with time zone - as $$ - select wt_add_seconds(sec, current_timestamp); - $$ language sql - stable - returns null on null input; - comment on function wt_add_seconds_to_now is - 'wt_add_seconds_to_now returns current_timestamp + sec.'; -`), - 7002: []byte(` -create domain wt_plugin_id as text - not null - check( - length(trim(value)) > 10 or value = 'pi_system' - ); - comment on domain wt_plugin_id is - '"pi_system", or random ID generated with github.com/hashicorp/go-secure-stdlib/base62'; - - create table plugin ( - public_id wt_plugin_id primary key - ); - - comment on table plugin is - 'plugin is a table where each row represents a unique plugin registered with Boundary.'; - - insert into plugin (public_id) - values - ('pi_system'); - - create trigger immutable_columns before update on plugin - for each row execute procedure immutable_columns('public_id'); - - create or replace function - disallow_system_plugin_deletion() - returns trigger - as $$ - begin - if old.public_id = 'pi_system' then - raise exception 'deletion of system plugin not allowed'; - end if; - return old; - end; - $$ language plpgsql; - - create trigger plugin_disallow_system_deletion before delete on plugin - for each row execute procedure disallow_system_plugin_deletion(); -`), - 7003: []byte(` -create table job ( - plugin_id wt_plugin_id not null - constraint plugin_fk - references plugin(public_id) - on delete cascade - on update cascade, - name wt_name not null, - description wt_description not null, - next_scheduled_run timestamp with time zone not null, - primary key (plugin_id, name) - ); - - comment on table job is - 'job is a table where each row represents a unique job that can only have one running instance at any specific time.'; - - create trigger immutable_columns before update on job - for each row execute procedure immutable_columns('plugin_id', 'name'); - - create table job_run_status_enm ( - name text not null primary key - constraint only_predefined_job_status_allowed - check(name in ('running', 'completed', 'failed', 'interrupted')) - ); - - comment on table job_run_status_enm is - 'job_run_status_enm is an enumeration table where each row contains a valid job run state.'; - - insert into job_run_status_enm (name) - values - ('running'), - ('completed'), - ('failed'), - ('interrupted'); - - create table job_run ( - private_id wh_dim_id primary key - default wh_dim_id(), - job_plugin_id wt_plugin_id not null, - job_name wt_name not null, - server_id wt_private_id - constraint server_fkey - references server(private_id) - on delete set null - on update cascade, - create_time wt_timestamp, - update_time wt_timestamp, - end_time timestamp with time zone, - completed_count int not null - default 0 - constraint completed_count_can_not_be_negative - check(completed_count >= 0), - total_count int not null - default 0 - constraint total_count_can_not_be_negative - check(total_count >= 0), - status text not null - default 'running' - constraint job_run_status_enm_fkey - references job_run_status_enm (name) - on delete restrict - on update cascade, - - constraint job_run_completed_count_less_than_equal_to_total_count - check(completed_count <= total_count), - - constraint job_fkey - foreign key (job_plugin_id, job_name) - references job (plugin_id, name) - on delete cascade - on update cascade - ); - - comment on table job_run is - 'job_run is a table where each row represents an instance of a job run that is either actively running or has already completed.'; - - create unique index job_run_status_constraint - on job_run (job_plugin_id, job_name) - where status = 'running'; - - create trigger update_time_column before update on job_run - for each row execute procedure update_time_column(); - - create trigger default_create_time_column before insert on job_run - for each row execute procedure default_create_time(); - - create trigger immutable_columns before update on job_run - for each row execute procedure immutable_columns('private_id', 'job_plugin_id', 'job_name', 'create_time'); - - create view job_jobs_to_run as - with - running_jobs (job_plugin_id, job_name) as ( - select job_plugin_id, job_name - from job_run - where status = 'running' - ), - final (job_plugin_id, job_name, next_scheduled_run) as ( - select plugin_id, name, next_scheduled_run - from job j - where next_scheduled_run <= current_timestamp - and not exists ( - select - from running_jobs - where job_plugin_id = j.plugin_id - and job_name = j.name - ) - ) - select job_plugin_id, job_name, next_scheduled_run from final; -`), - 8001: []byte(` -alter table session_connection - add column server_id text; - --- Note: here, and in the session table, we should add a trigger ensuring that --- if server_id goes to null, we mark connections as closed. See --- https://hashicorp.atlassian.net/browse/ICU-1495 -alter table session_connection - add constraint server_fkey - foreign key (server_id) - references server (private_id) - on delete set null - on update cascade; - --- We now populate the connection information from existing session information -update session_connection sc -set - server_id = s.server_id -from - session s -where - sc.session_id = s.public_id; -`), - 9001: []byte(` --- The base abstract table -create table auth_managed_group ( - public_id wt_public_id - primary key, - auth_method_id wt_public_id - not null, - -- Ensure that if the auth method is deleted (which will also happen if the - -- scope is deleted) this is deleted too - constraint auth_method_fkey - foreign key (auth_method_id) -- fk1 - references auth_method(public_id) - on delete cascade - on update cascade, - constraint auth_managed_group_auth_method_id_public_id_uq - unique(auth_method_id, public_id) -); -comment on table auth_managed_group is -'auth_managed_group is the abstract base table for managed groups.'; - --- Define the immutable fields of auth_managed_group -create trigger - immutable_columns -before -update on auth_managed_group - for each row execute procedure immutable_columns('public_id', 'auth_method_id'); - --- Function to insert into the base table when values are inserted into a --- concrete type table. This happens before inserts so the foreign keys in the --- concrete type will be valid. -create or replace function - insert_managed_group_subtype() - returns trigger -as $$ -begin - - insert into auth_managed_group - (public_id, auth_method_id) - values - (new.public_id, new.auth_method_id); - - return new; - -end; -$$ language plpgsql; - --- delete_managed_group_subtype() is an after delete trigger --- function for subtypes of managed_group -create or replace function delete_managed_group_subtype() - returns trigger -as $$ -begin - delete from auth_managed_group - where public_id = old.public_id; - return null; -- result is ignored since this is an after trigger -end; -$$ language plpgsql; -`), - 9002: []byte(` -create table auth_oidc_managed_group ( - public_id wt_public_id - primary key, - auth_method_id wt_public_id - not null, - name wt_name, - description wt_description, - create_time wt_timestamp, - update_time wt_timestamp, - version wt_version, - filter wt_bexprfilter - not null, - -- Ensure that this managed group relates to an oidc auth method, as opposed - -- to other types - constraint auth_oidc_method_fkey - foreign key (auth_method_id) -- fk1 - references auth_oidc_method (public_id) - on delete cascade - on update cascade, - -- Ensure it relates to an abstract managed group - constraint auth_managed_group_fkey - foreign key (auth_method_id, public_id) -- fk2 - references auth_managed_group (auth_method_id, public_id) - on delete cascade - on update cascade, - constraint auth_oidc_managed_group_auth_method_id_name_uq - unique(auth_method_id, name) -); -comment on table auth_oidc_managed_group is -'auth_oidc_managed_group entries are subtypes of auth_managed_group and represent an oidc managed group.'; - --- Define the immutable fields of auth_oidc_managed_group -create trigger - immutable_columns -before -update on auth_oidc_managed_group - for each row execute procedure immutable_columns('public_id', 'auth_method_id', 'create_time'); - --- Populate create time on insert -create trigger - default_create_time_column -before -insert on auth_oidc_managed_group - for each row execute procedure default_create_time(); - --- Generate update time on update -create trigger - update_time_column -before -update on auth_oidc_managed_group - for each row execute procedure update_time_column(); - --- Update version when something changes -create trigger - update_version_column -after -update on auth_oidc_managed_group - for each row execute procedure update_version_column(); - --- Add into the base table when inserting into the concrete table -create trigger - insert_managed_group_subtype -before insert on auth_oidc_managed_group - for each row execute procedure insert_managed_group_subtype(); - --- Ensure that deletions in the oidc subtype result in deletions to the base --- table. -create trigger - delete_managed_group_subtype -after -delete on auth_oidc_managed_group - for each row execute procedure delete_managed_group_subtype(); - --- The tickets for oplog are the subtypes not the base types because no updates --- are done to any values in the base types. -insert into oplog_ticket - (name, version) -values - ('auth_oidc_managed_group', 1); -`), - 9003: []byte(` --- Mappings of account to oidc managed groups. This is a non-abstract table with --- a view (below) so that it is a natural aggregate for the oplog (also below). -create table auth_oidc_managed_group_member_account ( - create_time wt_timestamp, - managed_group_id wt_public_id - references auth_oidc_managed_group(public_id) - on delete cascade - on update cascade, - member_id wt_public_id - references auth_oidc_account(public_id) - on delete cascade - on update cascade, - primary key (managed_group_id, member_id) -); -comment on table auth_oidc_managed_group_member_account is -'auth_oidc_managed_group_member_account is the join table for managed oidc groups and accounts.'; - --- auth_immutable_managed_oidc_group_member_account() ensures that group members are immutable. -create or replace function - auth_immutable_managed_oidc_group_member_account() - returns trigger -as $$ -begin - raise exception 'managed oidc group members are immutable'; -end; -$$ language plpgsql; - -create trigger - default_create_time_column -before -insert on auth_oidc_managed_group_member_account - for each row execute procedure default_create_time(); - -create trigger - auth_immutable_managed_oidc_group_member_account -before -update on auth_oidc_managed_group_member_account - for each row execute procedure auth_immutable_managed_oidc_group_member_account(); - --- Initially create the view with just oidc; eventually we can replace this view --- to union with other subtype tables. -create view auth_managed_group_member_account as -select - oidc.create_time, - oidc.managed_group_id, - oidc.member_id -from - auth_oidc_managed_group_member_account oidc; -`), - 9004: []byte(` --- iam_managed_group_role contains roles that have been assigned to managed --- groups. Managed groups can be from any scope. The rows in this table must be --- immutable after insert, which will be ensured with a before update trigger --- using iam_immutable_role_principal(). -create table iam_managed_group_role ( - create_time wt_timestamp, - role_id wt_role_id - references iam_role(public_id) - on delete cascade - on update cascade, - principal_id wt_public_id - references auth_managed_group(public_id) - on delete cascade - on update cascade, - primary key (role_id, principal_id) - ); - -create trigger immutable_role_principal -before update on iam_managed_group_role - for each row execute procedure iam_immutable_role_principal(); - -create trigger default_create_time_column -before insert on iam_managed_group_role - for each row execute procedure default_create_time(); - --- iam_principal_role provides a consolidated view all principal roles assigned --- (user and group roles). -create or replace view iam_principal_role as -select - ur.create_time, - ur.principal_id, - ur.role_id, - u.scope_id as principal_scope_id, - r.scope_id as role_scope_id, - get_scoped_principal_id(r.scope_id, u.scope_id, ur.principal_id) as scoped_principal_id, - 'user' as type -from - iam_user_role ur, - iam_role r, - iam_user u -where - ur.role_id = r.public_id and - u.public_id = ur.principal_id -union -select - gr.create_time, - gr.principal_id, - gr.role_id, - g.scope_id as principal_scope_id, - r.scope_id as role_scope_id, - get_scoped_principal_id(r.scope_id, g.scope_id, gr.principal_id) as scoped_principal_id, - 'group' as type -from - iam_group_role gr, - iam_role r, - iam_group g -where - gr.role_id = r.public_id and - g.public_id = gr.principal_id -union -select - mgr.create_time, - mgr.principal_id, - mgr.role_id, - (select scope_id from auth_method am where am.public_id = amg.auth_method_id) as principal_scope_id, - r.scope_id as role_scope_id, - get_scoped_principal_id(r.scope_id, (select scope_id from auth_method am where am.public_id = amg.auth_method_id), mgr.principal_id) as scoped_principal_id, - 'managed group' as type -from - iam_managed_group_role mgr, - iam_role r, - auth_managed_group amg -where - mgr.role_id = r.public_id and - amg.public_id = mgr.principal_id; -`), - }, - } -} diff --git a/internal/db/schema/schema.go b/internal/db/schema/schema.go index c33bd505d..f0180b852 100644 --- a/internal/db/schema/schema.go +++ b/internal/db/schema/schema.go @@ -9,11 +9,11 @@ import ( // MigrateStore executes the migrations needed to initialize the store. It // returns true if migrations actually ran; false if the database is already current -// or if there was an error. Supports the WithMigrationStates(...) option. -func MigrateStore(ctx context.Context, dialect string, url string, opt ...Option) (bool, error) { +// or if there was an error. Supports the WithEditions(...) option. +func MigrateStore(ctx context.Context, dialect Dialect, url string, opt ...Option) (bool, error) { const op = "schema.MigrateStore" - d, err := common.SqlOpen(dialect, url) + d, err := common.SqlOpen(dialect.String(), url) if err != nil { return false, errors.Wrap(ctx, err, op) } @@ -27,15 +27,12 @@ func MigrateStore(ctx context.Context, dialect string, url string, opt ...Option if err != nil { return false, errors.Wrap(ctx, err, op) } - if st.Dirty { - return false, errors.New(ctx, errors.MigrationIntegrity, op, "db marked dirty") - } - if st.InitializationStarted && st.DatabaseSchemaVersion == st.BinarySchemaVersion { + if st.Initialized && st.MigrationsApplied() { return false, nil } - if err := sMan.RollForward(ctx); err != nil { + if err := sMan.ApplyMigrations(ctx); err != nil { return false, errors.Wrap(ctx, err, op) } diff --git a/internal/db/schema/schema_test.go b/internal/db/schema/schema_test.go index ad2d354c9..db440d276 100644 --- a/internal/db/schema/schema_test.go +++ b/internal/db/schema/schema_test.go @@ -1,10 +1,11 @@ -package schema +package schema_test import ( "context" "testing" - "github.com/hashicorp/boundary/internal/db/common" + "github.com/hashicorp/boundary/internal/db/schema" + "github.com/hashicorp/boundary/internal/db/schema/internal/edition" "github.com/hashicorp/boundary/testing/dbtest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -20,55 +21,68 @@ func TestMigrateStore(t *testing.T) { require.NoError(t, c()) }) - // Set the possible migration state to be only part of the full migration - oState := migrationStates[dialect] - nState := createPartialMigrationState(oState, 8) - migrationStates[dialect] = nState - - ran, err := MigrateStore(ctx, dialect, u) + ran, err := schema.MigrateStore(ctx, schema.Dialect(dialect), u, schema.WithEditions( + edition.Editions{ + { + Name: "oss", + Dialect: schema.Postgres, + LatestVersion: 1, + Migrations: map[int][]byte{ + 1: []byte(`select 1`), + }, + Priority: 0, + }, + }, + )) assert.NoError(t, err) assert.True(t, ran) - ran, err = MigrateStore(ctx, dialect, u) + + ran, err = schema.MigrateStore(ctx, schema.Dialect(dialect), u, schema.WithEditions( + edition.Editions{ + { + Name: "oss", + Dialect: schema.Postgres, + LatestVersion: 1, + Migrations: map[int][]byte{ + 2: []byte(`select 1`), + }, + Priority: 0, + }, + }, + )) assert.NoError(t, err) assert.False(t, ran) - // Reset the possible migration state to contain everything - migrationStates[dialect] = oState - - ran, err = MigrateStore(ctx, dialect, u) + ran, err = schema.MigrateStore(ctx, schema.Dialect(dialect), u, schema.WithEditions( + edition.Editions{ + { + Name: "oss", + Dialect: schema.Postgres, + LatestVersion: 2, + Migrations: map[int][]byte{ + 1: []byte(`select 1`), + 2: []byte(`select 1`), + }, + Priority: 0, + }, + }, + )) assert.NoError(t, err) assert.True(t, ran) - ran, err = MigrateStore(ctx, dialect, u) + ran, err = schema.MigrateStore(ctx, schema.Dialect(dialect), u, schema.WithEditions( + edition.Editions{ + { + Name: "oss", + Dialect: schema.Postgres, + LatestVersion: 2, + Migrations: map[int][]byte{ + 1: []byte(`select 1`), + 2: []byte(`select 1`), + }, + Priority: 0, + }, + }, + )) assert.NoError(t, err) assert.False(t, ran) } - -func Test_MigrateStore_WithMigrationStates(t *testing.T) { - assert, require := assert.New(t), require.New(t) - dialect := dbtest.Postgres - ctx := context.Background() - - c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1)) - require.NoError(err) - t.Cleanup(func() { - require.NoError(c()) - }) - d, err := common.SqlOpen(dialect, u) - require.NoError(err) - - // migration to the prior migration (before the one we want to test) - oState := TestCloneMigrationStates(t) - nState := TestCreatePartialMigrationState(oState["postgres"], 8) - oState["postgres"] = nState - - ran, err := MigrateStore(ctx, dialect, u, WithMigrationStates(oState)) - assert.NoError(err) - assert.True(ran) - - m, err := NewManager(ctx, dialect, d, WithMigrationStates(oState)) - require.NoError(err) - state, err := m.CurrentState(ctx) - require.NoError(err) - assert.Equal(8, state.DatabaseSchemaVersion) - assert.False(state.Dirty) -} diff --git a/internal/db/schema/state.go b/internal/db/schema/state.go index 1dc1e3845..d330ac64b 100644 --- a/internal/db/schema/state.go +++ b/internal/db/schema/state.go @@ -1,58 +1,66 @@ package schema +import "github.com/hashicorp/boundary/internal/db/schema/internal/provider" + const nilVersion = -1 -// migrationState is meant to be populated by the generated migration code and -// contains the internal representation of a schema in the current binary. -type migrationState struct { - // binarySchemaVersion provides the database schema version supported by - // this binary. - binarySchemaVersion int - - upMigrations map[int][]byte +// State contains information regarding the current state of a boundary database's schema. +type State struct { + // Initialized indicates if the current database has been previously initialized. + Initialized bool + Editions []EditionState } -// migrationStates is populated by the generated migration code with the key being the dialect. -var migrationStates = make(map[string]migrationState) - -func getUpMigration(dialect string, opt ...Option) map[int][]byte { - opts := getOpts(opt...) - var ms migrationState - var ok bool - if opts.withMigrationStates != nil { - ms, ok = opts.withMigrationStates[dialect] - } else { - ms, ok = migrationStates[dialect] - } - if !ok { - return nil - } - return ms.upMigrations -} - -// BinarySchemaVersion provides the schema version that this binary supports for the provided dialect. -// If the binary doesn't support this dialect -1 is returned. -func BinarySchemaVersion(dialect string) int { - ms, ok := migrationStates[dialect] - if !ok { - return nilVersion - } - return ms.binarySchemaVersion -} - -func cloneMigrationStates(states map[string]migrationState) map[string]migrationState { - nStates := map[string]migrationState{} - for k, s := range states { - newState := migrationState{ - binarySchemaVersion: s.binarySchemaVersion, - upMigrations: map[int][]byte{}, +// MigrationsApplied checks to see that all Editions are in the Equal SchemaState. +func (s State) MigrationsApplied() bool { + for _, e := range s.Editions { + if e.DatabaseSchemaState != Equal { + return false } - for v, up := range s.upMigrations { - cp := make([]byte, len(up)) - copy(cp, up) - newState.upMigrations[v] = cp - } - nStates[k] = newState } - return nStates + return true +} + +func (s State) databaseState() provider.DatabaseState { + dbState := make(provider.DatabaseState) + for _, e := range s.Editions { + dbState[e.Name] = e.DatabaseSchemaVersion + } + return dbState +} + +// DatabaseState defines the state of the Database schema as compared to the +// latest version for the binary. +type DatabaseState int + +// Valid states. +const ( + Behind DatabaseState = iota // Database schema version is older then latest for the binary, so it needs migrations applied. + Ahead // Database schema version is newer then latest for the binary, so binary needs to be updated. + Equal // Database schema version matches latest version for the binary. +) + +// EditionState is the current state of a schema Edition. +type EditionState struct { + // Name is the identifier of the Edition. + Name string + + // DatabaseSchemaVersion is the schema version that is currently running in the database. + DatabaseSchemaVersion int + // BinarySchemaVersion is the schema version which this boundary binary supports. + BinarySchemaVersion int + + DatabaseSchemaState DatabaseState +} + +func compareVersions(d int, b int) DatabaseState { + if d == b { + return Equal + } + + if d < b { + return Behind + } + + return Ahead } diff --git a/internal/db/schema/state_test.go b/internal/db/schema/state_test.go deleted file mode 100644 index fa24919f5..000000000 --- a/internal/db/schema/state_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package schema - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestBinarySchemaVersion(t *testing.T) { - dialect := "test_binaryschemaversion" - migrationStates[dialect] = migrationState{binarySchemaVersion: 3} - assert.Equal(t, 3, BinarySchemaVersion(dialect)) - assert.Equal(t, nilVersion, BinarySchemaVersion("unknown_dialect")) -} diff --git a/internal/db/schema/statement.go b/internal/db/schema/statement.go deleted file mode 100644 index 63ce8028e..000000000 --- a/internal/db/schema/statement.go +++ /dev/null @@ -1,50 +0,0 @@ -package schema - -import ( - "sort" -) - -// statementProvider provides the migration statements in order. -// Next should be called prior to calling Version() or ReadUp() or sentinel -// values (-1 and nil) will be returned. -type statementProvider struct { - pos int - versions []int - up map[int][]byte -} - -func newStatementProvider(dialect string, curVer int, opt ...Option) *statementProvider { - const op = "schema.newStatementProvider" - qp := statementProvider{pos: -1} - qp.up = getUpMigration(dialect, opt...) - for k := range qp.up { - qp.versions = append(qp.versions, k) - } - sort.Ints(qp.versions) - - for len(qp.versions) > 0 && qp.versions[0] <= curVer { - qp.versions = qp.versions[1:] - } - - return &qp -} - -func (q *statementProvider) Next() bool { - q.pos++ - return len(q.versions) > q.pos -} - -func (q *statementProvider) Version() int { - if q.pos < 0 || q.pos >= len(q.versions) { - return -1 - } - return q.versions[q.pos] -} - -// ReadUp reads the current up migration -func (q *statementProvider) ReadUp() []byte { - if q.pos < 0 || q.pos >= len(q.versions) { - return nil - } - return q.up[q.versions[q.pos]] -} diff --git a/internal/db/schema/statement_test.go b/internal/db/schema/statement_test.go deleted file mode 100644 index 77ac31b56..000000000 --- a/internal/db/schema/statement_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package schema - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestStatementProvider(t *testing.T) { - testDialect := "test" - migrationStates[testDialect] = migrationState{ - binarySchemaVersion: 5, - upMigrations: map[int][]byte{ - 1: []byte("one"), - 2: []byte("two"), - 3: []byte("three"), - }, - } - - st := newStatementProvider(testDialect, 1) - assert.Equal(t, -1, st.Version()) - assert.Equal(t, []byte(nil), st.ReadUp()) - - assert.True(t, st.Next()) - assert.Equal(t, 2, st.Version()) - assert.Equal(t, []byte("two"), st.ReadUp()) - - assert.True(t, st.Next()) - assert.Equal(t, 3, st.Version()) - assert.Equal(t, []byte("three"), st.ReadUp()) - - assert.False(t, st.Next()) - assert.Equal(t, -1, st.Version()) - assert.Equal(t, []byte(nil), st.ReadUp()) - - assert.False(t, st.Next()) - assert.Equal(t, -1, st.Version()) - assert.Equal(t, []byte(nil), st.ReadUp()) - - st = newStatementProvider("unknown_dialect", nilVersion) - assert.False(t, st.Next()) -} diff --git a/internal/db/schema/testdata/one/0/01_initial.up.sql b/internal/db/schema/testdata/one/0/01_initial.up.sql new file mode 100644 index 000000000..a7dc0e211 --- /dev/null +++ b/internal/db/schema/testdata/one/0/01_initial.up.sql @@ -0,0 +1,6 @@ +begin; + create domain tt_public_id as text + check( + length(trim(value)) > 10 + ); +commit; diff --git a/internal/db/schema/testdata/three/0/01_initial.up.sql b/internal/db/schema/testdata/three/0/01_initial.up.sql new file mode 100644 index 000000000..395877ad3 --- /dev/null +++ b/internal/db/schema/testdata/three/0/01_initial.up.sql @@ -0,0 +1,6 @@ +begin; + create table test_three ( + id tt_public_id primary key, + two_id tt_public_id references test_two(id) + ); +commit; diff --git a/internal/db/schema/testdata/two/0/01_initial.up.sql b/internal/db/schema/testdata/two/0/01_initial.up.sql new file mode 100644 index 000000000..d4a924552 --- /dev/null +++ b/internal/db/schema/testdata/two/0/01_initial.up.sql @@ -0,0 +1,5 @@ +begin; + create table test_two ( + id tt_public_id primary key + ); +commit; diff --git a/internal/db/schema/testing.go b/internal/db/schema/testing.go index 38edea0a8..0b5a06432 100644 --- a/internal/db/schema/testing.go +++ b/internal/db/schema/testing.go @@ -1,27 +1,41 @@ package schema -import ( - "testing" -) +import "github.com/hashicorp/boundary/internal/db/schema/internal/edition" -// Creates a new migrationState only with the versions <= the provided maxVer -func TestCreatePartialMigrationState(om migrationState, maxVer int) migrationState { - nState := migrationState{ - upMigrations: make(map[int][]byte), - } - for k := range om.upMigrations { - if k > maxVer { - // Don't store any versions past our test version. - continue - } - nState.upMigrations[k] = om.upMigrations[k] - if nState.binarySchemaVersion < k { - nState.binarySchemaVersion = k +// PartialEditions is used by TestCreatePartialEditions. It is a map of edition +// names to the max version that should be included. +type PartialEditions map[string]int + +// TestCreatePartialEditions is used by tests to create a subset of the Edition migrations. +func TestCreatePartialEditions(dialect Dialect, p PartialEditions) edition.Editions { + editions.Lock() + defer editions.Unlock() + + e := make(edition.Editions, 0, len(p)) + for _, ee := range editions.m[dialect] { + maxVer, ok := p[ee.Name] + if ok { + edition := edition.Edition{ + Name: ee.Name, + Dialect: ee.Dialect, + Priority: ee.Priority, + LatestVersion: nilVersion, + Migrations: make(map[int][]byte), + } + + for k, b := range ee.Migrations { + if k > maxVer { + continue + } + + edition.Migrations[k] = b + if k > edition.LatestVersion { + edition.LatestVersion = k + } + } + e = append(e, edition) } } - return nState -} -func TestCloneMigrationStates(t *testing.T) map[string]migrationState { - return cloneMigrationStates(migrationStates) + return e } diff --git a/internal/db/testing.go b/internal/db/testing.go index c8b76a293..2fe197255 100644 --- a/internal/db/testing.go +++ b/internal/db/testing.go @@ -42,7 +42,7 @@ func TestSetup(t *testing.T, dialect string, opt ...TestOption) (*DB, string) { url = opts.withTestDatabaseUrl } - _, err = schema.MigrateStore(ctx, dialect, url) + _, err = schema.MigrateStore(ctx, schema.Dialect(dialect), url) if err != nil { t.Fatalf("Couldn't init store on existing db: %v", err) } diff --git a/internal/oplog/testing.go b/internal/oplog/testing.go index dac8f502c..ec677026f 100644 --- a/internal/oplog/testing.go +++ b/internal/oplog/testing.go @@ -88,7 +88,7 @@ func testInitStore(t *testing.T, cleanup func() error, url string) { d, err := common.SqlOpen(dialect, url) require.NoError(t, err) - sm, err := schema.NewManager(ctx, dialect, d) + sm, err := schema.NewManager(ctx, schema.Dialect(dialect), d) require.NoError(t, err) - require.NoError(t, sm.RollForward(ctx)) + require.NoError(t, sm.ApplyMigrations(ctx)) } From 2143940a4ac4c0c609404f4e5e83c1a2bfff1685 Mon Sep 17 00:00:00 2001 From: Timothy Messier Date: Mon, 13 Sep 2021 17:13:57 -0400 Subject: [PATCH 2/4] refactor(dbtest): Support new paths for migration files This updates the docker image used for tests to support the new location for sql migration files and the concept of editions. It continues to process the old paths for now if the new directory does not exist, so that the same image can be used during the transition. --- testing/dbtest/docker/Makefile | 2 +- testing/dbtest/docker/init-db.sh | 90 ++++++++++++++++++++++++-------- 2 files changed, 70 insertions(+), 22 deletions(-) diff --git a/testing/dbtest/docker/Makefile b/testing/dbtest/docker/Makefile index ea9db518d..93734b9f0 100644 --- a/testing/dbtest/docker/Makefile +++ b/testing/dbtest/docker/Makefile @@ -36,7 +36,7 @@ database-up: -e POSTGRES_DB=boundary \ -e PGDATA=/pgdata \ --mount type=tmpfs,destination=/pgdata \ - -v "$(CWD)/../../../internal/db/schema/migrations/postgres":/migrations \ + -v "$(CWD)/../../../internal/db/schema/migrations":/migrations \ $(IMAGE_TAG) \ -c 'config_file=/etc/postgresql/postgresql.conf' \ $(PG_OPTS) 1> /dev/null diff --git a/testing/dbtest/docker/init-db.sh b/testing/dbtest/docker/init-db.sh index ae18f8edc..a2512f572 100755 --- a/testing/dbtest/docker/init-db.sh +++ b/testing/dbtest/docker/init-db.sh @@ -2,43 +2,91 @@ set -e shopt -s globstar +# Create database to use as template psql -v "ON_ERROR_STOP=1" --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -q < Date: Wed, 15 Sep 2021 08:26:39 -0400 Subject: [PATCH 3/4] refactor(sqltest): Update init script to support migration editions This updates the docker image used for tests to support the new location for sql migration files and the concept of editions. It continues to process the old paths for now if the new directory does not exist, so that the same image can be used during the transition. --- internal/db/sqltest/Makefile | 4 +-- internal/db/sqltest/initdb.d/00_schema.sh | 33 +++++++++++++++++++---- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/internal/db/sqltest/Makefile b/internal/db/sqltest/Makefile index 5305a6bb4..04731bb21 100644 --- a/internal/db/sqltest/Makefile +++ b/internal/db/sqltest/Makefile @@ -33,7 +33,7 @@ test: -e POSTGRES_PASSWORD=boundary \ -e POSTGRES_USER=boundary \ -e POSTGRES_DB=boundary \ - -v "$(CWD)/../schema/migrations/postgres":/migrations \ + -v "$(CWD)/../schema/migrations":/migrations \ -v "$(CWD)/initdb.d":/docker-entrypoint-initdb.d/ \ $(POSTGRES_DOCKER_IMAGE) @docker run -it --rm \ @@ -56,7 +56,7 @@ database-up: -e POSTGRES_PASSWORD=boundary \ -e POSTGRES_USER=boundary \ -e POSTGRES_DB=boundary \ - -v "$(CWD)/../schema/migrations/postgres":/migrations \ + -v "$(CWD)/../schema/migrations":/migrations \ -v "$(CWD)/initdb.d":/docker-entrypoint-initdb.d/ \ $(POSTGRES_DOCKER_IMAGE) diff --git a/internal/db/sqltest/initdb.d/00_schema.sh b/internal/db/sqltest/initdb.d/00_schema.sh index b1cc3fdb7..a2ce2aa0b 100755 --- a/internal/db/sqltest/initdb.d/00_schema.sh +++ b/internal/db/sqltest/initdb.d/00_schema.sh @@ -26,8 +26,31 @@ docker_process_sql() { PGHOST= PGHOSTADDR= "${query_runner[@]}" "$@" } -# Run migrations in order. -for file in $(ls -v /migrations/**/*.sql); do - echo running "$file" - docker_process_sql -f "$file" -done +if [ -d /migrations/base ]; then + # Run migrations in order. + for file in $(ls -v /migrations/base/postgres/**/*.up.sql); do + echo running "$file" + docker_process_sql -f "$file" + done + + if [ -d /migrations/oss ]; then + # Run migrations in order. + for file in $(ls -v /migrations/oss/postgres/**/*.up.sql); do + echo running "$file" + docker_process_sql -f "$file" + done + fi + + for d in $(find /migrations/ -mindepth 1 -maxdepth 1 -type d -not -name 'oss' -not -name 'base'); do + for file in $(ls -v ${d}/postgres/**/*.up.sql); do + echo running "$file" + docker_process_sql -f "$file" + done + done +else + # Run migrations in order. + for file in $(ls -v /migrations/**/*.up.sql); do + echo running "$file" + docker_process_sql -f "$file" + done +fi From 06e6ab6c5766094168d4667f12e4055b72c62f30 Mon Sep 17 00:00:00 2001 From: Timothy Messier Date: Mon, 20 Sep 2021 09:39:43 -0400 Subject: [PATCH 4/4] build: Fix build to support switch to embed for migrations With the refactor to the schema package, and the switch to use embed, the sql files need to be included in CI when compiling the boundary binary. --- .circleci/config.yml | 176 ++++++++++++------------- .circleci/config/@build-release.yml | 176 ++++++++++++------------- packages-oss.lock/layers/layers.mk | 16 +-- packages-oss.lock/pkgs.yml | 198 ++++++++++++++-------------- packages-oss.yml | 2 +- 5 files changed, 284 insertions(+), 284 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 54e711bdd..ca89773db 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,49 +30,49 @@ jobs: command: mkdir -p .buildcache && echo "*" > .buildcache/.gitignore name: Ignore .buildcache - restore_cache: - key: package-2bb9488232c945d6800cc3a4a461d9b17503bccc-{{checksum ".buildcache/cache-keys/package-2bb9488232c945d6800cc3a4a461d9b17503bccc"}} + key: package-9801fb26ed9eb42980684234791a85a8d16c2713-{{checksum ".buildcache/cache-keys/package-9801fb26ed9eb42980684234791a85a8d16c2713"}} name: Restore package cache - restore_cache: - key: package-e9f9d34c2c14f59f68c31469b7bfa994074a48c2-{{checksum ".buildcache/cache-keys/package-e9f9d34c2c14f59f68c31469b7bfa994074a48c2"}} + key: package-c6687b4dfea47101b46520e0809c4e4135a9a3b8-{{checksum ".buildcache/cache-keys/package-c6687b4dfea47101b46520e0809c4e4135a9a3b8"}} name: Restore package cache - restore_cache: - key: package-3e2a1075872eae7608bf8be4b74f7291ca88d5b5-{{checksum ".buildcache/cache-keys/package-3e2a1075872eae7608bf8be4b74f7291ca88d5b5"}} + key: package-3d40792896ab78cd0e160df4bbd1c757c63cf985-{{checksum ".buildcache/cache-keys/package-3d40792896ab78cd0e160df4bbd1c757c63cf985"}} name: Restore package cache - restore_cache: - key: package-822b1aa9954becb7a9cb480784ff9f99783a8d23-{{checksum ".buildcache/cache-keys/package-822b1aa9954becb7a9cb480784ff9f99783a8d23"}} + key: package-fb9d5eb085b094774ff0f514490c9b8a5da28e24-{{checksum ".buildcache/cache-keys/package-fb9d5eb085b094774ff0f514490c9b8a5da28e24"}} name: Restore package cache - restore_cache: - key: package-7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a-{{checksum ".buildcache/cache-keys/package-7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a"}} + key: package-701ab62817cc79f8ec83299cefac126cb3c74d47-{{checksum ".buildcache/cache-keys/package-701ab62817cc79f8ec83299cefac126cb3c74d47"}} name: Restore package cache - restore_cache: - key: package-03d32b4b7349ceac506ca889be4772fe8239ecd8-{{checksum ".buildcache/cache-keys/package-03d32b4b7349ceac506ca889be4772fe8239ecd8"}} + key: package-524f6510ca269c805e992fa740ae8cd26db1b091-{{checksum ".buildcache/cache-keys/package-524f6510ca269c805e992fa740ae8cd26db1b091"}} name: Restore package cache - restore_cache: - key: package-ab3b6922ce2bd0fab200dddd6127544427b89fef-{{checksum ".buildcache/cache-keys/package-ab3b6922ce2bd0fab200dddd6127544427b89fef"}} + key: package-dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175-{{checksum ".buildcache/cache-keys/package-dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175"}} name: Restore package cache - restore_cache: - key: package-7f1216479517d6c6122fd8928ff44c046eb3a377-{{checksum ".buildcache/cache-keys/package-7f1216479517d6c6122fd8928ff44c046eb3a377"}} + key: package-d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2-{{checksum ".buildcache/cache-keys/package-d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2"}} name: Restore package cache - restore_cache: - key: package-7fae91402a49def23c2453fe147c262f1377eac3-{{checksum ".buildcache/cache-keys/package-7fae91402a49def23c2453fe147c262f1377eac3"}} + key: package-15db6440776ae92411d314100f394e7d59eb54dc-{{checksum ".buildcache/cache-keys/package-15db6440776ae92411d314100f394e7d59eb54dc"}} name: Restore package cache - restore_cache: - key: package-c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8-{{checksum ".buildcache/cache-keys/package-c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8"}} + key: package-0b0af5839c026648f6a9bee77d6769df9b76fd6c-{{checksum ".buildcache/cache-keys/package-0b0af5839c026648f6a9bee77d6769df9b76fd6c"}} name: Restore package cache - restore_cache: - key: package-b0201f2d2d1b04f519e0d091fbe23997252c5f70-{{checksum ".buildcache/cache-keys/package-b0201f2d2d1b04f519e0d091fbe23997252c5f70"}} + key: package-053de88ba7c9c7d3d7021774a040b4dba0edb1d3-{{checksum ".buildcache/cache-keys/package-053de88ba7c9c7d3d7021774a040b4dba0edb1d3"}} name: Restore package cache - restore_cache: - key: package-51ea70d1d613481fac829f239f1f081c5ad6c88b-{{checksum ".buildcache/cache-keys/package-51ea70d1d613481fac829f239f1f081c5ad6c88b"}} + key: package-34a4409cf043f4e9aedbb4ae8e68db90fa0552a0-{{checksum ".buildcache/cache-keys/package-34a4409cf043f4e9aedbb4ae8e68db90fa0552a0"}} name: Restore package cache - restore_cache: - key: package-19efae459acdaea98d804c26c554640ab3544553-{{checksum ".buildcache/cache-keys/package-19efae459acdaea98d804c26c554640ab3544553"}} + key: package-6d1801298f017e462e056af69b69e5b2f5fd2143-{{checksum ".buildcache/cache-keys/package-6d1801298f017e462e056af69b69e5b2f5fd2143"}} name: Restore package cache - restore_cache: - key: package-44f600485ac3823fa4e0dbba45763d932187225a-{{checksum ".buildcache/cache-keys/package-44f600485ac3823fa4e0dbba45763d932187225a"}} + key: package-c380c1e5206b2d02d9603a8cc068933436e05eb0-{{checksum ".buildcache/cache-keys/package-c380c1e5206b2d02d9603a8cc068933436e05eb0"}} name: Restore package cache - restore_cache: - key: package-d10bc84044e1ad705f48cd51591b05ee5e1e7dd5-{{checksum ".buildcache/cache-keys/package-d10bc84044e1ad705f48cd51591b05ee5e1e7dd5"}} + key: package-1df2a38d32be613f8179c22ffd0bf03c68f1a16d-{{checksum ".buildcache/cache-keys/package-1df2a38d32be613f8179c22ffd0bf03c68f1a16d"}} name: Restore package cache - run: command: make package-meta-all @@ -84,20 +84,20 @@ jobs: command: ls -lahR .buildcache name: List Build Cache - run: - command: cp packages*.lock/pkgs.yml lockfile-645fb387fb0a87de.yml + command: cp packages*.lock/pkgs.yml lockfile-84a71f97438578ee.yml name: Update Lockfile Name - run: - command: tar -czf packages-645fb387fb0a87de.tar.gz .buildcache/packages lockfile-645fb387fb0a87de.yml + command: tar -czf packages-84a71f97438578ee.tar.gz .buildcache/packages lockfile-84a71f97438578ee.yml name: Create Raw Package Tarball - run: - command: tar -czf meta-645fb387fb0a87de.tar.gz .buildcache/packages/store/*.json lockfile-645fb387fb0a87de.yml + command: tar -czf meta-84a71f97438578ee.tar.gz .buildcache/packages/store/*.json lockfile-84a71f97438578ee.yml name: Create Metadata Tarball - store_artifacts: - path: lockfile-645fb387fb0a87de.yml + path: lockfile-84a71f97438578ee.yml - store_artifacts: - path: packages-645fb387fb0a87de.tar.gz + path: packages-84a71f97438578ee.tar.gz - store_artifacts: - path: meta-645fb387fb0a87de.tar.gz + path: meta-84a71f97438578ee.tar.gz - store_artifacts: path: .buildcache/packages environment: @@ -112,7 +112,7 @@ jobs: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain - PRODUCT_REVISION: '' - - PACKAGE_SPEC_ID: 2bb9488232c945d6800cc3a4a461d9b17503bccc + - PACKAGE_SPEC_ID: 9801fb26ed9eb42980684234791a85a8d16c2713 steps: - setup_remote_docker: docker_layer_caching: false @@ -126,7 +126,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-2bb9488232c945d6800cc3a4a461d9b17503bccc-{{checksum ".buildcache/cache-keys/package-2bb9488232c945d6800cc3a4a461d9b17503bccc"}} + key: package-9801fb26ed9eb42980684234791a85a8d16c2713-{{checksum ".buildcache/cache-keys/package-9801fb26ed9eb42980684234791a85a8d16c2713"}} name: Restore package cache - run: command: |2 @@ -142,7 +142,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -172,7 +172,7 @@ jobs: command: ls -lahR .buildcache/packages name: List packages - save_cache: - key: package-2bb9488232c945d6800cc3a4a461d9b17503bccc-{{checksum ".buildcache/cache-keys/package-2bb9488232c945d6800cc3a4a461d9b17503bccc"}} + key: package-9801fb26ed9eb42980684234791a85a8d16c2713-{{checksum ".buildcache/cache-keys/package-9801fb26ed9eb42980684234791a85a8d16c2713"}} name: Save package cache paths: - .buildcache/packages/store @@ -184,7 +184,7 @@ jobs: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain - PRODUCT_REVISION: '' - - PACKAGE_SPEC_ID: 44f600485ac3823fa4e0dbba45763d932187225a + - PACKAGE_SPEC_ID: c380c1e5206b2d02d9603a8cc068933436e05eb0 steps: - setup_remote_docker: docker_layer_caching: false @@ -198,7 +198,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-44f600485ac3823fa4e0dbba45763d932187225a-{{checksum ".buildcache/cache-keys/package-44f600485ac3823fa4e0dbba45763d932187225a"}} + key: package-c380c1e5206b2d02d9603a8cc068933436e05eb0-{{checksum ".buildcache/cache-keys/package-c380c1e5206b2d02d9603a8cc068933436e05eb0"}} name: Restore package cache - run: command: |2 @@ -214,7 +214,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -244,7 +244,7 @@ jobs: command: ls -lahR .buildcache/packages name: List packages - save_cache: - key: package-44f600485ac3823fa4e0dbba45763d932187225a-{{checksum ".buildcache/cache-keys/package-44f600485ac3823fa4e0dbba45763d932187225a"}} + key: package-c380c1e5206b2d02d9603a8cc068933436e05eb0-{{checksum ".buildcache/cache-keys/package-c380c1e5206b2d02d9603a8cc068933436e05eb0"}} name: Save package cache paths: - .buildcache/packages/store @@ -274,14 +274,14 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} name: 'Restore Layer Cache: copy-source' - run: - command: LAYER_SPEC_ID=03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 make -C packages*.lock load-builder-cache + command: LAYER_SPEC_ID=03-copy-source-e59a64ba925a852a09535a8d252147336d635eff make -C packages*.lock load-builder-cache name: Load whatever builder cache we have (if any) into the Docker daemon - run: command: |2- @@ -345,7 +345,7 @@ jobs: echo " attempts to directly clone private repositories." fi - make -f packages*.lock/layer.mk 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7-image + make -f packages*.lock/layer.mk 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff-image name: Build copy-source layer - run: command: |2- @@ -361,13 +361,13 @@ jobs: echo " attempts to directly clone private repositories." fi - make -f packages*.lock/layer.mk 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7-save + make -f packages*.lock/layer.mk 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff-save name: Build copy-source layer - save_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} name: 'Save Layer Cache: copy-source' paths: - - .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + - .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz environment: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain @@ -380,7 +380,7 @@ jobs: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain - PRODUCT_REVISION: '' - - PACKAGE_SPEC_ID: 7f1216479517d6c6122fd8928ff44c046eb3a377 + - PACKAGE_SPEC_ID: d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2 steps: - setup_remote_docker: docker_layer_caching: false @@ -394,7 +394,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-7f1216479517d6c6122fd8928ff44c046eb3a377-{{checksum ".buildcache/cache-keys/package-7f1216479517d6c6122fd8928ff44c046eb3a377"}} + key: package-d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2-{{checksum ".buildcache/cache-keys/package-d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2"}} name: Restore package cache - run: command: |2 @@ -410,7 +410,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -440,7 +440,7 @@ jobs: command: ls -lahR .buildcache/packages name: List packages - save_cache: - key: package-7f1216479517d6c6122fd8928ff44c046eb3a377-{{checksum ".buildcache/cache-keys/package-7f1216479517d6c6122fd8928ff44c046eb3a377"}} + key: package-d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2-{{checksum ".buildcache/cache-keys/package-d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2"}} name: Save package cache paths: - .buildcache/packages/store @@ -452,7 +452,7 @@ jobs: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain - PRODUCT_REVISION: '' - - PACKAGE_SPEC_ID: ab3b6922ce2bd0fab200dddd6127544427b89fef + - PACKAGE_SPEC_ID: dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175 steps: - setup_remote_docker: docker_layer_caching: false @@ -466,7 +466,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-ab3b6922ce2bd0fab200dddd6127544427b89fef-{{checksum ".buildcache/cache-keys/package-ab3b6922ce2bd0fab200dddd6127544427b89fef"}} + key: package-dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175-{{checksum ".buildcache/cache-keys/package-dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175"}} name: Restore package cache - run: command: |2 @@ -482,7 +482,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -512,7 +512,7 @@ jobs: command: ls -lahR .buildcache/packages name: List packages - save_cache: - key: package-ab3b6922ce2bd0fab200dddd6127544427b89fef-{{checksum ".buildcache/cache-keys/package-ab3b6922ce2bd0fab200dddd6127544427b89fef"}} + key: package-dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175-{{checksum ".buildcache/cache-keys/package-dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175"}} name: Save package cache paths: - .buildcache/packages/store @@ -535,7 +535,7 @@ jobs: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain - PRODUCT_REVISION: '' - - PACKAGE_SPEC_ID: 7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a + - PACKAGE_SPEC_ID: 701ab62817cc79f8ec83299cefac126cb3c74d47 steps: - setup_remote_docker: docker_layer_caching: false @@ -549,7 +549,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a-{{checksum ".buildcache/cache-keys/package-7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a"}} + key: package-701ab62817cc79f8ec83299cefac126cb3c74d47-{{checksum ".buildcache/cache-keys/package-701ab62817cc79f8ec83299cefac126cb3c74d47"}} name: Restore package cache - run: command: |2 @@ -565,7 +565,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -595,7 +595,7 @@ jobs: command: ls -lahR .buildcache/packages name: List packages - save_cache: - key: package-7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a-{{checksum ".buildcache/cache-keys/package-7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a"}} + key: package-701ab62817cc79f8ec83299cefac126cb3c74d47-{{checksum ".buildcache/cache-keys/package-701ab62817cc79f8ec83299cefac126cb3c74d47"}} name: Save package cache paths: - .buildcache/packages/store @@ -607,7 +607,7 @@ jobs: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain - PRODUCT_REVISION: '' - - PACKAGE_SPEC_ID: e9f9d34c2c14f59f68c31469b7bfa994074a48c2 + - PACKAGE_SPEC_ID: c6687b4dfea47101b46520e0809c4e4135a9a3b8 steps: - setup_remote_docker: docker_layer_caching: false @@ -621,7 +621,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-e9f9d34c2c14f59f68c31469b7bfa994074a48c2-{{checksum ".buildcache/cache-keys/package-e9f9d34c2c14f59f68c31469b7bfa994074a48c2"}} + key: package-c6687b4dfea47101b46520e0809c4e4135a9a3b8-{{checksum ".buildcache/cache-keys/package-c6687b4dfea47101b46520e0809c4e4135a9a3b8"}} name: Restore package cache - run: command: |2 @@ -637,7 +637,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -667,7 +667,7 @@ jobs: command: ls -lahR .buildcache/packages name: List packages - save_cache: - key: package-e9f9d34c2c14f59f68c31469b7bfa994074a48c2-{{checksum ".buildcache/cache-keys/package-e9f9d34c2c14f59f68c31469b7bfa994074a48c2"}} + key: package-c6687b4dfea47101b46520e0809c4e4135a9a3b8-{{checksum ".buildcache/cache-keys/package-c6687b4dfea47101b46520e0809c4e4135a9a3b8"}} name: Save package cache paths: - .buildcache/packages/store @@ -679,7 +679,7 @@ jobs: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain - PRODUCT_REVISION: '' - - PACKAGE_SPEC_ID: 3e2a1075872eae7608bf8be4b74f7291ca88d5b5 + - PACKAGE_SPEC_ID: 3d40792896ab78cd0e160df4bbd1c757c63cf985 steps: - setup_remote_docker: docker_layer_caching: false @@ -693,7 +693,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-3e2a1075872eae7608bf8be4b74f7291ca88d5b5-{{checksum ".buildcache/cache-keys/package-3e2a1075872eae7608bf8be4b74f7291ca88d5b5"}} + key: package-3d40792896ab78cd0e160df4bbd1c757c63cf985-{{checksum ".buildcache/cache-keys/package-3d40792896ab78cd0e160df4bbd1c757c63cf985"}} name: Restore package cache - run: command: |2 @@ -709,7 +709,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -739,7 +739,7 @@ jobs: command: ls -lahR .buildcache/packages name: List packages - save_cache: - key: package-3e2a1075872eae7608bf8be4b74f7291ca88d5b5-{{checksum ".buildcache/cache-keys/package-3e2a1075872eae7608bf8be4b74f7291ca88d5b5"}} + key: package-3d40792896ab78cd0e160df4bbd1c757c63cf985-{{checksum ".buildcache/cache-keys/package-3d40792896ab78cd0e160df4bbd1c757c63cf985"}} name: Save package cache paths: - .buildcache/packages/store @@ -751,7 +751,7 @@ jobs: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain - PRODUCT_REVISION: '' - - PACKAGE_SPEC_ID: b0201f2d2d1b04f519e0d091fbe23997252c5f70 + - PACKAGE_SPEC_ID: 053de88ba7c9c7d3d7021774a040b4dba0edb1d3 steps: - setup_remote_docker: docker_layer_caching: false @@ -765,7 +765,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-b0201f2d2d1b04f519e0d091fbe23997252c5f70-{{checksum ".buildcache/cache-keys/package-b0201f2d2d1b04f519e0d091fbe23997252c5f70"}} + key: package-053de88ba7c9c7d3d7021774a040b4dba0edb1d3-{{checksum ".buildcache/cache-keys/package-053de88ba7c9c7d3d7021774a040b4dba0edb1d3"}} name: Restore package cache - run: command: |2 @@ -781,7 +781,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -811,7 +811,7 @@ jobs: command: ls -lahR .buildcache/packages name: List packages - save_cache: - key: package-b0201f2d2d1b04f519e0d091fbe23997252c5f70-{{checksum ".buildcache/cache-keys/package-b0201f2d2d1b04f519e0d091fbe23997252c5f70"}} + key: package-053de88ba7c9c7d3d7021774a040b4dba0edb1d3-{{checksum ".buildcache/cache-keys/package-053de88ba7c9c7d3d7021774a040b4dba0edb1d3"}} name: Save package cache paths: - .buildcache/packages/store @@ -823,7 +823,7 @@ jobs: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain - PRODUCT_REVISION: '' - - PACKAGE_SPEC_ID: 03d32b4b7349ceac506ca889be4772fe8239ecd8 + - PACKAGE_SPEC_ID: 524f6510ca269c805e992fa740ae8cd26db1b091 steps: - setup_remote_docker: docker_layer_caching: false @@ -837,7 +837,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-03d32b4b7349ceac506ca889be4772fe8239ecd8-{{checksum ".buildcache/cache-keys/package-03d32b4b7349ceac506ca889be4772fe8239ecd8"}} + key: package-524f6510ca269c805e992fa740ae8cd26db1b091-{{checksum ".buildcache/cache-keys/package-524f6510ca269c805e992fa740ae8cd26db1b091"}} name: Restore package cache - run: command: |2 @@ -853,7 +853,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -883,7 +883,7 @@ jobs: command: ls -lahR .buildcache/packages name: List packages - save_cache: - key: package-03d32b4b7349ceac506ca889be4772fe8239ecd8-{{checksum ".buildcache/cache-keys/package-03d32b4b7349ceac506ca889be4772fe8239ecd8"}} + key: package-524f6510ca269c805e992fa740ae8cd26db1b091-{{checksum ".buildcache/cache-keys/package-524f6510ca269c805e992fa740ae8cd26db1b091"}} name: Save package cache paths: - .buildcache/packages/store @@ -906,7 +906,7 @@ jobs: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain - PRODUCT_REVISION: '' - - PACKAGE_SPEC_ID: 19efae459acdaea98d804c26c554640ab3544553 + - PACKAGE_SPEC_ID: 6d1801298f017e462e056af69b69e5b2f5fd2143 steps: - setup_remote_docker: docker_layer_caching: false @@ -920,7 +920,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-19efae459acdaea98d804c26c554640ab3544553-{{checksum ".buildcache/cache-keys/package-19efae459acdaea98d804c26c554640ab3544553"}} + key: package-6d1801298f017e462e056af69b69e5b2f5fd2143-{{checksum ".buildcache/cache-keys/package-6d1801298f017e462e056af69b69e5b2f5fd2143"}} name: Restore package cache - run: command: |2 @@ -936,7 +936,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -966,7 +966,7 @@ jobs: command: ls -lahR .buildcache/packages name: List packages - save_cache: - key: package-19efae459acdaea98d804c26c554640ab3544553-{{checksum ".buildcache/cache-keys/package-19efae459acdaea98d804c26c554640ab3544553"}} + key: package-6d1801298f017e462e056af69b69e5b2f5fd2143-{{checksum ".buildcache/cache-keys/package-6d1801298f017e462e056af69b69e5b2f5fd2143"}} name: Save package cache paths: - .buildcache/packages/store @@ -1128,7 +1128,7 @@ jobs: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain - PRODUCT_REVISION: '' - - PACKAGE_SPEC_ID: c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8 + - PACKAGE_SPEC_ID: 0b0af5839c026648f6a9bee77d6769df9b76fd6c steps: - setup_remote_docker: docker_layer_caching: false @@ -1142,7 +1142,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8-{{checksum ".buildcache/cache-keys/package-c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8"}} + key: package-0b0af5839c026648f6a9bee77d6769df9b76fd6c-{{checksum ".buildcache/cache-keys/package-0b0af5839c026648f6a9bee77d6769df9b76fd6c"}} name: Restore package cache - run: command: |2 @@ -1158,7 +1158,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -1188,7 +1188,7 @@ jobs: command: ls -lahR .buildcache/packages name: List packages - save_cache: - key: package-c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8-{{checksum ".buildcache/cache-keys/package-c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8"}} + key: package-0b0af5839c026648f6a9bee77d6769df9b76fd6c-{{checksum ".buildcache/cache-keys/package-0b0af5839c026648f6a9bee77d6769df9b76fd6c"}} name: Save package cache paths: - .buildcache/packages/store @@ -1200,7 +1200,7 @@ jobs: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain - PRODUCT_REVISION: '' - - PACKAGE_SPEC_ID: 51ea70d1d613481fac829f239f1f081c5ad6c88b + - PACKAGE_SPEC_ID: 34a4409cf043f4e9aedbb4ae8e68db90fa0552a0 steps: - setup_remote_docker: docker_layer_caching: false @@ -1214,7 +1214,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-51ea70d1d613481fac829f239f1f081c5ad6c88b-{{checksum ".buildcache/cache-keys/package-51ea70d1d613481fac829f239f1f081c5ad6c88b"}} + key: package-34a4409cf043f4e9aedbb4ae8e68db90fa0552a0-{{checksum ".buildcache/cache-keys/package-34a4409cf043f4e9aedbb4ae8e68db90fa0552a0"}} name: Restore package cache - run: command: |2 @@ -1230,7 +1230,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -1260,7 +1260,7 @@ jobs: command: ls -lahR .buildcache/packages name: List packages - save_cache: - key: package-51ea70d1d613481fac829f239f1f081c5ad6c88b-{{checksum ".buildcache/cache-keys/package-51ea70d1d613481fac829f239f1f081c5ad6c88b"}} + key: package-34a4409cf043f4e9aedbb4ae8e68db90fa0552a0-{{checksum ".buildcache/cache-keys/package-34a4409cf043f4e9aedbb4ae8e68db90fa0552a0"}} name: Save package cache paths: - .buildcache/packages/store @@ -1272,7 +1272,7 @@ jobs: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain - PRODUCT_REVISION: '' - - PACKAGE_SPEC_ID: 7fae91402a49def23c2453fe147c262f1377eac3 + - PACKAGE_SPEC_ID: 15db6440776ae92411d314100f394e7d59eb54dc steps: - setup_remote_docker: docker_layer_caching: false @@ -1286,7 +1286,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-7fae91402a49def23c2453fe147c262f1377eac3-{{checksum ".buildcache/cache-keys/package-7fae91402a49def23c2453fe147c262f1377eac3"}} + key: package-15db6440776ae92411d314100f394e7d59eb54dc-{{checksum ".buildcache/cache-keys/package-15db6440776ae92411d314100f394e7d59eb54dc"}} name: Restore package cache - run: command: |2 @@ -1302,7 +1302,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -1332,7 +1332,7 @@ jobs: command: ls -lahR .buildcache/packages name: List packages - save_cache: - key: package-7fae91402a49def23c2453fe147c262f1377eac3-{{checksum ".buildcache/cache-keys/package-7fae91402a49def23c2453fe147c262f1377eac3"}} + key: package-15db6440776ae92411d314100f394e7d59eb54dc-{{checksum ".buildcache/cache-keys/package-15db6440776ae92411d314100f394e7d59eb54dc"}} name: Save package cache paths: - .buildcache/packages/store @@ -1344,7 +1344,7 @@ jobs: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain - PRODUCT_REVISION: '' - - PACKAGE_SPEC_ID: 822b1aa9954becb7a9cb480784ff9f99783a8d23 + - PACKAGE_SPEC_ID: fb9d5eb085b094774ff0f514490c9b8a5da28e24 steps: - setup_remote_docker: docker_layer_caching: false @@ -1358,7 +1358,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-822b1aa9954becb7a9cb480784ff9f99783a8d23-{{checksum ".buildcache/cache-keys/package-822b1aa9954becb7a9cb480784ff9f99783a8d23"}} + key: package-fb9d5eb085b094774ff0f514490c9b8a5da28e24-{{checksum ".buildcache/cache-keys/package-fb9d5eb085b094774ff0f514490c9b8a5da28e24"}} name: Restore package cache - run: command: |2 @@ -1374,7 +1374,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -1404,7 +1404,7 @@ jobs: command: ls -lahR .buildcache/packages name: List packages - save_cache: - key: package-822b1aa9954becb7a9cb480784ff9f99783a8d23-{{checksum ".buildcache/cache-keys/package-822b1aa9954becb7a9cb480784ff9f99783a8d23"}} + key: package-fb9d5eb085b094774ff0f514490c9b8a5da28e24-{{checksum ".buildcache/cache-keys/package-fb9d5eb085b094774ff0f514490c9b8a5da28e24"}} name: Save package cache paths: - .buildcache/packages/store @@ -1416,7 +1416,7 @@ jobs: - AUTO_INSTALL_TOOLS: 'YES' - BUILDKIT_PROGRESS: plain - PRODUCT_REVISION: '' - - PACKAGE_SPEC_ID: d10bc84044e1ad705f48cd51591b05ee5e1e7dd5 + - PACKAGE_SPEC_ID: 1df2a38d32be613f8179c22ffd0bf03c68f1a16d steps: - setup_remote_docker: docker_layer_caching: false @@ -1430,7 +1430,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-d10bc84044e1ad705f48cd51591b05ee5e1e7dd5-{{checksum ".buildcache/cache-keys/package-d10bc84044e1ad705f48cd51591b05ee5e1e7dd5"}} + key: package-1df2a38d32be613f8179c22ffd0bf03c68f1a16d-{{checksum ".buildcache/cache-keys/package-1df2a38d32be613f8179c22ffd0bf03c68f1a16d"}} name: Restore package cache - run: command: |2 @@ -1446,7 +1446,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -1476,7 +1476,7 @@ jobs: command: ls -lahR .buildcache/packages name: List packages - save_cache: - key: package-d10bc84044e1ad705f48cd51591b05ee5e1e7dd5-{{checksum ".buildcache/cache-keys/package-d10bc84044e1ad705f48cd51591b05ee5e1e7dd5"}} + key: package-1df2a38d32be613f8179c22ffd0bf03c68f1a16d-{{checksum ".buildcache/cache-keys/package-1df2a38d32be613f8179c22ffd0bf03c68f1a16d"}} name: Save package cache paths: - .buildcache/packages/store @@ -1492,7 +1492,7 @@ jobs: make test-sql POSTGRES_DOCKER_IMAGE_BASE=docker.mirror.hashicorp.services/postgres PG_DOCKER_TAG=latest name: Run SQL PgTap Tests workflows: - build-645fb387fb0a87de: + build-84a71f97438578ee: jobs: - build-common-layers - darwin_amd64_package: diff --git a/.circleci/config/@build-release.yml b/.circleci/config/@build-release.yml index 7432efd84..e5ae713c2 100644 --- a/.circleci/config/@build-release.yml +++ b/.circleci/config/@build-release.yml @@ -30,14 +30,14 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} name: 'Restore Layer Cache: copy-source' - run: - command: LAYER_SPEC_ID=03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 make -C packages*.lock load-builder-cache + command: LAYER_SPEC_ID=03-copy-source-e59a64ba925a852a09535a8d252147336d635eff make -C packages*.lock load-builder-cache name: Load whatever builder cache we have (if any) into the Docker daemon - run: command: |2- @@ -101,7 +101,7 @@ jobs: echo " attempts to directly clone private repositories." fi - make -f packages*.lock/layer.mk 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7-image + make -f packages*.lock/layer.mk 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff-image name: Build copy-source layer - run: command: |2- @@ -117,12 +117,12 @@ jobs: echo " attempts to directly clone private repositories." fi - make -f packages*.lock/layer.mk 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7-save + make -f packages*.lock/layer.mk 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff-save name: Build copy-source layer - save_cache: paths: - - .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} name: 'Save Layer Cache: copy-source' bundle-releases: executor: builder @@ -136,49 +136,49 @@ jobs: command: mkdir -p .buildcache && echo "*" > .buildcache/.gitignore name: Ignore .buildcache - restore_cache: - key: package-2bb9488232c945d6800cc3a4a461d9b17503bccc-{{checksum ".buildcache/cache-keys/package-2bb9488232c945d6800cc3a4a461d9b17503bccc"}} + key: package-9801fb26ed9eb42980684234791a85a8d16c2713-{{checksum ".buildcache/cache-keys/package-9801fb26ed9eb42980684234791a85a8d16c2713"}} name: Restore package cache - restore_cache: - key: package-e9f9d34c2c14f59f68c31469b7bfa994074a48c2-{{checksum ".buildcache/cache-keys/package-e9f9d34c2c14f59f68c31469b7bfa994074a48c2"}} + key: package-c6687b4dfea47101b46520e0809c4e4135a9a3b8-{{checksum ".buildcache/cache-keys/package-c6687b4dfea47101b46520e0809c4e4135a9a3b8"}} name: Restore package cache - restore_cache: - key: package-3e2a1075872eae7608bf8be4b74f7291ca88d5b5-{{checksum ".buildcache/cache-keys/package-3e2a1075872eae7608bf8be4b74f7291ca88d5b5"}} + key: package-3d40792896ab78cd0e160df4bbd1c757c63cf985-{{checksum ".buildcache/cache-keys/package-3d40792896ab78cd0e160df4bbd1c757c63cf985"}} name: Restore package cache - restore_cache: - key: package-822b1aa9954becb7a9cb480784ff9f99783a8d23-{{checksum ".buildcache/cache-keys/package-822b1aa9954becb7a9cb480784ff9f99783a8d23"}} + key: package-fb9d5eb085b094774ff0f514490c9b8a5da28e24-{{checksum ".buildcache/cache-keys/package-fb9d5eb085b094774ff0f514490c9b8a5da28e24"}} name: Restore package cache - restore_cache: - key: package-7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a-{{checksum ".buildcache/cache-keys/package-7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a"}} + key: package-701ab62817cc79f8ec83299cefac126cb3c74d47-{{checksum ".buildcache/cache-keys/package-701ab62817cc79f8ec83299cefac126cb3c74d47"}} name: Restore package cache - restore_cache: - key: package-03d32b4b7349ceac506ca889be4772fe8239ecd8-{{checksum ".buildcache/cache-keys/package-03d32b4b7349ceac506ca889be4772fe8239ecd8"}} + key: package-524f6510ca269c805e992fa740ae8cd26db1b091-{{checksum ".buildcache/cache-keys/package-524f6510ca269c805e992fa740ae8cd26db1b091"}} name: Restore package cache - restore_cache: - key: package-ab3b6922ce2bd0fab200dddd6127544427b89fef-{{checksum ".buildcache/cache-keys/package-ab3b6922ce2bd0fab200dddd6127544427b89fef"}} + key: package-dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175-{{checksum ".buildcache/cache-keys/package-dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175"}} name: Restore package cache - restore_cache: - key: package-7f1216479517d6c6122fd8928ff44c046eb3a377-{{checksum ".buildcache/cache-keys/package-7f1216479517d6c6122fd8928ff44c046eb3a377"}} + key: package-d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2-{{checksum ".buildcache/cache-keys/package-d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2"}} name: Restore package cache - restore_cache: - key: package-7fae91402a49def23c2453fe147c262f1377eac3-{{checksum ".buildcache/cache-keys/package-7fae91402a49def23c2453fe147c262f1377eac3"}} + key: package-15db6440776ae92411d314100f394e7d59eb54dc-{{checksum ".buildcache/cache-keys/package-15db6440776ae92411d314100f394e7d59eb54dc"}} name: Restore package cache - restore_cache: - key: package-c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8-{{checksum ".buildcache/cache-keys/package-c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8"}} + key: package-0b0af5839c026648f6a9bee77d6769df9b76fd6c-{{checksum ".buildcache/cache-keys/package-0b0af5839c026648f6a9bee77d6769df9b76fd6c"}} name: Restore package cache - restore_cache: - key: package-b0201f2d2d1b04f519e0d091fbe23997252c5f70-{{checksum ".buildcache/cache-keys/package-b0201f2d2d1b04f519e0d091fbe23997252c5f70"}} + key: package-053de88ba7c9c7d3d7021774a040b4dba0edb1d3-{{checksum ".buildcache/cache-keys/package-053de88ba7c9c7d3d7021774a040b4dba0edb1d3"}} name: Restore package cache - restore_cache: - key: package-51ea70d1d613481fac829f239f1f081c5ad6c88b-{{checksum ".buildcache/cache-keys/package-51ea70d1d613481fac829f239f1f081c5ad6c88b"}} + key: package-34a4409cf043f4e9aedbb4ae8e68db90fa0552a0-{{checksum ".buildcache/cache-keys/package-34a4409cf043f4e9aedbb4ae8e68db90fa0552a0"}} name: Restore package cache - restore_cache: - key: package-19efae459acdaea98d804c26c554640ab3544553-{{checksum ".buildcache/cache-keys/package-19efae459acdaea98d804c26c554640ab3544553"}} + key: package-6d1801298f017e462e056af69b69e5b2f5fd2143-{{checksum ".buildcache/cache-keys/package-6d1801298f017e462e056af69b69e5b2f5fd2143"}} name: Restore package cache - restore_cache: - key: package-44f600485ac3823fa4e0dbba45763d932187225a-{{checksum ".buildcache/cache-keys/package-44f600485ac3823fa4e0dbba45763d932187225a"}} + key: package-c380c1e5206b2d02d9603a8cc068933436e05eb0-{{checksum ".buildcache/cache-keys/package-c380c1e5206b2d02d9603a8cc068933436e05eb0"}} name: Restore package cache - restore_cache: - key: package-d10bc84044e1ad705f48cd51591b05ee5e1e7dd5-{{checksum ".buildcache/cache-keys/package-d10bc84044e1ad705f48cd51591b05ee5e1e7dd5"}} + key: package-1df2a38d32be613f8179c22ffd0bf03c68f1a16d-{{checksum ".buildcache/cache-keys/package-1df2a38d32be613f8179c22ffd0bf03c68f1a16d"}} name: Restore package cache - run: command: make package-meta-all @@ -190,20 +190,20 @@ jobs: command: ls -lahR .buildcache name: List Build Cache - run: - command: cp packages*.lock/pkgs.yml lockfile-645fb387fb0a87de.yml + command: cp packages*.lock/pkgs.yml lockfile-84a71f97438578ee.yml name: Update Lockfile Name - run: - command: tar -czf packages-645fb387fb0a87de.tar.gz .buildcache/packages lockfile-645fb387fb0a87de.yml + command: tar -czf packages-84a71f97438578ee.tar.gz .buildcache/packages lockfile-84a71f97438578ee.yml name: Create Raw Package Tarball - run: - command: tar -czf meta-645fb387fb0a87de.tar.gz .buildcache/packages/store/*.json lockfile-645fb387fb0a87de.yml + command: tar -czf meta-84a71f97438578ee.tar.gz .buildcache/packages/store/*.json lockfile-84a71f97438578ee.yml name: Create Metadata Tarball - store_artifacts: - path: lockfile-645fb387fb0a87de.yml + path: lockfile-84a71f97438578ee.yml - store_artifacts: - path: packages-645fb387fb0a87de.tar.gz + path: packages-84a71f97438578ee.tar.gz - store_artifacts: - path: meta-645fb387fb0a87de.tar.gz + path: meta-84a71f97438578ee.tar.gz - store_artifacts: path: .buildcache/packages darwin_amd64_package: @@ -221,7 +221,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-2bb9488232c945d6800cc3a4a461d9b17503bccc-{{checksum ".buildcache/cache-keys/package-2bb9488232c945d6800cc3a4a461d9b17503bccc"}} + key: package-9801fb26ed9eb42980684234791a85a8d16c2713-{{checksum ".buildcache/cache-keys/package-9801fb26ed9eb42980684234791a85a8d16c2713"}} name: Restore package cache - run: command: |2 @@ -237,7 +237,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -269,10 +269,10 @@ jobs: - save_cache: paths: - .buildcache/packages/store - key: package-2bb9488232c945d6800cc3a4a461d9b17503bccc-{{checksum ".buildcache/cache-keys/package-2bb9488232c945d6800cc3a4a461d9b17503bccc"}} + key: package-9801fb26ed9eb42980684234791a85a8d16c2713-{{checksum ".buildcache/cache-keys/package-9801fb26ed9eb42980684234791a85a8d16c2713"}} name: Save package cache environment: - PACKAGE_SPEC_ID: 2bb9488232c945d6800cc3a4a461d9b17503bccc + PACKAGE_SPEC_ID: 9801fb26ed9eb42980684234791a85a8d16c2713 freebsd_386_package: executor: builder steps: @@ -288,7 +288,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-e9f9d34c2c14f59f68c31469b7bfa994074a48c2-{{checksum ".buildcache/cache-keys/package-e9f9d34c2c14f59f68c31469b7bfa994074a48c2"}} + key: package-c6687b4dfea47101b46520e0809c4e4135a9a3b8-{{checksum ".buildcache/cache-keys/package-c6687b4dfea47101b46520e0809c4e4135a9a3b8"}} name: Restore package cache - run: command: |2 @@ -304,7 +304,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -336,10 +336,10 @@ jobs: - save_cache: paths: - .buildcache/packages/store - key: package-e9f9d34c2c14f59f68c31469b7bfa994074a48c2-{{checksum ".buildcache/cache-keys/package-e9f9d34c2c14f59f68c31469b7bfa994074a48c2"}} + key: package-c6687b4dfea47101b46520e0809c4e4135a9a3b8-{{checksum ".buildcache/cache-keys/package-c6687b4dfea47101b46520e0809c4e4135a9a3b8"}} name: Save package cache environment: - PACKAGE_SPEC_ID: e9f9d34c2c14f59f68c31469b7bfa994074a48c2 + PACKAGE_SPEC_ID: c6687b4dfea47101b46520e0809c4e4135a9a3b8 freebsd_amd64_package: executor: builder steps: @@ -355,7 +355,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-3e2a1075872eae7608bf8be4b74f7291ca88d5b5-{{checksum ".buildcache/cache-keys/package-3e2a1075872eae7608bf8be4b74f7291ca88d5b5"}} + key: package-3d40792896ab78cd0e160df4bbd1c757c63cf985-{{checksum ".buildcache/cache-keys/package-3d40792896ab78cd0e160df4bbd1c757c63cf985"}} name: Restore package cache - run: command: |2 @@ -371,7 +371,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -403,10 +403,10 @@ jobs: - save_cache: paths: - .buildcache/packages/store - key: package-3e2a1075872eae7608bf8be4b74f7291ca88d5b5-{{checksum ".buildcache/cache-keys/package-3e2a1075872eae7608bf8be4b74f7291ca88d5b5"}} + key: package-3d40792896ab78cd0e160df4bbd1c757c63cf985-{{checksum ".buildcache/cache-keys/package-3d40792896ab78cd0e160df4bbd1c757c63cf985"}} name: Save package cache environment: - PACKAGE_SPEC_ID: 3e2a1075872eae7608bf8be4b74f7291ca88d5b5 + PACKAGE_SPEC_ID: 3d40792896ab78cd0e160df4bbd1c757c63cf985 freebsd_arm_package: executor: builder steps: @@ -422,7 +422,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-822b1aa9954becb7a9cb480784ff9f99783a8d23-{{checksum ".buildcache/cache-keys/package-822b1aa9954becb7a9cb480784ff9f99783a8d23"}} + key: package-fb9d5eb085b094774ff0f514490c9b8a5da28e24-{{checksum ".buildcache/cache-keys/package-fb9d5eb085b094774ff0f514490c9b8a5da28e24"}} name: Restore package cache - run: command: |2 @@ -438,7 +438,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -470,10 +470,10 @@ jobs: - save_cache: paths: - .buildcache/packages/store - key: package-822b1aa9954becb7a9cb480784ff9f99783a8d23-{{checksum ".buildcache/cache-keys/package-822b1aa9954becb7a9cb480784ff9f99783a8d23"}} + key: package-fb9d5eb085b094774ff0f514490c9b8a5da28e24-{{checksum ".buildcache/cache-keys/package-fb9d5eb085b094774ff0f514490c9b8a5da28e24"}} name: Save package cache environment: - PACKAGE_SPEC_ID: 822b1aa9954becb7a9cb480784ff9f99783a8d23 + PACKAGE_SPEC_ID: fb9d5eb085b094774ff0f514490c9b8a5da28e24 linux_386_package: executor: builder steps: @@ -489,7 +489,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a-{{checksum ".buildcache/cache-keys/package-7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a"}} + key: package-701ab62817cc79f8ec83299cefac126cb3c74d47-{{checksum ".buildcache/cache-keys/package-701ab62817cc79f8ec83299cefac126cb3c74d47"}} name: Restore package cache - run: command: |2 @@ -505,7 +505,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -537,10 +537,10 @@ jobs: - save_cache: paths: - .buildcache/packages/store - key: package-7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a-{{checksum ".buildcache/cache-keys/package-7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a"}} + key: package-701ab62817cc79f8ec83299cefac126cb3c74d47-{{checksum ".buildcache/cache-keys/package-701ab62817cc79f8ec83299cefac126cb3c74d47"}} name: Save package cache environment: - PACKAGE_SPEC_ID: 7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a + PACKAGE_SPEC_ID: 701ab62817cc79f8ec83299cefac126cb3c74d47 linux_amd64_package: executor: builder steps: @@ -556,7 +556,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-03d32b4b7349ceac506ca889be4772fe8239ecd8-{{checksum ".buildcache/cache-keys/package-03d32b4b7349ceac506ca889be4772fe8239ecd8"}} + key: package-524f6510ca269c805e992fa740ae8cd26db1b091-{{checksum ".buildcache/cache-keys/package-524f6510ca269c805e992fa740ae8cd26db1b091"}} name: Restore package cache - run: command: |2 @@ -572,7 +572,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -604,10 +604,10 @@ jobs: - save_cache: paths: - .buildcache/packages/store - key: package-03d32b4b7349ceac506ca889be4772fe8239ecd8-{{checksum ".buildcache/cache-keys/package-03d32b4b7349ceac506ca889be4772fe8239ecd8"}} + key: package-524f6510ca269c805e992fa740ae8cd26db1b091-{{checksum ".buildcache/cache-keys/package-524f6510ca269c805e992fa740ae8cd26db1b091"}} name: Save package cache environment: - PACKAGE_SPEC_ID: 03d32b4b7349ceac506ca889be4772fe8239ecd8 + PACKAGE_SPEC_ID: 524f6510ca269c805e992fa740ae8cd26db1b091 linux_arm_package: executor: builder steps: @@ -623,7 +623,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-ab3b6922ce2bd0fab200dddd6127544427b89fef-{{checksum ".buildcache/cache-keys/package-ab3b6922ce2bd0fab200dddd6127544427b89fef"}} + key: package-dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175-{{checksum ".buildcache/cache-keys/package-dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175"}} name: Restore package cache - run: command: |2 @@ -639,7 +639,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -671,10 +671,10 @@ jobs: - save_cache: paths: - .buildcache/packages/store - key: package-ab3b6922ce2bd0fab200dddd6127544427b89fef-{{checksum ".buildcache/cache-keys/package-ab3b6922ce2bd0fab200dddd6127544427b89fef"}} + key: package-dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175-{{checksum ".buildcache/cache-keys/package-dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175"}} name: Save package cache environment: - PACKAGE_SPEC_ID: ab3b6922ce2bd0fab200dddd6127544427b89fef + PACKAGE_SPEC_ID: dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175 linux_arm64_package: executor: builder steps: @@ -690,7 +690,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-7f1216479517d6c6122fd8928ff44c046eb3a377-{{checksum ".buildcache/cache-keys/package-7f1216479517d6c6122fd8928ff44c046eb3a377"}} + key: package-d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2-{{checksum ".buildcache/cache-keys/package-d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2"}} name: Restore package cache - run: command: |2 @@ -706,7 +706,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -738,10 +738,10 @@ jobs: - save_cache: paths: - .buildcache/packages/store - key: package-7f1216479517d6c6122fd8928ff44c046eb3a377-{{checksum ".buildcache/cache-keys/package-7f1216479517d6c6122fd8928ff44c046eb3a377"}} + key: package-d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2-{{checksum ".buildcache/cache-keys/package-d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2"}} name: Save package cache environment: - PACKAGE_SPEC_ID: 7f1216479517d6c6122fd8928ff44c046eb3a377 + PACKAGE_SPEC_ID: d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2 netbsd_386_package: executor: builder steps: @@ -757,7 +757,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-7fae91402a49def23c2453fe147c262f1377eac3-{{checksum ".buildcache/cache-keys/package-7fae91402a49def23c2453fe147c262f1377eac3"}} + key: package-15db6440776ae92411d314100f394e7d59eb54dc-{{checksum ".buildcache/cache-keys/package-15db6440776ae92411d314100f394e7d59eb54dc"}} name: Restore package cache - run: command: |2 @@ -773,7 +773,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -805,10 +805,10 @@ jobs: - save_cache: paths: - .buildcache/packages/store - key: package-7fae91402a49def23c2453fe147c262f1377eac3-{{checksum ".buildcache/cache-keys/package-7fae91402a49def23c2453fe147c262f1377eac3"}} + key: package-15db6440776ae92411d314100f394e7d59eb54dc-{{checksum ".buildcache/cache-keys/package-15db6440776ae92411d314100f394e7d59eb54dc"}} name: Save package cache environment: - PACKAGE_SPEC_ID: 7fae91402a49def23c2453fe147c262f1377eac3 + PACKAGE_SPEC_ID: 15db6440776ae92411d314100f394e7d59eb54dc netbsd_amd64_package: executor: builder steps: @@ -824,7 +824,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8-{{checksum ".buildcache/cache-keys/package-c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8"}} + key: package-0b0af5839c026648f6a9bee77d6769df9b76fd6c-{{checksum ".buildcache/cache-keys/package-0b0af5839c026648f6a9bee77d6769df9b76fd6c"}} name: Restore package cache - run: command: |2 @@ -840,7 +840,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -872,10 +872,10 @@ jobs: - save_cache: paths: - .buildcache/packages/store - key: package-c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8-{{checksum ".buildcache/cache-keys/package-c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8"}} + key: package-0b0af5839c026648f6a9bee77d6769df9b76fd6c-{{checksum ".buildcache/cache-keys/package-0b0af5839c026648f6a9bee77d6769df9b76fd6c"}} name: Save package cache environment: - PACKAGE_SPEC_ID: c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8 + PACKAGE_SPEC_ID: 0b0af5839c026648f6a9bee77d6769df9b76fd6c openbsd_386_package: executor: builder steps: @@ -891,7 +891,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-b0201f2d2d1b04f519e0d091fbe23997252c5f70-{{checksum ".buildcache/cache-keys/package-b0201f2d2d1b04f519e0d091fbe23997252c5f70"}} + key: package-053de88ba7c9c7d3d7021774a040b4dba0edb1d3-{{checksum ".buildcache/cache-keys/package-053de88ba7c9c7d3d7021774a040b4dba0edb1d3"}} name: Restore package cache - run: command: |2 @@ -907,7 +907,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -939,10 +939,10 @@ jobs: - save_cache: paths: - .buildcache/packages/store - key: package-b0201f2d2d1b04f519e0d091fbe23997252c5f70-{{checksum ".buildcache/cache-keys/package-b0201f2d2d1b04f519e0d091fbe23997252c5f70"}} + key: package-053de88ba7c9c7d3d7021774a040b4dba0edb1d3-{{checksum ".buildcache/cache-keys/package-053de88ba7c9c7d3d7021774a040b4dba0edb1d3"}} name: Save package cache environment: - PACKAGE_SPEC_ID: b0201f2d2d1b04f519e0d091fbe23997252c5f70 + PACKAGE_SPEC_ID: 053de88ba7c9c7d3d7021774a040b4dba0edb1d3 openbsd_amd64_package: executor: builder steps: @@ -958,7 +958,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-51ea70d1d613481fac829f239f1f081c5ad6c88b-{{checksum ".buildcache/cache-keys/package-51ea70d1d613481fac829f239f1f081c5ad6c88b"}} + key: package-34a4409cf043f4e9aedbb4ae8e68db90fa0552a0-{{checksum ".buildcache/cache-keys/package-34a4409cf043f4e9aedbb4ae8e68db90fa0552a0"}} name: Restore package cache - run: command: |2 @@ -974,7 +974,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -1006,10 +1006,10 @@ jobs: - save_cache: paths: - .buildcache/packages/store - key: package-51ea70d1d613481fac829f239f1f081c5ad6c88b-{{checksum ".buildcache/cache-keys/package-51ea70d1d613481fac829f239f1f081c5ad6c88b"}} + key: package-34a4409cf043f4e9aedbb4ae8e68db90fa0552a0-{{checksum ".buildcache/cache-keys/package-34a4409cf043f4e9aedbb4ae8e68db90fa0552a0"}} name: Save package cache environment: - PACKAGE_SPEC_ID: 51ea70d1d613481fac829f239f1f081c5ad6c88b + PACKAGE_SPEC_ID: 34a4409cf043f4e9aedbb4ae8e68db90fa0552a0 solaris_amd64_package: executor: builder steps: @@ -1025,7 +1025,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-19efae459acdaea98d804c26c554640ab3544553-{{checksum ".buildcache/cache-keys/package-19efae459acdaea98d804c26c554640ab3544553"}} + key: package-6d1801298f017e462e056af69b69e5b2f5fd2143-{{checksum ".buildcache/cache-keys/package-6d1801298f017e462e056af69b69e5b2f5fd2143"}} name: Restore package cache - run: command: |2 @@ -1041,7 +1041,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -1073,10 +1073,10 @@ jobs: - save_cache: paths: - .buildcache/packages/store - key: package-19efae459acdaea98d804c26c554640ab3544553-{{checksum ".buildcache/cache-keys/package-19efae459acdaea98d804c26c554640ab3544553"}} + key: package-6d1801298f017e462e056af69b69e5b2f5fd2143-{{checksum ".buildcache/cache-keys/package-6d1801298f017e462e056af69b69e5b2f5fd2143"}} name: Save package cache environment: - PACKAGE_SPEC_ID: 19efae459acdaea98d804c26c554640ab3544553 + PACKAGE_SPEC_ID: 6d1801298f017e462e056af69b69e5b2f5fd2143 windows_386_package: executor: builder steps: @@ -1092,7 +1092,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-44f600485ac3823fa4e0dbba45763d932187225a-{{checksum ".buildcache/cache-keys/package-44f600485ac3823fa4e0dbba45763d932187225a"}} + key: package-c380c1e5206b2d02d9603a8cc068933436e05eb0-{{checksum ".buildcache/cache-keys/package-c380c1e5206b2d02d9603a8cc068933436e05eb0"}} name: Restore package cache - run: command: |2 @@ -1108,7 +1108,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -1140,10 +1140,10 @@ jobs: - save_cache: paths: - .buildcache/packages/store - key: package-44f600485ac3823fa4e0dbba45763d932187225a-{{checksum ".buildcache/cache-keys/package-44f600485ac3823fa4e0dbba45763d932187225a"}} + key: package-c380c1e5206b2d02d9603a8cc068933436e05eb0-{{checksum ".buildcache/cache-keys/package-c380c1e5206b2d02d9603a8cc068933436e05eb0"}} name: Save package cache environment: - PACKAGE_SPEC_ID: 44f600485ac3823fa4e0dbba45763d932187225a + PACKAGE_SPEC_ID: c380c1e5206b2d02d9603a8cc068933436e05eb0 windows_amd64_package: executor: builder steps: @@ -1159,7 +1159,7 @@ jobs: command: make -C packages*.lock write-package-cache-key name: Write package cache key - restore_cache: - key: package-d10bc84044e1ad705f48cd51591b05ee5e1e7dd5-{{checksum ".buildcache/cache-keys/package-d10bc84044e1ad705f48cd51591b05ee5e1e7dd5"}} + key: package-1df2a38d32be613f8179c22ffd0bf03c68f1a16d-{{checksum ".buildcache/cache-keys/package-1df2a38d32be613f8179c22ffd0bf03c68f1a16d"}} name: Restore package cache - run: command: |2 @@ -1175,7 +1175,7 @@ jobs: command: make -C packages*.lock write-builder-cache-keys name: Write builder layer cache keys - restore_cache: - key: copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + key: copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} keys: - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} @@ -1207,12 +1207,12 @@ jobs: - save_cache: paths: - .buildcache/packages/store - key: package-d10bc84044e1ad705f48cd51591b05ee5e1e7dd5-{{checksum ".buildcache/cache-keys/package-d10bc84044e1ad705f48cd51591b05ee5e1e7dd5"}} + key: package-1df2a38d32be613f8179c22ffd0bf03c68f1a16d-{{checksum ".buildcache/cache-keys/package-1df2a38d32be613f8179c22ffd0bf03c68f1a16d"}} name: Save package cache environment: - PACKAGE_SPEC_ID: d10bc84044e1ad705f48cd51591b05ee5e1e7dd5 + PACKAGE_SPEC_ID: 1df2a38d32be613f8179c22ffd0bf03c68f1a16d workflows: - build-645fb387fb0a87de: + build-84a71f97438578ee: jobs: - build-common-layers: {} - darwin_amd64_package: diff --git a/packages-oss.lock/layers/layers.mk b/packages-oss.lock/layers/layers.mk index 033224407..918e96c6d 100755 --- a/packages-oss.lock/layers/layers.mk +++ b/packages-oss.lock/layers/layers.mk @@ -29,11 +29,11 @@ LAYER_02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e_CACHE_KEY_FILE := . LAYER_02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e_ARCHIVE_FILE := .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz $(eval $(call LAYER,$(LAYER_02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e_ID),$(LAYER_02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e_TYPE),$(LAYER_02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e_BASE_LAYER),$(LAYER_02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e_SOURCE_INCLUDE),$(LAYER_02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e_SOURCE_EXCLUDE),$(LAYER_02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e_CACHE_KEY_FILE),$(LAYER_02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e_ARCHIVE_FILE))) -LAYER_03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7_ID := 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 -LAYER_03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7_TYPE := copy-source -LAYER_03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7_BASE_LAYER := 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e -LAYER_03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7_SOURCE_INCLUDE := *.go -LAYER_03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7_SOURCE_EXCLUDE := -LAYER_03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7_CACHE_KEY_FILE := .buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 -LAYER_03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7_ARCHIVE_FILE := .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz -$(eval $(call LAYER,$(LAYER_03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7_ID),$(LAYER_03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7_TYPE),$(LAYER_03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7_BASE_LAYER),$(LAYER_03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7_SOURCE_INCLUDE),$(LAYER_03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7_SOURCE_EXCLUDE),$(LAYER_03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7_CACHE_KEY_FILE),$(LAYER_03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7_ARCHIVE_FILE))) +LAYER_03-copy-source-e59a64ba925a852a09535a8d252147336d635eff_ID := 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff +LAYER_03-copy-source-e59a64ba925a852a09535a8d252147336d635eff_TYPE := copy-source +LAYER_03-copy-source-e59a64ba925a852a09535a8d252147336d635eff_BASE_LAYER := 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e +LAYER_03-copy-source-e59a64ba925a852a09535a8d252147336d635eff_SOURCE_INCLUDE := *.go *.up.sql +LAYER_03-copy-source-e59a64ba925a852a09535a8d252147336d635eff_SOURCE_EXCLUDE := +LAYER_03-copy-source-e59a64ba925a852a09535a8d252147336d635eff_CACHE_KEY_FILE := .buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff +LAYER_03-copy-source-e59a64ba925a852a09535a8d252147336d635eff_ARCHIVE_FILE := .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz +$(eval $(call LAYER,$(LAYER_03-copy-source-e59a64ba925a852a09535a8d252147336d635eff_ID),$(LAYER_03-copy-source-e59a64ba925a852a09535a8d252147336d635eff_TYPE),$(LAYER_03-copy-source-e59a64ba925a852a09535a8d252147336d635eff_BASE_LAYER),$(LAYER_03-copy-source-e59a64ba925a852a09535a8d252147336d635eff_SOURCE_INCLUDE),$(LAYER_03-copy-source-e59a64ba925a852a09535a8d252147336d635eff_SOURCE_EXCLUDE),$(LAYER_03-copy-source-e59a64ba925a852a09535a8d252147336d635eff_CACHE_KEY_FILE),$(LAYER_03-copy-source-e59a64ba925a852a09535a8d252147336d635eff_ARCHIVE_FILE))) diff --git a/packages-oss.lock/pkgs.yml b/packages-oss.lock/pkgs.yml index 8a21e83b1..ea6cdee96 100644 --- a/packages-oss.lock/pkgs.yml +++ b/packages-oss.lock/pkgs.yml @@ -1,11 +1,11 @@ # *** # WARNING: Do not EDIT or MERGE this file, it is generated by packagespec. # *** -lockid: 645fb387fb0a87de +lockid: 84a71f97438578ee packagespec-version: 0.2.6 cache-version: 10 packages: - - packagespecid: 2bb9488232c945d6800cc3a4a461d9b17503bccc + - packagespecid: 9801fb26ed9eb42980684234791a85a8d16c2713 inputs: BINARY_NAME: boundary BUILD_TAGS: ui @@ -22,8 +22,8 @@ packages: builtin: BUILD_LAYERS: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -33,22 +33,22 @@ packages: - type: base name: 00-base-2017dceda969e0eab6fd00cab602c9a05538de9c archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz - PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-2bb9488232c945d6800cc3a4a461d9b17503bccc + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-9801fb26ed9eb42980684234791a85a8d16c2713 POST_PROCESSORS: apple-notarize av-scan circleci: BUILDER_CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} - PACKAGE_CACHE_KEY: package-2bb9488232c945d6800cc3a4a461d9b17503bccc-{{checksum ".buildcache/cache-keys/package-2bb9488232c945d6800cc3a4a461d9b17503bccc"}} + PACKAGE_CACHE_KEY: package-9801fb26ed9eb42980684234791a85a8d16c2713-{{checksum ".buildcache/cache-keys/package-9801fb26ed9eb42980684234791a85a8d16c2713"}} build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID -X $VERSION_PKG_PATH.Version=0.6.2 -X $VERSION_PKG_PATH.VersionPrerelease=" -o $OUTPUT_DIR/boundary ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME boundary aliases: - type: local path: boundary_0.6.2_darwin_amd64.zip - type: public-hc-releases path: boundary/boundary_0.6.2/boundary_0.6.2_darwin_amd64.zip - - packagespecid: e9f9d34c2c14f59f68c31469b7bfa994074a48c2 + - packagespecid: c6687b4dfea47101b46520e0809c4e4135a9a3b8 inputs: BINARY_NAME: boundary BUILD_TAGS: ui @@ -65,8 +65,8 @@ packages: builtin: BUILD_LAYERS: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -76,22 +76,22 @@ packages: - type: base name: 00-base-2017dceda969e0eab6fd00cab602c9a05538de9c archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz - PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-e9f9d34c2c14f59f68c31469b7bfa994074a48c2 + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-c6687b4dfea47101b46520e0809c4e4135a9a3b8 POST_PROCESSORS: av-scan circleci: BUILDER_CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} - PACKAGE_CACHE_KEY: package-e9f9d34c2c14f59f68c31469b7bfa994074a48c2-{{checksum ".buildcache/cache-keys/package-e9f9d34c2c14f59f68c31469b7bfa994074a48c2"}} + PACKAGE_CACHE_KEY: package-c6687b4dfea47101b46520e0809c4e4135a9a3b8-{{checksum ".buildcache/cache-keys/package-c6687b4dfea47101b46520e0809c4e4135a9a3b8"}} build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID -X $VERSION_PKG_PATH.Version=0.6.2 -X $VERSION_PKG_PATH.VersionPrerelease=" -o $OUTPUT_DIR/boundary ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME boundary aliases: - type: local path: boundary_0.6.2_freebsd_386.zip - type: public-hc-releases path: boundary/boundary_0.6.2/boundary_0.6.2_freebsd_386.zip - - packagespecid: 3e2a1075872eae7608bf8be4b74f7291ca88d5b5 + - packagespecid: 3d40792896ab78cd0e160df4bbd1c757c63cf985 inputs: BINARY_NAME: boundary BUILD_TAGS: ui @@ -108,8 +108,8 @@ packages: builtin: BUILD_LAYERS: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -119,22 +119,22 @@ packages: - type: base name: 00-base-2017dceda969e0eab6fd00cab602c9a05538de9c archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz - PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-3e2a1075872eae7608bf8be4b74f7291ca88d5b5 + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-3d40792896ab78cd0e160df4bbd1c757c63cf985 POST_PROCESSORS: av-scan circleci: BUILDER_CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} - PACKAGE_CACHE_KEY: package-3e2a1075872eae7608bf8be4b74f7291ca88d5b5-{{checksum ".buildcache/cache-keys/package-3e2a1075872eae7608bf8be4b74f7291ca88d5b5"}} + PACKAGE_CACHE_KEY: package-3d40792896ab78cd0e160df4bbd1c757c63cf985-{{checksum ".buildcache/cache-keys/package-3d40792896ab78cd0e160df4bbd1c757c63cf985"}} build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID -X $VERSION_PKG_PATH.Version=0.6.2 -X $VERSION_PKG_PATH.VersionPrerelease=" -o $OUTPUT_DIR/boundary ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME boundary aliases: - type: local path: boundary_0.6.2_freebsd_amd64.zip - type: public-hc-releases path: boundary/boundary_0.6.2/boundary_0.6.2_freebsd_amd64.zip - - packagespecid: 822b1aa9954becb7a9cb480784ff9f99783a8d23 + - packagespecid: fb9d5eb085b094774ff0f514490c9b8a5da28e24 inputs: BINARY_NAME: boundary BUILD_TAGS: ui @@ -151,8 +151,8 @@ packages: builtin: BUILD_LAYERS: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -162,22 +162,22 @@ packages: - type: base name: 00-base-2017dceda969e0eab6fd00cab602c9a05538de9c archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz - PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-822b1aa9954becb7a9cb480784ff9f99783a8d23 + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-fb9d5eb085b094774ff0f514490c9b8a5da28e24 POST_PROCESSORS: av-scan circleci: BUILDER_CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} - PACKAGE_CACHE_KEY: package-822b1aa9954becb7a9cb480784ff9f99783a8d23-{{checksum ".buildcache/cache-keys/package-822b1aa9954becb7a9cb480784ff9f99783a8d23"}} + PACKAGE_CACHE_KEY: package-fb9d5eb085b094774ff0f514490c9b8a5da28e24-{{checksum ".buildcache/cache-keys/package-fb9d5eb085b094774ff0f514490c9b8a5da28e24"}} build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID -X $VERSION_PKG_PATH.Version=0.6.2 -X $VERSION_PKG_PATH.VersionPrerelease=" -o $OUTPUT_DIR/boundary ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME boundary aliases: - type: local path: boundary_0.6.2_freebsd_arm.zip - type: public-hc-releases path: boundary/boundary_0.6.2/boundary_0.6.2_freebsd_arm.zip - - packagespecid: 7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a + - packagespecid: 701ab62817cc79f8ec83299cefac126cb3c74d47 inputs: BINARY_NAME: boundary BUILD_TAGS: ui @@ -194,8 +194,8 @@ packages: builtin: BUILD_LAYERS: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -205,22 +205,22 @@ packages: - type: base name: 00-base-2017dceda969e0eab6fd00cab602c9a05538de9c archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz - PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-701ab62817cc79f8ec83299cefac126cb3c74d47 POST_PROCESSORS: av-scan circleci: BUILDER_CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} - PACKAGE_CACHE_KEY: package-7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a-{{checksum ".buildcache/cache-keys/package-7fd6bd8ba41907a8ad8d09a9ed3c43fb94801f2a"}} + PACKAGE_CACHE_KEY: package-701ab62817cc79f8ec83299cefac126cb3c74d47-{{checksum ".buildcache/cache-keys/package-701ab62817cc79f8ec83299cefac126cb3c74d47"}} build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID -X $VERSION_PKG_PATH.Version=0.6.2 -X $VERSION_PKG_PATH.VersionPrerelease=" -o $OUTPUT_DIR/boundary ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME boundary aliases: - type: local path: boundary_0.6.2_linux_386.zip - type: public-hc-releases path: boundary/boundary_0.6.2/boundary_0.6.2_linux_386.zip - - packagespecid: 03d32b4b7349ceac506ca889be4772fe8239ecd8 + - packagespecid: 524f6510ca269c805e992fa740ae8cd26db1b091 inputs: BINARY_NAME: boundary BUILD_TAGS: ui @@ -237,8 +237,8 @@ packages: builtin: BUILD_LAYERS: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -248,22 +248,22 @@ packages: - type: base name: 00-base-2017dceda969e0eab6fd00cab602c9a05538de9c archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz - PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-03d32b4b7349ceac506ca889be4772fe8239ecd8 + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-524f6510ca269c805e992fa740ae8cd26db1b091 POST_PROCESSORS: av-scan circleci: BUILDER_CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} - PACKAGE_CACHE_KEY: package-03d32b4b7349ceac506ca889be4772fe8239ecd8-{{checksum ".buildcache/cache-keys/package-03d32b4b7349ceac506ca889be4772fe8239ecd8"}} + PACKAGE_CACHE_KEY: package-524f6510ca269c805e992fa740ae8cd26db1b091-{{checksum ".buildcache/cache-keys/package-524f6510ca269c805e992fa740ae8cd26db1b091"}} build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID -X $VERSION_PKG_PATH.Version=0.6.2 -X $VERSION_PKG_PATH.VersionPrerelease=" -o $OUTPUT_DIR/boundary ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME boundary aliases: - type: local path: boundary_0.6.2_linux_amd64.zip - type: public-hc-releases path: boundary/boundary_0.6.2/boundary_0.6.2_linux_amd64.zip - - packagespecid: ab3b6922ce2bd0fab200dddd6127544427b89fef + - packagespecid: dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175 inputs: BINARY_NAME: boundary BUILD_TAGS: ui @@ -280,8 +280,8 @@ packages: builtin: BUILD_LAYERS: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -291,22 +291,22 @@ packages: - type: base name: 00-base-2017dceda969e0eab6fd00cab602c9a05538de9c archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz - PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-ab3b6922ce2bd0fab200dddd6127544427b89fef + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175 POST_PROCESSORS: av-scan circleci: BUILDER_CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} - PACKAGE_CACHE_KEY: package-ab3b6922ce2bd0fab200dddd6127544427b89fef-{{checksum ".buildcache/cache-keys/package-ab3b6922ce2bd0fab200dddd6127544427b89fef"}} + PACKAGE_CACHE_KEY: package-dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175-{{checksum ".buildcache/cache-keys/package-dc756c7b14fbf4ade5d1b63f03d2ca186f5eb175"}} build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID -X $VERSION_PKG_PATH.Version=0.6.2 -X $VERSION_PKG_PATH.VersionPrerelease=" -o $OUTPUT_DIR/boundary ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME boundary aliases: - type: local path: boundary_0.6.2_linux_arm.zip - type: public-hc-releases path: boundary/boundary_0.6.2/boundary_0.6.2_linux_arm.zip - - packagespecid: 7f1216479517d6c6122fd8928ff44c046eb3a377 + - packagespecid: d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2 inputs: BINARY_NAME: boundary BUILD_TAGS: ui @@ -323,8 +323,8 @@ packages: builtin: BUILD_LAYERS: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -334,22 +334,22 @@ packages: - type: base name: 00-base-2017dceda969e0eab6fd00cab602c9a05538de9c archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz - PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-7f1216479517d6c6122fd8928ff44c046eb3a377 + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2 POST_PROCESSORS: av-scan circleci: BUILDER_CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} - PACKAGE_CACHE_KEY: package-7f1216479517d6c6122fd8928ff44c046eb3a377-{{checksum ".buildcache/cache-keys/package-7f1216479517d6c6122fd8928ff44c046eb3a377"}} + PACKAGE_CACHE_KEY: package-d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2-{{checksum ".buildcache/cache-keys/package-d0d49ef910d2c6f6556cd3e86b36936b0d7c2ac2"}} build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID -X $VERSION_PKG_PATH.Version=0.6.2 -X $VERSION_PKG_PATH.VersionPrerelease=" -o $OUTPUT_DIR/boundary ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME boundary aliases: - type: local path: boundary_0.6.2_linux_arm64.zip - type: public-hc-releases path: boundary/boundary_0.6.2/boundary_0.6.2_linux_arm64.zip - - packagespecid: 7fae91402a49def23c2453fe147c262f1377eac3 + - packagespecid: 15db6440776ae92411d314100f394e7d59eb54dc inputs: BINARY_NAME: boundary BUILD_TAGS: ui @@ -366,8 +366,8 @@ packages: builtin: BUILD_LAYERS: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -377,22 +377,22 @@ packages: - type: base name: 00-base-2017dceda969e0eab6fd00cab602c9a05538de9c archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz - PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-7fae91402a49def23c2453fe147c262f1377eac3 + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-15db6440776ae92411d314100f394e7d59eb54dc POST_PROCESSORS: av-scan circleci: BUILDER_CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} - PACKAGE_CACHE_KEY: package-7fae91402a49def23c2453fe147c262f1377eac3-{{checksum ".buildcache/cache-keys/package-7fae91402a49def23c2453fe147c262f1377eac3"}} + PACKAGE_CACHE_KEY: package-15db6440776ae92411d314100f394e7d59eb54dc-{{checksum ".buildcache/cache-keys/package-15db6440776ae92411d314100f394e7d59eb54dc"}} build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID -X $VERSION_PKG_PATH.Version=0.6.2 -X $VERSION_PKG_PATH.VersionPrerelease=" -o $OUTPUT_DIR/boundary ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME boundary aliases: - type: local path: boundary_0.6.2_netbsd_386.zip - type: public-hc-releases path: boundary/boundary_0.6.2/boundary_0.6.2_netbsd_386.zip - - packagespecid: c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8 + - packagespecid: 0b0af5839c026648f6a9bee77d6769df9b76fd6c inputs: BINARY_NAME: boundary BUILD_TAGS: ui @@ -409,8 +409,8 @@ packages: builtin: BUILD_LAYERS: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -420,22 +420,22 @@ packages: - type: base name: 00-base-2017dceda969e0eab6fd00cab602c9a05538de9c archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz - PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8 + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-0b0af5839c026648f6a9bee77d6769df9b76fd6c POST_PROCESSORS: av-scan circleci: BUILDER_CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} - PACKAGE_CACHE_KEY: package-c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8-{{checksum ".buildcache/cache-keys/package-c9b8b9c5b6a1ab2f53599efe3792f7ebbd43a6b8"}} + PACKAGE_CACHE_KEY: package-0b0af5839c026648f6a9bee77d6769df9b76fd6c-{{checksum ".buildcache/cache-keys/package-0b0af5839c026648f6a9bee77d6769df9b76fd6c"}} build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID -X $VERSION_PKG_PATH.Version=0.6.2 -X $VERSION_PKG_PATH.VersionPrerelease=" -o $OUTPUT_DIR/boundary ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME boundary aliases: - type: local path: boundary_0.6.2_netbsd_amd64.zip - type: public-hc-releases path: boundary/boundary_0.6.2/boundary_0.6.2_netbsd_amd64.zip - - packagespecid: b0201f2d2d1b04f519e0d091fbe23997252c5f70 + - packagespecid: 053de88ba7c9c7d3d7021774a040b4dba0edb1d3 inputs: BINARY_NAME: boundary BUILD_TAGS: ui @@ -452,8 +452,8 @@ packages: builtin: BUILD_LAYERS: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -463,22 +463,22 @@ packages: - type: base name: 00-base-2017dceda969e0eab6fd00cab602c9a05538de9c archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz - PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-b0201f2d2d1b04f519e0d091fbe23997252c5f70 + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-053de88ba7c9c7d3d7021774a040b4dba0edb1d3 POST_PROCESSORS: av-scan circleci: BUILDER_CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} - PACKAGE_CACHE_KEY: package-b0201f2d2d1b04f519e0d091fbe23997252c5f70-{{checksum ".buildcache/cache-keys/package-b0201f2d2d1b04f519e0d091fbe23997252c5f70"}} + PACKAGE_CACHE_KEY: package-053de88ba7c9c7d3d7021774a040b4dba0edb1d3-{{checksum ".buildcache/cache-keys/package-053de88ba7c9c7d3d7021774a040b4dba0edb1d3"}} build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID -X $VERSION_PKG_PATH.Version=0.6.2 -X $VERSION_PKG_PATH.VersionPrerelease=" -o $OUTPUT_DIR/boundary ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME boundary aliases: - type: local path: boundary_0.6.2_openbsd_386.zip - type: public-hc-releases path: boundary/boundary_0.6.2/boundary_0.6.2_openbsd_386.zip - - packagespecid: 51ea70d1d613481fac829f239f1f081c5ad6c88b + - packagespecid: 34a4409cf043f4e9aedbb4ae8e68db90fa0552a0 inputs: BINARY_NAME: boundary BUILD_TAGS: ui @@ -495,8 +495,8 @@ packages: builtin: BUILD_LAYERS: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -506,22 +506,22 @@ packages: - type: base name: 00-base-2017dceda969e0eab6fd00cab602c9a05538de9c archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz - PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-51ea70d1d613481fac829f239f1f081c5ad6c88b + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-34a4409cf043f4e9aedbb4ae8e68db90fa0552a0 POST_PROCESSORS: av-scan circleci: BUILDER_CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} - PACKAGE_CACHE_KEY: package-51ea70d1d613481fac829f239f1f081c5ad6c88b-{{checksum ".buildcache/cache-keys/package-51ea70d1d613481fac829f239f1f081c5ad6c88b"}} + PACKAGE_CACHE_KEY: package-34a4409cf043f4e9aedbb4ae8e68db90fa0552a0-{{checksum ".buildcache/cache-keys/package-34a4409cf043f4e9aedbb4ae8e68db90fa0552a0"}} build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID -X $VERSION_PKG_PATH.Version=0.6.2 -X $VERSION_PKG_PATH.VersionPrerelease=" -o $OUTPUT_DIR/boundary ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME boundary aliases: - type: local path: boundary_0.6.2_openbsd_amd64.zip - type: public-hc-releases path: boundary/boundary_0.6.2/boundary_0.6.2_openbsd_amd64.zip - - packagespecid: 19efae459acdaea98d804c26c554640ab3544553 + - packagespecid: 6d1801298f017e462e056af69b69e5b2f5fd2143 inputs: BINARY_NAME: boundary BUILD_TAGS: ui @@ -538,8 +538,8 @@ packages: builtin: BUILD_LAYERS: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -549,22 +549,22 @@ packages: - type: base name: 00-base-2017dceda969e0eab6fd00cab602c9a05538de9c archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz - PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-19efae459acdaea98d804c26c554640ab3544553 + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-6d1801298f017e462e056af69b69e5b2f5fd2143 POST_PROCESSORS: av-scan circleci: BUILDER_CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} - PACKAGE_CACHE_KEY: package-19efae459acdaea98d804c26c554640ab3544553-{{checksum ".buildcache/cache-keys/package-19efae459acdaea98d804c26c554640ab3544553"}} + PACKAGE_CACHE_KEY: package-6d1801298f017e462e056af69b69e5b2f5fd2143-{{checksum ".buildcache/cache-keys/package-6d1801298f017e462e056af69b69e5b2f5fd2143"}} build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID -X $VERSION_PKG_PATH.Version=0.6.2 -X $VERSION_PKG_PATH.VersionPrerelease=" -o $OUTPUT_DIR/boundary ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME boundary aliases: - type: local path: boundary_0.6.2_solaris_amd64.zip - type: public-hc-releases path: boundary/boundary_0.6.2/boundary_0.6.2_solaris_amd64.zip - - packagespecid: 44f600485ac3823fa4e0dbba45763d932187225a + - packagespecid: c380c1e5206b2d02d9603a8cc068933436e05eb0 inputs: BINARY_NAME: boundary.exe BUILD_TAGS: ui @@ -581,8 +581,8 @@ packages: builtin: BUILD_LAYERS: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -592,22 +592,22 @@ packages: - type: base name: 00-base-2017dceda969e0eab6fd00cab602c9a05538de9c archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz - PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-44f600485ac3823fa4e0dbba45763d932187225a + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-c380c1e5206b2d02d9603a8cc068933436e05eb0 POST_PROCESSORS: microsoft-notarize av-scan circleci: BUILDER_CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} - PACKAGE_CACHE_KEY: package-44f600485ac3823fa4e0dbba45763d932187225a-{{checksum ".buildcache/cache-keys/package-44f600485ac3823fa4e0dbba45763d932187225a"}} + PACKAGE_CACHE_KEY: package-c380c1e5206b2d02d9603a8cc068933436e05eb0-{{checksum ".buildcache/cache-keys/package-c380c1e5206b2d02d9603a8cc068933436e05eb0"}} build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID -X $VERSION_PKG_PATH.Version=0.6.2 -X $VERSION_PKG_PATH.VersionPrerelease=" -o $OUTPUT_DIR/boundary.exe ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME boundary.exe aliases: - type: local path: boundary_0.6.2_windows_386.zip - type: public-hc-releases path: boundary/boundary_0.6.2/boundary_0.6.2_windows_386.zip - - packagespecid: d10bc84044e1ad705f48cd51591b05ee5e1e7dd5 + - packagespecid: 1df2a38d32be613f8179c22ffd0bf03c68f1a16d inputs: BINARY_NAME: boundary.exe BUILD_TAGS: ui @@ -624,8 +624,8 @@ packages: builtin: BUILD_LAYERS: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -635,15 +635,15 @@ packages: - type: base name: 00-base-2017dceda969e0eab6fd00cab602c9a05538de9c archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz - PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-d10bc84044e1ad705f48cd51591b05ee5e1e7dd5 + PACKAGE_CACHE_KEY_FILE: .buildcache/cache-keys/package-1df2a38d32be613f8179c22ffd0bf03c68f1a16d POST_PROCESSORS: microsoft-notarize av-scan circleci: BUILDER_CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} - PACKAGE_CACHE_KEY: package-d10bc84044e1ad705f48cd51591b05ee5e1e7dd5-{{checksum ".buildcache/cache-keys/package-d10bc84044e1ad705f48cd51591b05ee5e1e7dd5"}} + PACKAGE_CACHE_KEY: package-1df2a38d32be613f8179c22ffd0bf03c68f1a16d-{{checksum ".buildcache/cache-keys/package-1df2a38d32be613f8179c22ffd0bf03c68f1a16d"}} build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH; go build -v -tags 'ui' -ldflags "-X $VERSION_PKG_PATH.GitCommit=$PACKAGE_SOURCE_ID -X $VERSION_PKG_PATH.Version=0.6.2 -X $VERSION_PKG_PATH.VersionPrerelease=" -o $OUTPUT_DIR/boundary.exe ./cmd/boundary && cd $OUTPUT_DIR && zip $PACKAGE_ZIP_NAME boundary.exe aliases: - type: local @@ -773,15 +773,15 @@ layers: - depth: 3 final: true type: copy-source - id: 5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 + id: e59a64ba925a852a09535a8d252147336d635eff + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff parentname: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e parenttype: go-modules parentid: 0542ebe70aa49ea525c7cd85313d5ba08f90441e - cachekeyfile: .buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - sourceinclude: '*.go' + cachekeyfile: .buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff + sourceinclude: '*.go *.up.sql' sourceexclude: "" - archivefile: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + archivefile: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz dockerfile: | # syntax=docker.mirror.hashicorp.services/docker/dockerfile:1.1.7-experimental ARG BASE_IMAGE @@ -791,8 +791,8 @@ layers: builtin: LAYER_LIST: - type: copy-source - name: 03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7 - archive: .buildcache/archives/03-copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7.tar.gz + name: 03-copy-source-e59a64ba925a852a09535a8d252147336d635eff + archive: .buildcache/archives/03-copy-source-e59a64ba925a852a09535a8d252147336d635eff.tar.gz - type: go-modules name: 02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e archive: .buildcache/archives/02-go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e.tar.gz @@ -804,7 +804,7 @@ layers: archive: .buildcache/archives/00-base-2017dceda969e0eab6fd00cab602c9a05538de9c.tar.gz circleci: CACHE_KEY_PREFIX_LIST: - - copy-source_5f13_{{checksum ".buildcache/cache-keys/copy-source-5f13b2247825f6b1ba38bb75dd62a7a15469efe7"}} + - copy-source_e59a_{{checksum ".buildcache/cache-keys/copy-source-e59a64ba925a852a09535a8d252147336d635eff"}} - go-modules_0542_{{checksum ".buildcache/cache-keys/go-modules-0542ebe70aa49ea525c7cd85313d5ba08f90441e"}} - ui_f43e_{{checksum ".buildcache/cache-keys/ui-f43e20c1d27527cddb9fa2f5a08420e5058396f1"}} - base_2017_{{checksum ".buildcache/cache-keys/base-2017dceda969e0eab6fd00cab602c9a05538de9c"}} diff --git a/packages-oss.yml b/packages-oss.yml index 28b2445ed..2d30eb6e0 100644 --- a/packages-oss.yml +++ b/packages-oss.yml @@ -92,7 +92,7 @@ layers: RUN go mod download - name: copy-source - source-include: "*.go" + source-include: "*.go *.up.sql" build-command: VERSION_PKG_PATH=github.com/hashicorp/boundary/version; unset GOPATH;