feat(schema): Add support for migration hooks

This adds the ability to define hooks that will run prior to a schema
migration. The hooks should only be used when a migration would fail due
to data being in an invalid state, likely due to a bug in the previous
schema definition. For example if a column was missing a constraint, and
a new migration is attempting to add the constraint, there might already
be values that violate the new constraint. In such cases, we would want
the user to make a decision to address the data, and need to provide
them with details about what data is problematic in case they want to
manually address the problem. To help the user, if there is a sane way
to address the data, we should provide an option to automatically
"repair" the data, allowing the migration to continue. However, this
still should be an explicit opt-in from the user, and we need to clearly
describe what the repair option would do to the user so they understand
the consequences of opting in.

To accomplish this each migration can be provided a hook that contains a
CheckFunc, RepairFunc, and RepairDescription. These functions must
operate at the point-in-time of the migration immediately preceding the
current migration, as as such they should not use and domain packages or
repositories, instead they should use direct sql with the sql library.
The description is used to explain what the RepairFunc will do.

Hooks should be defined in `internal/db/schema/migrations/oss/oss.go`
and registered with the schema edition.

See the example in `internal/db/schema/manager_example_test.go` for more
details.
This commit is contained in:
Timothy Messier authored and Damian Debkowski committed 2022-08-22 12:15:20 -07:00
1 parent 1fa79e9aa1
commit 79866a287f
33 files changed
+1256 -156

No files matched your search

+22 -6
View File
@@ -3,6 +3,7 @@ package database
import (
"context"
"fmt"
"strings"
"github.com/hashicorp/boundary/internal/cmd/base"
"github.com/hashicorp/boundary/internal/db/common"
@@ -16,7 +17,7 @@ import (
// We expect the database already to be initialized iff initialized is set to true.
// Returns a cleanup function which must be called even if an error is returned and
// an error code where a non-zero value indicates an error happened.
func migrateDatabase(ctx context.Context, ui cli.Ui, dialect, u string, initialized bool, maxOpenConns int) (func(), int) {
func migrateDatabase(ctx context.Context, ui cli.Ui, dialect, u string, initialized bool, maxOpenConns int, selectedRepairs schema.RepairMigrations) (func(), int) {
noop := func() {}
// This database is used to keep an exclusive lock on the database for the
// remainder of the command
@@ -30,7 +31,7 @@ func migrateDatabase(ctx context.Context, ui cli.Ui, dialect, u string, initiali
ui.Error(fmt.Sprintf("Unable to connect to the database at %q", u))
return noop, 2
}
man, err := schema.NewManager(ctx, schema.Dialect(dialect), dBase)
man, err := schema.NewManager(ctx, schema.Dialect(dialect), dBase, schema.WithRepairMigrations(selectedRepairs))
if err != nil {
if errors.Match(errors.T(errors.MigrationLock), err) {
ui.Error("Unable to capture a lock on the database.")
@@ -62,21 +63,36 @@ func migrateDatabase(ctx context.Context, ui cli.Ui, dialect, u string, initiali
ui.Output(base.WrapAtLength("Database has already been initialized. Please use 'boundary database migrate' for any upgrade needs."))
return unlock, -1
}
if err := man.ApplyMigrations(ctx); err != nil {
repairLogs, err := man.ApplyMigrations(ctx)
if err != nil {
ui.Error(fmt.Errorf("Error running database migrations: %w", err).Error())
if checkErr, ok := err.(schema.MigrationCheckError); ok {
ui.Error(fmt.Errorf("%s", strings.Join(checkErr.Problems, "\n")).Error())
ui.Error(fmt.Sprintf("To automatically repair, use 'boundary database migrate -repair=%s:%d'. This will: %s", checkErr.Edition, checkErr.Version, checkErr.RepairDescription))
}
return unlock, 2
}
if base.Format(ui) == "table" {
ui.Info("Migrations successfully run.")
}
migrationLogs, err := man.GetMigrationLog(ctx)
if len(repairLogs) > 0 && base.Format(ui) == "table" {
ui.Info("Migration Repair logs...")
for _, e := range repairLogs {
ui.Info(fmt.Sprintf("%s:%d:", e.Edition, e.Version))
for _, entry := range e.Entry {
ui.Info(entry)
}
}
}
logs, err := man.GetMigrationLog(ctx)
if err != nil {
ui.Error(fmt.Errorf("Error retrieving database migration logs: %w", err).Error())
return unlock, 2
}
if len(migrationLogs) > 0 && base.Format(ui) == "table" {
if len(logs) > 0 && base.Format(ui) == "table" {
ui.Info("Migration Logs...")
for _, e := range migrationLogs {
for _, e := range logs {
ui.Info(e.Entry)
}
}
+7 -4
View File
@@ -57,7 +57,8 @@ func TestMigrateDatabase(t *testing.T) {
schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": earlyMigrationVersion}),
))
require.NoError(t, err)
require.NoError(t, man.ApplyMigrations(ctx))
_, err = man.ApplyMigrations(ctx)
require.NoError(t, err)
return u
},
expectedCode: 0,
@@ -81,7 +82,8 @@ func TestMigrateDatabase(t *testing.T) {
schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": earlyMigrationVersion}),
))
require.NoError(t, err)
require.NoError(t, man.ApplyMigrations(ctx))
_, err = man.ApplyMigrations(ctx)
require.NoError(t, err)
return u
},
expectedCode: -1,
@@ -193,7 +195,7 @@ func TestMigrateDatabase(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
u := tc.urlProvider()
ui := cli.NewMockUi()
clean, errCode := migrateDatabase(ctx, ui, dialect, u, tc.initialized, 10)
clean, errCode := migrateDatabase(ctx, ui, dialect, u, tc.initialized, 10, nil)
clean()
assert.EqualValues(t, tc.expectedCode, errCode)
assert.Equal(t, tc.expectedOutput, ui.OutputWriter.String())
@@ -218,7 +220,8 @@ func TestVerifyOplogIsEmpty(t *testing.T) {
man, err := schema.NewManager(ctx, schema.Dialect(dialect), dBase)
require.NoError(t, err)
require.NoError(t, man.ApplyMigrations(ctx))
_, err = man.ApplyMigrations(ctx)
require.NoError(t, err)
cmd := InitCommand{Server: base.NewServer(base.NewCommand(cli.NewMockUi()))}
+1 -1
View File
@@ -266,7 +266,7 @@ func (c *InitCommand) Run(args []string) (retCode int) {
return base.CommandUserError
}
clean, errCode := migrateDatabase(c.Context, c.UI, dialect, migrationUrl, false, c.DatabaseMaxOpenConnections)
clean, errCode := migrateDatabase(c.Context, c.UI, dialect, migrationUrl, false, c.DatabaseMaxOpenConnections, nil)
defer clean()
switch errCode {
case 0:
+39 -1
View File
@@ -4,10 +4,13 @@ import (
"context"
"fmt"
"os"
"strconv"
"strings"
"github.com/hashicorp/boundary/globals"
"github.com/hashicorp/boundary/internal/cmd/base"
"github.com/hashicorp/boundary/internal/cmd/config"
"github.com/hashicorp/boundary/internal/db/schema"
"github.com/hashicorp/boundary/internal/errors"
"github.com/hashicorp/boundary/internal/observability/event"
host_plugin_assets "github.com/hashicorp/boundary/plugins/host"
@@ -44,11 +47,14 @@ type MigrateCommand struct {
// deferred function on the Run method.
configWrapperCleanupFunc func() error
selectedRepairs schema.RepairMigrations
flagConfig string
flagConfigKms string
flagLogLevel string
flagLogFormat string
flagMigrationUrl string
flagRepairMigrations []string
flagAllowDevMigrations bool
}
@@ -117,6 +123,12 @@ func (c *MigrateCommand) Flags() *base.FlagSets {
Usage: `If set, overrides a migration URL set in config, and specifies the URL used to connect to the database for migration. This can allow different permissions for the user running initialization or migration vs. normal operation. This can refer to a file on disk (file://) from which a URL will be read; an env var (env://) from which the URL will be read; or a direct database URL.`,
})
f.StringSliceVar(&base.StringSliceVar{
Name: "repair",
Target: &c.flagRepairMigrations,
Usage: `Run the repair function for the provided migration version.`,
})
return set
}
@@ -256,7 +268,15 @@ plugins {
return base.CommandUserError
}
clean, errCode := migrateDatabase(c.Context, c.UI, dialect, migrationUrl, true, c.Config.Controller.Database.MaxOpenConnections)
clean, errCode := migrateDatabase(
c.Context,
c.UI,
dialect,
migrationUrl,
true,
c.Config.Controller.Database.MaxOpenConnections,
c.selectedRepairs,
)
defer clean()
if errCode != 0 {
return errCode
@@ -275,6 +295,24 @@ func (c *MigrateCommand) ParseFlagsAndConfig(args []string) int {
return base.CommandUserError
}
c.selectedRepairs = make(schema.RepairMigrations)
for _, r := range c.flagRepairMigrations {
parts := strings.SplitN(r, ":", 2)
if len(parts) != 2 {
c.UI.Error(fmt.Sprintf("Error parsing repair option, invalid format: %s", r))
return base.CommandUserError
}
edition := parts[0]
version, err := strconv.Atoi(parts[1])
if err != nil {
c.UI.Error(fmt.Sprintf("Error parsing repair option %s, %s", r, err.Error()))
return base.CommandUserError
}
c.selectedRepairs.Add(edition, version)
}
// Validation
switch {
case len(c.flagConfig) == 0: