CyberCoveLLC/threshold
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'tmessi/faster-testing'

Browse Source
This commit is contained in:
Timothy Messier committed 2021-09-10 08:08:02 -04:00
2 parents 811d0b63fd 2ee1d4e6e4
commit 30736b3bcd
38 files changed
+939 -502
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.circleci/config.yml
Generated
+17 -5
View File
@@ -7,7 +7,7 @@ version: 2
jobs:
test-sql-13-alpine:
machine:
image: ubuntu-1604:201903-01
image: ubuntu-2004:202107-02
steps:
- checkout
- run:
@@ -249,7 +249,7 @@ jobs:
- .buildcache/packages/store
test-sql-11-alpine:
machine:
image: ubuntu-1604:201903-01
image: ubuntu-2004:202107-02
steps:
- checkout
- run:
@@ -887,7 +887,7 @@ jobs:
- .buildcache/packages/store
test-sql-12-alpine:
machine:
image: ubuntu-1604:201903-01
image: ubuntu-2004:202107-02
steps:
- checkout
- run:
@@ -969,14 +969,26 @@ jobs:
- .buildcache/packages/store
build:
machine:
image: ubuntu-1604:201903-01
image: ubuntu-2004:202107-02
resource_class: large
steps:
- checkout
- run:
command: |
which pg_isready || sudo apt-get update && sudo apt-get install -y postgresql-client
make DOCKER_ARGS='-d' -C testing/dbtest/docker database-up
until pg_isready -h 127.0.0.1; do docker container inspect boundary-sql-tests &> /dev/null || exit -1; sleep 1; done
name: Initialize Test Database
- run:
command: |
make test-ci
name: Run Acceptance Tests
no_output_timeout: 15m
- run:
command: |
make -C testing/dbtest/docker clean
name: Cleanup
when: always
- run:
command: |
echo 'export SLACK_BUILD_STATUS="fail"' >> $BASH_ENV
@@ -1467,7 +1479,7 @@ jobs:
- .buildcache/packages/store
test-sql-latest:
machine:
image: ubuntu-1604:201903-01
image: ubuntu-2004:202107-02
steps:
- checkout
- run:
.circleci/config/jobs/build.yml
+13 -1
View File
@@ -1,13 +1,25 @@
resource_class: large
machine:
image: 'ubuntu-1604:201903-01'
image: 'ubuntu-2004:202107-02'
working_directory: ~/boundary
steps:
- checkout
- run:
name: "Initialize Test Database"
command: |
which pg_isready || sudo apt-get update && sudo apt-get install -y postgresql-client
make DOCKER_ARGS='-d' -C testing/dbtest/docker database-up
until pg_isready -h 127.0.0.1; do docker container inspect boundary-sql-tests &> /dev/null || exit -1; sleep 1; done
- run:
name: "Run Acceptance Tests"
no_output_timeout: 15m
command: |
make test-ci
- run:
name: "Cleanup"
when: always
command: |
make -C testing/dbtest/docker clean
- slack/status:
fail_only: true
only_for_branches: master
.circleci/config/jobs/test-sql.yml
+1 -1
View File
@@ -1,5 +1,5 @@
machine:
image: 'ubuntu-1604:201903-01'
image: 'ubuntu-2004:202107-02'
working_directory: ~/boundary
parameters:
postgres-version:
CONTRIBUTING.md
+96 -1
View File
@@ -70,7 +70,40 @@ Any bug fixes fall into this section.
## Testing
To run the entire test suite run this command in the root of the project:
Most tests require a postgres database instance to successfully run.
This is provided via docker
by running a customized postgres image that is optimized for boundary tests.
Before running the test suite, this docker container must be started:
```
$ make test-database-up
```
This can take a few seconds to initialize as it will
create a template database with the boundary migrations.
The progress can be checked b running:
```
$ docker logs -f boundary-sql-tests
```
Once a log line like the following is seen, the container is ready for running
tests:
```
PostgreSQL init process complete; ready for start up.
```
Alternatively if the `pg_isready` command is installed, it can be used to
determine if the container is ready, i.e.:
```
$ until pg_isready -h 127.0.0.1; do sleep 1; done
```
To run the entire test suite run this command in the root of the project
once the test database is ready:
```
$ make test
@@ -88,3 +121,65 @@ run:
```
$ go test -run TestAuthTokenAuthenticator -v ./internal/auth
```
### Stopping the test database container
The test database container can be shutdown using:
```
$ make test-database-down
```
Note that the container does *not* need to be shutdown between each run of
`make test` or `go test`.
### Test database container options
By default the container uses the host port of 5432.
This can changed using an environment variable:
```
$ export TEST_DB_PORT=5433
$ make test-database-up
$ make test
```
The default docker image is built using the `postgres:11` base image.
The image can be changed using a make option to test against other versions:
```
$ make IMAGE_TAG=docker.io/hashicorpboundary/postgres:12-alpine test-database-up
$ make IMAGE_TAG=docker.io/hashicorpboundary/postgres:13-alpine test-database-up
$ make IMAGE_TAG=docker.io/hashicorpboundary/postgres:alpine test-database-up
```
Additional options can be passed to postgres to customize and override the
configuration in the config file of the docker image.
See the troubleshooting section below for more details.
### Troubleshooting test database container
The postgres configuration file included in the image
is optimized to support running the full test suite in parallel in CI.
As such, there may be issues starting the container locally,
especially in cases where the container has less then 4GB of memory.
This is likely the case if the output of `docker logs boundary-sql-tests` shows:
```
pg_ctl: could not start server
```
In this case adjust the
[max_connections](https://www.postgresql.org/docs/11/runtime-config-connection.html#GUC-MAX-CONNECTIONS)
and/or
[shared_buffers](https://www.postgresql.org/docs/11/runtime-config-resource.html#GUC-SHARED-BUFFERS):
```
make PG_OPTS="-c max_connections=1000" test-database-up
```
Note that if `max_connections` is set too low, it may result in sporadic test
failures if a connection cannot be established. In this case, reduce the number
of concurrent tests via `GOMAXPROCS` or selectively run tests.
Makefile
+9 -3
View File
@@ -157,14 +157,20 @@ website-install:
website-start:
@npm start --prefix website/
test-database-up:
make -C testing/dbtest/docker database-up
test-database-down:
make -C testing/dbtest/docker clean
test-ci: install-go
~/.go/bin/go test ./... -v $(TESTARGS) -timeout 120m
test-sql:
$(MAKE) -C internal/db/sqltest/ test
test:
~/.go/bin/go test ./... -timeout 30m
test:
go test ./... -timeout 30m
install-go:
./ci/goinstall.sh
@@ -207,7 +213,7 @@ docker-publish:
docker push $(IMAGE_TAG)
docker push hashicorp/boundary:latest
.PHONY: api cli tools gen migrations proto website ci-config ci-verify set-ui-version docker docker-build docker-build-dev docker-publish
.PHONY: api cli tools gen migrations proto website ci-config ci-verify set-ui-version docker docker-build docker-build-dev docker-publish test-database-up test-database-down
.NOTPARALLEL:
internal/cmd/base/dev.go
+7 -2
View File
@@ -10,13 +10,14 @@ import (
"strings"
"github.com/hashicorp/boundary/internal/auth/oidc"
"github.com/hashicorp/boundary/internal/cmd/base/internal/docker"
"github.com/hashicorp/boundary/internal/db"
"github.com/hashicorp/boundary/internal/db/schema"
"github.com/hashicorp/boundary/internal/docker"
"github.com/hashicorp/boundary/internal/iam"
"github.com/hashicorp/boundary/internal/kms"
"github.com/hashicorp/boundary/internal/observability/event"
"github.com/hashicorp/boundary/internal/types/scope"
"github.com/hashicorp/boundary/testing/dbtest"
capoidc "github.com/hashicorp/cap/oidc"
"github.com/hashicorp/go-multierror"
)
@@ -39,7 +40,11 @@ func (b *Server) CreateDevDatabase(ctx context.Context, opt ...Option) error {
switch b.DatabaseUrl {
case "":
c, url, container, err = docker.StartDbInDocker(dialect, docker.WithContainerImage(opts.withContainerImage))
if opts.withDatabaseTemplate != "" {
c, url, _, err = dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(opts.withDatabaseTemplate))
} else {
c, url, container, err = docker.StartDbInDocker(dialect, docker.WithContainerImage(opts.withContainerImage))
}
// In case of an error, run the cleanup function. If we pass all errors, c should be set to a noop
// function before returning from this method
defer func() {
internal/docker/docker.go → internal/cmd/base/internal/docker/docker.go
View File
File renamed without changes.
internal/docker/option.go → internal/cmd/base/internal/docker/option.go
View File
File renamed without changes.
internal/docker/supported.go → internal/cmd/base/internal/docker/supported.go
View File
File renamed without changes.
internal/cmd/base/option.go
+9
View File
@@ -28,6 +28,7 @@ type Options struct {
withSkipTargetCreation bool
withContainerImage string
withDialect string
withDatabaseTemplate string
withEventerConfig *event.EventerConfig
withEventFlags *EventFlags
withAttributeFieldPrefix string
@@ -149,3 +150,11 @@ func WithStatusCode(statusCode int) Option {
o.withStatusCode = statusCode
}
}
// WithDatabaseTemplate allows for using an existing database template for
// initializing the boundary database.
func WithDatabaseTemplate(template string) Option {
return func(o *Options) {
o.withDatabaseTemplate = template
}
}
internal/cmd/commands/database/funcs_test.go
+10 -10
View File
@@ -6,8 +6,8 @@ import (
"testing"
"github.com/hashicorp/boundary/internal/cmd/base"
"github.com/hashicorp/boundary/internal/db"
"github.com/hashicorp/boundary/internal/db/schema"
"github.com/hashicorp/boundary/testing/dbtest"
"github.com/mitchellh/cli"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -15,7 +15,7 @@ import (
func TestMigrateDatabase(t *testing.T) {
ctx := context.Background()
dialect := "postgres"
dialect := dbtest.Postgres
cases := []struct {
name string
@@ -29,7 +29,7 @@ func TestMigrateDatabase(t *testing.T) {
name: "not_initialized_expected_not_intialized",
initialized: false,
urlProvider: func() string {
c, u, _, err := db.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
@@ -43,7 +43,7 @@ func TestMigrateDatabase(t *testing.T) {
name: "basic_initialized_expects_initialized",
initialized: true,
urlProvider: func() string {
c, u, _, err := db.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
@@ -69,7 +69,7 @@ func TestMigrateDatabase(t *testing.T) {
name: "basic_initialized_expects_not_initialized",
initialized: false,
urlProvider: func() string {
c, u, _, err := db.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
@@ -95,7 +95,7 @@ func TestMigrateDatabase(t *testing.T) {
name: "old_version_table_used_intialized",
initialized: true,
urlProvider: func() string {
c, u, _, err := db.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
@@ -115,7 +115,7 @@ func TestMigrateDatabase(t *testing.T) {
name: "old_version_table_used_not_intialized",
initialized: false,
urlProvider: func() string {
c, u, _, err := db.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
@@ -149,7 +149,7 @@ func TestMigrateDatabase(t *testing.T) {
name: "cant_get_lock_initialized",
initialized: true,
urlProvider: func() string {
c, u, _, err := db.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
@@ -172,7 +172,7 @@ func TestMigrateDatabase(t *testing.T) {
name: "cant_get_lock_not_initialized",
initialized: false,
urlProvider: func() string {
c, u, _, err := db.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
@@ -210,7 +210,7 @@ func TestVerifyOplogIsEmpty(t *testing.T) {
dialect := "postgres"
ctx := context.Background()
c, u, _, err := db.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
internal/cmd/commands/server/server_test.go
+1 -1
View File
@@ -67,7 +67,7 @@ func testServerCommand(t *testing.T, opts testServerCommandOpts) *Command {
cmd.Server.DevTargetId = defaultTestTargetId
}
err = cmd.CreateDevDatabase(cmd.Context, base.WithContainerImage("postgres"), base.WithSkipOidcAuthMethodCreation())
err = cmd.CreateDevDatabase(cmd.Context, base.WithDatabaseTemplate("boundary_template"), base.WithSkipOidcAuthMethodCreation())
if err != nil {
if cmd.DevDatabaseCleanupFunc != nil {
require.NoError(cmd.DevDatabaseCleanupFunc())
internal/db/db.go
-3
View File
@@ -7,7 +7,6 @@ import (
"math"
"time"
"github.com/hashicorp/boundary/internal/docker"
"github.com/hashicorp/boundary/internal/observability/event"
"github.com/hashicorp/go-hclog"
"github.com/jinzhu/gorm"
@@ -23,8 +22,6 @@ func init() {
pq.EnableInfinityTs(NegativeInfinityTS, PositiveInfinityTS)
}
var StartDbInDocker = docker.StartDbInDocker
type DbType int
const (
internal/db/db_test.go
+2 -1
View File
@@ -3,12 +3,13 @@ package db
import (
"testing"
"github.com/hashicorp/boundary/testing/dbtest"
"github.com/lib/pq"
"github.com/stretchr/testify/assert"
)
func TestOpen(t *testing.T) {
cleanup, url, _, err := StartDbInDocker("postgres")
cleanup, url, _, err := dbtest.StartUsingTemplate(dbtest.Postgres)
if err != nil {
t.Fatal(err)
}
internal/db/schema/manager_test.go
+19 -22
View File
@@ -8,16 +8,16 @@ import (
"testing"
"github.com/hashicorp/boundary/internal/db/schema/postgres"
"github.com/hashicorp/boundary/internal/docker"
"github.com/hashicorp/boundary/internal/errors"
"github.com/hashicorp/boundary/testing/dbtest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNewManager(t *testing.T) {
dialect := "postgres"
dialect := dbtest.Postgres
c, u, _, err := docker.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
@@ -37,18 +37,15 @@ func TestNewManager(t *testing.T) {
}
func TestCurrentState(t *testing.T) {
dialect := "postgres"
dialect := dbtest.Postgres
c, u, _, err := docker.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1))
t.Cleanup(func() {
if err := c(); err != nil {
t.Fatalf("Got error at cleanup: %v", err)
}
})
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
})
ctx := context.Background()
d, err := sql.Open(dialect, u)
require.NoError(t, err)
@@ -79,9 +76,9 @@ func TestCurrentState(t *testing.T) {
}
func TestRollForward(t *testing.T) {
dialect := "postgres"
dialect := dbtest.Postgres
c, u, _, err := docker.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
@@ -104,13 +101,13 @@ func TestRollForward(t *testing.T) {
}
func TestRollForward_NotFromFresh(t *testing.T) {
dialect := "postgres"
dialect := dbtest.Postgres
oState := migrationStates[dialect]
nState := createPartialMigrationState(oState, 8)
migrationStates[dialect] = nState
c, u, _, err := docker.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
@@ -142,9 +139,9 @@ func TestRollForward_NotFromFresh(t *testing.T) {
}
func TestRunMigration_canceledContext(t *testing.T) {
dialect := "postgres"
dialect := dbtest.Postgres
c, u, _, err := docker.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
@@ -163,7 +160,7 @@ func TestRunMigration_canceledContext(t *testing.T) {
}
func TestRollForward_BadSQL(t *testing.T) {
dialect := "postgres"
dialect := dbtest.Postgres
oState := migrationStates[dialect]
defer func() { migrationStates[dialect] = oState }()
@@ -172,7 +169,7 @@ func TestRollForward_BadSQL(t *testing.T) {
nState.upMigrations[10] = []byte("SELECT 1 FROM NonExistantTable;")
migrationStates[dialect] = nState
c, u, _, err := docker.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
@@ -194,9 +191,9 @@ func TestRollForward_BadSQL(t *testing.T) {
func TestManager_ExclusiveLock(t *testing.T) {
ctx := context.Background()
dialect := "postgres"
dialect := dbtest.Postgres
c, u, _, err := docker.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
@@ -219,9 +216,9 @@ func TestManager_ExclusiveLock(t *testing.T) {
func TestManager_SharedLock(t *testing.T) {
ctx := context.Background()
dialect := "postgres"
dialect := dbtest.Postgres
c, u, _, err := docker.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
@@ -248,9 +245,9 @@ func TestManager_SharedLock(t *testing.T) {
func Test_GetMigrationLog(t *testing.T) {
t.Parallel()
ctx := context.Background()
dialect := "postgres"
dialect := dbtest.Postgres
c, u, _, err := docker.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
internal/db/schema/migrations/postgres/11/01_server_type_enum_test.go
+3 -3
View File
@@ -7,7 +7,7 @@ import (
"time"
"github.com/hashicorp/boundary/internal/db/schema"
"github.com/hashicorp/boundary/internal/docker"
"github.com/hashicorp/boundary/testing/dbtest"
"github.com/stretchr/testify/require"
)
@@ -40,10 +40,10 @@ func Test_ServerEnumChanges(t *testing.T) {
require := require.New(t)
const priorMigration = 10007
const serverEnumMigration = 11001
dialect := "postgres"
dialect := dbtest.Postgres
ctx := context.Background()
c, u, _, err := docker.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1))
require.NoError(err)
t.Cleanup(func() {
require.NoError(c())
internal/db/schema/migrations/postgres/12/01_timestamp_sub_funcs_test.go
+3 -3
View File
@@ -7,7 +7,7 @@ import (
"time"
"github.com/hashicorp/boundary/internal/db/schema"
"github.com/hashicorp/boundary/internal/docker"
"github.com/hashicorp/boundary/testing/dbtest"
"github.com/stretchr/testify/require"
)
@@ -34,9 +34,9 @@ func testSetupDb(ctx context.Context, t *testing.T) *sql.DB {
t.Helper()
require := require.New(t)
dialect := "postgres"
dialect := dbtest.Postgres
c, u, _, err := docker.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1))
require.NoError(err)
t.Cleanup(func() {
require.NoError(c())
internal/db/schema/migrations/postgres/14/warehouse_user_dim_test.go
+3 -3
View File
@@ -9,12 +9,12 @@ import (
"github.com/hashicorp/boundary/internal/authtoken"
"github.com/hashicorp/boundary/internal/db"
"github.com/hashicorp/boundary/internal/db/schema"
"github.com/hashicorp/boundary/internal/docker"
"github.com/hashicorp/boundary/internal/host/static"
"github.com/hashicorp/boundary/internal/iam"
"github.com/hashicorp/boundary/internal/kms"
"github.com/hashicorp/boundary/internal/session"
"github.com/hashicorp/boundary/internal/target"
"github.com/hashicorp/boundary/testing/dbtest"
wrapping "github.com/hashicorp/go-kms-wrapping"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
@@ -30,9 +30,9 @@ func TestMigrations_UserDimension(t *testing.T) {
t.Parallel()
assert, require := assert.New(t), require.New(t)
ctx := context.Background()
dialect := "postgres"
dialect := dbtest.Postgres
c, u, _, err := docker.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1))
require.NoError(err)
t.Cleanup(func() {
require.NoError(c())
internal/db/schema/migrations/postgres/2/07_iam_test.go
+3 -3
View File
@@ -8,8 +8,8 @@ import (
"github.com/hashicorp/boundary/internal/auth/password"
"github.com/hashicorp/boundary/internal/db"
"github.com/hashicorp/boundary/internal/db/schema"
"github.com/hashicorp/boundary/internal/docker"
"github.com/hashicorp/boundary/internal/iam"
"github.com/hashicorp/boundary/testing/dbtest"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -30,10 +30,10 @@ func Test_PrimaryAuthMethodChanges(t *testing.T) {
//
// 4) asserting some bits about the state of the db.
assert, require := assert.New(t), require.New(t)
dialect := "postgres"
dialect := dbtest.Postgres
ctx := context.Background()
c, u, _, err := docker.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1))
require.NoError(err)
t.Cleanup(func() {
require.NoError(c())
internal/db/schema/migrations/postgres/2/10_auth_test.go
+1 -1
View File
@@ -18,7 +18,7 @@ func Test_AuthMethodSubtypes(t *testing.T) {
t.Parallel()
assert, require := assert.New(t), require.New(t)
ctx := context.Background()
conn, _ := db.TestSetup(t, "postgres")
conn, _ := db.TestSetup(t, "postgres", db.WithTemplate("template1"))
rw := db.New(conn)
rootWrapper := db.TestWrapper(t)
kmsCache := kms.TestKms(t, conn, rootWrapper)
internal/db/schema/migrations/postgres/8/08_connection_test.go
+1 -1
View File
@@ -38,7 +38,7 @@ func TestMigration(t *testing.T) {
selectQuery = `select session_id, server_id from session_connection_testing order by session_id`
)
conn, _ := db.TestSetup(t, "postgres")
conn, _ := db.TestSetup(t, "postgres", db.WithTemplate("template1"))
db := conn.DB()
_, err := db.Exec(createTables)
require.NoError(err)
internal/db/schema/postgres/postgres_test.go
+291 -429
View File
@@ -33,490 +33,352 @@ package postgres
import (
"bytes"
"context"
"database/sql"
sqldriver "database/sql/driver"
"fmt"
"io"
"log"
"strconv"
"strings"
"testing"
"github.com/dhui/dktest"
"github.com/golang-migrate/migrate/v4/dktesting"
"github.com/hashicorp/boundary/testing/dbtest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
pgPassword = "postgres"
)
var (
opts = dktest.Options{
Env: map[string]string{"POSTGRES_PASSWORD": pgPassword},
PortRequired: true, ReadyFunc: isReady,
}
// Supported versions: https://www.postgresql.org/support/versioning/
specs = []dktesting.ContainerSpec{
{ImageName: "postgres:12", Options: opts},
}
)
func pgConnectionString(host, port string) string {
return fmt.Sprintf("postgres://postgres:%s@%s:%s/postgres?sslmode=disable", pgPassword, host, port)
}
func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
ip, port, err := c.FirstPort()
if err != nil {
return false
}
db, err := sql.Open("postgres", pgConnectionString(ip, port))
if err != nil {
return false
}
defer func() {
if err := db.Close(); err != nil {
log.Println("close error:", err)
}
}()
if err = db.PingContext(ctx); err != nil {
switch err {
case sqldriver.ErrBadConn, io.EOF:
return false
default:
log.Println(err)
}
return false
}
return true
}
func TestDbStuff(t *testing.T) {
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
ctx := context.Background()
ip, port, err := c.FirstPort()
if err != nil {
t.Fatal(err)
}
addr := pgConnectionString(ip, port)
d, err := open(t, ctx, addr)
if err != nil {
t.Fatal(err)
}
defer func() {
if err := d.close(t); err != nil {
t.Error(err)
}
}()
test(t, d, []byte("SELECT 1"))
t.Parallel()
ctx := context.Background()
c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
})
d, err := open(t, ctx, u)
require.NoError(t, err)
test(t, d, []byte("SELECT 1"))
}
func TestCurrentState_NoVersionTable(t *testing.T) {
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
ctx := context.Background()
ip, port, err := c.FirstPort()
if err != nil {
t.Fatal(err)
}
addr := pgConnectionString(ip, port)
d, err := open(t, ctx, addr)
if err != nil {
t.Fatal(err)
}
defer func() {
if err := d.close(t); err != nil {
t.Error(err)
}
}()
// Drop the version table so calls to CurrentState don't rely on that
require.NoError(t, d.drop(ctx))
v, alreadyRan, dirt, err := d.CurrentState(ctx)
assert.NoError(t, err)
assert.False(t, alreadyRan)
assert.Equal(t, v, nilVersion)
assert.False(t, dirt)
t.Parallel()
ctx := context.Background()
c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
})
d, err := open(t, ctx, u)
require.NoError(t, err)
// Drop the version table so calls to CurrentState don't rely on that
require.NoError(t, d.drop(ctx))
v, alreadyRan, dirt, err := d.CurrentState(ctx)
assert.NoError(t, err)
assert.False(t, alreadyRan)
assert.Equal(t, v, nilVersion)
assert.False(t, dirt)
}
func TestCurrentState_ToManyTables(t *testing.T) {
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
ctx := context.Background()
ip, port, err := c.FirstPort()
if err != nil {
t.Fatal(err)
}
addr := pgConnectionString(ip, port)
d, err := open(t, ctx, addr)
if err != nil {
t.Fatal(err)
}
defer func() {
if err := d.close(t); err != nil {
t.Error(err)
}
}()
// Create the most recent table
require.NoError(t, d.EnsureVersionTable(ctx))
// Create the legacy version of the table.
oldTableCreate := `create table if not exists schema_migrations (version bigint primary key, dirty boolean not null)`
_, err = d.conn.ExecContext(ctx, oldTableCreate)
require.NoError(t, err)
v, alreadyRan, dirt, err := d.CurrentState(ctx)
assert.Error(t, err)
assert.True(t, alreadyRan)
assert.Equal(t, v, nilVersion)
assert.False(t, dirt)
t.Parallel()
ctx := context.Background()
c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
})
d, err := open(t, ctx, u)
require.NoError(t, err)
// Create the most recent table
require.NoError(t, d.EnsureVersionTable(ctx))
// Create the legacy version of the table.
oldTableCreate := `create table if not exists schema_migrations (version bigint primary key, dirty boolean not null)`
_, err = d.conn.ExecContext(ctx, oldTableCreate)
require.NoError(t, err)
v, alreadyRan, dirt, err := d.CurrentState(ctx)
assert.Error(t, err)
assert.True(t, alreadyRan)
assert.Equal(t, v, nilVersion)
assert.False(t, dirt)
}
func TestMultiStatement(t *testing.T) {
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
ctx := context.Background()
ip, port, err := c.FirstPort()
if err != nil {
t.Fatal(err)
}
addr := pgConnectionString(ip, port)
d, err := open(t, ctx, addr)
if err != nil {
t.Fatal(err)
}
defer func() {
if err := d.close(t); err != nil {
t.Error(err)
}
}()
if err := d.EnsureVersionTable(ctx); err != nil {
t.Fatalf("expected err to be nil, got %v", err)
}
if err := d.Run(ctx, strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);"), 2); err != nil {
t.Fatalf("expected err to be nil, got %v", err)
}
// make sure second table exists
var exists bool
if err := d.conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil {
t.Fatal(err)
}
if !exists {
t.Fatalf("expected table bar to exist")
}
t.Parallel()
ctx := context.Background()
c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
})
d, err := open(t, ctx, u)
require.NoError(t, err)
if err := d.EnsureVersionTable(ctx); err != nil {
t.Fatalf("expected err to be nil, got %v", err)
}
if err := d.Run(ctx, strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);"), 2); err != nil {
t.Fatalf("expected err to be nil, got %v", err)
}
// make sure second table exists
var exists bool
if err := d.conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil {
t.Fatal(err)
}
if !exists {
t.Fatalf("expected table bar to exist")
}
}
func TestTransaction(t *testing.T) {
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
ctx := context.Background()
ip, port, err := c.FirstPort()
if err != nil {
t.Fatal(err)
}
addr := pgConnectionString(ip, port)
d, err := open(t, ctx, addr)
if err != nil {
t.Fatal(err)
}
defer func() {
if err := d.close(t); err != nil {
t.Error(err)
}
}()
v, alreadyRan, dirty, err := d.CurrentState(ctx)
assert.NoError(t, err)
assert.False(t, alreadyRan)
assert.False(t, dirty)
assert.Equal(t, -1, v)
// Fail the initial setup of the db.
assert.NoError(t, d.StartRun(ctx))
assert.NoError(t, d.EnsureVersionTable(ctx))
assert.Error(t, d.Run(ctx, strings.NewReader("SELECT 1 from nonExistantTable"), 3))
assert.Error(t, d.CommitRun())
v, alreadyRan, dirty, err = d.CurrentState(ctx)
assert.NoError(t, err)
assert.False(t, alreadyRan)
assert.False(t, dirty)
assert.Equal(t, -1, v)
assert.NoError(t, d.StartRun(ctx))
assert.NoError(t, d.EnsureVersionTable(ctx))
assert.NoError(t, d.Run(ctx, strings.NewReader("CREATE TABLE foo (foo text);"), 2))
assert.NoError(t, d.Run(ctx, strings.NewReader("SELECT 1;"), 3))
assert.NoError(t, d.CommitRun())
v, alreadyRan, dirty, err = d.CurrentState(ctx)
assert.NoError(t, err)
assert.True(t, alreadyRan)
assert.False(t, dirty)
assert.Equal(t, 3, v)
assert.NoError(t, d.StartRun(ctx))
assert.NoError(t, d.Run(ctx, strings.NewReader("CREATE TABLE bar (bar text);"), 20))
assert.Error(t, d.Run(ctx, strings.NewReader("SELECT 1 FROM NonExistingTable"), 30))
assert.Error(t, d.CommitRun())
v, alreadyRan, dirty, err = d.CurrentState(ctx)
assert.NoError(t, err)
assert.True(t, alreadyRan)
assert.False(t, dirty)
assert.Equal(t, 3, v)
t.Parallel()
ctx := context.Background()
c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
})
d, err := open(t, ctx, u)
require.NoError(t, err)
v, alreadyRan, dirty, err := d.CurrentState(ctx)
assert.NoError(t, err)
assert.False(t, alreadyRan)
assert.False(t, dirty)
assert.Equal(t, -1, v)
// Fail the initial setup of the db.
assert.NoError(t, d.StartRun(ctx))
assert.NoError(t, d.EnsureVersionTable(ctx))
assert.Error(t, d.Run(ctx, strings.NewReader("SELECT 1 from nonExistantTable"), 3))
assert.Error(t, d.CommitRun())
v, alreadyRan, dirty, err = d.CurrentState(ctx)
assert.NoError(t, err)
assert.False(t, alreadyRan)
assert.False(t, dirty)
assert.Equal(t, -1, v)
assert.NoError(t, d.StartRun(ctx))
assert.NoError(t, d.EnsureVersionTable(ctx))
assert.NoError(t, d.Run(ctx, strings.NewReader("CREATE TABLE foo (foo text);"), 2))
assert.NoError(t, d.Run(ctx, strings.NewReader("SELECT 1;"), 3))
assert.NoError(t, d.CommitRun())
v, alreadyRan, dirty, err = d.CurrentState(ctx)
assert.NoError(t, err)
assert.True(t, alreadyRan)
assert.False(t, dirty)
assert.Equal(t, 3, v)
assert.NoError(t, d.StartRun(ctx))
assert.NoError(t, d.Run(ctx, strings.NewReader("CREATE TABLE bar (bar text);"), 20))
assert.Error(t, d.Run(ctx, strings.NewReader("SELECT 1 FROM NonExistingTable"), 30))
assert.Error(t, d.CommitRun())
v, alreadyRan, dirty, err = d.CurrentState(ctx)
assert.NoError(t, err)
assert.True(t, alreadyRan)
assert.False(t, dirty)
assert.Equal(t, 3, v)
}
func TestWithSchema(t *testing.T) {
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
ctx := context.Background()
ip, port, err := c.FirstPort()
require.NoError(t, err)
addr := pgConnectionString(ip, port)
d, err := open(t, ctx, addr)
require.NoError(t, err)
defer func() {
if err := d.close(t); err != nil {
t.Fatal(err)
}
}()
require.NoError(t, d.EnsureVersionTable(ctx))
// create foobar schema
require.NoError(t, d.Run(ctx, strings.NewReader("CREATE SCHEMA foobar AUTHORIZATION postgres"), 1))
// re-connect using that schema
d2, err := open(t, ctx, fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&search_path=foobar",
pgPassword, ip, port))
require.NoError(t, err)
defer func() {
if err := d2.close(t); err != nil {
t.Fatal(err)
}
}()
version, alreadyRan, _, err := d2.CurrentState(ctx)
require.NoError(t, err)
require.Equal(t, nilVersion, version)
assert.False(t, alreadyRan)
// now update CurrentState and compare
require.NoError(t, d2.EnsureVersionTable(ctx))
require.NoError(t, d2.setVersion(ctx, 2, false))
version, alreadyRan, _, err = d2.CurrentState(ctx)
require.NoError(t, err)
require.Equal(t, 2, version)
assert.True(t, alreadyRan)
// meanwhile, the public schema still has the other CurrentState
version, alreadyRan, _, err = d.CurrentState(ctx)
require.NoError(t, err)
require.Equal(t, 1, version)
assert.True(t, alreadyRan)
t.Parallel()
ctx := context.Background()
c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
})
d, err := open(t, ctx, u)
require.NoError(t, err)
require.NoError(t, d.EnsureVersionTable(ctx))
// create foobar schema
require.NoError(t, d.Run(ctx, strings.NewReader("CREATE SCHEMA foobar AUTHORIZATION postgres"), 1))
// re-connect using that schema
d2, err := open(t, ctx, fmt.Sprintf("%s&search_path=foobar", u))
require.NoError(t, err)
defer func() {
if err := d2.close(t); err != nil {
t.Fatal(err)
}
}()
version, alreadyRan, _, err := d2.CurrentState(ctx)
require.NoError(t, err)
require.Equal(t, nilVersion, version)
assert.False(t, alreadyRan)
// now update CurrentState and compare
require.NoError(t, d2.EnsureVersionTable(ctx))
require.NoError(t, d2.setVersion(ctx, 2, false))
version, alreadyRan, _, err = d2.CurrentState(ctx)
require.NoError(t, err)
require.Equal(t, 2, version)
assert.True(t, alreadyRan)
// meanwhile, the public schema still has the other CurrentState
version, alreadyRan, _, err = d.CurrentState(ctx)
require.NoError(t, err)
require.Equal(t, 1, version)
assert.True(t, alreadyRan)
}
func TestPostgres_Lock(t *testing.T) {
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
ctx := context.Background()
ip, port, err := c.FirstPort()
if err != nil {
t.Fatal(err)
}
addr := pgConnectionString(ip, port)
ps, err := open(t, ctx, addr)
if err != nil {
t.Fatal(err)
}
test(t, ps, []byte("SELECT 1"))
err = ps.Lock(ctx)
if err != nil {
t.Fatal(err)
}
err = ps.Unlock(ctx)
if err != nil {
t.Fatal(err)
}
err = ps.Lock(ctx)
if err != nil {
t.Fatal(err)
}
err = ps.Unlock(ctx)
if err != nil {
t.Fatal(err)
}
// make sure we call call Unlock in an idempotent manner.
err = ps.Unlock(ctx)
if err != nil {
t.Fatal(err)
}
t.Parallel()
ctx := context.Background()
c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
})
d, err := open(t, ctx, u)
require.NoError(t, err)
test(t, d, []byte("SELECT 1"))
err = d.Lock(ctx)
if err != nil {
t.Fatal(err)
}
err = d.Unlock(ctx)
if err != nil {
t.Fatal(err)
}
err = d.Lock(ctx)
if err != nil {
t.Fatal(err)
}
err = d.Unlock(ctx)
if err != nil {
t.Fatal(err)
}
// make sure we call call Unlock in an idempotent manner.
err = d.Unlock(ctx)
if err != nil {
t.Fatal(err)
}
}
func TestEnsureTable_Fresh(t *testing.T) {
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
ctx := context.Background()
ip, port, err := c.FirstPort()
if err != nil {
t.Fatal(err)
}
addr := pgConnectionString(ip, port)
p, err := open(t, ctx, addr)
if err != nil {
require.NoError(t, err)
}
t.Cleanup(func() {
require.NoError(t, p.close(t))
})
tableCreated := false
query := "SELECT exists (SELECT 1 FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_name = '" + defaultMigrationsTable + "')"
assert.NoError(t, p.db.QueryRowContext(ctx, query).Scan(&tableCreated))
assert.False(t, tableCreated)
assert.NoError(t, p.EnsureVersionTable(ctx))
assert.NoError(t, p.db.QueryRowContext(ctx, query).Scan(&tableCreated))
assert.True(t, tableCreated)
t.Parallel()
ctx := context.Background()
c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
})
d, err := open(t, ctx, u)
require.NoError(t, err)
tableCreated := false
query := "SELECT exists (SELECT 1 FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_name = '" + defaultMigrationsTable + "')"
assert.NoError(t, d.db.QueryRowContext(ctx, query).Scan(&tableCreated))
assert.False(t, tableCreated)
assert.NoError(t, d.EnsureVersionTable(ctx))
assert.NoError(t, d.db.QueryRowContext(ctx, query).Scan(&tableCreated))
assert.True(t, tableCreated)
}
func TestEnsureTable_ExistingTable(t *testing.T) {
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
ctx := context.Background()
ip, port, err := c.FirstPort()
if err != nil {
t.Fatal(err)
}
addr := pgConnectionString(ip, port)
p, err := open(t, ctx, addr)
if err != nil {
require.NoError(t, err)
}
t.Cleanup(func() {
require.NoError(t, p.close(t))
})
assert.NoError(t, p.EnsureVersionTable(ctx))
oldTableCreate := `CREATE TABLE IF NOT EXISTS schema_migrations (version bigint primary key, dirty boolean not null)`
_, err = p.db.ExecContext(ctx, oldTableCreate)
assert.NoError(t, err)
assert.NoError(t, p.EnsureVersionTable(ctx))
t.Parallel()
ctx := context.Background()
c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
})
d, err := open(t, ctx, u)
require.NoError(t, err)
assert.NoError(t, d.EnsureVersionTable(ctx))
oldTableCreate := `CREATE TABLE IF NOT EXISTS schema_migrations (version bigint primary key, dirty boolean not null)`
_, err = d.db.ExecContext(ctx, oldTableCreate)
assert.NoError(t, err)
assert.NoError(t, d.EnsureVersionTable(ctx))
}
func TestEnsureTable_OldTable(t *testing.T) {
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
ctx := context.Background()
ip, port, err := c.FirstPort()
if err != nil {
t.Fatal(err)
}
addr := pgConnectionString(ip, port)
p, err := open(t, ctx, addr)
if err != nil {
require.NoError(t, err)
}
t.Cleanup(func() {
require.NoError(t, p.close(t))
})
oldTableCreate := `CREATE TABLE IF NOT EXISTS schema_migrations (version bigint primary key, dirty boolean not null)`
_, err = p.db.ExecContext(ctx, oldTableCreate)
assert.NoError(t, err)
tableExists := false
oldTableCheck := "SELECT exists (SELECT 1 FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_name = 'schema_migrations')"
assert.NoError(t, p.db.QueryRowContext(ctx, oldTableCheck).Scan(&tableExists))
assert.True(t, tableExists)
query := "SELECT exists (SELECT 1 FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_name = '" + defaultMigrationsTable + "')"
assert.NoError(t, p.db.QueryRowContext(ctx, query).Scan(&tableExists))
assert.False(t, tableExists)
assert.NoError(t, p.EnsureVersionTable(ctx))
assert.NoError(t, p.db.QueryRowContext(ctx, oldTableCheck).Scan(&tableExists))
assert.False(t, tableExists)
assert.NoError(t, p.db.QueryRowContext(ctx, query).Scan(&tableExists))
assert.True(t, tableExists)
t.Parallel()
ctx := context.Background()
c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
})
d, err := open(t, ctx, u)
require.NoError(t, err)
oldTableCreate := `CREATE TABLE IF NOT EXISTS schema_migrations (version bigint primary key, dirty boolean not null)`
_, err = d.db.ExecContext(ctx, oldTableCreate)
assert.NoError(t, err)
tableExists := false
oldTableCheck := "SELECT exists (SELECT 1 FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_name = 'schema_migrations')"
assert.NoError(t, d.db.QueryRowContext(ctx, oldTableCheck).Scan(&tableExists))
assert.True(t, tableExists)
query := "SELECT exists (SELECT 1 FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_name = '" + defaultMigrationsTable + "')"
assert.NoError(t, d.db.QueryRowContext(ctx, query).Scan(&tableExists))
assert.False(t, tableExists)
assert.NoError(t, d.EnsureVersionTable(ctx))
assert.NoError(t, d.db.QueryRowContext(ctx, oldTableCheck).Scan(&tableExists))
assert.False(t, tableExists)
assert.NoError(t, d.db.QueryRowContext(ctx, query).Scan(&tableExists))
assert.True(t, tableExists)
}
func TestRollback(t *testing.T) {
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
ctx := context.Background()
ip, port, err := c.FirstPort()
if err != nil {
t.Fatal(err)
}
addr := pgConnectionString(ip, port)
p, err := open(t, ctx, addr)
if err != nil {
require.NoError(t, err)
}
t.Cleanup(func() {
require.NoError(t, p.close(t))
})
assert.NoError(t, p.StartRun(ctx))
assert.NoError(t, p.EnsureVersionTable(ctx))
assert.NoError(t, p.Run(ctx, bytes.NewReader([]byte("create table if not exists foo (foo text)")), 2))
var exists bool
query := "select exists (select 1 from information_schema.tables where table_name = 'foo' and table_schema = (select current_schema()))"
assert.NoError(t, p.conn.QueryRowContext(context.Background(), query).Scan(&exists))
assert.True(t, exists)
assert.NoError(t, p.Rollback())
assert.NoError(t, p.conn.QueryRowContext(context.Background(), query).Scan(&exists))
assert.False(t, exists)
t.Parallel()
ctx := context.Background()
c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
})
d, err := open(t, ctx, u)
require.NoError(t, err)
assert.NoError(t, d.StartRun(ctx))
assert.NoError(t, d.EnsureVersionTable(ctx))
assert.NoError(t, d.Run(ctx, bytes.NewReader([]byte("create table if not exists foo (foo text)")), 2))
var exists bool
query := "select exists (select 1 from information_schema.tables where table_name = 'foo' and table_schema = (select current_schema()))"
assert.NoError(t, d.conn.QueryRowContext(context.Background(), query).Scan(&exists))
assert.True(t, exists)
assert.NoError(t, d.Rollback())
assert.NoError(t, d.conn.QueryRowContext(context.Background(), query).Scan(&exists))
assert.False(t, exists)
}
func TestRun_Error(t *testing.T) {
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
ctx := context.Background()
ip, port, err := c.FirstPort()
if err != nil {
t.Fatal(err)
}
addr := pgConnectionString(ip, port)
p, err := open(t, ctx, addr)
if err != nil {
require.NoError(t, err)
}
t.Cleanup(func() {
require.NoError(t, p.close(t))
})
err = p.Run(ctx, bytes.NewReader([]byte("SELECT *\nFROM foo")), 2)
assert.Error(t, err)
t.Parallel()
ctx := context.Background()
c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
})
d, err := open(t, ctx, u)
require.NoError(t, err)
err = d.Run(ctx, bytes.NewReader([]byte("SELECT *\nFROM foo")), 2)
assert.Error(t, err)
}
func Test_computeLineFromPos(t *testing.T) {
internal/db/schema/schema_test.go
+5 -5
View File
@@ -5,16 +5,16 @@ import (
"database/sql"
"testing"
"github.com/hashicorp/boundary/internal/docker"
"github.com/hashicorp/boundary/testing/dbtest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMigrateStore(t *testing.T) {
dialect := "postgres"
dialect := dbtest.Postgres
ctx := context.Background()
c, u, _, err := docker.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
@@ -45,10 +45,10 @@ func TestMigrateStore(t *testing.T) {
func Test_MigrateStore_WithMigrationStates(t *testing.T) {
assert, require := assert.New(t), require.New(t)
dialect := "postgres"
dialect := dbtest.Postgres
ctx := context.Background()
c, u, _, err := docker.StartDbInDocker(dialect)
c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1))
require.NoError(err)
t.Cleanup(func() {
require.NoError(c())
internal/db/testing.go
+11 -1
View File
@@ -13,6 +13,7 @@ import (
"github.com/hashicorp/boundary/internal/db/schema"
"github.com/hashicorp/boundary/internal/oplog"
"github.com/hashicorp/boundary/internal/oplog/store"
"github.com/hashicorp/boundary/testing/dbtest"
wrapping "github.com/hashicorp/go-kms-wrapping"
"github.com/hashicorp/go-kms-wrapping/wrappers/aead"
"github.com/jinzhu/gorm"
@@ -30,7 +31,7 @@ func TestSetup(t *testing.T, dialect string, opt ...TestOption) (*gorm.DB, strin
switch opts.withTestDatabaseUrl {
case "":
cleanup, url, _, err = StartDbInDocker(dialect)
cleanup, url, _, err = dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(opts.withTemplate))
if err != nil {
t.Fatal(err)
}
@@ -172,6 +173,7 @@ type testOptions struct {
withOperation oplog.OpType
withTestDatabaseUrl string
withResourcePrivateId bool
withTemplate string
}
func getDefaultTestOptions() testOptions {
@@ -211,3 +213,11 @@ func WithResourcePrivateId(enable bool) TestOption {
o.withResourcePrivateId = enable
}
}
// WithTemplate provides a way to specify the source database template for creating
// a database.
func WithTemplate(template string) TestOption {
return func(o *testOptions) {
o.withTemplate = template
}
}
internal/oplog/testing.go
+2 -2
View File
@@ -7,8 +7,8 @@ import (
"testing"
"github.com/hashicorp/boundary/internal/db/schema"
"github.com/hashicorp/boundary/internal/docker"
"github.com/hashicorp/boundary/internal/oplog/oplog_test"
"github.com/hashicorp/boundary/testing/dbtest"
wrapping "github.com/hashicorp/go-kms-wrapping"
"github.com/hashicorp/go-kms-wrapping/wrappers/aead"
"github.com/hashicorp/go-uuid"
@@ -57,7 +57,7 @@ func testId(t *testing.T) string {
func testInitDbInDocker(t *testing.T) (cleanup func() error, retURL string, err error) {
t.Helper()
cleanup, retURL, _, err = docker.StartDbInDocker("postgres")
cleanup, retURL, _, err = dbtest.StartUsingTemplate(dbtest.Postgres)
if err != nil {
t.Fatal(err)
}
internal/servers/controller/handlers/credentialstores/credentialstore_service_test.go
+1 -1
View File
@@ -124,7 +124,7 @@ func TestList(t *testing.T) {
return
}
require.NoError(t, gErr)
assert.Empty(t, cmp.Diff(got, tc.res, protocmp.Transform()))
assert.ElementsMatch(t, got.Items, tc.res.Items)
// Test anonymous listing
got, gErr = s.ListCredentialStores(auth.DisabledAuthTestContext(iamRepoFn, tc.req.GetScopeId(), auth.WithUserId(auth.AnonymousUserId)), tc.req)
internal/servers/controller/testing.go
+1
View File
@@ -582,6 +582,7 @@ func NewTestController(t *testing.T, opts *TestControllerOpts) *TestController {
if opts.DisableOidcAuthMethodCreation {
createOpts = append(createOpts, base.WithSkipOidcAuthMethodCreation())
}
createOpts = append(createOpts, base.WithDatabaseTemplate("boundary_template"))
if err := tc.b.CreateDevDatabase(ctx, createOpts...); err != nil {
t.Fatal(err)
}
testing/dbtest/docker/Dockerfile.11-alpine
+6
View File
@@ -0,0 +1,6 @@
FROM postgres:11-alpine
ADD init-db.sh /docker-entrypoint-initdb.d/00-init-db.sh
ADD postgresql.conf /etc/postgresql/postgresql.conf
CMD ["postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"]
testing/dbtest/docker/Dockerfile.12-alpine
+6
View File
@@ -0,0 +1,6 @@
FROM postgres:12-alpine
ADD init-db.sh /docker-entrypoint-initdb.d/00-init-db.sh
ADD postgresql.conf /etc/postgresql/postgresql.conf
CMD ["postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"]
testing/dbtest/docker/Dockerfile.13-alpine
+6
View File
@@ -0,0 +1,6 @@
FROM postgres:13-alpine
ADD init-db.sh /docker-entrypoint-initdb.d/00-init-db.sh
ADD postgresql.conf /etc/postgresql/postgresql.conf
CMD ["postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"]
testing/dbtest/docker/Dockerfile.alpine
+6
View File
@@ -0,0 +1,6 @@
FROM postgres:alpine
ADD init-db.sh /docker-entrypoint-initdb.d/00-init-db.sh
ADD postgresql.conf /etc/postgresql/postgresql.conf
CMD ["postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"]
testing/dbtest/docker/Makefile
+50
View File
@@ -0,0 +1,50 @@
all: docker-build
CWD := $(shell pwd)
DOCKER_ARGS ?= -d
REGISTRY_NAME?=docker.io/hashicorpboundary
IMAGE_NAME=postgres
IMAGE_TAG ?= $(REGISTRY_NAME)/$(IMAGE_NAME):11-alpine
PG_OPTS ?=
TEST_DB_PORT ?= 5432
# Generate targets from dockerfiles
dockerfiles = $(wildcard Dockerfile.*)
docker-builds = $(patsubst Dockerfile.%,%-build, $(dockerfiles))
docker-publishes = $(patsubst Dockerfile.%,%-publish, $(dockerfiles))
${docker-builds}: %-build:
docker build -t $(REGISTRY_NAME)/$(IMAGE_NAME):$* -f Dockerfile.$* .
docker-build: ${docker-builds}
docker-publish: ${docker-publishes}
${docker-publishes}: %-publish:
docker push $(REGISTRY_NAME)/$(IMAGE_NAME):$*
database-up:
@echo "Using image: $(IMAGE_TAG)"
@echo "Additional postgres configuration: $(PG_OPTS)"
@docker run \
$(DOCKER_ARGS) \
--name boundary-sql-tests \
-p $(TEST_DB_PORT):5432 \
-e POSTGRES_PASSWORD=boundary \
-e POSTGRES_USER=boundary \
-e POSTGRES_DB=boundary \
-e PGDATA=/pgdata \
--mount type=tmpfs,destination=/pgdata \
-v "$(CWD)/../../../internal/db/schema/migrations/postgres":/migrations \
$(IMAGE_TAG) \
-c 'config_file=/etc/postgresql/postgresql.conf' \
$(PG_OPTS) 1> /dev/null
@echo "Test database available at: 127.0.0.1:$(TEST_DB_PORT)"
@echo "For database logs run:"
@echo " docker logs boundary-sql-test"
clean:
docker stop boundary-sql-tests || true
docker rm -v boundary-sql-tests || true
.PHONY: all docker-build database-up ${docker-builds} ${docker-publishes} clean
testing/dbtest/docker/init-db.sh
Executable
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/env bash
set -e
shopt -s globstar
psql -v "ON_ERROR_STOP=1" --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -q <<EOSQL
create user postgres superuser login;
create database boundary_template owner $POSTGRES_USER;
EOSQL
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --no-password --dbname boundary_template -q <<EOSQL
create table boundary_schema_version (
version bigint primary key,
dirty boolean not null
);
EOSQL
version=
# Run migrations in order.
for file in $(ls -v /migrations/**/*.sql); do
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --no-password --dbname boundary_template -f "$file"
IFS='/'
read -ra PARTS <<< "$file"
major="10#${PARTS[2]}"
IFS='_'
read -ra MINOR_PARTS <<< "${PARTS[3]}"
minor="10#${MINOR_PARTS[0]}"
let version=${major}*1000+${minor}
IFS=' '
done
echo "setting boundary_schema_version to ${version}";
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --no-password --dbname boundary_template -q <<EOSQL
insert into boundary_schema_version
(version, dirty)
values
(${version}, false);
EOSQL
psql -v "ON_ERROR_STOP=1" --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -q <<EOSQL
update pg_database set datistemplate = true, datallowconn = false where datname = 'boundary_template';
EOSQL
testing/dbtest/docker/postgresql.conf
+35
View File
@@ -0,0 +1,35 @@
listen_addresses = '*' # listen on all addresses so other containers can connect if need be.
max_connections = 200000 # set max connections to the limit to ensure tests can always get a connection.
# Note that this may need to be decreased if <4G are available for the container.
shared_buffers = 256MB # A more reasonable default compared to the 32MB default config.
# This gives postgres more memory for storing results. But given that short lived nature of the
# test databases and the relatively small data sets, there should not be a need to increase it much more then this.
dynamic_shared_memory_type = posix # Use POSIX shared memory allocated using shm_open
fsync = off # Do not flush, can cause data corruption, but since it is for tests and the data is dropped anyway...
synchronous_commit = off # Do not wait for WAL records to be written to disk before reporting success
full_page_writes = off # Also more performant, with the same risks as fsync being off
autovacuum = off # Disable autovacuum, since the test databases and tables are very short-lived, there is no need to vacuum.
log_destination = 'stderr' # Send logs to stderr, most logs are turned off by the following options, but if they are turned on, they will go to stderr
logging_collector = off # Disables the background log collector which is most useful in production environments with logs going to files
log_min_messages = panic # Only log panics to keep the noise and io down
log_min_error_statement = panic # Only log panics to keep the noise and io down
log_min_duration_statement = -1 # Disable slow query logs
log_temp_files = -1 # Disable logs about temp file usage
log_checkpoints = off
log_connections = off
log_disconnections = off
log_duration = off
log_hostname = off
log_lock_waits = off
log_parser_stats = off
log_planner_stats = off
log_executor_stats = off
log_statement_stats = off
log_timezone = 'UTC'
datestyle = 'iso, mdy'
timezone = 'Etc/UTC'
lc_messages = 'en_US.utf8'
lc_monetary = 'en_US.utf8'
lc_numeric = 'en_US.utf8'
lc_time = 'en_US.utf8'
default_text_search_config = 'pg_catalog.english'
testing/dbtest/docs.go
+25
View File
@@ -0,0 +1,25 @@
// Package dbtest provides a way to create a clean database for tests using
// template databases. For it to function properly a postgres database server
// must be running with the expected templates. This is typically done using
// the docker image provided in the docker directory of this package.
//
// To use this package in a test add something like the following in the
// beginning of the test:
//
// c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres)
// require.NoError(t, err)
// t.Cleanup(func() {
// require.NoError(t, c())
// })
// // use u to get a connection to the new database
// dBase, err := sql.Open("postgres", u)
//
// By default this uses a template that already has all of the boundary
// migrations run. If a test needs a database without any migrations, like in
// the case where the migration code needs to be tested, a different template
// can be specified. This should generally be template1:
//
// c, u, _, err := dbtest.StartUsingTemplate(dbtest.Postgres, dbtest.WithTemplate(dbtest.Template1))
//
// See https://www.postgresql.org/docs/13/manage-ag-templatedbs.html
package dbtest
testing/dbtest/option.go
+30
View File
@@ -0,0 +1,30 @@
package dbtest
// GetOpts - iterate the inbound Options and return a struct.
func GetOpts(opt ...Option) Options {
opts := getDefaultOptions()
for _, o := range opt {
o(&opts)
}
return opts
}
// Option - how Options are passed as arguments.
type Option func(*Options)
// Options - how Options are represented.
type Options struct {
withTemplate string
}
func getDefaultOptions() Options {
return Options{}
}
// WithTemplate tells the command with database template to use
// when creating a new test database.
func WithTemplate(template string) Option {
return func(o *Options) {
o.withTemplate = template
}
}
testing/dbtest/template.go
+150
View File
@@ -0,0 +1,150 @@
package dbtest
import (
"database/sql"
"fmt"
"math/rand"
"os"
"time"
// postgres dialect
_ "github.com/lib/pq"
)
var testDBPort = "5432"
var (
// StartUsingTemplate creates a new test database from a postgres template database.
StartUsingTemplate func(dialect string, opt ...Option) (func() error, string, string, error) = startUsingTemplate
)
func init() {
rand.Seed(time.Now().UnixNano())
p := os.Getenv("TEST_DB_PORT")
if p != "" {
testDBPort = p
}
}
// Dialects
const (
Postgres = "postgres"
)
// Template Database Names
const (
BoundaryTemplate = "boundary_template"
Template1 = "template1"
)
var supportedDialects = map[string]struct{}{
Postgres: struct{}{},
}
var supportedTemplates = map[string]struct{}{
BoundaryTemplate: struct{}{},
Template1: struct{}{},
}
const letterBytes = "abcdefghijklmnopqrstuvwxyz"
const (
letterIdxBits = 6 // 6 bits to represent a letter index
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
)
func randStr(n int) string {
b := make([]byte, n)
// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
for i, cache, remain := n-1, rand.Int63(), letterIdxMax; i >= 0; {
if remain == 0 {
cache, remain = rand.Int63(), letterIdxMax
}
if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
b[i] = letterBytes[idx]
i--
}
cache >>= letterIdxBits
remain--
}
return string(b)
}
func startUsingTemplate(dialect string, opt ...Option) (cleanup func() error, retURL, dbname string, err error) {
if _, ok := supportedDialects[dialect]; !ok {
return func() error { return nil }, "", "", fmt.Errorf("unsupported dialect: %s", dialect)
}
db, err := sql.Open(dialect, fmt.Sprintf("postgres://boundary:boundary@127.0.0.1:%s/boundary?sslmode=disable", testDBPort))
if err != nil {
return func() error { return nil }, "", "", fmt.Errorf("could not connect to source database: %w", err)
}
defer db.Close()
dbname = fmt.Sprintf("boundary_test_%s", randStr(16))
template := BoundaryTemplate
opts := GetOpts(opt...)
if opts.withTemplate != "" {
template = opts.withTemplate
}
if _, ok := supportedTemplates[template]; !ok {
return func() error { return nil }, "", "", fmt.Errorf("unsupported database template: %s", template)
}
_, err = db.Exec(fmt.Sprintf("create database %s template %s", dbname, template))
if err != nil {
return func() error { return nil }, "", "", fmt.Errorf("could not create test database: %w", err)
}
url := fmt.Sprintf("postgres://boundary:boundary@127.0.0.1:%s/%s?sslmode=disable", testDBPort, dbname)
cleanup = func() error {
return dropDatabase(dialect, dbname)
}
tdb, err := sql.Open(dialect, url)
if err != nil {
return func() error { return nil }, "", "", fmt.Errorf("could not create test database: %w", err)
}
defer tdb.Close()
if err := tdb.Ping(); err != nil {
return func() error { return nil }, "", "", fmt.Errorf("could not ping test database: %w", err)
}
return cleanup, url, dbname, nil
}
const killconns = `
select
pg_terminate_backend(pg_stat_activity.pid)
from
pg_stat_activity
where pg_stat_activity.datname = $1
and pid <> pg_backend_pid();
`
func dropDatabase(dialect, dbname string) error {
if dbname == "" {
return fmt.Errorf("empty dbname")
}
db, err := sql.Open(dialect, fmt.Sprintf("postgres://boundary:boundary@127.0.0.1:%s/boundary?sslmode=disable", testDBPort))
if err != nil {
return err
}
defer db.Close()
_, err = db.Exec(killconns, dbname)
if err != nil {
return err
}
_, err = db.Exec(fmt.Sprintf("drop database %s", dbname))
if err != nil {
return err
}
return nil
}
testing/dbtest/template_test.go
+72
View File
@@ -0,0 +1,72 @@
package dbtest_test
import (
"database/sql"
"errors"
"testing"
"github.com/hashicorp/boundary/testing/dbtest"
"github.com/stretchr/testify/require"
)
func TestStartUsingTemplate(t *testing.T) {
tests := []struct {
name string
dialect string
options []dbtest.Option
err error
}{
{
"PostgresNoOptions",
dbtest.Postgres,
[]dbtest.Option{},
nil,
},
{
"PostgresTemplate1",
dbtest.Postgres,
[]dbtest.Option{
dbtest.WithTemplate(dbtest.Template1),
},
nil,
},
{
"PostgresUnsupportedTemplate",
dbtest.Postgres,
[]dbtest.Option{
dbtest.WithTemplate("unsupportedTemplate"),
},
errors.New("unsupported database template: unsupportedTemplate"),
},
{
"UnsupportedDialect",
"unsupportedDialect",
[]dbtest.Option{},
errors.New("unsupported dialect: unsupportedDialect"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c, u, _, err := dbtest.StartUsingTemplate(tt.dialect, tt.options...)
if tt.err != nil {
require.EqualError(t, err, tt.err.Error())
require.NoError(t, c())
} else {
require.NoError(t, err)
db, err := sql.Open(tt.dialect, u)
require.NoError(t, err)
require.NoError(t, db.Ping())
db.Close()
require.NoError(t, c())
// ensure that database is gone after calling c
db, _ = sql.Open(tt.dialect, u)
require.Error(t, db.Ping())
}
})
}
}