Add job run cleaner (#2866)

* chore: remove duplicate import

* feat: Add job run cleaner

The new job cleans job runs older than a configured interval.
Configure the job to clean jobs older than 1 minute.
This commit is contained in:
Johan Brandhorst-Satzkorn authored and GitHub committed 2023-01-31 20:41:33 -05:00
1 parent bc332541c3
commit 55259029f1
5 files changed
+162 -5

No files matched your search

+8 -5
View File
@@ -28,8 +28,8 @@ import (
kmsjob "github.com/hashicorp/boundary/internal/kms/job"
"github.com/hashicorp/boundary/internal/observability/event"
"github.com/hashicorp/boundary/internal/plugin/host"
hostplugin "github.com/hashicorp/boundary/internal/plugin/host"
"github.com/hashicorp/boundary/internal/scheduler"
"github.com/hashicorp/boundary/internal/scheduler/cleaner"
"github.com/hashicorp/boundary/internal/scheduler/job"
"github.com/hashicorp/boundary/internal/server"
serversjob "github.com/hashicorp/boundary/internal/server/job"
@@ -239,9 +239,9 @@ func New(ctx context.Context, conf *Config) (*Controller, error) {
switch enabledPlugin {
case base.EnabledPluginHostLoopback:
plg := pluginhost.NewWrappingPluginClient(pluginhost.NewLoopbackPlugin())
opts := []hostplugin.Option{
hostplugin.WithDescription("Provides an initial loopback host plugin in Boundary"),
hostplugin.WithPublicId(conf.DevLoopbackHostPluginId),
opts := []host.Option{
host.WithDescription("Provides an initial loopback host plugin in Boundary"),
host.WithPublicId(conf.DevLoopbackHostPluginId),
}
if _, err = conf.RegisterHostPlugin(ctx, "loopback", plg, opts...); err != nil {
return nil, err
@@ -261,7 +261,7 @@ func New(ctx context.Context, conf *Config) (*Controller, error) {
return nil, fmt.Errorf("error creating %s host plugin: %w", pluginType, err)
}
conf.ShutdownFuncs = append(conf.ShutdownFuncs, cleanup)
if _, err := conf.RegisterHostPlugin(ctx, pluginType, client, hostplugin.WithDescription(fmt.Sprintf("Built-in %s host plugin", enabledPlugin.String()))); err != nil {
if _, err := conf.RegisterHostPlugin(ctx, pluginType, client, host.WithDescription(fmt.Sprintf("Built-in %s host plugin", enabledPlugin.String()))); err != nil {
return nil, fmt.Errorf("error registering %s host plugin: %w", pluginType, err)
}
}
@@ -516,6 +516,9 @@ func (c *Controller) registerJobs() error {
if err := kmsjob.RegisterJobs(c.baseContext, c.scheduler, c.kms); err != nil {
return err
}
if err := cleaner.RegisterJob(c.baseContext, c.scheduler, rw); err != nil {
return err
}
return nil
}
@@ -0,0 +1,12 @@
begin;
-- Delete any existing completed job_runs to make index
-- creation faster and avoid one very expensive first cleaner job.
delete from job_run where status='completed';
-- Create index for faster deletes.
create index job_run_status_ix on job_run (status);
comment on index job_run_status_ix is
'the job_run_status_ix is used by the job run cleaner job';
commit;
+27
View File
@@ -0,0 +1,27 @@
package cleaner
import (
"context"
"github.com/hashicorp/boundary/internal/db"
"github.com/hashicorp/boundary/internal/errors"
"github.com/hashicorp/boundary/internal/scheduler"
"github.com/hashicorp/boundary/internal/util"
)
// RegisterJob registers the cleaner job with the provided scheduler.
func RegisterJob(ctx context.Context, s *scheduler.Scheduler, w db.Writer) error {
const op = "cleaner.RegisterJob"
if s == nil {
return errors.New(ctx, errors.Internal, "nil scheduler", op, errors.WithoutEvent())
}
if util.IsNil(w) {
return errors.New(ctx, errors.Internal, "nil DB writer", op, errors.WithoutEvent())
}
if err := s.RegisterJob(ctx, newCleanerJob(w)); err != nil {
return errors.Wrap(ctx, err, op)
}
return nil
}
+54
View File
@@ -0,0 +1,54 @@
package cleaner
import (
"context"
"time"
"github.com/hashicorp/boundary/internal/db"
"github.com/hashicorp/boundary/internal/errors"
"github.com/hashicorp/boundary/internal/scheduler"
)
type cleanerJob struct {
w db.Writer
}
func newCleanerJob(w db.Writer) *cleanerJob {
return &cleanerJob{
w: w,
}
}
// Status reports the job’s current status.
func (c *cleanerJob) Status() scheduler.JobStatus {
return scheduler.JobStatus{}
}
// Run performs the required work depending on the implementation.
// The context is used to notify the job that it should exit early.
func (c *cleanerJob) Run(ctx context.Context) error {
const op = "cleaner.(cleanerJob).Run"
if _, err := c.w.Exec(ctx, "delete from job_run where status='completed'", nil); err != nil {
return errors.Wrap(ctx, err, op)
}
return nil
}
// NextRunIn returns the duration until the next job run should be scheduled.
// We report as ready immediately after a successful run. This doesn't mean that
// this job will run immediately, only about as often as the configured scheduler interval.
func (c *cleanerJob) NextRunIn(_ context.Context) (time.Duration, error) {
return 0, nil
}
// Name is the unique name of the job.
func (c *cleanerJob) Name() string {
return "job_run_cleaner"
}
// Description is the human readable description of the job.
func (c *cleanerJob) Description() string {
return "Cleans completed job runs"
}
@@ -0,0 +1,61 @@
package cleaner_test
import (
"context"
"sync"
"testing"
"time"
"github.com/hashicorp/boundary/internal/db"
"github.com/hashicorp/boundary/internal/scheduler"
"github.com/hashicorp/boundary/internal/scheduler/cleaner"
"github.com/hashicorp/boundary/internal/scheduler/job"
"github.com/stretchr/testify/require"
)
func TestCleanerJob(t *testing.T) {
conn, _ := db.TestSetup(t, "postgres")
rw := db.New(conn)
wrapper := db.TestWrapper(t)
s := scheduler.TestScheduler(t, conn, wrapper, scheduler.WithMonitorInterval(10*time.Millisecond))
err := cleaner.RegisterJob(context.Background(), s, rw)
require.NoError(t, err)
wg := &sync.WaitGroup{}
err = s.Start(context.Background(), wg)
require.NoError(t, err)
// Trigger some runs, waiting for the cleaner to run
for i := 0; i < 10; i++ {
s.RunNow()
// Wait to allow for the job to finish
time.Sleep(50 * time.Millisecond)
}
var jobRuns []*job.Run
err = rw.SearchWhere(context.Background(), &jobRuns, "", nil)
require.NoError(t, err)
// We should have run 10 times, as long as some of them
// have been cleaned we should succeed.
require.True(t, len(jobRuns) < 10, "expected fewer than 10 job_run rows, found %d", len(jobRuns))
}
func TestRegisterJob(t *testing.T) {
conn, _ := db.TestSetup(t, "postgres")
rw := db.New(conn)
wrapper := db.TestWrapper(t)
s := scheduler.TestScheduler(t, conn, wrapper)
t.Run("succeeds", func(t *testing.T) {
err := cleaner.RegisterJob(context.Background(), s, rw)
require.NoError(t, err)
})
t.Run("fails-on-nil-scheduler", func(t *testing.T) {
err := cleaner.RegisterJob(context.Background(), nil, rw)
require.Error(t, err)
})
t.Run("fails-on-nil-db-writer", func(t *testing.T) {
err := cleaner.RegisterJob(context.Background(), s, nil)
require.Error(t, err)
})
}