diff --git a/internal/daemon/controller/controller.go b/internal/daemon/controller/controller.go index 3ac24f483..f7567ac01 100644 --- a/internal/daemon/controller/controller.go +++ b/internal/daemon/controller/controller.go @@ -45,7 +45,6 @@ import ( "github.com/hashicorp/boundary/internal/ratelimit" "github.com/hashicorp/boundary/internal/recording" "github.com/hashicorp/boundary/internal/scheduler" - "github.com/hashicorp/boundary/internal/scheduler/cleaner" "github.com/hashicorp/boundary/internal/scheduler/job" "github.com/hashicorp/boundary/internal/server" serversjob "github.com/hashicorp/boundary/internal/server/job" @@ -637,9 +636,6 @@ func (c *Controller) registerJobs() error { if err := kmsjob.RegisterJobs(c.baseContext, c.scheduler, c.kms); err != nil { return err } - if err := cleaner.RegisterJob(c.baseContext, c.scheduler, rw); err != nil { - return err - } if err := snapshot.RegisterJob(c.baseContext, c.scheduler, rw, rw); err != nil { return err } diff --git a/internal/db/schema/migrations/oss/postgres/7/03_job.up.sql b/internal/db/schema/migrations/oss/postgres/7/03_job.up.sql index 472762ca4..b572eff62 100644 --- a/internal/db/schema/migrations/oss/postgres/7/03_job.up.sql +++ b/internal/db/schema/migrations/oss/postgres/7/03_job.up.sql @@ -20,6 +20,7 @@ begin; create trigger immutable_columns before update on job for each row execute procedure immutable_columns('plugin_id', 'name'); + -- updated in 93/01_job_run_clean.up.sql create table job_run_status_enm ( name text not null primary key constraint only_predefined_job_status_allowed @@ -28,6 +29,7 @@ begin; comment on table job_run_status_enm is 'job_run_status_enm is an enumeration table where each row contains a valid job run state.'; + -- updated in 93/01_job_run_clean.up.sql insert into job_run_status_enm (name) values ('running'), diff --git a/internal/db/schema/migrations/oss/postgres/93/01_job_run_clean.up.sql b/internal/db/schema/migrations/oss/postgres/93/01_job_run_clean.up.sql new file mode 100644 index 000000000..d09a19777 --- /dev/null +++ b/internal/db/schema/migrations/oss/postgres/93/01_job_run_clean.up.sql @@ -0,0 +1,37 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: BUSL-1.1 + +-- Boundary's design on removing entries from job_run has changed from having a +-- job that periodically cleans the table to a design where the scheduler +-- handles this by itself if the job is successful. It is possible that some +-- entries are left in the table with this change (eg: Boundary is stopped after +-- some jobs run but before the cleaner job runs). +-- +-- These entries would forever be stored, so this migration cleans them to +-- ensure no dangling rows are left behind. +-- +-- It also updates the valid statues enum to reflect the ones in use. + +begin; + delete from job_run where status = 'completed'; + + delete from job_run where job_name = 'job_run_cleaner'; + delete from job where name = 'job_run_cleaner'; + + comment on index job_run_status_ix is + 'the job_run_status_ix indexes the commonly-used status field'; + + comment on table job_run is + 'job_run is a table where each row represents an instance of a job run that is either actively running or has failed in some way.'; + + -- Since we don't set completed anymore, but rather remove the job_run entry, + -- remove 'completed' from the valid statuses. + -- updates 7/03_job.up.sql. + delete from job_run_status_enm where name = 'completed'; + + alter table job_run_status_enm + drop constraint only_predefined_job_status_allowed, + add constraint only_predefined_job_status_allowed + check(name in ('running', 'failed', 'interrupted')); + +commit; diff --git a/internal/proto/controller/storage/job/store/v1/job.proto b/internal/proto/controller/storage/job/store/v1/job.proto index 949c24927..e362b79ff 100644 --- a/internal/proto/controller/storage/job/store/v1/job.proto +++ b/internal/proto/controller/storage/job/store/v1/job.proto @@ -72,7 +72,7 @@ message JobRun { // @inject_tag: `gorm:"default:0"` uint32 retries_count = 12; - // status of the job run (running, completed, failed or interrupted). + // status of the job run (running, failed or interrupted). // @inject_tag: `gorm:"not_null"` string status = 10; diff --git a/internal/scheduler/additional_verification_test.go b/internal/scheduler/additional_verification_test.go index cacab68ea..bb5ec8c71 100644 --- a/internal/scheduler/additional_verification_test.go +++ b/internal/scheduler/additional_verification_test.go @@ -489,7 +489,7 @@ func TestSchedulerFinalStatusUpdate(t *testing.T) { repo, err := job.NewRepository(ctx, rw, rw, kmsCache) require.NoError(err) - run := waitForRunStatus(t, repo, runId, string(job.Failed)) + run := waitForRunStatus(t, repo, runId, job.Failed) assert.Equal(uint32(10), run.TotalCount) assert.Equal(uint32(10), run.CompletedCount) @@ -502,17 +502,9 @@ func TestSchedulerFinalStatusUpdate(t *testing.T) { runId = runJob.(*runningJob).runId // Complete job without error so CompleteRun is called + completeFn := waitForRunComplete(t, sched, repo, runId, tj.name) jobErr <- nil - - // Report status - jobStatus <- JobStatus{Total: 20, Completed: 20} - - repo, err = job.NewRepository(ctx, rw, rw, kmsCache) - require.NoError(err) - - run = waitForRunStatus(t, repo, runId, string(job.Completed)) - assert.Equal(uint32(20), run.TotalCount) - assert.Equal(uint32(20), run.CompletedCount) + completeFn() baseCnl() close(testDone) @@ -570,12 +562,13 @@ func TestSchedulerRunNow(t *testing.T) { require.True(ok) runId := runJob.(*runningJob).runId - // Complete job - jobCh <- struct{}{} - repo, err := job.NewRepository(ctx, rw, rw, kmsCache) require.NoError(err) - waitForRunStatus(t, repo, runId, string(job.Completed)) + + // Complete job + completeFn := waitForRunComplete(t, sched, repo, runId, tj.name) + jobCh <- struct{}{} + completeFn() // Update job to run immediately once scheduling loop is called err = sched.UpdateJobNextRunInAtLeast(context.Background(), tj.name, 0) @@ -600,9 +593,9 @@ func TestSchedulerRunNow(t *testing.T) { runId = runJob.(*runningJob).runId // Complete job + completeFn = waitForRunComplete(t, sched, repo, runId, tj.name) jobCh <- struct{}{} - - waitForRunStatus(t, repo, runId, string(job.Completed)) + completeFn() // Update job to run again with RunNow option err = sched.UpdateJobNextRunInAtLeast(context.Background(), tj.name, 0, WithRunNow(true)) @@ -620,7 +613,34 @@ func TestSchedulerRunNow(t *testing.T) { close(jobCh) } -func waitForRunStatus(t *testing.T, repo *job.Repository, runId, status string) *job.Run { +func waitForRunComplete(t *testing.T, sched *Scheduler, repo *job.Repository, runId, jobName string) func() { + r, err := repo.LookupRun(context.Background(), runId) + require.NoError(t, err) + require.EqualValues(t, job.Running, r.Status) + + return func() { + timeout := time.NewTimer(5 * time.Second) + for { + select { + case <-timeout.C: + t.Fatal(fmt.Errorf("timed out waiting for job run %q to be completed", runId)) + case <-time.After(100 * time.Millisecond): + } + + // A run is complete when we don't find it in the scheduler's + // running jobs list and also not in the job_run table. + _, ok := sched.runningJobs.Load(jobName) + if !ok { + r, err = repo.LookupRun(context.Background(), runId) + require.Nil(t, r) + require.Nil(t, err) + break + } + } + } +} + +func waitForRunStatus(t *testing.T, repo *job.Repository, runId string, status job.Status) *job.Run { t.Helper() var run *job.Run @@ -636,7 +656,7 @@ func waitForRunStatus(t *testing.T, repo *job.Repository, runId, status string) var err error run, err = repo.LookupRun(context.Background(), runId) require.NoError(t, err) - if run.Status == status { + if run.Status == string(status) { break } } diff --git a/internal/scheduler/cleaner/cleaner.go b/internal/scheduler/cleaner/cleaner.go deleted file mode 100644 index 86358cb58..000000000 --- a/internal/scheduler/cleaner/cleaner.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package cleaner - -import ( - "context" - - "github.com/hashicorp/boundary/internal/db" - "github.com/hashicorp/boundary/internal/errors" - "github.com/hashicorp/boundary/internal/scheduler" - "github.com/hashicorp/boundary/internal/util" -) - -// RegisterJob registers the cleaner job with the provided scheduler. -func RegisterJob(ctx context.Context, s *scheduler.Scheduler, w db.Writer) error { - const op = "cleaner.RegisterJob" - if s == nil { - return errors.New(ctx, errors.Internal, "nil scheduler", op, errors.WithoutEvent()) - } - if util.IsNil(w) { - return errors.New(ctx, errors.Internal, "nil DB writer", op, errors.WithoutEvent()) - } - - if err := s.RegisterJob(ctx, newCleanerJob(w)); err != nil { - return errors.Wrap(ctx, err, op) - } - - return nil -} diff --git a/internal/scheduler/cleaner/cleaner_job.go b/internal/scheduler/cleaner/cleaner_job.go deleted file mode 100644 index a7fa39abd..000000000 --- a/internal/scheduler/cleaner/cleaner_job.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package cleaner - -import ( - "context" - "time" - - "github.com/hashicorp/boundary/internal/db" - "github.com/hashicorp/boundary/internal/errors" - "github.com/hashicorp/boundary/internal/scheduler" -) - -type cleanerJob struct { - w db.Writer -} - -func newCleanerJob(w db.Writer) *cleanerJob { - return &cleanerJob{ - w: w, - } -} - -// Status reports the job’s current status. -func (c *cleanerJob) Status() scheduler.JobStatus { - return scheduler.JobStatus{} -} - -// Run performs the required work depending on the implementation. -// The context is used to notify the job that it should exit early. -func (c *cleanerJob) Run(ctx context.Context, _ time.Duration) error { - const op = "cleaner.(cleanerJob).Run" - - if _, err := c.w.Exec(ctx, "delete from job_run where status='completed'", nil); err != nil { - return errors.Wrap(ctx, err, op) - } - - return nil -} - -// NextRunIn returns the duration until the next job run should be scheduled. -// We report as ready immediately after a successful run. This doesn't mean that -// this job will run immediately, only about as often as the configured scheduler interval. -func (c *cleanerJob) NextRunIn(_ context.Context) (time.Duration, error) { - return 0, nil -} - -// Name is the unique name of the job. -func (c *cleanerJob) Name() string { - return "job_run_cleaner" -} - -// Description is the human readable description of the job. -func (c *cleanerJob) Description() string { - return "Cleans completed job runs" -} diff --git a/internal/scheduler/cleaner/cleaner_test.go b/internal/scheduler/cleaner/cleaner_test.go deleted file mode 100644 index 13eb4bbbd..000000000 --- a/internal/scheduler/cleaner/cleaner_test.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package cleaner_test - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/hashicorp/boundary/internal/db" - "github.com/hashicorp/boundary/internal/scheduler" - "github.com/hashicorp/boundary/internal/scheduler/cleaner" - "github.com/hashicorp/boundary/internal/scheduler/job" - "github.com/stretchr/testify/require" -) - -func TestCleanerJob(t *testing.T) { - conn, _ := db.TestSetup(t, "postgres") - rw := db.New(conn) - wrapper := db.TestWrapper(t) - s := scheduler.TestScheduler(t, conn, wrapper, scheduler.WithMonitorInterval(10*time.Millisecond)) - err := cleaner.RegisterJob(context.Background(), s, rw) - require.NoError(t, err) - wg := &sync.WaitGroup{} - err = s.Start(context.Background(), wg) - require.NoError(t, err) - - // Trigger some runs, waiting for the cleaner to run - for i := 0; i < 10; i++ { - s.RunNow() - // Wait to allow for the job to finish - time.Sleep(50 * time.Millisecond) - } - - var jobRuns []*job.Run - err = rw.SearchWhere(context.Background(), &jobRuns, "", nil) - require.NoError(t, err) - - // We should have run 10 times, as long as some of them - // have been cleaned we should succeed. - require.True(t, len(jobRuns) < 10, "expected fewer than 10 job_run rows, found %d", len(jobRuns)) -} - -func TestRegisterJob(t *testing.T) { - conn, _ := db.TestSetup(t, "postgres") - rw := db.New(conn) - wrapper := db.TestWrapper(t) - s := scheduler.TestScheduler(t, conn, wrapper) - - t.Run("succeeds", func(t *testing.T) { - err := cleaner.RegisterJob(context.Background(), s, rw) - require.NoError(t, err) - }) - t.Run("fails-on-nil-scheduler", func(t *testing.T) { - err := cleaner.RegisterJob(context.Background(), nil, rw) - require.Error(t, err) - }) - t.Run("fails-on-nil-db-writer", func(t *testing.T) { - err := cleaner.RegisterJob(context.Background(), s, nil) - require.Error(t, err) - }) -} diff --git a/internal/scheduler/job/additional_verification_test.go b/internal/scheduler/job/additional_verification_test.go index 0e9c351fd..6ec748efe 100644 --- a/internal/scheduler/job/additional_verification_test.go +++ b/internal/scheduler/job/additional_verification_test.go @@ -54,9 +54,8 @@ func TestJobWorkflow(t *testing.T) { require.NoError(err) assert.Nil(newRuns) - run, err = repo.CompleteRun(ctx, run.PrivateId, time.Hour, 0, 0, 0) + err = repo.CompleteRun(ctx, run.PrivateId, time.Hour) require.NoError(err) - assert.Equal(Completed.string(), run.Status) job, err = repo.LookupJob(ctx, job.Name) require.NoError(err) diff --git a/internal/scheduler/job/doc.go b/internal/scheduler/job/doc.go index dce76363e..b67e53e99 100644 --- a/internal/scheduler/job/doc.go +++ b/internal/scheduler/job/doc.go @@ -45,5 +45,5 @@ // nextJobRun = time.Now().Add(time.Hour) // // repo, _ = job.NewRepository(db, db, wrapper) -// run, _ = repo.CompleteRun(ctx, run.PrivateId, job.Completed, nextJobRun) +// _ = repo.CompleteRun(ctx, run.PrivateId nextJobRun) package job diff --git a/internal/scheduler/job/query.go b/internal/scheduler/job/query.go index 41bd0c318..be3975d9f 100644 --- a/internal/scheduler/job/query.go +++ b/internal/scheduler/job/query.go @@ -74,14 +74,7 @@ const updateProgressQuery = ` ` const completeRunQuery = ` - update - job_run - set - completed_count = ?, - total_count = ?, - retries_count = ?, - status = 'completed', - end_time = current_timestamp + delete from job_run where private_id = ? and status = 'running' diff --git a/internal/scheduler/job/repository_run.go b/internal/scheduler/job/repository_run.go index e31044e99..a13cfc9de 100644 --- a/internal/scheduler/job/repository_run.go +++ b/internal/scheduler/job/repository_run.go @@ -68,7 +68,7 @@ func (r *Repository) RunJobs(ctx context.Context, serverId string, opt ...Option // UpdateProgress updates the repository entry's completed and total counts for the provided runId. // -// Once a run has been persisted with a final run status (completed, failed or interrupted), +// Once a run has been persisted with a final run status (failed or interrupted), // any future UpdateProgress attempts will return an error with Code errors.InvalidJobRunState. // All options are ignored. func (r *Repository) UpdateProgress(ctx context.Context, runId string, completed, total, retries int, _ ...Option) (*Run, error) { @@ -123,32 +123,26 @@ func (r *Repository) UpdateProgress(ctx context.Context, runId string, completed return run, nil } -// CompleteRun updates the Run repository entry for the provided runId. -// It sets the status to 'completed', updates the run's EndTime to the current database -// time, and sets the completed and total counts. -// CompleteRun also updates the Job repository entry that is associated with this run, -// setting the job's NextScheduledRun to the current database time incremented by the nextRunIn +// CompleteRun is intended to be called when a job completes successfully. It +// deletes the job_run entry for the provided runId. It also updates the Job +// repository entry that is associated with this run, setting the job's +// NextScheduledRun to the current database time incremented by the nextRunIn // parameter. // -// Once a run has been persisted with a final run status (completed, failed -// or interrupted), any future calls to CompleteRun will return an error with Code -// errors.InvalidJobRunState. -// All options are ignored. -func (r *Repository) CompleteRun(ctx context.Context, runId string, nextRunIn time.Duration, completed, total, retries int, _ ...Option) (*Run, error) { +// If a run is persisted with a final run status (failed or interrupted), any +// calls to CompleteRun will return an error with Code +// errors.InvalidJobRunState. All options are ignored. +func (r *Repository) CompleteRun(ctx context.Context, runId string, nextRunIn time.Duration, _ ...Option) error { const op = "job.(Repository).CompleteRun" if runId == "" { - return nil, errors.New(ctx, errors.InvalidParameter, op, "missing run id") + return errors.New(ctx, errors.InvalidParameter, op, "missing run id") } run := allocRun() run.PrivateId = runId _, err := r.writer.DoTx(ctx, db.StdRetryCnt, db.ExpBackoff{}, func(r db.Reader, w db.Writer) error { - // TODO (lcr 07/2021) this can potentially overwrite completed and total values - // persisted by the scheduler's monitor jobs loop. - // Add an on update sql trigger to protect the job_run table, once progress - // values are used in the critical path. - rows, err := w.Query(ctx, completeRunQuery, []any{completed, total, retries, runId}) + rows, err := w.Query(ctx, completeRunQuery, []any{runId}) if err != nil { return errors.Wrap(ctx, err, op) } @@ -169,7 +163,8 @@ func (r *Repository) CompleteRun(ctx context.Context, runId string, nextRunIn ti return errors.Wrap(ctx, err, op, errors.WithMsg("unable to get next row for job run")) } if rowCnt == 0 { - // Failed to update run, either it does not exist or was in an invalid state + // No rows returned from the query: Either it's already been + // removed or was in a final state (not 'running'). if err = r.LookupById(ctx, run); err != nil { if errors.IsNotFoundError(err) { return errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("job run %q does not exist", runId))) @@ -206,17 +201,17 @@ func (r *Repository) CompleteRun(ctx context.Context, runId string, nextRunIn ti }, ) if err != nil { - return nil, errors.Wrap(ctx, err, op) + return errors.Wrap(ctx, err, op) } - return run, nil + return nil } // FailRun updates the Run repository entry for the provided runId. // It sets the status to 'failed' and updates the run's EndTime to the current database // time, and sets the completed and total counts. // -// Once a run has been persisted with a final run status (completed, failed +// Once a run has been persisted with a final run status (failed // or interrupted), any future calls to FailRun will return an error with Code // errors.InvalidJobRunState. // All options are ignored. @@ -280,7 +275,7 @@ func (r *Repository) FailRun(ctx context.Context, runId string, completed, total // updated for the provided interruptThreshold. It sets the status to 'interrupted' and // updates the run's EndTime to the current database time. // -// Once a run has been persisted with a final run status (completed, failed +// Once a run has been persisted with a final run status (failed // or interrupted), any future calls to InterruptRuns will return an error with Code // errors.InvalidJobRunState. // WithControllerId is the only valid option diff --git a/internal/scheduler/job/repository_run_test.go b/internal/scheduler/job/repository_run_test.go index fcc415e25..2b8e70f10 100644 --- a/internal/scheduler/job/repository_run_test.go +++ b/internal/scheduler/job/repository_run_test.go @@ -216,7 +216,7 @@ func TestRepository_RunJobsOrder(t *testing.T) { assert.Equal(run.JobPluginId, firstJob.PluginId) // End first job with time between last and middle - _, err = repo.CompleteRun(ctx, run.PrivateId, -6*time.Hour, 0, 0, 0) + err = repo.CompleteRun(ctx, run.PrivateId, -6*time.Hour) require.NoError(err) runs, err = repo.RunJobs(ctx, server.PrivateId) @@ -306,20 +306,6 @@ func TestRepository_UpdateProgress(t *testing.T) { wantErrCode: errors.InvalidJobRunState, wantErrMsg: "job.(Repository).UpdateProgress: db.DoTx: job.(Repository).UpdateProgress: job run was in a final run state: failed: integrity violation: error #115", }, - { - name: "status-already-completed", - orig: &Run{ - JobRun: &store.JobRun{ - JobName: job.Name, - JobPluginId: job.PluginId, - ControllerId: server.PrivateId, - Status: Completed.string(), - }, - }, - wantErr: true, - wantErrCode: errors.InvalidJobRunState, - wantErrMsg: "job.(Repository).UpdateProgress: db.DoTx: job.(Repository).UpdateProgress: job run was in a final run state: completed: integrity violation: error #115", - }, { name: "valid-no-changes", orig: &Run{ @@ -479,14 +465,10 @@ func TestRepository_CompleteRun(t *testing.T) { server := testController(t, conn, wrapper) job := testJob(t, conn, "name", "description", wrapper) - type args struct { - completed, total, retries int - } tests := []struct { name string orig *Run nextRunIn time.Duration - args args wantErr bool wantErrCode errors.Code wantErrMsg string @@ -525,20 +507,6 @@ func TestRepository_CompleteRun(t *testing.T) { wantErrCode: errors.InvalidJobRunState, wantErrMsg: "job.(Repository).CompleteRun: db.DoTx: job.(Repository).CompleteRun: job run was in a final run state: failed: integrity violation: error #115", }, - { - name: "status-already-completed", - orig: &Run{ - JobRun: &store.JobRun{ - JobName: job.Name, - JobPluginId: job.PluginId, - ControllerId: server.PrivateId, - Status: Completed.string(), - }, - }, - wantErr: true, - wantErrCode: errors.InvalidJobRunState, - wantErrMsg: "job.(Repository).CompleteRun: db.DoTx: job.(Repository).CompleteRun: job run was in a final run state: completed: integrity violation: error #115", - }, { name: "valid", orig: &Run{ @@ -561,7 +529,6 @@ func TestRepository_CompleteRun(t *testing.T) { Status: Running.string(), }, }, - args: args{completed: 10, total: 20, retries: 1}, }, } @@ -579,9 +546,13 @@ func TestRepository_CompleteRun(t *testing.T) { require.NoError(err) assert.Empty(tt.orig.EndTime) privateId = tt.orig.PrivateId + + r, err := repo.LookupRun(ctx, privateId) + require.NoError(err) + require.NotNil(r) } - got, err := repo.CompleteRun(ctx, privateId, tt.nextRunIn, tt.args.completed, tt.args.total, tt.args.retries) + err = repo.CompleteRun(ctx, privateId, tt.nextRunIn) if tt.wantErr { require.Error(err) assert.Truef(errors.Match(errors.T(tt.wantErrCode), err), "Unexpected error %s", err) @@ -596,27 +567,21 @@ func TestRepository_CompleteRun(t *testing.T) { return } assert.NoError(err) - require.NotNil(got) - assert.NotEmpty(got.EndTime) - assert.Equal(Completed.string(), got.Status) - assert.Equal(tt.args.completed, int(got.CompletedCount)) - assert.Equal(tt.args.total, int(got.TotalCount)) - assert.Equal(tt.args.retries, int(got.RetriesCount)) updatedJob, err := repo.LookupJob(ctx, tt.orig.JobName) assert.NoError(err) require.NotNil(updatedJob) - // The previous run is ended before the next run is scheduled, therefore the previous - // run end time incremented by the nextRunIn duration, should be less than or equal to the - // NextScheduledRun time that is persisted in the repo. - nextRunAt := updatedJob.NextScheduledRun.AsTime() - previousRunEnd := got.EndTime.AsTime() - assert.Equal(nextRunAt.Round(time.Minute), previousRunEnd.Add(tt.nextRunIn).Round(time.Minute)) + // The next run is expected to be ~ now + whatever duration was + // passed into CompleteRun. + expectedNextRunIn := time.Now().Add(tt.nextRunIn).Round(time.Minute).UTC() + actualNextRunIn := updatedJob.NextScheduledRun.AsTime().Round(time.Minute).UTC() + require.EqualValues(expectedNextRunIn, actualNextRunIn) - // Delete job run so it does not clash with future runs - _, err = repo.deleteRun(ctx, privateId) - assert.NoError(err) + // If we can't find the run, it means it was complete. + r, err := repo.LookupRun(ctx, privateId) + require.NoError(err) + require.Nil(r) }) } @@ -626,9 +591,8 @@ func TestRepository_CompleteRun(t *testing.T) { require.NoError(err) require.NotNil(repo) - got, err := repo.CompleteRun(ctx, "fake-run-id", time.Hour, 0, 0, 0) + err = repo.CompleteRun(ctx, "fake-run-id", time.Hour) require.Error(err) - require.Nil(got) assert.Truef(errors.Match(errors.T(errors.RecordNotFound), err), "Unexpected error %s", err) assert.Equal("job.(Repository).CompleteRun: db.DoTx: job.(Repository).CompleteRun: job run \"fake-run-id\" does not exist: db.LookupById: record not found, search issue: error #1100: dbw.LookupById: record not found", err.Error()) }) @@ -691,20 +655,6 @@ func TestRepository_FailRun(t *testing.T) { wantErrCode: errors.InvalidJobRunState, wantErrMsg: "job.(Repository).FailRun: db.DoTx: job.(Repository).FailRun: job run was in a final run state: failed: integrity violation: error #115", }, - { - name: "status-already-completed", - orig: &Run{ - JobRun: &store.JobRun{ - JobName: job.Name, - JobPluginId: job.PluginId, - ControllerId: server.PrivateId, - Status: Completed.string(), - }, - }, - wantErr: true, - wantErrCode: errors.InvalidJobRunState, - wantErrMsg: "job.(Repository).FailRun: db.DoTx: job.(Repository).FailRun: job run was in a final run state: completed: integrity violation: error #115", - }, { name: "valid", orig: &Run{ diff --git a/internal/scheduler/job/run.go b/internal/scheduler/job/run.go index 2128af7b4..0229c18fe 100644 --- a/internal/scheduler/job/run.go +++ b/internal/scheduler/job/run.go @@ -8,7 +8,7 @@ import ( "google.golang.org/protobuf/proto" ) -// Run represents an instance of a job that is either actively running or has already completed. +// Run represents an instance of a job that is either actively running or has failed in some way. type Run struct { *store.JobRun tableName string `gorm:"-"` diff --git a/internal/scheduler/job/status.go b/internal/scheduler/job/status.go index 5db19ff0d..4fe5df007 100644 --- a/internal/scheduler/job/status.go +++ b/internal/scheduler/job/status.go @@ -9,9 +9,6 @@ const ( // Running represents that the job run is actively running on a server Running Status = "running" - // Completed represents that the job run has successfully finished - Completed Status = "completed" - // Failed represent that the job run had an error during execution Failed Status = "failed" diff --git a/internal/scheduler/job/store/job.pb.go b/internal/scheduler/job/store/job.pb.go index c6d4020a8..f64fae2c2 100644 --- a/internal/scheduler/job/store/job.pb.go +++ b/internal/scheduler/job/store/job.pb.go @@ -138,7 +138,7 @@ type JobRun struct { // a job has retried work. // @inject_tag: `gorm:"default:0"` RetriesCount uint32 `protobuf:"varint,12,opt,name=retries_count,json=retriesCount,proto3" json:"retries_count,omitempty" gorm:"default:0"` - // status of the job run (running, completed, failed or interrupted). + // status of the job run (running, failed or interrupted). // @inject_tag: `gorm:"not_null"` Status string `protobuf:"bytes,10,opt,name=status,proto3" json:"status,omitempty" gorm:"not_null"` // The controller_id of the controller running the job and must be set. diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index b71f77b93..067854a90 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -262,8 +262,6 @@ func (s *Scheduler) runJob(ctx context.Context, wg *sync.WaitGroup, r *job.Run) defer wg.Done() runErr := j.Run(jobContext, s.interruptThreshold) - // Get final status report to update run progress with - status := j.Status() var updateErr error switch { case ctx.Err() != nil: @@ -273,9 +271,11 @@ func (s *Scheduler) runJob(ctx context.Context, wg *sync.WaitGroup, r *job.Run) if inner != nil { event.WriteError(ctx, op, inner, event.WithInfoMsg("error getting next run time", "name", j.Name())) } - _, updateErr = repo.CompleteRun(ctx, r.PrivateId, nextRun, status.Completed, status.Total, status.Retries) + updateErr = repo.CompleteRun(ctx, r.PrivateId, nextRun) default: event.WriteError(ctx, op, runErr, event.WithInfoMsg("job run failed", "run id", r.PrivateId, "name", j.Name())) + + status := j.Status() // Get final status report to update run progress with _, updateErr = repo.FailRun(ctx, r.PrivateId, status.Completed, status.Total, status.Retries) }