feat(internal/db): Storage policies implementation (#767)

This commit implements part of the storage policy's database model,
focusing specifically on the operational tables for this feature.

internal/db/schema/migrations/oss/postgres/100/01_storage_policies.up.sql

@@ -0,0 +1,178 @@
+-- Copyright (c) HashiCorp, Inc.
+-- SPDX-License-Identifier: BUSL-1.1
+
+begin;
+
+create table policy (
+  public_id wt_public_id primary key,
+  scope_id wt_scope_id not null
+  constraint policy_scope_id_fkey
+    references iam_scope(public_id)
+    on delete restrict
+    on update cascade
+);
+comment on table policy is
+'policy contains entries that represent an abstract policy';
+
+create table policy_storage_policy (
+  public_id wt_public_id primary key,
+  scope_id wt_scope_id not null
+  constraint policy_storage_policy_scope_id_fkey
+    references iam_scope(public_id)
+    on delete restrict
+    on update cascade,
+  retain_for_days integer not null,
+  retain_for_days_overridable boolean not null default true,
+  delete_after_days integer not null,
+  constraint delete_after_days_non_negative
+    check(delete_after_days >= 0),
+  constraint delete_after_days_greater_or_equal_than_retain_for_days
+    check(delete_after_days >= retain_for_days or delete_after_days = 0),
+  delete_after_days_overridable boolean not null default true,
+  name wt_name,
+  constraint policy_storage_policy_scope_id_name_uq
+    unique(scope_id, name),
+  description wt_description,
+  create_time wt_timestamp,
+  update_time wt_timestamp,
+  version wt_version
+);
+comment on table policy_storage_policy is
+'policy_storage_policy is a subtype of policy and contains entries that codify '
+'a storage policy and how long to keep session recordings';
+
+create trigger default_create_time_column before insert on policy_storage_policy
+  for each row execute procedure default_create_time();
+
+create trigger update_time_column before update on policy_storage_policy
+  for each row execute procedure update_time_column();
+
+create trigger update_version_column after update on policy_storage_policy
+  for each row execute procedure update_version_column();
+
+create trigger immutable_columns before update on policy_storage_policy
+  for each row execute procedure immutable_columns('public_id', 'scope_id', 'create_time');
+
+create function retain_for_days_and_delete_after_days_not_zero() returns trigger
+as $$
+begin
+  if new.retain_for_days = 0 and new.delete_after_days = 0 then
+    raise exception 'retain_for_days and delete_after_days are both zero';
+  end if;
+  return new;
+end;
+$$ language plpgsql;
+comment on function retain_for_days_and_delete_after_days_not_zero is
+'retain_for_days_and_delete_after_days_not_zero is a trigger function that '
+'ensures that retain_for_days and delete_after_days cannot both be zero';
+
+create trigger retain_for_days_and_delete_after_days_not_zero before insert or update on policy_storage_policy
+  for each row execute procedure retain_for_days_and_delete_after_days_not_zero();
+
+create function delete_after_days_zero_if_infinite_retain_for_days() returns trigger
+as $$
+begin
+  if new.retain_for_days < 0 and new.delete_after_days != 0 then
+    raise exception 'deletion period set on infinite retention period policy';
+  end if;
+  return new;
+end;
+$$ language plpgsql;
+comment on function delete_after_days_zero_if_infinite_retain_for_days is
+'delete_after_days_zero_if_infinite_retain_for_days is a trigger function that '
+'ensures that if retain_for_days is set to infinity (-1), delete_after_days is '
+'zero (working alongside the existing check constraints)';
+
+create trigger delete_after_days_zero_if_infinite_retain_for_days before insert or update on policy_storage_policy
+  for each row execute procedure delete_after_days_zero_if_infinite_retain_for_days();
+
+create function policy_storage_policy_scope_id_valid() returns trigger
+as $$
+begin
+  perform from iam_scope where public_id = new.scope_id and type in ('global', 'org');
+  if not found then
+    raise exception 'invalid scope type for storage policy creation';
+  end if;
+  return new;
+end;
+$$ language plpgsql;
+comment on function policy_storage_policy_scope_id_valid is
+'policy_storage_policy_scope_id_valid is a trigger function that checks that the '
+'scope_id being inserted is a global or org level scope.';
+
+create trigger policy_storage_policy_scope_id_valid before insert on policy_storage_policy
+  for each row execute procedure policy_storage_policy_scope_id_valid();
+
+create function insert_policy_subtype() returns trigger
+as $$
+begin
+  insert into policy
+    (public_id, scope_id)
+  values
+    (new.public_id, new.scope_id);
+  return new;
+end;
+$$ language plpgsql;
+
+create trigger insert_policy_subtype before insert on policy_storage_policy
+  for each row execute procedure insert_policy_subtype();
+
+create function delete_policy_subtype() returns trigger
+as $$
+begin
+  delete from policy where public_id = old.public_id;
+  return null;
+end;
+$$ language plpgsql;
+
+create trigger delete_policy_subtype after delete on policy_storage_policy
+  for each row execute procedure delete_policy_subtype();
+
+create table scope_policy_storage_policy (
+  scope_id wt_scope_id primary key
+  constraint scope_policy_storage_policy_scope_id_fkey
+    references iam_scope (public_id)
+    on update cascade
+    on delete cascade,
+  storage_policy_id wt_public_id not null
+  constraint scope_policy_storage_policy_storage_policy_id_fkey
+    references policy_storage_policy (public_id)
+    on update cascade
+    on delete cascade
+);
+comment on table scope_policy_storage_policy is
+'scope_policy_storage_policy is an association table between a scope and a storage '
+'policy. The scope can only be global or the same scope present in the storage '
+'policy to be associated';
+
+create trigger immutable_columns before update on scope_policy_storage_policy
+  for each row execute procedure immutable_columns('scope_id');
+
+create function scope_policy_storage_policy_scope_id_valid() returns trigger
+as $$
+begin
+  -- An org storage policy can only be applied to that specific org scope,
+  -- whereas a `global` storage policy can be applied to any scope.
+  perform from policy_storage_policy
+  where ( public_id = new.storage_policy_id and scope_id = new.scope_id ) or scope_id = 'global';
+  if not found then
+      raise exception 'invalid scope_id for scope_storage_policy association';
+  end if;
+  return new;
+end;
+$$ language plpgsql;
+comment on function scope_policy_storage_policy_scope_id_valid is
+'scope_policy_storage_policy_scope_id_valid is a trigger function that '
+'checks that the scope_id being inserted is either global or the same org as '
+'the storage policy we want to associate';
+
+create trigger scope_policy_storage_policy_scope_id_valid before insert on scope_policy_storage_policy
+  for each row execute procedure scope_policy_storage_policy_scope_id_valid();
+
+insert into oplog_ticket (name, version)
+values
+  ('policy', 1),
+  ('policy_storage_policy', 1),
+  ('scope_policy_storage_policy', 1);
+
+commit;

internal/db/sqltest/tests/storage/invalid_scope_policy_storage_policy.sql

@@ -0,0 +1,34 @@
+-- Copyright (c) HashiCorp, Inc.
+-- SPDX-License-Identifier: BUSL-1.1
+
+begin;
+  select plan(2);
+  select wtt_load('widgets', 'iam');
+
+  insert into iam_scope
+    (parent_id, type, public_id, name)
+  values
+    ('global', 'org', 'o_____widget2', 'Widget 2 Inc');
+
+  -- insert storage policy.
+  insert into policy_storage_policy
+    (public_id, scope_id, retain_for_days, delete_after_days, name, description)
+  values
+    ('pst____policy', 'o_____widget', -1, 0, 'Test Storage Policy', 'This is a test storage policy');
+
+  -- invalid scope_policy_storage_policy: scope is not global and does not match
+  -- the storage policy scope.
+  select throws_ok($$
+    insert into scope_policy_storage_policy
+      (scope_id, storage_policy_id)
+    values
+      ('o_____widget2', 'pst____policy')
+  $$, 'invalid scope_id for scope_storage_policy association');
+
+  select is(count(*), 0::bigint) from scope_policy_storage_policy
+  where
+    scope_id = 'o_____widget2' and
+    storage_policy_id = 'pst____policy';
+
+  delete from iam_scope where public_id = 'o_____widget2';
+rollback;

internal/db/sqltest/tests/storage/invalid_storage_policy_states.sql

@@ -0,0 +1,133 @@
+-- Copyright (c) HashiCorp, Inc.
+-- SPDX-License-Identifier: BUSL-1.1
+
+begin;
+  select plan(14);
+  select wtt_load('widgets', 'iam');
+
+  -- insert invalid storage policy: delete_after_days is lower than
+  -- retain_for_days.
+  select throws_ok($$
+    insert into policy_storage_policy
+      (public_id, scope_id, retain_for_days, delete_after_days, name, description)
+    values
+      ('pst____policy', 'o_____widget', 5, 1, 'Test Storage Policy', 'This is a test storage policy')
+  $$, 'new row for relation "policy_storage_policy" violates check constraint "delete_after_days_greater_or_equal_than_retain_for_days"');
+
+  -- 'policy' abstract table should not have a new association.
+  select is(count(*), 0::bigint) from policy
+  where
+    public_id = 'pst____policy' and
+    scope_id = 'o_____widget';
+
+  -- insert invalid storage policy: use project-level scope.
+  select throws_ok($$
+    insert into policy_storage_policy
+      (public_id, scope_id, retain_for_days, delete_after_days, name, description)
+    values
+      ('pst____policy', 'p____bwidget', -1, 0, 'Test Storage Policy', 'This is a test storage policy')
+  $$, 'invalid scope type for storage policy creation');
+
+  -- 'policy' abstract table should not have a new association.
+  select is(count(*), 0::bigint) from policy
+  where
+    public_id = 'pst____policy' and
+    scope_id = 'p____bwidget';
+
+  -- insert invalid storage policy: negative delete_after_days.
+  select throws_ok($$
+    insert into policy_storage_policy
+      (public_id, scope_id, retain_for_days, delete_after_days, name, description)
+    values
+      ('pst____policy', 'o_____widget', -1, -1, 'Test Storage Policy', 'This is a test storage policy')
+  $$, 'deletion period set on infinite retention period policy');
+
+  -- 'policy' abstract table should not have a new association.
+  select is(count(*), 0::bigint) from policy
+  where
+    public_id = 'pst____policy' and
+    scope_id = 'o_____widget';
+
+  -- insert valid storage policy to test using same policy name on a given scope.
+  insert into policy_storage_policy
+    (public_id, scope_id, retain_for_days, delete_after_days, name, description)
+  values
+    ('pst____policy', 'o_____widget', -1, 0, 'Test Storage Policy', 'This is a test storage policy');
+
+  -- since we already have a 'Test Storage Policy', this should fail.
+  select throws_ok($$
+    insert into policy_storage_policy
+      (public_id, scope_id, retain_for_days, delete_after_days, name, description)
+    values
+      ('pst____policy2', 'o_____widget', -1, 0, 'Test Storage Policy', 'This is a test storage policy')
+  $$, 'duplicate key value violates unique constraint "policy_storage_policy_scope_id_name_uq"');
+
+  -- cleanup so that we can run more tests with the same insert query below.
+  delete from policy_storage_policy where public_id = 'pst____policy';
+
+  -- 'policy' abstract table should not have a new association.
+  select is(count(*), 0::bigint) from policy
+  where
+    public_id = 'pst____policy' and
+    scope_id = 'o_____widget';
+
+  -- insert invalid storage policy: retain_for_days and delete_after_days are
+  -- both zero.
+  select throws_ok($$
+    insert into policy_storage_policy
+      (public_id, scope_id, retain_for_days, delete_after_days, name, description)
+    values
+      ('pst____policy', 'o_____widget', 0, 0, 'Test Storage Policy', 'This is a test storage policy')
+  $$, 'retain_for_days and delete_after_days are both zero');
+
+  -- insert valid storage policy to test update with both retain_for_days and
+  -- delete_after_days set to zero.
+  insert into policy_storage_policy
+    (public_id, scope_id, retain_for_days, delete_after_days, name, description)
+  values
+    ('pst____policy', 'o_____widget', -1, 0, 'Test Storage Policy', 'This is a test storage policy');
+
+  select throws_ok($$
+    update policy_storage_policy set (retain_for_days, delete_after_days) = (0, 0)
+      where public_id = 'pst____policy'
+  $$, 'retain_for_days and delete_after_days are both zero');
+
+  -- cleanup so that we can run more tests with the same insert query below.
+  delete from policy_storage_policy where public_id = 'pst____policy';
+
+  -- 'policy' abstract table should not have a new association.
+  select is(count(*), 0::bigint) from policy
+  where
+    public_id = 'pst____policy' and
+    scope_id = 'o_____widget';
+
+  -- insert invalid storage policy: infinite retention policy (-1) with a
+  -- deletion policy.
+  select throws_ok($$
+    insert into policy_storage_policy
+      (public_id, scope_id, retain_for_days, delete_after_days, name, description)
+    values
+      ('pst____policy', 'o_____widget', -1, 5, 'Test Storage Policy', 'This is a test storage policy')
+  $$, 'deletion period set on infinite retention period policy');
+
+  -- 'policy' abstract table should not have a new association.
+  select is(count(*), 0::bigint) from policy
+  where
+    public_id = 'pst____policy' and
+    scope_id = 'o_____widget';
+
+  -- insert valid storage policy to test updating infinite retention policy (-1)
+  -- with a deletion policy present.
+  insert into policy_storage_policy
+    (public_id, scope_id, retain_for_days, delete_after_days, name, description)
+  values
+    ('pst____policy', 'o_____widget', 5, 5, 'Test Storage Policy', 'This is a test storage policy');
+
+  select throws_ok($$
+    update policy_storage_policy set retain_for_days = -1
+      where public_id = 'pst____policy'
+  $$, 'deletion period set on infinite retention period policy');
+
+  -- cleanup so that we can run more tests with the same insert query below.
+  delete from policy_storage_policy where public_id = 'pst____policy';
+rollback;

internal/db/sqltest/tests/storage/valid_scope_policy_storage_policy.sql

@@ -0,0 +1,54 @@
+-- Copyright (c) HashiCorp, Inc.
+-- SPDX-License-Identifier: BUSL-1.1
+
+begin;
+  select plan(2);
+  select wtt_load('widgets', 'iam');
+
+  -- insert storage policy.
+  insert into policy_storage_policy
+    (public_id, scope_id, retain_for_days, delete_after_days, name, description)
+  values
+    ('pst____policy', 'o_____widget', -1, 0, 'Test Storage Policy', 'This is a test storage policy');
+
+  -- 'policy' abstract table should now have a new association due to insert
+  -- above.
+  select is(count(*), 1::bigint) from policy
+  where
+    public_id = 'pst____policy' and
+    scope_id = 'o_____widget';
+
+  -- insert into scope_policy_storage_policy association table.
+  insert into scope_policy_storage_policy
+    (scope_id, storage_policy_id)
+  values
+    ('o_____widget', 'pst____policy');
+
+  select is(count(*), 1::bigint) from scope_policy_storage_policy
+  where
+    scope_id = 'o_____widget' and
+    storage_policy_id = 'pst____policy';
+
+  -- Reset to allow for next test.
+  delete from scope_policy_storage_policy
+  where
+    scope_id = 'o_____widget' and
+    storage_policy_id = 'pst____policy';
+
+  -- A `global` storage policy should be able to be associated with any scope
+  -- insert storage policy.
+  insert into policy_storage_policy
+    (public_id, scope_id, retain_for_days, delete_after_days, name, description)
+  values
+    ('pst____policy2', 'global', -1, 0, 'Test Storage Policy 2', 'This is a test storage policy');
+
+  insert into scope_policy_storage_policy
+    (scope_id, storage_policy_id)
+  values
+    ('global', 'pst____policy2');
+
+  insert into scope_policy_storage_policy
+    (scope_id, storage_policy_id)
+  values
+    ('o_____widget', 'pst____policy2');
+rollback;

internal/db/sqltest/tests/storage/valid_storage_policy_lifecycle.sql

@@ -0,0 +1,66 @@
+-- Copyright (c) HashiCorp, Inc.
+-- SPDX-License-Identifier: BUSL-1.1
+
+begin;
+  select plan(5);
+  select wtt_load('widgets', 'iam');
+
+  -- insert storage policy.
+  insert into policy_storage_policy
+    (public_id, scope_id, retain_for_days, delete_after_days, name, description)
+  values
+    ('pst____policy', 'o_____widget', 5, 10, 'Test Storage Policy', 'This is a test storage policy');
+
+  -- 'policy' abstract table should now have a new association due to insert
+  -- above.
+  select is(count(*), 1::bigint) from policy
+  where
+    public_id = 'pst____policy' and
+    scope_id = 'o_____widget';
+
+  -- update a storage policy's retain_for_days
+  update policy_storage_policy set retain_for_days = 1
+  where
+    public_id = 'pst____policy' and
+    scope_id = 'o_____widget';
+
+  select is(retain_for_days, 1::integer) from policy_storage_policy
+  where
+    public_id = 'pst____policy' and
+    scope_id = 'o_____widget';
+
+  -- update a storage policy's delete_after_days
+  update policy_storage_policy set delete_after_days = 20
+  where
+    public_id = 'pst____policy' and
+    scope_id = 'o_____widget';
+
+  select is(delete_after_days, 20::integer) from policy_storage_policy
+  where
+    public_id = 'pst____policy' and
+    scope_id = 'o_____widget';
+
+  -- delete a storage policy.
+  delete from policy_storage_policy
+  where
+    public_id = 'pst____policy' and
+    scope_id = 'o_____widget';
+
+  -- row should be deleted from 'policy' abstract table.
+  select is(count(*), 0::bigint) from policy
+  where
+    public_id = 'pst____policy' and
+    scope_id = 'o_____widget';
+
+  -- delete_after_days is less than retain_for_days, however it is 0, which is a
+  -- valid carve-out on delete_after_days_greater_or_equal_than_retain_for_days.
+  insert into policy_storage_policy
+    (public_id, scope_id, retain_for_days, delete_after_days, name, description)
+  values
+    ('pst____policy2', 'o_____widget', 5, 0, 'Test Storage Policy 2', 'This is a test storage policy');
+
+  select is(count(*), 1::bigint) from policy
+  where
+    public_id = 'pst____policy2' and
+    scope_id = 'o_____widget';
+rollback;