From 8ae658a57088fb1d4b8bec06bca68741299ee929 Mon Sep 17 00:00:00 2001
From: Michael Gaffney <mike@gaffney.cc>
Date: Tue, 25 Apr 2023 12:28:06 -0400
Subject: [PATCH] feat(sql): Add history table and triggers for iam_scope

This adds a history table for `iam_scope`. It also includes reusable
triggers to maintain the included history table and future history
tables.

This also includes a test suite that runs on any table ending in `_hst`.
---
 .../postgres/80/02_history_triggers.up.sql    | 126 ++++++++++++++
 .../migrations/oss/postgres/80/03_iam.up.sql  |  40 +++++
 .../tests/history/history_table_tests.sql     | 158 ++++++++++++++++++
 .../db/sqltest/tests/history/hst_columns.sql  |  57 +++++++
 .../sqltest/tests/history/hst_table_name.sql  |  16 ++
 .../db/sqltest/tests/history/hst_triggers.sql | 116 +++++++++++++
 6 files changed, 513 insertions(+)
 create mode 100644 internal/db/schema/migrations/oss/postgres/80/02_history_triggers.up.sql
 create mode 100644 internal/db/schema/migrations/oss/postgres/80/03_iam.up.sql
 create mode 100644 internal/db/sqltest/tests/history/history_table_tests.sql
 create mode 100644 internal/db/sqltest/tests/history/hst_columns.sql
 create mode 100644 internal/db/sqltest/tests/history/hst_table_name.sql
 create mode 100644 internal/db/sqltest/tests/history/hst_triggers.sql

diff --git a/internal/db/schema/migrations/oss/postgres/80/02_history_triggers.up.sql b/internal/db/schema/migrations/oss/postgres/80/02_history_triggers.up.sql
new file mode 100644
index 000000000..0078f7511
--- /dev/null
+++ b/internal/db/schema/migrations/oss/postgres/80/02_history_triggers.up.sql
@@ -0,0 +1,126 @@
+-- Copyright (c) HashiCorp, Inc.
+-- SPDX-License-Identifier: MPL-2.0
+
+begin;
+
+  create function hst_table_name(table_schema name, operational_table_name name) returns text
+  as $$
+    select quote_ident(table_schema) || '.' || quote_ident(operational_table_name) || '_hst';
+  $$ language sql
+     immutable
+     parallel safe -- all of the functions called are parallel safe
+     cost 1        -- all of the functions called are cost 1
+     strict;       -- means the function returns null on null input
+  comment on function hst_table_name is
+    'Returns the history table name of the operational table';
+
+  create function hst_columns(table_schema name, operational_table_name name) returns setof text
+  as $$
+  declare
+    _opr_tbl regclass := quote_ident(table_schema) || '.' || quote_ident(operational_table_name);
+    _hst_tbl regclass := hst_table_name(table_schema, operational_table_name);
+  begin
+    return query
+         select quote_ident(attname)
+           from pg_attribute
+          where attrelid = _opr_tbl
+            and not attisdropped   -- no dropped (dead) columns
+            and attnum > 0         -- no system columns
+      intersect
+         select quote_ident(attname)
+           from pg_attribute
+          where attrelid = _hst_tbl
+            and not attisdropped   -- no dropped (dead) columns
+            and attnum > 0;        -- no system columns
+  end;
+  $$ language plpgsql
+     stable
+     parallel safe -- all of the functions called are parallel safe
+     -- cost ?     -- cost is unknown since it executes a query
+     strict;       -- means the function returns null on null input
+  comment on function hst_columns is
+    'Returns the intersection of columns between an operational table and it''s history table. '
+    'Raises an error if the operational table or history table do not exist.';
+
+  create function hst_on_insert() returns trigger
+  as $$
+  declare
+    _hst_tbl regclass := hst_table_name(tg_table_schema, tg_table_name);
+    _cols text;
+    _vals text;
+  begin
+
+    select into _cols, _vals
+         string_agg(quote_ident(hst_columns), ', '), string_agg('x.' || quote_ident(hst_columns), ', ')
+    from hst_columns(tg_table_schema, tg_table_name);
+
+    execute format('
+      insert into %s (%s)
+      select %s
+        from (select ($1).*) x', _hst_tbl, _cols, _vals)
+    using new;
+
+    return new;
+  end;
+  $$ language plpgsql;
+  comment on function hst_on_insert is
+    'hst_on_insert is an after insert trigger for any operational table that has a history table.';
+
+  create function hst_on_delete() returns trigger
+  as $$
+  declare
+    _hst_tbl regclass := hst_table_name(tg_table_schema, tg_table_name);
+  begin
+
+    execute format('
+      update %s
+         set valid_range = tstzrange(lower(valid_range), current_timestamp)
+       where public_id = $1.public_id
+         and valid_range @> current_timestamp', _hst_tbl)
+    using old;
+
+    return old;
+  end;
+  $$ language plpgsql;
+  comment on function hst_on_delete is
+    'hst_on_delete is an after delete trigger for any operational table that has a history table.';
+
+  create function hst_on_update() returns trigger
+  as $$
+  declare
+    _hst_tbl regclass := hst_table_name(tg_table_schema, tg_table_name);
+    _cols text;
+    _vals text;
+    _new_values record;
+    _old_values record;
+  begin
+
+    select into _cols, _vals
+         string_agg(quote_ident(hst_columns), ', '), string_agg('x.' || quote_ident(hst_columns), ', ')
+    from hst_columns(tg_table_schema, tg_table_name);
+
+    execute format('select %s from (select ($1).*) x', _cols, _vals) into _new_values using new;
+    execute format('select %s from (select ($1).*) x', _cols, _vals) into _old_values using old;
+    if _new_values is distinct from _old_values then
+    -- if one column has changed,
+      -- same query as on_delete
+      execute format('
+        update %s
+           set valid_range = tstzrange(lower(valid_range), current_timestamp)
+         where public_id = $1.public_id
+           and valid_range @> current_timestamp', _hst_tbl)
+      using new;
+      -- same query as on_insert
+      execute format('
+        insert into %s (%s)
+        select %s
+          from (select ($1).*) x', _hst_tbl, _cols, _vals)
+      using new;
+    end if;
+    return new;
+  end;
+  $$ language plpgsql;
+  comment on function hst_on_update is
+    'hst_on_update is an after update trigger for any operational table that has a history table.';
+
+commit;
diff --git a/internal/db/schema/migrations/oss/postgres/80/03_iam.up.sql b/internal/db/schema/migrations/oss/postgres/80/03_iam.up.sql
new file mode 100644
index 000000000..6ba5d6012
--- /dev/null
+++ b/internal/db/schema/migrations/oss/postgres/80/03_iam.up.sql
@@ -0,0 +1,40 @@
+-- Copyright (c) HashiCorp, Inc.
+-- SPDX-License-Identifier: MPL-2.0
+
+begin;
+
+  create extension if not exists btree_gist;
+
+  create table iam_scope_hst (
+    public_id wt_scope_id not null,
+    name text null,
+    description text null,
+    type text not null
+      constraint iam_scope_type_enm_fkey
+        references iam_scope_type_enm (string)
+        on delete restrict
+        on update cascade,
+    parent_id text null,
+    primary_auth_method_id wt_public_id null,
+    history_id wt_url_safe_id default wt_url_safe_id() primary key,
+    valid_range tstzrange not null default tstzrange(current_timestamp, null),
+    constraint iam_scope_hst_valid_range_excl
+      exclude using gist (public_id with =, valid_range with &&)
+  );
+  comment on table iam_scope_hst is
+    'iam_scope_hst is a history table where each row contains the values from a row '
+    'in the iam_scope table during the time range in the valid_range column.';
+
+  create trigger hst_on_insert after insert on iam_scope
+    for each row execute function hst_on_insert();
+  create trigger hst_on_update after update on iam_scope
+    for each row execute function hst_on_update();
+  create trigger hst_on_delete after delete on iam_scope
+    for each row execute function hst_on_delete();
+
+  insert into iam_scope_hst
+    (public_id, name, description, type, parent_id, primary_auth_method_id)
+  select public_id, name, description, type, parent_id, primary_auth_method_id
+    from iam_scope;
+
+commit;
diff --git a/internal/db/sqltest/tests/history/history_table_tests.sql b/internal/db/sqltest/tests/history/history_table_tests.sql
new file mode 100644
index 000000000..256b6fc10
--- /dev/null
+++ b/internal/db/sqltest/tests/history/history_table_tests.sql
@@ -0,0 +1,158 @@
+-- Copyright (c) HashiCorp, Inc.
+-- SPDX-License-Identifier: MPL-2.0
+
+begin;
+
+  create function get_history_tables() returns setof name
+  as $$
+    select c.relname
+      from pg_catalog.pg_class c
+     where c.relkind in ('r')
+       and c.relname operator(pg_catalog.~) '^(.*hst)$' collate pg_catalog.default
+       and pg_catalog.pg_table_is_visible(c.oid);
+  $$ language sql;
+
+  create function op_table(history_table_name name) returns text
+  as $$
+    select trim(trailing '_hst' from history_table_name);
+  $$ language sql;
+
+  create function has_operational_table(history_table_name name) returns text
+  as $$
+    select has_table(op_table(history_table_name));
+  $$ language sql;
+
+  create function get_col_type(table_name name, column_name name) returns text
+  as $$
+    select pg_catalog.format_type(a.atttypid, a.atttypmod)
+      from pg_catalog.pg_attribute a
+      join pg_catalog.pg_class c on a.attrelid = c.oid
+     where pg_catalog.pg_table_is_visible(c.oid)
+       and c.relname = table_name
+       and a.attname = column_name
+       and attnum > 0
+       and not a.attisdropped;
+  $$ language sql;
+
+  create function col_types_equal(history_table_name name, column_name name) returns text
+  as $$
+  declare
+    want_type text;
+  begin
+    want_type := get_col_type(op_table(history_table_name), column_name);
+    return col_type_is(history_table_name, column_name, want_type);
+  end;
+  $$ language plpgsql;
+
+  create function migrations_have_run(history_table_name name) returns text
+  as $$
+  declare
+    result text;
+  begin
+    execute format('select is(count(opt.*), count(hst.*)) '
+        'from %I opt, %I hst', op_table(history_table_name), history_table_name)
+    into result;
+    return result;
+  end;
+  $$ language plpgsql;
+
+  -- tests the tables exist and follow the required naming pattern
+  create function has_correct_tables(history_table_name name) returns text
+  as $$
+    select * from collect_tap(
+      has_table(history_table_name),
+      has_operational_table(history_table_name),
+      migrations_have_run(history_table_name)
+    );
+  $$ language sql;
+
+  -- tests the public_id column
+  create function has_public_id(history_table_name name) returns text
+  as $$
+    select * from collect_tap(
+      has_column(history_table_name, 'public_id'),
+      col_not_null(history_table_name, 'public_id'),
+      col_types_equal(history_table_name, 'public_id'), -- should be the same type as the operational table
+      col_hasnt_default(history_table_name, 'public_id')
+    );
+  $$ language sql;
+
+  -- tests the history_id column
+  create function has_history_id(history_table_name name) returns text
+  as $$
+    select * from collect_tap(
+      has_column(history_table_name, 'history_id'),
+      col_is_pk(history_table_name, 'history_id'),
+      col_type_is(history_table_name, 'history_id', 'wt_url_safe_id'),
+      col_has_default(history_table_name, 'history_id'),
+      col_default_is(history_table_name, 'history_id', 'wt_url_safe_id()')
+    );
+  $$ language sql;
+
+  -- tests the valid_range column
+  create function has_valid_range(history_table_name name) returns text
+  as $$
+    select * from collect_tap(
+      has_column(history_table_name, 'valid_range'),
+      col_not_null(history_table_name, 'valid_range'),
+      col_type_is(history_table_name, 'valid_range', 'tstzrange'),
+      col_has_default(history_table_name, 'valid_range'),
+      col_default_is(history_table_name, 'valid_range', 'tstzrange(CURRENT_TIMESTAMP, NULL::timestamp with time zone)')
+    );
+  $$ language sql;
+
+  -- tests for an exclusion index
+  create function has_exclusion_index(history_table_name name) returns text
+  as $$
+    select has_index(history_table_name, history_table_name || '_valid_range_excl', array['public_id', 'valid_range']);
+  $$ language sql;
+
+  -- tests to verify certain columns are not in the history table
+  create function hasnt_operational_columns(history_table_name name) returns text
+  as $$
+    select * from collect_tap(
+      hasnt_column(history_table_name, 'create_time'),
+      hasnt_column(history_table_name, 'update_time'),
+      hasnt_column(history_table_name, 'version')
+    );
+  $$ language sql;
+
+  -- tests that the operational table has the history triggers
+  create function has_history_triggers(history_table_name name) returns text
+  as $$
+    select * from collect_tap(
+      has_trigger(op_table(history_table_name), 'hst_on_insert'),
+      trigger_is(op_table(history_table_name), 'hst_on_insert', 'hst_on_insert'),
+      has_trigger(op_table(history_table_name), 'hst_on_update'),
+      trigger_is(op_table(history_table_name), 'hst_on_update', 'hst_on_update'),
+      has_trigger(op_table(history_table_name), 'hst_on_delete'),
+      trigger_is(op_table(history_table_name), 'hst_on_delete', 'hst_on_delete')
+    );
+  $$ language sql;
+
+  -- runs all the tests on a single history table
+  create function test_history_table(history_table_name name) returns text
+  as $$
+    select * from collect_tap(
+      has_correct_tables(history_table_name),
+      has_public_id(history_table_name),
+      has_history_id(history_table_name),
+      has_valid_range(history_table_name),
+      has_exclusion_index(history_table_name),
+      hasnt_operational_columns(history_table_name),
+      has_history_triggers(history_table_name)
+    );
+  $$ language sql;
+
+  -- 27 tests for each history table
+  select plan(a.table_count::integer)
+    from (
+      select 27 * count(*) as table_count
+        from get_history_tables()
+    ) as a;
+
+    select test_history_table(a)
+      from get_history_tables() a;
+
+  select * from finish();
+rollback;
diff --git a/internal/db/sqltest/tests/history/hst_columns.sql b/internal/db/sqltest/tests/history/hst_columns.sql
new file mode 100644
index 000000000..4d00ee999
--- /dev/null
+++ b/internal/db/sqltest/tests/history/hst_columns.sql
@@ -0,0 +1,57 @@
+-- Copyright (c) HashiCorp, Inc.
+-- SPDX-License-Identifier: MPL-2.0
+
+begin;
+  select plan(6);
+
+  -- Verify the function exists and is declared properly
+  select has_function('hst_columns', array['name', 'name']);
+  select volatility_is('hst_columns', 'stable');
+  select is_strict('hst_columns');
+
+  select is(count(*), 0::bigint, 'null input should return 0 rows') from hst_columns(null, null);
+
+  -- close to the iam_user table
+  create table test_r1 (
+    public_id wt_user_id primary key,
+    create_time wt_timestamp,
+    update_time wt_timestamp,
+    name text,
+    description text,
+    scope_id wt_scope_id not null
+      references iam_scope(public_id)
+      on delete cascade
+      on update cascade,
+    unique(name, scope_id),
+    version wt_version
+  );
+
+  create table test_r1_hst (
+    public_id wt_user_id not null,
+    name text,
+    description text,
+    scope_id text not null,
+    history_id wt_url_safe_id default wt_url_safe_id() primary key,
+    valid_range tstzrange not null default tstzrange(current_timestamp, null)
+  );
+
+  select is(count(*), 4::bigint) from hst_columns('public'::name, 'test_r1'::name);
+
+  create table test_r2 (
+    public_id wt_user_id primary key,
+    create_time wt_timestamp,
+    update_time wt_timestamp,
+    name text,
+    description text,
+    scope_id wt_scope_id not null
+      references iam_scope(public_id)
+      on delete cascade
+      on update cascade,
+    version wt_version
+  );
+
+  prepare select_no_history_table as select hst_columns('public'::name, 'test_r2'::name);
+  select throws_like('select_no_history_table', '%relation "public.test_r2_hst" does not exist', 'an operational table without a history table should raise an error');
+
+  select * from finish();
+rollback;
diff --git a/internal/db/sqltest/tests/history/hst_table_name.sql b/internal/db/sqltest/tests/history/hst_table_name.sql
new file mode 100644
index 000000000..2b04019eb
--- /dev/null
+++ b/internal/db/sqltest/tests/history/hst_table_name.sql
@@ -0,0 +1,16 @@
+-- Copyright (c) HashiCorp, Inc.
+-- SPDX-License-Identifier: MPL-2.0
+
+begin;
+  select plan(5);
+
+  -- Verify the function exists and is declared properly
+  select has_function('hst_table_name', array['name', 'name']);
+  select volatility_is('hst_table_name', 'immutable');
+  select is_strict('hst_table_name');
+
+  select is(hst_table_name('public', 'iam_scope'), 'public.iam_scope_hst');
+  select is(hst_table_name(null, null), null);
+
+  select * from finish();
+rollback;
diff --git a/internal/db/sqltest/tests/history/hst_triggers.sql b/internal/db/sqltest/tests/history/hst_triggers.sql
new file mode 100644
index 000000000..b03e0b4af
--- /dev/null
+++ b/internal/db/sqltest/tests/history/hst_triggers.sql
@@ -0,0 +1,116 @@
+-- Copyright (c) HashiCorp, Inc.
+-- SPDX-License-Identifier: MPL-2.0
+
+begin;
+  select plan(28);
+
+  -- Verify the trigger functions exist and are declared properly
+  select has_function('hst_on_insert');
+  select volatility_is('hst_on_insert', 'volatile');
+  select isnt_strict('hst_on_insert');
+
+  select has_function('hst_on_delete');
+  select volatility_is('hst_on_delete', 'volatile');
+  select isnt_strict('hst_on_delete');
+
+  select has_function('hst_on_update');
+  select volatility_is('hst_on_update', 'volatile');
+  select isnt_strict('hst_on_update');
+
+  select has_trigger('iam_scope', 'hst_on_insert');
+  select has_trigger('iam_scope', 'hst_on_update');
+  select has_trigger('iam_scope', 'hst_on_delete');
+
+  select is(count(a.*), count(b.*)) from iam_scope a, iam_scope_hst b;
+
+  -- test on_delete trigger
+  select is(count(*), 1::bigint)
+    from iam_scope_hst
+   where public_id = 'p____rcolors';
+
+  select is(upper(valid_range), null)
+    from iam_scope_hst
+   where public_id = 'p____rcolors';
+
+  select ok(isempty(valid_range) = false)
+    from iam_scope_hst
+   where public_id = 'p____rcolors';
+
+  delete
+    from iam_scope
+   where public_id = 'p____rcolors';
+
+  select is(count(*), 1::bigint)
+    from iam_scope_hst
+   where public_id = 'p____rcolors';
+
+  select is(upper(valid_range), now())
+    from iam_scope_hst
+   where public_id = 'p____rcolors';
+
+  select ok(isempty(valid_range) = false)
+    from iam_scope_hst
+   where public_id = 'p____rcolors';
+
+  -- test on_update trigger
+  select is(count(*), 1::bigint)
+    from iam_scope_hst
+   where public_id = 'p____bcolors';
+
+  update iam_scope
+     set description = 'updated description'
+   where public_id = 'p____bcolors';
+
+  select is(count(*), 2::bigint)
+    from iam_scope_hst
+   where public_id = 'p____bcolors';
+
+  select is(count(*), 1::bigint)
+    from iam_scope_hst
+   where public_id = 'p____bcolors'
+     and upper(valid_range) = now();
+
+  update iam_scope
+     set version = version + 1
+   where public_id = 'p____bcolors';
+
+  select is(count(*), 2::bigint, 'update to version column should not add a new history record')
+    from iam_scope_hst
+   where public_id = 'p____bcolors';
+
+  -- test a delete happening within the same transaction as an update
+  delete
+    from iam_scope
+   where public_id = 'p____bcolors';
+
+  select is(count(*), 2::bigint, 'delete of scope should not add a new history record')
+    from iam_scope_hst
+   where public_id = 'p____bcolors';
+
+ -- Doing an update and a delete in the same transaction produces a row in the
+ -- history table with a valid_range value of 'empty'.
+  select is(count(*), 1::bigint, 'an update and delete in the same transaction should result in an empty valid_range')
+    from iam_scope_hst
+   where public_id = 'p____bcolors'
+     and isempty(valid_range);
+
+  -- test on_insert trigger
+  insert into iam_scope
+    (parent_id, type, public_id, name)
+  values
+    ('o_____colors', 'project', 'p_______test', 'Test Color');
+
+  select is(count(*), 1::bigint)
+    from iam_scope_hst
+   where public_id = 'p_______test';
+
+  select is(lower(valid_range), now())
+    from iam_scope_hst
+   where public_id = 'p_______test';
+
+  select is(upper(valid_range), null)
+    from iam_scope_hst
+   where public_id = 'p_______test';
+
+  select * from finish();
+rollback;