CyberCoveLLC/threshold
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sql): Add history table and triggers for iam_scope

Browse Source 
This adds a history table for `iam_scope`. It also includes reusable
triggers to maintain the included history table and future history
tables.

This also includes a test suite that runs on any table ending in `_hst`.
This commit is contained in:
Michael Gaffney authored and Timothy Messier committed 2023-05-31 12:18:14 +00:00
1 parent 327d950a1e
commit 8ae658a570
6 files changed
+513
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/db/schema/migrations/oss/postgres/80/02_history_triggers.up.sql
+126
View File
@@ -0,0 +1,126 @@
-- Copyright (c) HashiCorp, Inc.
-- SPDX-License-Identifier: MPL-2.0
begin;
create function hst_table_name(table_schema name, operational_table_name name) returns text
as $$
select quote_ident(table_schema) || '.' || quote_ident(operational_table_name) || '_hst';
$$ language sql
immutable
parallel safe -- all of the functions called are parallel safe
cost 1 -- all of the functions called are cost 1
strict; -- means the function returns null on null input
comment on function hst_table_name is
'Returns the history table name of the operational table';
create function hst_columns(table_schema name, operational_table_name name) returns setof text
as $$
declare
_opr_tbl regclass := quote_ident(table_schema) || '.' || quote_ident(operational_table_name);
_hst_tbl regclass := hst_table_name(table_schema, operational_table_name);
begin
return query
select quote_ident(attname)
from pg_attribute
where attrelid = _opr_tbl
and not attisdropped -- no dropped (dead) columns
and attnum > 0 -- no system columns
intersect
select quote_ident(attname)
from pg_attribute
where attrelid = _hst_tbl
and not attisdropped -- no dropped (dead) columns
and attnum > 0; -- no system columns
end;
$$ language plpgsql
stable
parallel safe -- all of the functions called are parallel safe
-- cost ? -- cost is unknown since it executes a query
strict; -- means the function returns null on null input
comment on function hst_columns is
'Returns the intersection of columns between an operational table and it''s history table. '
'Raises an error if the operational table or history table do not exist.';
create function hst_on_insert() returns trigger
as $$
declare
_hst_tbl regclass := hst_table_name(tg_table_schema, tg_table_name);
_cols text;
_vals text;
begin
select into _cols, _vals
string_agg(quote_ident(hst_columns), ', '), string_agg('x.' || quote_ident(hst_columns), ', ')
from hst_columns(tg_table_schema, tg_table_name);
execute format('
insert into %s (%s)
select %s
from (select ($1).*) x', _hst_tbl, _cols, _vals)
using new;
return new;
end;
$$ language plpgsql;
comment on function hst_on_insert is
'hst_on_insert is an after insert trigger for any operational table that has a history table.';
create function hst_on_delete() returns trigger
as $$
declare
_hst_tbl regclass := hst_table_name(tg_table_schema, tg_table_name);
begin
execute format('
update %s
set valid_range = tstzrange(lower(valid_range), current_timestamp)
where public_id = $1.public_id
and valid_range @> current_timestamp', _hst_tbl)
using old;
return old;
end;
$$ language plpgsql;
comment on function hst_on_delete is
'hst_on_delete is an after delete trigger for any operational table that has a history table.';
create function hst_on_update() returns trigger
as $$
declare
_hst_tbl regclass := hst_table_name(tg_table_schema, tg_table_name);
_cols text;
_vals text;
_new_values record;
_old_values record;
begin
select into _cols, _vals
string_agg(quote_ident(hst_columns), ', '), string_agg('x.' || quote_ident(hst_columns), ', ')
from hst_columns(tg_table_schema, tg_table_name);
execute format('select %s from (select ($1).*) x', _cols, _vals) into _new_values using new;
execute format('select %s from (select ($1).*) x', _cols, _vals) into _old_values using old;
if _new_values is distinct from _old_values then
-- if one column has changed,
-- same query as on_delete
execute format('
update %s
set valid_range = tstzrange(lower(valid_range), current_timestamp)
where public_id = $1.public_id
and valid_range @> current_timestamp', _hst_tbl)
using new;
-- same query as on_insert
execute format('
insert into %s (%s)
select %s
from (select ($1).*) x', _hst_tbl, _cols, _vals)
using new;
end if;
return new;
end;
$$ language plpgsql;
comment on function hst_on_update is
'hst_on_update is an after update trigger for any operational table that has a history table.';
commit;
internal/db/schema/migrations/oss/postgres/80/03_iam.up.sql
+40
View File
@@ -0,0 +1,40 @@
-- Copyright (c) HashiCorp, Inc.
-- SPDX-License-Identifier: MPL-2.0
begin;
create extension if not exists btree_gist;
create table iam_scope_hst (
public_id wt_scope_id not null,
name text null,
description text null,
type text not null
constraint iam_scope_type_enm_fkey
references iam_scope_type_enm (string)
on delete restrict
on update cascade,
parent_id text null,
primary_auth_method_id wt_public_id null,
history_id wt_url_safe_id default wt_url_safe_id() primary key,
valid_range tstzrange not null default tstzrange(current_timestamp, null),
constraint iam_scope_hst_valid_range_excl
exclude using gist (public_id with =, valid_range with &&)
);
comment on table iam_scope_hst is
'iam_scope_hst is a history table where each row contains the values from a row '
'in the iam_scope table during the time range in the valid_range column.';
create trigger hst_on_insert after insert on iam_scope
for each row execute function hst_on_insert();
create trigger hst_on_update after update on iam_scope
for each row execute function hst_on_update();
create trigger hst_on_delete after delete on iam_scope
for each row execute function hst_on_delete();
insert into iam_scope_hst
(public_id, name, description, type, parent_id, primary_auth_method_id)
select public_id, name, description, type, parent_id, primary_auth_method_id
from iam_scope;
commit;
internal/db/sqltest/tests/history/history_table_tests.sql
+158
View File
@@ -0,0 +1,158 @@
-- Copyright (c) HashiCorp, Inc.
-- SPDX-License-Identifier: MPL-2.0
begin;
create function get_history_tables() returns setof name
as $$
select c.relname
from pg_catalog.pg_class c
where c.relkind in ('r')
and c.relname operator(pg_catalog.~) '^(.*hst)$' collate pg_catalog.default
and pg_catalog.pg_table_is_visible(c.oid);
$$ language sql;
create function op_table(history_table_name name) returns text
as $$
select trim(trailing '_hst' from history_table_name);
$$ language sql;
create function has_operational_table(history_table_name name) returns text
as $$
select has_table(op_table(history_table_name));
$$ language sql;
create function get_col_type(table_name name, column_name name) returns text
as $$
select pg_catalog.format_type(a.atttypid, a.atttypmod)
from pg_catalog.pg_attribute a
join pg_catalog.pg_class c on a.attrelid = c.oid
where pg_catalog.pg_table_is_visible(c.oid)
and c.relname = table_name
and a.attname = column_name
and attnum > 0
and not a.attisdropped;
$$ language sql;
create function col_types_equal(history_table_name name, column_name name) returns text
as $$
declare
want_type text;
begin
want_type := get_col_type(op_table(history_table_name), column_name);
return col_type_is(history_table_name, column_name, want_type);
end;
$$ language plpgsql;
create function migrations_have_run(history_table_name name) returns text
as $$
declare
result text;
begin
execute format('select is(count(opt.*), count(hst.*)) '
'from %I opt, %I hst', op_table(history_table_name), history_table_name)
into result;
return result;
end;
$$ language plpgsql;
-- tests the tables exist and follow the required naming pattern
create function has_correct_tables(history_table_name name) returns text
as $$
select * from collect_tap(
has_table(history_table_name),
has_operational_table(history_table_name),
migrations_have_run(history_table_name)
);
$$ language sql;
-- tests the public_id column
create function has_public_id(history_table_name name) returns text
as $$
select * from collect_tap(
has_column(history_table_name, 'public_id'),
col_not_null(history_table_name, 'public_id'),
col_types_equal(history_table_name, 'public_id'), -- should be the same type as the operational table
col_hasnt_default(history_table_name, 'public_id')
);
$$ language sql;
-- tests the history_id column
create function has_history_id(history_table_name name) returns text
as $$
select * from collect_tap(
has_column(history_table_name, 'history_id'),
col_is_pk(history_table_name, 'history_id'),
col_type_is(history_table_name, 'history_id', 'wt_url_safe_id'),
col_has_default(history_table_name, 'history_id'),
col_default_is(history_table_name, 'history_id', 'wt_url_safe_id()')
);
$$ language sql;
-- tests the valid_range column
create function has_valid_range(history_table_name name) returns text
as $$
select * from collect_tap(
has_column(history_table_name, 'valid_range'),
col_not_null(history_table_name, 'valid_range'),
col_type_is(history_table_name, 'valid_range', 'tstzrange'),
col_has_default(history_table_name, 'valid_range'),
col_default_is(history_table_name, 'valid_range', 'tstzrange(CURRENT_TIMESTAMP, NULL::timestamp with time zone)')
);
$$ language sql;
-- tests for an exclusion index
create function has_exclusion_index(history_table_name name) returns text
as $$
select has_index(history_table_name, history_table_name || '_valid_range_excl', array['public_id', 'valid_range']);
$$ language sql;
-- tests to verify certain columns are not in the history table
create function hasnt_operational_columns(history_table_name name) returns text
as $$
select * from collect_tap(
hasnt_column(history_table_name, 'create_time'),
hasnt_column(history_table_name, 'update_time'),
hasnt_column(history_table_name, 'version')
);
$$ language sql;
-- tests that the operational table has the history triggers
create function has_history_triggers(history_table_name name) returns text
as $$
select * from collect_tap(
has_trigger(op_table(history_table_name), 'hst_on_insert'),
trigger_is(op_table(history_table_name), 'hst_on_insert', 'hst_on_insert'),
has_trigger(op_table(history_table_name), 'hst_on_update'),
trigger_is(op_table(history_table_name), 'hst_on_update', 'hst_on_update'),
has_trigger(op_table(history_table_name), 'hst_on_delete'),
trigger_is(op_table(history_table_name), 'hst_on_delete', 'hst_on_delete')
);
$$ language sql;
-- runs all the tests on a single history table
create function test_history_table(history_table_name name) returns text
as $$
select * from collect_tap(
has_correct_tables(history_table_name),
has_public_id(history_table_name),
has_history_id(history_table_name),
has_valid_range(history_table_name),
has_exclusion_index(history_table_name),
hasnt_operational_columns(history_table_name),
has_history_triggers(history_table_name)
);
$$ language sql;
-- 27 tests for each history table
select plan(a.table_count::integer)
from (
select 27 * count(*) as table_count
from get_history_tables()
) as a;
select test_history_table(a)
from get_history_tables() a;
select * from finish();
rollback;
internal/db/sqltest/tests/history/hst_columns.sql
+57
View File
@@ -0,0 +1,57 @@
-- Copyright (c) HashiCorp, Inc.
-- SPDX-License-Identifier: MPL-2.0
begin;
select plan(6);
-- Verify the function exists and is declared properly
select has_function('hst_columns', array['name', 'name']);
select volatility_is('hst_columns', 'stable');
select is_strict('hst_columns');
select is(count(*), 0::bigint, 'null input should return 0 rows') from hst_columns(null, null);
-- close to the iam_user table
create table test_r1 (
public_id wt_user_id primary key,
create_time wt_timestamp,
update_time wt_timestamp,
name text,
description text,
scope_id wt_scope_id not null
references iam_scope(public_id)
on delete cascade
on update cascade,
unique(name, scope_id),
version wt_version
);
create table test_r1_hst (
public_id wt_user_id not null,
name text,
description text,
scope_id text not null,
history_id wt_url_safe_id default wt_url_safe_id() primary key,
valid_range tstzrange not null default tstzrange(current_timestamp, null)
);
select is(count(*), 4::bigint) from hst_columns('public'::name, 'test_r1'::name);
create table test_r2 (
public_id wt_user_id primary key,
create_time wt_timestamp,
update_time wt_timestamp,
name text,
description text,
scope_id wt_scope_id not null
references iam_scope(public_id)
on delete cascade
on update cascade,
version wt_version
);
prepare select_no_history_table as select hst_columns('public'::name, 'test_r2'::name);
select throws_like('select_no_history_table', '%relation "public.test_r2_hst" does not exist', 'an operational table without a history table should raise an error');
select * from finish();
rollback;
internal/db/sqltest/tests/history/hst_table_name.sql
+16
View File
@@ -0,0 +1,16 @@
-- Copyright (c) HashiCorp, Inc.
-- SPDX-License-Identifier: MPL-2.0
begin;
select plan(5);
-- Verify the function exists and is declared properly
select has_function('hst_table_name', array['name', 'name']);
select volatility_is('hst_table_name', 'immutable');
select is_strict('hst_table_name');
select is(hst_table_name('public', 'iam_scope'), 'public.iam_scope_hst');
select is(hst_table_name(null, null), null);
select * from finish();
rollback;
internal/db/sqltest/tests/history/hst_triggers.sql
+116
View File
@@ -0,0 +1,116 @@
-- Copyright (c) HashiCorp, Inc.
-- SPDX-License-Identifier: MPL-2.0
begin;
select plan(28);
-- Verify the trigger functions exist and are declared properly
select has_function('hst_on_insert');
select volatility_is('hst_on_insert', 'volatile');
select isnt_strict('hst_on_insert');
select has_function('hst_on_delete');
select volatility_is('hst_on_delete', 'volatile');
select isnt_strict('hst_on_delete');
select has_function('hst_on_update');
select volatility_is('hst_on_update', 'volatile');
select isnt_strict('hst_on_update');
select has_trigger('iam_scope', 'hst_on_insert');
select has_trigger('iam_scope', 'hst_on_update');
select has_trigger('iam_scope', 'hst_on_delete');
select is(count(a.*), count(b.*)) from iam_scope a, iam_scope_hst b;
-- test on_delete trigger
select is(count(*), 1::bigint)
from iam_scope_hst
where public_id = 'p____rcolors';
select is(upper(valid_range), null)
from iam_scope_hst
where public_id = 'p____rcolors';
select ok(isempty(valid_range) = false)
from iam_scope_hst
where public_id = 'p____rcolors';
delete
from iam_scope
where public_id = 'p____rcolors';
select is(count(*), 1::bigint)
from iam_scope_hst
where public_id = 'p____rcolors';
select is(upper(valid_range), now())
from iam_scope_hst
where public_id = 'p____rcolors';
select ok(isempty(valid_range) = false)
from iam_scope_hst
where public_id = 'p____rcolors';
-- test on_update trigger
select is(count(*), 1::bigint)
from iam_scope_hst
where public_id = 'p____bcolors';
update iam_scope
set description = 'updated description'
where public_id = 'p____bcolors';
select is(count(*), 2::bigint)
from iam_scope_hst
where public_id = 'p____bcolors';
select is(count(*), 1::bigint)
from iam_scope_hst
where public_id = 'p____bcolors'
and upper(valid_range) = now();
update iam_scope
set version = version + 1
where public_id = 'p____bcolors';
select is(count(*), 2::bigint, 'update to version column should not add a new history record')
from iam_scope_hst
where public_id = 'p____bcolors';
-- test a delete happening within the same transaction as an update
delete
from iam_scope
where public_id = 'p____bcolors';
select is(count(*), 2::bigint, 'delete of scope should not add a new history record')
from iam_scope_hst
where public_id = 'p____bcolors';
-- Doing an update and a delete in the same transaction produces a row in the
-- history table with a valid_range value of 'empty'.
select is(count(*), 1::bigint, 'an update and delete in the same transaction should result in an empty valid_range')
from iam_scope_hst
where public_id = 'p____bcolors'
and isempty(valid_range);
-- test on_insert trigger
insert into iam_scope
(parent_id, type, public_id, name)
values
('o_____colors', 'project', 'p_______test', 'Test Color');
select is(count(*), 1::bigint)
from iam_scope_hst
where public_id = 'p_______test';
select is(lower(valid_range), now())
from iam_scope_hst
where public_id = 'p_______test';
select is(upper(valid_range), null)
from iam_scope_hst
where public_id = 'p_______test';
select * from finish();
rollback;