From 37c6b8e5400143786a215a8e50bf15b15fbb8cf3 Mon Sep 17 00:00:00 2001 From: Michael Gaffney Date: Fri, 5 May 2023 15:18:39 -0400 Subject: [PATCH] feat(sql): Add history tables for hosts This adds history tables for `static_host` and `host_plugin_host`. This also adds a base table for the host history tables. This also adds a new test to the history table test suite. The new test verifies the contents of each history table against the contents of their operational table. --- .../oss/postgres/70/11_host_history.up.sql | 130 ++++++++++++++++++ .../db/sqltest/initdb.d/01_colors_persona.sql | 51 +++++++ .../sqltest/initdb.d/03_widgets_persona.sql | 3 +- .../tests/history/history_table_tests.sql | 57 +++++++- internal/db/sqltest/tests/history/hosts.sql | 38 +++++ 5 files changed, 272 insertions(+), 7 deletions(-) create mode 100644 internal/db/schema/migrations/oss/postgres/70/11_host_history.up.sql create mode 100644 internal/db/sqltest/tests/history/hosts.sql diff --git a/internal/db/schema/migrations/oss/postgres/70/11_host_history.up.sql b/internal/db/schema/migrations/oss/postgres/70/11_host_history.up.sql new file mode 100644 index 000000000..382b9c365 --- /dev/null +++ b/internal/db/schema/migrations/oss/postgres/70/11_host_history.up.sql @@ -0,0 +1,130 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + + create table host_history_base ( + history_id wt_url_safe_id primary key + ); + comment on table host_history_base is + 'host_history_base is a base history table ' + 'for host history tables.'; + + create function insert_host_history_subtype() returns trigger + as $$ + begin + insert into host_history_base + (history_id) + values + (new.history_id); + return new; + end; + $$ language plpgsql; + comment on function insert_host_history_subtype is + 'insert_host_history_subtype is a before insert trigger ' + 'function for subtypes of host_history_base.'; + + create function delete_host_history_subtype() returns trigger + as $$ + begin + delete + from host_history_base + where history_id = old.history_id; + return null; -- result is ignored since this is an after trigger + end; + $$ language plpgsql; + comment on function delete_host_history_subtype is + 'delete_host_history_subtype() is an after delete trigger ' + 'function for subtypes of host_history_base.'; + + create table no_host_history ( + history_id wt_url_safe_id primary key + constraint host_history_base_fkey + references host_history_base (history_id) + on delete restrict + on update restrict + ); + comment on table no_host_history is + 'no_host_history is a table with one row to represent ' + 'the case of a target with a direct address associated to it.'; + + insert into host_history_base values ('_______none'); + insert into no_host_history values ('_______none'); + + create trigger immutable_table before insert or update or delete on no_host_history + for each row execute procedure immutable_table(); + + create table static_host_hst ( + public_id wt_public_id not null, + name text null, + description text null, + catalog_id wt_public_id not null, + address text not null, + history_id wt_url_safe_id default wt_url_safe_id() primary key + constraint host_history_base_fkey + references host_history_base (history_id) + on delete cascade + on update cascade, + valid_range tstzrange not null default tstzrange(current_timestamp, null), + constraint static_host_hst_valid_range_excl + exclude using gist (public_id with =, valid_range with &&) + ); + comment on table static_host_hst is + 'static_host_hst is a history table where each row contains the values from a row ' + 'in the static_host table during the time range in the valid_range column.'; + + create trigger insert_host_history_subtype before insert on static_host_hst + for each row execute function insert_host_history_subtype(); + create trigger delete_host_history_subtype after delete on static_host_hst + for each row execute function delete_host_history_subtype(); + + create trigger hst_on_insert after insert on static_host + for each row execute function hst_on_insert(); + create trigger hst_on_update after update on static_host + for each row execute function hst_on_update(); + create trigger hst_on_delete after delete on static_host + for each row execute function hst_on_delete(); + + insert into static_host_hst + (public_id, name, description, catalog_id, address) + select public_id, name, description, catalog_id, address + from static_host; + + create table host_plugin_host_hst ( + public_id wt_public_id not null, + name wt_name null, + description text null, + catalog_id wt_public_id not null, + external_id text not null, + external_name text null, + history_id wt_url_safe_id default wt_url_safe_id() primary key + constraint host_history_base_fkey + references host_history_base (history_id) + on delete cascade + on update cascade, + valid_range tstzrange not null default tstzrange(current_timestamp, null), + constraint host_plugin_host_hst_valid_range_excl + exclude using gist (public_id with =, valid_range with &&) + ); + comment on table host_plugin_host_hst is + 'host_plugin_host_hst is a history table where each row contains the values from a row ' + 'in the host_plugin_host table during the time range in the valid_range column.'; + + create trigger insert_host_history_subtype before insert on host_plugin_host_hst + for each row execute function insert_host_history_subtype(); + create trigger delete_host_history_subtype after delete on host_plugin_host_hst + for each row execute function delete_host_history_subtype(); + + create trigger hst_on_insert after insert on host_plugin_host + for each row execute function hst_on_insert(); + create trigger hst_on_update after update on host_plugin_host + for each row execute function hst_on_update(); + create trigger hst_on_delete after delete on host_plugin_host + for each row execute function hst_on_delete(); + + insert into host_plugin_host_hst + (public_id, name, description, catalog_id, external_id, external_name) + select public_id, name, description, catalog_id, external_id, external_name + from host_plugin_host; + +commit; diff --git a/internal/db/sqltest/initdb.d/01_colors_persona.sql b/internal/db/sqltest/initdb.d/01_colors_persona.sql index 0fb1fe3a7..6bb487e81 100644 --- a/internal/db/sqltest/initdb.d/01_colors_persona.sql +++ b/internal/db/sqltest/initdb.d/01_colors_persona.sql @@ -295,6 +295,57 @@ begin; ('p____bcolors', 'plg_____chost', 'c___cb-plghcl', 'Blue Color Plugin Catalog', ''), ('p____rcolors', 'plg_____chost', 'c___cr-plghcl', 'Red Color Plugin Catalog', ''); + insert into host_plugin_host + (catalog_id, public_id, external_id) + values + ('c___cb-plghcl', 'h_____cb__01-plgh', '1 blue color'), + ('c___cb-plghcl', 'h_____cb__02-plgh', '2 blue color'), + ('c___cb-plghcl', 'h_____cb__03-plgh', '3 blue color'), + + ('c___cr-plghcl', 'h_____cr__01-plgh', '1 red color'), + ('c___cr-plghcl', 'h_____cr__02-plgh', '2 red color'), + ('c___cr-plghcl', 'h_____cr__03-plgh', '3 red color'); + + insert into host_dns_name + (host_id, name) + values + ('h_____cb__01-plgh', '1.blue.color'), + ('h_____cb__02-plgh', '2.blue.color'), + ('h_____cb__03-plgh', '3.blue.color'), + + ('h_____cr__01-plgh', '1.red.color'), + ('h_____cr__02-plgh', '2.red.color'), + ('h_____cr__03-plgh', '3.red.color'); + + insert into host_ip_address + (host_id, address) + values + ('h_____cb__01-plgh', '1.1.1.1'), + ('h_____cb__02-plgh', 'fe80::2222:2222:2222:2222'), + -- host 3 only has a dns name so the set of addresses are the same + -- between the static and plugin based host + + ('h_____cr__01-plgh', '11.11.11.11'), + ('h_____cr__02-plgh', '2001:4860:4860::2222'), + ('h_____cr__03-plgh', '33.33.33.33'); + + insert into host_plugin_set + (catalog_id, public_id, name, attributes, need_sync) + values + ('c___cb-plghcl', 's___1cb-plghs', 'Blue Color Plugin Set 1', '', false), + ('c___cb-plghcl', 's___2cb-plghs', 'Blue Color Plugin Set 2', '', false), + ('c___cr-plghcl', 's___1cr-plghs', 'Red Color Plugin Set 1', '', false), + ('c___cr-plghcl', 's___2cr-plghs', 'Red Color Plugin Set 2', '', false); + + insert + into host_plugin_set_member + ( host_id, set_id, catalog_id) + select h.public_id, s.public_id, s.catalog_id + from host_plugin_host as h, + host_plugin_set as s + where h.catalog_id = s.catalog_id + and h.external_id like '%color'; + insert into target_tcp (project_id, public_id, name) values diff --git a/internal/db/sqltest/initdb.d/03_widgets_persona.sql b/internal/db/sqltest/initdb.d/03_widgets_persona.sql index 5cc1e0102..6521cc319 100644 --- a/internal/db/sqltest/initdb.d/03_widgets_persona.sql +++ b/internal/db/sqltest/initdb.d/03_widgets_persona.sql @@ -334,7 +334,8 @@ begin; select h.public_id, s.public_id, s.catalog_id from host_plugin_host as h, host_plugin_set as s - where h.catalog_id = s.catalog_id; + where h.catalog_id = s.catalog_id + and h.external_id like '%widget'; end; $$ language plpgsql; diff --git a/internal/db/sqltest/tests/history/history_table_tests.sql b/internal/db/sqltest/tests/history/history_table_tests.sql index 6dcbd57b0..0c921f374 100644 --- a/internal/db/sqltest/tests/history/history_table_tests.sql +++ b/internal/db/sqltest/tests/history/history_table_tests.sql @@ -3,6 +3,24 @@ begin; + create function get_columns(history_table_name name) returns setof text + as $$ + begin + return query + select quote_ident(attname) + from pg_attribute + where attrelid = op_table(history_table_name)::regclass + and not attisdropped -- no dropped (dead) columns + and attnum > 0 -- no system columns + intersect + select quote_ident(attname) + from pg_attribute + where attrelid = history_table_name::regclass + and not attisdropped -- no dropped (dead) columns + and attnum > 0; -- no system columns + end; + $$ language plpgsql; + create function get_history_tables() returns setof name as $$ select c.relname @@ -44,25 +62,52 @@ begin; end; $$ language plpgsql; - create function migrations_have_run(history_table_name name) returns text + create function has_expected_row_count(history_table_name name) returns text as $$ declare result text; begin - execute format('select is(count(opt.*), count(hst.*)) ' - 'from %I opt, %I hst', op_table(history_table_name), history_table_name) + execute format('select results_eq( ' + ' ''select count(*) from %I'', ' + ' ''select count(*) from %I'') ', + op_table(history_table_name), history_table_name) into result; return result; end; $$ language plpgsql; + create function has_expected_content(history_table_name name) returns text + as $$ + declare + _cols text; + _q1 text; + _q2 text; + begin + select into _cols + string_agg(quote_ident(get_columns), ', ') + from get_columns(history_table_name); + select into _q1 format('select %s from %s', _cols, history_table_name); + select into _q2 format('select %s from %s', _cols, op_table(history_table_name)); + return results_eq(_q1, _q2); + + end; + $$ language plpgsql; + + create function has_expected_data(history_table_name name) returns text + as $$ + select * from collect_tap( + has_expected_row_count(history_table_name), + has_expected_content(history_table_name) + ); + $$ language sql; + -- tests the tables exist and follow the required naming pattern create function has_correct_tables(history_table_name name) returns text as $$ select * from collect_tap( has_table(history_table_name), has_operational_table(history_table_name), - migrations_have_run(history_table_name) + has_expected_data(history_table_name) ); $$ language sql; @@ -144,10 +189,10 @@ begin; ); $$ language sql; - -- 27 tests for each history table + -- 28 tests for each history table select plan(a.table_count::integer) from ( - select 27 * count(*) as table_count + select 28 * count(*) as table_count from get_history_tables() ) as a; diff --git a/internal/db/sqltest/tests/history/hosts.sql b/internal/db/sqltest/tests/history/hosts.sql new file mode 100644 index 000000000..0808cb788 --- /dev/null +++ b/internal/db/sqltest/tests/history/hosts.sql @@ -0,0 +1,38 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + + select plan(15); + + -- Verify the trigger functions exist and are declared properly + select has_function('insert_host_history_subtype'); + select volatility_is('insert_host_history_subtype', 'volatile'); + select isnt_strict('insert_host_history_subtype'); + + select has_function('delete_host_history_subtype'); + select volatility_is('delete_host_history_subtype', 'volatile'); + select isnt_strict('delete_host_history_subtype'); + + select has_trigger('static_host_hst', 'insert_host_history_subtype'); + select has_trigger('static_host_hst', 'delete_host_history_subtype'); + select fk_ok('static_host_hst', 'history_id', 'host_history_base' , 'history_id'); + + select has_trigger('host_plugin_host_hst', 'insert_host_history_subtype'); + select has_trigger('host_plugin_host_hst', 'delete_host_history_subtype'); + select fk_ok('host_plugin_host_hst', 'history_id', 'host_history_base' , 'history_id'); + + select is(count(*), 1::bigint) from no_host_history; + select fk_ok('no_host_history', 'history_id', 'host_history_base' , 'history_id'); + + select results_eq( + 'select ' + '(select count(*) from static_host_hst) + ' + '(select count(*) from host_plugin_host_hst) + ' + '(select count(*) from no_host_history)', + 'select count(*) from host_history_base' + ); + + select * from finish(); +rollback; +