CyberCoveLLC/threshold
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

internal/db: add deleted tables for all resources (#3885)

Browse Source 
Creates a new foo_deleted table for each subtype of each
resource that needs to support pagination. The deleted
tables are used to track deletions of resources and return
them to users of the new pagination API. We track only the
ID and delete_time of the resource, so there's no way to
filter out resources that the user could not see before it
was deleted. This is intentional to keep the design simple
and is not considered a security issue.
This commit is contained in:
Johan Brandhorst-Satzkorn committed 2024-01-05 07:43:19 -08:00
1 parent 263660a7cb
commit 38536f4eef
4 files changed
+470 -1
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/db/schema/migrations/oss/postgres/78/01_deleted_tables_and_triggers.up.sql
+341
View File
@@ -0,0 +1,341 @@
-- Copyright (c) HashiCorp, Inc.
-- SPDX-License-Identifier: BUSL-1.1
begin;
create table target_tcp_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table target_tcp_deleted is
'target_tcp_deleted holds the ID and delete_time of every deleted TCP target. '
'It is automatically trimmed of records older than 30 days by a job.';
create table target_ssh_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table target_ssh_deleted is
'target_ssh_deleted holds the ID and delete_time of every deleted SSH target. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Target repository list logic is implemented
-- against views, so use a view for the deleted
-- ids too.
-- Note: does not end in _deleted so as
-- not to be attached a trigger in the below function.
create view target_all_subtypes_deleted_view
as
select public_id, delete_time from target_tcp_deleted
union
select public_id, delete_time from target_ssh_deleted;
comment on view target_all_subtypes_deleted_view is
'target_all_subtypes_deleted_view holds the ID and delete_time of every deleted target.';
-- Sessions
create table session_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table session_deleted is
'session_deleted holds the ID and delete_time of every deleted session. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Auth methods
create table auth_oidc_method_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table auth_oidc_method_deleted is
'auth_oidc_method_deleted holds the ID and delete_time of every deleted OIDC auth method. '
'It is automatically trimmed of records older than 30 days by a job.';
create table auth_ldap_method_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table auth_ldap_method_deleted is
'auth_ldap_method_deleted holds the ID and delete_time of every deleted LDAP auth method. '
'It is automatically trimmed of records older than 30 days by a job.';
create table auth_password_method_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table auth_password_method_deleted is
'auth_password_method_deleted holds the ID and delete_time of every deleted password auth method. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Auth tokens
create table auth_token_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table auth_token_deleted is
'auth_token_deleted holds the ID and delete_time of every deleted auth token. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Auth accounts
create table auth_oidc_account_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table auth_oidc_account_deleted is
'auth_oidc_account_deleted holds the ID and delete_time of every deleted OIDC account. '
'It is automatically trimmed of records older than 30 days by a job.';
create table auth_ldap_account_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table auth_ldap_account_deleted is
'auth_ldap_account_deleted holds the ID and delete_time of every deleted LDAP account. '
'It is automatically trimmed of records older than 30 days by a job.';
create table auth_password_account_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table auth_password_account_deleted is
'auth_password_account_deleted holds the ID and delete_time of every deleted password account. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Auth managed groups
create table auth_oidc_managed_group_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table auth_oidc_managed_group_deleted is
'auth_oidc_managed_group_deleted holds the ID and delete_time of every deleted OIDC managed group. '
'It is automatically trimmed of records older than 30 days by a job.';
create table auth_ldap_managed_group_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table auth_ldap_managed_group_deleted is
'auth_ldap_managed_group_deleted holds the ID and delete_time of every deleted LDAP managed group. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Credential libraries
create table credential_vault_library_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table credential_vault_library_deleted is
'credential_vault_library_deleted holds the ID and delete_time of every deleted Vault credential library. '
'It is automatically trimmed of records older than 30 days by a job.';
create table credential_vault_ssh_cert_library_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table credential_vault_ssh_cert_library_deleted is
'credential_vault_ssh_cert_library_deleted holds the ID and delete_time of '
'every deleted Vault SSH certificate credential library. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Credentials
create table credential_static_json_credential_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table credential_static_json_credential_deleted is
'credential_static_json_credential_deleted holds the ID and delete_time '
'of every deleted static JSON credential. '
'It is automatically trimmed of records older than 30 days by a job.';
create table credential_static_username_password_credential_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table credential_static_username_password_credential_deleted is
'credential_static_username_password_credential_deleted holds the ID and delete_time '
'of every deleted static username password credential. '
'It is automatically trimmed of records older than 30 days by a job.';
create table credential_static_ssh_private_key_credential_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table credential_static_ssh_private_key_credential_deleted is
'credential_static_ssh_private_key_credential_deleted holds the ID and delete_time '
'of every deleted static ssh private key credential. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Credential stores
create table credential_vault_store_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table credential_vault_store_deleted is
'credential_vault_store_deleted holds the ID and delete_time '
'of every deleted Vault credential store. '
'It is automatically trimmed of records older than 30 days by a job.';
create table credential_static_store_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table credential_static_store_deleted is
'credential_static_store_deleted holds the ID and delete_time '
'of every deleted static credential store. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Groups
create table iam_group_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table iam_group_deleted is
'iam_group_deleted holds the ID and delete_time of every deleted group. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Roles
create table iam_role_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table iam_role_deleted is
'iam_role_deleted holds the ID and delete_time of every deleted role. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Scopes
create table iam_scope_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table iam_scope_deleted is
'iam_scope_deleted holds the ID and delete_time of every deleted scope. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Users
create table iam_user_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table iam_user_deleted is
'iam_user_deleted holds the ID and delete_time of every deleted user. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Host catalogs
create table host_plugin_catalog_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table host_plugin_catalog_deleted is
'host_plugin_catalog_deleted holds the ID and delete_time of every deleted plugin host catalog. '
'It is automatically trimmed of records older than 30 days by a job.';
create table static_host_catalog_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table static_host_catalog_deleted is
'static_host_catalog_deleted holds the ID and delete_time of every deleted static host catalog. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Host sets
create table host_plugin_set_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table host_plugin_set_deleted is
'host_plugin_set_deleted holds the ID and delete_time of every deleted plugin host set. '
'It is automatically trimmed of records older than 30 days by a job.';
create table static_host_set_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table static_host_set_deleted is
'static_host_set_deleted holds the ID and delete_time of every deleted static host set. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Hosts
create table host_plugin_host_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table host_plugin_host_deleted is
'host_plugin_host_deleted holds the ID and delete_time of every deleted plugin host. '
'It is automatically trimmed of records older than 30 days by a job.';
create table static_host_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table static_host_deleted is
'static_host_deleted holds the ID and delete_time of every deleted static host. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Storage buckets
create table storage_plugin_storage_bucket_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table storage_plugin_storage_bucket_deleted is
'storage_plugin_storage_bucket_deleted holds the ID and delete_time of every deleted storage bucket. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Session recordings
create table recording_session_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table recording_session_deleted is
'recording_session_deleted holds the ID and delete_time of every deleted session recording. '
'It is automatically trimmed of records older than 30 days by a job.';
-- Workers
create table server_worker_deleted (
public_id wt_public_id primary key,
delete_time wt_timestamp not null
);
comment on table server_worker_deleted is
'server_worker_deleted holds the ID and delete_time of every deleted worker. '
'It is automatically trimmed of records older than 30 days by a job.';
create function insert_deleted_id() returns trigger
as $$
begin
execute format('insert into %I (public_id, delete_time) values ($1, now()) on conflict (public_id) do update set delete_time = excluded.delete_time', tg_argv[0]) using old.public_id;
return old;
end;
$$ language plpgsql;
comment on function insert_deleted_id is
'insert_deleted_id is a function that inserts a record into the table '
'specified by the first trigger argument. It takes the public ID from the row '
'affected by the trigger and the current timestamp. It is used to populate rows '
'of the deleted tables.';
create function get_deletion_tables() returns setof name
as $$
select c.relname
from pg_catalog.pg_class c
where c.relkind in ('r')
and c.relname operator(pg_catalog.~) '^(.+_deleted)$' collate pg_catalog.default
and pg_catalog.pg_table_is_visible(c.oid);
$$ language sql;
comment on function get_deletion_tables is
'get_deletion_tables returns a set containing all the deleted table names by looking for '
'all tables that end in _deleted.';
-- Assign a "after delete" trigger on all deleted tables to run insert_deleted_id
do $$
declare
deletion_table_name text;
table_name text;
begin
for deletion_table_name in select get_deletion_tables() loop
table_name := split_part(deletion_table_name, '_deleted', 1);
execute format('create trigger insert_deleted_id after delete on %I for each row execute function insert_deleted_id(''%I'')', table_name, deletion_table_name);
execute format('create index %I_delete_time_idx on %I (delete_time)', deletion_table_name, deletion_table_name);
end loop;
end;
$$ language plpgsql;
-- These credential indices end up truncated by postgres, so we manually
-- rename them to something nicer.
alter index credential_static_username_password_credential_deleted_delete_t rename to credential_static_username_password_deleted_delete_time_idx;
alter index credential_static_ssh_private_key_credential_deleted_delete_tim rename to credential_static_ssh_private_key_deleted_delete_time_idx;
commit;
internal/db/sqltest/Makefile
+2 -1
View File
@@ -33,7 +33,8 @@ TESTS ?= tests/setup/*.sql \
tests/domain/*.sql \
tests/history/*.sql \
tests/recording/*.sql \
tests/auth/*/*.sql
tests/auth/*/*.sql \
tests/purge/*sql
POSTGRES_DOCKER_IMAGE_BASE ?= postgres
internal/db/sqltest/tests/purge/deleted_table_tests.sql
+86
View File
@@ -0,0 +1,86 @@
-- Copyright (c) HashiCorp, Inc.
-- SPDX-License-Identifier: BUSL-1.1
begin;
create function op_table(deletion_table_name name) returns text
as $$
select split_part(deletion_table_name, '_deleted', 1);
$$ language sql;
create function has_operational_table(deletion_table_name name) returns text
as $$
select has_table(op_table(deletion_table_name));
$$ language sql;
-- tests that the deletion table has the insert trigger
create function has_insert_trigger(deletion_table_name name) returns text
as $$
select * from collect_tap(
has_trigger(op_table(deletion_table_name), 'insert_deleted_id'),
trigger_is(op_table(deletion_table_name), 'insert_deleted_id', 'insert_deleted_id')
);
$$ language sql;
-- tests the public_id column
create function has_public_id(deletion_table_name name) returns text
as $$
select * from collect_tap(
has_column(deletion_table_name, 'public_id'),
col_not_null(deletion_table_name, 'public_id'),
col_hasnt_default(deletion_table_name, 'public_id')
);
$$ language sql;
-- tests the delete_time column
create function has_delete_time(deletion_table_name name) returns text
as $$
select * from collect_tap(
has_column(deletion_table_name, 'delete_time'),
col_not_null(deletion_table_name, 'delete_time'),
col_hasnt_default(deletion_table_name, 'delete_time')
);
$$ language sql;
-- tests for delete_time index
create function has_delete_time_index(deletion_table_name name) returns text
as $$
select case when length(deletion_table_name || '_delete_time_idx') > 63
then hasnt_index(deletion_table_name, deletion_table_name || '_delete_time_idx', 'Index name too long: ' || deletion_table_name || '_delete_time_idx')
else collect_tap(
has_index(deletion_table_name, deletion_table_name || '_delete_time_idx', 'delete_time')
) end;
$$ language sql;
-- tests the tables exist and follow the required naming pattern
create function has_correct_tables(deletetion_table_name name) returns text
as $$
select * from collect_tap(
has_table(deletetion_table_name),
has_operational_table(deletetion_table_name)
);
$$ language sql;
-- runs all the tests on a single deletion table
create function test_deletion_table(deletion_table_name name) returns text
as $$
select * from collect_tap(
has_correct_tables(deletion_table_name),
has_public_id(deletion_table_name),
has_delete_time(deletion_table_name),
has_delete_time_index(deletion_table_name),
has_insert_trigger(deletion_table_name)
);
$$ language sql;
-- 11 tests for each deletion table
select plan(a.table_count::integer)
from (
select 11 * count(*) as table_count
from get_deletion_tables()
) as a;
select test_deletion_table(a)
from get_deletion_tables() a;
select * from finish();
rollback;
internal/db/sqltest/tests/purge/trigger_insert_on_delete.sql
+41
View File
@@ -0,0 +1,41 @@
-- Copyright (c) HashiCorp, Inc.
-- SPDX-License-Identifier: BUSL-1.1
begin;
select plan(13);
-- Verify the trigger functions exist and are declared properly
select has_function('insert_deleted_id');
select volatility_is('insert_deleted_id', 'volatile');
select isnt_strict('insert_deleted_id');
select has_trigger('auth_token', 'insert_deleted_id');
-- To test the trigger that moves deleted rows into their appropriate tables, we'll use auth_token
-- Ensure session state table is populated, and session_deleted and auth_token_deleted tables are empty
select is(count(*), 1::bigint) from session_state where session_id = 's1_____clare';
select is(count(*), 0::bigint) from session_deleted;
select is(count(*), 0::bigint) from auth_token_deleted;
-- Check that we have a session for a particular auth token
select is(count(*), 2::bigint) from session where auth_token_id = 'tok____clare';
-- Delete auth tokens, expect no errors, check that the auth_token table is now populated
delete from auth_token where public_id = 'tok____clare';
select is(count(*), 0::bigint) from auth_token where public_id = 'tok____clare';
select is(count(*), 1::bigint) from auth_token_deleted;
-- Add new auth token with duplicate public_id, delete and check that there's no error
insert into auth_token
(key_id, auth_account_id, public_id, token)
values
('kdkv__colors', 'apa____clare', 'tok____clare', 'tok____clare'::bytea);
prepare delete_again as
delete from auth_token where public_id = 'tok____clare';
select lives_ok('delete_again');
select is(count(*), 0::bigint) from auth_token where public_id = 'tok____clare';
select is(count(*), 1::bigint) from auth_token_deleted;
select * from finish();
rollback;