CyberCoveLLC/threshold
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(db): Add function to update billing snapshot table

Browse Source 
This adds the `update_sessions_pending_daily_snapshot` SQL function
which updates the `sessions_pending_daily_snapshot` table by inserting
values for any days missing from the table.

This also adds a check constraint to the
`sessions_pending_daily_snapshot` table along with a `comment on table`.

This also removes the hcp daily billing views since they are no longer
needed.

Ref: ace2def49d
This commit is contained in:
Michael Gaffney committed 2023-06-05 12:48:15 -04:00
1 parent 41bdfb4406
commit 8159ddb08e
4 files changed
+185 -197
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/db/schema/migrations/oss/postgres/70/01_hcp_billing_daily.up.sql
+69 -52
View File
@@ -2,63 +2,80 @@
-- SPDX-License-Identifier: MPL-2.0
begin;
create table sessions_pending_daily_snapshot (
snapshot_date date primary key,
sessions_pending_count bigint not null
constraint sessions_pending_count_must_be_zero_or_positive
check(sessions_pending_count >= 0)
);
comment on table sessions_pending_daily_snapshot is
'sessions_pending_count is a table where each row contains the count of '
'sessions pending for snapshot_date for that date.';
create view hcp_billing_daily_sessions_yesterday as
with
daily_counts (day, sessions_pending_count) as (
select date_trunc('day', session_pending_time), count(*)
from wh_session_accumulating_fact
where session_pending_time >= date_trunc('day', timestamp 'yesterday')
and session_pending_time < date_trunc('day', timestamp 'today')
group by date_trunc('day', session_pending_time)
),
daily_range (day) as (
select date_trunc('day', timestamp 'yesterday')
),
final (day, sessions_pending_count) as (
select daily_range.day, coalesce(daily_counts.sessions_pending_count, 0)
from daily_range
left join daily_counts on daily_range.day = daily_counts.day
)
select day, sessions_pending_count
from final
order by day desc;
comment on view hcp_billing_daily_sessions_yesterday is
'hcp_billing_daily_sessions_yesterday is a view that contains '
'the sum of pending sessions '
'from the beginning of the previous day '
'until the start of the current day (exclusive).';
create function update_sessions_pending_daily_snapshot()
returns setof sessions_pending_daily_snapshot
as $$
begin
create view hcp_billing_daily_sessions_all as
with
daily_counts (day, sessions_pending_count) as (
select date_trunc('day', session_pending_time), count(*)
from wh_session_accumulating_fact
where session_pending_time < date_trunc('day', timestamp 'today')
group by date_trunc('day', session_pending_time)
),
daily_range (day) as (
select bucket
from generate_series(
date_trunc('day', (select min(session_pending_time) from wh_session_accumulating_fact)),
timestamp 'yesterday',
'1 day'::interval
-- already ran for today
if (date_trunc('day', now()) - '1 day'::interval) = (select max(snapshot_date) from sessions_pending_daily_snapshot)
then return;
end if;
-- never run before and there are only sessions starting from today
if (select count(*) from sessions_pending_daily_snapshot) = 0
and date_trunc('day', now()) = (select min(session_pending_time) from wh_session_accumulating_fact)
then return query
insert into sessions_pending_daily_snapshot
(snapshot_date, sessions_pending_count)
values
(date_trunc('day', now()) - '1 day'::interval, 0)
returning *;
return;
end if;
return query
with
daily_counts (day, sessions_pending_count) as (
select date_trunc('day', session_pending_time), count(*)
from wh_session_accumulating_fact
where session_pending_time < date_trunc('day', now()) -- before midnight today
and session_pending_time >= coalesce((select max(snapshot_date) from sessions_pending_daily_snapshot), '-infinity')
group by date_trunc('day', session_pending_time)
),
daily_range (day) as (
select bucket
from generate_series(
coalesce(date_trunc('day', (select min(session_pending_time) from wh_session_accumulating_fact)),
date_trunc('day', now()) - '1 day'::interval),
now() - '1 day'::interval,
'1 day'::interval
) as bucket
),
final (day, sessions_pending_count) as (
select daily_range.day::timestamp,
coalesce(daily_counts.sessions_pending_count, 0)
from daily_range
left join daily_counts on daily_range.day = daily_counts.day
)
select day, sessions_pending_count
from final
order by day desc;
comment on view hcp_billing_daily_sessions_all is
'hcp_billing_daily_sessions_all is a view that contains '
'the sum of pending sessions for yesterday and all previous days.';
),
missing (day, sessions_pending_count) as (
select daily_range.day::timestamp with time zone,
coalesce(daily_counts.sessions_pending_count, 0)
from daily_range
left join daily_counts on daily_range.day = daily_counts.day
),
final (day, sessions_pending_count) as (
insert into sessions_pending_daily_snapshot
(snapshot_date, sessions_pending_count)
select day::date, sessions_pending_count
from missing
returning *
)
select day, sessions_pending_count
from final
order by day desc;
end;
$$ language plpgsql
set timezone to 'utc';
comment on function update_sessions_pending_daily_snapshot is
'update_sessions_pending_daily_snapshot is a function that updates the sessions_pending_daily_snapshot table by '
'querying the data warehouse and inserting the session pending counts for any days since the max snapshot_date '
'and yesterday. '
'update_sessions_pending_daily_snapshot returns the rows inserted or null if no rows are inserted.';
commit;
internal/db/sqltest/tests/hcp/billing/daily_sessions_all.sql
-70
View File
@@ -1,70 +0,0 @@
-- Copyright (c) HashiCorp, Inc.
-- SPDX-License-Identifier: MPL-2.0
begin;
select plan(9);
select has_view('hcp_billing_daily_sessions_all', 'view for hcp billing does not exist');
select lives_ok('truncate wh_session_connection_accumulating_fact, wh_session_accumulating_fact',
'Truncate tables in preparation for testing');
-- validate the warehouse fact tables are empty
select is(count(*), 0::bigint, 'wh_session_connection_accumulating_fact is not empty') from wh_session_connection_accumulating_fact;
select is(count(*), 0::bigint, 'wh_session_accumulating_fact is not empty' ) from wh_session_accumulating_fact;
select is(count(*), 0::bigint, 'hcp_billing_daily_sessions_all should always return 0 rows when there are no sessions') from hcp_billing_daily_sessions_all;
-- insert one session per minute for the past 171 hours
-- 171 is 7.125 days, should return 8 days
with
dim_keys (host_key, user_key, credential_group_key) as (
select h.key, u.key, 'no credentials'
from (select key from wh_host_dimension limit 1) as h,
(select key from wh_user_dimension limit 1) as u
),
time_series (date_key, time_key, time) as (
select wh_date_key(time), wh_time_key(time), time
from generate_series(
now() - interval '171 hours',
now(),
interval '1 minute'
) as time
),
fake_sessions (session_id, auth_token_id,
host_key, user_key, credential_group_key,
session_pending_date_key, session_pending_time_key, session_pending_time) as (
select concat('s__________', t.date_key, t.time_key), concat('a__________', t.date_key, t.time_key),
k.host_key, k.user_key, k.credential_group_key,
t.date_key, t.time_key,t.time
from dim_keys as k,
time_series as t
)
insert into wh_session_accumulating_fact
(session_id, auth_token_id,
host_key, user_key, credential_group_key,
session_pending_date_key, session_pending_time_key, session_pending_time
)
select session_id, auth_token_id,
host_key, user_key, credential_group_key,
session_pending_date_key, session_pending_time_key, session_pending_time
from fake_sessions;
select is(count(*), 1::bigint, 'hcp_billing_daily_sessions_yesterday should always return 1 rows') from hcp_billing_daily_sessions_yesterday;
select is(count(*), 7::bigint, 'hcp_billing_daily_sessions_all should return 7 rows') from hcp_billing_daily_sessions_all;
select results_eq(
'select sessions_pending_count::bigint from hcp_billing_daily_sessions_all limit 1',
'select 1440::bigint',
'hcp_billing_daily_sessions_all: session count for the previous day is incorrect'
);
select results_eq(
'select count(*)::bigint from wh_session_accumulating_fact',
'select (select sum(sessions_pending_count)::bigint from hcp_billing_daily_sessions_all) + (select (select extract(minute from now())::bigint + 1) + (select extract(hour from now())::bigint * 60))',
'hcp_billing_daily_sessions_all sum of sessions is incorrect'
);
select * from finish();
rollback;
internal/db/sqltest/tests/hcp/billing/daily_sessions_yesterday.sql
-75
View File
@@ -1,75 +0,0 @@
-- Copyright (c) HashiCorp, Inc.
-- SPDX-License-Identifier: MPL-2.0
begin;
select plan(9);
select has_view('hcp_billing_daily_sessions_yesterday', 'view for hcp billing does not exist');
select lives_ok('truncate wh_session_connection_accumulating_fact, wh_session_accumulating_fact',
'Truncate tables in preparation for testing');
-- validate the warehouse fact tables are empty
select is(count(*), 0::bigint, 'wh_session_connection_accumulating_fact is not empty') from wh_session_connection_accumulating_fact;
select is(count(*), 0::bigint, 'wh_session_accumulating_fact is not empty' ) from wh_session_accumulating_fact;
select is(count(*), 1::bigint, 'hcp_billing_daily_sessions_yesterday should always return 1 row') from hcp_billing_daily_sessions_yesterday;
-- insert one session per minute from one minute before midnight of yesterday until one minute past midnight today
-- 1442 total sessions
with
dim_keys (host_key, user_key, credential_group_key) as (
select h.key, u.key, 'no credentials'
from (select key from wh_host_dimension limit 1) as h,
(select key from wh_user_dimension limit 1) as u
),
time_series (date_key, time_key, time) as (
select wh_date_key(time), wh_time_key(time), time
from generate_series(
timestamp 'yesterday' - interval '1 minute',
timestamp 'today' + interval '1 minute',
interval '1 minute'
) as time
),
fake_sessions (session_id, auth_token_id,
host_key, user_key, credential_group_key,
session_pending_date_key, session_pending_time_key, session_pending_time) as (
select concat('s__________', t.date_key, t.time_key), concat('a__________', t.date_key, t.time_key),
k.host_key, k.user_key, k.credential_group_key,
t.date_key, t.time_key,t.time
from dim_keys as k,
time_series as t
)
insert into wh_session_accumulating_fact
(session_id, auth_token_id,
host_key, user_key, credential_group_key,
session_pending_date_key, session_pending_time_key, session_pending_time
)
select session_id, auth_token_id,
host_key, user_key, credential_group_key,
session_pending_date_key, session_pending_time_key, session_pending_time
from fake_sessions;
select is(count(*), 1::bigint, 'hcp_billing_daily_sessions_yesterday should always return 1 row') from hcp_billing_daily_sessions_yesterday;
select results_eq(
'select count(*)::bigint from wh_session_accumulating_fact',
'select sum(sessions_pending_count)::bigint + 3 from hcp_billing_daily_sessions_yesterday',
'hcp_billing_daily_sessions_yesterday: the sum of sessions is incorrect'
);
select results_eq(
'select sessions_pending_count::bigint from hcp_billing_daily_sessions_yesterday limit 1',
'select 1440::bigint',
'hcp_billing_daily_sessions_yesterday: session count for the previous day is incorrect'
);
select results_eq(
'select * from hcp_billing_daily_sessions_yesterday',
'select * from hcp_billing_daily_sessions_all limit 1',
'hcp_billing_daily_sessions_yesterday and hcp_billing_daily_sessions_all: latest day should be equal'
);
select * from finish();
rollback;
internal/db/sqltest/tests/hcp/billing/update_sessions_pending_daily_snapshot.sql
+116
View File
@@ -0,0 +1,116 @@
-- Copyright (c) HashiCorp, Inc.
-- SPDX-License-Identifier: MPL-2.0
begin;
select plan(27);
select has_function('update_sessions_pending_daily_snapshot');
select volatility_is('update_sessions_pending_daily_snapshot', 'volatile');
select isnt_strict('update_sessions_pending_daily_snapshot');
prepare call_update_sessions_pending_daily_snapshot
as select * from update_sessions_pending_daily_snapshot();
create function test_add_session(ts timestamptz) returns void
as $$
with time_series (time) as (
select ts
),
dim_keys (host_key, user_key, credential_group_key) as (
select h.key, u.key, 'no credentials'
from (select key from wh_host_dimension limit 1) as h,
(select key from wh_user_dimension limit 1) as u
),
dim_time_series (date_key, time_key, time) as (
select wh_date_key(time), wh_time_key(time), time
from time_series
),
fake_sessions (session_id, auth_token_id,
host_key, user_key, credential_group_key,
session_pending_date_key, session_pending_time_key, session_pending_time) as (
select concat('s__________', t.date_key, t.time_key), concat('a__________', t.date_key, t.time_key),
k.host_key, k.user_key, k.credential_group_key,
t.date_key, t.time_key,t.time
from dim_keys as k,
dim_time_series as t
)
insert into wh_session_accumulating_fact
(session_id, auth_token_id,
host_key, user_key, credential_group_key,
session_pending_date_key, session_pending_time_key, session_pending_time
)
select session_id, auth_token_id,
host_key, user_key, credential_group_key,
session_pending_date_key, session_pending_time_key, session_pending_time
from fake_sessions;
$$ language sql;
create function today() returns timestamptz
as $$
select date_trunc('day', now(), 'utc');
$$ language sql;
create function yesterday() returns timestamptz
as $$
select today() - '1 day'::interval;
$$ language sql;
create function has_empty_table(table_name name) returns text
as $$
declare
result text;
begin
execute format('select is(count(*), 0::bigint) from %I', table_name) into result;
return result;
end;
$$ language plpgsql;
create table test_table_data (
snapshot_date date primary key,
sessions_pending_count bigint not null
);
prepare select_test_table_data as select * from test_table_data order by snapshot_date desc;
-- add 5 to plan every time this is called
create function reset_data() returns text
as $$
select * from collect_tap(
lives_ok('truncate wh_session_connection_accumulating_fact, wh_session_accumulating_fact, sessions_pending_daily_snapshot, test_table_data'),
has_empty_table('wh_session_connection_accumulating_fact'),
has_empty_table('wh_session_accumulating_fact'),
has_empty_table('sessions_pending_daily_snapshot'),
has_empty_table('test_table_data')
);
$$ language sql;
-- new install, no sessions
select reset_data();
-- no sessions
insert into test_table_data (snapshot_date, sessions_pending_count) select yesterday()::date, 0;
select results_eq('call_update_sessions_pending_daily_snapshot', 'select_test_table_data');
select is(t.*, null, 'update_sessions_pending_daily_snapshot should return null when it has already been run for the day') from update_sessions_pending_daily_snapshot() as t;
-- new install, only sessions are for today
select reset_data();
select test_add_session(today());
insert into test_table_data (snapshot_date, sessions_pending_count) select yesterday()::date, 0;
select results_eq('call_update_sessions_pending_daily_snapshot', 'select_test_table_data');
-- upgrade install, sessions are for today and yesterday
select reset_data();
select test_add_session(yesterday());
select test_add_session(today());
insert into test_table_data (snapshot_date, sessions_pending_count) select yesterday()::date, 1;
select results_eq('call_update_sessions_pending_daily_snapshot', 'select_test_table_data');
-- upgrade install, sessions are for today, yesterday, and 3 days ago
select reset_data();
select test_add_session(yesterday() - '1 day'::interval);
select test_add_session(yesterday());
select test_add_session(today());
insert into test_table_data (snapshot_date, sessions_pending_count) select yesterday()::date - '1 day'::interval, 1;
insert into test_table_data (snapshot_date, sessions_pending_count) select yesterday()::date, 1;
select results_eq('call_update_sessions_pending_daily_snapshot', 'select_test_table_data');
select * from finish();
rollback;