Files
threshold/internal/db/migrations/postgres/68_wh_session_facts.up.sql
T
Michael Gaffney a9e806b673 Data warehouse (#475)
* Define initial warehouse schema

* Add session id as the primary key to the session accumulating fact table

* Insert a row to use instead of nulls in the time and date dimensions

* Add missing attributes and a partial index

* Rename files

* Refactor: move fact table to a separate migration

* Add domain types for the warehouse

* Add session connection transaction fact table

* Change columns to match ingress data types

* Add views for host dimension ingress

* Create source and target views for the host and user dimensions

* Generate random ID for dimension table primary key

* Formatting changes

* Add current dimension id to target views

* Implement upsert function for host dimension

* Use domain types for upsert function parameters

* Fix indentation

* Add wh_public_id domain type

* Fix value of type columns when the other columns are null

* Implement upsert function for user dimension

* Add warehouse domain types for timestamps and text fields

* Move fact table migrations to a higher number

* Add functions for getting the current date and current time IDs

* Rename backend to endpoint in column names

* Rename fact tables

The fact tables are actually accumulating fact tables not transactional
fact tables.

* Add default value to connection_authorized_time column

* Update session fact table

* Add session_max_seconds, session_connection_limit, and default_port to wh_host_dimension

* Move session table above connection table

* Add after insert trigger to session table for warehouse ingress

* Add fk from connection warehouse table to session warehouse table

* Implement insert trigger for session connections

* Update warehouse for a user having multiple auth accounts

* Rename warehouse columns to be consistent with operational tables

* Add connection specific columns to the connection fact table

* Update session calculations in warehouse

* Rename functions and formatting changes

* Rename session columns to match names of session states

* Implement warehouse triggers for session and connection state changes

* Run 'make migrations'

* Use a static start and end date for the warehouse date dimension

* Document warehouse functions

* Remove outdated TODO comments

* Add TODO comments for additional dimensions and fact table columns
2020-09-30 14:09:40 -04:00

187 lines
7.4 KiB
PL/PgSQL

begin;
-- Column names for numeric fields that are not a measurement end in id or
-- number. This naming convention enables automatic field type detection in
-- certain data analysis tools.
-- https://help.tableau.com/current/pro/desktop/en-us/data_clean_adm.htm
-- The wh_session_accumulating_fact table is an accumulating snapshot.
-- The table wh_session_accumulating_fact is an accumulating fact table.
-- The grain of the fact table is one row per session.
create table wh_session_accumulating_fact (
session_id wt_public_id primary key,
-- auth token id is a degenerate dimension
auth_token_id wt_public_id not null,
-- foreign keys to the dimension tables
host_id wh_dim_id not null
references wh_host_dimension (id)
on delete restrict
on update cascade,
user_id wh_dim_id not null
references wh_user_dimension (id)
on delete restrict
on update cascade,
-- TODO(mgaffney) 09/2020: add dimension and foreign key for the session
-- termination reason
-- date and time foreign keys
session_pending_date_id integer not null
references wh_date_dimension (id)
on delete restrict
on update cascade,
session_pending_time_id integer not null
references wh_time_of_day_dimension (id)
on delete restrict
on update cascade,
session_pending_time wh_timestamp,
session_active_date_id integer default -1 not null
references wh_date_dimension (id)
on delete restrict
on update cascade,
session_active_time_id integer default -1 not null
references wh_time_of_day_dimension (id)
on delete restrict
on update cascade,
session_active_time wh_timestamp default 'infinity'::timestamptz,
session_canceling_date_id integer default -1 not null
references wh_date_dimension (id)
on delete restrict
on update cascade,
session_canceling_time_id integer default -1 not null
references wh_time_of_day_dimension (id)
on delete restrict
on update cascade,
session_canceling_time wh_timestamp default 'infinity'::timestamptz,
session_terminated_date_id integer default -1 not null
references wh_date_dimension (id)
on delete restrict
on update cascade,
session_terminated_time_id integer default -1 not null
references wh_time_of_day_dimension (id)
on delete restrict
on update cascade,
session_terminated_time wh_timestamp default 'infinity'::timestamptz,
-- TODO(mgaffney) 09/2020: add columns for session expiration
-- TODO(mgaffney) 09/2020: add connection limit. This may need a dimension
-- table and foreign key column to represent unlimited connections.
-- The total number of connections made during the session.
total_connection_count bigint, -- will be null until the first connection is created
-- The total number of bytes received by workers from the client and sent
-- to the endpoint for this session.
-- total_bytes_up is a fully additive measurement.
total_bytes_up wh_bytes_transmitted, -- will be null until the first connection is closed
-- The total number of bytes received by workers from the endpoint and sent
-- to the client for this session.
-- total_bytes_down is a fully additive measurement.
total_bytes_down wh_bytes_transmitted -- will be null until the first connection is closed
);
-- The wh_session_connection_accumulating_fact table is an accumulating fact table.
-- The grain of the fact table is one row per session connection.
create table wh_session_connection_accumulating_fact (
connection_id wt_public_id primary key,
-- session_id is a degenerate dimension
session_id wt_public_id not null
references wh_session_accumulating_fact (session_id)
on delete cascade
on update cascade,
-- foreign keys to the dimension tables
host_id wh_dim_id not null
references wh_host_dimension (id)
on delete restrict
on update cascade,
user_id wh_dim_id not null
references wh_user_dimension (id)
on delete restrict
on update cascade,
-- TODO(mgaffney) 09/2020: add dimension and foreign key for the connection
-- closed reason
-- date and time foreign keys and timestamps
connection_authorized_date_id integer not null
references wh_date_dimension (id)
on delete restrict
on update cascade,
connection_authorized_time_id integer not null
references wh_time_of_day_dimension (id)
on delete restrict
on update cascade,
connection_authorized_time wh_timestamp,
connection_connected_date_id integer default -1 not null
references wh_date_dimension (id)
on delete restrict
on update cascade,
connection_connected_time_id integer default -1 not null
references wh_time_of_day_dimension (id)
on delete restrict
on update cascade,
connection_connected_time wh_timestamp default 'infinity'::timestamptz,
connection_closed_date_id integer default -1 not null
references wh_date_dimension (id)
on delete restrict
on update cascade,
connection_closed_time_id integer default -1 not null
references wh_time_of_day_dimension (id)
on delete restrict
on update cascade,
connection_closed_time wh_timestamp default 'infinity'::timestamptz,
-- TODO(mgaffney) 09/2020: add a connection_duration_in_seconds column
-- The client address and port are degenerate dimensions
client_tcp_address inet, -- can be null
client_tcp_port_number wh_inet_port, -- can be null
-- The endpoint address and port are degenerate dimensions
endpoint_tcp_address inet, -- can be null
endpoint_tcp_port_number wh_inet_port, -- can be null
-- the connection_count must always be 1
-- this is a common pattern in data warehouse models
-- See The Data Warehouse Toolkit, Third Edition
-- by Ralph Kimball and Margy Ross for more information
connection_count smallint default 1 not null
constraint connection_count_must_be_1
check(connection_count = 1),
-- The total number of bytes received by the worker from the client and sent
-- to the endpoint for this connection.
-- bytes_up is a fully additive measurement.
bytes_up wh_bytes_transmitted, -- can be null
-- The total number of bytes received by the worker from the endpoint and sent
-- to the client for this connection.
-- bytes_down is a fully additive measurement.
bytes_down wh_bytes_transmitted -- can be null
);
-- TODO(mgaffney) 09/2020: Research and test if the comment fields are used by
-- data analysis tools.
comment on table wh_session_connection_accumulating_fact is
'The Wh Session Connection Accumulating Fact table is an accumulating fact table. '
'The grain of the fact table is one row per session connection.';
comment on column wh_session_connection_accumulating_fact.bytes_up is
'Bytes Up is the total number of bytes received by the worker from the '
'client and sent to the endpoint for this connection. Bytes Up is a fully '
'additive measurement.';
comment on column wh_session_connection_accumulating_fact.bytes_down is
'Bytes Down is the total number of bytes received by the worker from the '
'endpoint and sent to the client for this connection. Bytes Down is a fully '
'additive measurement.';
create index on wh_session_connection_accumulating_fact(session_id);
commit;