CyberCoveLLC/threshold
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add DW network address dimension tables (#1855)

Browse Source
This commit is contained in:
Todd authored and GitHub committed 2022-02-14 16:30:18 -07:00
1 parent 8cab9a4315
commit 11ce54f8fb
16 files changed
+979 -30
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

CHANGELOG.md
+5
View File
@@ -9,6 +9,11 @@ Canonical reference for changes, improvements, and bugfixes for Boundary.
* cli: Update authentication examples to remove password flag and make
subcommend selection a bit clearer
([PR](https://github.com/hashicorp/boundary/pull/1835))
* Data Warehouse: Add addresses on plugin based hosts to the database warehouse.
3 new dimension tables have been added including `wh_network_address_group`
(which is now referenced by `wh_host_dimension`),
`wh_network_address_dimension`, and `wh_network_address_group_membership`.
([PR](https://github.com/hashicorp/boundary/pull/1855))
### Bug Fixes
* targets: Specifying a plugin based host id when authorizing a session
internal/db/schema/migrations/oss/postgres/15/02_wh_rename_host_dimension_org.up.sql
+2
View File
@@ -56,6 +56,7 @@ begin;
;
-- replaces view from 15/01_wh_rename_key_columns.up.sql
-- replaced in 26/02_wh_network_address_dimensions.up.sql
drop view whx_host_dimension_target;
create view whx_host_dimension_target as
select key,
@@ -90,6 +91,7 @@ begin;
;
-- replaces function from 15/01_wh_rename_key_columns.up.sql
-- replaced in 26/03_wh_network_address_dimensions.up.sql
drop function wh_upsert_host;
create function wh_upsert_host(p_host_id wt_public_id, p_host_set_id wt_public_id, p_target_id wt_public_id)
returns wh_dim_key
internal/db/schema/migrations/oss/postgres/20/07_wh_session_dimensions.up.sql
+1
View File
@@ -1,6 +1,7 @@
begin;
-- replaces view from 15/02_wh_rename_host_dimension_org.up.sql
-- replaced by 26/02_wh_network_address_dimensions.up.sql
-- Updates whx_host_dimension_source to support plugin based host and related
-- resources. While this will return rows where there are host_sets where hosts
internal/db/schema/migrations/oss/postgres/26/01_wh_network_address_dimensions.up.sql
+162
View File
@@ -0,0 +1,162 @@
begin;
-- wh_network_address_dimension contains the addresses and calculated values
-- about those addresses.
create table wh_network_address_dimension (
address wh_dim_text primary key,
address_type wh_dim_text, -- (IP Address, DNS Name, Unknown)
ip_address_family wh_dim_text, -- (IPv4, IPv6, Not Applicable)
private_ip_address_indicator wh_dim_text, -- ("Public IP address", "Private IP address", "Not Applicable")
dns_name wh_dim_text,
ip4_address wh_dim_text,
ip6_address wh_dim_text
);
-- wh_network_address_group is referenced by the wh_host_dimension to id
-- the group of addresses on a host at the time a session is created.
create table wh_network_address_group (
key wh_dim_key primary key default wh_dim_key()
);
-- wh_network_address_group_membership groups the addresses
create table wh_network_address_group_membership (
network_address_group_key wh_dim_key
constraint wh_network_address_group_fkey
references wh_network_address_group (key)
on delete restrict
on update cascade,
network_address wh_dim_text
constraint wh_network_address_dimension_fkey
references wh_network_address_dimension (address)
on delete restrict
on update cascade,
primary key(network_address_group_key, network_address)
);
-- Get all the previously used warehouse specific labels for addresses which
-- could not be captured by the warehousing system at some point in time.
insert into wh_network_address_dimension(
address, address_type, ip_address_family, private_ip_address_indicator,
dns_name, ip4_address, ip6_address
)
values
('Unsupported', 'Unknown', 'Not Applicable', 'Not Applicable', 'Not Applicable', 'Not Applicable', 'Not Applicable'),
('Unknown', 'Unknown', 'Not Applicable', 'Not Applicable', 'Not Applicable', 'Not Applicable', 'Not Applicable');
-- prepare a group which can be referenced by the newly created
-- wh_host_dimension column.
insert into wh_network_address_group(key)
values('Unknown'), ('Unsupported'), ('No Addresses');
insert into wh_network_address_group_membership(network_address_group_key, network_address)
values('Unknown', 'Unknown'), ('Unsupported', 'Unsupported');
-- Allow the host dimension to reference a group of addresses as referenced
-- above.
alter table wh_host_dimension
add column network_address_group_key wh_dim_key not null
default 'Unknown'
constraint wh_network_address_group_fkey
references wh_network_address_group(key)
on delete restrict
on update cascade;
alter table wh_host_dimension
alter column network_address_group_key drop default;
-- wh_try_cast_inet returns either the provided text cast into inet or a null.
create function wh_try_cast_inet(text)
returns inet
as $$
begin
return cast($1 as inet);
exception when others then
return null::inet;
end;
$$ language plpgsql
immutable
returns null on null input;
-- wh_private_address_indicator returns a warehouse appropriate string
-- representing if the address is private or public.
create function wh_private_address_indicator(inet) returns text
as $$
begin
case
when $1 << any ('{10.0.0.0/8, 192.168.0.0/16, 172.16.0.0/12, fc00::/7, fe80::/10}'::cidr[]) then
return 'Private IP address';
else
return 'Public IP address';
end case;
end;
$$ language plpgsql
immutable
returns null on null input;
-- Migrate all the ip addresses and ignore any addresses which aren't ip.
with
ip_addresses(address, inet_address) as (
select hd.host_address as address, wh_try_cast_inet(hd.host_address) as inet_address
from wh_host_dimension as hd
where wh_try_cast_inet(hd.host_address) is not null
)
insert into wh_network_address_dimension(
address,
address_type,
ip_address_family,
private_ip_address_indicator,
dns_name,
ip4_address,
ip6_address
)
select
address,
'IP Address',
case
when family(inet_address) = 4 then 'IPv4'
when family(inet_address) = 6 then 'IPv6'
else 'Not Applicable'
end,
wh_private_address_indicator(inet_address),
'Not Applicable',
case
when family(inet_address) = 4 then address
else 'Not Applicable'
end,
case
when family(inet_address) = 6 then address
else 'Not Applicable'
end
from ip_addresses;
-- Everything else left to migrate is a dns name.
insert into wh_network_address_dimension(
address, address_type,
ip_address_family, private_ip_address_indicator,
dns_name,
ip4_address, ip6_address
)
select
whd.host_address, 'DNS Name',
'Not Applicable', 'Not Applicable',
whd.host_address,
'Not Applicable', 'Not Applicable'
from wh_host_dimension as whd
where
whd.host_address not in (select address from wh_network_address_dimension);
insert into wh_network_address_group (key)
select distinct
host_address
from wh_host_dimension
where host_address not in ('Unknown', 'Unsupported');
insert into wh_network_address_group_membership(network_address_group_key, network_address)
select distinct
host_address, host_address
from wh_host_dimension
where host_address not in ('Unknown', 'Unsupported');
update wh_host_dimension
set network_address_group_key = host_address;
commit;
internal/db/schema/migrations/oss/postgres/26/02_wh_network_address_dimensions.up.sql
+184
View File
@@ -0,0 +1,184 @@
begin;
-- replaces 20/07_wh_session_dimensions.up.sql
-- Updates whx_host_dimension_source to not have the host address on the
-- view and instead rely on a reference to the wh_network_address_group
-- referenced in the wh_upsert_host function query that uses this view.
drop view whx_host_dimension_source;
create view whx_host_dimension_source as
select -- id is the first column in the target view
h.public_id as host_id,
case when sh.public_id is not null then 'static host'
when ph.public_id is not null then 'plugin host'
else 'Unknown' end as host_type,
case when sh.public_id is not null then coalesce(sh.name, 'None')
when ph.public_id is not null then coalesce(ph.name, 'None')
else 'Unknown' end as host_name,
case when sh.public_id is not null then coalesce(sh.description, 'None')
when ph.public_id is not null then coalesce(ph.description, 'None')
else 'Unknown' end as host_description,
hs.public_id as host_set_id,
case when shs.public_id is not null then 'static host set'
when phs.public_id is not null then 'plugin host set'
else 'Unknown' end as host_set_type,
case
when shs.public_id is not null then coalesce(shs.name, 'None')
when phs.public_id is not null then coalesce(phs.name, 'None')
else 'None'
end as host_set_name,
case
when shs.public_id is not null then coalesce(shs.description, 'None')
when phs.public_id is not null then coalesce(phs.description, 'None')
else 'None'
end as host_set_description,
hc.public_id as host_catalog_id,
case when shc.public_id is not null then 'static host catalog'
when phc.public_id is not null then 'plugin host catalog'
else 'Unknown' end as host_catalog_type,
case
when shc.public_id is not null then coalesce(shc.name, 'None')
when phc.public_id is not null then coalesce(phc.name, 'None')
else 'None'
end as host_catalog_name,
case
when shc.public_id is not null then coalesce(shc.description, 'None')
when phc.public_id is not null then coalesce(phc.description, 'None')
else 'None'
end as host_catalog_description,
t.public_id as target_id,
'tcp target' as target_type,
coalesce(t.name, 'None') as target_name,
coalesce(t.description, 'None') as target_description,
coalesce(t.default_port, 0) as target_default_port_number,
t.session_max_seconds as target_session_max_seconds,
t.session_connection_limit as target_session_connection_limit,
p.public_id as project_id,
coalesce(p.name, 'None') as project_name,
coalesce(p.description, 'None') as project_description,
o.public_id as organization_id,
coalesce(o.name, 'None') as organization_name,
coalesce(o.description, 'None') as organization_description
from host as h
join host_catalog as hc on h.catalog_id = hc.public_id
join host_set as hs on h.catalog_id = hs.catalog_id
join target_host_set as ts on hs.public_id = ts.host_set_id
join target_tcp as t on ts.target_id = t.public_id
join iam_scope as p on t.scope_id = p.public_id and p.type = 'project'
join iam_scope as o on p.parent_id = o.public_id and o.type = 'org'
left join static_host as sh on sh.public_id = h.public_id
left join host_plugin_host as ph on ph.public_id = h.public_id
left join static_host_catalog as shc on shc.public_id = hc.public_id
left join host_plugin_catalog as phc on phc.public_id = hc.public_id
left join static_host_set as shs on shs.public_id = hs.public_id
left join host_plugin_set as phs on phs.public_id = hs.public_id
;
-- replaces view from 15/02_wh_rename_host_dimension_org.up.sql
-- adds the network_address_group_key.
drop view whx_host_dimension_target;
create view whx_host_dimension_target as
select key,
network_address_group_key,
host_id,
host_type,
host_name,
host_description,
host_set_id,
host_set_type,
host_set_name,
host_set_description,
host_catalog_id,
host_catalog_type,
host_catalog_name,
host_catalog_description,
target_id,
target_type,
target_name,
target_description,
target_default_port_number,
target_session_max_seconds,
target_session_connection_limit,
project_id,
project_name,
project_description,
organization_id,
organization_name,
organization_description
from wh_host_dimension
where current_row_indicator = 'Current'
;
create view whx_network_address_dimension_source as
select
hdns.host_id as host_id,
hdns.name as address,
'DNS Name' as address_type,
'Not Applicable' as ip_address_family,
'Not Applicable' as private_ip_address_indicator,
hdns.name as dns_name,
'Not Applicable' as ip4_address,
'Not Applicable' as ip6_address
from host_dns_name as hdns
union
select -- id is the first column in the target view
hip.host_id as host_id,
host(hip.address) as address,
'IP Address' as address_type,
case
when family(hip.address) = 4 then 'IPv4'
when family(hip.address) = 6 then 'IPv6'
else 'Not Applicable'
end as ip_address_family,
wh_private_address_indicator(hip.address) as private_ip_address_indicator,
'Not Applicable' as dns_name,
case
when hip.address is not null and family(hip.address) = 4 then host(hip.address)
else 'Not Applicable'
end as ip4_address,
case
when hip.address is not null and family(hip.address) = 6 then host(hip.address)
else 'Not Applicable'
end as ip6_address
from host_ip_address as hip
union
select
sh.public_id as host_id,
sh.address as address,
'DNS Name' as address_type,
'Not Applicable' as ip_address_family,
'Not Applicable' as private_ip_address_indicator,
sh.address as dns_name,
'Not Applicable' as ip4_address,
'Not Applicable' as ip6_address
from static_host as sh
where wh_try_cast_inet(sh.address) is null
union
select
sh.public_id as host_id,
host(wh_try_cast_inet(sh.address)) as address,
'IP Address' as address_type,
case
when family(wh_try_cast_inet(sh.address)) = 4 then 'IPv4'
when family(wh_try_cast_inet(sh.address)) = 6 then 'IPv6'
else 'Not Applicable'
end as ip_address_family,
wh_private_address_indicator(wh_try_cast_inet(sh.address)) as private_ip_address_indicator,
'Not Applicable' as dns_name,
case
when family(wh_try_cast_inet(sh.address)) = 4 then host(wh_try_cast_inet(sh.address))
else 'Not Applicable'
end as ip4_address,
case
when family(wh_try_cast_inet(sh.address)) = 6 then host(wh_try_cast_inet(sh.address))
else 'Not Applicable'
end as ip6_address
from static_host as sh
where wh_try_cast_inet(sh.address) is not null;
alter table wh_host_dimension
drop column host_address;
commit;
internal/db/schema/migrations/oss/postgres/26/03_wh_network_address_dimensions.up.sql
+145
View File
@@ -0,0 +1,145 @@
begin;
-- wh_upsert_network_address_dimension function writes all addresses for a
-- host to wh_network_address_dimension if they aren't already there, creates
-- a network address group which contains only the addresses for the
-- provided host if one doesn't exist and returns the group key which contains
-- exactly the set of addresses which the provided host contains.
create function wh_upsert_network_address_dimension(p_host_id wt_public_id)
returns wh_dim_key
as $$
declare
nag_key wh_dim_key;
a_key wh_dim_key;
begin
if not exists (
select 1
from whx_network_address_dimension_source
where host_id = p_host_id) then
return 'No Addresses';
end if;
insert into wh_network_address_dimension (
address, address_type, ip_address_family, private_ip_address_indicator, dns_name, ip4_address, ip6_address
)
select
address, address_type, ip_address_family, private_ip_address_indicator, dns_name, ip4_address, ip6_address
from whx_network_address_dimension_source
where host_id = p_host_id
on conflict do nothing;
with address_list (address) as (
select address
from whx_network_address_dimension_source
where host_id = p_host_id
)
select distinct network_address_group_key into nag_key
from wh_network_address_group_membership o
-- At least 1 address is in the group which is also in the address list
where o.network_address in (select address from address_list)
-- The number of unique addresses in this group is the same as is in the
-- address_list.
and (select count(address) from address_list) =
(
select count(i.network_address)
from wh_network_address_group_membership i
where o.network_address_group_key = i.network_address_group_key
)
-- There are no other addresses in this group which are not in the
-- address_list.
and not exists
(
select 1
from wh_network_address_group_membership i
where o.network_address_group_key = i.network_address_group_key
and i.network_address not in (select address from address_list)
)
;
if nag_key is null then
insert into wh_network_address_group default values returning key into nag_key;
insert into wh_network_address_group_membership
(network_address_group_key, network_address)
select nag_key, address
from whx_network_address_dimension_source
where host_id = p_host_id;
end if;
return nag_key;
end
$$ language plpgsql;
-- replaces function from 15/02_wh_rename_key_columns.up.sql
-- adds the network address key to the host dimension table.
drop function wh_upsert_host;
create function wh_upsert_host(p_host_id wt_public_id, p_host_set_id wt_public_id, p_target_id wt_public_id)
returns wh_dim_key
as $$
declare
src whx_host_dimension_target%rowtype;
target whx_host_dimension_target%rowtype;
new_row wh_host_dimension%rowtype;
addr_group_key wh_dim_key;
begin
select * into target
from whx_host_dimension_target as t
where t.host_id = p_host_id
and t.host_set_id = p_host_set_id
and t.target_id = p_target_id;
select wh_upsert_network_address_dimension(p_host_id) into addr_group_key;
select target.key, addr_group_key, t.* into src
from whx_host_dimension_source as t
where t.host_id = p_host_id
and t.host_set_id = p_host_set_id
and t.target_id = p_target_id;
if src is distinct from target then
-- expire the current row
update wh_host_dimension
set current_row_indicator = 'Expired',
row_expiration_time = current_timestamp
where host_id = p_host_id
and host_set_id = p_host_set_id
and target_id = p_target_id
and current_row_indicator = 'Current';
-- insert a new row
insert into wh_host_dimension (
host_id, host_type, host_name, host_description,
network_address_group_key,
host_set_id, host_set_type, host_set_name, host_set_description,
host_catalog_id, host_catalog_type, host_catalog_name, host_catalog_description,
target_id, target_type, target_name, target_description,
target_default_port_number, target_session_max_seconds, target_session_connection_limit,
project_id, project_name, project_description,
organization_id, organization_name, organization_description,
current_row_indicator, row_effective_time, row_expiration_time
)
select host_id, host_type, host_name, host_description,
addr_group_key,
host_set_id, host_set_type, host_set_name, host_set_description,
host_catalog_id, host_catalog_type, host_catalog_name, host_catalog_description,
target_id, target_type, target_name, target_description,
target_default_port_number, target_session_max_seconds, target_session_connection_limit,
project_id, project_name, project_description,
organization_id, organization_name, organization_description,
'Current', current_timestamp, 'infinity'::timestamptz
from whx_host_dimension_source
where host_id = p_host_id
and host_set_id = p_host_set_id
and target_id = p_target_id
returning * into new_row;
return new_row.key;
end if;
return target.key;
end;
$$ language plpgsql;
commit;
internal/db/schema/migrations/oss/postgres_26_01_test.go
+224
View File
@@ -0,0 +1,224 @@
package oss_test
import (
"context"
"testing"
"github.com/hashicorp/boundary/internal/db"
"github.com/hashicorp/boundary/internal/db/common"
"github.com/hashicorp/boundary/internal/db/schema"
"github.com/hashicorp/boundary/testing/dbtest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMigrations_WareHouse_HostAddresses(t *testing.T) {
const (
priorMigration = 25001
currentMigration = 26001
)
t.Parallel()
ctx := context.Background()
dialect := dbtest.Postgres
c, u, _, err := dbtest.StartUsingTemplate(dialect, dbtest.WithTemplate(dbtest.Template1))
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, c())
})
d, err := common.SqlOpen(dialect, u)
require.NoError(t, err)
// migration to the prior migration (before the one we want to test)
m, err := schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions(
schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": priorMigration}),
))
require.NoError(t, err)
require.NoError(t, m.ApplyMigrations(ctx))
state, err := m.CurrentState(ctx)
require.NoError(t, err)
want := &schema.State{
Initialized: true,
Editions: []schema.EditionState{
{
Name: "oss",
BinarySchemaVersion: priorMigration,
DatabaseSchemaVersion: priorMigration,
DatabaseSchemaState: schema.Equal,
},
},
}
require.Equal(t, want, state)
// get a connection
dbType, err := db.StringToDbType(dialect)
require.NoError(t, err)
conn, err := db.Open(dbType, u)
require.NoError(t, err)
rw := db.New(conn)
type whAddress struct {
Address string
AddressType string
IpAddressFamily string
PrivateIpAddressIndicator string
DnsName string
Ip4Address string
Ip6Address string
}
addresses := []whAddress{
{
Address: "10.0.0.1",
AddressType: "IP Address",
IpAddressFamily: "IPv4",
PrivateIpAddressIndicator: "Private IP address",
DnsName: "Not Applicable",
Ip4Address: "10.0.0.1",
Ip6Address: "Not Applicable",
},
{
Address: "12.3.4.5",
AddressType: "IP Address",
IpAddressFamily: "IPv4",
PrivateIpAddressIndicator: "Public IP address",
DnsName: "Not Applicable",
Ip4Address: "12.3.4.5",
Ip6Address: "Not Applicable",
},
{
Address: "fe80::1234:5678:1234:5678",
AddressType: "IP Address",
IpAddressFamily: "IPv6",
PrivateIpAddressIndicator: "Private IP address",
DnsName: "Not Applicable",
Ip4Address: "Not Applicable",
Ip6Address: "fe80::1234:5678:1234:5678",
},
{
Address: "2001:4860:4860::8888",
AddressType: "IP Address",
IpAddressFamily: "IPv6",
PrivateIpAddressIndicator: "Public IP address",
DnsName: "Not Applicable",
Ip4Address: "Not Applicable",
Ip6Address: "2001:4860:4860::8888",
},
{
Address: "foo",
AddressType: "DNS Name",
IpAddressFamily: "Not Applicable",
PrivateIpAddressIndicator: "Not Applicable",
DnsName: "foo",
Ip4Address: "Not Applicable",
Ip6Address: "Not Applicable",
},
{
Address: "something.com",
AddressType: "DNS Name",
IpAddressFamily: "Not Applicable",
PrivateIpAddressIndicator: "Not Applicable",
DnsName: "something.com",
Ip4Address: "Not Applicable",
Ip6Address: "Not Applicable",
},
{
Address: "10.0.foo.com",
AddressType: "DNS Name",
IpAddressFamily: "Not Applicable",
PrivateIpAddressIndicator: "Not Applicable",
DnsName: "10.0.foo.com",
Ip4Address: "Not Applicable",
Ip6Address: "Not Applicable",
},
{
Address: "not valid anything",
AddressType: "DNS Name",
IpAddressFamily: "Not Applicable",
PrivateIpAddressIndicator: "Not Applicable",
DnsName: "not valid anything",
Ip4Address: "Not Applicable",
Ip6Address: "Not Applicable",
},
{
Address: "Unsupported",
AddressType: "Unknown",
IpAddressFamily: "Not Applicable",
PrivateIpAddressIndicator: "Not Applicable",
DnsName: "Not Applicable",
Ip4Address: "Not Applicable",
Ip6Address: "Not Applicable",
},
{
Address: "Unknown",
AddressType: "Unknown",
IpAddressFamily: "Not Applicable",
PrivateIpAddressIndicator: "Not Applicable",
DnsName: "Not Applicable",
Ip4Address: "Not Applicable",
Ip6Address: "Not Applicable",
},
}
{
q := `
insert into wh_host_dimension
(host_id, host_type, host_name, host_description, host_address,
host_set_id, host_set_type, host_set_name, host_set_description,
host_catalog_id, host_catalog_type, host_catalog_name, host_catalog_description,
target_id, target_type, target_name, target_description, target_default_port_number,
target_session_max_seconds, target_session_connection_limit,
project_id, project_name, project_description,
organization_id, organization_name, organization_description,
current_row_indicator, row_effective_time, row_expiration_time)
values
('h_1234567890', 'static', 'None', 'None', ?,
'hs_1234567890', 'static', 'None', 'None',
'hc_1234567890', 'static', 'None', 'None',
't_1234567890', 'tcp', 'None', 'None', 0,
30, 1,
'p_1234567890', 'None', 'None',
'o_1234567890', 'None', 'None',
'Expired', current_timestamp, current_timestamp)
`
for _, a := range addresses {
_, err := rw.Exec(ctx, q, []interface{}{a.Address})
require.NoError(t, err)
}
}
// now we're ready for the migration we want to test.
m, err = schema.NewManager(ctx, schema.Dialect(dialect), d, schema.WithEditions(
schema.TestCreatePartialEditions(schema.Dialect(dialect), schema.PartialEditions{"oss": currentMigration}),
))
require.NoError(t, err)
require.NoError(t, m.ApplyMigrations(ctx))
state, err = m.CurrentState(ctx)
require.NoError(t, err)
want = &schema.State{
Initialized: true,
Editions: []schema.EditionState{
{
Name: "oss",
BinarySchemaVersion: currentMigration,
DatabaseSchemaVersion: currentMigration,
DatabaseSchemaState: schema.Equal,
},
},
}
require.Equal(t, want, state)
// Now read all the converted rows and see if we have transformed and
// calculated the values as expected.
{
rows, err := rw.Query(ctx, "select * from wh_network_address_dimension", nil)
require.NoError(t, err)
var results []whAddress
for rows.Next() {
var addr whAddress
require.NoError(t, rw.ScanRows(context.Background(), rows, &addr))
results = append(results, addr)
}
assert.ElementsMatch(t, results, addresses)
}
}
internal/db/sqltest/README.md
+3 -3
View File
@@ -45,13 +45,13 @@ You can also run individual tests:
```bash
# run a single test file
make TEST=tests/setup/wtt_load.sql
make TESTS=tests/setup/wtt_load.sql
# run a single test file with the database already created.
make run-tests TEST=tests/setup/wtt_load.sql
make run-tests TESTS=tests/setup/wtt_load.sql
# run a directory of tests
make TEST=tests/setup/*.sql
make TESTS=tests/setup/*.sql
```
You can pass through options to `pg_prove`.
internal/db/sqltest/initdb.d/01_colors_persona.sql
+46
View File
@@ -148,6 +148,52 @@ begin;
('c___cr-sthcl', 'h_____cr__08', '8.red.color'),
('c___cr-sthcl', 'h_____cr__09', '9.red.color');
insert into host_dns_name
(host_id, name)
values
-- note there are no dns names for blue 1
('h_____cb__02', '2.blue.color'),
('h_____cb__03', '3.blue.color'),
('h_____cb__04', '4.blue.color'),
('h_____cb__05', '5.blue.color'),
('h_____cb__06', '6.blue.color'),
('h_____cb__07', '7.blue.color'),
('h_____cb__08', '8.blue.color'),
('h_____cb__09', '9.blue.color'),
('h_____cr__02', '1.red.color'),
('h_____cr__02', '2.red.color'),
('h_____cr__03', '3.red.color'),
('h_____cr__04', '4.red.color'),
('h_____cr__05', '5.red.color'),
('h_____cr__06', '6.red.color'),
('h_____cr__07', '7.red.color'),
('h_____cr__08', '8.red.color'),
('h_____cr__09', '9.red.color');
insert into host_ip_address
(host_id, address)
values
-- note there are no address for blue 1
('h_____cb__02', '10.0.0.2'),
('h_____cb__03', 'fe80::3333:3333:3333:3333'),
('h_____cb__04', '10.0.0.4'),
('h_____cb__05', 'fe80::5555:5555:5555:5555'),
('h_____cb__06', '10.0.0.6'),
('h_____cb__07', 'fe80::7777:7777:7777:7777'),
('h_____cb__08', '10.0.0.8'),
('h_____cb__09', 'fe80::9999:9999:9999:9999'),
('h_____cr__01', '11.11.11.11'),
('h_____cr__02', '2001:4860:4860::2222'),
('h_____cr__03', '33.33.33.33'),
('h_____cr__04', '2001:4860:4860::4444'),
('h_____cr__05', '55.55.55.55'),
('h_____cr__06', '2001:4860:4860::6666'),
('h_____cr__07', '77.77.77.77'),
('h_____cr__08', '2001:4860:4860::8888'),
('h_____cr__09', '99.99.99.99');
insert into static_host_set
(catalog_id, public_id, name)
values
internal/db/sqltest/initdb.d/03_widgets_persona.sql
+23
View File
@@ -259,6 +259,29 @@ begin;
('c___ws-plghcl', 'h_____ws__02-plgh', '2 small widget'),
('c___ws-plghcl', 'h_____ws__03-plgh', '3 small widget');
insert into host_dns_name
(host_id, name)
values
('h_____wb__01-plgh', '1.big.widget'),
('h_____wb__02-plgh', '2.big.widget'),
('h_____wb__03-plgh', '3.big.widget'),
('h_____ws__01-plgh', '1.small.widget'),
('h_____ws__02-plgh', '2.small.widget'),
('h_____ws__03-plgh', '3.small.widget');
insert into host_ip_address
(host_id, address)
values
('h_____wb__01-plgh', '1.1.1.1'),
('h_____wb__02-plgh', 'fe80::2222:2222:2222:2222'),
-- host 3 only has a dns name so the set of addresses are the same
-- between the static and plugin based host
('h_____ws__01-plgh', '11.11.11.11'),
('h_____ws__02-plgh', '2001:4860:4860::2222'),
('h_____ws__03-plgh', '33.33.33.33');
insert into host_plugin_set
(catalog_id, public_id, name, attributes, need_sync)
values
internal/db/sqltest/tests/wh/host_dimension/session_multiple_sessions.sql
+43 -7
View File
@@ -1,18 +1,18 @@
-- session_multiple_sessions tests the wh_host_dimesion when
-- multiple sessions are created using the same user and auth method.
begin;
select plan(4);
select plan(6);
select wtt_load('widgets', 'iam', 'kms', 'auth', 'hosts', 'targets');
-- ensure no existing dimensions
select is(count(*), 0::bigint) from wh_host_dimension where organization_id = 'o_____widget';
-- insert first session, should result in a new user dimension
-- insert first session, should result in a new host dimension
insert into session
( scope_id , target_id , host_set_id , host_id , user_id , auth_token_id , certificate , endpoint , public_id)
values
('p____bwidget' , 't_________wb' , 's___1wb-sths' , 'h_____wb__01' , 'u_____walter' , 'tok___walter' , 'abc'::bytea , 'ep1' , 's1____walter');
('p____bwidget' , 't_________wb' , 's___1wb-plghs' , 'h_____wb__02-plgh' , 'u_____walter' , 'tok___walter' , 'abc'::bytea , 'ep1' , 's1____walter');
select is(count(*), 1::bigint) from wh_host_dimension where organization_id = 'o_____widget';
@@ -20,11 +20,11 @@ begin;
-- * same user
-- * same auth
-- * same host
-- should not result in a new user dimension
-- should not result in a new host dimension
insert into session
( scope_id , target_id , host_set_id , host_id , user_id , auth_token_id , certificate , endpoint , public_id)
values
('p____bwidget' , 't_________wb' , 's___1wb-sths' , 'h_____wb__01' , 'u_____walter' , 'tok___walter' , 'abc'::bytea , 'ep1' , 's2____walter');
('p____bwidget' , 't_________wb' , 's___1wb-plghs' , 'h_____wb__02-plgh' , 'u_____walter' , 'tok___walter' , 'abc'::bytea , 'ep1' , 's2____walter');
select is(count(*), 1::bigint) from wh_host_dimension where organization_id = 'o_____widget';
@@ -32,14 +32,50 @@ begin;
-- * different user
-- * same auth
-- * same host
-- should not result in a new user dimension
-- should not result in a new host dimension
insert into session
( scope_id , target_id , host_set_id , host_id , user_id , auth_token_id , certificate , endpoint , public_id)
values
('p____bwidget' , 't_________wb' , 's___1wb-sths' , 'h_____wb__01' , 'u_____warren' , 'tok___warren' , 'abc'::bytea , 'ep1' , 's3____walter');
('p____bwidget' , 't_________wb' , 's___1wb-plghs' , 'h_____wb__02-plgh' , 'u_____warren' , 'tok___warren' , 'abc'::bytea , 'ep1' , 's3____walter');
select is(count(*), 1::bigint) from wh_host_dimension where organization_id = 'o_____widget';
-- another session with:
-- * same user
-- * same auth
-- * same host
-- * host has a different set of addresses
insert into host_dns_name
(host_id, name)
values
('h_____wb__02-plgh', 'new.big.widget');
-- should result in a new host dimension
insert into session
( scope_id , target_id , host_set_id , host_id , user_id , auth_token_id , certificate , endpoint , public_id)
values
('p____bwidget' , 't_________wb' , 's___1wb-plghs' , 'h_____wb__02-plgh' , 'u_____walter' , 'tok___walter' , 'abc'::bytea , 'ep1' , 's4____walter');
select is(count(*), 2::bigint) from wh_host_dimension where organization_id = 'o_____widget';
-- another session with:
-- * same user
-- * same auth
-- * same host
-- * host has a different set of addresses with ipv6
insert into host_ip_address
(host_id, address)
values
('h_____wb__02-plgh', 'fe80::beef:1111:2222:333');
-- should result in a new host dimension
insert into session
( scope_id , target_id , host_set_id , host_id , user_id , auth_token_id , certificate , endpoint , public_id)
values
('p____bwidget' , 't_________wb' , 's___1wb-plghs' , 'h_____wb__02-plgh' , 'u_____walter' , 'tok___walter' , 'abc'::bytea , 'ep1' , 's5____walter');
select is(count(*), 3::bigint) from wh_host_dimension where organization_id = 'o_____widget';
select * from finish();
rollback;
internal/db/sqltest/tests/wh/host_dimension/session_update.sql
+16 -3
View File
@@ -1,18 +1,18 @@
-- session_update tests the wh_host_dimesion when
-- a session is inserted and then updated.
begin;
select plan(3);
select plan(4);
select wtt_load('widgets', 'iam', 'kms', 'auth', 'hosts', 'targets');
-- ensure no existing dimensions
select is(count(*), 0::bigint) from wh_host_dimension where organization_id = 'o_____widget';
-- insert first session, should result in a new user dimension
-- insert first session, should result in a new host dimension
insert into session
( scope_id , target_id , host_set_id , host_id , user_id , auth_token_id , certificate , endpoint , public_id)
values
('p____bwidget' , 't_________wb' , 's___1wb-sths' , 'h_____wb__01' , 'u_____walter' , 'tok___walter' , 'abc'::bytea , 'ep1' , 's1____walter');
('p____bwidget' , 't_________wb' , 's___1wb-plghs' , 'h_____wb__01-plgh' , 'u_____walter' , 'tok___walter' , 'abc'::bytea , 'ep1' , 's1____walter');
select is(count(*), 1::bigint) from wh_host_dimension where organization_id = 'o_____widget';
@@ -24,5 +24,18 @@ begin;
select is(count(*), 1::bigint) from wh_host_dimension where organization_id = 'o_____widget';
insert into host_dns_name
(host_id, name)
values
('h_____wb__01-plgh', 'new.big.widget');
-- update session, should not impact wh_host_dimension
update session set
version = 2
where
public_id = 's1____walter';
select is(count(*), 1::bigint) from wh_host_dimension where organization_id = 'o_____widget';
select * from finish();
rollback;
internal/db/sqltest/tests/wh/host_dimension_views/source.sql
+21 -3
View File
@@ -1,11 +1,11 @@
-- source tests the whx_host_dimension_source view.
begin;
select plan(2);
select plan(3);
select wtt_load('widgets', 'iam', 'kms', 'auth', 'hosts', 'targets');
-- Static hosts
select is(s.*, row(
'h_____wb__01', 'static host', 'None', 'None', '1.big.widget',
'h_____wb__01', 'static host', 'None', 'None',
's___2wb-sths', 'static host set', 'Big Widget Static Set 2', 'None',
'c___wb-sthcl', 'static host catalog', 'Big Widget Static Catalog', 'None',
't_________wb', 'tcp target', 'Big Widget Target', 'None', 0, 28800, 1,
@@ -19,7 +19,7 @@ begin;
-- Plugin based hosts
select is(s.*, row(
'h_____wb__01-plgh', 'plugin host', 'None', 'None', 'Unsupported',
'h_____wb__01-plgh', 'plugin host', 'None', 'None',
's___2wb-plghs', 'plugin host set', 'Big Widget Plugin Set 2', 'None',
'c___wb-plghcl', 'plugin host catalog', 'Big Widget Plugin Catalog', 'None',
't_________wb', 'tcp target', 'Big Widget Target', 'None', 0, 28800, 1,
@@ -31,6 +31,24 @@ begin;
and s.host_set_id = 's___2wb-plghs'
and s.target_id = 't_________wb';
-- network address dimension
declare cwant cursor for select
address, address_type, ip_address_family, private_ip_address_indicator,
dns_name, ip4_address, ip6_address
from whx_network_address_dimension_source
where host_id = 'h_____wb__02-plgh'
order by address;
select results_eq(
'cwant'::refcursor,
$$VALUES
('2.big.widget', 'DNS Name', 'Not Applicable', 'Not Applicable',
'2.big.widget', 'Not Applicable', 'Not Applicable'),
('fe80::2222:2222:2222:2222', 'IP Address', 'IPv6', 'Private IP address',
'Not Applicable', 'Not Applicable', 'fe80::2222:2222:2222:2222')
$$);
select * from finish();
rollback;
internal/db/sqltest/tests/wh/host_dimension_views/target.sql
+29 -14
View File
@@ -9,21 +9,36 @@ begin;
and target_id = 't_________wb';
$$);
insert into wh_network_address_dimension
(address, address_type, ip_address_family, private_ip_address_indicator, dns_name, ip4_address, ip6_address)
values
('1.big.widget', 'DNS Name', 'Not Applicable', 'Not Applicable', '1.big.widget', 'Not Applicable', 'Not Applicable');
insert into wh_network_address_group
(key)
values
('nagk____1_big_widget');
insert into wh_network_address_group_membership
(network_address_group_key, network_address)
values
('nagk____1_big_widget', '1.big.widget');
insert into wh_host_dimension
(
key,
host_id, host_type, host_name, host_description, host_address,
host_set_id, host_set_type, host_set_name, host_set_description,
host_catalog_id, host_catalog_type, host_catalog_name, host_catalog_description,
target_id, target_type, target_name, target_description, target_default_port_number, target_session_max_seconds, target_session_connection_limit,
project_id, project_name, project_description,
organization_id, organization_name, organization_description,
current_row_indicator, row_effective_time, row_expiration_time
key, network_address_group_key,
host_id, host_type, host_name, host_description,
host_set_id, host_set_type, host_set_name, host_set_description,
host_catalog_id, host_catalog_type, host_catalog_name, host_catalog_description,
target_id, target_type, target_name, target_description, target_default_port_number, target_session_max_seconds, target_session_connection_limit,
project_id, project_name, project_description,
organization_id, organization_name, organization_description,
current_row_indicator, row_effective_time, row_expiration_time
)
values
(
'whd_____1',
'h_____wb__01', 'static host', 'None', 'None', '1.big.widget',
'whd_____1', 'nagk____1_big_widget',
'h_____wb__01', 'static host', 'None', 'None',
's___2wb-sths', 'static host set', 'Big Widget Static Set 2', 'None',
'c___wb-sthcl', 'static host catalog', 'Big Widget Static Catalog', 'None',
't_________wb', 'tcp target', 'Big Widget Target', 'None', 0, 28800, 1,
@@ -32,8 +47,8 @@ begin;
'Expired', '2021-07-21T11:01'::timestamptz, '2021-07-21T12:01'::timestamptz
),
(
'whd_____2',
'h_____wb__01', 'static host', 'None', 'None', '1.big.widget',
'whd_____2', 'nagk____1_big_widget',
'h_____wb__01', 'static host', 'None', 'None',
's___2wb-sths', 'static host set', 'Big Widget Static Set 2', 'None',
'c___wb-sthcl', 'static host catalog', 'Big Widget Static Catalog', 'None',
't_________wb', 'tcp target', 'Big Widget Target', 'None', 0, 28800, 1,
@@ -43,8 +58,8 @@ begin;
);
select is(t.*, row(
'whd_____2',
'h_____wb__01', 'static host', 'None', 'None', '1.big.widget',
'whd_____2', 'nagk____1_big_widget',
'h_____wb__01', 'static host', 'None', 'None',
's___2wb-sths', 'static host set', 'Big Widget Static Set 2', 'None',
'c___wb-sthcl', 'static host catalog', 'Big Widget Static Catalog', 'None',
't_________wb', 'tcp target', 'Big Widget Target', 'None', 0, 28800, 1,
internal/db/sqltest/tests/wh/network_address_dimension/helper_functions.sql
+49
View File
@@ -0,0 +1,49 @@
-- Tests that the helper methods for network address dimension work as expected.
begin;
select plan(33);
-- test wh_private_address_status
select is(wh_private_address_indicator('10.0.0.1'::inet), 'Private IP address');
select is(wh_private_address_indicator('192.168.0.1'::inet), 'Private IP address');
select is(wh_private_address_indicator('172.16.0.1'::inet), 'Private IP address');
select is(wh_private_address_indicator('73.2.3.4'::inet), 'Public IP address');
select is(wh_private_address_indicator('2001:4860:4860::8888'::inet), 'Public IP address');
select is(wh_private_address_indicator('fe80::1234:5678:1234:5678'::inet), 'Private IP address');
select is(wh_private_address_indicator(null::inet), null);
-- test wh_try_cast_inet
-- ipv4
select is(wh_try_cast_inet('127.0.0.1'), '127.0.0.1'::inet);
select is(wh_try_cast_inet('1.2.3.4'), '1.2.3.4'::inet);
select is(wh_try_cast_inet('0.0.0.0'), '0.0.0.0'::inet);
select is(wh_try_cast_inet('1.2.3'), null::inet);
select is(wh_try_cast_inet('256.256.256.256'), null::inet);
select is(wh_try_cast_inet(null), null::inet);
select is(wh_try_cast_inet('not.an.ip.address'), null::inet);
select is(wh_try_cast_inet('not even a dns name'), null::inet);
-- ipv6
select is(wh_try_cast_inet('fe80::1234:1234:1234:1234'), 'fe80::1234:1234:1234:1234'::inet);
select is(wh_try_cast_inet('2001:0db8:0000:0000:0000:ff00:0042:8329'), '2001:0db8:0000:0000:0000:ff00:0042:8329'::inet);
select is(wh_try_cast_inet('2001:db8:0:0:0:ff00:42:8329'), '2001:db8:0:0:0:ff00:42:8329'::inet);
select is(wh_try_cast_inet('0000:0000:0000:0000:0000:0000:0000:0001'), '0000:0000:0000:0000:0000:0000:0000:0001'::inet);
select is(wh_try_cast_inet('::1'), '::1'::inet);
select is(wh_try_cast_inet('::'), '::'::inet);
select is(wh_try_cast_inet('::ffff:129.144.52.38'), '::ffff:129.144.52.38'::inet);
select is(wh_try_cast_inet('::129.144.52.38'), '::129.144.52.38'::inet);
select is(wh_try_cast_inet('::ffff:d'), '::ffff:d'::inet);
select is(wh_try_cast_inet('1080:0:0:0:8:800:200c:417a'), '1080:0:0:0:8:800:200c:417a'::inet);
select is(wh_try_cast_inet('::129.144.52.38'), '::129.144.52.38'::inet);
select is(wh_try_cast_inet('0:0:0:0:0:0:192.1.56.11'), '0:0:0:0:0:0:192.1.56.11'::inet);
select is(wh_try_cast_inet('abcd:abcd:abcd:abcd:abcd:abcd:192.168.158.190'), 'abcd:abcd:abcd:abcd:abcd:abcd:192.168.158.190'::inet);
select is(wh_try_cast_inet('::ffff:192.1.56.10'), '::ffff:192.1.56.10'::inet);
select is(wh_try_cast_inet('::ffff:d.d.d'), null::inet);
select is(wh_try_cast_inet('::ffff:d.d'), null::inet);
select is(wh_try_cast_inet('::d.d.d'), null::inet);
select is(wh_try_cast_inet('::d.d'), null::inet);
select * from finish();
rollback;
internal/db/sqltest/tests/wh/network_address_dimension/upsert_network_address_group_key.sql
+26
View File
@@ -0,0 +1,26 @@
-- test the upsert network address dimension function
begin;
select plan(4);
select wtt_load('widgets', 'iam', 'kms', 'auth', 'hosts');
select is(wh_upsert_network_address_dimension('non-existant-key'), 'No Addresses');
-- The network address group key remains the same for the same set of
-- addresses.
with nag (key) as (
select wh_upsert_network_address_dimension('h_____wb__03')
)
select is(wh_upsert_network_address_dimension('h_____wb__03'), key) from nag;
with nag (key) as (
select wh_upsert_network_address_dimension('h_____wb__03')
)
select is(wh_upsert_network_address_dimension('h_____wb__03-plgh'), key) from nag;
-- different set of addresses means a different group key
with nag (key) as (
select wh_upsert_network_address_dimension('h_____wb__01')
)
select isnt(wh_upsert_network_address_dimension('h_____wb__02'), key) from nag;
select * from finish();
rollback;