paulj/hpr-tools
1
0
Fork 0
forked from HPR/hpr-tools
Code Pull Requests Activity
Files
0f1e72748719e08783cee5edf714e6a9cd370363
hpr-tools/PostgreSQL_Database/copy_mysql_pg_2

2288 lines
69 KiB
Perl
Executable File
Raw Blame History

#!/usr/bin/env perl
#===============================================================================
#
# FILE: copy_mysql_pg_2
#
# USAGE: ./copy_mysql_pg_2
#
# DESCRIPTION: Copies HPR show data from the MariaDB database to an
# experimental PostgreSQL database (second version "HPR2")
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.1.5
# CREATED: 2017-10-23 19:11:48
# REVISION: 2019-06-01 16:10:23
#
#===============================================================================
use 5.010;
use strict;
use warnings;
use utf8;
use Getopt::Long;
use Pod::Usage;
use Config::General;
use List::MoreUtils qw{uniq apply};
use Text::CSV;
use DBI;
use SQL::Abstract;
use Data::Dumper;
#
# Version number (manually incremented)
#
our $VERSION = '0.1.5';
#
# Script and directory names
#
( my $PROG = $0 ) =~ s|.*/||mx;
( my $DIR = $0 ) =~ s|/?[^/]*$||mx;
$DIR = '.' unless $DIR;
#-------------------------------------------------------------------------------
# Declarations
#-------------------------------------------------------------------------------
#
# Constants and other declarations
#
my $basedir = "$ENV{HOME}/HPR/PostgreSQL_Database";
my $configfile1 = "$basedir/.hpr_db.cfg";
my $configfile2 = "$basedir/.hpr_pg2.cfg";
my $database3 = "$basedir/ia.db"; # soft link
my $email_template = 'host_%s@hackerpublicradio.org';
my $default_licence = 'CC-BY-SA';
my ( $dbh1, $sth1, $h1, $rv1 );
my ( $dbh2, $sth2, $h2, $rv2 );
my ( $dbh3, $sth3, $h3, $rv3 );
my (@phase_choices);
my ( %eps_tags, %data );
my @phases = (
'episodes', 'hosts', 'eh_xref', 'series', 'es_xref', 'tags',
'comments', 'archived', 'assets', 'epilogue'
);
#
# Enable Unicode mode
#
binmode STDOUT, ":encoding(UTF-8)";
binmode STDERR, ":encoding(UTF-8)";
#-------------------------------------------------------------------------------
# Options and arguments
#-------------------------------------------------------------------------------
my $DEF_DEBUG = 0;
#
# Process options
#
my %options;
Options( \%options );
#
# Default help
#
pod2usage( -msg => "$PROG version $VERSION\n", -exitval => 1 )
if ( $options{'help'} );
#
# Collect options
#
my $DEBUG = ( defined( $options{debug} ) ? $options{debug} : $DEF_DEBUG );
my $cfgfile1
= ( defined( $options{config} ) ? $options{config} : $configfile1 );
my $verbose = ( defined( $options{verbose} ) ? $options{verbose} : 0 );
my $dry_run = ( defined( $options{'dry-run'} ) ? $options{'dry-run'} : 0 );
#
# This option is a list, provided as a CSV
#
my $phase_choices = $options{phases};
if ( defined($phase_choices) ) {
#
# We have a list which we'll parse, validate, sort, make unique and filter
#
my $lcsv = Text::CSV_XS->new( { binary => 1, } );
if ( $lcsv->parse($phase_choices) ) {
# Sort fields
@phase_choices = uniq( sort { $a cmp $b } $lcsv->fields() );
# Trim leading and trailing spaces
@phase_choices = apply { $_ =~ s/(^\s*|\s*$)// } @phase_choices;
# Make a list of invalid keywords
my %tmp = map { $_ => 1 } @phases;
my @bad = grep { not exists $tmp{$_} } @phase_choices;
# Deal with all errors
die "Invalid list; no elements\n" if scalar(@phase_choices) == 0;
die "Invalid list; too many elements\n"
if scalar(@phase_choices) > scalar(@phases);
die "Invalid list elements: ", join( ",", @bad ) . "\n"
if scalar(@bad) > 0;
}
else {
die "Failed to parse -list='$phase_choices'\n"
. $lcsv->error_diag() . "\n";
}
}
else {
#
# By default we do all phases
#
@phase_choices = @phases;
}
#-------------------------------------------------------------------------------
# Configuration file for MySQL/MariaDB - load data
#-------------------------------------------------------------------------------
my $conf1 = Config::General->new(
-ConfigFile => $cfgfile1,
-InterPolateVars => 1,
-ExtendedAccess => 1
);
my %config1 = $conf1->getall();
my $conf2 = Config::General->new(
-ConfigFile => $configfile2,
-InterPolateVars => 1,
-ExtendedAccess => 1
);
my %config2 = $conf2->getall();
#-------------------------------------------------------------------------------
# Connect to the MariaDB database
#-------------------------------------------------------------------------------
my $dbtype1 = $config1{database}->{type} // 'mysql';
my $dbhost1 = $config1{database}->{host} // '127.0.0.1';
my $dbport1 = $config1{database}->{port} // 3306;
my $dbname1 = $config1{database}->{name};
my $dbuser1 = $config1{database}->{user};
my $dbpwd1 = $config1{database}->{password};
$dbh1
= DBI->connect( "dbi:$dbtype1:host=$dbhost1;port=$dbport1;database=$dbname1",
$dbuser1, $dbpwd1, { AutoCommit => 1 } )
or die $DBI::errstr;
#
# Enable client-side UTF8
#
$dbh1->{mysql_enable_utf8} = 1;
#-------------------------------------------------------------------------------
# Connect to the PostgreSQL database
#-------------------------------------------------------------------------------
my $dbtype2 = $config2{database}->{type} // 'Pg';
my $dbhost2 = $config2{database}->{host} // '127.0.0.1';
my $dbport2 = $config2{database}->{port} // 5432;
my $dbname2 = $config2{database}->{name};
my $dbuser2 = $config2{database}->{user};
my $dbpwd2 = $config2{database}->{password};
$dbh2 = DBI->connect( "dbi:$dbtype2:host=$dbhost2;database=$dbname2;port=$dbport2",
$dbuser2, $dbpwd2, { PrintError => 0, AutoCommit => 1 } )
or die $DBI::errstr;
#
# Enable client-side UTF8
#
$dbh2->{pg_enable_utf8} = 1;
#-------------------------------------------------------------------------------
# Connect to the SQLite database
#-------------------------------------------------------------------------------
$dbh3 = DBI->connect( "dbi:SQLite:dbname=$database3", "", "" );
my %choices = map { $_ => 1 } @phase_choices;
#
# Perform phases in order, omitting those that are not in the list
#
for my $phase (@phases) {
#---------------------------------------------------------------------------
# Copy the 'eps' table to 'episodes'
#---------------------------------------------------------------------------
if ( $phase eq 'episodes' && exists( $choices{$phase} ) ) {
print "Build episodes table\n" if ( $verbose > 0 );
if ( check_table( $dbh2, 'episodes' ) ) {
build_episodes_table( $dbh1, $dbh2, $verbose ) unless $dry_run;
}
else {
print "** Table 'episodes' is not empty\n";
}
}
#---------------------------------------------------------------------------
# Copy the 'hosts' table to 'hosts'
#---------------------------------------------------------------------------
elsif ( $phase eq 'hosts' && exists( $choices{$phase} ) ) {
print "Build hosts table\n" if ( $verbose > 0 );
if ( check_table( $dbh2, 'hosts' ) ) {
build_hosts_table( $dbh1, $dbh2, $email_template, $verbose )
unless $dry_run;
}
else {
print "** Table 'hosts' is not empty\n";
}
}
#---------------------------------------------------------------------------
# Generate the 'episodes_hosts_xref' table
#---------------------------------------------------------------------------
elsif ( $phase eq 'eh_xref' && exists( $choices{$phase} ) ) {
print "Build episodes_hosts_xref table\n" if ( $verbose > 0 );
if ( check_table( $dbh2, 'episodes_hosts_xref' ) ) {
build_episodes_hosts_xref_table( $dbh1, $dbh2, $verbose )
unless $dry_run;
}
else {
print "** Table 'episodes_hosts_xref' is not empty\n";
}
}
#---------------------------------------------------------------------------
# Copy the 'miniseries' table to 'series'
#---------------------------------------------------------------------------
elsif ( $phase eq 'series' && exists( $choices{$phase} ) ) {
print "Build series table\n" if ( $verbose > 0 );
if ( check_table( $dbh2, 'series' ) ) {
build_series_table( $dbh1, $dbh2, $verbose ) unless $dry_run;
}
else {
print "** Table 'series' is not empty\n";
}
}
#---------------------------------------------------------------------------
# Generate the 'episodes_series_xref' table
#---------------------------------------------------------------------------
elsif ( $phase eq 'es_xref' && exists( $choices{$phase} ) ) {
print "Build episodes_series_xref table\n" if ( $verbose > 0 );
if ( check_table( $dbh2, 'episodes_series_xref' ) ) {
build_episodes_series_xref_table( $dbh1, $dbh2, $verbose )
unless $dry_run;
}
else {
print "** Table 'episodes_series_xref' is not empty\n";
}
}
#---------------------------------------------------------------------------
# Collect and store the id numbers and tags from the MySQL 'eps' table,
# then add them to the PostgreSQL tables.
#---------------------------------------------------------------------------
elsif ( $phase eq 'tags' && exists( $choices{$phase} ) ) {
print "Build tags and episodes_tags_xref tables\n" if ( $verbose > 0 );
if ( check_table( $dbh2, 'tags' )
&& check_table( $dbh2, 'episodes_tags_xref' ) )
{
unless ($dry_run) {
%eps_tags = %{ collect_eps_tags( $dbh1, $verbose ) };
if (%eps_tags) {
build_tags_table( $dbh2, $verbose, \%eps_tags );
}
}
}
else {
print "** Tables 'tags' and/or 'episodes_tags_xref' are not empty\n";
}
}
#---------------------------------------------------------------------------
# Copy the 'comments' table to 'comments'
#---------------------------------------------------------------------------
elsif ( $phase eq 'comments' && exists( $choices{$phase} ) ) {
print "Build comments table\n" if ( $verbose > 0 );
if (check_table($dbh2,'comments')) {
build_comments_table( $dbh1, $dbh2, $verbose ) unless $dry_run;
}
else {
print "** Table 'comments' is not empty\n";
}
}
#---------------------------------------------------------------------------
# Fill in archive-related fields the 'episodes' table from 'ia.db'
#---------------------------------------------------------------------------
elsif ( $phase eq 'archived' && exists( $choices{$phase} ) ) {
print "Copy archive-related fields to episodes table\n" if ( $verbose > 0 );
update_episodes_table( $dbh3, $dbh2, $verbose ) unless $dry_run;
}
#---------------------------------------------------------------------------
# Copy the 'assets' table in 'ia.db' to 'assets'
#---------------------------------------------------------------------------
elsif ( $phase eq 'assets' && exists( $choices{$phase} ) ) {
print "Build assets table\n" if ( $verbose > 0 );
if (check_table($dbh2,'assets')) {
build_assets_table( $dbh3, $dbh2, $verbose ) unless $dry_run;
}
else {
print "** Table 'assets' is not empty\n";
}
}
#---------------------------------------------------------------------------
# Perform the 'epilogue' actions
#---------------------------------------------------------------------------
elsif ( $phase eq 'epilogue' && exists( $choices{$phase} ) ) {
print "Perform epilogue actions\n" if ( $verbose > 0 );
#
# Resolve the "double host" problems. We have *not* copied these
# across from the MySQL 'hosts' table, so can resolve them now.
#
resolve_double_hosts( $dbh1, $dbh2, $email_template, $default_licence,
$verbose ) unless $dry_run;
#
# Determine the first show date per host, assuming that's when they
# were added to the database
#
compute_host_date_added($dbh2) unless $dry_run;
}
}
exit;
#=== FUNCTION ================================================================
# NAME: build_episodes_table
# PURPOSE: Copy the data from the MariaDB 'eps' table to the Pg
# 'episodes' table
# PARAMETERS: $dbh1 Handle for the MariaDB database
# $dbh2 Handle for the Pg database
# $verbose Verbosity level
# RETURNS: Nothing
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub build_episodes_table {
my ( $dbh1, $dbh2, $verbose ) = @_;
my $count = 0;
$count += load_twat_episodes_table( $dbh1, $dbh2, 'twat', $verbose );
$count += load_eps_table( $dbh1, $dbh2, 'hpr', $verbose );
print "Added $count records to the 'episodes' table\n"
if ( $verbose > 0 );
#
# Set the sequence to the correct value
#
alter_seq( $dbh2, 'episodes', 'episode_id', 'episode_seq' );
}
#=== FUNCTION ================================================================
# NAME: load_twat_episodes_table
# PURPOSE: Load the 'twat_episodes' table from the MariaDB database into
# the PostgreSQL 'episodes' table.
# PARAMETERS: $dbh1 Handle for the MariaDB database
# $dbh2 Handle for the Pg database
# $key_prefix String to prefix the 'episode_key' field with
# $verbose Verbosity level
# RETURNS: Number of rows added
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub load_twat_episodes_table {
my ( $dbh1, $dbh2, $key_prefix, $verbose ) = @_;
my ( $sth1, $h1, $sth2, $count );
#
# Query to collect the entire 'twat_episodes' table
#
$sth1 = $dbh1->prepare('SELECT * FROM twat_episodes ORDER BY ep_num')
or die $DBI::errstr;
if ( $dbh1->err ) {
warn $dbh1->errstr;
}
#
# Query to insert into the PostgreSQL 'episodes' table
#
$sth2 = $dbh2->prepare(
q{INSERT INTO episodes
(episode_key, release_date, title, notes, explicit, license)
VALUES (?,date(to_timestamp(?)),?,?,TRUE,
id_in_licenses('CC-BY-SA'))}
) or die $DBI::errstr;
if ( $dbh2->err ) {
warn $dbh2->errstr;
}
#
# Query MariaDB for the entire 'twat_episodes' table
#
$sth1->execute;
if ( $dbh1->err ) {
die $dbh1->errstr;
}
#
# Loop though 'twat_episodes' table rows writing them to the PostgreSQL
# 'episodes' table
#
$count = 0;
while ( $h1 = $sth1->fetchrow_hashref ) {
$count++;
$sth2->execute(
sprintf( "%s%04d", $key_prefix, $h1->{ep_num} ),
$h1->{date},
coalesce( $h1->{topic}, '[undefined]' ),
coalesce( $h1->{writeup}, '<p>No notes</p>' ),
);
if ( $dbh2->err ) {
die $dbh2->errstr;
}
}
print "Copied $count records from 'twat_episodes'\n" if ( $verbose > 0 );
return $count;
}
#=== FUNCTION ================================================================
# NAME: load_eps_table
# PURPOSE: Load the 'eps' table from the MariaDB database into the
# PostgreSQL 'episodes' table.
# PARAMETERS: $dbh1 Handle for the MariaDB database
# $dbh2 Handle for the Pg database
# $key_prefix String to prefix the 'episode_key' field with
# $verbose Verbosity level
# RETURNS: Number of rows added
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub load_eps_table {
my ( $dbh1, $dbh2, $key_prefix, $verbose ) = @_;
my ( $sth1, $h1, $sth2, $count );
#
# Query to collect rows from the old dfatabase
#
$sth1 = $dbh1->prepare('SELECT * FROM eps ORDER BY id')
or die $DBI::errstr;
if ( $dbh1->err ) {
warn $dbh1->errstr;
}
#
# Query to insert rows into the new database
#
$sth2 = $dbh2->prepare(
q{INSERT INTO episodes
(episode_key, release_date, title, summary, notes, explicit,
license, duration, downloads)
VALUES (?,?,?,?,?,?,id_in_licenses(?),? * '1 second'::interval,?)}
) or die $DBI::errstr;
if ( $dbh2->err ) {
warn $dbh2->errstr;
}
#
# Query MariaDB for the entire 'eps' table
#
$sth1->execute;
if ( $dbh1->err ) {
die $dbh1->errstr;
}
#
# Loop though 'eps' table rows writing them to the PostgreSQL 'episodes'
# table
#
$count = 0;
while ( $h1 = $sth1->fetchrow_hashref ) {
$count++;
$sth2->execute(
sprintf( "%s%04d", $key_prefix, $h1->{id} ),
$h1->{date},
$h1->{title},
nullif( $h1->{summary}, '^\s*$' ),
$h1->{notes},
$h1->{explicit},
$h1->{license},
$h1->{duration},
$h1->{downloads},
);
if ( $dbh2->err ) {
die $dbh2->errstr;
}
}
print "Copied $count records from 'eps'\n" if ( $verbose > 0 );
return $count;
}
#=== FUNCTION ================================================================
# NAME: build_hosts_table
# PURPOSE: Copy the data from the Mariadb 'hosts' table to the Pg 'hosts'
# table
# PARAMETERS: $dbh1 Handle for the MariaDB table
# $dbh2 Handle for the Pg database
# $template Template for building the default email
# $verbose Verbosity level
# RETURNS: Nothing
# DESCRIPTION: Copies records from one table into the other. However, some
# transformations are made along the way: blank emails and
# 'admin@hackerpublicradio.org' are converted to 'host_NNN@hpr',
# empty profiles and GPG strings are turned into NULL values,
# and licence settings are converted to the id of the licence
# in the 'licenses' table (in the database).
# The double host problem is an issue though. Since the email
# address field is unique we get errors since the double hosts
# have duplicate addresses.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub build_hosts_table {
my ( $dbh1, $dbh2, $template, $verbose ) = @_;
my ( $sth1, $h1, $sth2, $rv, $count );
$sth1 = $dbh1->prepare('SELECT * FROM hosts') or die $DBI::errstr;
if ( $dbh1->err ) {
warn $dbh1->errstr;
}
$sth2 = $dbh2->prepare(
q{INSERT INTO hosts
(host_id, host, email, profile, license, local_image, gpg, espeak_name)
VALUES (?,?,?,?,id_in_licenses(?),?,?,?)}
) or die $DBI::errstr;
if ( $dbh2->err ) {
warn $dbh2->errstr;
}
$sth1->execute;
if ( $dbh1->err ) {
die $dbh1->errstr;
}
$count = 0;
while ( $h1 = $sth1->fetchrow_hashref ) {
#
# Ignore "double host" entries
#
if ( $h1->{host} !~ /^(.+)\s+and\s+(.+)$/ ) {
$count++;
$sth2->execute(
$h1->{hostid},
$h1->{host},
default_email(
$h1->{email}, '^(\s*|admin@hackerpublicradio.org)$',
$template, $h1->{hostid}
),
nullif( $h1->{profile}, '^\s*$' ),
$h1->{license},
$h1->{local_image},
nullif( $h1->{gpg}, '^\s*$' ),
$h1->{espeak_name},
);
if ( $dbh2->err ) {
die $dbh2->errstr;
}
}
}
print "Copied $count records\n" if ( $verbose > 0 );
#
# Set the sequence to the correct value
#
alter_seq( $dbh2, 'hosts', 'host_id', 'host_seq' );
}
#=== FUNCTION ================================================================
# NAME: build_episodes_hosts_xref_table
# PURPOSE: Generates the cross reference table by examining the 'eps' and
# 'hosts' tables in the MariaDB database.
# PARAMETERS: $dbh1 Handle for the MariaDB table
# $dbh2 Handle for the Pg database
# $verbose Verbosity level
# RETURNS: Nothing
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub build_episodes_hosts_xref_table {
my ( $dbh1, $dbh2, $verbose ) = @_;
my $count = 0;
$count += link_twat_episodes( $dbh1, $dbh2, $verbose );
$count += link_hpr_episodes( $dbh1, $dbh2, $verbose );
print "Added a total of $count links\n" if ( $verbose > 0 );
}
#=== FUNCTION ================================================================
# NAME: link_twat_episodes
# PURPOSE: Generate cross reference links between TwaT episodes and hosts
# PARAMETERS: $dbh1 Handle for the MariaDB database
# $dbh2 Handle for the Pg database
# $verbose Verbosity level
# RETURNS: The count of links
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub link_twat_episodes {
my ( $dbh1, $dbh2, $verbose ) = @_;
my ( $sth1, $h1, $sth2, $count, $episode_key );
#
# Query to find the TwaT episode and hostid from the old database. Use
# LEFT JOIN so we see when there's no host match.
#
$sth1 = $dbh1->prepare(
# q{SELECT t.ep_num,h.hostid
# FROM twat_episodes t, hosts h
# WHERE t.host = h.host}
q{SELECT t.ep_num, h.hostid
FROM twat_episodes t
LEFT JOIN hosts h ON (t.host = h.host)}
) or die $DBI::errstr;
if ( $dbh1->err ) {
warn $dbh1->errstr;
}
#
# Query to add a cross reference link to the new database
#
$sth2 = $dbh2->prepare(
q{INSERT INTO episodes_hosts_xref
SELECT e.episode_id, h.host_id
FROM episodes e, hosts h
WHERE e.episode_key = ? AND h.host_id = ?}
) or die $DBI::errstr;
if ( $dbh2->err ) {
warn $dbh2->errstr;
}
#
# Collect all the TwaT episodes and host id numbers
#
$sth1->execute;
if ( $dbh1->err ) {
die $dbh1->errstr;
}
#
# Loop through the result of the query, adding links to the new database.
# If the returned hostid is NULL then we don't have this host in the
# 'hosts' table and need to warn about it.
#
$count = 0;
while ( $h1 = $sth1->fetchrow_hashref ) {
$episode_key = sprintf( "twat%04d", $h1->{ep_num} );
if (defined($h1->{hostid})) {
$count++;
$sth2->execute( $episode_key, $h1->{hostid}, );
if ( $dbh2->err ) {
die $dbh2->errstr;
}
}
else {
warn "Unable to link $episode_key to a host\n";
}
}
print "Added $count links to TwaT episodes\n" if ( $verbose > 0 );
return $count;
}
#=== FUNCTION ================================================================
# NAME: link_hpr_episodes
# PURPOSE: Generate cross reference links between HPR episodes and hosts
# PARAMETERS: $dbh1 Handle for the MariaDB database
# $dbh2 Handle for the Pg database
# $verbose Verbosity level
# RETURNS: The count of links
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub link_hpr_episodes {
my ( $dbh1, $dbh2, $verbose ) = @_;
my ( $sth1, $h1, $sth2, $count, $episode_key );
#
# Query to find the HPR episode and hostid from the old database
#
$sth1 = $dbh1->prepare(
q{SELECT e.id,h.hostid
FROM eps e, hosts h
WHERE e.hostid = h.hostid}
) or die $DBI::errstr;
if ( $dbh1->err ) {
warn $dbh1->errstr;
}
#
# Query to add a cross reference link to the new database
#
$sth2 = $dbh2->prepare(
q{INSERT INTO episodes_hosts_xref
SELECT e.episode_id, h.host_id
FROM episodes e, hosts h
WHERE e.episode_key = ? AND h.host_id = ?}
) or die $DBI::errstr;
if ( $dbh2->err ) {
warn $dbh2->errstr;
}
#
# Collect all the HPR episodes and host id numbers
#
$sth1->execute;
if ( $dbh1->err ) {
die $dbh1->errstr;
}
#
# Loop through the result of the query, adding links to the new database
#
$count = 0;
while ( $h1 = $sth1->fetchrow_hashref ) {
$count++;
$episode_key = sprintf( "hpr%04d", $h1->{id} );
$sth2->execute( $episode_key, $h1->{hostid}, );
if ( $dbh2->err ) {
die $dbh2->errstr;
}
}
print "Added $count links to HPR episodes\n" if ( $verbose > 0 );
return $count;
}
#=== FUNCTION ================================================================
# NAME: build_series_table
# PURPOSE: Copy the data from the Mariadb 'miniseries' table to the Pg
# 'series' table
# PARAMETERS: $dbh1 Handle for the MariaDB database
# $dbh2 Handle for the Pg database
# $verbose Verbosity level
# RETURNS: Nothing
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub build_series_table {
my ( $dbh1, $dbh2, $verbose ) = @_;
my ( $sth1, $h1, $sth2, $count );
$sth1 = $dbh1->prepare('SELECT * FROM miniseries') or die $DBI::errstr;
if ( $dbh1->err ) {
warn $dbh1->errstr;
}
$sth2 = $dbh2->prepare('INSERT INTO series VALUES (?,?,?,?,?)')
or die $DBI::errstr;
if ( $dbh2->err ) {
warn $dbh2->errstr;
}
#
# Query MariaDB for the entire 'miniseries' table
#
$sth1->execute;
if ( $dbh1->err ) {
die $dbh1->errstr;
}
#
# Loop though 'miniseries' table rows writing them to the PostgreSQL
# 'series' table
#
$count = 0;
while ( $h1 = $sth1->fetchrow_hashref ) {
$count++;
$sth2->execute(
$h1->{id}, $h1->{name},
$h1->{description}, $h1->{private},
nullif( $h1->{image}, '^\s*$' ),
# $h1->{valid},
);
if ( $dbh2->err ) {
die $dbh2->errstr;
}
}
print "Copied $count records\n" if ($verbose > 0);
#
# Set the sequence to the correct value
#
alter_seq( $dbh2, 'series', 'series_id', 'series_seq' );
}
#=== FUNCTION ================================================================
# NAME: build_episodes_series_xref_table
# PURPOSE: Generates the cross reference table by examining the 'eps' and
# 'series' tables in the MariaDB database.
# PARAMETERS: $dbh1 Handle for the MariaDB table
# $dbh2 Handle for the Pg database
# $verbose Verbosity level
# RETURNS: Nothing
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub build_episodes_series_xref_table {
my ( $dbh1, $dbh2, $verbose ) = @_;
my ( $sth1, $h1, $sth2, $count, $episode_key );
#
# Query to find the HPR episode and series id from the old database
#
$sth1 = $dbh1->prepare(
q{SELECT e.id AS epid, m.id AS msid
FROM eps e, miniseries m
WHERE e.series = m.id}
) or die $DBI::errstr;
if ( $dbh1->err ) {
warn $dbh1->errstr;
}
#
# Query to add a cross reference link to the new database
#
$sth2 = $dbh2->prepare(
q{INSERT INTO episodes_series_xref
SELECT e.episode_id, s.series_id
FROM episodes e, series s
WHERE e.episode_key = ? AND s.series_id = ?}
) or die $DBI::errstr;
if ( $dbh2->err ) {
warn $dbh2->errstr;
}
#
# Collect all the HPR episodes and host id numbers (no TwaT shows are in
# series)
#
$sth1->execute;
if ( $dbh1->err ) {
die $dbh1->errstr;
}
#
# Loop through the result of the query, adding links to the new database
#
$count = 0;
while ( $h1 = $sth1->fetchrow_hashref ) {
$count++;
$episode_key = sprintf( "hpr%04d", $h1->{epid} );
$sth2->execute( $episode_key, $h1->{msid} );
if ( $dbh2->err ) {
die $dbh2->errstr;
}
}
print "Added a total of $count series links\n" if ( $verbose > 0 );
}
#=== FUNCTION ================================================================
# NAME: build_tags_table
# PURPOSE: Using the data structure built from the MariaDB database
# populate the many-to-many table in the Pg database
# PARAMETERS: $dbh Handle for the Pg database
# $verbose Verbosity level
# $tag_hash Reference to a hash of episode ids and tags
# for each episode
# RETURNS: Nothing
# DESCRIPTION: Before being called the tags in the MariaDB database are
# gathered into a hash which is passed to this function as an
# argument. The hash is keyed by episode number and each value
# consists of an arrayref containing the tag strings. These tags
# are then processed to add to the PostgreSQL database.
# For each episode (in sorted order) the tag array is processed.
# If a tag already exists in the 'tags' table the tag id is
# stored for later, otherwise the tag is added to the 'tags'
# table. The tag id from either source is used to add to the
# cross reference table 'episodes_tags_xref'.
# Adding a link requires the provision of the episode key, which
# in this case will be 'hprNNNN' where 'NNNN' is a 4 digit zero
# padded number. This is converted to the episode id by
# a PostgreSQL function.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub build_tags_table {
my ( $dbh, $verbose, $tag_hash ) = @_;
my ( $sth1, $h1, $sth2, $sth3, $rv);
my ( $tags, $tid, $episode_key, $count1, $count2 );
#
# Query to find if a tag already exists in the PostgreSQL database
#
$sth1 = $dbh->prepare(q{SELECT * FROM tags WHERE tag = ?});
#
# Query to add a new tag
#
$sth2 = $dbh->prepare(q{INSERT INTO tags (tag) VALUES(?)});
#
# Query to add a new joining row
#
$sth3 = $dbh->prepare(
q{INSERT INTO episodes_tags_xref
VALUES(id_in_episodes(?),?)}
);
$count1 = $count2 = 0;
foreach my $id ( sort { $a <=> $b } keys( %{$tag_hash} ) ) {
#
# The episode key is now more than a number
#
$episode_key = sprintf("hpr%04d", $id);
#
# Get the array of tags for this episode id
#
$tags = $tag_hash->{$id};
#
# Loop through the array of tags (using an integer so we can index the
# current tag)
#
for my $i ( 0 .. $#$tags ) {
#
# Look to see if this tag exists
#
$sth1->execute( $tags->[$i] );
if ( $dbh->err ) {
warn $dbh->errstr;
}
#
# If it's already in the table just store the id for later
# otherwise add a new entry
#
if ( $h1 = $sth1->fetchrow_hashref ) {
$tid = $h1->{tag_id};
}
else {
#
# Add the tag to 'tags'
#
$count1++;
$rv = $sth2->execute( $tags->[$i] );
if ( $dbh->err ) {
warn $dbh->errstr;
}
$rv = 0 if ( $rv eq '0E0' );
#
# Get the id number of the inserted tag
#
$tid = $dbh->last_insert_id( undef, undef, undef, undef,
{ sequence => 'tag_seq' } );
}
$count2++;
$rv = $sth3->execute( $episode_key, $tid );
if ( $dbh->err ) {
warn $dbh->errstr;
}
$rv = 0 if ( $rv eq '0E0' );
}
}
if ($verbose > 0) {
print "Added $count1 tags\n";
print "Added $count2 cross references\n";
}
}
#=== FUNCTION ================================================================
# NAME: collect_eps_tags
# PURPOSE: Collects the tags from the eps.tags field
# PARAMETERS: $dbh Database handle
# $verbose Verbosity level
# RETURNS: A reference to the hash created by collecting all the tags
# DESCRIPTION: Queries the MariaDB 'eps' table for all of the rows containing
# tags, returning the comma-separated list with the id number.
# Each CSV list is then parsed and the result turned into a hash
# keyed on the id number and containing a sorted array of tags.
# If the level of verbosity is greater than 2 the tags hash is
# dumped (ironically, as a CSV list!).
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub collect_eps_tags {
my ( $dbh, $verbose ) = @_;
my ( $status, @fields, %hash );
my ( $sth, $h );
#
# For parsing the field as CSV
#
my $csv = Text::CSV_XS->new;
#
# Query the MariaDB 'eps' table for all the id and tags
#
$sth = $dbh->prepare(
q{SELECT id,tags FROM eps
WHERE length(tags) > 0
ORDER BY id}
) or die $DBI::errstr;
if ( $dbh->err ) {
warn $dbh->errstr;
}
$sth->execute;
if ( $dbh->err ) {
warn $dbh->errstr;
}
#
# Loop through what we got
#
while ( $h = $sth->fetchrow_hashref ) {
#
# Parse the tag list
#
$status = $csv->parse( $h->{tags} );
unless ($status) {
#
# Report any errors
#
print "Parse error on episode ", $h->{id}, "\n";
print $csv->error_input(), "\n";
next;
}
@fields = $csv->fields();
next unless (@fields);
#
# Trim all tags (don't alter $_ when doing it)
#
@fields = map {
my $t = $_;
$t =~ s/(^\s+|\s+$)//g;
$t;
} @fields;
#print "$h->{id}: ",join(",",@fields),"\n";
#
# Save the id and its tags, sorted for comparison
#
$hash{ $h->{id} } = [ sort @fields ];
}
#
# Dump all id numbers and tags if the verbose level is high enough
#
if ( $verbose >= 2 ) {
print "\nTags collected from the 'eps' table\n\n";
foreach my $id ( sort { $a <=> $b } keys(%hash) ) {
printf "%04d: %s\n", $id, join( ",", @{ $hash{$id} } );
}
}
return \%hash;
}
#=== FUNCTION ================================================================
# NAME: build_comments_table
# PURPOSE: Copy the data from the Mariadb 'comments' table to the Pg
# 'comments' table
# PARAMETERS: $dbh1 Handle for the MariaDB database
# $dbh2 Handle for the Pg database
# $verbose Verbosity level
# RETURNS: Nothing
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub build_comments_table {
my ( $dbh1, $dbh2, $verbose ) = @_;
my ( $sth1, $h1, $sth2, $count );
#
# Query to collect comments from the old database
#
$sth1 = $dbh1->prepare('SELECT * FROM comments') or die $DBI::errstr;
if ( $dbh1->err ) {
warn $dbh1->errstr;
}
#
# Query to insert comment data into the new database
#
$sth2 = $dbh2->prepare(
q{INSERT INTO comments VALUES (?,id_in_episodes(?),?,?,?,?,?)}
) or die $DBI::errstr;
if ( $dbh2->err ) {
warn $dbh2->errstr;
}
#
# Query MariaDB for the entire 'comments' table
#
$sth1->execute;
if ( $dbh1->err ) {
die $dbh1->errstr;
}
#
# Loop though 'comments' table rows writing them to the PostgreSQL
# 'comments' table
#
$count = 0;
while ( $h1 = $sth1->fetchrow_hashref ) {
$count++;
$sth2->execute(
$h1->{id},
sprintf("hpr%04d",$h1->{eps_id}),
$h1->{comment_timestamp},
nullif( $h1->{comment_author_name}, '^\s*$' ),
nullif( $h1->{comment_title}, '^\s*$' ),
$h1->{comment_text},
$h1->{last_changed},
);
if ( $dbh2->err ) {
die $dbh2->errstr;
}
}
print "Copied $count comments\n" if ($verbose > 0);
#
# Set the sequence to the correct value
#
alter_seq( $dbh2, 'comments', 'comment_id', 'comment_seq' );
}
#=== FUNCTION ================================================================
# NAME: update_episodes_table
# PURPOSE: Initialise archiv-related fields in the Pg 'episodes' table
# from the SQLite 'episodes' table
# PARAMETERS: $dbh1 Handle for the SQLite database
# $dbh2 Handle for the Pg database
# $verbose Verbosity level
# RETURNS: Nothing
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub update_episodes_table {
my ( $dbh1, $dbh2, $verbose ) = @_;
my ( $sth1, $h1, $sth2, $count );
#
# Query to collect episodes from the SQLite database
#
$sth1 = $dbh1->prepare('SELECT * FROM episodes') or die $DBI::errstr;
if ( $dbh1->err ) {
warn $dbh1->errstr;
}
#
# Query to insert fields into the Pg database 'episodes' table. We just
# write whatever comes in, no check for existing data.
#
$sth2 = $dbh2->prepare(
q{UPDATE episodes SET
archived = ?,
archive_date = ?,
IA_URL = ?
WHERE episode_key = ?}
) or die $DBI::errstr;
if ( $dbh2->err ) {
warn $dbh2->errstr;
}
#
# Query SQLite for the entire 'episodes' table
#
$sth1->execute;
if ( $dbh1->err ) {
die $dbh1->errstr;
}
#
# Loop though 'episodes' table rows writing selected fields to the
# PostgreSQL 'episodes' table
#
$count = 0;
while ( $h1 = $sth1->fetchrow_hashref ) {
$count++;
$sth2->execute(
$h1->{uploaded},
$h1->{archive_date},
$h1->{IA_URL},
sprintf("hpr%04d",$h1->{id}),
);
if ( $dbh2->err ) {
die $dbh2->errstr;
}
}
print "Updated $count rows in the episodes table\n" if ($verbose > 0);
}
#=== FUNCTION ================================================================
# NAME: build_assets_table
# PURPOSE: Copy the data from the SQLite 'assets' table to the Pg
# 'assets' table
# PARAMETERS: $dbh1 Handle for the SQLite database
# $dbh2 Handle for the Pg database
# $verbose Verbosity level
# RETURNS: Nothing
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub build_assets_table {
my ( $dbh1, $dbh2, $verbose ) = @_;
my ( $sth1, $h1, $sth2, $count );
#
# Query to collect assets from the SQLite database
#
$sth1 = $dbh1->prepare('SELECT * FROM assets') or die $DBI::errstr;
if ( $dbh1->err ) {
warn $dbh1->errstr;
}
#
# Query to insert asset data into the new database
#
$sth2 = $dbh2->prepare(
q{INSERT INTO assets VALUES (DEFAULT,id_in_episodes(?),?,?,?)}
) or die $DBI::errstr;
if ( $dbh2->err ) {
warn $dbh2->errstr;
}
#
# Query SQLite for the entire 'assets' table
#
$sth1->execute;
if ( $dbh1->err ) {
die $dbh1->errstr;
}
#
# Loop though 'assets' table rows writing them to the PostgreSQL
# 'assets' table
#
$count = 0;
while ( $h1 = $sth1->fetchrow_hashref ) {
$count++;
$sth2->execute(
sprintf("hpr%04d",$h1->{episode_id}),
$h1->{URL},
$h1->{filename},
$h1->{uploaded},
);
if ( $dbh2->err ) {
die $dbh2->errstr;
}
}
print "Copied $count assets\n" if ($verbose > 0);
#
# Set the sequence to the correct value
#
alter_seq( $dbh2, 'assets', 'asset_id', 'asset_seq' );
}
#=== FUNCTION ================================================================
# NAME: resolve_double_hosts
# PURPOSE: Turn any double host entries in the 'hosts' table into
# "singletons"
# PARAMETERS: $dbh1 Handle for the MariaDB database
# $dbh2 Handle for the Pg database
# $template Template for building the default email
# $licence string version of the short CC licence name
# $verbose Verbosity level
# RETURNS: Nothing
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub resolve_double_hosts {
my ( $dbh1, $dbh2, $template, $licence, $verbose ) = @_;
my ( $sth1, $h1, $rv1, $sth2, $h2, $rv2, $sth3, $h3, $rv3, $sth4, $h4, $rv4 );
my ( %doubles, @h, %hosts, $count, $unknown );
#
# Search the 'hosts' table in the old database for "host1 and host2"
# strings
#
$count = find_double_hosts($dbh1, \%doubles, \%hosts);
#
# If no doubles there's nothing to do
#
if ($count == 0) {
print "No doubles found\n" if ($verbose > 0);
return;
}
#
# Scan the list of individual hosts we stored earlier and find them in
# the 'hosts' table (assuming that the hostnames we have extracted match
# exactly)
#
$unknown = find_hosts($dbh1, \%doubles, \%hosts);
#
# Allocate all unknown hosts a host id in the PostgreSQL database, and give an
# unique email address.
#
if ( $unknown > 0 ) {
register_unknown($dbh2, \%doubles, \%hosts, $template,
$licence);
}
#
# Now %doubles contains all the original names and host ids and %hosts
# contains the parsed out names and their ids. We can look for shows
# attributed to the double hosts and re-attribute them to the single
# hosts.
#
link_double_twat_episodes($dbh1,$dbh2,\%doubles,$verbose);
link_double_hpr_episodes($dbh1,$dbh2,\%doubles,$verbose);
return;
}
#=== FUNCTION ================================================================
# NAME: find_double_hosts
# PURPOSE: Search the 'hosts' table in the old database for "host1 and
# host2" strings.
# PARAMETERS: $dbh handle for the MariaDB database
# $doubles hashref where details will be stored
# $hosts hashref to contain single host details
# RETURNS: The number of double hosts found
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub find_double_hosts {
my ($dbh, $doubles, $hosts) = @_;
my ($sth, $h, @h, $count);
#
# Find the double hosts in the old database
#
$sth = $dbh->prepare(
q{ SELECT hostid, host FROM hosts
WHERE host regexp '[[:<:]]and[[:>:]]'
ORDER BY hostid
}
) or die $DBI::errstr;
if ( $dbh->err ) {
warn $dbh->errstr;
return 0;
}
$sth->execute;
if ( $dbh->err ) {
die $dbh->errstr;
}
#
# Loop through the doubles we found, counting them as we go
#
$count = 0;
while ( $h = $sth->fetchrow_hashref ) {
#
# Each hash value is a hash containing the original id, and, in a sub-hash
# the replacement ids
#
$doubles->{$h->{host}} = {
double => $h->{hostid},
singles => {},
};
#
# Parse the double host string into an array
#
@h = ( $h->{host} =~ /^(.+)\s+and\s+(.+)$/ );
#
# Initialise the entries for %doubles and %hosts
#
for my $host (@h) {
$doubles->{$h->{host}}->{singles}->{$host} = undef;
unless ( exists( $hosts->{$host} ) ) {
$hosts->{$host} = 0;
}
}
$count++;
}
return $count;
}
#=== FUNCTION ================================================================
# NAME: find_hosts
# PURPOSE: Find individual hosts in the old MariaDB database using the
# names we found as double hosts
# PARAMETERS: $dbh handle for the MariaDB database
# $doubles hashref where details will be stored
# $hosts hashref to contain single host details
# RETURNS: The number of unknown hosts
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub find_hosts {
my ($dbh, $doubles, $hosts) = @_;
my ($sth, $h, $rv, @h, $unknown);
#
# Query to find the host by name in the old database
#
$sth = $dbh->prepare(q{SELECT hostid FROM hosts WHERE host = ?})
or die $DBI::errstr;
if ( $dbh->err ) {
warn $dbh->errstr;
}
#
# Scan the list of individual hosts we stored earlier and find them in
# the 'hosts' table (assuming that the hostnames we have extracted match
# exactly)
#
$unknown = 0;
foreach my $host ( sort(keys(%{$hosts})) ) {
$rv = $sth->execute($host);
if ( $dbh->err ) {
die $dbh->errstr;
}
$rv = 0 if ( $rv eq '0E0' );
if ($rv) {
#
# Found id for host. Save in the %doubles hash
#
$h = $sth->fetchrow_hashref;
$hosts->{$host} = $h->{hostid};
save_hostid($doubles,$host,$h->{hostid});
}
else {
#
# Can't find this host
#
$unknown++;
}
}
return $unknown;
}
#=== FUNCTION ================================================================
# NAME: register_unknown
# PURPOSE: Create an entry in the 'hosts' table of the new database
# wherever there's a host that's unknown (has a zero id in the
# %hosts hash).
# PARAMETERS: $dbh handle for the MariaDB database
# $doubles hashref where details will be stored
# $hosts hashref to contain single host details
# $email_template template for making an email address
# $licence string version of the short CC licence name
# RETURNS: Nothing
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub register_unknown {
my ( $dbh, $doubles, $hosts, $email_template, $licence ) = @_;
my ( $sth, $h, $rv, $new_email );
#
# PostgreSQL query to register an unknown host
#
$sth = $dbh->prepare(
q{INSERT INTO hosts (host,email,license)
VALUES (?,?,id_in_licenses(?))}
) or die $DBI::errstr;
if ( $dbh->err ) {
warn $dbh->errstr;
}
foreach my $host ( sort( keys( %{$hosts} ) ) ) {
if ( $hosts->{$host} == 0 ) {
#
# Write a row to the 'hosts' table. There has to be an email,
# but we can't compute one until the row has been added.
#
$rv = $sth->execute( $host, 'placeholder', $licence );
if ( $dbh->err ) {
die $dbh->errstr;
}
#
# Save the id number we just generated as the new host id
#
my $newid = $dbh->last_insert_id( undef, undef, undef, undef,
{ sequence => 'host_seq' } );
$hosts->{$host} = $newid;
save_hostid( $doubles, $host, $newid );
printf "Created new host %s (%d)\n", $host, $newid
if ( $verbose > 0 );
#
# Give the new host entry a default email address
#
$new_email = sprintf( $email_template, $newid );
$rv = $dbh->do( 'UPDATE hosts SET email = ? WHERE host_id = ?',
undef, $new_email, $newid );
if ( $dbh->err ) {
warn $dbh->errstr;
}
$rv = 0 if ( $rv eq '0E0' );
warn "Failed to set email address $new_email for $host\n"
unless ( defined($rv) );
}
}
}
#=== FUNCTION ================================================================
# NAME: link_double_twat_episodes
# PURPOSE: Find and link TwaT episodes with double hosts
# PARAMETERS: $dbh1 Handle for the MariaDB database
# $dbh2 Handle for the Pg database
# $doubles hashref holding double host details
# $verbose Verbosity level
# RETURNS: Nothing
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub link_double_twat_episodes {
my ( $dbh1, $dbh2, $doubles, $verbose ) = @_;
my ( $sth, $h, $rv );
#
# Query to find TwaT shows with particular host names in the old database
#
$sth = $dbh1->prepare(
q{
SELECT ep_num
FROM twat_episodes
WHERE host = ?
}
) or die $DBI::errstr;
if ( $dbh1->err ) {
warn $dbh1->errstr;
}
#
# Loop through the double hosts we collected
#
foreach my $double ( sort( keys(%{$doubles}) ) ) {
#
# Extract saved double and corresponding single details
#
my ( $doubleid, @newhosts ) = (
$doubles->{$double}->{double},
keys( %{ $doubles->{$double}->{singles} } )
);
#
# Find TwaT shows marked as belonging to this double-host (in the old
# database)
#
$sth->execute($double);
if ( $dbh1->err ) {
die $dbh1->errstr;
}
#
# Process all the shows we found
#
while ( $h = $sth->fetchrow_hashref ) {
my $eps_id = $h->{ep_num};
my $episode_key = sprintf("twat%04d",$eps_id);
#
# If one insert into the new database fails they all do
#
$dbh2->begin_work();
#
# Add links for the single hosts to the new database
#
foreach my $host (@newhosts) {
$rv = $dbh2->do(
q{INSERT INTO episodes_hosts_xref
SELECT e.episode_id, h.host_id
FROM episodes e, hosts h
WHERE e.episode_key = ? AND h.host = ?},
undef, $episode_key, $host);
if ( $dbh2->err ) {
warn $dbh2->errstr;
}
$rv = 0 if ( $rv eq '0E0' );
unless ( defined($rv) ) {
warn "Problem adding link to episodes_hosts_xref for "
. "$episode_key,$host\n";
}
else {
printf "Linked show %s for host %s\n", $episode_key, $host;
}
}
#
# Commit the inserts above
#
$dbh2->commit();
}
}
}
#=== FUNCTION ================================================================
# NAME: link_double_hpr_episodes
# PURPOSE: Find and link HPR episodes with double hosts
# PARAMETERS: $dbh1 Handle for the MariaDB database
# $dbh2 Handle for the Pg database
# $doubles hashref holding double host details
# $verbose Verbosity level
# RETURNS: Nothing
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub link_double_hpr_episodes {
my ( $dbh1, $dbh2, $doubles, $verbose ) = @_;
my ( $sth, $h, $rv );
#
# Query to find HPR shows with particular host ids in the old database
#
$sth = $dbh1->prepare(
q{
SELECT id AS eps_id
FROM eps
WHERE hostid = ?
}
) or die $DBI::errstr;
if ( $dbh1->err ) {
warn $dbh1->errstr;
}
#
# Loop through the double hosts we collected
#
foreach my $double ( sort( keys(%{$doubles}) ) ) {
#
# Extract saved double and corresponding single details
#
my ( $doubleid, @newhosts ) = (
$doubles->{$double}->{double},
keys( %{ $doubles->{$double}->{singles} } )
);
#
# Find HPR shows marked as belonging to this double-host (in the old
# database)
#
$sth->execute($doubleid);
if ( $dbh1->err ) {
die $dbh1->errstr;
}
#
# Process all the shows we found
#
while ( $h = $sth->fetchrow_hashref ) {
my $eps_id = $h->{eps_id};
my $episode_key = sprintf("hpr%04d",$eps_id);
#
# If one insert into the new database fails they all do
#
$dbh2->begin_work();
#
# Add links for the single hosts to the new database
#
foreach my $host (@newhosts) {
$rv = $dbh2->do(
q{INSERT INTO episodes_hosts_xref
SELECT e.episode_id, h.host_id
FROM episodes e, hosts h
WHERE e.episode_key = ? AND h.host = ?},
undef, $episode_key, $host);
if ( $dbh2->err ) {
warn $dbh2->errstr;
}
$rv = 0 if ( $rv eq '0E0' );
unless ( defined($rv) ) {
warn "Problem adding link to episodes_hosts_xref for "
. "$episode_key,$host\n";
}
else {
printf "Linked show %s for host %s\n", $episode_key, $host;
}
}
#
# Commit the inserts above
#
$dbh2->commit();
}
}
}
#=== FUNCTION ================================================================
# NAME: save_hostid
# PURPOSE: Saves the host id after searching for the key in the %doubles
# hash
# PARAMETERS: $doubles hashref to %doubles
# $host host key
# $hostid host id number
# RETURNS: Nothing
# DESCRIPTION: Searches the %doubles hash for particular keys in the
# 'singles' sub-hash. If found saves the corresponding host id
# there.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub save_hostid {
my ( $doubles, $host, $hostid ) = @_;
foreach my $key ( keys(%$doubles) ) {
if ( exists( $doubles->{$key}->{singles}->{$host} ) ) {
$doubles->{$key}->{singles}->{$host} = $hostid;
}
}
}
#=== FUNCTION ================================================================
# NAME: compute_host_date_added
# PURPOSE: Determine the 'hosts.when_added' field once the database is
# fully populated.
# PARAMETERS: $dbh Handle for the PostgreSQL database
# RETURNS: Nothing
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub compute_host_date_added {
my ($dbh) = @_;
my $rv;
#
# Allocate when_added values where possible
#
$rv = $dbh->do(
q{
UPDATE hosts
SET when_added = sq.when_added
FROM (
SELECT h.host_id,min(e.release_date) AS when_added
FROM episodes e
JOIN episodes_hosts_xref eh ON (e.episode_id = eh.episode_id)
JOIN hosts h ON (h.host_id = eh.host_id)
GROUP BY h.host_id
ORDER by min(e.release_date)) AS sq
WHERE hosts.host_id = sq.host_id
}
);
if ( $dbh->err ) {
warn $dbh->errstr;
}
$rv = 0 if ( $rv eq '0E0' );
print "Added $rv dates to the 'when_added' column\n";
}
#=== FUNCTION ================================================================
# NAME: alter_seq
# PURPOSE: Ensure the PostgreSQL sequence associated with a table has the
# correct value.
# PARAMETERS: $dbh Handle for the PostgreSQL database
# $table Table name for the query
# $idname Id field name (primary key)
# $sequence Sequence name
# RETURNS: Nothing
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub alter_seq {
my ( $dbh, $table, $idname, $sequence ) = @_;
my ( $sth, $h, $rv, $maxid );
#
# Find the maximum id number in the table
#
$sth = $dbh->prepare("SELECT max($idname) as maxid FROM $table")
or die $DBI::errstr;
if ( $dbh->err ) {
warn $dbh->errstr;
}
$sth->execute;
if ( $dbh->err ) {
die $dbh->errstr;
}
#
# Save the maximum
#
if ( $h = $sth->fetchrow_hashref ) {
$maxid = $h->{maxid};
$sth->finish;
}
#
# Reset the sequence one more than the maximum
#
$maxid++;
$rv = $dbh->do("ALTER SEQUENCE $sequence RESTART WITH $maxid");
if ( $dbh->err ) {
warn $dbh->errstr;
}
$rv = 0 if ( $rv eq '0E0' );
warn "Failed to reset $sequence\n" unless (defined($rv));
return;
}
#=== FUNCTION ================================================================
# NAME: check_table
# PURPOSE: Check that a given PostgreSQL table is empty
# PARAMETERS: $dbh Handle for the PostgreSQL database
# $table Name of table
# RETURNS: True if empty, otherwise false
# DESCRIPTION: Simply perform a query on the nominated table which counts
# rows. If the table does not exist a DBI method will fail (the
# execute?), so we treat this as a 'no empty' to make the caller
# take error action.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub check_table {
my ( $dbh, $table ) = @_;
my ( $sth, $h, $count );
$sth = $dbh->prepare("SELECT count(*) AS count FROM $table")
or die $DBI::errstr;
if ( $dbh->err ) {
warn $dbh->errstr;
return 0;
}
$sth->execute;
if ( $dbh->err ) {
warn $dbh->errstr;
return 0;
}
if ( $h = $sth->fetchrow_hashref ) {
$count = $h->{count};
$sth->finish;
}
return $count == 0;
}
#=== FUNCTION ================================================================
# NAME: default_email
# PURPOSE: Make a default email address for hosts with none
# PARAMETERS: $email Original email address
# $regex Regular expression to check the email against
# $template Template for building the default
# $hostid Host id number to use in the default
# RETURNS: The email address to be used
# DESCRIPTION: If the email address matches a regular expression then
# generate a default from the template and the host id,
# otherwise just return the address untouched.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub default_email {
my ( $email, $regex, $template, $hostid ) = @_;
return (
$email =~ $regex
? sprintf( $template, $hostid )
: $email
);
}
#=== FUNCTION ================================================================
# NAME: nullif
# PURPOSE: Tests a value and makes it 'undef' (equivalent to NULL in the
# database) if it matches a regular expression.
# PARAMETERS: $value Value to test
# $regex Regular expression to match against
# RETURNS: 'undef' if the values match, otherwise the original value
# DESCRIPTION: This is very simple, just a wrapper around the test.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub nullif {
my ( $value, $regex ) = @_;
return $value unless defined($value);
return ( $value =~ $regex ? undef : $value );
}
#=== FUNCTION ================================================================
# NAME: coalesce
# PURPOSE: To find the first defined argument and return it
# PARAMETERS: Arbitrary number of arguments
# RETURNS: The first defined argument or undef if there are none
# DESCRIPTION: Modelled on the SQL function of the same name. It takes a list
# of arguments, scans it for the first one that is not undefined
# and returns it. If an argument is defined and it's an arrayref
# then the referenced array is returned comma-delimited. This
# allows calls such as "coalesce($var,'undef')" which returns
# the value of $var if it's defined, and 'undef' if not and
# doesn't break anything along the way.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub coalesce {
foreach (@_) {
if ( defined($_) ) {
if ( ref($_) eq 'ARRAY' ) {
return join( ',', @{$_} );
}
else {
return $_;
}
}
}
return; # implicit undef
}
#=== FUNCTION ================================================================
# NAME: Options
# PURPOSE: Processes command-line options
# PARAMETERS: $optref Hash reference to hold the options
# RETURNS: Undef
# DESCRIPTION:
# THROWS: no exceptions
# COMMENTS: none
# SEE ALSO: n/a
#===============================================================================
sub Options {
my ($optref) = @_;
my @options = ( "help", "config=s", "debug=i", "dry-run!", "verbose+",
"phases=s" );
if ( !GetOptions( $optref, @options ) ) {
pod2usage( -msg => "$PROG version $VERSION\n", -exitval => 1 );
}
return;
}
__END__
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# Application Documentation
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#{{{
=head1 NAME
copy_mysql_pg - copy the HPR database from MySQL to PostgreSQL
=head1 VERSION
This documentation refers to B<copy_mysql_pg> version 0.1.5
=head1 USAGE
copy_mysql_pg2 -verbose
copy_mysql_pg2 -config=.hpr_livedb.cfg -verbose
copy_mysql_pg2 -verbose \
-phase='episodes,hosts,eh_xref,series,es_xref,tags,comments,archived,assets,epilogue'
=head1 OPTIONS
=over 8
=item B<-help>
Prints a brief help message describing the usage of the program, and then exits.
=item B<-debug=N>
Selects a level of debugging. Debug information consists of a line or series
of lines prefixed with the characters 'D>':
=over 4
=item B<0>
No debug output is generated: this is the default
=back
=item B<-[no]dry-run>
When enabled (B<-dry-run>) the script will report what it would do, but will
make no changes to the target database. In the default state (B<-nodry-run>)
then changes are made.
=item B<-verbose>
Makes the script verbose resulting in the production of more information about
what it is doing.
The option may be repeated to increase the level of verbosity. The levels are:
=over 4
=item B<0>
No output is generated (apart from errors and warnings if appropriate). This
is the default level.
=item B<1>
A message is generated per phase to indicate which actions are taking place.
This includes a report of the number of rows copied from the MySQL database to
the PostgreSQL one.
=item B<2>
Following the process of collecting the CSV tags from the MySQL 'episodes' table
these are reported as a list per episode. This output will be long!
=back
=item B<-phase=CSV_LIST>
This option allows the phases of the copying process to be selected
individually. The argument B<CSV_LIST> is a list of phase names, which have to
be typed exactly. The order is not important since the script will scan its
version of the list of phases in its own order and will check to see if each
has been selected.
The phase names are:
=over 4
=item B<episodes>
Causes the B<episodes> table to be filled.
=item B<hosts>
Causes the B<hosts> table to be filled.
=item B<eh_xref>
Causes the B<episodes_hosts_xref> table to be filled.
=item B<series>
Causes the B<series> table to be filled.
=item B<es_xref>
Causes the B<episodes_series_xref> table to be filled.
=item B<tags>
Causes the B<tags> and the B<episodes_tags_xref> tables to be filled.
=item B<comments>
Causes the B<comments> table to be filled.
=item B<archived>
Causes the archive-related fields of the B<episodes> table to be filled from
the SQLite database B<ia.db>.
=item B<assets>
Causes the B<assets> table to be filled from the SQLite database B<ia.db>.
=item B<epilogue>
Runs various tasks that can only be carried out after the database has been
populated.
=back
=back
=item B<-config=FILE>
This option allows an alternative configuration file to be used. This file
defines the location of the database, its port, its name and the username and
password to be used to access it. This feature was added to allow the script
to access alternative databases or the live database over an SSH tunnel.
See the CONFIGURATION AND ENVIRONMENT section below for the file format.
If the option is omitted the default file is used: B<.hpr_db.cfg>
=head1 DESCRIPTION
A full description of the application and its features.
May include numerous subsections (i.e. =head2, =head3, etc.)
=head1 DIAGNOSTICS
A list of every error and warning message that the application can generate
(even the ones that will "never happen"), with a full explanation of each
problem, one or more likely causes, and any suggested remedies. If the
application generates exit status codes (e.g. under Unix) then list the exit
status associated with each error.
=head1 CONFIGURATION AND ENVIRONMENT
The script obtains the credentials it requires to open the HPR database from
a configuration file. The name of the file it expects is B<.hpr_db.cfg> in the
directory holding the script. To change this will require changing the script.
The configuration file format is as follows:
<database>
host = 127.0.0.1
port = PORT
name = DATABASE
user = USERNAME
password = PASSWORD
</database>
=head1 DEPENDENCIES
A list of all the other modules that this module relies upon, including any
restrictions on versions, and an indication whether these required modules are
part of the standard Perl distribution, part of the module's distribution,
or must be installed separately.
=head1 BUGS AND LIMITATIONS
There are no known bugs in this module.
Please report problems to <Maintainer name(s)> (<contact address>)
Patches are welcome.
=head1 AUTHOR
Dave Morriss (Dave.Morriss@gmail.com)
=head1 LICENCE AND COPYRIGHT
Copyright (c) 2017-2019 Dave Morriss (Dave.Morriss@gmail.com). All rights reserved.
This module is free software; you can redistribute it and/or
modify it under the same terms as Perl itself. See perldoc perlartistic.
This program is distributed in the hope that it will be useful
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
=cut
#}}}
# [zo to open fold, zc to close]
# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker
View Git Blame Copy Permalink