forked from HPR/hpr-tools
Compare commits
14 Commits
Author | SHA1 | Date | |
---|---|---|---|
|
2000398ad8 | ||
|
a83e945c08 | ||
|
ee4a174233 | ||
|
0f1e727487 | ||
6621e67703 | |||
7fe9f60205 | |||
5cfdd42b11 | |||
e1df438111 | |||
5feaed5f46 | |||
|
4feae03fee | ||
|
a3c8586730 | ||
|
2f350dd1db | ||
|
e0c4545295 | ||
|
37567bfd16 |
21
Community_News/.make_email.cfg
Normal file
21
Community_News/.make_email.cfg
Normal file
@ -0,0 +1,21 @@
|
||||
<email>
|
||||
server = chatter.skyehaven.net
|
||||
port = 64738
|
||||
room = HPR
|
||||
# Default day of the week
|
||||
dayname = Friday
|
||||
# Times are UTC
|
||||
starttime = 15:00:00
|
||||
endtime = 17:00:00
|
||||
# Usually 2 hours are set aside. This value is used when a different start
|
||||
# time is provided and no end time.
|
||||
duration = 02 # hours
|
||||
# Template is in the current directory
|
||||
template = make_email_template.tpl
|
||||
</email>
|
||||
<database>
|
||||
name = hpr.db
|
||||
</database>
|
||||
<recdates>
|
||||
name = recording_dates.dat
|
||||
</recdates>
|
45
Community_News/.make_shownotes.cfg
Normal file
45
Community_News/.make_shownotes.cfg
Normal file
@ -0,0 +1,45 @@
|
||||
#
|
||||
# .make_shownotes.cfg (2025-03-27)
|
||||
# Configuration file for make_shownotes version >= 4
|
||||
#
|
||||
<settings>
|
||||
title_template = HPR Community News for %s %s
|
||||
summary_template = HPR Volunteers talk about shows released and comments posted in %s %s
|
||||
# Repeat the following line with each of the desired tags to make an
|
||||
# array-like structure
|
||||
tags = Community News
|
||||
|
||||
# Host id for HPR Volunteers
|
||||
# Series id for HPR Community News series
|
||||
hostid = 159
|
||||
series_id = 47
|
||||
|
||||
# Day the Community News show is released
|
||||
releaseday = Monday
|
||||
|
||||
# Default day of the week for the recording
|
||||
recordingday = Friday
|
||||
|
||||
# Recording times are UTC
|
||||
starttime = 16:00:00
|
||||
endtime = 17:00:00
|
||||
|
||||
# cache of previous recording dates and times
|
||||
cache = recording_dates.dat
|
||||
|
||||
# Templates
|
||||
# ---------
|
||||
|
||||
# Main note template (actually a soft link)
|
||||
main_template = shownote_template.tpl
|
||||
|
||||
# Used to make a stand-alone HTML file from the default HTML
|
||||
# fragment
|
||||
container_template = shownotes_container.tpl
|
||||
|
||||
</settings>
|
||||
|
||||
<database>
|
||||
# Assume a local file
|
||||
name = hpr.db
|
||||
</database>
|
@ -241,7 +241,8 @@ AOBMKD="$BASEDIR/aob_$startdate.mkd"
|
||||
# --define "table=$TMP1" "$AOBMKD" |\
|
||||
# pandoc -f markdown-smart -t html5 -o "${AOBMKD%mkd}html"; then
|
||||
#
|
||||
if pandoc -f markdown-smart -t html5 "$AOBMKD" -o "${AOBMKD%mkd}html"; then
|
||||
if tpage "$AOBMKD" |\
|
||||
pandoc -f markdown-smart -t html5 -o "${AOBMKD%mkd}html"; then
|
||||
echo "Converted $AOBMKD to HTML"
|
||||
else
|
||||
echo "Conversion of $AOBMKD to HTML failed"
|
||||
|
122
Community_News/collect_HPR_database
Executable file
122
Community_News/collect_HPR_database
Executable file
@ -0,0 +1,122 @@
|
||||
#!/bin/bash -
|
||||
#===============================================================================
|
||||
#
|
||||
# FILE: collect_HPR_database
|
||||
#
|
||||
# USAGE: ./collect_HPR_database
|
||||
#
|
||||
# DESCRIPTION: Collects the SQL dump of the public copy of the HPR database
|
||||
# from the website and converts it to SQLite.
|
||||
#
|
||||
# OPTIONS: None
|
||||
# REQUIREMENTS: ---
|
||||
# BUGS: ---
|
||||
# NOTES: There are dependencies on mysql2sqlite and sqlite3. The former
|
||||
# comes from https://github.com/mysql2sqlite and is expected to
|
||||
# be in the same directory as this script. The sqlite3 package
|
||||
# needs to be installed from the repository appropriate to the
|
||||
# local OS. It is assumed that wget is available. The script
|
||||
# uses auto-deleted temporary files for the MySQL dump, and the
|
||||
# dump converted for SQLite.
|
||||
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
||||
# VERSION: 0.0.1
|
||||
# CREATED: 2025-02-22 16:52:40
|
||||
# REVISION: 2025-02-22 17:56:00
|
||||
#
|
||||
#===============================================================================
|
||||
|
||||
set -o nounset # Treat unset variables as an error
|
||||
|
||||
SCRIPT=${0##*/}
|
||||
BASEDIR=${0%/*}
|
||||
|
||||
VERSION="0.0.1"
|
||||
|
||||
# {{{ -- Functions: cleanup_temp
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: cleanup_temp
|
||||
# DESCRIPTION: Cleanup temporary files in case of a keyboard interrupt
|
||||
# (SIGINT) or a termination signal (SIGTERM) and at script
|
||||
# exit. Expects to be called from 'trap' so it can just exit
|
||||
# (assuming it's all that's called)
|
||||
# PARAMETERS: * - names of temporary files to delete
|
||||
# RETURNS: Nothing
|
||||
#===============================================================================
|
||||
function cleanup_temp {
|
||||
for tmp; do
|
||||
[ -e "$tmp" ] && rm --force "$tmp"
|
||||
done
|
||||
exit 0
|
||||
}
|
||||
# }}}
|
||||
|
||||
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
#
|
||||
# Make sure we're where the script lives
|
||||
#
|
||||
cd "$BASEDIR" || { echo "$SCRIPT: Failed to cd to $BASEDIR"; exit 1; }
|
||||
|
||||
#
|
||||
# Make temporary files and set traps to delete them
|
||||
#
|
||||
TMP1=$(mktemp) || {
|
||||
echo "$SCRIPT: creation of temporary file failed!"
|
||||
exit 1
|
||||
}
|
||||
TMP2=$(mktemp) || {
|
||||
echo "$SCRIPT: creation of temporary file failed!"
|
||||
exit 1
|
||||
}
|
||||
trap 'cleanup_temp $TMP1 $TMP2' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
|
||||
|
||||
#
|
||||
# Definition of files
|
||||
#
|
||||
#-------------------------------------------------------------------------------
|
||||
mysql2sqlite="$BASEDIR/mysql2sqlite"
|
||||
snapshot_url="https://www.hackerpublicradio.org/hpr.sql"
|
||||
db_name="hpr.db"
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
#
|
||||
# Sanity check
|
||||
#
|
||||
[ -e "$mysql2sqlite" ] || {
|
||||
echo "$SCRIPT: Unable to find mandatory script $mysql2sqlite"
|
||||
exit 1
|
||||
}
|
||||
|
||||
#
|
||||
# Collect the SQL dump into a temporary file`
|
||||
#
|
||||
if ! wget -q "$snapshot_url" -O "$TMP1"; then
|
||||
echo "$SCRIPT: Failed to download from $snapshot_url"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
#
|
||||
# Delete the SQLite database if it exists (otherwise the new data is merged
|
||||
# with it causing chaos)
|
||||
#
|
||||
[ -e "$db_name" ] && rm -f "$db_name"
|
||||
|
||||
#
|
||||
# Convert the MySQL/MariaDB dump. First run sed on it, then run mysql2sqlite
|
||||
# (from https://github.com/mysql2sqlite) to do the SQL dump conversion. Use
|
||||
# the result to make a SQLite database.
|
||||
#
|
||||
sed '/^DELIMITER ;;/,/^DELIMITER ;/d' "$TMP1" > "$TMP2"
|
||||
$mysql2sqlite "$TMP2" | sqlite3 "$db_name"
|
||||
|
||||
#
|
||||
# Report success if the new database exists
|
||||
#
|
||||
if [[ -e "$db_name" ]]; then
|
||||
echo "Created SQLite database '$db_name'"
|
||||
else
|
||||
echo "Failed to find the expected new database '$db_name'"
|
||||
fi
|
||||
|
||||
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker
|
||||
|
@ -3,56 +3,60 @@
|
||||
#
|
||||
# FILE: make_email
|
||||
#
|
||||
# USAGE: ./make_email [-debug=N] [-month=DATE] [-from=ADDRESS]
|
||||
# [-to=ADDRESS] [-[no]mail] [-date=DATE] [-start=START_TIME]
|
||||
# [-end=END_TIME] [-config=FILE] [-dbconfig=FILE]
|
||||
# USAGE: ./make_email [-debug=N] [-month=DATE] [-date=DATE]
|
||||
# [-start=START_TIME] [-end=END_TIME] [-output[=FILE]]
|
||||
# [-config=FILE]
|
||||
#
|
||||
# DESCRIPTION: Make and send an invitation email for the next Community News
|
||||
# DESCRIPTION: Make an invitation email for the next Community News
|
||||
# with times per timezone.
|
||||
#
|
||||
# The configuration file (.make_email.cfg) defines the name of
|
||||
# the email template and the defaults used when generating the
|
||||
# message. The date of the recording is computed from the
|
||||
# current month (Saturday before the first Monday of the month
|
||||
# current month (Friday before the first Monday of the month
|
||||
# when the show will be posted). It can also be specified
|
||||
# through the -date=DATE option.
|
||||
#
|
||||
# The month the email relates to can be changed through the
|
||||
# -month=DATA option, though this is rarely used. Use a date of
|
||||
# -month=DATE option, though this is rarely used. Use a date of
|
||||
# the format 'YYYY-MM-DD' here. The day is ignored but the year
|
||||
# and month are used in the computation. The month specified
|
||||
# must be in the future.
|
||||
#
|
||||
# The database configuration file defines the database to be
|
||||
# used to compute the date and show number. Use .hpr_db.cfg for
|
||||
# the local MariaDB copy (for testing) and .hpr_livedb.cfg for
|
||||
# the live database (over the ssh tunnel, which must have been
|
||||
# opened already).
|
||||
#
|
||||
# OPTIONS: ---
|
||||
# REQUIREMENTS: ---
|
||||
# BUGS: ---
|
||||
#
|
||||
# NOTES: Does not send the email at present. Needs work
|
||||
# 2022-02-28: DBD::MariaDB has vanished, had to revert to MySQL
|
||||
# again
|
||||
# again.
|
||||
# 2025-02-22: Moved to SQLite for the database
|
||||
# 2025-02-23: Dropped Mail::Mailer and all related code. The
|
||||
# output is now written to STDOUT or an output file.
|
||||
#
|
||||
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
||||
# VERSION: 0.2.7
|
||||
# VERSION: 0.3.3
|
||||
# CREATED: 2013-10-28 20:35:22
|
||||
# REVISION: 2024-05-24 18:53:17
|
||||
# REVISION: 2025-02-28 14:40:28
|
||||
#
|
||||
#===============================================================================
|
||||
|
||||
use 5.010;
|
||||
use strict;
|
||||
use warnings;
|
||||
use v5.36;
|
||||
use utf8;
|
||||
use feature qw{ try };
|
||||
no warnings qw{ experimental::try };
|
||||
|
||||
use open ':std', ':encoding(UTF-8)'; # Make all IO UTF-8
|
||||
|
||||
use Cwd qw( abs_path );
|
||||
|
||||
use Getopt::Long;
|
||||
use Pod::Usage;
|
||||
|
||||
use Config::General;
|
||||
|
||||
use File::Copy;
|
||||
|
||||
use Date::Parse;
|
||||
|
||||
use DateTime;
|
||||
@ -63,8 +67,6 @@ use Date::Calc qw{:all};
|
||||
|
||||
use Template;
|
||||
|
||||
use Mail::Mailer;
|
||||
|
||||
use DBI;
|
||||
|
||||
use Data::Dumper;
|
||||
@ -72,7 +74,7 @@ use Data::Dumper;
|
||||
#
|
||||
# Version number (manually incremented)
|
||||
#
|
||||
our $VERSION = '0.2.7';
|
||||
our $VERSION = '0.3.3';
|
||||
|
||||
#
|
||||
# Script name
|
||||
@ -85,11 +87,16 @@ our $VERSION = '0.2.7';
|
||||
#
|
||||
# Constants and other declarations
|
||||
#
|
||||
my $basedir = "$ENV{HOME}/HPR/Community_News";
|
||||
my $configfile1 = "$basedir/.${PROG}.cfg";
|
||||
my $configfile2 = "$basedir/.hpr_db.cfg";
|
||||
( my $basedir = abs_path($0) ) =~ s|/?[^/]*$||mx;
|
||||
my $configfile = "$basedir/.${PROG}.cfg";
|
||||
|
||||
my ( $dbh, $sth1, $sth2, $sth3, $h1, $h2, $rv );
|
||||
my ( $dbh, $sth1, $h1, $h2, $rv );
|
||||
my ( %recdates, $rdfh );
|
||||
|
||||
#
|
||||
# Run in the script's directory
|
||||
#
|
||||
chdir($basedir);
|
||||
|
||||
#
|
||||
# The timezones we want to report. These were generated with
|
||||
@ -495,18 +502,10 @@ my @zones = (
|
||||
#}}}
|
||||
);
|
||||
|
||||
#
|
||||
# Enable Unicode mode
|
||||
#
|
||||
binmode STDOUT, ":encoding(UTF-8)";
|
||||
binmode STDERR, ":encoding(UTF-8)";
|
||||
|
||||
#
|
||||
# Defaults for options
|
||||
#
|
||||
my $DEF_DEBUG = 0;
|
||||
my $DEF_FROM = 'Dave.Morriss@gmail.com';
|
||||
my $DEF_TO = 'perloid@autistici.org';
|
||||
|
||||
#
|
||||
# Options and arguments
|
||||
@ -521,43 +520,26 @@ pod2usage( -msg => "$PROG version $VERSION\n", -verbose => 0, -exitval => 1 )
|
||||
if ( $options{'help'} );
|
||||
|
||||
#
|
||||
# Full documentation if requested with -doc
|
||||
# Full documentation if requested with -doc[umentation] or -man
|
||||
#
|
||||
pod2usage(
|
||||
-msg => "$PROG version $VERSION\n",
|
||||
-verbose => 2,
|
||||
-exitval => 1,
|
||||
# -noperldoc => 0,
|
||||
) if ( $options{'documentation'} );
|
||||
|
||||
#
|
||||
# Collect options
|
||||
#
|
||||
my $DEBUG = ( defined( $options{debug} ) ? $options{debug} : $DEF_DEBUG );
|
||||
my $month = $options{month};
|
||||
my $mail = ( defined( $options{mail} ) ? $options{mail} : 0 );
|
||||
my $from_address = (
|
||||
defined( $options{fromaddress} ) ? $options{fromaddress} : $DEF_FROM );
|
||||
my $to_address
|
||||
= ( defined( $options{toaddress} ) ? $options{toaddress} : $DEF_TO );
|
||||
my $date = $options{date};
|
||||
my $start = $options{starttime};
|
||||
my $end = $options{endtime};
|
||||
|
||||
# This value is in the configuration file and can't be overridden. The planned
|
||||
# end time can be specified however.
|
||||
#my $duration = $options{duration};
|
||||
my $DEBUG = ( defined( $options{debug} ) ? $options{debug} : $DEF_DEBUG );
|
||||
my $month = $options{month};
|
||||
my $date = $options{date};
|
||||
my $start = $options{starttime};
|
||||
my $end = $options{endtime};
|
||||
my $outfile = $options{output};
|
||||
|
||||
my $cfgfile
|
||||
= ( defined( $options{config} ) ? $options{config} : $configfile1 );
|
||||
my $dbcfgfile
|
||||
= ( defined( $options{dbconfig} ) ? $options{dbconfig} : $configfile2 );
|
||||
|
||||
#
|
||||
# Use the 'testfile' mailer if option -nomail was chosen. This writes the file
|
||||
# 'mailer.testfile' and sends no message
|
||||
#
|
||||
my $mailertype = ( $mail ? 'sendmail' : 'testfile' );
|
||||
= ( defined( $options{config} ) ? $options{config} : $configfile );
|
||||
|
||||
#
|
||||
# Sanity checking the options
|
||||
@ -565,39 +547,26 @@ my $mailertype = ( $mail ? 'sendmail' : 'testfile' );
|
||||
die "Unable to find $cfgfile\n" unless ( -e $cfgfile );
|
||||
die "Use only one of -month=MONTH or -date=DATE\n"
|
||||
if (defined($month) && defined($date));
|
||||
#die "Use only one of -endtime=TIME or -duration=HOURS\n"
|
||||
# if (defined($end) && defined($duration));
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Load script and database configuration data
|
||||
# Load configuration data
|
||||
#-------------------------------------------------------------------------------
|
||||
my $conf = new Config::General(
|
||||
my $conf = Config::General->new(
|
||||
-ConfigFile => $cfgfile,
|
||||
-InterPolateVars => 1,
|
||||
-ExtendedAccess => 1
|
||||
);
|
||||
my %config = $conf->getall();
|
||||
#print Dumper( \%config ), "\n";
|
||||
_debug( $DEBUG >= 2, '%config: ' . Dumper( \%config ) );
|
||||
|
||||
#
|
||||
# Load database configuration data
|
||||
#
|
||||
my $dbconf = new Config::General(
|
||||
-ConfigFile => $dbcfgfile,
|
||||
-InterPolateVars => 1,
|
||||
-ExtendedAccess => 1
|
||||
);
|
||||
my %dbconfig = $dbconf->getall();
|
||||
#print Dumper( \%dbconfig ), "\n";
|
||||
|
||||
#
|
||||
# Configuration file values with defaults and/or checks
|
||||
# Configuration file values for the email text with defaults and/or checks
|
||||
#
|
||||
my $server = $config{email}->{server} // 'chatter.skyehaven.net';
|
||||
my $port = $config{email}->{port} // 64738;
|
||||
my $room = $config{email}->{room} // 'Hacker Public Radio';
|
||||
my $duration = $config{email}->{duration} // 2;
|
||||
my $dayname = $config{email}->{dayname} // 'Sunday';
|
||||
my $duration = $config{email}->{duration} // 2; # Hours
|
||||
my $dayname = $config{email}->{dayname} // 'Friday';
|
||||
|
||||
#
|
||||
# If we had a start time specified then check it and ensure the end time makes
|
||||
@ -637,9 +606,33 @@ my @starttime = split( ':', $start );
|
||||
my @endtime = split( ':', $end );
|
||||
die "Missing start/end time(s)\n" unless ( @starttime && @endtime );
|
||||
|
||||
#
|
||||
# The template from the configuration file
|
||||
#
|
||||
my $template = $config{email}->{template};
|
||||
die "Missing template file $template\n" unless (-e $template);
|
||||
|
||||
#
|
||||
# Recording date cache filename in the configuration file
|
||||
#
|
||||
my $recdatefile = $config{recdates}->{name};
|
||||
unless ($recdatefile) {
|
||||
warn "No recording date file defined in configuration";
|
||||
say STDERR "Continuing without this file";
|
||||
}
|
||||
elsif ( ! -e $recdatefile) {
|
||||
warn "Can't find recording date file $recdatefile";
|
||||
say STDERR "Continuing without this file";
|
||||
$recdatefile = undef;
|
||||
}
|
||||
|
||||
#
|
||||
# Load the recording dates
|
||||
#
|
||||
if ($recdatefile) {
|
||||
%recdates = load_cache($recdatefile, $rdfh);
|
||||
}
|
||||
|
||||
_debug($DEBUG >= 2,
|
||||
'$start: ' . coalesce($start,''),
|
||||
'$end: ' . coalesce($end,''),
|
||||
@ -655,24 +648,17 @@ if ($DEBUG >= 1) {
|
||||
# 2021-12-24: moved to MariaDB
|
||||
# 2022-02-28: the MariaDB driver has gone away apparently. Reverted to MySQL
|
||||
# again
|
||||
# 2025-02-22: Converted to SQLite
|
||||
#-------------------------------------------------------------------------------
|
||||
my $dbhost = $dbconfig{database}->{host} // '127.0.0.1';
|
||||
my $dbport = $dbconfig{database}->{port} // 3306;
|
||||
my $dbname = $dbconfig{database}->{name};
|
||||
my $dbuser = $dbconfig{database}->{user};
|
||||
my $dbpwd = $dbconfig{database}->{password};
|
||||
#$dbh = DBI->connect( "DBI:MariaDB:database=$dbname;host=$dbhost;port=$dbport",
|
||||
# $dbuser, $dbpwd, { AutoCommit => 1 } )
|
||||
# or croak $DBI::errstr;
|
||||
my $dbname = $config{database}->{name};
|
||||
|
||||
$dbh = DBI->connect( "dbi:mysql:host=$dbhost;port=$dbport;database=$dbname",
|
||||
$dbuser, $dbpwd, { AutoCommit => 1 } )
|
||||
or die $DBI::errstr;
|
||||
$dbh = DBI->connect( "DBI:SQLite:dbname=$dbname",
|
||||
"", "", { AutoCommit => 1 } );
|
||||
|
||||
#
|
||||
# Enable client-side UTF8
|
||||
#
|
||||
$dbh->{mysql_enable_utf8} = 1;
|
||||
$dbh->{sqlite_unicode} = 1;
|
||||
|
||||
#
|
||||
# Date and time values using Date::Calc format
|
||||
@ -681,7 +667,7 @@ my @today = Today();
|
||||
my @startdate;
|
||||
my @startmonth;
|
||||
my @reviewdate;
|
||||
my $monday = 1; # Day of week number 1-7, Monday-Sunday
|
||||
my $monday = 1; # Day of week number 1-7, Monday-Sunday
|
||||
my $offset = day_offset($dayname)->{offset};
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
@ -689,7 +675,7 @@ my $offset = day_offset($dayname)->{offset};
|
||||
# or the current date.
|
||||
#-------------------------------------------------------------------------------
|
||||
#
|
||||
# If there's an argument then it'll be an override for the start date
|
||||
# If there's a -date=DATE option then it'll be an override for the start date
|
||||
# otherwise we'll compute it.
|
||||
#
|
||||
if ( defined($date) ) {
|
||||
@ -726,7 +712,8 @@ elsif ( defined($month) ) {
|
||||
|
||||
#
|
||||
# Compute the next meeting date from now (by finding the next first Monday
|
||||
# of the month then backing up two days to the Saturday).
|
||||
# of the month then backing up two days to the desired day - default
|
||||
# Friday).
|
||||
#
|
||||
@startdate = make_date( \@startmonth, $monday, 1, $offset );
|
||||
}
|
||||
@ -745,9 +732,11 @@ _debug($DEBUG >= 2, '@startdate: ' . join(',',@startdate));
|
||||
# before.
|
||||
#
|
||||
if ( $startdate[1] eq $today[1] ) {
|
||||
# Same month
|
||||
@reviewdate = @startdate;
|
||||
}
|
||||
else {
|
||||
# Previous month - backup 1 month
|
||||
@reviewdate = Add_Delta_YM( @startdate, 0, -1 );
|
||||
}
|
||||
|
||||
@ -755,7 +744,8 @@ _debug($DEBUG >= 2, '@reviewdate: ' . join(',',@reviewdate));
|
||||
|
||||
#
|
||||
# Transfer Date::Calc values into hashes for initialising DateTime objects so
|
||||
# we can play time zone games
|
||||
# we can play time zone games. (Note: %dtargs is a hash and we're using hash
|
||||
# slicing to initialise it).
|
||||
#
|
||||
my ( %dtargs, $dtstart, $dtend );
|
||||
@dtargs{ 'year', 'month', 'day', 'hour', 'minute', 'second', 'time_zone' }
|
||||
@ -775,14 +765,15 @@ my $days = $dtf->format_duration($dtoffset);
|
||||
#
|
||||
# Formatted dates for the mail message body
|
||||
#
|
||||
my ( $year, $monthname, $nicedate, $starttime, $endtime ) = (
|
||||
$dtstart->strftime("%Y"), Month_to_Text( $reviewdate[1] ),
|
||||
$dtstart->strftime("%A, %B %d %Y"), $dtstart->strftime("%R (%Z)"),
|
||||
$dtend->strftime("%R (%Z)"),
|
||||
my ( $year, $monthno, $monthname, $nicedate, $starttime, $endtime ) = (
|
||||
$dtstart->strftime("%Y"), $dtstart->strftime("%m"),
|
||||
Month_to_Text( $reviewdate[1] ), $dtstart->strftime("%A, %B %d %Y"),
|
||||
$dtstart->strftime("%R (%Z)"), $dtend->strftime("%R (%Z)"),
|
||||
);
|
||||
|
||||
_debug($DEBUG >= 2,
|
||||
"\$year: $year",
|
||||
"\$monthno: $monthno",
|
||||
"\$monthname: $monthname",
|
||||
"\$nicedate: $nicedate",
|
||||
"\$starttime: $starttime",
|
||||
@ -800,38 +791,50 @@ my $subject = $dtstart->strftime(
|
||||
|
||||
_debug( $DEBUG >= 2, "\$subject: $subject" );
|
||||
|
||||
#
|
||||
# Prepare to send mail
|
||||
#
|
||||
my $mailer = Mail::Mailer->new($mailertype);
|
||||
#-------------------------------------------------------------------------------
|
||||
# Open the output file (or STDOUT) - we may need the year and month number to
|
||||
# do it, if the file name contains '%s'.
|
||||
#-------------------------------------------------------------------------------
|
||||
my $outfh;
|
||||
if ($outfile) {
|
||||
$outfile = sprintf( $outfile, sprintf( "%d-%02d", $year, $monthno ) )
|
||||
if ( $outfile =~ /%s/ );
|
||||
|
||||
#
|
||||
# Generate the headers we need
|
||||
#
|
||||
$mailer->open(
|
||||
{ To => $to_address,
|
||||
From => $from_address,
|
||||
Subject => $subject,
|
||||
}
|
||||
);
|
||||
open( $outfh, ">:encoding(UTF-8)", $outfile )
|
||||
or die "Unable to open $outfile for writing: $!\n";
|
||||
}
|
||||
else {
|
||||
open( $outfh, ">&", \*STDOUT )
|
||||
or die "Unable to initialise for writing: $!\n";
|
||||
}
|
||||
|
||||
#
|
||||
# Build an array of timezone data for the template
|
||||
#
|
||||
my @timezones;
|
||||
for my $tz (@zones) {
|
||||
push( @timezones, storeTZ( $dtstart, $dtend, $tz ) );
|
||||
push( @timezones, storeTZ( $dtstart, $dtend, $tz ) );
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Find the number of the show with the notes. Take care because the recording
|
||||
# date might not be on the weekend before the show is released.
|
||||
# TODO: If this search fails (because in future Community News shows will not
|
||||
# be reserved), then the date needs to be computed.
|
||||
#-------------------------------------------------------------------------------
|
||||
my $isodate = $dtstart->ymd;
|
||||
#$sth1 = $dbh->prepare(q{
|
||||
# SELECT id FROM eps
|
||||
# WHERE date > ?
|
||||
# AND date_format(date,"%W") = 'Monday'
|
||||
# AND title LIKE 'HPR Community News%'
|
||||
# ORDER BY date
|
||||
# LIMIT 1
|
||||
#});
|
||||
$sth1 = $dbh->prepare(q{
|
||||
SELECT id FROM eps
|
||||
WHERE date > ?
|
||||
AND date_format(date,"%W") = 'Monday'
|
||||
AND strftime("%u", date) = '1'
|
||||
AND title LIKE 'HPR Community News%'
|
||||
ORDER BY date
|
||||
LIMIT 1
|
||||
@ -848,13 +851,38 @@ unless ( $h1 = $sth1->fetchrow_hashref ) {
|
||||
exit 1;
|
||||
}
|
||||
|
||||
my $shownotes = $h1->{id};
|
||||
my $episode = $h1->{id};
|
||||
|
||||
_debug( $DEBUG >= 2, "\$shownotes (slot): $shownotes" );
|
||||
_debug( $DEBUG >= 2, "\$episode (slot): $episode" );
|
||||
|
||||
$sth1->finish;
|
||||
$dbh->disconnect;
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Update the date cache now we have the date and time details we need.
|
||||
#-------------------------------------------------------------------------------
|
||||
( my $monthkey = $dtstart->ymd ) =~ s/\d+$/01/;
|
||||
my $datestamp = $dtstart->strftime("%F %T");
|
||||
|
||||
if (exists($recdates{$monthkey})) {
|
||||
#
|
||||
# It exists. Is it different?
|
||||
#
|
||||
unless ( $recdates{$monthkey} eq $datestamp ) {
|
||||
#
|
||||
# Save the new data (assuming it's correct)
|
||||
#
|
||||
$recdates{$monthkey} = $datestamp;
|
||||
update_cache( $recdatefile, \%recdates );
|
||||
}
|
||||
}
|
||||
else {
|
||||
#
|
||||
# Add a new record to the end of the cache file
|
||||
#
|
||||
append_cache( $recdatefile, sprintf( "%s,%s", $monthkey, $datestamp ) );
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Fill the template
|
||||
#-------------------------------------------------------------------------------
|
||||
@ -865,12 +893,10 @@ my $tt = Template->new(
|
||||
);
|
||||
|
||||
my $vars = {
|
||||
# subject => $subject,
|
||||
# from => $from_address,
|
||||
# to => $to_address,
|
||||
server => $server,
|
||||
port => $port,
|
||||
room => $room,
|
||||
subject => $subject,
|
||||
timezones => \@timezones,
|
||||
utc => {
|
||||
days => $days,
|
||||
@ -880,7 +906,7 @@ my $vars = {
|
||||
start => $starttime,
|
||||
end => $endtime,
|
||||
},
|
||||
shownotes => $shownotes,
|
||||
episode => $episode, # show number
|
||||
};
|
||||
|
||||
my $document;
|
||||
@ -889,23 +915,136 @@ $tt->process( $template,
|
||||
|| die $tt->error(), "\n";
|
||||
|
||||
#
|
||||
# Add the template-generated body to the mail message
|
||||
# Write to the output file
|
||||
#
|
||||
print $mailer $document;
|
||||
print $outfh $document;
|
||||
|
||||
#
|
||||
# Send the message
|
||||
# Report the output file name if there is one
|
||||
#
|
||||
$mailer->close
|
||||
or die "Couldn't send message: $!\n";
|
||||
|
||||
unless ($mail) {
|
||||
print "Message was not sent since -nomail was selected (or defaulted).\n";
|
||||
print "Look in 'mailer.testfile' for the output\n";
|
||||
if ($outfile) {
|
||||
say "Output is in $outfile";
|
||||
}
|
||||
|
||||
exit;
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: load_cache
|
||||
# PURPOSE: Load the date cache into a hash
|
||||
# PARAMETERS: $cache_name Name of file holding the cache
|
||||
# RETURNS: Contents of cache as a hash
|
||||
# DESCRIPTION: Opens the nominated file, parses each record, and adds the
|
||||
# data to a hash. The record should contain the following:
|
||||
# * 'YYYY-MM-01' the month for which the details are being
|
||||
# recorded
|
||||
# * ',' comma field separator
|
||||
# * 'YYYY-MM-DD HH:MM:SS' timestamp of the recording
|
||||
# The file is closed once it has been scanned. The function
|
||||
# returns the completed hash.
|
||||
# THROWS: No exceptions
|
||||
# COMMENTS: None
|
||||
# SEE ALSO: N/A
|
||||
#===============================================================================
|
||||
sub load_cache {
|
||||
my ($cache_name) = @_;
|
||||
|
||||
my ( $month, $datetime, %result );
|
||||
|
||||
#
|
||||
# Open the file in read mode
|
||||
#
|
||||
open( my $dcfh, '<', $cache_name )
|
||||
or die "$PROG: failed to open '$cache_name': $!\n";
|
||||
|
||||
while ( my $line = <$dcfh> ) {
|
||||
chomp($line);
|
||||
if ( ( $month, $datetime )
|
||||
= ( $line =~ /^(\d{4}-\d{2}-\d{2}),(.*)$/ ) )
|
||||
{
|
||||
$result{$month} = $datetime;
|
||||
}
|
||||
# TODO: Report any errors found in the file
|
||||
}
|
||||
|
||||
close($dcfh)
|
||||
or warn "$PROG: failed to close '$cache_name': $!\n";
|
||||
|
||||
return %result;
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: append_cache
|
||||
# PURPOSE: Append a new line to the cache
|
||||
# PARAMETERS: $cache_name Name of file holding the cache
|
||||
# $line New record to add
|
||||
# RETURNS: Nothing
|
||||
# DESCRIPTION: Opens the nominated file and appends the new record in $line.
|
||||
# The file is closed once it has been updated.
|
||||
# THROWS: No exceptions
|
||||
# COMMENTS: None
|
||||
# SEE ALSO: N/A
|
||||
#===============================================================================
|
||||
sub append_cache {
|
||||
my ( $cache_name, $line ) = @_;
|
||||
|
||||
#
|
||||
# Open the file in append mode
|
||||
#
|
||||
open( my $dcfh, '>>', $cache_name )
|
||||
or die "$PROG: failed to open '$cache_name': $!\n";
|
||||
|
||||
say $dcfh $line;
|
||||
|
||||
close($dcfh)
|
||||
or warn "$PROG: failed to close '$cache_name': $!\n";
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: update_cache
|
||||
# PURPOSE: Make changes to an existing line in the cache
|
||||
# PARAMETERS: $cache_name Name of file holding the cache
|
||||
# $rhash Hashref holding the updated cache contents
|
||||
# RETURNS: Nothing
|
||||
# DESCRIPTION: Makes a backup of the nominated file. Opens, truncates it and
|
||||
# positions for writing (using 'seek'). The now empty file is
|
||||
# filled with data from the hash and closed.
|
||||
# THROWS: No exceptions
|
||||
# COMMENTS: None
|
||||
# SEE ALSO: N/A
|
||||
#===============================================================================
|
||||
sub update_cache {
|
||||
my ( $cache_name, $rhash ) = @_;
|
||||
|
||||
#
|
||||
# Copy the cache file to a backup
|
||||
#
|
||||
copy($cache_name,"${cache_name}~")
|
||||
or die "Unable to back up $cache_name\n";
|
||||
|
||||
#
|
||||
# Open the original file in write mode
|
||||
#
|
||||
open( my $dcfh, '>', $cache_name )
|
||||
or die "$PROG: failed to open '$cache_name': $!\n";
|
||||
|
||||
#
|
||||
# Truncate the file and seek to the start again
|
||||
#
|
||||
truncate($dcfh,0)
|
||||
or die "$PROG: failed to truncate '$cache_name': $!\n";
|
||||
seek($dcfh,0,0)
|
||||
or die "$PROG: failed to seek in '$cache_name': $!\n";
|
||||
|
||||
#
|
||||
# Write the cache data to the file
|
||||
#
|
||||
for my $key (sort(keys(%$rhash))) {
|
||||
say $dcfh sprintf("%s,%s",$key, $rhash->{$key});
|
||||
}
|
||||
|
||||
close($dcfh)
|
||||
or warn "$PROG: failed to close '$cache_name': $!\n";
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: report_settings
|
||||
@ -921,36 +1060,18 @@ sub report_settings {
|
||||
my $fmt = "D> %-14s = %s\n";
|
||||
print "D> Settings from options or default values:\n";
|
||||
printf $fmt, "Month", coalesce($month,'undef');
|
||||
printf $fmt, "Mail", coalesce($mail,'undef');
|
||||
printf $fmt, "From", coalesce($from_address,'undef');
|
||||
printf $fmt, "To", coalesce($to_address,'undef');
|
||||
printf $fmt, "Meeting date", coalesce($date,'undef');
|
||||
printf $fmt, "Start time", join(':',@starttime);
|
||||
printf $fmt, "End time", join(':',@endtime);
|
||||
printf $fmt, "Config file", coalesce($cfgfile,'undef');
|
||||
printf $fmt, "DB config file", coalesce($dbcfgfile,'undef');
|
||||
printf $fmt, "Server", coalesce($server,'undef');
|
||||
printf $fmt, "Port", coalesce($port,'undef');
|
||||
printf $fmt, "Room", coalesce($room,'undef');
|
||||
printf $fmt, "Template", coalesce($template,'undef');
|
||||
printf $fmt, "Recording date file", coalesce($recdatefile,'undef');
|
||||
print "D> ----\n";
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: compute_endtime
|
||||
# PURPOSE: Given a start time and a duration computes the end time
|
||||
# PARAMETERS: $rdate arrayref for the date
|
||||
# $rstime arrayref for the start time
|
||||
# $rduration arrayref for the duration [HH,MM,SS]
|
||||
# RETURNS: The end time as a string (HH:MM:SS)
|
||||
# DESCRIPTION:
|
||||
# THROWS: No exceptions
|
||||
# COMMENTS: Decided not to implement this, may do so in future.
|
||||
# SEE ALSO: N/A
|
||||
#===============================================================================
|
||||
#sub compute_endtime {
|
||||
#}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: validate_time
|
||||
# PURPOSE: Validates a time in HH:MM:SS format
|
||||
@ -1029,7 +1150,7 @@ sub make_date {
|
||||
}
|
||||
|
||||
#
|
||||
# Apply the day offset
|
||||
# Apply any day offset
|
||||
#
|
||||
@date = Add_Delta_Days( @date, $offset ) if $offset;
|
||||
|
||||
@ -1047,7 +1168,8 @@ sub make_date {
|
||||
# $end DateTime object containing the ending datetime
|
||||
# as UTC
|
||||
# $tz The textual time zone (need this to be valid)
|
||||
# RETURNS: A hash containing the timezone name and start and end times
|
||||
# RETURNS: Reference to a hash containing the timezone name and start and
|
||||
# end times
|
||||
# DESCRIPTION: Relies on DateTime::TimeZone to do the work to turn a UTC time
|
||||
# into a time in a different time zone. Uses the DateTime
|
||||
# strftime method to format the dates and times for printing.
|
||||
@ -1076,49 +1198,15 @@ sub storeTZ {
|
||||
return \%result;
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: printTZ
|
||||
# PURPOSE: Print start/end times for a timezone
|
||||
# PARAMETERS: $fh File handle for writing
|
||||
# $start DateTime object containing the starting
|
||||
# datetime as UTC
|
||||
# $end DateTime object containing the ending datetime
|
||||
# as UTC
|
||||
# $tz The textual time zone (need this to be valid)
|
||||
# RETURNS: Nothing
|
||||
# DESCRIPTION: Relies on DateTime::TimeZone to do the work to turn a UTC time
|
||||
# into a time in a different time zone. Uses the DateTime
|
||||
# strftime method to format the dates and times for printing.
|
||||
# THROWS: No exceptions
|
||||
# COMMENTS: None
|
||||
# SEE ALSO:
|
||||
#===============================================================================
|
||||
sub printTZ {
|
||||
my ( $fh, $start, $end, $tz ) = @_;
|
||||
|
||||
#
|
||||
# Adjust time zone
|
||||
#
|
||||
$start->set_time_zone($tz);
|
||||
$end->set_time_zone($tz);
|
||||
|
||||
#
|
||||
# Print time zone and start/end times in that zone
|
||||
#
|
||||
print $fh "$tz\n";
|
||||
print $fh $start->strftime("Start: %H:%S %a, %b %d %Y\n");
|
||||
print $fh $end->strftime("End: %H:%S %a, %b %d %Y\n\n");
|
||||
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: day_offset
|
||||
# PURPOSE: Given a day name computes day attributes including the
|
||||
# (negative) offset in days from the target Monday to the
|
||||
# recording date.
|
||||
# (negative) offset in days from the target release day (Monday)
|
||||
# to the recording date.
|
||||
# PARAMETERS: $dayname Name of a day of the week
|
||||
# RETURNS: Hashref containing the full day name, the weekday number and
|
||||
# the integer offset from Monday to the recording day, or undef.
|
||||
# RETURNS: Hashref containing the full day name (dayname), the weekday
|
||||
# number (wday) and the integer offset (offset) from Monday to
|
||||
# the recording day, or undef.
|
||||
# DESCRIPTION: Uses the hash '%matches' keyed by regular expressions matching
|
||||
# day names. The argument '$dayname' is matched against each
|
||||
# regex in turn and if it matches the sub-hash is returned. This
|
||||
@ -1239,14 +1327,13 @@ sub Options {
|
||||
my ($optref) = @_;
|
||||
|
||||
my @options = (
|
||||
"help", "documentation|man",
|
||||
"debug=i", "mail!",
|
||||
"fromaddress=s", "toaddress=s",
|
||||
"date=s", "starttime=s",
|
||||
"endtime=s", "month=s",
|
||||
"config=s", "dbconfig=s",
|
||||
"help", "documentation|man",
|
||||
"debug=i", "date=s",
|
||||
"starttime=s", "endtime=s",
|
||||
"month=s", "output:s",
|
||||
"config=s",
|
||||
);
|
||||
# "duration=s",
|
||||
# "mail!", "fromaddress=s", "toaddress=s", "duration=s",
|
||||
|
||||
if ( !GetOptions( $optref, @options ) ) {
|
||||
pod2usage( -msg => "Version $VERSION\n", -verbose => 0, -exitval => 1 );
|
||||
@ -1264,20 +1351,20 @@ __END__
|
||||
|
||||
=head1 NAME
|
||||
|
||||
make_email - generates an HPR Community News recording invitation email
|
||||
make_email - generates the text of an HPR Community News recording invitation
|
||||
email
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
This documentation refers to make_email version 0.2.7
|
||||
This documentation refers to make_email version 0.3.3
|
||||
|
||||
|
||||
=head1 USAGE
|
||||
|
||||
make_email [-help] [-documentation] [-debug=N] [-month=DATE] [-[no]mail]
|
||||
[-from=FROM_ADDRESS] [-to=TO_ADDRESS] [-date=DATE] [-start=START_TIME]
|
||||
[-end=END_TIME] [-config=FILE] [-dbconfig=FILE]
|
||||
make_email [-help] [-documentation] [-debug=N] [-month=DATE] [-date=DATE]
|
||||
[-start=START_TIME] [-end=END_TIME] [-config=FILE]
|
||||
|
||||
./make_email -dbconf=$HOME/HPR/.hpr_livedb.cfg -date=2022-12-27
|
||||
./make_email -date=2022-12-27
|
||||
|
||||
=head1 OPTIONS
|
||||
|
||||
@ -1293,7 +1380,7 @@ Prints the entire embedded documentation for the program, then exits.
|
||||
|
||||
Another way to see the full documentation use:
|
||||
|
||||
B<perldoc ./make_email>
|
||||
perldoc ./make_email
|
||||
|
||||
=item B<-debug=N>
|
||||
|
||||
@ -1351,30 +1438,6 @@ a ISO8601 date such as 2014-03-08 (meaning March 2014) or 1-Jan-2017 (meaning
|
||||
January 2017). Only the year and month parts are used but a valid day must be
|
||||
present.
|
||||
|
||||
=item B<-[no]mail>
|
||||
|
||||
** NOTE ** The sending of mail does not work at present, and B<-nomail> should
|
||||
always be used.
|
||||
|
||||
Causes mail to be sent (B<-mail>) or not sent (B<-nomail>). If the mail is
|
||||
sent then it is sent via the local MTA (in the assumption that there is one).
|
||||
If this option is omitted, the default is B<-nomail>, in which case the
|
||||
message is appended to the file B<mailer.testfile> in the current directory.
|
||||
|
||||
=item B<-from=FROM_ADDRESS>
|
||||
|
||||
** NOTE ** The sending of mail does not work at present.
|
||||
|
||||
This option defines the address from which the message is to be sent. This
|
||||
address is used in the message header; the message envelope will contain the
|
||||
I<real> sender.
|
||||
|
||||
=item B<-to=TO_ADDRESS>
|
||||
|
||||
** NOTE ** The sending of mail does not work at present.
|
||||
|
||||
This option defines the address to which the message is to be sent.
|
||||
|
||||
=item B<-date=DATE>
|
||||
|
||||
This is an option provides a non-default date for the recording. Normally the
|
||||
@ -1405,30 +1468,35 @@ The default end time is defined in the configuration file, but if it is
|
||||
necessary to change it temporarily, this option can be used to do it. The
|
||||
B<END_TIME> value must be a valid B<HH:MM> time specification.
|
||||
|
||||
=item B<-output=FILE>
|
||||
|
||||
This option defines an output file to receive the mail message text. If the option is
|
||||
omitted the notes are written to STDOUT, allowing them to be redirected if
|
||||
required.
|
||||
|
||||
The output file name may contain the characters 'B<%s>'. This denotes the point
|
||||
at which the year and month in the format B<YYYY-MM> are inserted. For example
|
||||
if the script is being run for February 2025 the option:
|
||||
|
||||
-out=HPR_email_%s.txt
|
||||
|
||||
will cause the generation of the file:
|
||||
|
||||
HPR_email_2025-02.txt
|
||||
|
||||
=item B<-config=FILE>
|
||||
|
||||
This option defines a configuration file other than the default
|
||||
B<.make_email.cfg>. The file must be formatted as described below in the
|
||||
section I<CONFIGURATION AND ENVIRONMENT>.
|
||||
|
||||
=item B<-dbconfig=FILE>
|
||||
|
||||
This option defines a database configuration file other than the default
|
||||
B<.hpr_db.cfg>. The file must be formatted as described below in the section
|
||||
I<CONFIGURATION AND ENVIRONMENT>.
|
||||
|
||||
The default file is configured to open a local copy of the HPR database. An
|
||||
alternative is B<.hpr_livedb.cfg> which assumes an SSH tunnel to the live
|
||||
database and attempts to connect to it. Use the script I<open_tunnel> to open
|
||||
the SSH tunnel.
|
||||
|
||||
=back
|
||||
|
||||
=head1 DESCRIPTION
|
||||
|
||||
Makes and sends(*) an invitation email for the next Community News with times per
|
||||
timezone. The message is structured by a Template Toolkit template, so its
|
||||
content can be adjusted without changing this script.
|
||||
Makes an invitation email for the next Community News with times per timezone.
|
||||
The message is structured by a Template Toolkit template, so its contents can
|
||||
be adjusted without changing this script.
|
||||
|
||||
In normal operation the script computes the date of the next recording using
|
||||
the algorithm "Saturday before the first Monday of the next month" starting
|
||||
@ -1446,9 +1514,6 @@ seen as the shows are visited and discussed.
|
||||
The email generated by the script is sent to the HPR mailing list, usually on
|
||||
the Monday prior to the weekend of the recording.
|
||||
|
||||
Notes:
|
||||
* Mail sending does not work at present.
|
||||
|
||||
=head1 DIAGNOSTICS
|
||||
|
||||
=over 8
|
||||
@ -1496,11 +1561,6 @@ The month specified in B<-month=DATE> is in the past.
|
||||
|
||||
The program can generate warning messages from the Template.
|
||||
|
||||
=item B<Couldn't send message: ...>
|
||||
|
||||
The email mesage has been constructed but could not be sent. See the error
|
||||
returned by the mail subsystem for more information.
|
||||
|
||||
=back
|
||||
|
||||
=head1 CONFIGURATION AND ENVIRONMENT
|
||||
@ -1524,31 +1584,64 @@ needs to contain the following data:
|
||||
|
||||
=head2 DATABASE CONFIGURATION
|
||||
|
||||
The program obtains the credentials it requires for connecting to the HPR
|
||||
database by loading them from a configuration file. The default file is called
|
||||
B<.hpr_db.cfg> and should contain the following data:
|
||||
The program also obtains the details it requires for connecting to a SQLite
|
||||
copy of the HPR database by loading them from the same configuration file in
|
||||
a separate section. The data is as follows:
|
||||
|
||||
<database>
|
||||
host = 127.0.0.1
|
||||
port = PORT
|
||||
name = DBNAME
|
||||
user = USER
|
||||
password = PASSWORD
|
||||
</database>
|
||||
|
||||
The file B<.hpr_livedb.cfg> should be available to allow access to the
|
||||
database over an SSH tunnel which has been previously opened.
|
||||
=head2 DATE CACHE
|
||||
|
||||
<recdates>
|
||||
name = recording_dates.dat
|
||||
</recdates>
|
||||
|
||||
The program will update a cache of recording dates and times per month. This
|
||||
is useful for the script B<make_shownotes> which needs to know about the
|
||||
Community News show recording time so it can determine how to display
|
||||
information about comments. See the details for this script.
|
||||
|
||||
The format of the lines in the file is:
|
||||
|
||||
MONTH,TIMESTAMP
|
||||
|
||||
Note the separating comma. The month is an ISO8601 date where the day part is
|
||||
always B<01>. The timestamp part is the date and time of the recording in the
|
||||
format:
|
||||
|
||||
YYYY-MM-DD HH:MM:SS
|
||||
|
||||
For example:
|
||||
|
||||
2024-12-01,2025-01-03 15:00:00
|
||||
2025-01-01,2025-01-31 15:00:00
|
||||
2025-02-01,2025-02-28 16:00:00
|
||||
|
||||
The dates and times are derived from the configuration file defaults (and those
|
||||
computed in the script), or the options given when running the script.
|
||||
|
||||
The contents of this cache are loaded into the B<make_email> script. If there
|
||||
is no record for the month being processed, one is appended to the file. If
|
||||
the details already exist they are updated unless they are the same as those
|
||||
stored.
|
||||
|
||||
A backup of the file is made if the data in a record is being updated.
|
||||
|
||||
=head1 DEPENDENCIES
|
||||
|
||||
Config::General
|
||||
Cwd
|
||||
DBI
|
||||
Data::Dumper
|
||||
Date::Calc
|
||||
Date::Parse
|
||||
DateTime
|
||||
DateTime::Format::Duration
|
||||
DateTime::TimeZone
|
||||
File::Copy
|
||||
Getopt::Long
|
||||
Mail::Mailer
|
||||
Pod::Usage
|
||||
Template
|
||||
|
||||
@ -1560,7 +1653,7 @@ Patches are welcome.
|
||||
|
||||
=head1 AUTHOR
|
||||
|
||||
Dave Morriss (Dave.Morriss@gmail.com) 2013 - 2024
|
||||
Dave Morriss (Dave.Morriss@gmail.com) 2013 - 2025
|
||||
|
||||
=head1 LICENCE AND COPYRIGHT
|
||||
|
||||
|
76
Community_News/make_email_template.tpl
Normal file
76
Community_News/make_email_template.tpl
Normal file
@ -0,0 +1,76 @@
|
||||
[%# make_email_template.tpl 2025-02-23 -%]
|
||||
[%# Community News email template -%]
|
||||
[% USE wrap -%]
|
||||
[% subject %]
|
||||
|
||||
[% FILTER replace('\n', ' ') -%]
|
||||
[% IF utc.days > 6 -%]
|
||||
The Community News for [% utc.month %] will be recorded using Mumble on
|
||||
[% ELSE -%]
|
||||
The next Community News will be recorded using Mumble on
|
||||
[% END -%]
|
||||
[% utc.date %] between [% utc.start %] and [% utc.end %] in the '[% room %]' room on [% server %] port [% port %].
|
||||
[% END %]
|
||||
|
||||
[% FILTER replace('\n', ' ') -%]
|
||||
During the recording HPR Volunteers will review the shows released during
|
||||
[% utc.month %] [% utc.year %], they will read comments submitted during that
|
||||
month, as well as summarising email sent to the HPR mailing list.
|
||||
[% END %]
|
||||
|
||||
[% FILTER replace('\n', ' ') -%]
|
||||
All HPR listeners are welcome to join in, but we ask that you listen to all
|
||||
the shows in [% utc.month %] before you do so.
|
||||
[% END %]
|
||||
|
||||
[% FILTER replace('\n', ' ') -%]
|
||||
Occasionally, due to local factors, we might need to change the time, or even
|
||||
the date, without warning. If you're planning on joining in it might be a good
|
||||
idea to let us know in advance - email admin@hackerpublicradio.org. Then we
|
||||
can contact you with the new schedule in the rare event that we have to
|
||||
make a change.
|
||||
[% END %]
|
||||
|
||||
[% FILTER replace('\n', ' ') -%]
|
||||
The notes for the recording are an extended version of the show notes. These
|
||||
extended elements are removed before the show is made fully available on the
|
||||
HPR site (and on archive.org). Comments which might have been missed in the
|
||||
last recording will be marked in red. Comments which would normally be in this
|
||||
month, but which were read out in the last show are marked in green. Comments
|
||||
made in the past month to older shows will be displayed in full (so they are
|
||||
easier to read).
|
||||
[% END %]
|
||||
|
||||
[% FILTER replace('\n', ' ') -%]
|
||||
Look here for the notes for this recording:
|
||||
https://hackerpublicradio.org/eps/hpr[% episode %]/index.html
|
||||
[% END %]
|
||||
|
||||
Summary:
|
||||
[% FILTER indent(' ') -%]
|
||||
Date of recording: [% utc.date %]
|
||||
Start and end times: [% utc.start %] and [% utc.end %]
|
||||
Mumble server: [% server %]
|
||||
Port: [% port %]
|
||||
Room: [% room %]
|
||||
[% END -%]
|
||||
|
||||
Refer to https://hackerpublicradio.org/recording.html for how to use Mumble.
|
||||
|
||||
[% FILTER replace('\n', ' ') -%]
|
||||
There is an iCal file on the HPR site that you can load into a compatible
|
||||
calendar which will remind you of the next 12 upcoming recording dates.
|
||||
Access it from https://hackerpublicradio.org/HPR_Community_News_schedule.ics
|
||||
[% END %]
|
||||
|
||||
See below for start and end times in various international timezones.
|
||||
|
||||
[% FOREACH tz IN timezones -%]
|
||||
[% tz.name %]
|
||||
Start: [% tz.start %]
|
||||
End: [% tz.end %]
|
||||
|
||||
[% END -%]
|
||||
[%#
|
||||
# vim: syntax=tt2:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
|
||||
-%]
|
@ -17,9 +17,9 @@
|
||||
# Hacking"
|
||||
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
||||
# LICENCE: Copyright (c) year 2012-2024 Dave Morriss
|
||||
# VERSION: 0.2.2
|
||||
# VERSION: 0.2.3
|
||||
# CREATED: 2012-10-13 15:34:01
|
||||
# REVISION: 2024-05-24 22:45:56
|
||||
# REVISION: 2024-10-28 13:17:44
|
||||
#
|
||||
#===============================================================================
|
||||
|
||||
@ -42,7 +42,7 @@ use Date::ICal;
|
||||
#
|
||||
# Version number (manually incremented)
|
||||
#
|
||||
our $VERSION = '0.2.2';
|
||||
our $VERSION = '0.2.3';
|
||||
|
||||
#
|
||||
# Script name
|
||||
@ -65,8 +65,6 @@ my ( @startdate, @rdate, @events );
|
||||
#
|
||||
# Attributes for the calendar message
|
||||
#
|
||||
#my $server = 'ch1.teamspeak.cc';
|
||||
#my $port = 64747;
|
||||
my $server = 'chatter.skyehaven.net';
|
||||
my $port = 64738;
|
||||
|
||||
@ -129,11 +127,12 @@ else {
|
||||
# of having a time zone defined (default UTC, as now).
|
||||
#
|
||||
my $monday = 1; # Day of week number 1-7, Monday-Sunday
|
||||
my $offset = -3; # Offset from the target date (-3 is Friday)
|
||||
|
||||
my @starttime = ( 13, 00, 00 ); # UTC
|
||||
my @endtime = ( 15, 00, 00 );
|
||||
my @starttime = ( 15, 00, 00 ); # UTC
|
||||
my @endtime = ( 17, 00, 00 );
|
||||
|
||||
my @todostart = ( 9, 00, 00 ); # UTC
|
||||
my @todostart = ( 9, 00, 00 ); # UTC
|
||||
my @todoend = ( 17, 00, 00 );
|
||||
|
||||
#
|
||||
@ -161,12 +160,13 @@ http://hackerpublicradio.org/recording.php
|
||||
ENDDESC
|
||||
|
||||
#
|
||||
# Compute the next recording date from the starting date (@startdate will be
|
||||
# today's date or the start of the explicitly selected month provided via
|
||||
# -from=DATE. We want day of the week to be Monday, the first in the month,
|
||||
# then to go back 1 day from that to get to the Sunday! Simple)
|
||||
# Compute the next recording date from the starting date.
|
||||
# Now @startdate will be today's date or the start of the explicitly selected
|
||||
# month provided via -from=DATE. We want day of the week to be Monday, the
|
||||
# first in the month, then to go back $offset days from that to get to the
|
||||
# recording day! Simple.
|
||||
#
|
||||
@startdate = make_date( \@startdate, $monday, 1, -1 );
|
||||
@startdate = make_date( \@startdate, $monday, 1, $offset );
|
||||
@rdate = @startdate;
|
||||
|
||||
#
|
||||
@ -208,7 +208,7 @@ for my $i ( 1 .. $count ) {
|
||||
#
|
||||
# Recording date computation from the start of the month
|
||||
#
|
||||
@rdate = make_date( \@rdate, $monday, 1, -1 );
|
||||
@rdate = make_date( \@rdate, $monday, 1, $offset );
|
||||
|
||||
#
|
||||
# Save the current recording date to make an array of arrayrefs
|
||||
@ -305,8 +305,9 @@ exit;
|
||||
# RETURNS: The start of the month in the textual date in Date::Calc
|
||||
# format
|
||||
# DESCRIPTION: Parses the date string and makes a Date::Calc date from the
|
||||
# result where the day part is 1. Optionally checks that the
|
||||
# date isn't in the past, though $force = 1 ignores this check.
|
||||
# result where the day part is forced to be 1. Optionally
|
||||
# checks that the date isn't in the past, though $force
|
||||
# = 1 ignores this check.
|
||||
# THROWS: No exceptions
|
||||
# COMMENTS: Requires Date::Calc and Date::Parse
|
||||
# Note the validation 'die' has a non-generic message
|
||||
|
File diff suppressed because it is too large
Load Diff
33
Community_News/recording_dates.dat
Normal file
33
Community_News/recording_dates.dat
Normal file
@ -0,0 +1,33 @@
|
||||
2021-10-01,2021-10-30 14:00:00
|
||||
2022-06-01,2022-07-02 14:00:00
|
||||
2022-07-01,2022-07-30 14:00:00
|
||||
2022-08-01,2022-09-03 14:00:00
|
||||
2022-09-01,2022-10-01 14:00:00
|
||||
2022-10-01,2022-11-05 14:00:00
|
||||
2022-11-01,2022-12-03 15:00:00
|
||||
2022-12-01,2022-12-27 15:00:00
|
||||
2023-01-01,2023-02-04 15:00:00
|
||||
2023-02-01,2023-03-04 15:00:00
|
||||
2023-03-01,2023-04-01 15:00:00
|
||||
2023-04-01,2023-04-29 15:00:00
|
||||
2023-05-01,2023-06-03 15:00:00
|
||||
2023-07-01,2023-08-05 15:00:00
|
||||
2023-08-01,2023-09-02 15:00:00
|
||||
2023-09-01,2023-09-30 15:00:00
|
||||
2023-10-01,2023-11-04 15:00:00
|
||||
2023-11-01,2023-12-02 15:00:00
|
||||
2023-12-01,2023-12-30 15:00:00
|
||||
2024-01-01,2024-02-04 15:00:00
|
||||
2024-02-01,2024-03-02 15:00:00
|
||||
2024-03-01,2024-03-30 15:00:00
|
||||
2024-05-01,2024-06-02 13:00:00
|
||||
2024-06-01,2024-06-28 15:00:00
|
||||
2024-07-01,2024-08-02 15:00:00
|
||||
2024-08-01,2024-08-29 15:00:00
|
||||
2024-09-01,2024-10-04 15:00:00
|
||||
2024-10-01,2024-11-01 15:00:00
|
||||
2024-11-01,2024-11-29 15:00:00
|
||||
2024-12-01,2025-01-03 15:00:00
|
||||
2025-01-01,2025-01-31 15:00:00
|
||||
2025-02-01,2025-02-28 16:00:00
|
||||
2025-03-01,2025-04-04 15:00:00
|
@ -1 +1 @@
|
||||
shownote_template11.tpl
|
||||
shownote_template12.tpl
|
@ -1,11 +1,15 @@
|
||||
[%# shownote_template11.tpl 2024-05-07 -%]
|
||||
[%# HTML snippet for insertion into the database -%]
|
||||
[%# This one uses the new format for the mailing list data, and partitions -%]
|
||||
[%# comments into past and current. It also marks comments that don't need -%]
|
||||
[%# to be read when -markcomments is selected. It requires make_shownotes >= V0.0.30 -%]
|
||||
[%# shownote_template11.tpl Updated: 2025-03-31 -%]
|
||||
[%# -------------------------------------------------------------------------------- -%]
|
||||
[%# Makes either an HTML snippet for insertion into the database or a full -%]
|
||||
[%# listing with full comments for circulation to the hosts recording the episode -%]
|
||||
[%# This one uses the new format for the mailing list data, and partitions -%]
|
||||
[%# comments into past and current. It also marks comments that don't need -%]
|
||||
[%# to be read when mark_comments is true. It requires make_shownotes >= V0.0.30 -%]
|
||||
[%# -------------------------------------------------------------------------------- -%]
|
||||
[%- USE date -%]
|
||||
[%- USE pad4 = format('%04d') -%]
|
||||
[%- correspondents = "https://hackerpublicradio.org/correspondents"
|
||||
mailinglist = "https://lists.hackerpublicradio.com/mailman/listinfo/hpr"
|
||||
mailbase="https://lists.hackerpublicradio.com/pipermail/hpr"
|
||||
mailthreads = "$mailbase/$review_year-$review_month/thread.html" -%]
|
||||
[%- DEFAULT skip_comments = 0
|
||||
@ -15,8 +19,9 @@
|
||||
missed_count = 0
|
||||
past_count = 0
|
||||
-%]
|
||||
[%# Embedded CSS. The 'table' and 'hr' settings are always there but the rest is only for if -%]
|
||||
[%# we are marking comments -%]
|
||||
[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
|
||||
[%# Embedded CSS. The 'table' and 'hr' settings are always there but the rest is -%]
|
||||
[%# only for when we are marking comments -%]
|
||||
<style>
|
||||
table td.shrink {
|
||||
white-space:nowrap
|
||||
@ -40,12 +45,7 @@ div#highlight {
|
||||
}
|
||||
[%- END %]
|
||||
</style>
|
||||
[%# For the '-mailnotes' option without a file we generate our own inclusion. -%]
|
||||
[%# We pretend 'default_mail' is a filename in the calling script. Messy. -%]
|
||||
[% BLOCK default_mail -%]
|
||||
<a href="[% mailthreads %]" target="_blank">[% mailthreads %]</a>
|
||||
[% END -%]
|
||||
|
||||
[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
|
||||
<h2>New hosts</h2>
|
||||
<p>
|
||||
[% IF hosts.size > 0 -%]
|
||||
@ -88,13 +88,13 @@ There were no new hosts this month.
|
||||
[%# Skip comments if told to by the caller -%]
|
||||
[%- IF skip_comments == 0 -%]
|
||||
[%# Handle any missed comments if mark_comments is true -%]
|
||||
[%- IF mark_comments == 1 && missed_count > 0 -%]
|
||||
[%- IF (mark_comments == 1) AND (missed_count > 0) -%]
|
||||
<br/><div id="highlight">
|
||||
<h2>Missed comment[%- missed_comments.size > 1 ? 's' : '' -%] last month</h2>
|
||||
<p><b>Note to Volunteers</b>: These are comments for shows last month that were not read in the last show because they arrived on or after the recording day. This section will be removed before these notes are released.</p>
|
||||
<p><b>Note to Volunteers</b>: These are comments for shows last month that were not read in the last show because they arrived on or after the recording started. This section will be removed before these notes are released.</p>
|
||||
<ul>
|
||||
[%- FOREACH comment IN missed_comments -%]
|
||||
<li><strong><a href="[% comment.identifier_url %]#comments" target="_blank">hpr[% comment.episode %]</a></strong>
|
||||
<li><strong><a href="[% comment.identifier_url %]#comments" target="_blank">hpr[% pad4(comment.episode) %]</a></strong>
|
||||
([% comment.date %]) "<em>[% comment.title %]</em>" by <a href="[% correspondents %]/[% pad4(comment.hostid) %].html" target="_blank">[% comment.host %]</a>.<br/>
|
||||
<small>Summary: "<em>[% comment.summary %]</em>"</small><br/>
|
||||
From: [% comment.comment_author_name FILTER html_entity -%] on [% date.format(comment.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
|
||||
@ -112,7 +112,7 @@ From: [% comment.comment_author_name FILTER html_entity -%] on [% date.format(co
|
||||
<h2>Comments this month</h2>
|
||||
|
||||
[% IF comment_count > 0 -%]
|
||||
[%- IF mark_comments == 1 && ignore_count > 0 -%]
|
||||
[%- IF (mark_comments == 1) AND (ignore_count > 0) -%]
|
||||
<p id="ignore"><b>Note to Volunteers</b>: Comments marked in green were read in the last
|
||||
Community News show and should be ignored in this one.</p>
|
||||
[%- END -%]
|
||||
@ -124,9 +124,9 @@ There [%- comment_count == 1 ? "is $comment_count comment" : "are $comment_count
|
||||
[% past.size %] previous [% past.size == 1 ? "show" : "shows" %]:</p>
|
||||
<ul>
|
||||
[%# Loop through by episode then by comment relating to that episode -%]
|
||||
[%- FOREACH ep IN past.keys.sort -%]
|
||||
[%- FOREACH ep IN past.keys.nsort -%]
|
||||
[%- arr = past.$ep -%]
|
||||
<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% arr.0.episode %]</a></strong>
|
||||
<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
|
||||
([% arr.0.date %]) "<em>[% arr.0.title %]</em>"
|
||||
by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.<br/>
|
||||
[%- IF mark_comments == 1 || ctext == 1 -%]
|
||||
@ -142,7 +142,7 @@ by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">
|
||||
[%- ELSE %]
|
||||
<li>
|
||||
[%- END %]
|
||||
<a href="[% row.identifier_url %]#[% row.index %]" target="_blank">Comment [% row.index %]</a>:
|
||||
<a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
|
||||
[% row.comment_author_name FILTER html_entity -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
|
||||
[%- IF row.comment_title.length > 0 %]
|
||||
"[% row.comment_title %]"
|
||||
@ -158,7 +158,7 @@ by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">
|
||||
[%- END -%]
|
||||
[%- END -%]
|
||||
</ul><br/>
|
||||
</limage>
|
||||
</li>
|
||||
[%- END -%]
|
||||
</ul>
|
||||
[%- IF mark_comments == 1 || ctext == 1 -%]
|
||||
@ -171,15 +171,15 @@ by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">
|
||||
<h3>This month's shows</h3>
|
||||
<p>There [% cc == 1 ? "is $cc comment" : "are $cc comments" %] on [% current.size %] of this month's shows:</p>
|
||||
<ul>
|
||||
[%- FOREACH ep IN current.keys.sort -%]
|
||||
[%- FOREACH ep IN current.keys.nsort -%]
|
||||
[%- arr = current.$ep -%]
|
||||
<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% arr.0.episode %]</a></strong>
|
||||
<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
|
||||
([% arr.0.date %]) "<em>[% arr.0.title %]</em>"
|
||||
by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.</li>
|
||||
<li style="list-style: none; display: inline">
|
||||
<ul>
|
||||
[%- FOREACH row IN arr -%]
|
||||
<li><a href="[% row.identifier_url %]#[% row.index %]" target="_blank">Comment [% row.index %]</a>:
|
||||
<li><a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
|
||||
[% row.comment_author_name FILTER html_entity -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
|
||||
[%- IF row.comment_title.length > 0 %]
|
||||
"[% row.comment_title %]"
|
||||
@ -199,17 +199,16 @@ There were no comments this month.
|
||||
[%- END %]
|
||||
|
||||
[%# ---------------------------------------------------------------------------------------- -%]
|
||||
[%- IF includefile.defined -%]
|
||||
[%- IF mailnotes == 1 -%]
|
||||
<h2>Mailing List discussions</h2>
|
||||
<p>
|
||||
Policy decisions surrounding HPR are taken by the community as a whole. This
|
||||
discussion takes place on the <a href="https://hackerpublicradio.org/maillist"
|
||||
target="_blank">Mail List</a> which is open to all HPR listeners and
|
||||
contributors. The discussions are open and available on the HPR server under
|
||||
<a href="[% mailbase %]">Mailman</a>.
|
||||
discussion takes place on the <a href="[% mailinglist %]" target="_blank">Mailing List</a>
|
||||
which is open to all HPR listeners and contributors. The discussions are open
|
||||
and available on the HPR server under <a href="[% mailbase %]">Mailman</a>.
|
||||
</p>
|
||||
<p>The threaded discussions this month can be found here:</p>
|
||||
[% INCLUDE $includefile -%]
|
||||
<a href="[% mailthreads %]" target="_blank">[% mailthreads %]</a>
|
||||
[%- END %]
|
||||
|
||||
[%# ---------------------------------------------------------------------------------------- -%]
|
||||
@ -231,4 +230,3 @@ page.</blockquote>
|
||||
[%#
|
||||
# vim: syntax=tt2:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
|
||||
-%]
|
||||
|
||||
|
226
Community_News/shownote_template12.tpl
Normal file
226
Community_News/shownote_template12.tpl
Normal file
@ -0,0 +1,226 @@
|
||||
[%# shownote_template12.tpl Updated: 2025-03-31 -%]
|
||||
[%# -------------------------------------------------------------------------------- -%]
|
||||
[%# Makes either an HTML snippet for insertion into the database or a full -%]
|
||||
[%# listing with full comments for circulation to the hosts recording the episode -%]
|
||||
[%# This one uses the new format for the mailing list data, and partitions -%]
|
||||
[%# comments into past and current. It also marks comments that don't need -%]
|
||||
[%# to be read when mark_comments is true. It requires make_shownotes >= V0.0.30 -%]
|
||||
[%# -------------------------------------------------------------------------------- -%]
|
||||
[%- USE date -%]
|
||||
[%- USE pad4 = format('%04d') -%]
|
||||
[%- correspondents = "https://hackerpublicradio.org/correspondents"
|
||||
mailinglist = "https://lists.hackerpublicradio.com/mailman/listinfo/hpr"
|
||||
mailbase="https://lists.hackerpublicradio.com/pipermail/hpr"
|
||||
mailthreads = "$mailbase/$review_year-$review_month/thread.html" -%]
|
||||
[%- DEFAULT skip_comments = 0
|
||||
mark_comments = 0
|
||||
ctext = 0
|
||||
ignore_count = 0
|
||||
missed_count = 0
|
||||
past_count = 0
|
||||
-%]
|
||||
[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
|
||||
[%# Embedded CSS. The 'table' and 'hr' settings are always there but the rest is -%]
|
||||
[%# only for when we are marking comments -%]
|
||||
<style>
|
||||
table td.shrink {
|
||||
white-space:nowrap
|
||||
}
|
||||
hr.thin {
|
||||
border: 0;
|
||||
height: 0;
|
||||
border-top: 1px solid rgba(0, 0, 0, 0.1);
|
||||
border-bottom: 1px solid rgba(255, 255, 255, 0.3);
|
||||
}
|
||||
[%- IF mark_comments == 1 %]
|
||||
p#ignore, li#ignore {
|
||||
background-color: lightgreen;
|
||||
color:maroon;
|
||||
}
|
||||
div#highlight {
|
||||
border-style: solid;
|
||||
border-color: red;
|
||||
padding-right: 20px;
|
||||
padding-left: 20px;
|
||||
}
|
||||
[%- END %]
|
||||
</style>
|
||||
[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
|
||||
<h2>New hosts</h2>
|
||||
<p>
|
||||
[% IF hosts.size > 0 -%]
|
||||
Welcome to our new host[%- hosts.size > 1 ? 's' : '' -%]: <br />
|
||||
[%- count = 0 %]
|
||||
[%# List the new hosts. If a name contains a comma quote it. -%]
|
||||
[%- FOREACH row IN hosts %]
|
||||
[%- count = count + 1 %]
|
||||
[%- hostname = (row.host.search(',') ? row.host.replace('^(.*)$','"$1"') : row.host) %]
|
||||
<a href="[% correspondents %]/[% pad4(row.hostid) %].html" target="_blank">[% hostname %]</a>
|
||||
[%- count < hosts.size ? ', ' : '.' %]
|
||||
[%- END %]
|
||||
[% ELSE -%]
|
||||
There were no new hosts this month.
|
||||
[% END -%]
|
||||
</p>
|
||||
|
||||
<h2>Last Month's Shows</h2>
|
||||
[%# The id 't01' is in the HPR CSS but might give trouble on the IA -%]
|
||||
<table id="t01">
|
||||
<tr>
|
||||
<th>Id</th>
|
||||
<th>Day</th>
|
||||
<th>Date</th>
|
||||
<th>Title</th>
|
||||
<th>Host</th>
|
||||
</tr>
|
||||
[%- FOREACH row IN shows %]
|
||||
<tr>
|
||||
<td><strong><a href="https://hackerpublicradio.org/eps/hpr[% pad4(row.eps_id) %]/index.html" target="_blank">[% row.eps_id %]</a></strong></td>
|
||||
<td>[% date.format(row.date,'%a') %]</td>
|
||||
<td class="shrink">[% date.format(row.date,'%Y-%m-%d') %]</td>
|
||||
<td><a href="https://hackerpublicradio.org/eps/hpr[% pad4(row.eps_id) %]/index.html" target="_blank">[% row.title %]</a></td>
|
||||
<td><a href="[% correspondents %]/[% pad4(row.ho_hostid) %].html" target="_blank">[% row.ho_host FILTER html_entity %]</a></td>
|
||||
</tr>
|
||||
[%- END %]
|
||||
</table>
|
||||
|
||||
[%# ---------------------------------------------------------------------------------------- -%]
|
||||
[%# Skip comments if told to by the caller -%]
|
||||
[%- IF skip_comments == 0 -%]
|
||||
[%# Handle any missed comments if mark_comments is true -%]
|
||||
[%- IF (mark_comments == 1) AND (missed_count > 0) -%]
|
||||
<br/><div id="highlight">
|
||||
<h2>Missed comment[%- missed_comments.size > 1 ? 's' : '' -%] last month</h2>
|
||||
<p><b>Note to Volunteers</b>: These are comments for shows last month that were not read in the last show because they arrived on or after the recording started. This section will be removed before these notes are released.</p>
|
||||
<ul>
|
||||
[%- FOREACH comment IN missed_comments -%]
|
||||
<li><strong><a href="[% comment.identifier_url %]#comments" target="_blank">hpr[% pad4(comment.episode) %]</a></strong>
|
||||
([% comment.date %]) "<em>[% comment.title %]</em>" by <a href="[% correspondents %]/[% pad4(comment.hostid) %].html" target="_blank">[% comment.host %]</a>.<br/>
|
||||
<small>Summary: "<em>[% comment.summary %]</em>"</small><br/>
|
||||
From: [% comment.comment_author_name FILTER html_entity -%] on [% date.format(comment.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
|
||||
[%- IF comment.comment_title.length > 0 %]
|
||||
"[% comment.comment_title %]"
|
||||
[%- ELSE -%]
|
||||
"[no title]"
|
||||
[%- END -%]
|
||||
<br/><hr class="thin">[% comment.comment_text FILTER html_line_break %]
|
||||
</li><br/>
|
||||
[%- END -%]
|
||||
</ul></div>
|
||||
[%- END -%]
|
||||
[%# ---------------------------------------------------------------------------------------- -%]
|
||||
<h2>Comments this month</h2>
|
||||
|
||||
[% IF comment_count > 0 -%]
|
||||
[%- IF (mark_comments == 1) AND (ignore_count > 0) -%]
|
||||
<p id="ignore"><b>Note to Volunteers</b>: Comments marked in green were read in the last
|
||||
Community News show and should be ignored in this one.</p>
|
||||
[%- END -%]
|
||||
<p>These are comments which have been made during the past month, either to shows released during the month or to past shows.
|
||||
There [%- comment_count == 1 ? "is $comment_count comment" : "are $comment_count comments" -%] in total.</p>
|
||||
[% IF past_count > 0 -%]
|
||||
<h3>Past shows</h3>
|
||||
<p>There [% past_count == 1 ? "is $past_count comment" : "are $past_count comments" %] on
|
||||
[% past.size %] previous [% past.size == 1 ? "show" : "shows" %]:</p>
|
||||
<ul>
|
||||
[%# Loop through by episode then by comment relating to that episode -%]
|
||||
[%- FOREACH ep IN past.keys.nsort -%]
|
||||
[%- arr = past.$ep -%]
|
||||
<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
|
||||
([% arr.0.date %]) "<em>[% arr.0.title %]</em>"
|
||||
by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.<br/>
|
||||
[%- IF mark_comments == 1 || ctext == 1 -%]
|
||||
<small>Summary: "<em>[% arr.0.summary %]</em>"</small></li>
|
||||
[%- END %]
|
||||
<li style="list-style: none; display: inline">
|
||||
<ul>
|
||||
[%- FOREACH row IN arr -%]
|
||||
[%# IF mark_comments == 1 && ((row.comment_timestamp_ut <= last_recording) && (arr.0.date.substr(0,7) == last_month)) -%]
|
||||
[%# IF mark_comments == 1 && ((row.comment_released_ut <= last_recording) && (arr.0.date.substr(0,7) == last_month)) -%]
|
||||
[%- IF mark_comments == 1 && row.ignore == 1 -%]
|
||||
<li id="ignore">
|
||||
[%- ELSE %]
|
||||
<li>
|
||||
[%- END %]
|
||||
<a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
|
||||
[% row.comment_author_name FILTER html_entity -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
|
||||
[%- IF row.comment_title.length > 0 %]
|
||||
"[% row.comment_title %]"
|
||||
[%- ELSE -%]
|
||||
"[no title]"
|
||||
[%- END -%]
|
||||
[%# Add the comment body in too if ctext is true -%]
|
||||
[%- IF ctext == 1 %]
|
||||
<br/><hr class="thin">[% row.comment_text FILTER html_line_break %]
|
||||
</li><br/>
|
||||
[%- ELSE -%]
|
||||
</li>
|
||||
[%- END -%]
|
||||
[%- END -%]
|
||||
</ul><br/>
|
||||
</li>
|
||||
[%- END -%]
|
||||
</ul>
|
||||
[%- IF mark_comments == 1 || ctext == 1 -%]
|
||||
<p><small><small><em>Updated on [% date.format(date.now,'%Y-%m-%d %H:%M:%S') %]</em></small></small></p>
|
||||
[%- END -%]
|
||||
[%- END %]
|
||||
[%# ---------------------------------------------------------------------------------------- -%]
|
||||
[% cc = (comment_count - past_count) -%]
|
||||
[% IF cc > 0 -%]
|
||||
<h3>This month's shows</h3>
|
||||
<p>There [% cc == 1 ? "is $cc comment" : "are $cc comments" %] on [% current.size %] of this month's shows:</p>
|
||||
<ul>
|
||||
[%- FOREACH ep IN current.keys.nsort -%]
|
||||
[%- arr = current.$ep -%]
|
||||
<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
|
||||
([% arr.0.date %]) "<em>[% arr.0.title %]</em>"
|
||||
by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.</li>
|
||||
<li style="list-style: none; display: inline">
|
||||
<ul>
|
||||
[%- FOREACH row IN arr -%]
|
||||
<li><a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
|
||||
[% row.comment_author_name FILTER html_entity -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
|
||||
[%- IF row.comment_title.length > 0 %]
|
||||
"[% row.comment_title %]"
|
||||
[%- ELSE -%]
|
||||
"[no title]"
|
||||
[%- END -%]
|
||||
</li>
|
||||
[%- END -%]
|
||||
</ul><br/>
|
||||
</li>
|
||||
[%- END -%]
|
||||
</ul>
|
||||
[%- END %]
|
||||
[%- ELSE %]
|
||||
There were no comments this month.
|
||||
[%- END %]
|
||||
[%- END %]
|
||||
|
||||
[%# ---------------------------------------------------------------------------------------- -%]
|
||||
[%- IF mailnotes == 1 -%]
|
||||
<h2>Mailing List discussions</h2>
|
||||
<p>
|
||||
Policy decisions surrounding HPR are taken by the community as a whole. This
|
||||
discussion takes place on the <a href="[% mailinglist %]" target="_blank">Mailing List</a>
|
||||
which is open to all HPR listeners and contributors. The discussions are open
|
||||
and available on the HPR server under <a href="[% mailbase %]">Mailman</a>.
|
||||
</p>
|
||||
<p>The threaded discussions this month can be found here:</p>
|
||||
<a href="[% mailthreads %]" target="_blank">[% mailthreads %]</a>
|
||||
[%- END %]
|
||||
|
||||
[%# ---------------------------------------------------------------------------------------- -%]
|
||||
<h2>Events Calendar</h2>
|
||||
<p>With the kind permission of <strong>LWN.net</strong> we are linking to
|
||||
<a href="https://lwn.net/Calendar/" target="_blank">The LWN.net Community Calendar</a>.</p>
|
||||
<p>Quoting the site:</p>
|
||||
<blockquote>This is the LWN.net community event calendar, where we track
|
||||
events of interest to people using and developing Linux and free software.
|
||||
Clicking on individual events will take you to the appropriate web
|
||||
page.</blockquote>
|
||||
[%# ---------------------------------------------------------------------------------------- -%]
|
||||
[%#
|
||||
# vim: syntax=tt2:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
|
||||
-%]
|
@ -1,9 +1,10 @@
|
||||
#!/bin/bash -
|
||||
# shellcheck disable=SC2317
|
||||
#===============================================================================
|
||||
#
|
||||
# FILE: future_upload
|
||||
#
|
||||
# USAGE: ./future_upload
|
||||
# USAGE: ./future_upload [-h] [-v] [-D] [-d {0|1}] [-F] [-r] [-l cp]
|
||||
#
|
||||
# DESCRIPTION: Uploads future HPR shows based on what is in the upload area
|
||||
#
|
||||
@ -13,9 +14,9 @@
|
||||
# NOTES: Contains methods from 'delete_uploaded' and 'weekly_upload' as
|
||||
# well as 'update_state'
|
||||
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
||||
# VERSION: 0.0.15
|
||||
# VERSION: 0.0.17
|
||||
# CREATED: 2021-01-07 12:11:02
|
||||
# REVISION: 2024-07-29 23:17:45
|
||||
# REVISION: 2025-01-06 17:51:57
|
||||
#
|
||||
#===============================================================================
|
||||
|
||||
@ -26,7 +27,7 @@ SCRIPT=${0##*/}
|
||||
|
||||
STDOUT="/dev/fd/2"
|
||||
|
||||
VERSION="0.0.15"
|
||||
VERSION="0.0.17"
|
||||
|
||||
#
|
||||
# Load library functions
|
||||
@ -36,7 +37,7 @@ LIB="$HOME/bin/function_lib.sh"
|
||||
# shellcheck disable=SC1090
|
||||
source "$LIB"
|
||||
|
||||
# {{{ -- Functions -- check_uploads, _log, _usage
|
||||
# {{{ -- Functions -- check_uploads, update_show_state, _log, _usage
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: check_uploads
|
||||
@ -51,13 +52,54 @@ check_uploads () {
|
||||
#
|
||||
# Look for files called hpr1234.flac and so on. Don't bother with the
|
||||
# hpr1234_source.flac one. As soon as a file is missing return with false.
|
||||
# 2025-01-01: Dropped 'spx' from the list
|
||||
#
|
||||
for suff in flac mp3 ogg opus spx wav; do
|
||||
for suff in flac mp3 ogg opus wav; do
|
||||
if [[ ! -e $UPLOADS/$prefix.$suff ]]; then
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
#
|
||||
# Transcripts are (currently) in a sub-directory with the same name as the
|
||||
# IA item. We only cater for two types as of 2025.
|
||||
#
|
||||
for suff in txt srt; do
|
||||
if [[ ! -e $UPLOADS/$prefix/$prefix.$suff ]]; then
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: update_show_state
|
||||
# DESCRIPTION: Updates the status of a single show in the HPR database.
|
||||
# It is assumed the caller has found the show number in the
|
||||
# 'reservations' table with the required status of
|
||||
# 'MEDIA_TRANSCODED'. All this function does is to change this
|
||||
# to 'UPLOADED_TO_IA', returning true if successful, otherwise
|
||||
# false.
|
||||
# PARAMETERS: $show Show number to update
|
||||
# RETURNS: True if the update worked, otherwise false
|
||||
#===============================================================================
|
||||
update_show_state () {
|
||||
local show=${1:?Usage: update_state show}
|
||||
local BASECOM URL QUERY COMMAND RES
|
||||
|
||||
BASECOM='curl -K ./.hpradmin_curlrc -s'
|
||||
URL="https://hub.hackerpublicradio.org/cms/status.php"
|
||||
QUERY="${BASECOM} ${URL}"
|
||||
|
||||
COMMAND="${QUERY}?ep_num=${show}&status=UPLOADED_TO_IA"
|
||||
|
||||
$COMMAND
|
||||
RES=$?
|
||||
if [[ $RES -ne 0 ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
@ -72,7 +114,7 @@ check_uploads () {
|
||||
# PARAMETERS: 1 - the message to write
|
||||
# RETURNS: Nothing
|
||||
#===============================================================================
|
||||
# shellcheck disable=SC2317 disable=SC2059
|
||||
# shellcheck disable=SC2059
|
||||
_log () {
|
||||
local msg="$1"
|
||||
|
||||
@ -169,7 +211,7 @@ BASECOM='curl -K ./.hpradmin_curlrc -s'
|
||||
URL="https://hub.hackerpublicradio.org/cms/status.php"
|
||||
# QUERY1="${BASECOM} ${URL}"
|
||||
QUERY2="${BASECOM} -o - ${URL}"
|
||||
UPSTATE="$BASEDIR/update_state"
|
||||
# UPSTATE="$BASEDIR/update_state"
|
||||
|
||||
#
|
||||
# Fallback URL
|
||||
@ -188,10 +230,10 @@ ia=$(command -v ia)
|
||||
echo "Needs the 'make_metadata' script"
|
||||
exit 1
|
||||
}
|
||||
[ -e "$UPSTATE" ] || {
|
||||
echo "Needs the 'update_state' script"
|
||||
exit 1
|
||||
}
|
||||
# [ -e "$UPSTATE" ] || {
|
||||
# echo "Needs the 'update_state' script"
|
||||
# exit 1
|
||||
# }
|
||||
|
||||
#
|
||||
# File of processed shows
|
||||
@ -223,6 +265,9 @@ do
|
||||
done
|
||||
shift $((OPTIND - 1))
|
||||
|
||||
#
|
||||
# Check and set option variables
|
||||
#
|
||||
DRYRUN=${DRYRUN:-1}
|
||||
if [[ $DRYRUN -ne 0 && $DRYRUN -ne 1 ]]; then
|
||||
echo "** Use '-d 0' or '-d 1'"
|
||||
@ -261,6 +306,7 @@ fi
|
||||
|
||||
#
|
||||
# Declarations
|
||||
# ------------
|
||||
#
|
||||
declare -A processed
|
||||
declare -A ready
|
||||
@ -271,6 +317,7 @@ lastitem=
|
||||
|
||||
#
|
||||
# Load array of processed shows
|
||||
# ---- ----- -- --------- -----
|
||||
#
|
||||
while read -r item; do
|
||||
processed+=([$item]=1)
|
||||
@ -278,46 +325,17 @@ done < "$PROCFILE"
|
||||
[ "$VERBOSE" -eq 1 ] && echo "Number of shows in cache: ${#processed[@]}"
|
||||
|
||||
#
|
||||
# TODO: Create the associative array 'ready' containing the numbers of shows
|
||||
# ready for upload. This is a way to ensure that we don't try and upload shows
|
||||
# in transit to the upload area.
|
||||
# Populate the associative array 'ready' with the numbers of shows ready for
|
||||
# upload. This is a way to ensure that we don't try and upload shows in
|
||||
# transit to the upload area. Only do this if force mode is off.
|
||||
#
|
||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
# Proposed code. Not sure what the actual URL will be nor what will be
|
||||
# returned if nothing is ready for upload yet
|
||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
#
|
||||
# json=$(curl http://hackerpublicradio.org/queue.php -s -o -)
|
||||
# while read -r showno; do
|
||||
# ready+=([$showno]=1)
|
||||
# done < <(echo "${json}" | jq '.READY_FOR_IA_UPLOAD[] | tonumber')
|
||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
# Change of plan. Now we have a list of CSV values, so we need to do something
|
||||
# like this:
|
||||
#
|
||||
# reservations=$($BASECOM -o - $URL)
|
||||
# while read -r line; do
|
||||
# if [[ $line =~ ^([^,]+),([^,]+),([^,]+),([^,]+),([^,]+),.*$ ]]; then
|
||||
# state="${BASH_REMATCH[5]}"
|
||||
# show="${BASH_REMATCH[2]}"
|
||||
# fi
|
||||
# if [[ $state = 'MEDIA_TRANSCODED' ]]; then
|
||||
# ready+=([$show]=1)
|
||||
# fi
|
||||
# done <<< $reservations
|
||||
#
|
||||
# At the end of this the associative array 'ready' will contain the keys of
|
||||
# shows that are ready for upload (presumably) so we can look in this array to
|
||||
# double check.
|
||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
if [[ $FORCE -eq 0 ]]; then
|
||||
#
|
||||
# Collect the current table of shows requiring work. We expect something like:
|
||||
# timestamp_epoc,ep_num,ep_date,key,status,email
|
||||
# 1651286617,3617,2022-06-14,fda088e0e3bd5d0353ea6b7569e93b87626ca25976a0a,UPLOADED_TO_IA,lurkingprion@gmail.com
|
||||
# 1651648589,3619,2022-06-16,e7d3810afa098863d81663418d8640276272284de68f1,UPLOADED_TO_IA,monochromec@gmail.com
|
||||
# TODO: Check for a failure in the query?A
|
||||
# TODO: Reinstate the check for a failure in the query? Se update_state
|
||||
# NOTE: Problem encountered 2022-09-23 because the SSL certificate has expired
|
||||
#
|
||||
reservations=$($QUERY2) || {
|
||||
@ -342,8 +360,8 @@ if [[ $FORCE -eq 0 ]]; then
|
||||
fi
|
||||
|
||||
#
|
||||
# The query returns the bare number, but we're using 'hprxxxx' as the key in
|
||||
# the 'ready' array.
|
||||
# The query returns the bare show number, but we're using 'hprxxxx' as the
|
||||
# key in the 'ready' array.
|
||||
#
|
||||
while read -r line; do
|
||||
if [[ $line =~ ^([^,]+),([^,]+),([^,]+),([^,]+),([^,]+),.*$ ]]; then
|
||||
@ -363,7 +381,10 @@ fi
|
||||
|
||||
#
|
||||
# Process files. There will be several with the same prefix so look for
|
||||
# a change of prefix
|
||||
# a change of prefix.
|
||||
#
|
||||
# The loop is reading from the following pipeline:
|
||||
# find "$UPLOADS" -regextype posix-extended -regex '.*hpr[0-9]{4}.*' | sort
|
||||
#
|
||||
while read -r path; do
|
||||
#
|
||||
@ -379,8 +400,8 @@ while read -r path; do
|
||||
_DEBUG "Item $item"
|
||||
|
||||
#
|
||||
# Detect that the item prefix has changed. If it has we're processing
|
||||
# a new IA identifier, so work on this one
|
||||
# Detect that the item prefix has changed. If it has we've found a new IA
|
||||
# identifier, so work on the previous one
|
||||
#
|
||||
if [[ $item != "$lastitem" ]]; then
|
||||
lastitem=$item
|
||||
@ -414,7 +435,8 @@ while read -r path; do
|
||||
processed+=([$lastitem]=1)
|
||||
else
|
||||
#
|
||||
# Is the show ready for upload?
|
||||
# Is the show ready for upload? We don't check if force mode
|
||||
# is on. If not ready we skip this show.
|
||||
#
|
||||
if [[ $FORCE -eq 0 ]]; then
|
||||
if [[ ! -v "ready[$lastitem]" ]]; then
|
||||
@ -461,10 +483,9 @@ while read -r path; do
|
||||
|
||||
done < <(find "$UPLOADS" -regextype posix-extended -regex '.*hpr[0-9]{4}.*' | sort)
|
||||
|
||||
#
|
||||
#-------------------------------------------------------------------------------
|
||||
# Write the processed array to the cache file unless in dry-run mode
|
||||
#
|
||||
# [ $DEBUG -eq 1 ] && { echo -n 'D> '; declare -p processed; }
|
||||
#-------------------------------------------------------------------------------
|
||||
_DEBUG "processed = ${!processed[*]}"
|
||||
[ "$VERBOSE" -eq 1 ] && echo "Number of shows in cache: ${#processed[@]}"
|
||||
if [[ $DRYRUN -ne 1 ]]; then
|
||||
@ -473,24 +494,26 @@ if [[ $DRYRUN -ne 1 ]]; then
|
||||
done < <(printf '%s\n' "${!processed[@]}" | sort -u ) > "$PROCFILE"
|
||||
fi
|
||||
|
||||
#
|
||||
#-------------------------------------------------------------------------------
|
||||
# Generate the list of uploads for the 'make_metadata' option '-list=1,2,3'.
|
||||
# The show numbers are keys in the associative array 'uploads'. The
|
||||
# end-product is a comma-separated list of the keys in the variable '$list'.
|
||||
# Order is unimportant because make_metadata sorts internally.
|
||||
#
|
||||
#-------------------------------------------------------------------------------
|
||||
_DEBUG "uploads = ${!uploads[*]}"
|
||||
[ "$VERBOSE" -eq 1 ] && echo "Number of shows for upload: ${#uploads[@]}"
|
||||
printf -v list '%s,' "${!uploads[@]}"
|
||||
list="${list:0:-1}"
|
||||
|
||||
#
|
||||
#-------------------------------------------------------------------------------
|
||||
# If there are no uploads to do we can stop
|
||||
#
|
||||
#-------------------------------------------------------------------------------
|
||||
[[ ! -v uploads[@] ]] && { echo "Nothing to do!"; exit; }
|
||||
|
||||
#
|
||||
#-------------------------------------------------------------------------------
|
||||
# Check that the shows being uploaded have all their files and log what is
|
||||
# happening.
|
||||
#
|
||||
#-------------------------------------------------------------------------------
|
||||
while read -r show; do
|
||||
echo "$(date +%Y%m%d%H%M%S) preparing to upload hpr$show" >> "$LOGFILE"
|
||||
|
||||
@ -501,10 +524,10 @@ while read -r show; do
|
||||
fi
|
||||
done < <(printf '%s\n' "${!uploads[@]}" | sort)
|
||||
|
||||
#
|
||||
#-------------------------------------------------------------------------------
|
||||
# Define output files. If the list contains one element then it's a different
|
||||
# name from the multi-element case (make_metadata does this too).
|
||||
#
|
||||
#-------------------------------------------------------------------------------
|
||||
if [[ ${#uploads[@]} -eq 1 ]]; then
|
||||
metadata="metadata_${minshow}.csv"
|
||||
script="script_${minshow}.sh"
|
||||
@ -513,9 +536,9 @@ else
|
||||
script="script_${minshow}-${maxshow}.sh"
|
||||
fi
|
||||
|
||||
#
|
||||
#-------------------------------------------------------------------------------
|
||||
# Perform the uploads or report what would be done
|
||||
#
|
||||
#-------------------------------------------------------------------------------
|
||||
if [[ $DRYRUN -eq 1 ]]; then
|
||||
echo "Dry run: Would have uploaded list '$list'"
|
||||
echo "Dry run: Would have created $metadata and $script"
|
||||
@ -562,17 +585,17 @@ else
|
||||
echo "$(date +%Y%m%d%H%M%S) ${#uploads[@]} uploads completed" >> "$LOGFILE"
|
||||
|
||||
#
|
||||
# Update the state in the HPR database, unless we're using
|
||||
# FORCE. Pass the limit used here to this script so it can
|
||||
# stop looking for work unnecessarily
|
||||
# Update the state of all the shows being processed in the
|
||||
# HPR database, unless we're using FORCE.
|
||||
#
|
||||
if [[ $FORCE -eq 0 ]]; then
|
||||
$UPSTATE -l$LIMIT
|
||||
RES=$?
|
||||
if [[ $RES -ne 0 ]]; then
|
||||
echo "Problem updating database state"
|
||||
exit 1
|
||||
fi
|
||||
while read -r show; do
|
||||
if update_show_state $show; then
|
||||
echo "Updated state for show $show"
|
||||
else
|
||||
echo "Failed to update state for show $show"
|
||||
fi
|
||||
done < <(printf '%s\n' "${!uploads[@]}" | sort)
|
||||
else
|
||||
echo "Not updating the database, FORCE mode is on"
|
||||
fi
|
||||
|
245
InternetArchive/reformat_html
Executable file
245
InternetArchive/reformat_html
Executable file
@ -0,0 +1,245 @@
|
||||
#!/usr/bin/env perl
|
||||
#===============================================================================
|
||||
#
|
||||
# FILE: reformat_html
|
||||
#
|
||||
# USAGE: ./reformat_html < input.html > output.html
|
||||
#
|
||||
# DESCRIPTION: Reformats the HTML found in the HPR database in the 'notes'
|
||||
# field to the format required in the 'description' field of an
|
||||
# item on the IA. It reads from STDIN and writes to STDOUT.
|
||||
#
|
||||
# OPTIONS: ---
|
||||
# REQUIREMENTS: ---
|
||||
# BUGS: ---
|
||||
# NOTES: ---
|
||||
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
||||
# VERSION: 0.0.1
|
||||
# CREATED: 2025-02-09 22:56:30
|
||||
# REVISION: 2025-02-13 11:13:37
|
||||
#
|
||||
#===============================================================================
|
||||
|
||||
use v5.36;
|
||||
use strict;
|
||||
use warnings;
|
||||
use feature qw{ say try };
|
||||
no warnings qw{ experimental::try };
|
||||
|
||||
use open ':std', ':encoding(UTF-8)'; # Make all IO UTF-8
|
||||
|
||||
use HTML::TreeBuilder 5 -weak;
|
||||
use HTML::Entities;
|
||||
|
||||
#
|
||||
# Version number (Incremented by Vim)
|
||||
#
|
||||
our $VERSION = '0.0.1';
|
||||
|
||||
#
|
||||
# Declarations
|
||||
#
|
||||
my ($verbose, @notes, $notes, $tree);
|
||||
|
||||
#
|
||||
# Read the input data into an array
|
||||
#
|
||||
try {
|
||||
@notes = <STDIN>;
|
||||
}
|
||||
catch ($e) {
|
||||
warn "Problem reading input HTML; $e";
|
||||
exit 1;
|
||||
}
|
||||
|
||||
die "No input HTML detected\n" unless @notes;
|
||||
|
||||
#
|
||||
# Turn the array into a scalar
|
||||
#
|
||||
$notes = join( '', @notes );
|
||||
|
||||
#
|
||||
# Get ready to parse the array
|
||||
#
|
||||
$tree = HTML::TreeBuilder->new;
|
||||
$tree->ignore_unknown(0);
|
||||
$tree->no_expand_entities(1);
|
||||
$tree->p_strict(1);
|
||||
$tree->store_comments(1); # Necessary?
|
||||
$tree->warn(1);
|
||||
|
||||
#
|
||||
# Parse HTML to the tree structure
|
||||
#
|
||||
$tree->parse_content($notes)
|
||||
or die "HTML::TreeBuilder failed to parse input HTML: $!\n";
|
||||
|
||||
#
|
||||
# Flatten all <pre> tags and add <br/> tags
|
||||
#
|
||||
$notes = flatten_pre($tree);
|
||||
|
||||
#
|
||||
# Deal with non-ASCII
|
||||
#
|
||||
$notes = encode_entities( $notes, '^\n&\x20-\x25\x27-\x7e' );
|
||||
|
||||
#
|
||||
# Remove all newlines
|
||||
#
|
||||
$notes =~ s/\n//g;
|
||||
|
||||
#
|
||||
# Write the end result to the STDOUT
|
||||
#
|
||||
say $notes;
|
||||
|
||||
exit;
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: flatten_pre
|
||||
# PURPOSE: Process notes "flattening" <pre> contents
|
||||
# PARAMETERS: $tree HTML::TreeBuilder object containing parsed and
|
||||
# partially processed notes
|
||||
# RETURNS: Processed notes
|
||||
# DESCRIPTION: The HTML "<pre>" tag encloses preformatted text. It can also
|
||||
# contain some formatting tags like <em> and <code>, but spaces
|
||||
# and newlines are significant. The Internet Archive upload API
|
||||
# uses HTTP headers which are text strings without newlines, so
|
||||
# when these tags are uploaded through this route some
|
||||
# formatting is lost. What this routine does is parse the
|
||||
# contents of all <pre> sections in $notes, adding <br/> tags
|
||||
# to replace newlines. It has to perform a full parse
|
||||
# since the contents may include HTML tags and these need to be
|
||||
# passed through intact. It calls the subroutine 'flatten_item' to
|
||||
# deal with the recursive nature of HTML tags.
|
||||
# THROWS: No exceptions
|
||||
# COMMENTS: None
|
||||
# SEE ALSO: N/A
|
||||
#===============================================================================
|
||||
sub flatten_pre {
|
||||
my ($tree) = @_;
|
||||
|
||||
#
|
||||
# Find all the <pre> tags
|
||||
#
|
||||
my @pre_tags = $tree->look_down( _tag => 'pre', );
|
||||
|
||||
#
|
||||
# Walk the various <pre> elements in the document
|
||||
#
|
||||
foreach my $tag (@pre_tags) {
|
||||
#
|
||||
# Save the tag and empty the original
|
||||
#
|
||||
my $saved = $tag->clone();
|
||||
$tag->delete_content();
|
||||
|
||||
#
|
||||
# Walk the saved content and rebuild the tag into $atag using the
|
||||
# nested arrayref structure permitted by HTML::Element for
|
||||
# convenience (the alternative is a little nasty). See the
|
||||
# documentation for 'new_from_lol' in HTML::Element.
|
||||
#
|
||||
my $atag;
|
||||
foreach my $item ( @{ $saved->content_array_ref } ) {
|
||||
push( @$atag, flatten_item($item) );
|
||||
}
|
||||
|
||||
#
|
||||
# Rebuild the tag from the arrayref we built. We treat the arrayref
|
||||
# structure we just built as an array because otherwise the top level
|
||||
# is interpreted as a spurious <null> tag.
|
||||
#
|
||||
$tag->push_content(@$atag);
|
||||
}
|
||||
|
||||
#
|
||||
# Trim out the original notes from the enclosing tags we added earlier
|
||||
#
|
||||
my $body = $tree->look_down( _tag => 'body' );
|
||||
( my $result = $body->as_HTML( undef, ' ', {} ) )
|
||||
=~ s{(^<body[^>]*>|</body>$)}{}gi;
|
||||
|
||||
return $result;
|
||||
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: flatten_item
|
||||
# PURPOSE: Recursively "flatten" items within the enclosing <pre>
|
||||
# PARAMETERS: $item an HTML::Element item parsed from the original
|
||||
# <pre> section
|
||||
# RETURNS: An arrayref if the last seen item was a tag, otherwise a list
|
||||
# DESCRIPTION: Since <pre> sections can contain inline elements which change
|
||||
# the rendering of the text we need to parse these as we add
|
||||
# <br/> tags. This routine does this by recursively descending
|
||||
# through the contents. A common tag sequence is <pre><code> for
|
||||
# scripts and the like. This routine deals with such sequences.
|
||||
# It expects to receive the contents in sequence and builds the
|
||||
# result as a nested arrayref structure.
|
||||
# THROWS: No exceptions
|
||||
# COMMENTS: None
|
||||
# SEE ALSO: N/A
|
||||
#===============================================================================
|
||||
sub flatten_item {
|
||||
my ($item) = @_;
|
||||
|
||||
return unless defined($item);
|
||||
|
||||
my ( @result, %attr );
|
||||
|
||||
#
|
||||
# Is it a sub-tag or non-tag content?
|
||||
#
|
||||
if ( ref($item) ) {
|
||||
#
|
||||
# It's a tag. Save the tag name and any attributes and recurse into
|
||||
# it. Return an arrayref
|
||||
#
|
||||
push( @result, $item->tag() );
|
||||
%attr = $item->all_external_attr();
|
||||
push( @result, \%attr ) if %attr;
|
||||
for my $child ( $item->content_list() ) {
|
||||
push( @result, flatten_item($child) );
|
||||
}
|
||||
return \@result;
|
||||
}
|
||||
else {
|
||||
#
|
||||
# It's non-tag content. Join the lines with <br/> tags. Return an
|
||||
# array (since this is a simple list).
|
||||
#
|
||||
# Note that we split with a LIMIT of -1 which causes any trailing list
|
||||
# items to be returned; default behaviour is to drop them.
|
||||
#
|
||||
$item =~ s/\r//g;
|
||||
my @content = split( /\n/, $item, -1 );
|
||||
if (@content) {
|
||||
#
|
||||
# Remove a leading blank line - usually the result of
|
||||
# a "<pre>'NL'text" sequence
|
||||
#
|
||||
shift(@content) if ( $content[0] =~ /^\s*$/ );
|
||||
|
||||
#
|
||||
# Join back the lines with <br/> tags between them.
|
||||
#
|
||||
foreach my $txt (@content) {
|
||||
push( @result, $txt, ['br'] );
|
||||
}
|
||||
|
||||
#
|
||||
# Remove the <br/> at the end, it's spurious
|
||||
#
|
||||
pop(@result);
|
||||
}
|
||||
|
||||
return (@result);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker
|
||||
|
@ -6,7 +6,7 @@
|
||||
# USAGE: ./tidy_uploaded [-h] [-v] [-d {0|1}] [-c COUNT]
|
||||
#
|
||||
# DESCRIPTION: Relocates HPR audio and other show-related files on 'borg'
|
||||
# after their shows have been uploaded to the Internet Archive
|
||||
# after their shows have been uploaded to the Internet Archive.
|
||||
#
|
||||
# OPTIONS: ---
|
||||
# REQUIREMENTS: ---
|
||||
@ -43,7 +43,7 @@ TMP1=$(mktemp) || { echo "$SCRIPT: creation of temporary file failed!"; exit 1;
|
||||
trap 'cleanup_temp $TMP1' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
|
||||
|
||||
#
|
||||
# Configure depending whether local or on the VPS
|
||||
# Configure depending whether local or on 'borg'
|
||||
#
|
||||
case $HOSTNAME in
|
||||
borg) BASEDIR="$HOME/InternetArchive"
|
||||
@ -95,7 +95,7 @@ queued_tasks () {
|
||||
# NAME: movefile
|
||||
# DESCRIPTION: Moves a file to a new place, catering for any directories in
|
||||
# the path
|
||||
# PARAMETERS: $1 directory to move form
|
||||
# PARAMETERS: $1 directory to move from
|
||||
# $2 directory to move to
|
||||
# $3 file (or sub-path to move)
|
||||
# RETURNS: True if a move was done, otherwise False
|
||||
@ -356,7 +356,7 @@ while read -r path; do
|
||||
#
|
||||
tasks=$(queued_tasks "$item")
|
||||
if [[ $tasks -gt 0 ]]; then
|
||||
echo "** Item $item still has $tasks unfinished " \
|
||||
echo "** Item $item still has $tasks unfinished" \
|
||||
"$(ngettext task tasks "$tasks")"
|
||||
echo "** Skipping to the next item"
|
||||
continue
|
||||
@ -434,9 +434,6 @@ while read -r path; do
|
||||
|
||||
done < <(find "$UPLOADS" -regextype posix-extended -regex '.*hpr[0-9]{4}.*' -printf "%CY%Cm%Cd%CH%CM%CS %p\n" | sort | cut -f2 -d' ')
|
||||
|
||||
# Old 'find' used:
|
||||
# done < <(find "$UPLOADS" -regextype posix-extended -regex '.*hpr[0-9]{4}.*' | sort)
|
||||
|
||||
#
|
||||
# No shows processed? There was nothing to do
|
||||
#
|
||||
|
@ -3,7 +3,7 @@
|
||||
#
|
||||
# FILE: update_state
|
||||
#
|
||||
# USAGE: ./update_state
|
||||
# USAGE: ./update_state [-h] [-D] [-d] [-F] [-l N] [-m]
|
||||
#
|
||||
# DESCRIPTION: A script to update the state of shows which have been sent to
|
||||
# the IA. It looks at the current state of the 'reservations'
|
||||
@ -136,7 +136,6 @@ esac
|
||||
|
||||
cd "$BASEDIR" || { echo "Can't cd to $BASEDIR"; exit 1; }
|
||||
|
||||
|
||||
#
|
||||
# Tools
|
||||
#
|
||||
|
Binary file not shown.
739
Show_Submission/extract_images
Executable file
739
Show_Submission/extract_images
Executable file
@ -0,0 +1,739 @@
|
||||
#!/usr/bin/env perl
|
||||
#===============================================================================
|
||||
#
|
||||
# FILE: extract_images
|
||||
#
|
||||
# USAGE: ./extract_images [--help] [--documentation|man]
|
||||
# [--prefix=STRING] [--[no-]backup] [--force] [--[no]silent]
|
||||
# HTML_file [ [HTML_file_2] [HTML_file_3] ... ]
|
||||
#
|
||||
# DESCRIPTION: Processes HTML files which may have 'data' scheme URIs containing
|
||||
# images, and extracts these images into files in the same
|
||||
# directory. The 'data' scheme links are converted to 'https'
|
||||
# and reference the extracted files. The modified HTML is
|
||||
# output, and the original will be saved as a backup if
|
||||
# requested.
|
||||
#
|
||||
# The 'data' URI scheme is specified in RFC 2397.
|
||||
#
|
||||
# OPTIONS: ---
|
||||
# REQUIREMENTS: ---
|
||||
# BUGS: ---
|
||||
# NOTES: ---
|
||||
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
||||
# VERSION: 0.0.3
|
||||
# CREATED: 2024-12-25 10:53:15
|
||||
# REVISION: 2024-12-31 20:56:50
|
||||
#
|
||||
#===============================================================================
|
||||
|
||||
use v5.36;
|
||||
use strict;
|
||||
use warnings;
|
||||
use feature qw{ say state try };
|
||||
no warnings qw{ experimental::try };
|
||||
|
||||
use open ':std', ':encoding(UTF-8)'; # Make all IO UTF-8
|
||||
|
||||
use Getopt::Long;
|
||||
use Pod::Usage;
|
||||
|
||||
use HTML::TreeBuilder 5 -weak;
|
||||
use URI;
|
||||
use MIME::Types;
|
||||
|
||||
use Path::Tiny;
|
||||
use File::Copy;
|
||||
|
||||
use Data::Dumper;
|
||||
|
||||
#
|
||||
# Version number (Incremented by Vim)
|
||||
#
|
||||
our $VERSION = '0.0.3';
|
||||
|
||||
#
|
||||
# Script and directory names
|
||||
#
|
||||
( my $PROG = $0 ) =~ s|.*/||mx;
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Declarations
|
||||
#-------------------------------------------------------------------------------
|
||||
my ( $notes, $typename, $uri );
|
||||
my ( $fcount, $basename, $filename, $suffix, $fh );
|
||||
my ( $updates, $bsuffix, $newURL );
|
||||
|
||||
my $backupcount = 5;
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Options and arguments
|
||||
#-------------------------------------------------------------------------------
|
||||
# {{{
|
||||
#
|
||||
# Option defaults
|
||||
#
|
||||
my $DEFDEBUG = 0;
|
||||
my $DEFPREFIX = 'image';
|
||||
|
||||
my %options;
|
||||
Options( \%options );
|
||||
|
||||
#
|
||||
# Default help shows minimal information
|
||||
#
|
||||
pod2usage( -msg => "$PROG version $VERSION\n", -exitval => 1, -verbose => 0 )
|
||||
if ( $options{'help'} );
|
||||
|
||||
#
|
||||
# The -documentation or -man option shows the full POD documentation through
|
||||
# a pager for convenience
|
||||
#
|
||||
pod2usage( -msg => "$PROG version $VERSION\n", -exitval => 1, -verbose => 2 )
|
||||
if ( $options{'documentation'} );
|
||||
|
||||
#
|
||||
# Collect options
|
||||
#
|
||||
my $DEBUG = ( defined( $options{debug} ) ? $options{debug} : $DEFDEBUG );
|
||||
my $silent = ( defined( $options{silent} ) ? $options{silent} : 0 );
|
||||
my $prefix = ( defined( $options{prefix} ) ? $options{prefix} : $DEFPREFIX );
|
||||
my $backup = ( defined( $options{backup} ) ? $options{backup} : 1 );
|
||||
my $force = ( defined( $options{force} ) ? $options{force} : 0 );
|
||||
|
||||
#
|
||||
# Check we have arguments
|
||||
#
|
||||
pod2usage(
|
||||
-msg => "Missing arguments. One or more HTML file names are needed\n",
|
||||
-exitval => 1,
|
||||
-verbose => 0
|
||||
)
|
||||
unless ( scalar(@ARGV) > 0 );
|
||||
|
||||
#
|
||||
# Clean up the prefix
|
||||
#
|
||||
$prefix = ($prefix =~ /(.*)_$/ ? $1 : $prefix);
|
||||
|
||||
#
|
||||
# Backup suffix (the dot gets added later)
|
||||
#
|
||||
$bsuffix = 'bak';
|
||||
|
||||
#
|
||||
# Debug the options
|
||||
#
|
||||
_debug(
|
||||
$DEBUG > 1,
|
||||
'$silent = ' . $silent,
|
||||
'$prefix = ' . $prefix,
|
||||
'$backup = ' . $backup,
|
||||
'$force = ' . $force,
|
||||
);
|
||||
|
||||
# }}}
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Prepare a MIME::Types object
|
||||
#-------------------------------------------------------------------------------
|
||||
my $mt = MIME::Types->new;
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Loop through the arguments
|
||||
#-------------------------------------------------------------------------------
|
||||
foreach my $notesfile (@ARGV) {
|
||||
|
||||
#
|
||||
# Get the absolute path of the argument
|
||||
#
|
||||
my $abs_nf = path($notesfile)->absolute;
|
||||
|
||||
#
|
||||
# Get the 'dirname' from the absolute path
|
||||
#
|
||||
my $dirname = path($abs_nf)->parent->stringify;
|
||||
|
||||
unless (-e $abs_nf) {
|
||||
warn "Unable to find $notesfile\n";
|
||||
next;
|
||||
}
|
||||
|
||||
#
|
||||
# Force the MIME type of $notesfile to a string
|
||||
#
|
||||
$typename = "" . coalesce( $mt->mimeTypeOf("$abs_nf"), '' );
|
||||
|
||||
#
|
||||
# Check the MIME type and reject non-HTML
|
||||
#
|
||||
unless ($typename eq 'text/html') {
|
||||
warn "File $abs_nf is not HTML\n";
|
||||
next
|
||||
}
|
||||
|
||||
say "Reading from $abs_nf\n" unless $silent;
|
||||
|
||||
#
|
||||
# Get HTML file basename (no path) without the suffix for building filenames
|
||||
#
|
||||
$basename = path($abs_nf)->basename('.html');
|
||||
|
||||
#
|
||||
# Read the HTML
|
||||
#
|
||||
open (my $nfh, '<', $notesfile) or die "Unable to open $notesfile\n";
|
||||
$notes = <$nfh>;
|
||||
close($nfh);
|
||||
|
||||
#
|
||||
# Image files are to have an index/sequence number
|
||||
#
|
||||
$fcount = 0;
|
||||
|
||||
#
|
||||
# Keep a note of HTML updates
|
||||
#
|
||||
$updates = 0;
|
||||
|
||||
#
|
||||
# Initialise the TreeBuilder
|
||||
#
|
||||
my $tree = HTML::TreeBuilder->new;
|
||||
$tree->ignore_unknown(0);
|
||||
$tree->no_expand_entities(1);
|
||||
$tree->p_strict(1);
|
||||
$tree->store_comments(1);
|
||||
$tree->warn(1);
|
||||
|
||||
#
|
||||
# Load the HTML obtained from the file into the TreeBuilder
|
||||
#
|
||||
$tree->parse_content($notes)
|
||||
or die "HTML::TreeBuilder failed to parse notes: $!\n";
|
||||
|
||||
#
|
||||
# Loop through the tree looking for 'data' scheme images
|
||||
#
|
||||
for ( @{ $tree->extract_links('img') } ) {
|
||||
my ( $link, $element, $attr, $tag ) = @$_;
|
||||
|
||||
$uri = URI->new($link);
|
||||
unless ($silent) {
|
||||
say "Scheme: ", $uri->scheme;
|
||||
say "Media type: ", $uri->media_type;
|
||||
}
|
||||
|
||||
#
|
||||
# We only care about 'data' scheme stuff - for now anyway, and only
|
||||
# images within this set
|
||||
#
|
||||
if ( $uri->scheme eq 'data' ) {
|
||||
#
|
||||
# Only images
|
||||
#
|
||||
if ( $uri->media_type =~ /^image/ ) {
|
||||
#
|
||||
# Extract the file name suffix from the MIME string, and give it
|
||||
# a leading '.'
|
||||
#
|
||||
( $suffix = $uri->media_type ) =~ s{^.*/}{.};
|
||||
|
||||
#
|
||||
# Construct the filename for this image making sure it's in
|
||||
# the directory where the HTML is located.
|
||||
#
|
||||
# ${fileprefix}_${prefix}_${increment}.${extension}
|
||||
#
|
||||
$fcount++;
|
||||
$filename
|
||||
= "$dirname/${basename}_${prefix}_${fcount}${suffix}";
|
||||
say "Writing to: $filename" unless $silent;
|
||||
|
||||
say '-' x 40 unless $silent;
|
||||
|
||||
#
|
||||
# If the file exists and --force is not active, warn and skip.
|
||||
# Otherwise write the file.
|
||||
#
|
||||
if ( -e $filename && !$force ) {
|
||||
warn "File $filename exists; not overwriting\n";
|
||||
}
|
||||
else {
|
||||
#
|
||||
# Write the data to the file in binary format. The URI
|
||||
# module does the conversion so it's already binary.
|
||||
#
|
||||
$fh = path($filename)->openw_raw;
|
||||
print $fh $uri->data;
|
||||
close($fh);
|
||||
}
|
||||
|
||||
#
|
||||
# Update the HTML with a link to the file we created (or that
|
||||
# was already there from an earlier time).
|
||||
#
|
||||
$updates++;
|
||||
$newURL = path($filename)->basename;
|
||||
$element->attr( $attr, $newURL );
|
||||
}
|
||||
}
|
||||
|
||||
} # extract_links loop
|
||||
|
||||
#
|
||||
# Output the changed HTML turning what became standalone back into
|
||||
# a <body> fragment.
|
||||
#
|
||||
if ($updates > 0) {
|
||||
my $body = $tree->look_down( _tag => 'body' );
|
||||
( my $result = $body->as_HTML( undef, ' ', {} ) )
|
||||
=~ s{(^<body[^>]*>|</body>$)}{}gi;
|
||||
|
||||
#$notesfile = path($notesfile)->basename;
|
||||
if ($backup) {
|
||||
if (_backup( $abs_nf, $bsuffix, $backupcount )) {
|
||||
say "$notesfile backed up" unless $silent;
|
||||
}
|
||||
}
|
||||
else {
|
||||
say "$notesfile not backed up" unless $silent;
|
||||
}
|
||||
|
||||
$fh = path($notesfile)->openw;
|
||||
say $fh $result;
|
||||
close($fh);
|
||||
|
||||
say "$updates images converted, $notesfile updated" unless $silent;
|
||||
}
|
||||
else {
|
||||
say "No images found, no changes made to $notesfile" unless $silent;
|
||||
}
|
||||
|
||||
} # $notesfile loop
|
||||
|
||||
exit;
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: _backup
|
||||
# PURPOSE: Given a file, make a backup of it by appending $suffix, and
|
||||
# handle previous backups
|
||||
# PARAMETERS: $filename path of file to backup
|
||||
# $suffix suffix to use, default 'bck'
|
||||
# $limit number of backups to keep
|
||||
# RETURNS: True or false depending on success
|
||||
# DESCRIPTION: Checks that the file exists and returns FALSE if it doesn't.
|
||||
# Rationalises the $limit to avoid it being 0 or less.
|
||||
# THROWS: No exceptions
|
||||
# COMMENTS: None
|
||||
# SEE ALSO: N/A
|
||||
#===============================================================================
|
||||
sub _backup {
|
||||
my ( $filename, $suffix, $limit ) = @_;
|
||||
|
||||
my ( $backupname, $limitname );
|
||||
|
||||
#
|
||||
# Check the target file exists
|
||||
#
|
||||
unless ( -e $filename ) {
|
||||
warn "Unable to find $filename to backup\n";
|
||||
return 0;
|
||||
}
|
||||
|
||||
#
|
||||
# Handle invalid $limit values
|
||||
#
|
||||
$limit = 1 unless ( $limit >= 1 );
|
||||
|
||||
#
|
||||
# Defaults
|
||||
#
|
||||
$suffix = 'bck' unless $suffix;
|
||||
$limitname = "$filename.$suffix.$limit";
|
||||
$backupname = "$filename.$suffix";
|
||||
|
||||
#
|
||||
# Look for existing backups
|
||||
#
|
||||
if ( -e $backupname ) {
|
||||
|
||||
#
|
||||
# If maximum version exists delete it
|
||||
#
|
||||
if ( -e $limitname ) {
|
||||
unlink($limitname);
|
||||
}
|
||||
|
||||
#
|
||||
# Go backwards through the versions incrementing their version numbers
|
||||
#
|
||||
for ( my $vsn = $limit - 1; $vsn > 0; $vsn-- ) {
|
||||
if ( -e "$filename.$suffix.$vsn" ) {
|
||||
move( "$filename.$suffix.$vsn",
|
||||
sprintf( '%s.%s.%s', $filename, $suffix, $vsn + 1 ) );
|
||||
}
|
||||
}
|
||||
|
||||
#
|
||||
# Make $filename.bck into $filename.bck.1
|
||||
#
|
||||
move( "$filename.$suffix", "$filename.$suffix.1" );
|
||||
|
||||
}
|
||||
|
||||
#
|
||||
# Finally save the $filename as $filename.bck
|
||||
#
|
||||
move( $filename, "$filename.$suffix" );
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: coalesce
|
||||
# PURPOSE: To find the first defined argument and return it
|
||||
# PARAMETERS: Arbitrary number of arguments
|
||||
# RETURNS: The first defined argument or undef if there are none
|
||||
# DESCRIPTION: Just a simple way of ensuring an 'undef' value is never
|
||||
# returned when doing so might be a problem.
|
||||
# THROWS: No exceptions
|
||||
# COMMENTS: None
|
||||
# SEE ALSO: N/A
|
||||
#===============================================================================
|
||||
sub coalesce {
|
||||
foreach (@_) {
|
||||
return $_ if defined($_);
|
||||
}
|
||||
return undef; ## no critic
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: _debug
|
||||
# PURPOSE: Prints debug reports
|
||||
# PARAMETERS: $active Boolean: 1 for print, 0 for no print
|
||||
# $messages... Arbitrary list of messages to print
|
||||
# RETURNS: Nothing
|
||||
# DESCRIPTION: Outputs messages if $active is true. It removes any trailing
|
||||
# newline from each one and then adds one in the 'print' to the
|
||||
# caller doesn't have to bother. Prepends each message with 'D>'
|
||||
# to show it's a debug message.
|
||||
# THROWS: No exceptions
|
||||
# COMMENTS: Differs from other functions of the same name
|
||||
# SEE ALSO: N/A
|
||||
#===============================================================================
|
||||
sub _debug {
|
||||
my $active = shift;
|
||||
|
||||
my $message;
|
||||
return unless $active;
|
||||
|
||||
while ($message = shift) {
|
||||
chomp($message);
|
||||
print STDERR "D> $message\n";
|
||||
}
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: Options
|
||||
# PURPOSE: Processes command-line options
|
||||
# PARAMETERS: $optref Hash reference to hold the options
|
||||
# RETURNS: Undef
|
||||
# DESCRIPTION: Process the options we want to offer. See the documentation
|
||||
# for details
|
||||
# THROWS: no exceptions
|
||||
# COMMENTS: none
|
||||
# SEE ALSO: n/a
|
||||
#===============================================================================
|
||||
sub Options {
|
||||
my ($optref) = @_;
|
||||
|
||||
my @options = (
|
||||
"help", "documentation|man", "debug=i", "silent!",
|
||||
"prefix=s", "backup!", "force!",
|
||||
# "config=s",
|
||||
);
|
||||
|
||||
if ( !GetOptions( $optref, @options ) ) {
|
||||
pod2usage(
|
||||
-msg => "$PROG version $VERSION\n",
|
||||
-exitval => 1,
|
||||
-verbose => 0
|
||||
);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
__END__
|
||||
|
||||
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
# Application Documentation
|
||||
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
#{{{
|
||||
|
||||
=head1 NAME
|
||||
|
||||
extract_images - extract embedded images from HTML and save as files
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
This documentation refers to extract_images version 0.0.3
|
||||
|
||||
=head1 USAGE
|
||||
|
||||
extract_images
|
||||
[--help] [--documentation|man] [-debug=N]
|
||||
[--prefix=STRING] [--[no]backup] [--[no]force] [--[no]silent]
|
||||
HTML_file [ [HTML_file_2] [ [HTML_file_3] ... ] ]
|
||||
|
||||
Examples:
|
||||
|
||||
extract_images --prefix=picture_ --backup index.html
|
||||
|
||||
extract_images --silent --nobackup shownotes.html
|
||||
|
||||
=head1 REQUIRED ARGUMENTS
|
||||
|
||||
The script requires the names of one or more HTML files which will be scanned
|
||||
for embedded images.
|
||||
|
||||
=head1 OPTIONS
|
||||
|
||||
Note that all on/off options can be written as B<--silent>, B<--nosilent> or
|
||||
B<--no-silent>. A single hyphen can be used at the start or a pair.
|
||||
|
||||
=over 8
|
||||
|
||||
=item B<--help>
|
||||
|
||||
Reports brief information about how to use the script and exits. To see the
|
||||
full documentation use the option B<--documentation> or B<--man>. Alternatively,
|
||||
to generate a PDF version use the I<pod2pdf> tool from
|
||||
I<http://search.cpan.org/~jonallen/pod2pdf-0.42/bin/pod2pdf>. This can be
|
||||
installed with the cpan tool as App::pod2pdf.
|
||||
|
||||
=item B<--documentation> or B<--man>
|
||||
|
||||
Reports full information about how to use the script and exits. Alternatively,
|
||||
to generate a PDF version use the I<pod2pdf> tool from
|
||||
I<http://search.cpan.org/~jonallen/pod2pdf-0.42/bin/pod2pdf>. This can be
|
||||
installed with the cpan tool as App::pod2pdf.
|
||||
|
||||
=item B<--debug=N>
|
||||
|
||||
Run in debug mode at the level specified by I<N>. Possible values are:
|
||||
|
||||
=over 4
|
||||
|
||||
=item B<0>
|
||||
|
||||
No debugging (the default).
|
||||
|
||||
=item B<1>
|
||||
|
||||
TBA
|
||||
|
||||
=item B<2>
|
||||
|
||||
Displays the internal variables used to store the options.
|
||||
|
||||
=item B<3>
|
||||
|
||||
TBA
|
||||
|
||||
=back
|
||||
|
||||
=item B<--prefix=STRING>
|
||||
|
||||
Since embedded images have no names, when they are written to image files
|
||||
names are generated for them. These names are built from the following
|
||||
components:
|
||||
|
||||
=over 4
|
||||
|
||||
=item B<HTML filename>
|
||||
|
||||
The name of the HTML file without a suffix. So, if the name is 'index.html',
|
||||
the 'index' part will be used.
|
||||
|
||||
=item B<prefix>
|
||||
|
||||
The prefix string provided by this option, or 'image' if not specified.
|
||||
|
||||
=item B<counter>
|
||||
|
||||
The images found in the HTML are counted, starting from 1, and this number is
|
||||
used here.
|
||||
|
||||
=item B<extension>
|
||||
|
||||
The image format extension, taken from the embedded image information.
|
||||
Examples would be 'jpg', 'jpeg' and 'png'.
|
||||
|
||||
=back
|
||||
|
||||
The B<HTML filename>, B<prefix>, B<counter> and B<extension> are joined
|
||||
together to make the file name as follows:
|
||||
|
||||
<HTML filename>_<prefix>_<counter>.<extension>
|
||||
|
||||
So the following run of the script would generate the first file shown below
|
||||
assuming the first embedded picture was in 'PNG' format:
|
||||
|
||||
extract_images --prefix=picture hpr4283.html
|
||||
|
||||
hpr4283_picture_1.png
|
||||
|
||||
=item B<--[no]backup>
|
||||
|
||||
The HTML is modified to leave placeholders referring to any embedded images
|
||||
found as it it processed. The new HTML is written to the original file if
|
||||
B<--nobackup> is specified, or a backup of the original file is made, before
|
||||
writing the new HTML. Making a backup is the default behaviour.
|
||||
|
||||
=item B<--[no]force>
|
||||
|
||||
Images are written to the current directory. If an image already exists, and
|
||||
B<--force> has not been specified, the script will stop processing and exit.
|
||||
The default setting of this option is B<--noforce>.
|
||||
|
||||
=item B<--[no]silent>
|
||||
|
||||
By default the script reports its progress as it processes an HTML file, but
|
||||
this can be turned off by using this option.
|
||||
|
||||
=back
|
||||
|
||||
=head1 DESCRIPTION
|
||||
|
||||
This Perl script B<extract_images> parses HTML looking for embedded images.
|
||||
Such images use a URI with the scheme 'data' followed by image details
|
||||
including the encoded binary contents. See the Wikipedia article
|
||||
https://en.wikipedia.org/wiki/Data_URI_scheme for details.
|
||||
|
||||
When such images are found they are written as files (with generated names),
|
||||
and the HTML is updated with these names replacing the original URI. Further
|
||||
work is likely to be required to build full image links, but the majority of
|
||||
work will have been done.
|
||||
|
||||
The script will not overwrite image files unless the B<--force> option is
|
||||
used, but will overwrite the original HTML file unless B<--backup> is
|
||||
specified.
|
||||
|
||||
By default details of progress are written to standard output, but this can be
|
||||
preveneted by using the B<--silent> option.
|
||||
|
||||
=head2 GENERATED IMAGE FILE NAMES
|
||||
|
||||
These names are built from various elements:
|
||||
|
||||
<HTML filename>_<prefix>_<counter>.<extension>
|
||||
|
||||
Where <HTML filename> is the base name of the HTML input file, <prefix> is the
|
||||
string provided with the B<--prefix=STRING> option (or B<image> by default),
|
||||
<counter> is a count of image files found during the scan of the HTML, and
|
||||
<extension> is the appropriate file extension for the type of image being
|
||||
converted.
|
||||
|
||||
See the option B<--prefix=STRING> for details of file name generation.
|
||||
|
||||
=head1 DIAGNOSTICS
|
||||
|
||||
=over 4
|
||||
|
||||
=item B<Unable to find ...>
|
||||
|
||||
Type: warning
|
||||
|
||||
The script is attempting to find a file presented to it as an argument, but
|
||||
cannot. It will skip to the next argument and continue.
|
||||
|
||||
=item B<File ... is not HTML>
|
||||
|
||||
Type: warning
|
||||
|
||||
The script is checking files presented to it as arguments. The named file has
|
||||
been checked to see if it contains HTML, and it appears it does not. It will
|
||||
skip to the next argument and continue.
|
||||
|
||||
=item B<Unable to open ...>
|
||||
|
||||
Type: fatal
|
||||
|
||||
The script is attempting to open a file presented to it as an argument. Its
|
||||
existence has already been checked but it cannot be opened, possibly due to
|
||||
a permissions issue.
|
||||
|
||||
=item B<HTML::TreeBuilder failed to parse notes: ...>
|
||||
|
||||
Type: fatal
|
||||
|
||||
The script has attempted to parse the HTML in a file presented to it. This has
|
||||
failed. The error message includes a report from the module used to do this.
|
||||
|
||||
=item B<File ... exists; not overwriting>
|
||||
|
||||
Type: warning
|
||||
|
||||
The script is attempting to write an image file copied from the HTML, but has
|
||||
found it already exists. Using the option B<--force> would cause it to be
|
||||
overwritten, but this option is not enabled.
|
||||
|
||||
The script will not write the file, however, it will still modify the HTML to
|
||||
reference the existing image file.
|
||||
|
||||
=item B<Unable to find ... to backup>
|
||||
|
||||
Type: warning
|
||||
|
||||
The script is attempting to make a backup of one of the HTML file arguments,
|
||||
but is unable to find it. Since this is a rather unusual circumstance (has the
|
||||
file been deleted?), it is assumed it can be regenerated if needed, so the
|
||||
writing of the modified HTML is continued.
|
||||
|
||||
=head1 CONFIGURATION AND ENVIRONMENT
|
||||
|
||||
TBA
|
||||
|
||||
=head1 DEPENDENCIES
|
||||
|
||||
Data::Dumper
|
||||
File::Copy
|
||||
Getopt::Long
|
||||
HTML::TreeBuilder
|
||||
MIME::Types
|
||||
Path::Tiny
|
||||
Pod::Usage
|
||||
URI
|
||||
|
||||
=head1 BUGS AND LIMITATIONS
|
||||
|
||||
There are no known bugs in this module.
|
||||
Please report problems to Dave Morriss (Dave.Morriss@gmail.com)
|
||||
Patches are welcome.
|
||||
|
||||
=head1 AUTHOR
|
||||
|
||||
Dave Morriss (Dave.Morriss@gmail.com)
|
||||
|
||||
=head1 LICENCE AND COPYRIGHT
|
||||
|
||||
Copyright (c) 2024 Dave Morriss (Dave.Morriss@gmail.com).
|
||||
All rights reserved.
|
||||
|
||||
This module is free software; you can redistribute it and/or
|
||||
modify it under the same terms as Perl itself. See perldoc perlartistic.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
||||
|
||||
=cut
|
||||
|
||||
#}}}
|
||||
|
||||
# [zo to open fold, zc to close]
|
||||
|
||||
# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker
|
||||
|
@ -361,7 +361,8 @@ try {
|
||||
$content = decode_json($json_text);
|
||||
}
|
||||
catch ($e) {
|
||||
die colored( "Failed to decode the JSON in $infile", 'red' ) . "\n"
|
||||
warn colored( "Failed to decode the JSON in $infile", 'red' ) . "\n";
|
||||
die "Error was: $e\n";
|
||||
}
|
||||
|
||||
$log->info( $showno, "[$VERSION] Processing $infile" );
|
||||
|
@ -1 +0,0 @@
|
||||
/home/cendjm/HPR/Database/query2csv
|
137
Show_Submission/query2csv
Executable file
137
Show_Submission/query2csv
Executable file
@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env perl
|
||||
#===============================================================================
|
||||
#
|
||||
# FILE: query2csv
|
||||
#
|
||||
# USAGE: ./query2csv query
|
||||
#
|
||||
# DESCRIPTION: Runs a query given as the only argument. Caution is needed
|
||||
# since *any* query will be run. The result of the query is
|
||||
# output in CSV form on STDOUT. The CSV is always quoted to
|
||||
# cater for the more simplistic consumers.
|
||||
#
|
||||
# OPTIONS: ---
|
||||
# REQUIREMENTS: ---
|
||||
# BUGS: ---
|
||||
# NOTES: Had to revert to MySQL because of a problem with DBD::MariaDB
|
||||
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
||||
# VERSION: 0.0.2
|
||||
# CREATED: 2015-07-11 15:53:01
|
||||
# REVISION: 2022-02-16 23:17:16
|
||||
#
|
||||
#===============================================================================
|
||||
|
||||
use 5.010;
|
||||
use strict;
|
||||
use warnings;
|
||||
use utf8;
|
||||
|
||||
use Config::General;
|
||||
use Text::CSV_XS;
|
||||
use DBI;
|
||||
|
||||
use Data::Dumper;
|
||||
|
||||
#
|
||||
# Version number (manually incremented)
|
||||
#
|
||||
our $VERSION = '0.0.2';
|
||||
|
||||
#
|
||||
# Script and directory names
|
||||
#
|
||||
( my $PROG = $0 ) =~ s|.*/||mx;
|
||||
( my $DIR = $0 ) =~ s|/?[^/]*$||mx;
|
||||
$DIR = '.' unless $DIR;
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Declarations
|
||||
#-------------------------------------------------------------------------------
|
||||
#
|
||||
# Constants and other declarations
|
||||
#
|
||||
my $basedir = "$ENV{HOME}/HPR/Database";
|
||||
my $configfile = "$basedir/.hpr_livedb.cfg";
|
||||
|
||||
my ( $dbh, $sth1, $aref1 );
|
||||
my ( $query, $csv );
|
||||
|
||||
#
|
||||
# Enable Unicode mode
|
||||
#
|
||||
binmode STDOUT, ":encoding(UTF-8)";
|
||||
binmode STDERR, ":encoding(UTF-8)";
|
||||
|
||||
#
|
||||
# Load database configuration data
|
||||
#
|
||||
my $conf = Config::General->new(
|
||||
-ConfigFile => $configfile,
|
||||
-InterPolateVars => 1,
|
||||
-ExtendedAccess => 1
|
||||
);
|
||||
my %config = $conf->getall();
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Options and arguments
|
||||
#-------------------------------------------------------------------------------
|
||||
$query = shift;
|
||||
die "Usage: $PROG query\n" unless $query;
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Connect to the database
|
||||
#-------------------------------------------------------------------------------
|
||||
my $dbhost = $config{database}->{host} // '127.0.0.1';
|
||||
my $dbport = $config{database}->{port} // 3306;
|
||||
my $dbname = $config{database}->{name};
|
||||
my $dbuser = $config{database}->{user};
|
||||
my $dbpwd = $config{database}->{password};
|
||||
#$dbh = DBI->connect( "DBI:MariaDB:host=$dbhost;port=$dbport;database=$dbname",
|
||||
# $dbuser, $dbpwd, { AutoCommit => 1 } )
|
||||
# or die $DBI::errstr;
|
||||
|
||||
$dbh = DBI->connect( "dbi:mysql:host=$dbhost;port=$dbport;database=$dbname",
|
||||
$dbuser, $dbpwd, { AutoCommit => 1 } )
|
||||
or die $DBI::errstr;
|
||||
|
||||
#
|
||||
# Enable client-side UTF8
|
||||
#
|
||||
$dbh->{mysql_enable_utf8} = 1;
|
||||
|
||||
#
|
||||
# Set up the query
|
||||
#
|
||||
$sth1 = $dbh->prepare($query) or die $DBI::errstr;
|
||||
if ( $dbh->err ) {
|
||||
warn $dbh->errstr;
|
||||
}
|
||||
|
||||
#
|
||||
# Perform the query
|
||||
#
|
||||
$sth1->execute;
|
||||
if ( $dbh->err ) {
|
||||
warn $dbh->errstr;
|
||||
}
|
||||
|
||||
#
|
||||
# Prepare to make CSV. Not sure if always quoting is the best idea though
|
||||
#
|
||||
$csv = Text::CSV_XS->new(
|
||||
# { always_quote => 1 }
|
||||
);
|
||||
|
||||
#
|
||||
# Loop through the returned rows making and printing CSV. Each row is returned
|
||||
# as an arrayref to make it easy to join everything.
|
||||
#
|
||||
while ( $aref1 = $sth1->fetchrow_arrayref ) {
|
||||
$csv->combine(@$aref1);
|
||||
print $csv->string(), "\n";
|
||||
}
|
||||
|
||||
exit;
|
||||
|
||||
# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker
|
||||
|
@ -1 +0,0 @@
|
||||
/home/cendjm/HPR/Database/query2json
|
134
Show_Submission/query2json
Executable file
134
Show_Submission/query2json
Executable file
@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env perl
|
||||
#===============================================================================
|
||||
#
|
||||
# FILE: query2json
|
||||
#
|
||||
# USAGE: ./query2json query
|
||||
#
|
||||
# DESCRIPTION: Runs a query given as the only argument. Caution is needed
|
||||
# since *any* query will be run. The result of the query is
|
||||
# output in JSON form on STDOUT.
|
||||
#
|
||||
# OPTIONS: ---
|
||||
# REQUIREMENTS: ---
|
||||
# BUGS: ---
|
||||
# NOTES: Had to revert to MySQL because of a problem with DBD::MariaDB
|
||||
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
||||
# VERSION: 0.0.2
|
||||
# CREATED: 2021-06-18 13:24:49
|
||||
# REVISION: 2023-01-05 16:17:24
|
||||
#
|
||||
#===============================================================================
|
||||
|
||||
use 5.010;
|
||||
use strict;
|
||||
use warnings;
|
||||
use utf8;
|
||||
|
||||
use Config::General;
|
||||
use JSON;
|
||||
use DBI;
|
||||
|
||||
use Data::Dumper;
|
||||
|
||||
#
|
||||
# Version number (manually incremented)
|
||||
#
|
||||
our $VERSION = '0.0.2';
|
||||
|
||||
#
|
||||
# Script and directory names
|
||||
#
|
||||
( my $PROG = $0 ) =~ s|.*/||mx;
|
||||
( my $DIR = $0 ) =~ s|/?[^/]*$||mx;
|
||||
$DIR = '.' unless $DIR;
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Declarations
|
||||
#-------------------------------------------------------------------------------
|
||||
#
|
||||
# Constants and other declarations
|
||||
#
|
||||
my $basedir = "$ENV{HOME}/HPR/Database";
|
||||
my $configfile = "$basedir/.hpr_livedb.cfg";
|
||||
|
||||
my ( $dbh, $sth1, $aref1 );
|
||||
my ( $query, $result, $json );
|
||||
|
||||
#
|
||||
# Enable Unicode mode
|
||||
#
|
||||
binmode STDOUT, ":encoding(UTF-8)";
|
||||
binmode STDERR, ":encoding(UTF-8)";
|
||||
|
||||
#
|
||||
# Load database configuration data
|
||||
#
|
||||
my $conf = Config::General->new(
|
||||
-ConfigFile => $configfile,
|
||||
-InterPolateVars => 1,
|
||||
-ExtendedAccess => 1
|
||||
);
|
||||
my %config = $conf->getall();
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Options and arguments
|
||||
#-------------------------------------------------------------------------------
|
||||
$query = shift;
|
||||
die "Usage: $PROG query\n" unless $query;
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Connect to the database
|
||||
#-------------------------------------------------------------------------------
|
||||
my $dbhost = $config{database}->{host} // '127.0.0.1';
|
||||
my $dbport = $config{database}->{port} // 3306;
|
||||
my $dbname = $config{database}->{name};
|
||||
my $dbuser = $config{database}->{user};
|
||||
my $dbpwd = $config{database}->{password};
|
||||
#$dbh = DBI->connect( "DBI:MariaDB:host=$dbhost;port=$dbport;database=$dbname",
|
||||
# $dbuser, $dbpwd, { AutoCommit => 1 } )
|
||||
# or die $DBI::errstr;
|
||||
|
||||
$dbh = DBI->connect( "dbi:mysql:host=$dbhost;port=$dbport;database=$dbname",
|
||||
$dbuser, $dbpwd, { AutoCommit => 1 } )
|
||||
or die $DBI::errstr;
|
||||
|
||||
#
|
||||
# Enable client-side UTF8
|
||||
#
|
||||
$dbh->{mysql_enable_utf8} = 1;
|
||||
|
||||
#
|
||||
# Set up the query
|
||||
#
|
||||
$sth1 = $dbh->prepare($query) or die $DBI::errstr;
|
||||
if ( $dbh->err ) {
|
||||
warn $dbh->errstr;
|
||||
}
|
||||
|
||||
#
|
||||
# Perform the query
|
||||
#
|
||||
$sth1->execute;
|
||||
if ( $dbh->err ) {
|
||||
warn $dbh->errstr;
|
||||
}
|
||||
|
||||
#
|
||||
# Grab everything as an arrayref of hashrefs
|
||||
#
|
||||
$result = $sth1->fetchall_arrayref( {} );
|
||||
|
||||
#
|
||||
# Prepare for JSON, forcing object key sorting (expensive)
|
||||
#
|
||||
$json = JSON->new->utf8->canonical;
|
||||
|
||||
#
|
||||
# Encode the Perl structure to JSON
|
||||
#
|
||||
print $json->encode($result), "\n";
|
||||
|
||||
exit;
|
||||
|
||||
# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker
|
BIN
hpr_tags/fix_tags.bin
Executable file
BIN
hpr_tags/fix_tags.bin
Executable file
Binary file not shown.
BIN
workflow/fix_tags_20250201170019.bin
Normal file
BIN
workflow/fix_tags_20250201170019.bin
Normal file
Binary file not shown.
@ -1,44 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/
|
||||
#============================================================
|
||||
|
||||
git_dir="$HOME/tmp/hpr/hpr_generator/sourcecode"
|
||||
if [ ! -d "${git_dir}/.git" ]
|
||||
then
|
||||
git clone gitea@repo.anhonesthost.net:HPR/hpr_generator.git "${git_dir}"
|
||||
fi
|
||||
|
||||
cd "${git_dir}"
|
||||
|
||||
git pull
|
||||
|
||||
ssh hpr -t "ls -al /home/hpr/www/hpr.sql;md5sum /home/hpr/www/hpr.sql"
|
||||
ssh hpr -t "/home/hpr/bin/hpr_db_backup.bash"
|
||||
ssh hpr -t "ls -al /home/hpr/www/hpr.sql;md5sum /home/hpr/www/hpr.sql"
|
||||
|
||||
./utils/update-hpr-db.sh
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
echo 'Terminating...' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
./site-generator --all --verbose
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
echo 'Terminating...' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rsync -av --partial --progress "${git_dir}/public_html/" hpr:/home/hpr/public_html/
|
||||
rsync -av --partial --progress "${git_dir}/public_html/" hobbypublicradio.org:hobbypublicradio.org/
|
||||
|
||||
cd $HOME/sourcecode/hpr/hpr_hub/
|
||||
git pull
|
||||
cd $HOME/sourcecode/hpr/hpr_hub/sql
|
||||
split --hex-suffixes --lines=1000 --additional-suffix=.sql hpr.sql hpr-db-part-
|
||||
cd $HOME/sourcecode/hpr/hpr_hub/
|
||||
git add $HOME/sourcecode/hpr/hpr_hub/sql/hpr*sql
|
||||
git commit -m "$(\date -u +%Y-%m-%d_%H-%M-%SZ_%A ) database changed"
|
||||
git push
|
||||
#xdg-open https://hackerpublicradio.org/
|
@ -1,685 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/
|
||||
#============================================================
|
||||
|
||||
PATH=$PATH:$HOME/sourcecode/hpr/hpr_hub/bin/
|
||||
source $HOME/tmp/pip3.9/bin/activate
|
||||
# https://hub.tcno.co/ai/whisper/install/
|
||||
|
||||
TEMP_DIR="/var/tmp/"
|
||||
CHANNELS="2"
|
||||
FIXAUDIO="1"
|
||||
ARTIST="EMPTY"
|
||||
TITLE="EMPTY"
|
||||
YEAR="EMPTY"
|
||||
SLOT="EMPTY"
|
||||
basedir="/var/IA"
|
||||
upload_dir="${basedir}/uploads"
|
||||
theme="${basedir}/theme.wav"
|
||||
outro="${basedir}/2022-03-07-outro.wav"
|
||||
outro_c2="${basedir}/2022-03-07-outro_c2.wav"
|
||||
ttsserver="http://localhost:5500"
|
||||
processing_dir="$HOME/tmp/hpr/processing"
|
||||
git_image_dir="$HOME/sourcecode/hpr/HPR_Public_Code/www/images/hosts"
|
||||
acceptable_duration_difference="2" # Seconds
|
||||
thedate=$(/usr/bin/date -u +%Y-%m-%d_%H-%M-%SZ_%A)
|
||||
echo "Processing the next HPR Show in the queue"
|
||||
|
||||
|
||||
if [ $( curl -s -o /dev/null -w "%{http_code}" -X 'GET' "${ttsserver}/api/voices" -H 'accept: */*' ) != "200" ]
|
||||
then
|
||||
echo "Please start the tts-server \"podman run -it -p 5500:5500 synesthesiam/opentts:en\""
|
||||
exit
|
||||
fi
|
||||
|
||||
function abs_diff {
|
||||
echo $(($1 >= $2 ? $1 - $2 : $2 - $1))
|
||||
}
|
||||
|
||||
function create_tts_summary {
|
||||
HPR_summary="$( cat "hpr${ep_num}_summary.txt" )"
|
||||
echo "INFO: Converting text \"${HPR_summary}\" to speech."
|
||||
curl -X 'GET' -G --data-urlencode "voice=coqui-tts:en_ljspeech" --data-urlencode "text=${HPR_summary}" --data-urlencode "vocoder=high" --data-urlencode "denoiserStrength=0.03" --data-urlencode "cache=false" ${ttsserver}/api/tts -H 'accept: */*' --output ~hpr${ep_num}_summary.wav
|
||||
}
|
||||
|
||||
###################
|
||||
# Locate media directory
|
||||
#
|
||||
#
|
||||
show_type=""
|
||||
if [[ -f "${1}" && -n "${2}" ]]
|
||||
then
|
||||
echo "DEBUG: Show type is local" 1>&2
|
||||
show_type="local"
|
||||
mediafile="${1}"
|
||||
media_dir="$( dirname "${mediafile}" )"
|
||||
ep_num="${2}"
|
||||
echo "The duration is \"$( \date -ud "1970-01-01 $( ffprobe -i "${mediafile}" 2>&1| awk -F ': |, ' '/Duration:/ { print $2 }' )" +%s )\"."
|
||||
else
|
||||
echo "DEBUG: Show type is remote" 1>&2
|
||||
show_type="remote"
|
||||
response=$( curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | \
|
||||
grep 'SHOW_POSTED' | \
|
||||
head -1 | \
|
||||
sed 's/,/ /g' )
|
||||
|
||||
echo "DEBUG: Getting server response" 1>&2
|
||||
if [ -z "${response}" ]
|
||||
then
|
||||
echo "INFO: There appear to be no more shows with the status \"SHOW_POSTED\"."
|
||||
echo "Getting a list of all the reservations."
|
||||
curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
timestamp_epoc="$( echo ${response} | awk '{print $1}' )"
|
||||
ep_num="$( echo ${response} | awk '{print $2}' )"
|
||||
ep_date="$( echo ${response} | awk '{print $3}' )"
|
||||
key="$( echo ${response} | awk '{print $4}' )"
|
||||
status="$( echo ${response} | awk '{print $5}' )"
|
||||
email="$( echo ${response} | awk '{print $6}' )"
|
||||
|
||||
#source_dir="hpr:/home/hpr/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
|
||||
dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
|
||||
media_dir="${processing_dir}/${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
|
||||
|
||||
fi
|
||||
|
||||
upload_dir_ep_num="${upload_dir}/hpr${ep_num}"
|
||||
mkdir -p "${upload_dir_ep_num}" 2>/dev/null
|
||||
if [ ! -d "${upload_dir_ep_num}" ]
|
||||
then
|
||||
echo "DEBUG: Missing directory \"upload_dir_ep_num\" \"${upload_dir}\", /hpr and \"${ep_num}\"" 1>&2
|
||||
exit
|
||||
fi
|
||||
|
||||
echo "DEBUG: Checking variables" 1>&2
|
||||
if [ -z "${show_type}" ]
|
||||
then
|
||||
echo "sorry, variable \"show_type\" is not set."
|
||||
exit
|
||||
fi
|
||||
|
||||
if [ -z "${media_dir}" ]
|
||||
then
|
||||
echo "sorry, variable \"media_dir\" is not set."
|
||||
exit
|
||||
fi
|
||||
|
||||
if [ ! -d "${media_dir}" ]
|
||||
then
|
||||
echo "sorry, directory \"media_dir: ${media_dir}\" does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo detox -v "${media_dir}/"
|
||||
detox -vr "${media_dir}/"
|
||||
|
||||
echo "DEBUG: Using media directory as \"${media_dir}\""
|
||||
|
||||
if [[ "$( find "${media_dir}" \( -iname "hpr${ep_num}_sandwitch.wav" -o -iname "hpr${ep_num}.mp3" -o -iname "hpr${ep_num}.ogg" -o -iname "hpr${ep_num}.spx" -o -iname "hpr${ep_num}_summary.txt" -o -iname "hpr${ep_num}_summary.wav" -o -iname "hpr${ep_num}_sandwitch.wav" -o -iname "*_mez.wav" -o -iname "*_sox_norm.wav" -o -iname "*_mezzanine.wav" -o -iname "*_tmp.pcm" -o -iname "hpr${ep_num}_intro*.wav" -o -iname "~hpr${ep_num}_summary.wav" -o -iname "~~hpr${ep_num}_summary.wav" -o -iname "*_tmp.log" -o -iname "silence.wav" -o -iname "hpr${ep_num}_norm.wav" \) | wc -l )" -ne 0 ]]
|
||||
then
|
||||
echo "Files for this episode already exist."
|
||||
find "${media_dir}" \( -iname "hpr${ep_num}_sandwitch.wav" -o -iname "hpr${ep_num}.mp3" -o -iname "hpr${ep_num}.ogg" -o -iname "hpr${ep_num}.spx" -o -iname "hpr${ep_num}_summary.txt" -o -iname "hpr${ep_num}_summary.wav" -o -iname "hpr${ep_num}_sandwitch.wav" -o -iname "*_mez.wav" -o -iname "*_sox_norm.wav" -o -iname "*_mezzanine.wav" -o -iname "*_tmp.pcm" -o -iname "hpr${ep_num}_intro*.wav" -o -iname "~hpr${ep_num}_summary.wav" -o -iname "~~hpr${ep_num}_summary.wav" -o -iname "*_tmp.log" -o -iname "silence.wav" -o -iname "hpr${ep_num}_norm.wav" \)
|
||||
read -p "Shall I remove them ? (N|y) ? " -n 1 -r
|
||||
echo # (optional) move to a new line
|
||||
if [[ ! $REPLY =~ ^[yY]$ ]]
|
||||
then
|
||||
echo "skipping...."
|
||||
exit
|
||||
else
|
||||
echo "Removing old files ...."
|
||||
find "${media_dir}" \( -iname "hpr${ep_num}_sandwitch.wav" -o -iname "hpr${ep_num}.mp3" -o -iname "hpr${ep_num}.ogg" -o -iname "hpr${ep_num}.spx" -o -iname "hpr${ep_num}_summary.txt" -o -iname "hpr${ep_num}_summary.wav" -o -iname "hpr${ep_num}_sandwitch.wav" -o -iname "*_mez.wav" -o -iname "*_sox_norm.wav" -o -iname "*_mezzanine.wav" -o -iname "*_tmp.pcm" -o -iname "hpr${ep_num}_intro*.wav" -o -iname "~hpr${ep_num}_summary.wav" -o -iname "~~hpr${ep_num}_summary.wav" -o -iname "*_tmp.log" -o -iname "silence.wav" -o -iname "hpr${ep_num}_norm.wav" \) -delete -print
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
#####################################################
|
||||
# Process media
|
||||
|
||||
media=$( find "${media_dir}" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' )
|
||||
if [ -z "${media}" ]
|
||||
then
|
||||
echo "ERROR: Can't find any media in \"${media_dir}/\""
|
||||
find "${media_dir}/" -type f
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mediafile=""
|
||||
|
||||
echo "Found more than one media file. Please select which one to use ?"
|
||||
if [ "$( echo "${media}" | wc -l )" -ne 1 ]
|
||||
then
|
||||
select this_media in $( echo "${media}" )
|
||||
do
|
||||
echo "INFO: You selected \"${this_media}\"."
|
||||
ls -al "${this_media}"
|
||||
mediafile="${this_media}"
|
||||
break
|
||||
done
|
||||
else
|
||||
echo "INFO: Selecting media as \"${media}\"."
|
||||
mediafile="${media}"
|
||||
fi
|
||||
|
||||
# extract file name and extension
|
||||
fname="$( basename "${mediafile%.*}" )"
|
||||
ext="${mediafile/*./}"
|
||||
|
||||
cd "${media_dir}/"
|
||||
pwd
|
||||
|
||||
echo "INFO: Processing hpr${ep_num} from ${email}"
|
||||
echo "INFO: Working directory is \"${media_dir}/\""
|
||||
echo ""
|
||||
|
||||
|
||||
if [ -z "${mediafile}" ]
|
||||
then
|
||||
echo "sorry, variable \"mediafile\" is not set."
|
||||
exit
|
||||
fi
|
||||
|
||||
if [ -z "${fname}" ]
|
||||
then
|
||||
echo "sorry, variable \"fname\" is not set."
|
||||
exit
|
||||
fi
|
||||
|
||||
if [ -z "${ext}" ]
|
||||
then
|
||||
echo "sorry, variable \"ext\" is not set."
|
||||
exit
|
||||
fi
|
||||
|
||||
if [ ! -f "${mediafile}" ]
|
||||
then
|
||||
echo "sorry, media file \"${mediafile}\" does not exist"
|
||||
exit
|
||||
fi
|
||||
|
||||
echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
|
||||
echo "File information for \"${mediafile}\"" | tee -a ${fname}_tmp.log
|
||||
ffprobe ${mediafile} 2>&1 | grep Audio:
|
||||
mediainfo ${mediafile}
|
||||
audio2image.bash ${mediafile}
|
||||
xdg-open ${mediafile%.*}.png >/dev/null 2>&1 &
|
||||
unset REPLY
|
||||
until [[ $REPLY =~ ^[yYnN]$ ]]
|
||||
do
|
||||
read -p "Source Waveform look ok ? (N|y) ? " -n 1 -r
|
||||
echo ""
|
||||
done
|
||||
|
||||
if [[ ! $REPLY =~ ^[yY]$ ]]
|
||||
then
|
||||
echo "skipping.... $REPLY"
|
||||
exit
|
||||
fi
|
||||
|
||||
|
||||
re='^[0-9]+$'
|
||||
if ! [[ $ep_num =~ $re ]]
|
||||
then
|
||||
echo "error: episode \"${ep_num}\" is not a number"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "${outro}" ]
|
||||
then
|
||||
echo "sorry, file \"${outro}\" does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "${outro_c2}" ]
|
||||
then
|
||||
echo "sorry, file \"${outro_c2}\" does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
if [ ! -f "${theme}" ]
|
||||
then
|
||||
echo "sorry, file \"${theme}\" does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -r "${mediafile}" ]
|
||||
then
|
||||
echo "sorry, media file \"${mediafile}\" is not readable"
|
||||
exit
|
||||
fi
|
||||
|
||||
if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ]
|
||||
then
|
||||
echo "sorry, media file \"${mediafile}\" has no audio track"
|
||||
exit
|
||||
fi
|
||||
|
||||
xdg-open "https://hackerpublicradio.org/eps/hpr${ep_num}/index.html" >/dev/null 2>&1 &
|
||||
|
||||
mpv -vo=null "${mediafile}"
|
||||
|
||||
unset REPLY
|
||||
until [[ $REPLY =~ ^[yYnN]$ ]]
|
||||
do
|
||||
read -p "Is the audio ok (n|Y) ? " -n 1 -r
|
||||
echo ""
|
||||
done
|
||||
|
||||
if [[ ! $REPLY =~ ^[yY]$ ]]
|
||||
then
|
||||
echo "skipping.... $REPLY"
|
||||
exit
|
||||
fi
|
||||
|
||||
echo "--------------------------------------------------------------------------------"
|
||||
echo "Geting metadata for hpr${ep_num}"
|
||||
|
||||
while read -r line
|
||||
do
|
||||
field=$(echo $line | awk -F ':' '{print $1}')
|
||||
case $field in
|
||||
"HPR_summary")
|
||||
HPR_summary=$(echo $line | grep "HPR_summary: " | cut -c 14- )
|
||||
continue
|
||||
;;
|
||||
"HPR_album")
|
||||
HPR_album=$(echo $line | grep "HPR_album: " | cut -c 12- )
|
||||
continue
|
||||
;;
|
||||
"HPR_artist")
|
||||
HPR_artist=$(echo $line | grep "HPR_artist: " | cut -c 13- )
|
||||
continue
|
||||
;;
|
||||
"HPR_comment")
|
||||
HPR_comment=$(echo $line | grep "HPR_comment: " | cut -c 14- )
|
||||
continue
|
||||
;;
|
||||
"HPR_genre")
|
||||
HPR_genre=$(echo $line | grep "HPR_genre: " | cut -c 12- )
|
||||
continue
|
||||
;;
|
||||
"HPR_title")
|
||||
HPR_title=$(echo $line | grep "HPR_title: " | cut -c 12- )
|
||||
continue
|
||||
;;
|
||||
"HPR_track")
|
||||
HPR_track=$(echo $line | grep "HPR_track: " | cut -c 12- )
|
||||
continue
|
||||
;;
|
||||
"HPR_year")
|
||||
HPR_year=$(echo $line | grep "HPR_year: " | cut -c 11- )
|
||||
continue
|
||||
;;
|
||||
"HPR_duration")
|
||||
HPR_duration=$(echo $line | grep "HPR_duration: " | cut -c 15- )
|
||||
continue
|
||||
;;
|
||||
"HPR_explicit")
|
||||
HPR_explicit=$(echo $line | grep "HPR_explicit: " | cut -c 15- )
|
||||
continue
|
||||
;;
|
||||
"HPR_license")
|
||||
HPR_license=$(echo $line | grep "HPR_license: " | cut -c 14- )
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
done < <( curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num}" )
|
||||
|
||||
if [[ -z "$HPR_album" || -z "$HPR_artist" || -z "$HPR_comment" || -z "$HPR_genre" || -z "$HPR_title" || -z "$HPR_track" || -z "$HPR_year" || -z "$HPR_summary" || -z "$HPR_duration" || -z "$HPR_explicit" || -z "$HPR_license" ]]
|
||||
then
|
||||
echo "Could not find information on ${ep_num}. Has the show been posted ?"
|
||||
exit;
|
||||
fi
|
||||
echo "--------------------------------------------------------------------------------"
|
||||
echo "album : $HPR_album"
|
||||
echo "artist : $HPR_artist"
|
||||
echo "comment : $HPR_comment"
|
||||
echo "genre : $HPR_genre"
|
||||
echo "title : $HPR_title"
|
||||
echo "track : $HPR_track"
|
||||
echo "year : $HPR_year"
|
||||
echo "summary : $HPR_summary"
|
||||
echo "duration : $HPR_duration"
|
||||
echo "explicit : $HPR_explicit"
|
||||
echo "license : $HPR_license"
|
||||
echo "media_dir : ${media_dir}"
|
||||
echo "mediafile : ${mediafile}"
|
||||
echo "fname : ${fname}"
|
||||
echo "ext : ${ext}"
|
||||
|
||||
if [[ $HPR_duration == "0" ]]
|
||||
then
|
||||
echo "The duration is set to 0. Please update the show with the correct time."
|
||||
exit;
|
||||
fi
|
||||
|
||||
#============================================================
|
||||
# Preproc`s the source file
|
||||
|
||||
echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
|
||||
echo "Prepare mezzanine file" | tee -a ${fname}_tmp.log
|
||||
ffmpeg -y -i ${mediafile} -ar 44100 -ac $CHANNELS ${fname}_mezzanine.wav >> ${fname}_tmp.log 2>&1
|
||||
|
||||
echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
|
||||
echo "Add HPR Branding" | tee -a ${fname}_tmp.log
|
||||
|
||||
echo "Creating the summary" | tee -a ${fname}_tmp.log
|
||||
#echo "$HPR_summary" - | text2wave -F 44100 - -o hpr${ep_num}_summary.wav #festival --tts
|
||||
#echo "$HPR_summary" - | espeak -w ~hpr${ep_num}_summary.wav
|
||||
|
||||
echo "$HPR_summary" > "hpr${ep_num}_summary.txt"
|
||||
|
||||
create_tts_summary_ok="not-ok"
|
||||
|
||||
while [ "${create_tts_summary_ok}" != "OK" ]
|
||||
do
|
||||
create_tts_summary
|
||||
|
||||
xdg-open "hpr${ep_num}_summary.txt" 2>&1 &
|
||||
mpv --speed=1.8 ~hpr${ep_num}_summary.wav
|
||||
|
||||
read -p "Is the text to speech correct (y|N) ? " -n 1 -r
|
||||
echo # (optional) move to a new line
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]
|
||||
then
|
||||
create_tts_summary_ok="OK"
|
||||
else
|
||||
echo "WARN: Please correct the text and try again."
|
||||
inotifywait --event modify "hpr${ep_num}_summary.txt"
|
||||
fi
|
||||
done
|
||||
echo "INFO: TTS complete."
|
||||
|
||||
ffmpeg -y -i ~hpr${ep_num}_summary.wav -ar 44100 ~~hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1
|
||||
|
||||
sox -V2 -n -r 44100 -c 1 silence.wav trim 0.0 6.0 >> ${fname}_tmp.log 2>&1
|
||||
|
||||
sox -V2 silence.wav ~~hpr${ep_num}_summary.wav hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1
|
||||
|
||||
echo "Adding the theme" | tee -a ${fname}_tmp.log
|
||||
sox -V2 -m "hpr${ep_num}_summary.wav" "${theme}" "hpr${ep_num}_intro.wav" >> ${fname}_tmp.log 2>&1
|
||||
|
||||
|
||||
if [ ${CHANNELS} -eq "2" ]
|
||||
then
|
||||
echo "Converting mono to Stero" | tee -a ${fname}_tmp.log
|
||||
ffmpeg -y -i "hpr${ep_num}_intro.wav" -ac 2 "hpr${ep_num}_intro_c2.wav"
|
||||
echo "Creating the sandwitch: \"hpr${ep_num}_intro_c2.wav\" \"${fname}_mezzanine.wav\" \"${outro}\" \"hpr${ep_num}_sandwitch.wav\" " | tee -a ${fname}_tmp.log
|
||||
sox -V2 "hpr${ep_num}_intro_c2.wav" "${fname}_mezzanine.wav" "${outro_c2}" "hpr${ep_num}_sandwitch.wav" >> ${fname}_tmp.log 2>&1
|
||||
else
|
||||
echo "Creating the sandwitch: \"hpr${ep_num}_intro.wav\" \"${fname}_mezzanine.wav\" \"${outro}\" \"hpr${ep_num}_sandwitch.wav\" " | tee -a ${fname}_tmp.log
|
||||
sox -V2 "hpr${ep_num}_intro.wav" "${fname}_mezzanine.wav" "${outro}" "hpr${ep_num}_sandwitch.wav" >> ${fname}_tmp.log 2>&1
|
||||
fi
|
||||
|
||||
|
||||
|
||||
echo "Normalizing the wav files" | tee -a ${fname}_tmp.log
|
||||
#sox --temp "${TEMP_DIR}" --norm hpr${ep_num}_sandwitch.wav hpr${ep_num}_norm.wav
|
||||
#ffmpeg -y -i hpr${ep_num}_sandwitch.wav -filter:a "dynaudnorm=p=0.9:s=5" hpr${ep_num}_norm.wav >> ${fname}_tmp.log 2>&1
|
||||
ffmpeg -y -i hpr${ep_num}.wav -af loudnorm=I=-16:LRA=11:TP=-1.5 hpr${ep_num}_norm.wav >> ${fname}_tmp.log 2>&1
|
||||
|
||||
if [ ! -d "${upload_dir_ep_num}" ]
|
||||
then
|
||||
echo "DEBUG: Missing directory \"upload_dir_ep_num\" \"${upload_dir}\", /hpr and \"${ep_num}\"" 1>&2
|
||||
exit
|
||||
fi
|
||||
|
||||
if [ ! -e "hpr${ep_num}_norm.wav" ]
|
||||
then
|
||||
echo "DEBUG: Missing file \"hpr${ep_num}_norm.wav\"" 1>&2
|
||||
exit
|
||||
fi
|
||||
|
||||
cp -v hpr${ep_num}_norm.wav ${upload_dir_ep_num}/hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1
|
||||
|
||||
if [ ! -e "${upload_dir_ep_num}/hpr${ep_num}.wav" ]
|
||||
then
|
||||
echo "DEBUG: Missing file \"${upload_dir_ep_num}/hpr${ep_num}.wav\"" 1>&2
|
||||
exit
|
||||
fi
|
||||
|
||||
|
||||
echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
|
||||
echo "File information" | tee -a ${fname}_tmp.log
|
||||
ffprobe ${upload_dir_ep_num}/hpr${ep_num}.wav 2>&1 | grep Audio:
|
||||
mediainfo ${upload_dir_ep_num}/hpr${ep_num}.wav
|
||||
audio2image.bash ${upload_dir_ep_num}/hpr${ep_num}.wav
|
||||
xdg-open ${upload_dir_ep_num}/hpr${ep_num}.png >/dev/null 2>&1 &
|
||||
read -p "Processed Waveform look ok ? (N|y) ? " -n 1 -r
|
||||
echo # (optional) move to a new line
|
||||
if [[ ! $REPLY =~ ^[yY]$ ]]
|
||||
then
|
||||
echo "skipping...."
|
||||
exit
|
||||
fi
|
||||
#rm -v ${upload_dir_ep_num}/hpr${ep_num}.png
|
||||
|
||||
|
||||
echo "--------------------------------------------------------------------------------"
|
||||
echo "Geting transcript of hpr${ep_num}"
|
||||
|
||||
echo whisper --model tiny --language en --output_dir "${media_dir}" "${upload_dir_ep_num}/hpr${ep_num}.wav" | tee -a ${fname}_tmp.log
|
||||
whisper --model tiny --language en --output_dir "${media_dir}" "${upload_dir_ep_num}/hpr${ep_num}.wav" | tee -a ${fname}_tmp.log
|
||||
|
||||
ls -al "${media_dir}/hpr${ep_num}".*
|
||||
xdg-open "${media_dir}/hpr${ep_num}".txt 2>&1 &
|
||||
echo mpv --no-audio-display --audio-channels=stereo --speed="3.5" "${media_dir}/hpr${ep_num}".txt
|
||||
unset REPLY
|
||||
until [[ $REPLY =~ ^[yYnN]$ ]]
|
||||
do
|
||||
read -p "Processed transcript look ok ? (N|y) ? " -n 1 -r
|
||||
echo ""
|
||||
done
|
||||
|
||||
if [[ ! $REPLY =~ ^[yY]$ ]]
|
||||
then
|
||||
echo "skipping.... $REPLY"
|
||||
exit
|
||||
fi
|
||||
|
||||
mkdir "${upload_dir_ep_num}/hpr${ep_num}" >/dev/null 2>&1
|
||||
|
||||
cp -v "${media_dir}/hpr${ep_num}".vtt "${upload_dir_ep_num}/hpr${ep_num}.vtt" | tee -a ${fname}_tmp.log
|
||||
cp -v "${media_dir}/hpr${ep_num}".srt "${upload_dir_ep_num}/hpr${ep_num}.srt" | tee -a ${fname}_tmp.log
|
||||
cp -v "${media_dir}/hpr${ep_num}".txt "${upload_dir_ep_num}/hpr${ep_num}.txt" | tee -a ${fname}_tmp.log
|
||||
cp -v "${media_dir}/hpr${ep_num}".tsv "${upload_dir_ep_num}/hpr${ep_num}.tsv" | tee -a ${fname}_tmp.log
|
||||
cp -v "${media_dir}/hpr${ep_num}".json "${upload_dir_ep_num}/hpr${ep_num}.json" | tee -a ${fname}_tmp.log
|
||||
|
||||
echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
|
||||
echo "Convert to opus" | tee -a ${fname}_tmp.log
|
||||
ffmpeg -y -i ${upload_dir_ep_num}/hpr${ep_num}.wav ${upload_dir_ep_num}/hpr${ep_num}.opus 2>> ${fname}_tmp.log 1>&2
|
||||
|
||||
echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
|
||||
echo "Convert to flac" | tee -a ${fname}_tmp.log
|
||||
ffmpeg -y -i ${upload_dir_ep_num}/hpr${ep_num}.wav ${upload_dir_ep_num}/hpr${ep_num}.flac 2>> ${fname}_tmp.log 1>&2
|
||||
|
||||
echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
|
||||
echo "Convert to mp3" | tee -a ${fname}_tmp.log
|
||||
ffmpeg -y -i ${upload_dir_ep_num}/hpr${ep_num}.wav ${upload_dir_ep_num}/hpr${ep_num}.mp3 2>> ${fname}_tmp.log 1>&2
|
||||
|
||||
echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
|
||||
echo "Convert to ogg" | tee -a ${fname}_tmp.log
|
||||
#ffmpeg -y -i ${upload_dir_ep_num}/hpr${ep_num}.wav ${upload_dir_ep_num}/hpr${ep_num}.ogg 2>> ${fname}_tmp.log 1>&2
|
||||
ffmpeg -y -i ${upload_dir_ep_num}/hpr${ep_num}.wav -acodec libopus -f ogg ${upload_dir_ep_num}/hpr${ep_num}.ogg 2>> ${fname}_tmp.log 1>&2
|
||||
|
||||
echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
|
||||
echo "Convert to spx" | tee -a ${fname}_tmp.log
|
||||
ffmpeg -y -i ${upload_dir_ep_num}/hpr${ep_num}.wav ${upload_dir_ep_num}/hpr${ep_num}.spx 2>> ${fname}_tmp.log 1>&2
|
||||
|
||||
### End conversion
|
||||
|
||||
intro_duration="$( mediainfo --Output=XML --Full "hpr${ep_num}_intro.wav" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
|
||||
outro_duration="$( mediainfo --Output=XML --Full "${outro}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
|
||||
source_duration="$( mediainfo --Output=XML --Full "${mediafile}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
|
||||
expected_duration=$(( ${intro_duration} + ${HPR_duration} + ${outro_duration} ))
|
||||
|
||||
echo "Intro(${intro_duration}) + Show(${HPR_duration}) + Outro(${outro_duration}) = ${expected_duration}"
|
||||
|
||||
media_error="0"
|
||||
|
||||
for check_file in \
|
||||
${upload_dir_ep_num}/hpr${ep_num}.wav \
|
||||
${upload_dir_ep_num}/hpr${ep_num}.opus \
|
||||
${upload_dir_ep_num}/hpr${ep_num}.flac \
|
||||
${upload_dir_ep_num}/hpr${ep_num}.mp3 \
|
||||
${upload_dir_ep_num}/hpr${ep_num}.spx \
|
||||
${upload_dir_ep_num}/hpr${ep_num}.ogg
|
||||
do
|
||||
# ${upload_dir_ep_num}/hpr${ep_num}.spx
|
||||
echo "INFO: Processing the file \"${check_file}\""
|
||||
if [[ ! -s "${check_file}" ]]
|
||||
then
|
||||
echo "ERROR: Something went wrong encoding of the file \"${check_file}\""
|
||||
exit
|
||||
else
|
||||
mediainfo --Output=XML --Full "${check_file}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}'
|
||||
this_duration=$( mediainfo --Output=XML --Full "${check_file}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )
|
||||
if [[ $(abs_diff "${this_duration}" "${expected_duration}" ) -le "${acceptable_duration_difference}" ]]
|
||||
then
|
||||
echo "INFO: The file \"${check_file}\" duration of ${this_duration} () is close enough to ${expected_duration}"
|
||||
else
|
||||
echo "ERROR: The file \"${check_file}\" actual duration of ${this_duration} is not close enough to posted duration of ${expected_duration}."
|
||||
echo " Fix or update the posted duration to ${source_duration}."
|
||||
media_error="1"
|
||||
fi
|
||||
#${expected_duration}
|
||||
fi
|
||||
done
|
||||
echo "Source: ${source_duration}"
|
||||
|
||||
if [[ "${media_error}" -eq "1" ]]
|
||||
then
|
||||
echo "ERROR: Media is not encoded correctly"
|
||||
exit
|
||||
else
|
||||
echo "INFO: Media duration is correct"
|
||||
fi
|
||||
|
||||
|
||||
if [[ ! -s ${upload_dir_ep_num}/hpr${ep_num}.wav ]] || [[ ! -s ${upload_dir_ep_num}/hpr${ep_num}.opus ]] || [[ ! -s ${upload_dir_ep_num}/hpr${ep_num}.flac ]] || [[ ! -s ${upload_dir_ep_num}/hpr${ep_num}.mp3 ]] || [[ ! -s ${upload_dir_ep_num}/hpr${ep_num}.ogg ]] || [[ ! -s ${upload_dir_ep_num}/hpr${ep_num}.spx ]]
|
||||
then
|
||||
echo "ERROR: Something went wrong encoding the files"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
|
||||
echo "Fixing Tags" | tee -a ${fname}_tmp.log
|
||||
|
||||
fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="${HPR_comment} The license is ${HPR_license}" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" ${upload_dir_ep_num}/hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2
|
||||
|
||||
fix_tags ${upload_dir_ep_num}/hpr${ep_num}*
|
||||
|
||||
#echo "Changing the file dates to the time of upload"
|
||||
touch -r ${mediafile} hpr${ep_num}*
|
||||
touch -r ${mediafile} ${upload_dir_ep_num}/hpr${ep_num}*
|
||||
|
||||
ls -al hpr${ep_num}* ${upload_dir_ep_num}/hpr${ep_num}* | tee -a ${fname}_tmp.log
|
||||
|
||||
# # echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
|
||||
# # echo "Getting info for the asset table" | tee -a ${fname}_tmp.log
|
||||
# #
|
||||
# # echo "INSERT INTO assets (episode_id,filename,extension,size,sha1sum,mime_type,file_type) VALUES"
|
||||
# #
|
||||
# # for asset_file in \
|
||||
# # ${upload_dir_ep_num}/hpr${ep_num}.wav \
|
||||
# # ${upload_dir_ep_num}/hpr${ep_num}.opus \
|
||||
# # ${upload_dir_ep_num}/hpr${ep_num}.flac \
|
||||
# # ${upload_dir_ep_num}/hpr${ep_num}.mp3 \
|
||||
# # ${upload_dir_ep_num}/hpr${ep_num}.spx \
|
||||
# # ${upload_dir_ep_num}/hpr${ep_num}.ogg
|
||||
# # do
|
||||
# # size="$( ls -al "${asset_file}" | awk '{print $5}' )"
|
||||
# # sha1sum="$( sha1sum "${asset_file}" | awk '{print $1}' )"
|
||||
# # mime_type=$( file --dereference --brief --mime "${asset_file}" )
|
||||
# # file_type=$( file --dereference --brief "${asset_file}" )
|
||||
# # echo "(${ep_num},'${filename}','${extension}','${size}','${sha1sum}','${mime_type}','${file_type}'),"
|
||||
# # done
|
||||
|
||||
echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
|
||||
echo "Transferring files to Servers" | tee -a ${fname}_tmp.log
|
||||
|
||||
|
||||
#rsync -ave ssh --partial --progress --ignore-existing ${upload_dir_ep_num}/hpr${ep_num}.mp3 ${upload_dir_ep_num}/hpr${ep_num}.ogg ${upload_dir_ep_num}/hpr${ep_num}.spx hpr:www/eps/
|
||||
|
||||
#firefox "https://hackerpublicradio.org/local/hpr${ep_num}.mp3" >/dev/null 2>&1 &
|
||||
#firefox "https://hackerpublicradio.org/local/hpr${ep_num}.ogg" >/dev/null 2>&1 &
|
||||
firefox "file://${upload_dir_ep_num}/hpr${ep_num}.mp3" >/dev/null 2>&1 &
|
||||
firefox "file://${upload_dir_ep_num}/hpr${ep_num}.ogg" >/dev/null 2>&1 &
|
||||
#firefox "https://hackerpublicradio.org/eps.php?id=${ep_num}" >/dev/null 2>&1 &
|
||||
|
||||
#mpv "http://hackerpublicradio.org/local/hpr${ep_num}.spx" "http://hackerpublicradio.org/local/hpr${ep_num}.ogg" "http://hackerpublicradio.org/local/hpr${ep_num}.mp3" "file://${upload_dir_ep_num}/hpr${ep_num}.spx" "file://${upload_dir_ep_num}/hpr${ep_num}.opus" "file://${upload_dir_ep_num}/hpr${ep_num}.ogg" "file://${upload_dir_ep_num}/hpr${ep_num}.mp3" "file://${upload_dir_ep_num}/hpr${ep_num}.flac"
|
||||
mpv "file://${upload_dir_ep_num}/hpr${ep_num}.spx" "file://${upload_dir_ep_num}/hpr${ep_num}.opus" "file://${upload_dir_ep_num}/hpr${ep_num}.ogg" "file://${upload_dir_ep_num}/hpr${ep_num}.mp3" "file://${upload_dir_ep_num}/hpr${ep_num}.flac"
|
||||
|
||||
read -p "Remove files for \"${fname}\" (y|N) ? " -n 1 -r
|
||||
echo # (optional) move to a new line
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]
|
||||
then
|
||||
mediafilename=$(basename "${mediafile}")
|
||||
mediaextension="${mediafilename##*.}" # "
|
||||
# ssh hpr -t "mkdir /home/hpr/www/eps/hpr${ep_num}" >/dev/null 2>&1
|
||||
# rsync -ave ssh --partial --progress --ignore-existing "${mediafile}" hpr:www/eps/hpr${ep_num}/hpr${ep_num}_source.${mediaextension} | tee -a ${fname}_tmp.log
|
||||
#
|
||||
# rsync -ave ssh --partial --progress --ignore-existing "${media_dir}/hpr${ep_num}.wav".vtt hpr:www/eps/hpr${ep_num}/hpr${ep_num}.vtt | tee -a ${fname}_tmp.log
|
||||
# rsync -ave ssh --partial --progress --ignore-existing "${media_dir}/hpr${ep_num}.wav".srt hpr:www/eps/hpr${ep_num}/hpr${ep_num}.srt | tee -a ${fname}_tmp.log
|
||||
# rsync -ave ssh --partial --progress --ignore-existing "${media_dir}/hpr${ep_num}.wav".txt hpr:www/eps/hpr${ep_num}/hpr${ep_num}.txt | tee -a ${fname}_tmp.log
|
||||
# f
|
||||
# ssh hpr -t "ls -al /home/hpr/www/eps/hpr${ep_num}*"
|
||||
cp -v "${mediafile}" "${upload_dir_ep_num}/hpr${ep_num}_source.${mediaextension}"
|
||||
|
||||
#echo "Remove temp files"
|
||||
# rm -v ~hpr${ep_num}_summary.wav ~~hpr${ep_num}_summary.wav silence.wav
|
||||
# rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav
|
||||
#mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/
|
||||
|
||||
# wget --timeout=0 -q "http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1" -O - | xmlstarlet val --err -
|
||||
# wget --timeout=0 -q "http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1" -O - | xmlstarlet val --err -
|
||||
# wget --timeout=0 -q "http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1" -O - | xmlstarlet val --err -
|
||||
# wget --timeout=0 -q "http://hackerpublicradio.org/rss-future.php" -O - | xmlstarlet val --err -
|
||||
|
||||
echo "INFO: rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* borg:/data/IA/uploads/"
|
||||
rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* borg:/data/IA/uploads/
|
||||
echo "INFO: rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* rsync.net:processing/"
|
||||
rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* rsync.net:processing/
|
||||
echo "$( ssh borg -t "ls -al /data/IA/uploads/hpr${ep_num}*" ; ssh rsync.net -t "ls -al rsync.net:processing/hpr${ep_num}*" ; ls -al ${upload_dir}/hpr${ep_num}* )" | grep "hpr${ep_num}" | awk '{print $5, $NF}' | sort
|
||||
# ken@kalani:
|
||||
# rsync -av --partial --progress /var/IA/uploads/ vger:processing/
|
||||
# $HOME/sourcecode/hpr/hpr_hub.wip/bin/hpr-assets.bash
|
||||
|
||||
# root@vger:~#
|
||||
# rsync -av --progress --partial $HOME/processing/ /mnt/data/hpr/eps/
|
||||
# find /mnt/data/hpr/eps/ -mindepth 1 -maxdepth 1 -type f | grep -vE 'assets.csv|assets.json' | while read i;do mv -fv ${i} $(dirname ${i})/$(basename ${i%.*} | cut -c -7 )/;done
|
||||
# find /mnt/data/hpr/eps/ -type f \( -iname "*source*" -o -iname "*flac*" -o -iname "*wav*" \) -delete -print
|
||||
# chcon -R --type=httpd_sys_rw_content_t /mnt/data/hpr/
|
||||
# chown apache:apache /mnt/data/hpr/
|
||||
|
||||
# ken@kalani:
|
||||
# sshfs vger:/ /mnt/sshfs/vger/
|
||||
# sshfs rsync.net: /mnt/sshfs/rsync.net/
|
||||
# rsync -av --partial --progress /mnt/sshfs/vger/mnt/data/hpr/eps/ /mnt/sshfs/rsync.net/hpr/eps/
|
||||
|
||||
# https://hpr.nyc3.cdn.digitaloceanspaces.com/eps/hpr1404/hpr1404.mp3
|
||||
# https://alpha.nl.eu.mirror.hackerpublicradio.org/eps/hpr4271/hpr4271.mp3
|
||||
# https://archive.org/download/hpr4268/hpr4268.ogg
|
||||
|
||||
# set eps.valid=1
|
||||
|
||||
if [ ${show_type} == "remote" ]
|
||||
then
|
||||
echo "INFO: Setting the status"
|
||||
# SHOW_SUBMITTED → METADATA_PROCESSED → SHOW_POSTED → MEDIA_TRANSCODED → UPLOADED_TO_IA → UPLOADED_TO_RSYNC_NET
|
||||
curl --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php?ep_num=${ep_num}&status=MEDIA_TRANSCODED"
|
||||
curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php"
|
||||
fi
|
||||
else
|
||||
echo "skipping...."
|
||||
echo "cp -v \"${mediafile}\" \"${upload_dir_ep_num}/hpr${ep_num}_source.${mediaextension}\""
|
||||
# echo "rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav"
|
||||
#echo "mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/"
|
||||
echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
|
||||
echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
|
||||
echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
|
||||
echo "rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* borg:/data/IA/uploads/"
|
||||
fi
|
20
workflow/intro.srt
Normal file
20
workflow/intro.srt
Normal file
@ -0,0 +1,20 @@
|
||||
1
|
||||
00:00:00,000 --> 00:00:13,200
|
||||
REPLACE_LINE_1
|
||||
|
||||
2
|
||||
00:00:13,200 --> 00:00:16,400
|
||||
REPLACE_LINE_2
|
||||
|
||||
3
|
||||
00:00:16,400 --> 00:00:20,040
|
||||
REPLACE_LINE_3
|
||||
|
||||
4
|
||||
00:00:20,040 --> 00:00:22,080
|
||||
REPLACE_LINE_4
|
||||
|
||||
5
|
||||
00:00:22,080 --> 00:00:27,320
|
||||
REPLACE_LINE_5
|
||||
|
BIN
workflow/outro.flac
Normal file
BIN
workflow/outro.flac
Normal file
Binary file not shown.
31
workflow/outro.srt
Normal file
31
workflow/outro.srt
Normal file
@ -0,0 +1,31 @@
|
||||
1
|
||||
00:00:00,000 --> 00:00:07,720
|
||||
You have been listening to Hacker Public Radio at Hacker Public Radio.org.
|
||||
|
||||
2
|
||||
00:00:07,720 --> 00:00:11,520
|
||||
Today's show was contributed by a HPR listener like yourself.
|
||||
|
||||
3
|
||||
00:00:11,520 --> 00:00:18,000
|
||||
If you ever thought of recording podcast, click on our upload link
|
||||
|
||||
4
|
||||
00:00:18,000 --> 00:00:19,000
|
||||
to find out how easy it is.
|
||||
|
||||
5
|
||||
00:00:19,000 --> 00:00:26,480
|
||||
Hosting for HPR has been kindly provided by an AnHonestHost.com, the Internet Archive,
|
||||
|
||||
6
|
||||
00:00:26,480 --> 00:00:27,480
|
||||
rsync.net, and our mirror network.
|
||||
|
||||
7
|
||||
00:00:27,480 --> 00:00:33,480
|
||||
Unless otherwise stated, today's show is released under a Creative Commons
|
||||
|
||||
8
|
||||
00:00:33,480 --> 00:00:36,480
|
||||
Attribution-ShareAlike 4.0 International (CC BY-SA 4.0) license.
|
@ -1,339 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/
|
||||
#============================================================
|
||||
|
||||
processing_dir="$HOME/tmp/hpr/processing" # The directory where the files will be copied to for processing
|
||||
|
||||
if [ ! -d "${processing_dir}" ]
|
||||
then
|
||||
echo "ERROR: The application \"${this_program}\" is required but is not installed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
###################
|
||||
# Check that all the programs are installed
|
||||
|
||||
function is_installed () {
|
||||
for this_program in "$@"
|
||||
do
|
||||
if ! command -v ${this_program} 2>&1 >/dev/null
|
||||
then
|
||||
echo "ERROR: The application \"${this_program}\" is required but is not installed."
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
is_installed awk base64 cat curl curl date detox eval ffprobe file find grep grep head jq jq kate magick mediainfo mv mv rsync rsync seamonkey sed sed sort sponge ssh touch touch wget
|
||||
|
||||
echo "Processing the next HPR Show in the queue"
|
||||
|
||||
###################
|
||||
# Get the show
|
||||
#
|
||||
# Replaced METADATA_PROCESSED with SHOW_SUBMITTED
|
||||
response=$( curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | \
|
||||
grep ',SHOW_SUBMITTED,' | \
|
||||
head -1 | \
|
||||
sed 's/,/ /g' )
|
||||
|
||||
if [ -z "${response}" ]
|
||||
then
|
||||
echo "INFO: There appear to be no more shows with the status \"SHOW_SUBMITTED\"."
|
||||
echo "Getting a list of all the reservations."
|
||||
curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | sort -n
|
||||
exit 3
|
||||
fi
|
||||
|
||||
timestamp_epoc="$( echo ${response} | awk '{print $1}' )"
|
||||
ep_num="$( echo ${response} | awk '{print $2}' )"
|
||||
ep_date="$( echo ${response} | awk '{print $3}' )"
|
||||
key="$( echo ${response} | awk '{print $4}' )"
|
||||
status="$( echo ${response} | awk '{print $5}' )"
|
||||
email="$( echo ${response} | awk '{print $6}' )"
|
||||
email_unpadded="$( echo $email | sed 's/.nospam@nospam./@/g' )"
|
||||
|
||||
upload_dir="/home/hpr/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
|
||||
source_dir="hpr:${upload_dir}"
|
||||
publish_dir="hpr:www/eps/hpr${ep_num}"
|
||||
dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
|
||||
|
||||
ssh hpr -t "detox -v ${upload_dir}/"
|
||||
|
||||
echo "INFO: Downloading hpr${ep_num} from ${email_unpadded}"
|
||||
echo ""
|
||||
echo rsync -ave ssh --partial --progress ${source_dir}/ ${processing_dir}/${dest_dir}/
|
||||
rsync -ave ssh --partial --progress ${source_dir}/ ${processing_dir}/${dest_dir}/
|
||||
echo ""
|
||||
echo "INFO: Working directory is \"${processing_dir}/${dest_dir}/\""
|
||||
echo ""
|
||||
|
||||
shownotes_json="${processing_dir}/${dest_dir}/shownotes.json"
|
||||
|
||||
if [ ! -s "${shownotes_json}" ]
|
||||
then
|
||||
echo "ERROR: \"${shownotes_json}\" is missing"
|
||||
exit 4
|
||||
fi
|
||||
|
||||
if [ "$( file "${shownotes_json}" | grep -ic "text" )" -eq 0 ]
|
||||
then
|
||||
echo "ERROR: \"${shownotes_json}\" is not a text file"
|
||||
exit 5
|
||||
fi
|
||||
|
||||
mv -v "${shownotes_json}" "${shownotes_json%.*}_origional.json"
|
||||
|
||||
jq '.' "${shownotes_json%.*}_origional.json" > "${shownotes_json}"
|
||||
|
||||
###################
|
||||
# Get the media
|
||||
#
|
||||
|
||||
remote_media="$( jq --raw-output '.metadata.url' "${shownotes_json}" )"
|
||||
|
||||
if [ -n "${remote_media}" ]
|
||||
then
|
||||
echo "INFO: Fetching remote media from \"${remote_media}\""
|
||||
wget --timestamping --directory-prefix="${processing_dir}/${dest_dir}/" "${remote_media}"
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
echo "ERROR: Could not get the remote media"
|
||||
exit 6
|
||||
fi
|
||||
fi
|
||||
|
||||
shownotes_html="${processing_dir}/${dest_dir}/shownotes.html"
|
||||
|
||||
jq --raw-output '.episode.Show_Notes' "${shownotes_json}" > "${shownotes_html}"
|
||||
|
||||
if [ ! -s "${shownotes_html}" ]
|
||||
then
|
||||
echo "ERROR: \"${shownotes_html}\" is missing"
|
||||
exit 7
|
||||
fi
|
||||
|
||||
# Process Shownotes
|
||||
sed "s#>#>\n#g" "${shownotes_html}" | sponge "${shownotes_html}"
|
||||
|
||||
# Extract Images
|
||||
## TODO Temp fix until https://repo.anhonesthost.net/HPR/hpr-tools/issues/3
|
||||
|
||||
image_count="1"
|
||||
|
||||
for image in $( grep --color=never --perl-regexp --only-matching 'data:image/[^;]*;base64,\K[a-zA-Z0-9+/=]*' "${shownotes_html}" )
|
||||
do
|
||||
this_image="${processing_dir}/${dest_dir}/hpr${ep_num}_${image_count}"
|
||||
echo -n "$image" | base64 -di > ${this_image}
|
||||
this_ext="$( file --mime-type ${this_image} | awk -F '/' '{print $NF}' )"
|
||||
mv -v "${this_image}" "${this_image}.${this_ext}"
|
||||
this_width="$( mediainfo "${this_image}.${this_ext}" | grep Width | awk -F ': | pixels' '{print $2}' | sed 's/ //g' )"
|
||||
if [ "${this_width}" -gt "400" ]
|
||||
then
|
||||
magick "${this_image}.${this_ext}" -resize 400x "${this_image}_tn.${this_ext}"
|
||||
fi
|
||||
((image_count=image_count+1))
|
||||
done
|
||||
|
||||
for image in $( grep --color=never --perl-regexp --only-matching '<img.*src.*http.*>' "${shownotes_html}" | awk -F 'src=' '{print $2}' | awk -F '"' '{print $2}' )
|
||||
do
|
||||
this_image="${processing_dir}/${dest_dir}/hpr${ep_num}_${image_count}"
|
||||
wget "${image}" --output-document=${this_image}
|
||||
this_ext="$( file --mime-type ${this_image} | awk -F '/' '{print $NF}' )"
|
||||
if [ ! -e "${this_image}.${this_ext}" ]
|
||||
then
|
||||
mv -v "${this_image%.*}" "${this_image}.${this_ext}"
|
||||
fi
|
||||
this_width="$( mediainfo "${this_image}.${this_ext}" | grep Width | awk -F ': | pixels' '{print $2}' | sed 's/ //g' )"
|
||||
if [ "${this_width}" -gt "400" ]
|
||||
then
|
||||
magick "${this_image}.${this_ext}" -resize 400x "${this_image}_tn.${this_ext}"
|
||||
fi
|
||||
((image_count=image_count+1))
|
||||
done
|
||||
|
||||
## TODO End Temp fix until https://repo.anhonesthost.net/HPR/hpr-tools/issues/3
|
||||
|
||||
ls -al "${processing_dir}/${dest_dir}/"
|
||||
|
||||
## Manually edit the shownotes to fix issues
|
||||
|
||||
kate "${shownotes_html}" >/dev/null 2>&1 &
|
||||
# librewolf "${shownotes_html}" >/dev/null 2>&1 &
|
||||
seamonkey "${shownotes_html}" >/dev/null 2>&1 &
|
||||
# bluefish "${shownotes_html}" >/dev/null 2>&1 &
|
||||
|
||||
read -p "Does the metadata 'look ok ? (N|y) ? " -n 1 -r
|
||||
echo # (optional) move to a new line
|
||||
if [[ ! $REPLY =~ ^[yY]$ ]]
|
||||
then
|
||||
echo "skipping...."
|
||||
exit 8
|
||||
fi
|
||||
|
||||
media=$( find "${processing_dir}/${dest_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' )
|
||||
if [ -z "${media}" ]
|
||||
then
|
||||
echo "ERROR: Can't find any media in \"${processing_dir}/${dest_dir}/\""
|
||||
find "${processing_dir}/${dest_dir}/" -type f
|
||||
exit 9
|
||||
fi
|
||||
|
||||
the_show_media=""
|
||||
|
||||
if [ "$( echo "${media}" | wc -l )" -ne 1 ]
|
||||
then
|
||||
echo "Multiple files found. Which one do you want to use ?"
|
||||
select this_media in $( echo "${media}" )
|
||||
do
|
||||
echo "INFO: You selected \"${this_media}\"."
|
||||
ls -al "${this_media}"
|
||||
the_show_media="${this_media}"
|
||||
break
|
||||
done
|
||||
else
|
||||
echo "INFO: Selecting media as \"${media}\"."
|
||||
the_show_media="${media}"
|
||||
fi
|
||||
|
||||
duration=$( \date -ud "1970-01-01 $( ffprobe -i "${the_show_media}" 2>&1| awk -F ': |, ' '/Duration:/ { print $2 }' )" +%s )
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
echo 'ERROR: Invalid duration found in '\"${media}\" >&2
|
||||
exit 10
|
||||
fi
|
||||
|
||||
###################
|
||||
# Gather episode information
|
||||
#
|
||||
|
||||
if [ "$( curl --silent --write-out '%{http_code}' http://hackerpublicradio.org/say.php?id=${ep_num} --output /dev/null )" == 200 ]
|
||||
then
|
||||
echo "ERROR: The Episode hpr${ep_num} has already been posted"
|
||||
exit 11
|
||||
fi
|
||||
|
||||
if [ "$( jq --raw-output '.metadata.Episode_Number' ${shownotes_json} )" != "${ep_num}" ]
|
||||
then
|
||||
echo "ERROR: The Episode_Number: \"${ep_num}\" was not found in \"${shownotes_json}\""
|
||||
exit 12
|
||||
fi
|
||||
|
||||
if [ "$( jq --raw-output '.metadata.Episode_Date' ${shownotes_json} )" != "${ep_date}" ]
|
||||
then
|
||||
echo "ERROR: The Episode_Date: \"${ep_date}\" was not found in \"${shownotes_json}\""
|
||||
exit 13
|
||||
fi
|
||||
|
||||
if [ "$( jq --raw-output '.host.Host_Email' ${shownotes_json} )" != "${email_unpadded}" ]
|
||||
then
|
||||
echo "ERROR: The Host_Email: \"${email_unpadded}\" was not found in \"${shownotes_json}\""
|
||||
exit 14
|
||||
fi
|
||||
|
||||
###################
|
||||
# Assemble the components
|
||||
#
|
||||
# https://newbedev.com/how-to-urlencode-data-for-curl-command/
|
||||
hostid="$( jq --raw-output '.host.Host_ID' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
|
||||
host_name="$( jq --raw-output '.host.Host_Name' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
|
||||
title=$( jq --raw-output '.episode.Title' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )
|
||||
summary="$( jq --raw-output '.episode.Summary' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
|
||||
series_id="$( jq --raw-output '.episode.Series' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
|
||||
series_name="$( jq --raw-output '.episode.Series_Name' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
|
||||
explicit="$( jq --raw-output '.episode.Explicit' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
|
||||
episode_license="$( jq --raw-output '.episode.Show_License' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
|
||||
tags="$( jq --raw-output '.episode.Tags' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
|
||||
host_license=$( jq --raw-output '.host.Host_License' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )
|
||||
host_profile=$( jq --raw-output '.host.Host_Profile' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )
|
||||
|
||||
notes="$( cat "${shownotes_html}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
|
||||
|
||||
if [ $# -gt 0 ]
|
||||
then
|
||||
declare -A hash
|
||||
for argument
|
||||
do
|
||||
if [[ $argument =~ ^[^=]+=.*$ ]]
|
||||
then
|
||||
this_key="${argument%=*}" # "${} Kate format hack
|
||||
this_value="${argument#*=}" # "${} Kate format hack
|
||||
this_value="$( echo "${this_value}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
|
||||
eval "${this_key}=${this_value}"
|
||||
echo "INFO: Replacing \"${this_key}\" with \"${this_value}\"."
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "${hostid}" == '0' ]
|
||||
then
|
||||
echo "ERROR: The hostid is 0. Create the host and use \"hostid=???\" to override"
|
||||
exit 15
|
||||
fi
|
||||
|
||||
###################
|
||||
# Post show to HPR
|
||||
#
|
||||
|
||||
post_show_json="${processing_dir}/${dest_dir}/post_show.json"
|
||||
post_show_response="${processing_dir}/${dest_dir}/post_show_response.txt"
|
||||
|
||||
echo "Sending:"
|
||||
cat "${post_show}"
|
||||
echo "key=${key}
|
||||
ep_num=${ep_num}
|
||||
ep_date=${ep_date}
|
||||
email=${email}
|
||||
title=${title}
|
||||
duration=${duration}
|
||||
summary=${summary}
|
||||
series_id=${series_id}
|
||||
series_name=${series_name}
|
||||
explicit=${explicit}
|
||||
episode_license=${episode_license}
|
||||
tags=${tags}
|
||||
hostid=${hostid}
|
||||
host_name=${host_name}
|
||||
host_license=${host_license}
|
||||
host_profile=${host_profile}
|
||||
notes=${notes}"
|
||||
|
||||
echo "{
|
||||
\"key\": \"${key}\",
|
||||
\"ep_num\": \"${ep_num}\",
|
||||
\"ep_date\": \"${ep_date}\",
|
||||
\"email\": \"${email}\",
|
||||
\"title\": \"${title}\",
|
||||
\"duration\": \"${duration}\",
|
||||
\"summary\": \"${summary}\",
|
||||
\"series_id\": \"${series_id}\",
|
||||
\"series_name\": \"${series_name}\",
|
||||
\"explicit\": \"${explicit}\",
|
||||
\"episode_license\": \"${episode_license}\",
|
||||
\"tags\": \"${tags}\",
|
||||
\"hostid\": \"${hostid}\",
|
||||
\"host_name\": \"${host_name}\",
|
||||
\"host_license\": \"${host_license}\",
|
||||
\"host_profile\": \"${host_profile}\",
|
||||
\"notes\": \"${notes}\"
|
||||
}" | tee "${post_show_json}"
|
||||
|
||||
echo "INFO: Uploading processed files to \"${processing_dir}/${dest_dir}/\""
|
||||
echo ""
|
||||
echo rsync -ave ssh --partial --progress ${processing_dir}/${dest_dir}/ ${source_dir}/
|
||||
rsync -ave ssh --partial --progress ${processing_dir}/${dest_dir}/ ${source_dir}/
|
||||
echo ""
|
||||
|
||||
curl --netrc --include --request POST "https://hub.hackerpublicradio.org/cms/add_show_json.php" --header "Content-Type: application/json" --data-binary "@${post_show_json}"
|
||||
|
||||
if [ "$( curl --silent --write-out '%{http_code}' http://hackerpublicradio.org/say.php?id=${ep_num} --output /dev/null )" != 200 ]
|
||||
then
|
||||
echo "ERROR: The Episode hpr${ep_num} has not been posted"
|
||||
exit 16
|
||||
fi
|
||||
|
||||
echo "INFO: Uploading completed files to \"${publish_dir}/\""
|
||||
echo ""
|
||||
echo "rsync -ave ssh --partial --progress \"${processing_dir}/${dest_dir}/hpr${ep_num}\"* \"${publish_dir}/\""
|
||||
rsync -ave ssh --partial --progress "${processing_dir}/${dest_dir}/hpr${ep_num}"* "${publish_dir}/"
|
||||
echo ""
|
1561
workflow/process_episode.bash
Executable file
1561
workflow/process_episode.bash
Executable file
File diff suppressed because it is too large
Load Diff
12
workflow/remove-image.pl
Executable file
12
workflow/remove-image.pl
Executable file
@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env perl
|
||||
|
||||
use strict;
|
||||
|
||||
while (<>) {
|
||||
s/(<img.*src.*data:image\/[^;]*;base64,[a-zA-Z0-9+\/=]*)/<img src="LOCAL_IMAGE_REMOVED/g;
|
||||
s/(<img.*src.*http.*>)/<img src="REMOTE_IMAGE_REMOVED" \/>/g;
|
||||
print;
|
||||
}
|
||||
|
||||
exit
|
||||
# <img src="data:image/jpeg;base64,/9j/4QnuRXhpZgAATU
|
BIN
workflow/silence.flac
Normal file
BIN
workflow/silence.flac
Normal file
Binary file not shown.
BIN
workflow/theme.flac
Normal file
BIN
workflow/theme.flac
Normal file
Binary file not shown.
Loading…
x
Reference in New Issue
Block a user