Abort early if shownotes missing, and allign wave with play controls

Reviewed-on: #15

Community_News/.make_email.cfg

@@ -0,0 +1,18 @@
+<email>
+    server = chatter.skyehaven.net
+    port   = 64738
+    room   = HPR
+    # Default day of the week for the recording
+    dayname = Friday
+    # Times are UTC
+    starttime = 16:00:00
+    endtime = 18:00:00
+    # Usually 2 hours are set aside. This value is used when a different start
+    # time is provided and no end time.
+    duration = 02 # hours
+    # Template is in the current directory
+    template = make_email_template.tpl
+</email>
+<recdates>
+    name = recording_dates.dat
+</recdates>

Community_News/.make_shownotes.cfg

@@ -0,0 +1,47 @@
+#
+# .make_shownotes.cfg (2025-04-13)
+# Configuration file for make_shownotes version >= 4
+#
+<settings>
+    # Format strings (using 'printf' formatting) for building certain required strings
+    title_template   = HPR Community News for %s %s
+    summary_template = HPR Volunteers talk about shows released and comments posted in %s %s
+
+    # Repeat the following line with each of the desired tags to make an
+    # array-like structure
+    tags             = Community News
+
+    # Host id for HPR Volunteers
+    # Series id for HPR Community News series
+    hostid = 159
+    series_id = 47
+
+    # Day the Community News show is released
+    releaseday = Monday
+
+    # Default day of the week for the recording
+    recordingday = Friday
+
+    # Recording times are UTC
+    starttime = 16:00:00
+    endtime = 17:00:00
+
+    # cache of previous recording dates and times
+    cache = recording_dates.dat
+
+    # Template Toolkit templates
+    # --------------------------
+
+    # Main note template (actually a soft link)
+    main_template = shownote_template.tpl
+
+    # Used to make a stand-alone HTML file from the default HTML
+    # fragment
+    container_template = shownotes_container.tpl
+
+</settings>
+
+<database>
+    # Assume a local file
+    name = hpr.db
+</database>

Community_News/build_AOB

@@ -241,7 +241,8 @@ AOBMKD="$BASEDIR/aob_$startdate.mkd"
 #     --define "table=$TMP1" "$AOBMKD" |\
 #     pandoc -f markdown-smart -t html5 -o "${AOBMKD%mkd}html"; then
 #
-if pandoc -f markdown-smart -t html5 "$AOBMKD" -o "${AOBMKD%mkd}html"; then
+if tpage "$AOBMKD" |\
+    pandoc -f markdown-smart -t html5 -o "${AOBMKD%mkd}html"; then
     echo "Converted $AOBMKD to HTML"
 else
     echo "Conversion of $AOBMKD to HTML failed"

Community_News/collect_HPR_database

@@ -0,0 +1,122 @@
+#!/bin/bash -
+#===============================================================================
+#
+#         FILE: collect_HPR_database
+#
+#        USAGE: ./collect_HPR_database
+#
+#  DESCRIPTION: Collects the SQL dump of the public copy of the HPR database
+#               from the website and converts it to SQLite.
+#
+#      OPTIONS: None
+# REQUIREMENTS: ---
+#         BUGS: ---
+#        NOTES: There are dependencies on mysql2sqlite and sqlite3. The former
+#               comes from https://github.com/mysql2sqlite and is expected to
+#               be in the same directory as this script. The sqlite3 package
+#               needs to be installed from the repository appropriate to the
+#               local OS. It is assumed that wget is available. The script
+#               uses auto-deleted temporary files for the MySQL dump, and the
+#               dump converted for SQLite.
+#       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
+#      VERSION: 0.0.1
+#      CREATED: 2025-02-22 16:52:40
+#     REVISION: 2025-02-22 17:56:00
+#
+#===============================================================================
+
+set -o nounset                              # Treat unset variables as an error
+
+SCRIPT=${0##*/}
+BASEDIR=${0%/*}
+
+VERSION="0.0.1"
+
+# {{{ -- Functions: cleanup_temp
+#===  FUNCTION  ================================================================
+#          NAME:  cleanup_temp
+#   DESCRIPTION:  Cleanup temporary files in case of a keyboard interrupt
+#                 (SIGINT) or a termination signal (SIGTERM) and at script
+#                 exit. Expects to be called from 'trap' so it can just exit
+#                 (assuming it's all that's called)
+#    PARAMETERS:  * - names of temporary files to delete
+#       RETURNS:  Nothing
+#===============================================================================
+function cleanup_temp {
+    for tmp; do
+        [ -e "$tmp" ] && rm --force "$tmp"
+    done
+    exit 0
+}
+# }}}
+
+#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+#
+# Make sure we're where the script lives
+#
+cd "$BASEDIR" || { echo "$SCRIPT: Failed to cd to $BASEDIR"; exit 1; }
+
+#
+# Make temporary files and set traps to delete them
+#
+TMP1=$(mktemp) || {
+    echo "$SCRIPT: creation of temporary file failed!"
+    exit 1
+}
+TMP2=$(mktemp) || {
+    echo "$SCRIPT: creation of temporary file failed!"
+    exit 1
+}
+trap 'cleanup_temp $TMP1 $TMP2' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
+
+#
+# Definition of files
+#
+#-------------------------------------------------------------------------------
+mysql2sqlite="$BASEDIR/mysql2sqlite"
+snapshot_url="https://www.hackerpublicradio.org/hpr.sql"
+db_name="hpr.db"
+#-------------------------------------------------------------------------------
+
+#
+# Sanity check
+#
+[ -e "$mysql2sqlite" ] || {
+    echo "$SCRIPT: Unable to find mandatory script $mysql2sqlite"
+    exit 1
+}
+
+#
+# Collect the SQL dump into a temporary file`
+#
+if ! wget -q "$snapshot_url" -O "$TMP1"; then
+    echo "$SCRIPT: Failed to download from $snapshot_url"
+    exit 1
+fi
+
+#
+# Delete the SQLite database if it exists (otherwise the new data is merged
+# with it causing chaos)
+#
+[ -e "$db_name" ] && rm -f "$db_name"
+
+#
+# Convert the MySQL/MariaDB dump. First run sed on it, then run mysql2sqlite
+# (from https://github.com/mysql2sqlite) to do the SQL dump conversion. Use
+# the result to make a SQLite database.
+#
+sed '/^DELIMITER ;;/,/^DELIMITER ;/d' "$TMP1" > "$TMP2"
+$mysql2sqlite "$TMP2" | sqlite3 "$db_name"
+
+#
+# Report success if the new database exists
+#
+if [[ -e "$db_name" ]]; then
+    echo "Created SQLite database '$db_name'"
+else
+    echo "Failed to find the expected new database '$db_name'"
+fi
+
+# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker
+

Community_News/make_email_template.tpl

@@ -0,0 +1,76 @@
+[%# make_email_template.tpl 2025-04-03 -%]
+[%# Community News email template -%]
+[% USE wrap -%]
+[% subject %]
+
+[% FILTER replace('\n', ' ') -%]
+[% IF utc.days > 6 -%]
+The Community News for [% utc.revmonth %] will be recorded using Mumble on
+[% ELSE -%]
+The next Community News will be recorded using Mumble on
+[% END -%]
+[% utc.date %] between [% utc.start %] and [% utc.end %] in the '[% room %]' room on [% server %] port [% port %].
+[% END %]
+
+[% FILTER replace('\n', ' ') -%]
+During the recording HPR Volunteers will review the shows released during
+[% utc.revmonth %] [% utc.revyear %], they will read comments submitted during that
+month, as well as summarising email sent to the HPR mailing list.
+[% END %]
+
+[% FILTER replace('\n', ' ') -%]
+All HPR listeners are welcome to join in, but we ask that you listen to all
+the shows in [% utc.revmonth %] before you do so.
+[% END %]
+
+[% FILTER replace('\n', ' ') -%]
+Occasionally, due to local factors, we might need to change the time, or even
+the date, without warning. If you're planning on joining in it might be a good
+idea to let us know in advance - email admin@hackerpublicradio.org. Then we
+can contact you with the new schedule in the rare event that we have to
+make a change.
+[% END %]
+
+[% FILTER replace('\n', ' ') -%]
+The notes for the recording are an extended version of the show notes.
+A version without these extended elements is made for release on the HPR site
+(and on archive.org). Comments which might have been missed in the last
+recording will be marked in red in the extended version. Comments which would
+normally be in this month, but which were read out in the last show are marked
+in green. Comments made in the past month to older shows will be displayed in
+full (so they are easier to read).
+[% END %]
+
+[% FILTER replace('\n', ' ') -%]
+The notes for this recording will be made available to participants before the
+recording begins.
+[% END %]
+
+Summary:
+[% FILTER indent('   ') -%]
+Date of recording:      [% utc.date %]
+Start and end times:    [% utc.start %] and [% utc.end %]
+Mumble server:          [% server %]
+Port:                   [% port %]
+Room:                   [% room %]
+[% END -%]
+
+Refer to https://hackerpublicradio.org/recording.html for how to use Mumble.
+
+[% FILTER replace('\n', ' ') -%]
+There is an iCal file on the HPR site that you can load into a compatible
+calendar which will remind you of the next 12 upcoming recording dates.
+Access it from https://hackerpublicradio.org/HPR_Community_News_schedule.ics
+[% END %]
+
+See below for start and end times in various international timezones.
+
+[% FOREACH tz IN timezones -%]
+[% tz.name %]
+Start: [% tz.start %]
+End:   [% tz.end %]
+
+[% END -%]
+[%#
+  # vim: syntax=tt2:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
+-%]

Community_News/make_meeting

@@ -17,9 +17,9 @@
 #               Hacking"
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
 #      LICENCE: Copyright (c) year 2012-2024 Dave Morriss
-#      VERSION: 0.2.2
+#      VERSION: 0.2.3
 #      CREATED: 2012-10-13 15:34:01
-#     REVISION: 2024-05-24 22:45:56
+#     REVISION: 2024-10-28 13:17:44
 #
 #===============================================================================
 
@@ -42,7 +42,7 @@ use Date::ICal;
 #
 # Version number (manually incremented)
 #
-our $VERSION = '0.2.2';
+our $VERSION = '0.2.3';
 
 #
 # Script name
@@ -65,8 +65,6 @@ my ( @startdate, @rdate, @events );
 #
 # Attributes for the calendar message
 #
-#my $server = 'ch1.teamspeak.cc';
-#my $port   = 64747;
 my $server = 'chatter.skyehaven.net';
 my $port   = 64738;
 
@@ -129,11 +127,12 @@ else {
 # of having a time zone defined (default UTC, as now).
 #
 my $monday = 1;    # Day of week number 1-7, Monday-Sunday
+my $offset = -3;   # Offset from the target date (-3 is Friday)
 
-my @starttime = ( 13, 00, 00 );    # UTC
-my @endtime   = ( 15, 00, 00 );
+my @starttime = ( 15, 00, 00 );    # UTC
+my @endtime   = ( 17, 00, 00 );
 
-my @todostart = ( 9,  00, 00 );    # UTC
+my @todostart = (  9, 00, 00 );    # UTC
 my @todoend   = ( 17, 00, 00 );
 
 #
@@ -161,12 +160,13 @@ http://hackerpublicradio.org/recording.php
 ENDDESC
 
 #
-# Compute the next recording date from the starting date (@startdate will be
-# today's date or the start of the explicitly selected month provided via
-# -from=DATE. We want day of the week to be Monday, the first in the month,
-# then to go back 1 day from that to get to the Sunday! Simple)
+# Compute the next recording date from the starting date.
+# Now @startdate will be today's date or the start of the explicitly selected
+# month provided via -from=DATE. We want day of the week to be Monday, the
+# first in the month, then to go back $offset days from that to get to the
+# recording day! Simple.
 #
-@startdate = make_date( \@startdate, $monday, 1, -1 );
+@startdate = make_date( \@startdate, $monday, 1, $offset );
 @rdate = @startdate;
 
 #
@@ -208,7 +208,7 @@ for my $i ( 1 .. $count ) {
     #
     # Recording date computation from the start of the month
     #
-    @rdate = make_date( \@rdate, $monday, 1, -1 );
+    @rdate = make_date( \@rdate, $monday, 1, $offset );
 
     #
     # Save the current recording date to make an array of arrayrefs
@@ -305,8 +305,9 @@ exit;
 #      RETURNS: The start of the month in the textual date in Date::Calc
 #               format
 #  DESCRIPTION: Parses the date string and makes a Date::Calc date from the
-#               result where the day part is 1. Optionally checks that the
-#               date isn't in the past, though $force = 1 ignores this check.
+#               result where the day part is forced to be 1. Optionally
+#               checks that the date isn't in the past, though $force
+#               = 1 ignores this check.
 #       THROWS: No exceptions
 #     COMMENTS: Requires Date::Calc and Date::Parse
 #               Note the validation 'die' has a non-generic message

Community_News/recording_dates.dat

@@ -0,0 +1,34 @@
+2021-10-01,2021-10-30 14:00:00
+2022-06-01,2022-07-02 14:00:00
+2022-07-01,2022-07-30 14:00:00
+2022-08-01,2022-09-03 14:00:00
+2022-09-01,2022-10-01 14:00:00
+2022-10-01,2022-11-05 14:00:00
+2022-11-01,2022-12-03 15:00:00
+2022-12-01,2022-12-27 15:00:00
+2023-01-01,2023-02-04 15:00:00
+2023-02-01,2023-03-04 15:00:00
+2023-03-01,2023-04-01 15:00:00
+2023-04-01,2023-04-29 15:00:00
+2023-05-01,2023-06-03 15:00:00
+2023-07-01,2023-08-05 15:00:00
+2023-08-01,2023-09-02 15:00:00
+2023-09-01,2023-09-30 15:00:00
+2023-10-01,2023-11-04 15:00:00
+2023-11-01,2023-12-02 15:00:00
+2023-12-01,2023-12-30 15:00:00
+2024-01-01,2024-02-04 15:00:00
+2024-02-01,2024-03-02 15:00:00
+2024-03-01,2024-03-30 15:00:00
+2024-05-01,2024-06-02 13:00:00
+2024-06-01,2024-06-28 15:00:00
+2024-07-01,2024-08-02 15:00:00
+2024-08-01,2024-08-29 15:00:00
+2024-09-01,2024-10-04 15:00:00
+2024-10-01,2024-11-01 15:00:00
+2024-11-01,2024-11-29 15:00:00
+2024-12-01,2025-01-03 15:00:00
+2025-01-01,2025-01-31 15:00:00
+2025-02-01,2025-02-28 16:00:00
+2025-03-01,2025-04-04 16:00:00
+2025-04-01,2025-05-02 16:00:00

Community_News/shownote_template.tpl

@@ -1 +1 @@
-shownote_template11.tpl
+shownote_template12.tpl

Community_News/shownote_template11.tpl

@@ -1,11 +1,15 @@
-[%# shownote_template11.tpl 2024-05-07 -%]
-[%# HTML snippet for insertion into the database -%]
-[%# This one uses the new format for the mailing list data, and partitions -%]
-[%# comments into past and current. It also marks comments that don't need -%]
-[%# to be read when -markcomments is selected. It requires make_shownotes >= V0.0.30 -%]
+[%# shownote_template11.tpl Updated: 2025-03-31 -%]
+[%# -------------------------------------------------------------------------------- -%]
+[%# Makes either an HTML snippet for insertion into the database or a full           -%]
+[%# listing with full comments for circulation to the hosts recording the episode    -%]
+[%# This one uses the new format for the mailing list data, and partitions           -%]
+[%# comments into past and current. It also marks comments that don't need           -%]
+[%# to be read when mark_comments is true. It requires make_shownotes >= V0.0.30     -%]
+[%# -------------------------------------------------------------------------------- -%]
 [%- USE date -%]
 [%- USE pad4 = format('%04d') -%]
 [%- correspondents = "https://hackerpublicradio.org/correspondents"
+    mailinglist = "https://lists.hackerpublicradio.com/mailman/listinfo/hpr"
     mailbase="https://lists.hackerpublicradio.com/pipermail/hpr"
     mailthreads = "$mailbase/$review_year-$review_month/thread.html" -%]
 [%- DEFAULT skip_comments = 0
@@ -15,8 +19,9 @@
     missed_count = 0
     past_count = 0
 -%]
-[%# Embedded CSS. The 'table' and 'hr' settings are always there but the rest is only for if -%]
-[%# we are marking comments                                                                  -%]
+[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
+[%# Embedded CSS. The 'table' and 'hr' settings are always there but the rest is     -%]
+[%# only for when we are marking comments                                            -%]
 <style>
 table td.shrink {
     white-space:nowrap
@@ -40,12 +45,7 @@ div#highlight {
 }
 [%- END %]
 </style>
-[%# For the '-mailnotes' option without a file we generate our own inclusion. -%]
-[%# We pretend 'default_mail' is a filename in the calling script. Messy.     -%]
-[% BLOCK default_mail -%]
-<a href="[% mailthreads %]" target="_blank">[% mailthreads %]</a>
-[% END -%]
-
+[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
 <h2>New hosts</h2>
 <p>
 [% IF hosts.size > 0 -%]
@@ -88,13 +88,13 @@ There were no new hosts this month.
 [%# Skip comments if told to by the caller -%]
 [%- IF skip_comments == 0 -%]
 [%# Handle any missed comments if mark_comments is true -%]
-[%- IF mark_comments == 1 && missed_count > 0 -%]
+[%- IF (mark_comments == 1) AND (missed_count > 0) -%]
 <br/><div id="highlight">
 <h2>Missed comment[%- missed_comments.size > 1 ? 's' : '' -%] last month</h2>
-<p><b>Note to Volunteers</b>: These are comments for shows last month that were not read in the last show because they arrived on or after the recording day. This section will be removed before these notes are released.</p>
+<p><b>Note to Volunteers</b>: These are comments for shows last month that were not read in the last show because they arrived on or after the recording started. This section will be removed before these notes are released.</p>
 <ul>
 [%- FOREACH comment IN missed_comments -%]
-<li><strong><a href="[% comment.identifier_url %]#comments" target="_blank">hpr[% comment.episode %]</a></strong>
+<li><strong><a href="[% comment.identifier_url %]#comments" target="_blank">hpr[% pad4(comment.episode) %]</a></strong>
 ([% comment.date %]) "<em>[% comment.title %]</em>" by <a href="[% correspondents %]/[% pad4(comment.hostid) %].html" target="_blank">[% comment.host %]</a>.<br/>
 <small>Summary: "<em>[% comment.summary %]</em>"</small><br/>
 From: [% comment.comment_author_name FILTER html_entity -%] on [% date.format(comment.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
@@ -112,7 +112,7 @@ From: [% comment.comment_author_name FILTER html_entity -%] on [% date.format(co
 <h2>Comments this month</h2>
 
 [% IF comment_count > 0 -%]
-[%- IF mark_comments == 1 && ignore_count > 0 -%]
+[%- IF (mark_comments == 1) AND (ignore_count > 0) -%]
 <p id="ignore"><b>Note to Volunteers</b>: Comments marked in green were read in the last
 Community News show and should be ignored in this one.</p>
 [%- END -%]
@@ -124,9 +124,9 @@ There [%- comment_count == 1 ? "is $comment_count comment" : "are $comment_count
 [% past.size %] previous [% past.size == 1 ? "show" : "shows" %]:</p>
 <ul>
 [%# Loop through by episode then by comment relating to that episode -%]
-[%-    FOREACH ep IN past.keys.sort -%]
+[%-    FOREACH ep IN past.keys.nsort -%]
 [%-       arr = past.$ep -%]
-<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% arr.0.episode %]</a></strong>
+<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
 ([%       arr.0.date %]) "<em>[% arr.0.title %]</em>"
 by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.<br/>
 [%-       IF mark_comments == 1 || ctext == 1 -%]
@@ -142,7 +142,7 @@ by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">
 [%-          ELSE %]
 <li>
 [%-          END %]
-<a href="[% row.identifier_url %]#[% row.index %]" target="_blank">Comment [% row.index %]</a>:
+<a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
 [%        row.comment_author_name FILTER html_entity -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
 [%-          IF row.comment_title.length > 0 %]
 "[%             row.comment_title %]"
@@ -158,7 +158,7 @@ by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">
 [%-          END -%]
 [%-       END -%]
 </ul><br/>
-</limage>
+</li>
 [%-    END -%]
 </ul>
 [%-    IF mark_comments == 1 || ctext == 1 -%]
@@ -171,15 +171,15 @@ by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">
 <h3>This month's shows</h3>
 <p>There [% cc == 1 ? "is $cc comment" : "are $cc comments" %] on [% current.size %] of this month's shows:</p>
 <ul>
-[%- FOREACH ep IN current.keys.sort -%]
+[%- FOREACH ep IN current.keys.nsort -%]
 [%- arr = current.$ep -%]
-<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% arr.0.episode %]</a></strong>
+<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
 ([% arr.0.date %]) "<em>[% arr.0.title %]</em>"
 by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.</li>
 <li style="list-style: none; display: inline">
 <ul>
 [%- FOREACH row IN arr -%]
-<li><a href="[% row.identifier_url %]#[% row.index %]" target="_blank">Comment [% row.index %]</a>:
+<li><a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
 [% row.comment_author_name FILTER html_entity -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
 [%- IF row.comment_title.length > 0 %]
 "[% row.comment_title %]"
@@ -199,17 +199,16 @@ There were no comments this month.
 [%- END %]
 
 [%# ---------------------------------------------------------------------------------------- -%]
-[%- IF includefile.defined -%]
+[%- IF mailnotes == 1 -%]
 <h2>Mailing List discussions</h2>
 <p>
 Policy decisions surrounding HPR are taken by the community as a whole. This
-discussion takes place on the <a href="https://hackerpublicradio.org/maillist"
-target="_blank">Mail List</a> which is open to all HPR listeners and
-contributors. The discussions are open and available on the HPR server under
-<a href="[% mailbase %]">Mailman</a>.
+discussion takes place on the <a href="[% mailinglist %]" target="_blank">Mailing List</a>
+which is open to all HPR listeners and contributors. The discussions are open
+and available on the HPR server under <a href="[% mailbase %]">Mailman</a>.
 </p>
 <p>The threaded discussions this month can be found here:</p>
-[% INCLUDE $includefile -%]
+<a href="[% mailthreads %]" target="_blank">[% mailthreads %]</a>
 [%- END %]
 
 [%# ---------------------------------------------------------------------------------------- -%]
@@ -231,4 +230,3 @@ page.</blockquote>
 [%#
   # vim: syntax=tt2:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
 -%]
-

Community_News/shownote_template12.tpl

@@ -0,0 +1,226 @@
+[%# shownote_template12.tpl Updated: 2025-03-31 -%]
+[%# -------------------------------------------------------------------------------- -%]
+[%# Makes either an HTML snippet for insertion into the database or a full           -%]
+[%# listing with full comments for circulation to the hosts recording the episode    -%]
+[%# This one uses the new format for the mailing list data, and partitions           -%]
+[%# comments into past and current. It also marks comments that don't need           -%]
+[%# to be read when mark_comments is true. It requires make_shownotes >= V0.0.30     -%]
+[%# -------------------------------------------------------------------------------- -%]
+[%- USE date -%]
+[%- USE pad4 = format('%04d') -%]
+[%- correspondents = "https://hackerpublicradio.org/correspondents"
+    mailinglist = "https://lists.hackerpublicradio.com/mailman/listinfo/hpr"
+    mailbase="https://lists.hackerpublicradio.com/pipermail/hpr"
+    mailthreads = "$mailbase/$review_year-$review_month/thread.html" -%]
+[%- DEFAULT skip_comments = 0
+    mark_comments = 0
+    ctext = 0
+    ignore_count = 0
+    missed_count = 0
+    past_count = 0
+-%]
+[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
+[%# Embedded CSS. The 'table' and 'hr' settings are always there but the rest is     -%]
+[%# only for when we are marking comments                                            -%]
+<style>
+table td.shrink {
+    white-space:nowrap
+}
+hr.thin {
+    border: 0;
+    height: 0;
+    border-top: 1px solid rgba(0, 0, 0, 0.1);
+    border-bottom: 1px solid rgba(255, 255, 255, 0.3);
+}
+[%- IF mark_comments == 1 %]
+p#ignore, li#ignore {
+    background-color: lightgreen;
+    color:maroon;
+}
+div#highlight {
+  border-style: solid;
+  border-color: red;
+  padding-right: 20px;
+  padding-left: 20px;
+}
+[%- END %]
+</style>
+[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
+<h2>New hosts</h2>
+<p>
+[% IF hosts.size > 0 -%]
+Welcome to our new host[%- hosts.size > 1 ? 's' : '' -%]: <br />
+[%- count = 0 %]
+[%# List the new hosts. If a name contains a comma quote it.                  -%]
+[%- FOREACH row IN hosts %]
+    [%- count = count + 1 %]
+    [%- hostname = (row.host.search(',') ? row.host.replace('^(.*)$','"$1"') : row.host) %]
+    <a href="[% correspondents %]/[% pad4(row.hostid) %].html" target="_blank">[% hostname %]</a>
+    [%- count < hosts.size ? ', ' : '.' %]
+[%- END %]
+[% ELSE -%]
+There were no new hosts this month.
+[% END -%]
+</p>
+
+<h2>Last Month's Shows</h2>
+[%# The id 't01' is in the HPR CSS but might give trouble on the IA -%]
+<table id="t01">
+<tr>
+<th>Id</th>
+<th>Day</th>
+<th>Date</th>
+<th>Title</th>
+<th>Host</th>
+</tr>
+[%- FOREACH row IN shows %]
+<tr>
+<td><strong><a href="https://hackerpublicradio.org/eps/hpr[% pad4(row.eps_id) %]/index.html" target="_blank">[% row.eps_id %]</a></strong></td>
+<td>[% date.format(row.date,'%a') %]</td>
+<td class="shrink">[% date.format(row.date,'%Y-%m-%d') %]</td>
+<td><a href="https://hackerpublicradio.org/eps/hpr[% pad4(row.eps_id) %]/index.html" target="_blank">[% row.title %]</a></td>
+<td><a href="[% correspondents %]/[% pad4(row.ho_hostid) %].html" target="_blank">[% row.ho_host FILTER html_entity %]</a></td>
+</tr>
+[%- END %]
+</table>
+
+[%# ---------------------------------------------------------------------------------------- -%]
+[%# Skip comments if told to by the caller -%]
+[%- IF skip_comments == 0 -%]
+[%# Handle any missed comments if mark_comments is true -%]
+[%- IF (mark_comments == 1) AND (missed_count > 0) -%]
+<br/><div id="highlight">
+<h2>Missed comment[%- missed_comments.size > 1 ? 's' : '' -%] last month</h2>
+<p><b>Note to Volunteers</b>: These are comments for shows last month that were not read in the last show because they arrived on or after the recording started. This section will be removed before these notes are released.</p>
+<ul>
+[%- FOREACH comment IN missed_comments -%]
+<li><strong><a href="[% comment.identifier_url %]#comments" target="_blank">hpr[% pad4(comment.episode) %]</a></strong>
+([% comment.date %]) "<em>[% comment.title %]</em>" by <a href="[% correspondents %]/[% pad4(comment.hostid) %].html" target="_blank">[% comment.host %]</a>.<br/>
+<small>Summary: "<em>[% comment.summary %]</em>"</small><br/>
+From: [% comment.comment_author_name FILTER html_entity -%] on [% date.format(comment.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
+[%- IF comment.comment_title.length > 0 %]
+"[% comment.comment_title %]"
+[%- ELSE -%]
+"[no title]"
+[%- END -%]
+<br/><hr class="thin">[% comment.comment_text FILTER html_line_break %]
+</li><br/>
+[%- END -%]
+</ul></div>
+[%- END -%]
+[%# ---------------------------------------------------------------------------------------- -%]
+<h2>Comments this month</h2>
+
+[% IF comment_count > 0 -%]
+[%- IF (mark_comments == 1) AND (ignore_count > 0) -%]
+<p id="ignore"><b>Note to Volunteers</b>: Comments marked in green were read in the last
+Community News show and should be ignored in this one.</p>
+[%- END -%]
+<p>These are comments which have been made during the past month, either to shows released during the month or to past shows.
+There [%- comment_count == 1 ? "is $comment_count comment" : "are $comment_count comments" -%] in total.</p>
+[% IF past_count > 0 -%]
+<h3>Past shows</h3>
+<p>There [% past_count == 1 ? "is $past_count comment" : "are $past_count comments" %] on
+[% past.size %] previous [% past.size == 1 ? "show" : "shows" %]:</p>
+<ul>
+[%# Loop through by episode then by comment relating to that episode -%]
+[%-    FOREACH ep IN past.keys.nsort -%]
+[%-       arr = past.$ep -%]
+<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
+([%       arr.0.date %]) "<em>[% arr.0.title %]</em>"
+by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.<br/>
+[%-       IF mark_comments == 1 || ctext == 1 -%]
+<small>Summary: "<em>[% arr.0.summary %]</em>"</small></li>
+[%-       END %]
+<li style="list-style: none; display: inline">
+<ul>
+[%-       FOREACH row IN arr -%]
+[%#          IF mark_comments == 1 && ((row.comment_timestamp_ut <= last_recording) && (arr.0.date.substr(0,7) == last_month)) -%]
+[%#          IF mark_comments == 1 && ((row.comment_released_ut <= last_recording) && (arr.0.date.substr(0,7) == last_month)) -%]
+[%-          IF mark_comments == 1 && row.ignore == 1 -%]
+<li id="ignore">
+[%-          ELSE %]
+<li>
+[%-          END %]
+<a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
+[%        row.comment_author_name FILTER html_entity -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
+[%-          IF row.comment_title.length > 0 %]
+"[%             row.comment_title %]"
+[%-          ELSE -%]
+"[no title]"
+[%-          END -%]
+[%# Add the comment body in too if ctext is true -%]
+[%-          IF ctext == 1 %]
+<br/><hr class="thin">[% row.comment_text FILTER html_line_break %]
+</li><br/>
+[%-          ELSE -%]
+</li>
+[%-          END -%]
+[%-       END -%]
+</ul><br/>
+</li>
+[%-    END -%]
+</ul>
+[%-    IF mark_comments == 1 || ctext == 1 -%]
+<p><small><small><em>Updated on [% date.format(date.now,'%Y-%m-%d %H:%M:%S') %]</em></small></small></p>
+[%-    END -%]
+[%- END %]
+[%# ---------------------------------------------------------------------------------------- -%]
+[% cc = (comment_count - past_count) -%]
+[% IF cc > 0 -%]
+<h3>This month's shows</h3>
+<p>There [% cc == 1 ? "is $cc comment" : "are $cc comments" %] on [% current.size %] of this month's shows:</p>
+<ul>
+[%- FOREACH ep IN current.keys.nsort -%]
+[%- arr = current.$ep -%]
+<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
+([% arr.0.date %]) "<em>[% arr.0.title %]</em>"
+by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.</li>
+<li style="list-style: none; display: inline">
+<ul>
+[%- FOREACH row IN arr -%]
+<li><a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
+[% row.comment_author_name FILTER html_entity -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
+[%- IF row.comment_title.length > 0 %]
+"[% row.comment_title %]"
+[%- ELSE -%]
+"[no title]"
+[%- END -%]
+</li>
+[%- END -%]
+</ul><br/>
+</li>
+[%- END -%]
+</ul>
+[%- END %]
+[%- ELSE %]
+There were no comments this month.
+[%- END %]
+[%- END %]
+
+[%# ---------------------------------------------------------------------------------------- -%]
+[%- IF mailnotes == 1 -%]
+<h2>Mailing List discussions</h2>
+<p>
+Policy decisions surrounding HPR are taken by the community as a whole. This
+discussion takes place on the <a href="[% mailinglist %]" target="_blank">Mailing List</a>
+which is open to all HPR listeners and contributors. The discussions are open
+and available on the HPR server under <a href="[% mailbase %]">Mailman</a>.
+</p>
+<p>The threaded discussions this month can be found here:</p>
+<a href="[% mailthreads %]" target="_blank">[% mailthreads %]</a>
+[%- END %]
+
+[%# ---------------------------------------------------------------------------------------- -%]
+<h2>Events Calendar</h2>
+<p>With the kind permission of <strong>LWN.net</strong> we are linking to
+<a href="https://lwn.net/Calendar/" target="_blank">The LWN.net Community Calendar</a>.</p>
+<p>Quoting the site:</p>
+<blockquote>This is the LWN.net community event calendar, where we track
+events of interest to people using and developing Linux and free software.
+Clicking on individual events will take you to the appropriate web
+page.</blockquote>
+[%# ---------------------------------------------------------------------------------------- -%]
+[%#
+  # vim: syntax=tt2:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
+-%]

Community_News/shownotes_container.tpl

@@ -0,0 +1,255 @@
+[%# /home/cendjm/HPR/Community_News/shownotes_container.tpl 2024-06-22 -%]
+[%# Container to display Community News shownotes as an HTML page -%]
+[% DEFAULT shownotes = ""
+   episode = ""
+   month_year = ""
+-%]
+<!DOCTYPE HTML>
+<html lang="en">
+<head>
+  <title>Hacker Public Radio ~ The Technology Community Podcast</title>  <base href="https://hackerpublicradio.org/">  <meta charset="utf-8" />
+  <meta http-equiv="X-Clacks-Overhead" content="GNU Terry Pratchett" />
+  <meta http-equiv="last-modified" content="Sat, 22 Jun 2024 16:04:21 +0000">
+  <meta name="keywords" content="Technology, Tech News, Education, Training" />
+  <meta name="description" content="Hacker Public Radio is a podcast that releases shows every weekday Monday through Friday. Our shows are produced by the community (you) and can be on any topic that is of interest to hackers and hobbyists." />
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <!-- Internal CSS -->
+  <style type="text/css">
+  article, aside, dialog, figure, footer, header, hgroup, menu, nav, section {
+          display: block;
+  }
+  #list1, #list2, #list3 {
+          display:none;
+  }
+  </style>
+  <link rel="shortcut icon" href="https://hackerpublicradio.org/hpr.ico" >
+  <link rel="alternate" type="application/rss+xml" title="Hacker Public Radio Ogg Vorbis RSS" href="./hpr_ogg_rss.php" />
+  <link rel="alternate" type="application/rss+xml" title="Hacker Public Radio Speex RSS" href="./hpr_spx_rss.php" />
+  <link rel="alternate" type="application/rss+xml" title="Hacker Public Radio MP3 RSS" href="./hpr_mp3_rss.php" />
+  <link rel="alternate" type="application/rss+xml" title="Hacker Public Radio Comments RSS" href="./comments.rss" />
+  <link rel="license" title="CC BY-SA 4.0"  href="https://creativecommons.org/licenses/by-sa/4.0/" />
+  <link href="./css/hpr.css" rel="stylesheet" />
+  <!--[if IE]>
+  <link rel="stylesheet" href="./css/hpr.css" media="screen" type="text/css" />
+  <script src="/JavaScript/html5.js"></script>
+  <![endif]-->
+  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.5, user-scalable=yes"/>
+</head>
+
+<body id="give">
+<div id="container" class="shadow">
+<header>
+  <a href="./"><img id="hprlogo" src="./images/hpr_logo.png" alt="hprlogo"></a>
+  <div id="hpr_banner">
+    <p id="accessible_menu">
+      <a href="./sitemap.html">Site Map</a>
+      - <a href="#maincontent">skip to main content</a>
+      </p>
+      <h1 id="sitename">
+        <a href="./correspondents/index.html">H</a>acker
+        <a href="./comments_viewer.html">P</a>ublic
+        <a href="./syndication.html">R</a>adio
+      </h1>
+    <h2>Your ideas, projects, opinions - podcasted.</h2>
+    <h3>New episodes every weekday Monday through Friday.<br />
+    <em><small>Temporary version of the Community News notes for hosts to use when recording</small></em></h3>
+
+  </div>
+  <hr />
+
+  <nav class="menu" role="navigation">		<ul>
+			<li><a href="https://hub.hackerpublicradio.org/"><strong>⇧Upload⇧</strong></a></li>
+			<li><a href="./index.html"><strong>Home</strong></a></li>
+			<li><a href="./syndication.html"><strong>Get Shows »</strong></a></li>
+			<li><a href="./eps/index.html">Full Episode Guide</a></li>
+			<li><a href="./series/index.html">In-Depth Series</a></li>
+			<li><a href="./download.html">Download Options</a></li>
+		</ul>
+  </nav>
+</header>
+
+<main id="maincontent">
+
+
+
+<article>
+	<header>
+		<h1>hpr[% episode %] :: HPR Community News for [% month_year %]</h1>
+		<h3>HPR Volunteers talk about shows released and comments posted in [% month_year %]</h3>
+		<p class="meta"><small><a href="./eps/hpr0001/index.html" rel="first">&lt;&lt; First</a>,    <a href="./eps/hpr4190/index.html" rel="previous">&lt; Previous</a>,    Next <span>&gt;</span>    <a href="./eps/hpr4145/index.html" rel="last">Latest &gt;&gt;</a></small>		</p>
+<p><a href="./correspondents/0159.html"><img src="./images/hosts/159.png" height="80" width="80" alt="Thumbnail of HPR Volunteers"  /></a><br>Hosted by <a href="./correspondents/0159.html">HPR Volunteers</a> on <span>2024-09-02</span> is flagged as <span>Explicit</span> and is released under a <span>CC-BY-SA license</span>. <br>    <span><label>Tags:</label> <em>        <a href="./tags.html#community news">Community News</a>.</em>
+    </span>
+<label>Comments: </label><a href="./eps/hpr[% episode %]/index.html#comments" aria-label="Comments for hpr[% episode %]">(Be the first)</a>. <br>
+The show is available on the Internet Archive at: <a href="https://archive.org/details/hpr[% episode %]">https://archive.org/details/hpr[% episode %]</a><p> Listen in <a href="https://archive.org/download/hpr[% episode %]/hpr[% episode %].ogg" aria-label="Download hpr[% episode %] as">ogg</a>,
+<a href="https://archive.org/download/hpr[% episode %]/hpr[% episode %].spx" aria-label="Download hpr[% episode %] as">spx</a>,
+or <a href="https://archive.org/download/hpr[% episode %]/hpr[% episode %].mp3" aria-label="Download hpr[% episode %] as">mp3</a> format. Play now:<br>
+	<audio controls preload="none">
+	<source src="https://archive.org/download/hpr[% episode %]/hpr[% episode %].ogg" type="audio/ogg" >
+		<source src="https://archive.org/download/hpr[% episode %]/hpr[% episode %].mp3" type="audio/mpeg" >
+	</audio><br>
+	Duration: 00:00:00</p></p>
+<h3>	<label>Part of the series:</label> <a href="./series/0047.html">HPR Community News</a>.</h3>
+<p><em>A monthly look at what has been going on in the HPR community. This is a regular show scheduled for the first Monday of the month.</em></p>
+	</header>
+	<div>
+	[% INCLUDE $shownotes -%]
+	</div>
+<footer><h2>Show Transcript</h2>
+<p>Automatically generated using <a href="https://github.com/openai/whisper">whisper</a>
+<pre><code>whisper --model tiny --language en hpr[% episode %].wav</code></pre></p>
+<p>
+You can save these subtitle files to the same location as the HPR Episode, and they will automatically show in players like <a href="https://mpv.io/">mpv</a>, <a href="https://www.videolan.org/vlc/">vlc</a>. Some players allow you to specify the subtitle file location.
+</p>
+<ul>
+	<li>Text: <a href="https://archive.org/download/hpr[% episode %]/hpr[% episode %]/hpr[% episode %].txt">hpr[% episode %].txt</a></li>
+	<li><a href="https://en.wikipedia.org/wiki/WebVTT">WebVTT</a>: <a href="https://archive.org/download/hpr[% episode %]/hpr[% episode %]/hpr[% episode %].vtt">hpr[% episode %].vtt</a></li>
+	<li><a href="https://en.wikipedia.org/wiki/SubRip">SubRip</a>: <a href="https://archive.org/download/hpr[% episode %]/hpr[% episode %]/hpr[% episode %].srt">hpr[% episode %].srt</a></li>
+</ul><p><small><a href="./eps/hpr0001/index.html" rel="first">&lt;&lt; First</a>,    <a href="./eps/hpr4190/index.html" rel="previous">&lt; Previous</a>,    Next <span>&gt;</span>    <a href="./eps/hpr4145/index.html" rel="last">Latest &gt;&gt;</a></small></p>
+    </footer></article><hr />
+<h1>Comments</h1>
+<p id="comments">
+Subscribe to the comments <a href="./comments.rss">RSS</a> feed.
+</p>
+        <h2>Leave Comment</h2>
+        <p>
+        <strong>Note to Verbose Commenters</strong><br />
+        If you can't fit everything you want to say in the comment below then you really should <a href="https://hackerpublicradio.org/about.html#so_you_want_to_record_a_podcast">record</a> a response show instead.
+        </p>
+        <p>
+        <strong>Note to Spammers</strong><br />
+		All comments are moderated. All links are checked by humans. We strip out all html. Feel free to <a href="https://hackerpublicradio.org/about.html#so_you_want_to_record_a_podcast">record</a> a show about yourself, or your industry, or any other topic we may find interesting. <em>We also check shows for spam :)</em>.
+        </p>
+        <form method="POST" action="https://hub.hackerpublicradio.org/comment_confirm.php">
+        <fieldset>
+        <legend>Provide feedback</legend>
+        <table>
+        <tr>
+            <td>Your Name/Handle:</td>
+            <td><input required type="text" name="comment_author_name" size="40" maxlength="40" placeholder="Enter your name" ></td>
+        </tr>
+        <tr>
+            <td>Title:</td>
+            <td><input required type="text" name="comment_title" size="50" maxlength="100" placeholder="What is your comment about?"></td>
+        </tr>
+        <tr>
+            <td>Comment:</td>
+            <td><textarea required name="comment_text" maxlength="2000" rows="10" cols="50" placeholder="Place the comment here."></textarea></td>
+        </tr>
+        <tr>
+            <td>Anti Spam Question:</td>
+            <td>
+            What does the letter <strong>P</strong> in <em>HPR</em> stand for? <br />
+            <input required type="text" name="anti_spam_question" size="50" maxlength="100" placeholder="Type out what the P in HPR stands for."></td>
+	</tr><!-- . -->
+	<tr>
+	<td>Are you a spammer?</td>
+	<td>
+	<input required checked="checked" type="radio" name="spammer" id="spammer_yes" value="Yes">
+	<label for="spammer_yes">Yes</label>
+	<input required type="radio" name="spammer" id="spammer_no" value="No">
+	<label for="spammer_no">No</label>
+	</td>
+	</tr>
+<!-- . -->
+	<tr>
+	<td>What is the <strong>HOST_ID</strong> for the host of this show?</td>
+	<td>
+	<input required type="text" name="hostid" size="20" maxlength="5" placeholder="Type the host number"></td>
+	<td>
+<!-- . -->
+	<tr>
+	<td>What does HPR mean to you?</td>
+	<td><textarea required name="justification" maxlength="200" rows="4" cols="50" placeholder="Convince us you are part of the community."></textarea></td>
+	</tr>	<tr><td>
+        <input type="hidden" name="eps_id" value="[% episode %]">
+	</td></tr>
+        </table>
+        <input type="submit" value="Next">
+        </fieldset>
+        </form>
+</main>
+
+<footer id="footer_page">
+  <h1 class="thick_bar"><span style="padding-left: 1em;">More Information...</span></h1>
+  <div id="more_info">
+    <nav class="column">
+      <h2>Ancestry</h2>
+      <ul>
+        <li><a href="http://audio.textfiles.com/shows/radiofreekamerica/">Radio Freek America</a></li>
+        <li><a href="http://audio.textfiles.com/shows/binrev/">BinRev Radio</a></li>
+        <li><a href="http://audio.textfiles.com/shows/infonomicon/">Infonomicon</a></li>
+        <li><a href="http://audio.textfiles.com/shows/twat/">Today With a Techie</a></li>
+      </ul>
+    </nav>
+    <nav class="column">
+      <h2>Social</h2>
+      <ul>
+        <li><a href="https://hackerpublicradio.org/maillist" >Mailing list</a></li>
+        <li><a href="https://botsin.space/@hpr" >Mastodon</a></li>
+        <li><a href="https://matrix.to/#/#hpr:matrix.org" >Matrix</a></li>
+        <li><a href="mumble://chatter.skyehaven.net:64738/Hacker%20Public%20Radio?version=1.2.0" >Mumble</a></li>
+        <li><a href="https://web.libera.chat/gamja/?channels=oggcastplanet" target="_blank">#oggcastplanet</a></li>
+        <li><a href="https://t.me/+6fEhQrf5IEc4ZGU8">Telegram</a></li>
+        <li><a href="https://twitter.com/HPR">Twitter.com</a></li>
+        <li><a href="https://www.facebook.com/HenryPartickReilly" target="_blank">Facebook</a></li>
+        <li><a href="https://www.linkedin.com/company/hackerpublicradio/" target="_blank">Linked-In</a></li>
+      </ul>
+    </nav>
+    <nav class="column">
+      <h2>Unaffiliates</h2>
+      <ul>
+        <li><a href="https://archive.org/details/hackerpublicradio">Archive.org</a></li>
+        <li><a href="https://music.amazon.fr/podcasts/9d9e6211-ff78-4501-93b6-6a9e560c4dbd/hacker-public-radio">Amazon Music</a></li>
+        <li><a href="https://podcasts.google.com/feed/aHR0cDovL2hhY2tlcnB1YmxpY3JhZGlvLm9yZy9ocHJfcnNzLnBocA">Google Podcasts</a></li>
+        <li><a href="https://www.iheart.com/podcast/256-hacker-public-radio-30994513/" target="_blank">iHeart Radio</a></li>
+        <li><a href="https://podcasts.apple.com/us/podcast/hacker-public-radio/id281699640">iTunes</a></li>
+        <li><a href="https://www.listennotes.com/de/podcasts/hacker-public-radio-hacker-public-radio-mNH-jsI7LcJ/">Listen Notes</a></li>
+        <li><a href="https://www.mixcloud.com/hackerpublicradio/">MixCloud</a></li>
+        <li><a href="https://player.fm/series/hacker-public-radio">PlayerFM</a></li>
+        <li><a href="https://www.podchaser.com/podcasts/hacker-public-radio-76781">Podchaser</a></li>
+        <li><a href="https://nl.radio.net/podcast/hacker-public-radio">Radio.net</a></li>
+        <li><a href="https://open.spotify.com/show/7e2hYcnHj9vKgUzsIOf4r3">Spotify</a></li>
+        <li><a href="https://toppodcast.com/podcast_feeds/hacker-public-radio/">Top Podcasts</a></li>
+      </ul>
+    </nav>
+    <nav class="column">
+      <h2>Commons</h2>
+      <ul>
+        <li><a href="https://freeculturepodcasts.org/">Free Culture Podcasts</a></li>
+        <li><a href="https://archive.org/details/hackerpublicradio">archive.org</a></li>
+        <li><a href="https://repo.anhonesthost.net/explore/repos" >HPR Source Code</a></li>
+        <li><a href="https://cchits.net/">cchits.net</a></li>
+        <li><a href="https://freesound.org/">freesound.org</a></li>
+        <li><a href="https://librivox.org/">librivox.org</a></li>
+        <li><a href="https://openclipart.org/">openclipart.org</a></li>
+        <li><a href="https://openfontlibrary.org/">openfontlibrary.org</a></li>
+        <li><a href="https://www.openrouteservice.org/">openrouteservice.org/</a></li>
+        <li><a href="https://pixabay.com/">pixabay.com/</a></li>
+      </ul>
+    </nav>
+    <nav class="column">
+      <h2>Patrons</h2>
+      <ul>
+        <li><a href="https://anhonesthost.com/hosting/shared-hosting">AnHonestHost.com</a></li>
+        <li><a href="https://archive.org/donate/">Archive.org</a></li>
+        <li><a href="https://rsync.net/">rsync.net</a></li>
+      </ul>
+    </nav>
+  </div><!-- more_info -->
+  <h1 class="thick_bar"><span style="padding-left: 1em;">Copyright Information</span></h1>
+  <div id="copyright">
+    <p>
+    Unless otherwise stated, our shows are released under a <a rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/">
+    Creative Commons Attribution-ShareAlike 4.0 International (CC BY-SA 4.0)</a> license.</p>
+    <p>
+    The <span property="dct:title">HPR Website Design</span> is released to the <a rel="license" href="https://creativecommons.org/publicdomain/mark/1.0/">Public Domain</a>.
+    </p>
+    <hr />
+  </div><!-- copyright -->
+  <hr />
+</footer>
+</div>
+<!-- shadow -->
+[%#
+vim: syntax=html:ts=8:sw=4:ai:et:tw=78:fo=tcqn:fdm=marker:com+=fb\:-
+-%]

Containerfile

@@ -0,0 +1,62 @@
+FROM perl:5.40.1 As base
+
+LABEL author="sgoti" \
+      email="lyunpaw@gmail.com" \
+      project="Hacker Public Radio" \
+      forge="https://repo.anhonesthost.net/HPR"
+
+ARG unprivilegedUser="janitor"
+
+RUN apt update && apt upgrade --yes;
+
+RUN apt install --no-install-recommends sqlite3 git --yes \
+    && rm --recursive --force /var/lib/apt/lists/*;
+
+RUN mkdir --verbose --parent /opt/hpr /tmp/hpr;
+
+RUN groupadd --system ${unprivilegedUser} \
+    && useradd --system --no-log-init --gid ${unprivilegedUser} ${unprivilegedUser};
+
+RUN chown --recursive ${unprivilegedUser}:${unprivilegedUser} /opt/hpr \
+    && chown --recursive ${unprivilegedUser}:${unprivilegedUser} /tmp/hpr;
+
+#Bill of particulars.
+
+##Meta::CPAN (Comprehensive Perl Archive Network)
+RUN cpanm Config::General \
+    DBD::SQLite \
+    DBI \
+    Data::Dumper \
+    Date::Calc \
+    Date::Parse \
+    DateTime \
+    DateTime::Duration \
+    DateTime::Format::Duration \
+    DateTime::TimeZone \
+    HTML::Entities \
+    JSON \
+    Template \
+    Template::Filters;
+
+##Included perl core modules (standard library).
+##Carp
+##Cwd
+##Getopt::Long
+##Pod::Usage
+##File::Copy
+
+USER ${unprivilegedUser}
+
+WORKDIR /opt/hpr
+
+RUN git clone https://repo.anhonesthost.net/HPR/hpr-tools.git \
+    && git clone https://repo.anhonesthost.net/HPR/hpr_hub.git \
+    && git clone https://repo.anhonesthost.net/HPR/hpr_generator.git \
+    && git clone https://repo.anhonesthost.net/HPR/hpr_documentation.git;
+
+WORKDIR /opt/hpr/hpr-tools/Community_News/
+RUN ln --symbolic /opt/hpr/hpr_generator/utils/mysql2sqlite /opt/hpr/hpr-tools/Community_News/mysql2sqlite;
+RUN ./collect_HPR_database;
+
+CMD bash;
+

Database/edit_episode

@@ -14,9 +14,9 @@
 #         BUGS: ---
 #        NOTES: Had to revert to MySQL due to a problem with DBD::MariaDB
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
-#      VERSION: 0.1.3
+#      VERSION: 0.1.4
 #      CREATED: 2015-06-17 23:17:50
-#     REVISION: 2022-02-16 20:07:45
+#     REVISION: 2024-07-20 11:21:19
 #
 #===============================================================================
 
@@ -39,7 +39,7 @@ use Data::Dumper;
 #
 # Version number (manually incremented)
 #
-our $VERSION = '0.1.3';
+our $VERSION = '0.1.4';
 
 #
 # Script and directory names
@@ -301,11 +301,11 @@ sub change_episode {
             #
             if ($rv) {
                 my $ccount = scalar( keys(%changes) );
-                printf "Updated database (%d %s to the eps row)\n",
-                    $ccount, _plural( 'change', $ccount );
+                printf "Updated database (%d %s to the eps row for show %s)\n",
+                    $ccount, _plural( 'change', $ccount ), $show;
             }
             else {
-                print "Episode not updated due to error\n";
+                print "Episode $show not updated due to error\n";
             }
 
         }
@@ -603,7 +603,7 @@ edit_episode - edit one or more fields in the database for a given HPR show
 
 =head1 VERSION
 
-This documentation refers to edit_episode version 0.1.3
+This documentation refers to edit_episode version 0.1.4
 
 
 =head1 USAGE

Database/hosts_list.tpl

@@ -0,0 +1,39 @@
+[%# ==========================================================================
+This is the TT2 file for making a list of hosts contributing to HPR in the
+    current year which is run in conjunction with 'query2tt2'. It's invoked
+    thus:
+        year="2022" # or whatever
+        ./query2tt2 -query=$HOME/HPR/Community_News/hosts_showcount.sql \
+            -conf=$HOME/HPR/.hpr_livedb.cfg \
+            -dbargs "${year}-01-01" -dbargs "${year}-12-31" -def year=${year} \
+            -template=$HOME/HPR/Community_News/hosts_list.tpl \
+            > $HOME/HPR/Community_News/hosts_showcount_${year}.html
+
+    [We can't use the planned pure TT2 version since Template::Plugin::DBI
+    can't run over the SSH tunnel.]
+
+2023-10-30 The correspondent URL has changed with the static site, and needs
+the hostid to be zero-padded.
+========================================================================== -%]
+[%- USE date -%]
+[%- DEFAULT
+    year = date.format(date.now,'%Y','UTC')
+-%]
+<h3>Thanks to all [% result.size %] HPR contributors in [% year %]!</h3>
+
+[% limit = 8 -%]
+[% count = 0 -%]
+<p><ul><li>
+[% FOREACH h = result -%]
+<a href="https://hackerpublicradio.org/correspondents/[% h.hostid %].html">[% h.hostname %]</a>
+[%- IF loop.count mod limit == 0 || loop.count == result.size -%].[% ELSE %],[% END %]
+[% count = count + 1 -%]
+[% IF count == limit -%]
+[% count = 0 -%]
+</li><li>
+[% END -%]
+[% END -%]
+</li></ul></p>
+[%#
+  # vim: syntax=tt2:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker
+-%]

Database/hosts_showcount.sqlite.sql

@@ -0,0 +1,15 @@
+--
+-- Query for use with 'query2tt2' to generate a list of hosts who contributed
+-- shows in a particular year. Designed to be used with the 'hosts_list.tpl'
+-- template.
+-- The two '?' placeholders in the query are to be filled with 'YYYY-01-01'
+-- for the start of the year and 'YYYY-12-31'. The values can be passed using
+-- the '-dbargs' option to 'query2tt2'.
+--
+SELECT DISTINCT
+    printf('%04d',h.hostid) AS hostid, h.host AS hostname
+FROM eps e
+JOIN hosts h ON e.hostid = h.hostid
+WHERE e.date BETWEEN ? AND ?
+AND title != 'Reserved'
+ORDER BY h.host

Database/query2csv

@@ -3,30 +3,41 @@
 #
 #         FILE: query2csv
 #
-#        USAGE: ./query2csv query
+#        USAGE: ./query2csv [-help] [-documentation|-man] [-debug=N]
+#               [-config=FILE] [-query=FILE]
+#               [-dbarg=ARG1 [-dbarg=ARG2] ...] [-[no-]header] [QUERY]
 #
-#  DESCRIPTION: Runs a query given as the only argument. Caution is needed
-#               since *any* query will be run. The result of the query is
-#               output in CSV form on STDOUT. The CSV is always quoted to
-#               cater for the more simplistic consumers.
+#  DESCRIPTION: Runs a query given as the only argument, or provided in
+#               a file. Caution is needed since *any* query will be run. The
+#               result of the query is output in CSV form on STDOUT or to
+#               a file. The -header option allows a CSV header of column names
+#               to be added (default not added). If the query contains '?'
+#               placeholders they can be filled with -dbarg=ARG options.
 #
 #      OPTIONS: ---
 # REQUIREMENTS: ---
 #         BUGS: ---
-#        NOTES: Had to revert to MySQL because of a problem with DBD::MariaDB
+#        NOTES: ---
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
-#      VERSION: 0.0.2
+#      VERSION: 0.0.5
 #      CREATED: 2015-07-11 15:53:01
-#     REVISION: 2022-02-16 23:17:16
+#     REVISION: 2025-05-25 18:26:13
 #
 #===============================================================================
 
-use 5.010;
-use strict;
-use warnings;
+use v5.40;
 use utf8;
+use open ':std', ':encoding(UTF-8)';
+
+use feature qw{ say try };
+
+use Cwd qw( getcwd abs_path );                 # Detecting where the script lives
+
+use Getopt::Long;
+use Pod::Usage;
 
 use Config::General;
+use File::Slurper qw{ read_text };
 use Text::CSV_XS;
 use DBI;
 
@@ -35,14 +46,12 @@ use Data::Dumper;
 #
 # Version number (manually incremented)
 #
-our $VERSION = '0.0.2';
+our $VERSION = '0.0.5';
 
 #
-# Script and directory names
+# Script name
 #
 ( my $PROG = $0 ) =~ s|.*/||mx;
-( my $DIR  = $0 ) =~ s|/?[^/]*$||mx;
-$DIR = '.' unless $DIR;
 
 #-------------------------------------------------------------------------------
 # Declarations
@@ -50,58 +59,147 @@ $DIR = '.' unless $DIR;
 #
 # Constants and other declarations
 #
-my $basedir    = "$ENV{HOME}/HPR/Database";
-my $configfile = "$basedir/.hpr_livedb.cfg";
+#
+# Make a variable to hold the working directory where the script is located
+#
+( my $basedir = abs_path($0) ) =~ s|/?[^/]*$||mx;
+
+my $configfile = "$basedir/.hpr_sqlite.cfg";
 
 my ( $dbh, $sth1, $aref1 );
-my ( $query, $csv );
-
-#
-# Enable Unicode mode
-#
-binmode STDOUT, ":encoding(UTF-8)";
-binmode STDERR, ":encoding(UTF-8)";
-
-#
-# Load database configuration data
-#
-my $conf = Config::General->new(
-    -ConfigFile      => $configfile,
-    -InterPolateVars => 1,
-    -ExtendedAccess  => 1
-);
-my %config = $conf->getall();
+my ( $query, @names, $csv );
+my ( $pcount, $acount );
 
 #-------------------------------------------------------------------------------
 # Options and arguments
 #-------------------------------------------------------------------------------
-$query = shift;
-die "Usage: $PROG query\n" unless $query;
+my %options;
+Options( \%options );
+
+#
+# Default help
+#
+pod2usage( -msg => "Version $VERSION\n", -exitval => 1, -verbose => 0 )
+    if ( $options{'help'} );
+
+#
+# Full documentation if requested with -doc
+#
+pod2usage(
+    -msg => "$PROG version $VERSION\n",
+    -verbose => 2,
+    -exitval => 1,
+    -noperldoc => 0,
+) if ( $options{'documentation'} );
+
+
+#
+# Collect options
+#
+my $DEBUG = ( $options{'debug'} ? $options{'debug'} : 0 );
+
+my $cfgfile
+    = ( defined( $options{config} ) ? $options{config} : $configfile );
+
+my $queryfile = $options{query};
+
+my $header = ( defined( $options{header} ) ? $options{header} : 0 );
+
+my $outfile = $options{output};
+_debug( $DEBUG >= 3, '$outfile: ' . $outfile ) if ($outfile);
+
+my @dbargs = _dbargs( \%options );
+_debug( $DEBUG >= 3, '@dbargs: ' . join( ',', @dbargs ) );
+
+#-------------------------------------------------------------------------------
+# Option checks and defaults
+#-------------------------------------------------------------------------------
+die "Unable to find configuration file $cfgfile\n" unless ( -e $cfgfile );
+_debug( $DEBUG >= 3, '$cfgfile: ' . $cfgfile );
+
+#
+# Query is an argument string or is in a file
+#
+if ($queryfile) {
+    die "Unable to find query file $queryfile\n" unless ( -e $queryfile );
+    $query = read_text($queryfile);
+}
+else {
+    $query = shift;
+    pod2usage(
+        -msg     => "Please specify a SQL query\n",
+        -exitval => 1,
+        -verbose => 0
+        )
+        unless $query;
+}
+_debug( $DEBUG >= 3, '$query: ' . Dumper(\$query) );
+
+#
+# Strip SQL comments
+#
+$query = strip_sql_comments($query);
+
+#
+# Count placeholders in the query and the arguments provided. First remove all
+# comments which may contain '?' characters, then count any that are left.
+#
+#$query = join("\n", grep {!/^--/} split( "\n", $query ) );
+$pcount = grep {/\?/} split( '', $query );
+$acount = scalar(@dbargs);
+
+#
+# Check the placeholder and argument counts are the same
+#
+if ( $pcount ne $acount) {
+    say STDERR "Query placeholder vs argument mismatch";
+    say STDERR "Placeholders = $pcount, Arguments = $acount";
+    pod2usage(
+        -msg       => "Wrong number of DB arguments\n",
+        -exitvalue => 1,
+        -verbose   => 0
+    );
+}
+
+#-------------------------------------------------------------------------------
+# Open the output file (or STDOUT)
+#-------------------------------------------------------------------------------
+my $outfh;
+if ($outfile) {
+    open( $outfh, ">:encoding(UTF-8)", $outfile )
+        or die "Unable to open $outfile for writing: $!";
+}
+else {
+    open( $outfh, ">&", \*STDOUT )
+        or die "Unable to initialise for writing: $!";
+}
+
+#-------------------------------------------------------------------------------
+# Load database configuration data; allow environment variables
+#-------------------------------------------------------------------------------
+my $conf = Config::General->new(
+    -ConfigFile      => $cfgfile,
+    -InterPolateVars => 1,
+    -InterPolateEnv  => 1,
+    -ExtendedAccess  => 1
+);
+my %config = $conf->getall();
+
+#
+# Set defaults in case values have been omitted
+#
+$config{database}->{dbtype} //= 'SQLite';
+$config{database}->{host}   //= '127.0.0.1';
+$config{database}->{port}   //= 3306;
 
 #-------------------------------------------------------------------------------
 # Connect to the database
 #-------------------------------------------------------------------------------
-my $dbhost = $config{database}->{host} // '127.0.0.1';
-my $dbport = $config{database}->{port} // 3306;
-my $dbname = $config{database}->{name};
-my $dbuser = $config{database}->{user};
-my $dbpwd  = $config{database}->{password};
-#$dbh = DBI->connect( "DBI:MariaDB:host=$dbhost;port=$dbport;database=$dbname",
-#    $dbuser, $dbpwd, { AutoCommit => 1 } )
-#    or die $DBI::errstr;
+$dbh = db_connect(\%config);
 
-$dbh = DBI->connect( "dbi:mysql:host=$dbhost;port=$dbport;database=$dbname",
-    $dbuser, $dbpwd, { AutoCommit => 1 } )
-    or die $DBI::errstr;
-
-#
-# Enable client-side UTF8
-#
-$dbh->{mysql_enable_utf8} = 1;
-
-#
-# Set up the query
-#
+#-------------------------------------------------------------------------------
+# Set up and perform the query
+#-------------------------------------------------------------------------------
 $sth1 = $dbh->prepare($query) or die $DBI::errstr;
 if ( $dbh->err ) {
     warn $dbh->errstr;
@@ -110,9 +208,18 @@ if ( $dbh->err ) {
 #
 # Perform the query
 #
-$sth1->execute;
-if ( $dbh->err ) {
-    warn $dbh->errstr;
+try {
+    $sth1->execute(@dbargs);
+    if ( $dbh->err ) {
+        warn $dbh->errstr;
+    }
+}
+catch ($e) {
+    #
+    # The 'die' above was triggered. The error is in $_.
+    #
+    say STDERR "Failed to execute query.";
+    exit 1;
 }
 
 #
@@ -122,16 +229,430 @@ $csv = Text::CSV_XS->new(
 #    { always_quote => 1 }
 );
 
+#
+# Collect field names  and output them at the start of the CSV if requested
+#
+if ($header) {
+    @names = @{$sth1->{NAME}};
+    _debug( $DEBUG >= 3, '@names: ' . Dumper(\@names) );
+
+    $csv->combine(@names);
+    say $outfh $csv->string();
+}
+
 #
 # Loop through the returned rows making and printing CSV. Each row is returned
 # as an arrayref to make it easy to join everything.
 #
 while ( $aref1 = $sth1->fetchrow_arrayref ) {
     $csv->combine(@$aref1);
-    print $csv->string(), "\n";
+    say $outfh $csv->string();
 }
+close($outfh);
 
 exit;
 
+#===  FUNCTION  ================================================================
+#         NAME: db_connect
+#      PURPOSE: Connects to a database using configuration settings including
+#               the database type
+#   PARAMETERS: $cfg            Config::General object
+#      RETURNS: Database handle
+#  DESCRIPTION: The 'dbtype' field in the configuration file gets a default,
+#               but the 'name' field is mandatory. Depending on the
+#               (lowercase) type name a different form of database connect is
+#               performed after '$dbtype' is set to the format the DBD driver
+#               needs. The database handle is returned (unless there's an
+#               error).
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub db_connect {                                                            #{{{
+    my ($cfg) = @_;
+
+    my ( $dbh, $dbtype, $dbname );
+
+    $dbtype = $cfg->{database}->{dbtype};
+    $dbname = $cfg->{database}->{name};
+    die "Database name is mandatory\n" unless $dbname;
+
+    #
+    # Connect according to the database type
+    #
+    if ($dbtype =~ /sqlite/i) {
+        #
+        # The name for the SQLite driver is 'DBD:SQLite'
+        #
+        $dbtype = 'SQLite';
+        _debug( $DEBUG >= 3, '$dbtype: ' . $dbtype, '$dbname: ' . $dbname );
+
+        $dbh = DBI->connect( "DBI:$dbtype:dbname=$dbname",
+            "", "", { AutoCommit => 1, sqlite_unicode => 1,  } )
+            or die $DBI::errstr;
+    }
+    elsif ($dbtype =~ /mysql/i) {
+        #
+        # The name for the MySQL driver is 'DBD:mysql'
+        #
+        $dbtype = 'mysql';
+
+        my $dbhost = $cfg->{database}->{host};
+        my $dbport = $cfg->{database}->{port};
+        my $dbuser = $cfg->{database}->{user};
+        my $dbpwd  = $cfg->{database}->{password};
+
+        $dbh = DBI->connect( "DBI:$dbtype:host=$dbhost;port=$dbport;database=$dbname",
+            $dbuser, $dbpwd, { AutoCommit => 1 } )
+            or die $DBI::errstr;
+
+        #
+        # Enable client-side UTF8
+        #
+        $dbh->{mysql_enable_utf8} = 1;
+
+        # $dbh->trace('2|SQL');
+    }
+    elsif ($dbtype =~ /pg/i) {
+        #
+        # The name for the PostgreSQL driver is 'DBD:Pg'
+        #
+        $dbtype = 'Pg';
+
+        die "The PostgreSQL database type is not catered for yet.\n";
+    }
+    else {
+        die "Unknown database type: $dbtype\n";
+    }
+
+    return $dbh;
+}                                                                           #}}}
+
+#===  FUNCTION  ================================================================
+#         NAME: strip_sql_comments
+#      PURPOSE: Given a query as a scalar, strips all SQL comments
+#   PARAMETERS: $query          string containing a query
+#      RETURNS: Stripped string
+#  DESCRIPTION: Two types of comments might exist in the query: the C-style
+#               and the SQL style. The string is treated as a single string
+#               even though it's multi-line, and any C-style comments are
+#               removed. Then the string is treated as multi-line and each
+#               line is scanned for SQL comments (which end at the end of the
+#               line), and these are stripped. Blank lines are skipped too to
+#               compress the output a little.
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub strip_sql_comments {                                                    #{{{
+    my ($query) = @_;
+
+    my $result;
+
+    #
+    # Strip C-style comments
+    #
+    $query =~ s/\/\*.*?\*\///sg;
+
+    #
+    # Strip SQL line-oriented comments
+    #
+    foreach my $line (split(/\n/,$query)) {
+        next if $line =~ /^\s*$/;
+        $line =~ s/--.*$//;
+        $result .= "$line\n";
+    }
+
+    return $result;
+}                                                                           #}}}
+
+#===  FUNCTION  ================================================================
+#         NAME: _debug
+#      PURPOSE: Prints debug reports
+#   PARAMETERS: $active         Boolean: 1 for print, 0 for no print
+#               @messages       Messages to print
+#      RETURNS: Nothing
+#  DESCRIPTION: Outputs messages if $active is true. It removes any trailing
+#               newlines and then adds one to each line so the caller doesn't
+#               have to bother. Prepends 'D> ' to each message to show it's
+#               a debug message.
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub _debug {                                                                #{{{
+    my ( $active, @messages ) = @_;
+
+    if ($active) {
+        chomp(@messages);
+        say STDERR "D> ", join( "\nD> ", @messages );
+    }
+}                                                                           #}}}
+
+#===  FUNCTION  ================================================================
+#         NAME: _dbargs
+#      PURPOSE: Collects database arguments for the main query
+#   PARAMETERS: $opts   hash reference holding the options
+#      RETURNS: An array holding all of the arguments
+#  DESCRIPTION: If there are -dbargs options they will be an array in the hash
+#               returned by Getopt::Long. We return the array to the caller.
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub _dbargs {                                                               #{{{
+    my ($opts) = @_;
+
+    my @args;
+
+    if ( defined( $opts->{dbarg} ) ) {
+        @args = @{ $opts->{dbarg} };
+    }
+
+    return (@args);
+}                                                                           #}}}
+
+#===  FUNCTION  ================================================================
+#         NAME: Options
+#      PURPOSE: Processes command-line options
+#   PARAMETERS: $optref     Hash reference to hold the options
+#      RETURNS: Undef
+#  DESCRIPTION:
+#       THROWS: no exceptions
+#     COMMENTS: none
+#     SEE ALSO: n/a
+#===============================================================================
+sub Options {                                                               #{{{
+    my ($optref) = @_;
+
+    my @options = (
+        "help",     "documentation|man", "debug=i",   "config=s",
+        "output=s", "query=s",           "dbarg=s@", "header!",
+    );
+
+    if ( !GetOptions( $optref, @options ) ) {
+        pod2usage( -msg => "Version $VERSION\n", -exitval => 1 );
+    }
+
+    return;
+}                                                                           #}}}
+
+__END__
+
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+#  Application Documentation
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+#{{{
+
+=head1 NAME
+
+query2csv - A script for generating CSV from database query
+
+=head1 VERSION
+
+This documentation refers to query2csv version 0.0.5
+
+=head1 USAGE
+
+    query2csv [-help] [-documentation|-man] [-debug=N] [-config=FILE]
+         [-query=FILE] [-output=FILE] [-[no]header] [QUERY]
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<-help>
+
+Prints a brief help message describing the usage of the program, and then exits.
+
+=item B<-documentation> or B<-man>
+
+Displays the entirety of the documentation (using a pager), and then exits. To
+generate a PDF version use:
+
+    pod2pdf query2csv --out=query2csv.pdf
+
+=item B<-debug=N>
+
+Selects a level of debugging. Debug information consists of a line or series
+of lines prefixed with the characters 'D>':
+
+=over 4
+
+=item B<0>
+
+No debug output is generated: this is the default
+
+=item B<3>
+
+Prints all data structures from options or from the database
+
+=back
+
+(The debug levels need work!)
+
+=item B<-config=FILE>
+
+This option allows an alternative configuration file to be used. This file
+defines the location of the database, its port, its name and the username and
+password to be used to access it. This feature was added to allow the script
+to access alternative databases.
+
+See the CONFIGURATION AND ENVIRONMENT section below for the file format.
+
+If the option is omitted the default file is used: B<.hpr_sqlite.cfg>
+
+=item B<-output=FILE>
+
+This option defines an output file to receive the result of the query. If the
+option is omitted the notes are written to STDOUT, allowing them to be
+redirected if required.
+
+=item B<-query=FILE>
+
+The script needs an SQL query to be applied to the database. This may be
+supplied as a file, in which case this option gives the name of the file.
+
+Alternatively the query can be given as a delimited string on the command
+line.
+
+If neither method is used the script aborts with an error message.
+
+=item B<-dbarg=ARG> [ B<-dbarg=ARG> ... ]
+
+The query can have place holders ('?') in it and the corresponding values for
+these placeholders can be passed to the script through the B<-dbarg=ARG>
+option. The option can be repeated as many times as required and the order of
+B<ARG> values is preserved.
+
+=item B<-[no-]header>
+
+This option allows a header to be added to the CSV output with the names of
+the database columns in CSV format. By default this is not produced.
+
+=back
+
+=head1 DESCRIPTION
+
+The purpose of the script is to run a query against a local database. It will
+query SQLite or MySQL/MariaDB databases.  The database choice is made via
+a configuration file using B<dbtype = TYPE>. The default file points to the
+local SQLite copy of the HPR database, but the alternative (discussed later)
+can access the equivalent MySQL database.
+
+The data returned from the query is then passed through a CSV conversion
+library and the results output to the terminal or to a file.
+
+=head1 DIAGNOSTICS
+
+=over 4
+
+=item B<Unable to find configuration file ...>
+
+The nominated (or default) configuration file could not be found.
+
+=item B<Unable to find query file ...>
+
+The nominated query file could not be found.
+
+=item B<Couldn't open ...: ...>
+
+The nominated query file could not be opened.
+
+=item B<various database errors>
+
+An error has occurred while performing a database operation.
+
+=item B<Failed to execute query.>
+
+There is a mismatch between the number of placeholders in the query ('?'
+characters) and the number of arguments provided through the B<-dbargs=ARG>
+option. The script will attempt to analyse whether there are too many or too
+few arguments
+
+=back
+
+=head1 CONFIGURATION AND ENVIRONMENT
+
+The script obtains the credentials it requires to open the SQLite or MariaDB database
+from a configuration file. No credentials are required for the SQLite format.
+The name of the MySQL database or the SQLite file it expects is specified as
+a simple name or as an absolute file path. This configuration file can be overridden
+using the B<-config=FILE> option as described above.
+
+The configuration file formats are as follows:
+
+SQLite format:
+
+ <database>
+     dbtype = SQLite
+     name   = /home/cendjm/HPR/Community_News/hpr.db
+ </database>
+
+MySQL/MariaDB format:
+
+ <database>
+     dbtype   = MySQL
+     host     = 127.0.0.1
+     name     = DBNAME
+     user     = DBUSER
+     password = PASSWORD
+ </database>
+
+=head1 EXAMPLES
+
+ query2csv -help
+
+ # Find tags fields shorter than 5 characters between shows 1 and 2000
+ query2csv -config=.hpr_sqlite.cfg \
+    'SELECT id,summary,tags FROM eps WHERE id BETWEEN 1 AND 2000
+    AND length(tags) < 5 ORDER BY id'
+
+ # Find hosts who have released HPR shows during 2025. The database
+ # arguments are the dates of the start and end of the year
+ year=2025
+ query2csv -query=hosts_showcount.sqlite.sql \
+    -dbargs "${year}-01-01" -dbargs "${year}-12-31"
+
+=head1 DEPENDENCIES
+
+    Config::General
+    Cwd
+    Data::Dumper
+    DBI
+    File::Slurper
+    Getopt::Long
+    Pod::Usage
+    Text::CSV_XS
+
+=head1 BUGS AND LIMITATIONS
+
+There are no known bugs in this module.
+Please report problems to Dave Morriss (Dave.Morriss@gmail.com)
+Patches are welcome.
+
+=head1 AUTHOR
+
+Dave Morriss (Dave.Morriss@gmail.com)
+
+
+=head1 LICENCE AND COPYRIGHT
+
+Copyright (c) 2021, 2022, 2024, 2025 Dave Morriss (Dave.Morriss@gmail.com).
+All rights reserved.
+
+This module is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself. See perldoc perlartistic.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+=cut
+
+#}}}
+
+# [zo to open fold, zc to close]
+
 # vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker
 

Database/query2json

@@ -3,29 +3,40 @@
 #
 #         FILE: query2json
 #
-#        USAGE: ./query2json query
+#        USAGE: ./query2json [-help] [-documentation|-man] [-debug=N]
+#               [-config=FILE] [-query=FILE]
+#               [-dbarg=ARG1 [-dbarg=ARG2] ...] [QUERY]
 #
-#  DESCRIPTION: Runs a query given as the only argument. Caution is needed
-#               since *any* query will be run. The result of the query is
-#               output in JSON form on STDOUT.
+#  DESCRIPTION: Runs a query given as the only argument, or provided in
+#               a file. Caution is needed since *any* query will be run. The
+#               result of the query is output in JSON form on STDOUT or to
+#               a file. If the query contains '?' placeholders they can be
+#               filled with -dbarg=ARG options.
 #
 #      OPTIONS: ---
 # REQUIREMENTS: ---
 #         BUGS: ---
-#        NOTES: Had to revert to MySQL because of a problem with DBD::MariaDB
+#        NOTES: ---
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
-#      VERSION: 0.0.2
+#      VERSION: 0.0.4
 #      CREATED: 2021-06-18 13:24:49
-#     REVISION: 2023-01-05 16:17:24
+#     REVISION: 2025-05-25 18:48:59
 #
 #===============================================================================
 
-use 5.010;
-use strict;
-use warnings;
+use v5.40;
 use utf8;
+use open ':std', ':encoding(UTF-8)';
+
+use feature qw{ say try };
+
+use Cwd qw( getcwd abs_path );                 # Detecting where the script lives
+
+use Getopt::Long;
+use Pod::Usage;
 
 use Config::General;
+use File::Slurper qw{ read_text };
 use JSON;
 use DBI;
 
@@ -34,14 +45,12 @@ use Data::Dumper;
 #
 # Version number (manually incremented)
 #
-our $VERSION = '0.0.2';
+our $VERSION = '0.0.4';
 
 #
-# Script and directory names
+# Script name
 #
 ( my $PROG = $0 ) =~ s|.*/||mx;
-( my $DIR  = $0 ) =~ s|/?[^/]*$||mx;
-$DIR = '.' unless $DIR;
 
 #-------------------------------------------------------------------------------
 # Declarations
@@ -49,58 +58,144 @@ $DIR = '.' unless $DIR;
 #
 # Constants and other declarations
 #
-my $basedir    = "$ENV{HOME}/HPR/Database";
-my $configfile = "$basedir/.hpr_livedb.cfg";
+#
+# Make a variable to hold the working directory where the script is located
+#
+( my $basedir = abs_path($0) ) =~ s|/?[^/]*$||mx;
+
+my $configfile = "$basedir/.hpr_sqlite.cfg";
 
 my ( $dbh, $sth1, $aref1 );
 my ( $query, $result, $json );
-
-#
-# Enable Unicode mode
-#
-binmode STDOUT, ":encoding(UTF-8)";
-binmode STDERR, ":encoding(UTF-8)";
-
-#
-# Load database configuration data
-#
-my $conf = Config::General->new(
-    -ConfigFile      => $configfile,
-    -InterPolateVars => 1,
-    -ExtendedAccess  => 1
-);
-my %config = $conf->getall();
+my ( $pcount, $acount );
 
 #-------------------------------------------------------------------------------
 # Options and arguments
 #-------------------------------------------------------------------------------
-$query = shift;
-die "Usage: $PROG query\n" unless $query;
+my %options;
+Options( \%options );
+
+#
+# Default help
+#
+pod2usage( -msg => "Version $VERSION\n", -exitval => 1, -verbose => 0 )
+    if ( $options{'help'} );
+
+#
+# Full documentation if requested with -doc
+#
+pod2usage(
+    -msg => "$PROG version $VERSION\n",
+    -verbose => 2,
+    -exitval => 1,
+    -noperldoc => 0,
+) if ( $options{'documentation'} );
+
+#
+# Collect options
+#
+my $DEBUG = ( $options{'debug'} ? $options{'debug'} : 0 );
+
+my $cfgfile
+    = ( defined( $options{config} ) ? $options{config} : $configfile );
+
+my $queryfile = $options{query};
+
+my $outfile = $options{output};
+_debug( $DEBUG >= 3, '$outfile: ' . $outfile ) if ($outfile);
+
+my @dbargs = _dbargs( \%options );
+_debug( $DEBUG >= 3, '@dbargs: ' . join( ',', @dbargs ) );
+
+#-------------------------------------------------------------------------------
+# Option checks and defaults
+#-------------------------------------------------------------------------------
+die "Unable to find configuration file $cfgfile\n" unless ( -e $cfgfile );
+_debug( $DEBUG >= 3, '$cfgfile: ' . $cfgfile );
+
+#
+# Query is an argument string or is in a file
+#
+if ($queryfile) {
+    die "Unable to find query file $queryfile\n" unless ( -e $queryfile );
+    $query = read_text($queryfile);
+}
+else {
+    $query = shift;
+    pod2usage(
+        -msg     => "Please specify a SQL query\n",
+        -exitval => 1,
+        -verbose => 0
+        )
+        unless $query;
+}
+_debug( $DEBUG >= 3, '$query: ' . Dumper(\$query) );
+
+#
+# Strip SQL comments
+#
+$query = strip_sql_comments($query);
+
+#
+# Count placeholders in the query and the arguments provided. First remove all
+# comments which may contain '?' characters, then count any that are left.
+#
+#$query = join("\n", grep {!/^--/} split( "\n", $query ) );
+$pcount = grep {/\?/} split( '', $query );
+$acount = scalar(@dbargs);
+
+#
+# Check the placeholder and argument counts are the same
+#
+if ( $pcount ne $acount) {
+    say STDERR "Query placeholder vs argument mismatch";
+    say STDERR "Placeholders = $pcount, Arguments = $acount";
+    pod2usage(
+        -msg       => "Wrong number of DB arguments\n",
+        -exitvalue => 1,
+        -verbose   => 0
+    );
+}
+
+#-------------------------------------------------------------------------------
+# Open the output file (or STDOUT)
+#-------------------------------------------------------------------------------
+my $outfh;
+if ($outfile) {
+    open( $outfh, ">:encoding(UTF-8)", $outfile )
+        or die "Unable to open $outfile for writing: $!";
+}
+else {
+    open( $outfh, ">&", \*STDOUT )
+        or die "Unable to initialise for writing: $!";
+}
+
+#-------------------------------------------------------------------------------
+# Load database configuration data; allow environment variables
+#-------------------------------------------------------------------------------
+my $conf = Config::General->new(
+    -ConfigFile      => $configfile,
+    -InterPolateVars => 1,
+    -InterPolateEnv  => 1,
+    -ExtendedAccess  => 1
+);
+my %config = $conf->getall();
+
+#
+# Set defaults in case values have been omitted
+#
+$config{database}->{dbtype} //= 'SQLite';
+$config{database}->{host}   //= '127.0.0.1';
+$config{database}->{port}   //= 3306;
 
 #-------------------------------------------------------------------------------
 # Connect to the database
 #-------------------------------------------------------------------------------
-my $dbhost = $config{database}->{host} // '127.0.0.1';
-my $dbport = $config{database}->{port} // 3306;
-my $dbname = $config{database}->{name};
-my $dbuser = $config{database}->{user};
-my $dbpwd  = $config{database}->{password};
-#$dbh = DBI->connect( "DBI:MariaDB:host=$dbhost;port=$dbport;database=$dbname",
-#    $dbuser, $dbpwd, { AutoCommit => 1 } )
-#    or die $DBI::errstr;
+$dbh = db_connect(\%config);
 
-$dbh = DBI->connect( "dbi:mysql:host=$dbhost;port=$dbport;database=$dbname",
-    $dbuser, $dbpwd, { AutoCommit => 1 } )
-    or die $DBI::errstr;
-
-#
-# Enable client-side UTF8
-#
-$dbh->{mysql_enable_utf8} = 1;
-
-#
-# Set up the query
-#
+#-------------------------------------------------------------------------------
+# Set up and perform the query
+#-------------------------------------------------------------------------------
 $sth1 = $dbh->prepare($query) or die $DBI::errstr;
 if ( $dbh->err ) {
     warn $dbh->errstr;
@@ -109,26 +204,434 @@ if ( $dbh->err ) {
 #
 # Perform the query
 #
-$sth1->execute;
-if ( $dbh->err ) {
-    warn $dbh->errstr;
+try {
+    $sth1->execute(@dbargs);
+    if ( $dbh->err ) {
+        warn $dbh->errstr;
+    }
 }
+catch ($e) {
+    #
+    # The 'die' above was triggered. The error is in $_.
+    #
+    say STDERR "Failed to execute query.";
+    exit 1;
+}
+
+#
+# Prepare for JSON, forcing object key sorting (expensive), and prettification
+#
+$json = JSON->new->utf8->canonical->pretty;
 
 #
 # Grab everything as an arrayref of hashrefs
 #
 $result = $sth1->fetchall_arrayref( {} );
 
-#
-# Prepare for JSON, forcing object key sorting (expensive)
-#
-$json = JSON->new->utf8->canonical;
-
 #
 # Encode the Perl structure to JSON
 #
-print $json->encode($result), "\n";
+say $outfh $json->encode($result);
+close($outfh);
 
 exit;
 
+#===  FUNCTION  ================================================================
+#         NAME: db_connect
+#      PURPOSE: Connects to a database using configuration settings including
+#               the database type
+#   PARAMETERS: $cfg            Config::General object
+#      RETURNS: Database handle
+#  DESCRIPTION: The 'dbtype' field in the configuration file gets a default,
+#               but the 'name' field is mandatory. Depending on the
+#               (lowercase) type name a different form of database connect is
+#               performed after '$dbtype' is set to the format the DBD driver
+#               needs. The database handle is returned (unless there's an
+#               error).
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub db_connect {                                                            #{{{
+    my ($cfg) = @_;
+
+    my ( $dbh, $dbtype, $dbname );
+
+    $dbtype = $cfg->{database}->{dbtype};
+    $dbname = $cfg->{database}->{name};
+    die "Database name is mandatory\n" unless $dbname;
+
+    #
+    # Connect according to the database type
+    #
+    if ($dbtype =~ /sqlite/i) {
+        #
+        # The name for the SQLite driver is 'DBD:SQLite'
+        #
+        $dbtype = 'SQLite';
+        _debug( $DEBUG >= 3, '$dbtype: ' . $dbtype, '$dbname: ' . $dbname );
+
+        $dbh = DBI->connect( "DBI:$dbtype:dbname=$dbname",
+            "", "", { AutoCommit => 1, sqlite_unicode => 1,  } )
+            or die $DBI::errstr;
+    }
+    elsif ($dbtype =~ /mysql/i) {
+        #
+        # The name for the MySQL driver is 'DBD:mysql'
+        #
+        $dbtype = 'mysql';
+
+        my $dbhost = $cfg->{database}->{host};
+        my $dbport = $cfg->{database}->{port};
+        my $dbuser = $cfg->{database}->{user};
+        my $dbpwd  = $cfg->{database}->{password};
+
+        $dbh = DBI->connect( "DBI:$dbtype:host=$dbhost;port=$dbport;database=$dbname",
+            $dbuser, $dbpwd, { AutoCommit => 1 } )
+            or die $DBI::errstr;
+
+        #
+        # Enable client-side UTF8
+        #
+        $dbh->{mysql_enable_utf8} = 1;
+
+        # $dbh->trace('2|SQL');
+    }
+    elsif ($dbtype =~ /pg/i) {
+        #
+        # The name for the PostgreSQL driver is 'DBD:Pg'
+        #
+        $dbtype = 'Pg';
+
+        die "The PostgreSQL database type is not catered for yet.\n";
+    }
+    else {
+        die "Unknown database type: $dbtype\n";
+    }
+
+    return $dbh;
+}                                                                           #}}}
+
+#===  FUNCTION  ================================================================
+#         NAME: strip_sql_comments
+#      PURPOSE: Given a query as a scalar, strips all SQL comments
+#   PARAMETERS: $query          string containing a query
+#      RETURNS: Stripped string
+#  DESCRIPTION: Two types of comments might exist in the query: the C-style
+#               and the SQL style. The string is treated as a single string
+#               even though it's multi-line, and any C-style comments are
+#               removed. Then the string is treated as multi-line and each
+#               line is scanned for SQL comments (which end at the end of the
+#               line), and these are stripped. Blank lines are skipped too to
+#               compress the output a little.
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub strip_sql_comments {                                                    #{{{
+    my ($query) = @_;
+
+    my $result;
+
+    #
+    # Strip C-style comments
+    #
+    $query =~ s/\/\*.*?\*\///sg;
+
+    #
+    # Strip SQL line-oriented comments
+    #
+    foreach my $line (split(/\n/,$query)) {
+        next if $line =~ /^\s*$/;
+        $line =~ s/--.*$//;
+        $result .= "$line\n";
+    }
+
+    return $result;
+}                                                                           #}}}
+
+#===  FUNCTION  ================================================================
+#         NAME: _debug
+#      PURPOSE: Prints debug reports
+#   PARAMETERS: $active         Boolean: 1 for print, 0 for no print
+#               @messages       Messages to print
+#      RETURNS: Nothing
+#  DESCRIPTION: Outputs messages if $active is true. It removes any trailing
+#               newlines and then adds one to each line so the caller doesn't
+#               have to bother. Prepends 'D> ' to each message to show it's
+#               a debug message.
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub _debug {                                                                #{{{
+    my ( $active, @messages ) = @_;
+
+    if ($active) {
+        chomp(@messages);
+        say STDERR "D> ", join( "\nD> ", @messages );
+    }
+}                                                                           #}}}
+
+#===  FUNCTION  ================================================================
+#         NAME: _dbargs
+#      PURPOSE: Collects database arguments for the main query
+#   PARAMETERS: $opts   hash reference holding the options
+#      RETURNS: An array holding all of the arguments
+#  DESCRIPTION: If there are -dbargs options they will be an array in the hash
+#               returned by Getopt::Long. We return the array to the caller.
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub _dbargs {                                                               #{{{
+    my ($opts) = @_;
+
+    my @args;
+
+    if ( defined( $opts->{dbarg} ) ) {
+        @args = @{ $opts->{dbarg} };
+    }
+
+    return (@args);
+}                                                                           #}}}
+
+#===  FUNCTION  ================================================================
+#         NAME: Options
+#      PURPOSE: Processes command-line options
+#   PARAMETERS: $optref     Hash reference to hold the options
+#      RETURNS: Undef
+#  DESCRIPTION:
+#       THROWS: no exceptions
+#     COMMENTS: none
+#     SEE ALSO: n/a
+#===============================================================================
+sub Options {                                                               #{{{
+    my ($optref) = @_;
+
+    my @options = (
+        "help",     "documentation|man", "debug=i", "config=s",
+        "output=s", "query=s",           "dbargs=s@",
+    );
+
+    if ( !GetOptions( $optref, @options ) ) {
+        pod2usage( -msg => "Version $VERSION\n", -exitval => 1 );
+    }
+
+    return;
+}                                                                           #}}}
+
+__END__
+
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+#  Application Documentation
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+#{{{
+
+=head1 NAME
+
+query2json - A script for generating CSV from database query
+
+=head1 VERSION
+
+This documentation refers to query2json version 0.0.4
+
+=head1 USAGE
+
+    query2json [-help] [-documentation|-man] [-debug=N] [-config=FILE]
+         [-query=FILE] [-output=FILE] [QUERY]
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<-help>
+
+Prints a brief help message describing the usage of the program, and then exits.
+
+=item B<-documentation> or B<-man>
+
+Displays the entirety of the documentation (using a pager), and then exits. To
+generate a PDF version use:
+
+    pod2pdf query2json --out=query2json.pdf
+
+=item B<-debug=N>
+
+Selects a level of debugging. Debug information consists of a line or series
+of lines prefixed with the characters 'D>':
+
+=over 4
+
+=item B<0>
+
+No debug output is generated: this is the default
+
+=item B<3>
+
+Prints all data structures from options or from the database
+
+=back
+
+(The debug levels need work!)
+
+=item B<-config=FILE>
+
+This option allows an alternative configuration file to be used. This file
+defines the location of the database, its port, its name and the username and
+password to be used to access it. This feature was added to allow the script
+to access alternative databases.
+
+See the CONFIGURATION AND ENVIRONMENT section below for the file format.
+
+If the option is omitted the default file is used: B<.hpr_sqlite.cfg>
+
+=item B<-output=FILE>
+
+This option defines an output file to receive the result of the query. If the
+option is omitted the notes are written to STDOUT, allowing them to be
+redirected if required.
+
+=item B<-query=FILE>
+
+The script needs an SQL query to be applied to the database. This may be
+supplied as a file, in which case this option gives the name of the file.
+
+Alternatively the query can be given as a delimited string on the command
+line.
+
+If neither method is used the script aborts with an error message.
+
+=item B<-dbarg=ARG> [ B<-dbarg=ARG> ... ]
+
+The query can have place holders ('?') in it and the corresponding values for
+these placeholders can be passed to the script through the B<-dbarg=ARG>
+option. The option can be repeated as many times as required and the order of
+B<ARG> values is preserved.
+
+=back
+
+=head1 DESCRIPTION
+
+The purpose of the script is to run a query against a local database. It will
+query SQLite or MySQL/MariaDB databases.  The database choice is made via
+a configuration file using B<dbtype = TYPE>. The default file points to the
+local SQLite copy of the HPR database, but the alternative (discussed later)
+can access the equivalent MySQL database.
+
+The data returned from the query is then passed through a JSON conversion
+library and the results output to the terminal or to a file.
+
+=head1 DIAGNOSTICS
+
+=over 4
+
+=item B<Unable to find configuration file ...>
+
+The nominated (or default) configuration file could not be found.
+
+=item B<Unable to find query file ...>
+
+The nominated query file could not be found.
+
+=item B<Couldn't open ...: ...>
+
+The nominated query file could not be opened.
+
+=item B<various database errors>
+
+An error has occurred while performing a database operation.
+
+=item B<Failed to execute query.>
+
+There is a mismatch between the number of placeholders in the query ('?'
+characters) and the number of arguments provided through the B<-dbargs=ARG>
+option. The script will attempt to analyse whether there are too many or too
+few arguments
+
+=back
+
+=head1 CONFIGURATION AND ENVIRONMENT
+
+The script obtains the credentials it requires to open the SQLite or MariaDB database
+from a configuration file. No credentials are required for the SQLite format.
+The name of the MySQL database or the SQLite file it expects is specified as
+a simple name or as an absolute file path. This configuration file can be overridden
+using the B<-config=FILE> option as described above.
+
+The configuration file formats are as follows:
+
+SQLite format:
+
+ <database>
+     dbtype = SQLite
+     name   = /home/cendjm/HPR/Community_News/hpr.db
+ </database>
+
+MySQL/MariaDB format:
+
+ <database>
+     dbtype   = MySQL
+     host     = 127.0.0.1
+     name     = DBNAME
+     user     = DBUSER
+     password = PASSWORD
+ </database>
+
+=head1 EXAMPLES
+
+ query2json -help
+
+ # Find tags fields shorter than 5 characters between shows 1 and 2000
+ query2json -config=.hpr_sqlite.cfg \
+    'SELECT id,summary,tags FROM eps WHERE id BETWEEN 1 AND 2000
+    AND length(tags) < 5 ORDER BY id'
+
+ # Find hosts who have released HPR shows during 2025. The database
+ # arguments are the dates of the start and end of the year
+ year=2025
+ query2json -query=hosts_showcount.sqlite.sql \
+    -dbargs "${year}-01-01" -dbargs "${year}-12-31"
+
+=head1 DEPENDENCIES
+
+    Config::General
+    Cwd
+    Data::Dumper
+    DBI
+    File::Slurper
+    Getopt::Long
+    Pod::Usage
+    Text::CSV_XS
+
+=head1 BUGS AND LIMITATIONS
+
+There are no known bugs in this module.
+Please report problems to Dave Morriss (Dave.Morriss@gmail.com)
+Patches are welcome.
+
+=head1 AUTHOR
+
+Dave Morriss (Dave.Morriss@gmail.com)
+
+
+=head1 LICENCE AND COPYRIGHT
+
+Copyright (c) 2021, 2022, 2024, 2025 Dave Morriss (Dave.Morriss@gmail.com).
+All rights reserved.
+
+This module is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself. See perldoc perlartistic.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+=cut
+
+#}}}
+
+# [zo to open fold, zc to close]
+
+
 # vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker

Database/query2tt2

@@ -3,11 +3,11 @@
 #
 #         FILE: query2tt2
 #
-#        USAGE: ./query2tt2 [-help] [-debug=N] [-config=FILE] [-query=FILE]
-#               [-template=FILE]
+#        USAGE: ./query2tt2 [-help] [-documentation|-man] [-debug=N]
+#               [-config=FILE] [-query=FILE] [-template=FILE]
 #               [-dbarg=ARG1 [-dbarg=ARG2] ...]
-#               [-define KEY1=VALUE1 [-define KEY2=VALUE2] ...
-#               [-define KEYn=VALUEn]] [QUERY]
+#               [-define KEY1=VALUE1 [-define KEY2=VALUE2] ...]
+#               [QUERY]
 #
 #  DESCRIPTION: Built for use with the Hacker Public Radio database, but could
 #               be used in any context with a MariaDB database.
@@ -24,31 +24,26 @@
 #      OPTIONS: ---
 # REQUIREMENTS: ---
 #         BUGS: ---
-#        NOTES: Had to revert to MySQL because of a problem with DBD::MariaDB
+#        NOTES: ---
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
-#      VERSION: 0.0.5
+#      VERSION: 0.0.9
 #      CREATED: 2021-06-18 13:24:49
-#     REVISION: 2024-06-29 18:42:49
+#     REVISION: 2025-05-25 18:41:09
 #
 #===============================================================================
 
-use 5.010;
-use strict;
-use warnings;
+use v5.40;
 use utf8;
-use open ':encoding(UTF-8)';
+use open ':std', ':encoding(UTF-8)';
 
-# Using experimental features, some of which require warnings to be turned off
 use feature qw{ say try };
-no warnings qw{
-    experimental::try
-};
+
+use Cwd qw( getcwd abs_path );                 # Detecting where the script lives
 
 use Getopt::Long;
 use Pod::Usage;
 
 use Config::General;
-#use Try::Tiny;
 use File::Slurper qw{ read_text };
 use Hash::Merge;
 use Template;
@@ -59,14 +54,12 @@ use Data::Dumper;
 #
 # Version number (manually incremented)
 #
-our $VERSION = '0.0.5';
+our $VERSION = '0.0.9';
 
 #
 # Script and directory names
 #
 ( my $PROG = $0 ) =~ s|.*/||mx;
-( my $DIR  = $0 ) =~ s|/?[^/]*$||mx;
-$DIR = '.' unless $DIR;
 
 #-------------------------------------------------------------------------------
 # Declarations
@@ -74,11 +67,16 @@ $DIR = '.' unless $DIR;
 #
 # Constants and other declarations
 #
-my $basedir    = "$ENV{HOME}/HPR/Database";
-my $configfile = "$basedir/.hpr_db.cfg";
+#
+# Make a variable to hold the working directory where the script is located
+#
+( my $basedir = abs_path($0) ) =~ s|/?[^/]*$||mx;
+
+my $configfile = "$basedir/.hpr_sqlite.cfg";
 
 my ( $dbh, $sth1 );
 my ( $query, $result, @names, $document );
+my ( $pcount, $acount );
 
 #
 # Default template iterates through all rows in the 'result' matrix and for
@@ -94,15 +92,10 @@ my $def_template = <<'ENDTPL';
 [% END -%]
 ENDTPL
 
-#-------------------------------------------------------------------------------
+################################################################################
 # There should be no need to edit anything after this point
-#-------------------------------------------------------------------------------
+################################################################################
 
-#
-# Enable Unicode mode
-#
-#binmode STDOUT, ":encoding(UTF-8)";
-#binmode STDERR, ":encoding(UTF-8)";
 
 #-------------------------------------------------------------------------------
 # Options and arguments
@@ -124,8 +117,7 @@ pod2usage(
     -verbose => 2,
     -exitval => 1,
     -noperldoc => 0,
-) if ( $options{'doc'} );
-
+) if ( $options{'documentation'} );
 
 #
 # Collect options
@@ -143,6 +135,9 @@ my %defs   = _define( \%options );
 _debug( $DEBUG >= 3, '@dbargs: ' . join( ',', @dbargs ) );
 _debug( $DEBUG >= 3, '%defs: ' . Dumper(\%defs) );
 
+my $outfile = $options{output};
+_debug( $DEBUG >= 3, '$outfile: ' . $outfile ) if ($outfile);
+
 #-------------------------------------------------------------------------------
 # Option checks and defaults
 #-------------------------------------------------------------------------------
@@ -158,11 +153,41 @@ if ($queryfile) {
 }
 else {
     $query = shift;
-    pod2usage( -msg => "Please specify a SQL query\n", -exitval => 1 )
+    pod2usage(
+        -msg     => "Please specify a SQL query\n",
+        -exitval => 1,
+        -verbose => 0
+        )
         unless $query;
 }
 _debug( $DEBUG >= 3, '$query: ' . Dumper(\$query) );
 
+#
+# Strip SQL comments
+#
+$query = strip_sql_comments($query);
+
+#
+# Count placeholders in the query and the arguments provided. First remove all
+# comments which may contain '?' characters, then count any that are left.
+#
+#$query = join("\n", grep {!/^--/} split( "\n", $query ) );
+$pcount = grep {/\?/} split( '', $query );
+$acount = scalar(@dbargs);
+
+#
+# Check the placeholder and argument counts are the same
+#
+if ( $pcount ne $acount) {
+    say STDERR "Query placeholder vs argument mismatch";
+    say STDERR "Placeholders = $pcount, Arguments = $acount";
+    pod2usage(
+        -msg       => "Wrong number of DB arguments\n",
+        -exitvalue => 1,
+        -verbose   => 0
+    );
+}
+
 #
 # Template is the default pre-defined string or a filename
 #
@@ -176,41 +201,49 @@ _debug(
     $DEBUG >= 3,
     '$template: '
         . (ref($template) eq ''
-        ? "filename $template"
+        ? "filename = $template"
         : "reference to string\n$$template")
 );
 
 #-------------------------------------------------------------------------------
-# Load database configuration data
+# Open the output file (or STDOUT)
+#-------------------------------------------------------------------------------
+my $outfh;
+if ($outfile) {
+    open( $outfh, ">:encoding(UTF-8)", $outfile )
+        or die "Unable to open $outfile for writing: $!";
+}
+else {
+    open( $outfh, ">&", \*STDOUT )
+        or die "Unable to initialise for writing: $!";
+}
+
+#-------------------------------------------------------------------------------
+# Load database configuration data; allow environment variables
 #-------------------------------------------------------------------------------
 my $conf = Config::General->new(
     -ConfigFile      => $cfgfile,
     -InterPolateVars => 1,
+    -InterPolateEnv  => 1,
     -ExtendedAccess  => 1
 );
 my %config = $conf->getall();
 
+#
+# Set defaults
+#
+$config{database}->{dbtype} //= 'SQLite';
+$config{database}->{host}   //= '127.0.0.1';
+$config{database}->{port}   //= 3306;
+
 #-------------------------------------------------------------------------------
 # Connect to the database
 #-------------------------------------------------------------------------------
-my $dbhost = $config{database}->{host} // '127.0.0.1';
-my $dbport = $config{database}->{port} // 3306;
-my $dbname = $config{database}->{name};
-my $dbuser = $config{database}->{user};
-my $dbpwd  = $config{database}->{password};
+$dbh = db_connect(\%config);
 
-$dbh = DBI->connect( "dbi:mysql:host=$dbhost;port=$dbport;database=$dbname",
-    $dbuser, $dbpwd, { AutoCommit => 1 } )
-    or die $DBI::errstr;
-
-#
-# Enable client-side UTF8
-#
-$dbh->{mysql_enable_utf8} = 1;
-
-#
-# Set up the query
-#
+#-------------------------------------------------------------------------------
+# Set up and perform the query
+#-------------------------------------------------------------------------------
 $sth1 = $dbh->prepare($query) or die $DBI::errstr;
 if ( $dbh->err ) {
     die $dbh->errstr;
@@ -229,11 +262,8 @@ catch ($e) {
     #
     # The 'die' above was triggered. The error is in $_.
     #
-    my $pcount = grep {/\?/} split( '', $query );
-    my $acount = scalar(@dbargs);
-    print STDERR "Failed to execute query.\n";
-    print STDERR "Placeholder/Argument mismatch: $pcount/$acount\n";
-    exit;
+    say STDERR "Failed to execute query.";
+    exit 1;
 }
 
 #
@@ -249,14 +279,16 @@ _debug( $DEBUG >= 3, '$result: ' . Dumper($result) );
 _debug( $DEBUG >= 3, '@names: ' . Dumper(\@names) );
 
 #
-# Set up the template
+# Set up the template object. Look for template files where the script lives
+# and in the current directory.
 #
 my $tt = Template->new(
     {   ABSOLUTE     => 1,
+        RELATIVE     => 1,
         ENCODING     => 'utf8',
-        INCLUDE_PATH => $basedir,
+        INCLUDE_PATH => [ $basedir, getcwd() ],
     }
-);
+) || die Template->error(), "\n";
 
 #
 # Send collected data to the template
@@ -274,31 +306,148 @@ _debug( $DEBUG >= 3, '$vars: ' . Dumper($vars) );
 
 $tt->process( $template, $vars, \$document, { binmode => ':utf8' } )
     || die $tt->error(), "\n";
-print $document;
+print $outfh $document;
+close($outfh);
+_debug( $DEBUG >= 3, '$document: ' . Dumper($document) );
 
 exit;
 
+#===  FUNCTION  ================================================================
+#         NAME: db_connect
+#      PURPOSE: Connects to a database using configuration settings including
+#               the database type
+#   PARAMETERS: $cfg            Config::General object
+#      RETURNS: Database handle
+#  DESCRIPTION: The 'dbtype' field in the configuration file gets a default,
+#               but the 'name' field is mandatory. Depending on the
+#               (lowercase) type name a different form of database connect is
+#               performed after '$dbtype' is set to the format the DBD driver
+#               needs. The database handle is returned (unless there's an
+#               error).
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub db_connect {                                                            #{{{
+    my ($cfg) = @_;
+
+    my ( $dbh, $dbtype, $dbname );
+
+    $dbtype = $cfg->{database}->{dbtype};
+    $dbname = $cfg->{database}->{name};
+    die "Database name is mandatory\n" unless $dbname;
+
+    #
+    # Connect according to the database type
+    #
+    if ($dbtype =~ /sqlite/i) {
+        #
+        # The name for the SQLite driver is 'DBD:SQLite'
+        #
+        $dbtype = 'SQLite';
+        _debug( $DEBUG >= 3, '$dbtype: ' . $dbtype, '$dbname: ' . $dbname );
+
+        $dbh = DBI->connect( "DBI:$dbtype:dbname=$dbname",
+            "", "", { AutoCommit => 1, sqlite_unicode => 1,  } )
+            or die $DBI::errstr;
+    }
+    elsif ($dbtype =~ /mysql/i) {
+        #
+        # The name for the MySQL driver is 'DBD:mysql'
+        #
+        $dbtype = 'mysql';
+
+        my $dbhost = $cfg->{database}->{host};
+        my $dbport = $cfg->{database}->{port};
+        my $dbuser = $cfg->{database}->{user};
+        my $dbpwd  = $cfg->{database}->{password};
+
+        $dbh = DBI->connect( "DBI:$dbtype:host=$dbhost;port=$dbport;database=$dbname",
+            $dbuser, $dbpwd, { AutoCommit => 1 } )
+            or die $DBI::errstr;
+
+        #
+        # Enable client-side UTF8
+        #
+        $dbh->{mysql_enable_utf8} = 1;
+
+        # $dbh->trace('2|SQL');
+    }
+    elsif ($dbtype =~ /pg/i) {
+        #
+        # The name for the PostgreSQL driver is 'DBD:Pg'
+        #
+        $dbtype = 'Pg';
+
+        die "The PostgreSQL database type is not catered for yet.\n";
+    }
+    else {
+        die "Unknown database type: $dbtype\n";
+    }
+
+    return $dbh;
+}                                                                           #}}}
+
+#===  FUNCTION  ================================================================
+#         NAME: strip_sql_comments
+#      PURPOSE: Given a query as a scalar, strips all SQL comments
+#   PARAMETERS: $query          string containing a query
+#      RETURNS: Stripped string
+#  DESCRIPTION: Two types of comments might exist in the query: the C-style
+#               and the SQL style. The string is treated as a single string
+#               even though it's multi-line, and any C-style comments are
+#               removed. Then the string is treated as multi-line and each
+#               line is scanned for SQL comments (which end at the end of the
+#               line), and these are stripped. Blank lines are skipped too to
+#               compress the output a little.
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub strip_sql_comments {                                                    #{{{
+    my ($query) = @_;
+
+    my $result;
+
+    #
+    # Strip C-style comments
+    #
+    $query =~ s/\/\*.*?\*\///sg;
+
+    #
+    # Strip SQL line-oriented comments
+    #
+    foreach my $line (split(/\n/,$query)) {
+        next if $line =~ /^\s*$/;
+        $line =~ s/--.*$//;
+        $result .= "$line\n";
+    }
+
+    return $result;
+}                                                                           #}}}
+
 #===  FUNCTION  ================================================================
 #         NAME: _debug
 #      PURPOSE: Prints debug reports
 #   PARAMETERS: $active         Boolean: 1 for print, 0 for no print
-#               $message        Message to print
+#               @messages       Messages to print
 #      RETURNS: Nothing
-#  DESCRIPTION: Outputs a message if $active is true. It removes any trailing
-#               newline and then adds one in the 'print' to the caller doesn't
-#               have to bother. Prepends the message with 'D> ' to show it's
+#  DESCRIPTION: Outputs messages if $active is true. It removes any trailing
+#               newlines and then adds one to each line so the caller doesn't
+#               have to bother. Prepends 'D> ' to each message to show it's
 #               a debug message.
 #       THROWS: No exceptions
 #     COMMENTS: None
 #     SEE ALSO: N/A
 #===============================================================================
-sub _debug {
-    my ( $active, $message ) = @_;
-
-    chomp($message);
-    print STDERR "D> $message\n" if $active;
-}
+sub _debug {                                                                #{{{
+    my ( $active, @messages ) = @_;
 
+    if ($active) {
+        chomp(@messages);
+        say STDERR "D> ", join( "\nD> ", @messages );
+    }
+}                                                                           #}}}
 
 #===  FUNCTION  ================================================================
 #         NAME: _dbargs
@@ -311,17 +460,17 @@ sub _debug {
 #     COMMENTS: None
 #     SEE ALSO: N/A
 #===============================================================================
-sub _dbargs {
+sub _dbargs {                                                               #{{{
     my ($opts) = @_;
 
     my @args;
 
-    if ( defined( $opts->{dbargs} ) ) {
-        @args = @{ $opts->{dbargs} };
+    if ( defined( $opts->{dbarg} ) ) {
+        @args = @{ $opts->{dbarg} };
     }
 
     return (@args);
-}
+}                                                                           #}}}
 
 #===  FUNCTION  ================================================================
 #         NAME: _define
@@ -336,7 +485,7 @@ sub _dbargs {
 #     COMMENTS: None
 #     SEE ALSO:
 #===============================================================================
-sub _define {
+sub _define {                                                               #{{{
     my ($opts) = @_;
 
     my %defs;
@@ -346,7 +495,7 @@ sub _define {
     }
 
     return (%defs);
-}
+}                                                                           #}}}
 
 #===  FUNCTION  ================================================================
 #         NAME: Options
@@ -358,12 +507,15 @@ sub _define {
 #     COMMENTS: none
 #     SEE ALSO: n/a
 #===============================================================================
-sub Options {
+sub Options {                                                               #{{{
     my ($optref) = @_;
 
     my @options = (
-        "help",    "doc",        "debug=i",   "config=s",
-        "query=s", "template=s", "dbargs=s@", "define=s%",
+        "help",       "documentation|man",
+        "debug=i",    "config=s",
+        "output=s",   "query=s",
+        "template=s", "dbarg=s@",
+        "define=s%",
     );
 
     if ( !GetOptions( $optref, @options ) ) {
@@ -371,7 +523,7 @@ sub Options {
     }
 
     return;
-}
+}                                                                           #}}}
 
 __END__
 
@@ -386,26 +538,14 @@ query2tt2 - A script for formatting a report from database query using a templat
 
 =head1 VERSION
 
-This documentation refers to query2tt2 version 0.0.5
+This documentation refers to query2tt2 version 0.0.9
 
 =head1 USAGE
 
-    query2tt2 [-help] [-doc] [-debug=N] [-config=FILE] [-query=FILE]
-         [-template=FILE] [-dbargs=ARG1 [-dbarg=ARG2] ...]
+    query2tt2 [-help] [-documentation|-man] [-debug=N] [-config=FILE]
+         [-query=FILE] [-template=FILE] [-dbargs=ARG1 [-dbarg=ARG2] ...]
          [define KEY1=VALUE [define key2=VALUE2] ...] [QUERY]
 
-    query2tt2 -help
-
-    query2tt2 -query=tag_query_580-589.sql
-
-    query2tt2 -config=.hpr_livedb.cfg -template=query2tt2_taglist.tpl \
-    'select id,summary,tags from eps where id between 580 AND 589 AND (length(summary) = 0 or length(tags) = 0) ORDER BY id'
-
-    query2tt2 -config=.hpr_livedb.cfg -query=hosts_showcount.sql \
-        -dbargs '2021-01-01' -dbargs '2021-12-31' \
-        -def year=2021 -template=~/HPR/Community_News/hosts_list.tpl
-
-
 =head1 OPTIONS
 
 =over 4
@@ -414,7 +554,7 @@ This documentation refers to query2tt2 version 0.0.5
 
 Prints a brief help message describing the usage of the program, and then exits.
 
-=item B<-doc>
+=item B<-documentation> or B<-man>
 
 Displays the entirety of the documentation (using a pager), and then exits. To
 generate a PDF version use:
@@ -445,11 +585,17 @@ Prints all data structures from options or from the database
 This option allows an alternative configuration file to be used. This file
 defines the location of the database, its port, its name and the username and
 password to be used to access it. This feature was added to allow the script
-to access alternative databases or the live HPR database over an SSH tunnel.
+to access alternative databases.
 
 See the CONFIGURATION AND ENVIRONMENT section below for the file format.
 
-If the option is omitted the default file is used: B<.hpr_db.cfg>
+If the option is omitted the default file is used: B<.hpr_sqlite.cfg>
+
+=item B<-output=FILE>
+
+This option defines an output file to receive the result of the query. If the
+option is omitted the notes are written to STDOUT, allowing them to be
+redirected if required.
 
 =item B<-query=FILE>
 
@@ -474,30 +620,32 @@ The results of the query are fed to the Template Toolkit system for
 reformatting. This option provides the name of the template definition file.
 If this option is omitted then the script uses a very simple internal template
 which is roughly equivalent to the effect in MySQL/MariaDB of ending a query
-with I<\G>.
+with I<\G> or using I<.mode list> with SQLite.
 
 See below in the B<DESCRIPTION> section for the constraints imposed on the
 contents of the template.
 
-Output from the template is written to STDOUT.
+Output from the template is written to STDOUT or to the file designated with
+the B<-out=FILE> option.
 
 =item B<-define KEY1=VALUE1> [ B<-define KEY2=VALUE2> ... B<-define KEYn=VALUEn> ]
 
 The Template Toolkit (TT2) template may receive values from the command line
 using this option. The argument to the B<-define> option is a B<key=value>
 pair. Keys should be unique otherwise they will overwrite one another. They
-should also not be 'names' or 'result' because these keys are used internally
-(for the data from the database). See below for more details. The keys will
-become TT2 variables and the values will be assigned to them.
+should also not be I<'names'> or I<'result'> because these keys are used
+internally (for the data from the database). See below for more details. The
+keys will become TT2 variables and the values will be assigned to them.
 
 =back
 
 =head1 DESCRIPTION
 
 The purpose of the script is to run a query against the HPR database (a local
-copy or the live one on the server over an SSH tunnel). The database choice is
-made via a configuration file. The default file points to the local database,
-but the alternative (discussed later) accesses the live database.
+copy in SQLite or MySQL form or the live one on the server over an SSH
+tunnel). The database choice is made via a configuration file. The default
+file points to the local database, but the alternative (discussed later)
+accesses the live database.
 
 The data returned from the query is then passed through a Template Toolkit
 template so that it can be formatted. There are many ways in which this can be
@@ -566,26 +714,64 @@ An error has occurred while processing the template.
 
 =head1 CONFIGURATION AND ENVIRONMENT
 
-The script obtains the credentials it requires to open the MariaDB database
-from a configuration file. The name of the file it expects is B<.hpr_db.cfg>
-in the directory holding the script. This configuration file can be overridden
+The script obtains the credentials it requires to open the SQLite or MariaDB database
+from a configuration file. No credentials are required for the SQLite format.
+The name of the MySQL database or the SQLite file it expects is specified as
+a simple name or as an absolute file path. This configuration file can be overridden
 using the B<-config=FILE> option as described above.
 
-The configuration file format is as follows:
+The configuration file formats are as follows:
+
+SQLite format:
 
  <database>
-     host = 127.0.0.1
-     port = PORT
-     name = DATABASE
-     user = USERNAME
+     dbtype = SQLite
+     name   = /home/cendjm/HPR/Community_News/hpr.db
+ </database>
+
+MySQL/MariaDB format:
+
+ <database>
+     dbtype   = MySQL
+     host     = 127.0.0.1
+     name     = hpr_hpr
+     user     = hpradmin
      password = PASSWORD
  </database>
 
+=head1 EXAMPLES
+
+ # Request minimal help
+ query2tt2 -help
+
+ # Request full internal documentation
+ query2tt2 -man
+
+ # Run a query from a file and output in the default format
+ query2tt2 -query=tag_query_580-589.sql
+
+ # Run the query on the command line and process the results using
+ # a specific template
+ query2tt2 -config=.hpr_sqlite.cfg -template=query2tt2_taglist.tpl \
+     'select id,summary,tags from eps where id between 580 AND 589
+     AND (length(summary) = 0 or length(tags) = 0) ORDER BY id'
+
+ # Run a query from a file. The query has two placeholders which receive
+ # values from the '-dbarg' options. A template processes the output and
+ # takes a TT2 variable 'year' which is used by the template. Output is in
+ # HTML format and is written to a file.
+ query2tt2 -config=.hpr_sqlite.cfg -query=hosts_showcount.sql \
+     -dbargs '2024-01-01' -dbargs '2024-12-31' \
+     -def year=2024 -template=~/HPR/Community_News/hosts_list.tpl \
+     -out=host_showcount.html
+
+
 =head1 DEPENDENCIES
 
     Config::General
-    DBI
+    Cwd
     Data::Dumper
+    DBI
     File::Slurper
     Getopt::Long
     Hash::Merge
@@ -605,8 +791,8 @@ Dave Morriss (Dave.Morriss@gmail.com)
 
 =head1 LICENCE AND COPYRIGHT
 
-Copyright (c) 2021, 2022, 2024 Dave Morriss (Dave.Morriss@gmail.com). All
-rights reserved.
+Copyright (c) 2021, 2022, 2024, 2025 Dave Morriss (Dave.Morriss@gmail.com).
+All rights reserved.
 
 This module is free software; you can redistribute it and/or
 modify it under the same terms as Perl itself. See perldoc perlartistic.

InternetArchive/future_upload

@@ -1,9 +1,10 @@
 #!/bin/bash -
+# shellcheck disable=SC2317
 #===============================================================================
 #
 #         FILE: future_upload
 #
-#        USAGE: ./future_upload
+#        USAGE: ./future_upload [-h] [-v] [-D] [-d {0|1}] [-F] [-r] [-l cp]
 #
 #  DESCRIPTION: Uploads future HPR shows based on what is in the upload area
 #
@@ -13,9 +14,9 @@
 #        NOTES: Contains methods from 'delete_uploaded' and 'weekly_upload' as
 #               well as 'update_state'
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
-#      VERSION: 0.0.15
+#      VERSION: 0.0.17
 #      CREATED: 2021-01-07 12:11:02
-#     REVISION: 2024-07-29 23:17:45
+#     REVISION: 2025-01-06 17:51:57
 #
 #===============================================================================
 
@@ -26,7 +27,7 @@ SCRIPT=${0##*/}
 
 STDOUT="/dev/fd/2"
 
-VERSION="0.0.15"
+VERSION="0.0.17"
 
 #
 # Load library functions
@@ -36,7 +37,7 @@ LIB="$HOME/bin/function_lib.sh"
 # shellcheck disable=SC1090
 source "$LIB"
 
-# {{{ -- Functions -- check_uploads, _log, _usage
+# {{{ -- Functions -- check_uploads, update_show_state, _log, _usage
 
 #===  FUNCTION  ================================================================
 #         NAME: check_uploads
@@ -51,13 +52,54 @@ check_uploads () {
     #
     # Look for files called hpr1234.flac and so on. Don't bother with the
     # hpr1234_source.flac one. As soon as a file is missing return with false.
+    # 2025-01-01: Dropped 'spx' from the list
     #
-    for suff in flac mp3 ogg opus spx wav; do
+    for suff in flac mp3 ogg opus wav; do
         if [[ ! -e $UPLOADS/$prefix.$suff ]]; then
             return 1
         fi
     done
 
+    #
+    # Transcripts are (currently) in a sub-directory with the same name as the
+    # IA item. We only cater for two types as of 2025.
+    #
+    for suff in txt srt; do
+        if [[ ! -e $UPLOADS/$prefix/$prefix.$suff ]]; then
+            return 1
+        fi
+    done
+
+    return 0
+}
+
+#===  FUNCTION  ================================================================
+#         NAME: update_show_state
+#  DESCRIPTION: Updates the status of a single show in the HPR database.
+#               It is assumed the caller has found the show number in the
+#               'reservations' table with the required status of
+#               'MEDIA_TRANSCODED'. All this function does is to change this
+#               to 'UPLOADED_TO_IA', returning true if successful, otherwise
+#               false.
+#   PARAMETERS: $show           Show number to update
+#      RETURNS: True if the update worked, otherwise false
+#===============================================================================
+update_show_state () {
+    local show=${1:?Usage: update_state show}
+    local BASECOM URL QUERY COMMAND RES
+
+    BASECOM='curl -K ./.hpradmin_curlrc -s'
+    URL="https://hub.hackerpublicradio.org/cms/status.php"
+    QUERY="${BASECOM} ${URL}"
+
+    COMMAND="${QUERY}?ep_num=${show}&status=UPLOADED_TO_IA"
+
+    $COMMAND
+    RES=$?
+    if [[ $RES -ne 0 ]]; then
+        return 1
+    fi
+
     return 0
 }
 
@@ -72,7 +114,7 @@ check_uploads () {
 #   PARAMETERS: 1 - the message to write
 #      RETURNS: Nothing
 #===============================================================================
-# shellcheck disable=SC2317 disable=SC2059
+# shellcheck disable=SC2059
 _log () {
     local msg="$1"
 
@@ -169,7 +211,7 @@ BASECOM='curl -K ./.hpradmin_curlrc -s'
 URL="https://hub.hackerpublicradio.org/cms/status.php"
 # QUERY1="${BASECOM} ${URL}"
 QUERY2="${BASECOM} -o - ${URL}"
-UPSTATE="$BASEDIR/update_state"
+# UPSTATE="$BASEDIR/update_state"
 
 #
 # Fallback URL
@@ -188,10 +230,10 @@ ia=$(command -v ia)
     echo "Needs the 'make_metadata' script"
     exit 1
 }
-[ -e "$UPSTATE" ] || {
-    echo "Needs the 'update_state' script"
-    exit 1
-}
+# [ -e "$UPSTATE" ] || {
+#     echo "Needs the 'update_state' script"
+#     exit 1
+# }
 
 #
 # File of processed shows
@@ -223,6 +265,9 @@ do
 done
 shift $((OPTIND - 1))
 
+#
+# Check and set option variables
+#
 DRYRUN=${DRYRUN:-1}
 if [[ $DRYRUN -ne 0 && $DRYRUN -ne 1 ]]; then
     echo "** Use '-d 0' or '-d 1'"
@@ -261,6 +306,7 @@ fi
 
 #
 # Declarations
+# ------------
 #
 declare -A processed
 declare -A ready
@@ -271,6 +317,7 @@ lastitem=
 
 #
 # Load array of processed shows
+# ---- ----- -- --------- -----
 #
 while read -r item; do
     processed+=([$item]=1)
@@ -278,46 +325,17 @@ done < "$PROCFILE"
 [ "$VERBOSE" -eq 1 ] && echo "Number of shows in cache: ${#processed[@]}"
 
 #
-# TODO: Create the associative array 'ready' containing the numbers of shows
-# ready for upload. This is a way to ensure that we don't try and upload shows
-# in transit to the upload area.
+# Populate the associative array 'ready' with the numbers of shows ready for
+# upload. This is a way to ensure that we don't try and upload shows in
+# transit to the upload area. Only do this if force mode is off.
 #
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# Proposed code. Not sure what the actual URL will be nor what will be
-# returned if nothing is ready for upload yet
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-#
-# json=$(curl http://hackerpublicradio.org/queue.php -s -o -)
-# while read -r showno; do
-#     ready+=([$showno]=1)
-# done < <(echo "${json}" | jq '.READY_FOR_IA_UPLOAD[] | tonumber')
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# Change of plan. Now we have a list of CSV values, so we need to do something
-# like this:
-#
-# reservations=$($BASECOM -o - $URL)
-# while read -r line; do
-#    if [[ $line =~ ^([^,]+),([^,]+),([^,]+),([^,]+),([^,]+),.*$ ]]; then
-#       state="${BASH_REMATCH[5]}"
-#       show="${BASH_REMATCH[2]}"
-#    fi
-#    if [[ $state = 'MEDIA_TRANSCODED' ]]; then
-#       ready+=([$show]=1)
-#    fi
-# done <<< $reservations
-#
-# At the end of this the associative array 'ready' will contain the keys of
-# shows that are ready for upload (presumably) so we can look in this array to
-# double check.
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
 if [[ $FORCE -eq 0 ]]; then
     #
     # Collect the current table of shows requiring work. We expect something like:
     # timestamp_epoc,ep_num,ep_date,key,status,email
     # 1651286617,3617,2022-06-14,fda088e0e3bd5d0353ea6b7569e93b87626ca25976a0a,UPLOADED_TO_IA,lurkingprion@gmail.com
     # 1651648589,3619,2022-06-16,e7d3810afa098863d81663418d8640276272284de68f1,UPLOADED_TO_IA,monochromec@gmail.com
-    # TODO: Check for a failure in the query?A
+    # TODO: Reinstate the check for a failure in the query? Se update_state
     # NOTE: Problem encountered 2022-09-23 because the SSL certificate has expired
     #
     reservations=$($QUERY2) || {
@@ -342,8 +360,8 @@ if [[ $FORCE -eq 0 ]]; then
     fi
 
     #
-    # The query returns the bare number, but we're using 'hprxxxx' as the key in
-    # the 'ready' array.
+    # The query returns the bare show number, but we're using 'hprxxxx' as the
+    # key in the 'ready' array.
     #
     while read -r line; do
         if [[ $line =~ ^([^,]+),([^,]+),([^,]+),([^,]+),([^,]+),.*$ ]]; then
@@ -363,7 +381,10 @@ fi
 
 #
 # Process files. There will be several with the same prefix so look for
-# a change of prefix
+# a change of prefix.
+#
+# The loop is reading from the following pipeline:
+# find "$UPLOADS" -regextype posix-extended -regex '.*hpr[0-9]{4}.*' | sort
 #
 while read -r path; do
     #
@@ -379,8 +400,8 @@ while read -r path; do
     _DEBUG "Item $item"
 
     #
-    # Detect that the item prefix has changed. If it has we're processing
-    # a new IA identifier, so work on this one
+    # Detect that the item prefix has changed. If it has we've found a new IA
+    # identifier, so work on the previous one
     #
     if [[ $item != "$lastitem" ]]; then
         lastitem=$item
@@ -414,7 +435,8 @@ while read -r path; do
                 processed+=([$lastitem]=1)
             else
                 #
-                # Is the show ready for upload?
+                # Is the show ready for upload? We don't check if force mode
+                # is on. If not ready we skip this show.
                 #
                 if [[ $FORCE -eq 0 ]]; then
                     if [[ ! -v "ready[$lastitem]" ]]; then
@@ -461,10 +483,9 @@ while read -r path; do
 
 done < <(find "$UPLOADS" -regextype posix-extended -regex '.*hpr[0-9]{4}.*' | sort)
 
-#
+#-------------------------------------------------------------------------------
 # Write the processed array to the cache file unless in dry-run mode
-#
-# [ $DEBUG -eq 1 ] && { echo -n 'D> '; declare -p processed; }
+#-------------------------------------------------------------------------------
 _DEBUG "processed = ${!processed[*]}"
 [ "$VERBOSE" -eq 1 ] && echo "Number of shows in cache: ${#processed[@]}"
 if [[ $DRYRUN -ne 1 ]]; then
@@ -473,24 +494,26 @@ if [[ $DRYRUN -ne 1 ]]; then
     done < <(printf '%s\n' "${!processed[@]}" | sort -u ) > "$PROCFILE"
 fi
 
-#
+#-------------------------------------------------------------------------------
 # Generate the list of uploads for the 'make_metadata' option '-list=1,2,3'.
+# The show numbers are keys in the associative array 'uploads'. The
+# end-product is a comma-separated list of the keys in the variable '$list'.
 # Order is unimportant because make_metadata sorts internally.
-#
+#-------------------------------------------------------------------------------
 _DEBUG "uploads = ${!uploads[*]}"
 [ "$VERBOSE" -eq 1 ] && echo "Number of shows for upload: ${#uploads[@]}"
 printf -v list '%s,' "${!uploads[@]}"
 list="${list:0:-1}"
 
-#
+#-------------------------------------------------------------------------------
 # If there are no uploads to do we can stop
-#
+#-------------------------------------------------------------------------------
 [[ ! -v uploads[@] ]] && { echo "Nothing to do!"; exit; }
 
-#
+#-------------------------------------------------------------------------------
 # Check that the shows being uploaded have all their files and log what is
 # happening.
-#
+#-------------------------------------------------------------------------------
 while read -r show; do
     echo "$(date +%Y%m%d%H%M%S) preparing to upload hpr$show" >> "$LOGFILE"
 
@@ -501,10 +524,10 @@ while read -r show; do
     fi
 done < <(printf '%s\n' "${!uploads[@]}" | sort)
 
-#
+#-------------------------------------------------------------------------------
 # Define output files. If the list contains one element then it's a different
 # name from the multi-element case (make_metadata does this too).
-#
+#-------------------------------------------------------------------------------
 if [[ ${#uploads[@]} -eq 1 ]]; then
     metadata="metadata_${minshow}.csv"
     script="script_${minshow}.sh"
@@ -513,9 +536,9 @@ else
     script="script_${minshow}-${maxshow}.sh"
 fi
 
-#
+#-------------------------------------------------------------------------------
 # Perform the uploads or report what would be done
-#
+#-------------------------------------------------------------------------------
 if [[ $DRYRUN -eq 1 ]]; then
     echo "Dry run: Would have uploaded list '$list'"
     echo "Dry run: Would have created $metadata and $script"
@@ -562,17 +585,17 @@ else
                 echo "$(date +%Y%m%d%H%M%S) ${#uploads[@]} uploads completed" >> "$LOGFILE"
 
                 #
-                # Update the state in the HPR database, unless we're using
-                # FORCE. Pass the limit used here to this script so it can
-                # stop looking for work unnecessarily
+                # Update the state  of all the shows being processed in the
+                # HPR database, unless we're using FORCE.
                 #
                 if [[ $FORCE -eq 0 ]]; then
-                    $UPSTATE -l$LIMIT
-                    RES=$?
-                    if [[ $RES -ne 0 ]]; then
-                        echo "Problem updating database state"
-                        exit 1
-                    fi
+                    while read -r show; do
+                        if update_show_state $show; then
+                            echo "Updated state for show $show"
+                        else
+                            echo "Failed to update state for show $show"
+                        fi
+                    done < <(printf '%s\n' "${!uploads[@]}" | sort)
                 else
                     echo "Not updating the database, FORCE mode is on"
                 fi

InternetArchive/recover_transcripts

@@ -307,7 +307,7 @@ VERBOSE=${VERBOSE:-0}
 #
 # Should have one argument
 #
-if [[ $# != 1 ]]; then
+if [[ $# -ne 1 ]]; then
     coloured 'red' "Missing argument"
     _usage 1
 fi

InternetArchive/reformat_html

@@ -0,0 +1,245 @@
+#!/usr/bin/env perl
+#===============================================================================
+#
+#         FILE: reformat_html
+#
+#        USAGE: ./reformat_html < input.html > output.html
+#
+#  DESCRIPTION: Reformats the HTML found in the HPR database in the 'notes'
+#               field to the format required in the 'description' field of an
+#               item on the IA. It reads from STDIN and writes to STDOUT.
+#
+#      OPTIONS: ---
+# REQUIREMENTS: ---
+#         BUGS: ---
+#        NOTES: ---
+#       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
+#      VERSION: 0.0.1
+#      CREATED: 2025-02-09 22:56:30
+#     REVISION: 2025-02-13 11:13:37
+#
+#===============================================================================
+
+use v5.36;
+use strict;
+use warnings;
+use feature qw{ say try };
+no warnings qw{ experimental::try };
+
+use open ':std', ':encoding(UTF-8)'; # Make all IO UTF-8
+
+use HTML::TreeBuilder 5 -weak;
+use HTML::Entities;
+
+#
+# Version number (Incremented by Vim)
+#
+our $VERSION = '0.0.1';
+
+#
+# Declarations
+#
+my ($verbose, @notes, $notes, $tree);
+
+#
+# Read the input data into an array
+#
+try {
+    @notes = <STDIN>;
+}
+catch ($e) {
+    warn "Problem reading input HTML; $e";
+    exit 1;
+}
+
+die "No input HTML detected\n" unless @notes;
+
+#
+# Turn the array into a scalar
+#
+$notes = join( '', @notes );
+
+#
+# Get ready to parse the array
+#
+$tree = HTML::TreeBuilder->new;
+$tree->ignore_unknown(0);
+$tree->no_expand_entities(1);
+$tree->p_strict(1);
+$tree->store_comments(1);               # Necessary?
+$tree->warn(1);
+
+#
+# Parse HTML to the tree structure
+#
+$tree->parse_content($notes)
+    or die "HTML::TreeBuilder failed to parse input HTML: $!\n";
+
+#
+# Flatten all <pre> tags and add <br/> tags
+#
+$notes = flatten_pre($tree);
+
+#
+# Deal with non-ASCII
+#
+$notes = encode_entities( $notes, '^\n&\x20-\x25\x27-\x7e' );
+
+#
+# Remove all newlines
+#
+$notes =~ s/\n//g;
+
+#
+# Write the end result to the STDOUT
+#
+say $notes;
+
+exit;
+
+#===  FUNCTION  ================================================================
+#         NAME: flatten_pre
+#      PURPOSE: Process notes "flattening" <pre> contents
+#   PARAMETERS: $tree   HTML::TreeBuilder object containing parsed and
+#                       partially processed notes
+#      RETURNS: Processed notes
+#  DESCRIPTION: The HTML "<pre>" tag encloses preformatted text. It can also
+#               contain some formatting tags like <em> and <code>, but spaces
+#               and newlines are significant. The Internet Archive upload API
+#               uses HTTP headers which are text strings without newlines, so
+#               when these tags are uploaded through this route some
+#               formatting is lost. What this routine does is parse the
+#               contents of all <pre> sections in $notes, adding <br/> tags
+#               to replace newlines. It has to perform a full parse
+#               since the contents may include HTML tags and these need to be
+#               passed through intact. It calls the subroutine 'flatten_item' to
+#               deal with the recursive nature of HTML tags.
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub flatten_pre {
+    my ($tree) = @_;
+
+    #
+    # Find all the <pre> tags
+    #
+    my @pre_tags = $tree->look_down( _tag => 'pre', );
+
+    #
+    # Walk the various <pre> elements in the document
+    #
+    foreach my $tag (@pre_tags) {
+        #
+        # Save the tag and empty the original
+        #
+        my $saved = $tag->clone();
+        $tag->delete_content();
+
+        #
+        # Walk the saved content and rebuild the tag into $atag using the
+        # nested arrayref structure permitted by HTML::Element for
+        # convenience (the alternative is a little nasty). See the
+        # documentation for 'new_from_lol' in HTML::Element.
+        #
+        my $atag;
+        foreach my $item ( @{ $saved->content_array_ref } ) {
+            push( @$atag, flatten_item($item) );
+        }
+
+        #
+        # Rebuild the tag from the arrayref we built. We treat the arrayref
+        # structure we just built as an array because otherwise the top level
+        # is interpreted as a spurious <null> tag.
+        #
+        $tag->push_content(@$atag);
+    }
+
+    #
+    # Trim out the original notes from the enclosing tags we added earlier
+    #
+    my $body = $tree->look_down( _tag => 'body' );
+    ( my $result = $body->as_HTML( undef, ' ', {} ) )
+        =~ s{(^<body[^>]*>|</body>$)}{}gi;
+
+    return $result;
+
+}
+
+#===  FUNCTION  ================================================================
+#         NAME: flatten_item
+#      PURPOSE: Recursively "flatten" items within the enclosing <pre>
+#   PARAMETERS: $item   an HTML::Element item parsed from the original
+#                       <pre> section
+#      RETURNS: An arrayref if the last seen item was a tag, otherwise a list
+#  DESCRIPTION: Since <pre> sections can contain inline elements which change
+#               the rendering of the text we need to parse these as we add
+#               <br/> tags. This routine does this by recursively descending
+#               through the contents. A common tag sequence is <pre><code> for
+#               scripts and the like. This routine deals with such sequences.
+#               It expects to receive the contents in sequence and builds the
+#               result as a nested arrayref structure.
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub flatten_item {
+    my ($item) = @_;
+
+    return unless defined($item);
+
+    my ( @result, %attr );
+
+    #
+    # Is it a sub-tag or non-tag content?
+    #
+    if ( ref($item) ) {
+        #
+        # It's a tag. Save the tag name and any attributes and recurse into
+        # it. Return an arrayref
+        #
+        push( @result, $item->tag() );
+        %attr = $item->all_external_attr();
+        push( @result, \%attr ) if %attr;
+        for my $child ( $item->content_list() ) {
+            push( @result, flatten_item($child) );
+        }
+        return \@result;
+    }
+    else {
+        #
+        # It's non-tag content. Join the lines with <br/> tags.  Return an
+        # array (since this is a simple list).
+        #
+        # Note that we split with a LIMIT of -1 which causes any trailing list
+        # items to be returned; default behaviour is to drop them.
+        #
+        $item =~ s/\r//g;
+        my @content = split( /\n/, $item, -1 );
+        if (@content) {
+            #
+            # Remove a leading blank line - usually the result of
+            # a "<pre>'NL'text" sequence
+            #
+            shift(@content) if ( $content[0] =~ /^\s*$/ );
+
+            #
+            # Join back the lines with <br/> tags between them.
+            #
+            foreach my $txt (@content) {
+                push( @result, $txt, ['br'] );
+            }
+
+            #
+            # Remove the <br/> at the end, it's spurious
+            #
+            pop(@result);
+        }
+
+        return (@result);
+    }
+
+}
+
+# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker
+

InternetArchive/repair_assets

@@ -6,7 +6,7 @@
 #        USAGE: ./repair_assets showid
 #
 #  DESCRIPTION: Given a show where there was a directory of asset files on the
-#               old HPR server whichj got lost in the migration, rebuild it
+#               old HPR server which got lost in the migration, rebuild it
 #               and fill it with assets from the IA. Modify the show notes to
 #               point to these recovered assets.
 #
@@ -15,15 +15,15 @@
 #         BUGS: ---
 #        NOTES: ---
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
-#      VERSION: 0.0.8
+#      VERSION: 0.0.10
 #      CREATED: 2024-05-10 21:26:31
-#     REVISION: 2024-08-23 11:55:25
+#     REVISION: 2024-10-02 17:34:47
 #
 #===============================================================================
 
 # set -o nounset                              # Treat unset variables as an error
 
-VERSION="0.0.8"
+VERSION="0.0.10"
 
 SCRIPT=${0##*/}
 # DIR=${0%/*}
@@ -96,6 +96,38 @@ trap 'cleanup_temp $TMP1 $TMP2' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
 #               $3      Name of array to receive list of missing assets
 #      RETURNS: Nothing
 #===============================================================================
+# find_missing () {
+#     local -n IA="${1}"
+#     local -n HPR="${2}"
+#     local output="${3}"
+#
+#     local -A hIA hHPR
+#     local i key
+#
+#     #
+#     # Make a hash keyed by the IA file base names from an indexed array
+#     #
+#     for (( i=0; i<${#IA[@]}; i++ )); do
+#         hIA+=([${IA[$i]##*/}]=${IA[$i]})
+#     done
+#
+#     #
+#     # Make a hash keyed by the HPR file base names from an indexed array
+#     #
+#     for (( i=0; i<${#HPR[@]}; i++ )); do
+#         hHPR+=([${HPR[$i]##*/}]=${HPR[$i]})
+#     done
+#
+#     #
+#     # Use the basename keys to check what's missing, but return the full path
+#     # names.
+#     #
+#     for key in "${!hIA[@]}"; do
+#         if ! exists_in hHPR "$key"; then
+#             eval "$output+=('${hIA[$key]}')"
+#         fi
+#     done
+# }
 find_missing () {
     local -n IA="${1}"
     local -n HPR="${2}"
@@ -105,26 +137,29 @@ find_missing () {
     local i key
 
     #
-    # Make a hash keyed by the IA file base names from an indexed array
+    # Make a hash keyed by the full IA paths from an indexed array
     #
     for (( i=0; i<${#IA[@]}; i++ )); do
-        hIA+=([${IA[$i]##*/}]=${IA[$i]})
+        hIA+=([${IA[$i]}]=$i)
     done
 
     #
-    # Make a hash keyed by the HPR file base names from an indexed array
+    # Make a hash keyed by the HPR file paths from an indexed array, but
+    # remove the first element for parity with the IA paths. We are going to
+    # copy the IA paths, not these, so we never need the full paths again
+    # here.
     #
     for (( i=0; i<${#HPR[@]}; i++ )); do
-        hHPR+=([${HPR[$i]##*/}]=${HPR[$i]})
+        hHPR+=([${HPR[$i]#*/}]=$i)
     done
 
     #
-    # Use the basename keys to check what's missing, but return the full path
-    # names.
+    # Use the full path keys to check what's missing, and return the IA full
+    # path names.
     #
     for key in "${!hIA[@]}"; do
         if ! exists_in hHPR "$key"; then
-            eval "$output+=('${hIA[$key]}')"
+            eval "$output+=('$key')"
         fi
     done
 }
@@ -267,10 +302,13 @@ fi
 show="${1,,}"
 
 #
-# Ensure show id is correctly formatted. We want it to be 'hpr1234'
+# Ensure show id is correctly formatted. We want it to be 'hpr1234' but we
+# allow the 'hpr' bit to be omitted, as well as any leading zeroes. We need to
+# handle the weirdness of "leading zero means octal" though, but we always
+# store it as 'hpr1234' once processed.
 #
 if [[ $show =~ (hpr)?([0-9]+) ]]; then
-    printf -v show 'hpr%04d' "${BASH_REMATCH[2]}"
+    printf -v show 'hpr%04d' "$((10#${BASH_REMATCH[2]}))"
 else
     coloured 'red' "Incorrect show specification: $show"
     coloured 'yellow' "Use 'hpr9999' or '9999' format"
@@ -443,37 +481,45 @@ ignore_re="index.html$"
 # Run the command and save the output. Save the asset names returned in an
 # array. TODO: Handle errors from the command
 #
-if [[ $DRYRUN -eq 0 ]]; then
-    eval "$command" > "$TMP2"
-    RES=$?
-    if [[ $RES -eq 0 ]]; then
-        _verbose "$(coloured 'green' "Remote command successful")"
-        while read -r hprfile; do
-            if [[ ! $hprfile =~ $ignore_re ]]; then
-                hpr_asset+=("${hprfile}")
-            fi
-        done < "$TMP2"
-        _verbose "$(coloured 'green' "Assets found on HPR server = ${#hpr_asset[@]}")"
-        _verbose "$(printf '%s\n' "${hpr_asset[@]}")"
-        _log "Assets found on HPR server = ${#hpr_asset[@]}"
-    else
-        coloured 'red' "Remote command failed"
-        _log "Failed while searching for HPR assets"
-        exit 1
-    fi
+#
+# NOTE: We also want to interrogate the HPR state in dry-run mode
+#
+# if [[ $DRYRUN -eq 0 ]]; then
+# else
+#     coloured 'yellow' "Would have searched for assets on the HPR server"
+# fi
+
+eval "$command" > "$TMP2"
+RES=$?
+if [[ $RES -eq 0 ]]; then
+    _verbose "$(coloured 'green' "Remote command successful")"
+    while read -r hprfile; do
+        if [[ ! $hprfile =~ $ignore_re ]]; then
+            hpr_asset+=("${hprfile}")
+        fi
+    done < "$TMP2"
+    _verbose "$(coloured 'green' "Assets found on HPR server = ${#hpr_asset[@]}")"
+    _verbose "$(printf '%s\n' "${hpr_asset[@]}")"
+    _log "Assets found on HPR server = ${#hpr_asset[@]}"
 else
-    coloured 'yellow' "Would have searched for assets on the HPR server"
+    coloured 'red' "Remote command failed"
+    _log "Failed while searching for HPR assets"
+    exit 1
 fi
 
 #-------------------------------------------------------------------------------
 # Compare the two asset lists and return what's missing on the HPR server
 #-------------------------------------------------------------------------------
-# TODO: This algorithm does not handle the instance where there are pictures
-# in one directory and a lower directory containing thumbnails, AND THE FILE
-# NAMES ARE THE SAME!
+# TODO: The algorithm in find_missing does not handle the instance where there
+# are pictures in one directory and a lower directory containing thumbnails,
+# AND THE FILE NAMES ARE THE SAME!
 #
 declare -a missing
-find_missing ia_asset hpr_asset missing
+if [[ ${#hpr_asset[@]} -eq 0 ]]; then
+    missing=( "${ia_asset[@]}" )
+else
+    find_missing ia_asset hpr_asset missing
+fi
 _verbose "$(coloured 'cyan' "** missing (${#missing[@]}):")"
 _verbose "$(printf '%s\n' "${missing[@]}")"
 

InternetArchive/repair_item

@@ -23,7 +23,7 @@
 #               temporarily on 'borg') and determines which have not been
 #               uploaded, then takes steps to perform the uploads.
 #
-#               Version 0.0.11 onwards has the capability to repair an IA item
+#               Version 0.0.12 onwards has the capability to repair an IA item
 #               from the HPR backup disk. This seems to be necessary because
 #               the transcripts were not carried over (although we are
 #               adding them to the IA for new shows now, older ones were never
@@ -44,15 +44,15 @@
 #         BUGS: ---
 #        NOTES: ---
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
-#      VERSION: 0.0.11
+#      VERSION: 0.0.12
 #      CREATED: 2020-01-05 22:42:46
-#     REVISION: 2024-07-20 17:06:10
+#     REVISION: 2024-09-13 18:19:59
 #
 #===============================================================================
 
 #set -o nounset                              # Treat unset variables as an error
 
-VERSION="0.0.11"
+VERSION="0.0.12"
 
 SCRIPT=${0##*/}
 # DIR=${0%/*}
@@ -492,9 +492,9 @@ else
 
         #
         # Stop the missed file loop if we have reached the limiting number, in
-        # dry-run and live mode
+        # dry-run and live mode, but not extended mode
         #
-        [[ $upload_count -eq $LIMIT ]] && {
+        [[ $EXTENDED -eq 0 && $upload_count -eq $LIMIT ]] && {
             coloured 'blue' "Upload limit ($LIMIT) reached"
             break
         }

InternetArchive/snapshot_metadata

@@ -5,23 +5,28 @@
 #
 #        USAGE: ./snapshot_metadata episode_number
 #
-#  DESCRIPTION: Collects metadata from the IA for a given show and stores it
-#               in the cache.
+#  DESCRIPTION: Collects JSON metadata from the IA for a given show and stores
+#               it in the cache. Runs 'view_derivatives' on the JSON to
+#               display the derivatives if any, and to save their names if
+#               found, for deletion.
+#               Deletion is performed thus (external to this script):
+#
+#               cat assets/hpr$(./next_repair)/derived.lis | xargs ia delete hpr$(./next_repair) --no-backup
 #
 #      OPTIONS: ---
 # REQUIREMENTS: ---
 #         BUGS: ---
 #        NOTES: ---
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
-#      VERSION: 0.0.2
+#      VERSION: 0.0.3
 #      CREATED: 2024-08-16 20:36:51
-#     REVISION: 2024-08-17 10:31:15
+#     REVISION: 2024-10-02 17:40:13
 #
 #===============================================================================
 
 set -o nounset                              # Treat unset variables as an error
 
-VERSION="0.0.2"
+VERSION="0.0.3"
 
 SCRIPT=${0##*/}
 # DIR=${0%/*}
@@ -126,17 +131,19 @@ fi
 show="${1,,}"
 
 #
-# Ensure show id is correctly formatted. We want it to be 'hpr1234'
+# Ensure show id is correctly formatted. We want it to be 'hpr1234' but we
+# allow the 'hpr' bit to be omitted, as well as any leading zeroes. We need to
+# handle the weirdness of "leading zero means octal" though, but we always
+# store it as 'hpr1234' once processed.
 #
 if [[ $show =~ (hpr)?([0-9]+) ]]; then
-    printf -v show 'hpr%04d' "${BASH_REMATCH[2]}"
+    printf -v show 'hpr%04d' "$((10#${BASH_REMATCH[2]}))"
 else
     coloured 'red' "Incorrect show specification: $show"
     coloured 'yellow' "Use 'hpr9999' or '9999' format"
     exit 1
 fi
 
-
 #-------------------------------------------------------------------------------
 # Setting up paths
 #-------------------------------------------------------------------------------

InternetArchive/tidy_uploaded

@@ -6,7 +6,7 @@
 #        USAGE: ./tidy_uploaded [-h] [-v] [-d {0|1}] [-c COUNT]
 #
 #  DESCRIPTION: Relocates HPR audio and other show-related files on 'borg'
-#               after their shows have been uploaded to the Internet Archive
+#               after their shows have been uploaded to the Internet Archive.
 #
 #      OPTIONS: ---
 # REQUIREMENTS: ---
@@ -43,7 +43,7 @@ TMP1=$(mktemp) || { echo "$SCRIPT: creation of temporary file failed!"; exit 1;
 trap 'cleanup_temp $TMP1' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
 
 #
-# Configure depending whether local or on the VPS
+# Configure depending whether local or on 'borg'
 #
 case $HOSTNAME in
     borg)       BASEDIR="$HOME/InternetArchive"
@@ -95,7 +95,7 @@ queued_tasks () {
 #         NAME: movefile
 #  DESCRIPTION: Moves a file to a new place, catering for any directories in
 #               the path
-#   PARAMETERS: $1      directory to move form
+#   PARAMETERS: $1      directory to move from
 #               $2      directory to move to
 #               $3      file (or sub-path to move)
 #      RETURNS: True if a move was done, otherwise False
@@ -356,7 +356,7 @@ while read -r path; do
         #
         tasks=$(queued_tasks "$item")
         if [[ $tasks -gt 0 ]]; then
-            echo "** Item $item still has $tasks unfinished " \
+            echo "** Item $item still has $tasks unfinished" \
                 "$(ngettext task tasks "$tasks")"
             echo "** Skipping to the next item"
             continue
@@ -434,9 +434,6 @@ while read -r path; do
 
 done < <(find "$UPLOADS" -regextype posix-extended -regex '.*hpr[0-9]{4}.*' -printf "%CY%Cm%Cd%CH%CM%CS %p\n" | sort  | cut -f2 -d' ')
 
-# Old 'find' used:
-# done < <(find "$UPLOADS" -regextype posix-extended -regex '.*hpr[0-9]{4}.*' | sort)
-
 #
 # No shows processed? There was nothing to do
 #

InternetArchive/update_state

@@ -3,7 +3,7 @@
 #
 #         FILE: update_state
 #
-#        USAGE: ./update_state
+#        USAGE: ./update_state [-h] [-D] [-d] [-F] [-l N] [-m]
 #
 #  DESCRIPTION: A script to update the state of shows which have been sent to
 #               the IA. It looks at the current state of the 'reservations'
@@ -136,7 +136,6 @@ esac
 
 cd "$BASEDIR" || { echo "Can't cd to $BASEDIR"; exit 1; }
 
-
 #
 # Tools
 #

InternetArchive/view_derivatives

@@ -44,9 +44,9 @@
 #         BUGS: ---
 #        NOTES: ---
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
-#      VERSION: 0.0.2
+#      VERSION: 0.0.4
 #      CREATED: 2024-08-12 16:26:29
-#     REVISION: 2024-08-17 13:44:44
+#     REVISION: 2024-09-17 17:03:27
 #
 #===============================================================================
 
@@ -71,7 +71,7 @@ use Data::Dumper;
 #
 # Version number (Incremented by Vim)
 #
-our $VERSION = '0.0.2';
+our $VERSION = '0.0.4';
 
 #
 # Script and directory names
@@ -170,11 +170,12 @@ die "Empty JSON?\n" unless (@jsonbuffer);
 my $md = $jsonbuffer[0];
 
 #
-# Collect the identifier from the parsed JSON and define the one derived file
-# we don't want to delete.
+# Collect the identifier from the parsed JSON and define the derived files we
+# don't want to delete. (Found cases of audio files being "derived" in 1672
+# and 1664)
 #
 my $identifier = $md->{metadata}->{identifier};
-my $item_png = "${identifier}.png";
+my $skip_re = qr{^${identifier}\.(flac|mp3|ogg|opus|png|spx|wav)$};
 
 #
 # Build a hash from the original and derived files referenced in the metadata.
@@ -257,9 +258,13 @@ if ($verbose > 0) {
     say '-' x 10;
 }
 
+#
+# List derived files that can be deleted, being careful not to delete the
+# audio or the PNG image created by IA code.
+#
 if ($list_derived) {
     foreach my $file ( sort(@derived) ) {
-        say "$file" unless ($file eq $item_png);
+        say "$file" unless ($file =~ $skip_re);
     }
 }
 
@@ -377,49 +382,157 @@ __END__
 
 =head1 NAME
 
-view_derivatives - <One line description of application's purpose>
+view_derivatives - a tool to analyse IA metadata
 
 =head1 VERSION
 
-The initial template usually just has:
-
-This documentation refers to view_derivatives version 0.0.2
-
+This documentation refers to view_derivatives version 0.0.4
 
 =head1 USAGE
 
-    # Brief working invocation example(s) here showing the most common usage(s)
+    view_derivatives [-help] [-documentation|-man] [-debug=N] [-[no]dry-run]
+        [-verbose [-verbose] ...] [-[no]list_derived] metadata_file
 
-    # This section will be as far as many users ever read
-    # so make it as educational and exemplary as possible.
+    # Parse the metadata and report the relationships beteen files
+    view_derivatives -verb METADATA
 
+    # Parse the metadata and write out a list of derived files for potential
+    # deletion.
+    view_derivatives -list_derived METADATA > FILE
 
 =head1 REQUIRED ARGUMENTS
 
-A complete list of every argument that must appear on the command line.
-when the application  is invoked, explaining what each of them does, any
-restrictions on where each one may appear (i.e. flags that must appear
-before or after filenames), and how the various arguments and options
-may interact (e.g. mutual exclusions, required combinations, etc.)
+The name of a file created by the following command:
 
-If all of the application's arguments are optional this section
-may be omitted entirely.
+    ia metadata "show" > metadata_file
 
+The file is expected to contain one JSON object (in a one-element array). If
+it contains more objects only the first will be processed.
 
 =head1 OPTIONS
 
-A complete list of every available option with which the application
-can be invoked, explaining what each does, and listing any restrictions,
-or interactions.
+=over 4
 
-If the application has no options this section may be omitted entirely.
+=item B<-help>
 
+Prints a brief help message describing the usage of the program, and then exits.
+
+=item B<-documentation> B<-man>
+
+Displays the entirety of the documentation (using a pager), and then exits. To
+generate a PDF version use the I<pod2pdf> tool from
+I<http://search.cpan.org/~jonallen/pod2pdf-0.42/bin/pod2pdf>. This can be
+installed with the cpan tool as App::pod2pdf. Use the command:
+
+    pod2pdf view_derivatives --out=view_derivatives.pdf
+
+=item B<-debug=N>
+
+Selects a level of debugging. Debug information consists of a line or series
+of lines prefixed with the characters 'D>':
+
+=over 4
+
+=item B<0>
+
+No debug output is generated: this is the default
+
+=item B<3>
+
+Prints all data structures from options
+
+=back
+
+(The debug levels need work!)
+
+=item B<-[no]dry-run>
+
+Enable/disable dry run mode (default off)
+
+=item B<-verbose>
+
+Sets the verbosity level. If the option is omitted then the level is zero (no
+verbose output). Thereafter, for each occurrence of the option the verbosity
+level is incremented. Only levels 1 and 2 are currently catered for. Any
+levels above 2 produce the same result as level 2.
+
+=item B<-[no]list_derived>
+
+This option is off by default. Turning it on causes the script to write all
+derived files to standard output. If the verbosity level is zero this is the
+only output from the script.
+
+The idea is that at verbosity level 1 or 2 information is displayed about the
+relationship of files in the metadata, for human consumption. If
+B<-nolist_derived> is the setting (or default) then this is all that is shown.
+
+If the verbosity level is zero and B<-list_derived> is on then only the list
+of derived files will be generated, and this can be used to delete the files
+from the IA.
 
 =head1 DESCRIPTION
 
-A full description of the application and its features.
-May include numerous subsections (i.e. =head2, =head3, etc.)
+=head2 OVERVIEW
 
+Items on the IA (Internet Archive, or I<archive.org>) consist of metadata and
+files. Each item generated for HPR is a show or episode. Most files comprising
+the episode on the IA are those which are part of the episode on the HPR
+server. A few extra files are created by the IA software, but these are part
+of the metadata (HTML details, upload date, etc.)
+
+By default the IA software will create additional files which are derived from
+the original files. Typical examples are other audio formats, such as Ogg or
+Mp3. We have been disabling this derivation process for several years for
+various reasons, preferring to generate our own derivatives. IA-generated
+audio derivatives do not have ID3 and similar tags, whereas HPR-generated
+audio formats do.
+
+Historically it was difficult to disable the derivation process. Even though
+there were settings to do this they apparently didn't work on all of the
+servers making up the IA, and so older items may have many derived files.
+
+This script assists with identifying unwanted derivatives and with their
+deletion.
+
+=head2 METADATA
+
+The metadata for an item can be obtained (by a registered user) from the IA
+using the B<ia> tool. Its format is JSON, and this script uses a JSON module
+to parse it.
+
+=head2 FILE RELATIONSHIPS
+
+The JSON metadata contains details of all files comprising the IA item.
+It contains details such as the name, size, and type of each file. It also
+categorises files into groups such as I<original> and I<derived>. Files which
+are derived have parents. The script uses this to build tree-like data
+structures of derived files based on the original files. All children of an
+original file will be derived, but some derived files may also have children.
+
+The derivatives can be classified simply as children of original files or of
+derived files. These are what are listed if required and what are used in the
+deletion process.
+
+=head2 DELETING UNWANTED DERIVATIVES
+
+The simplest method is to pipe the output from the script with verbose level
+zero and with B<-list_derived> enabled into B<xargs> in order to run a command
+which will delete the unwanted derivatives.
+
+One usage is:
+
+    ./view_derivatives -list_derived metadata.json |\
+        xargs ia delete hpr1234 --no-backup
+
+This will generate a list of files to be deleted, then pipe them to B<xargs>
+which will construct a command by appending the names to the command template.
+
+This approach is not ideal since it does not handle the case where there is
+nothing to delete. The script B<snapshot_metadata> manages this situation by
+generating the metadata and saving it in a file, then it runs
+B<view_derivatives> on this file and generates a file of derivatives. If this
+file is not empty it can be used to perform the deletions, and otherwise no
+attempt will be made.
 
 =head1 DIAGNOSTICS
 
@@ -468,23 +581,18 @@ special cases that are not (yet) handled, etc.
 The initial template usually just has:
 
 There are no known bugs in this module.
-Please report problems to <Maintainer name(s)>  (<contact address>)
+Please report problems to Dave Morriss (dave.morriss@gmail.com)
 Patches are welcome.
 
+
 =head1 AUTHOR
 
-<Author name(s)>  (<contact address>)
+Dave Morriss (dave.morriss@gmail.com)
 
 
 =head1 LICENCE AND COPYRIGHT
 
-Copyright (c) <year> <copyright holder> (<contact address>). All rights reserved.
-
-Followed by whatever licence you wish to release it under.
-For Perl code that is often just:
-
-This module is free software; you can redistribute it and/or
-modify it under the same terms as Perl itself. See perldoc perlartistic.
+Copyright (c) 2024 Dave Morriss (dave.morriss@gmail.com). All rights reserved.
 
 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of

Processing-Show-Notes.md

@@ -0,0 +1,13 @@
+- A `cron` job runs periodically to flag whether there are any notes requiring work (`cronjob_scrape`). The HPR website is scraped with a Perl script to determine this (`scrape_HPR`) by looking for entries in the 'processing' state on the calendar page.
+- If there is work to do the partial copy of the `upload` directory on the HPR server kept locally is synchronised with `rsync` over SSH (`sync_hpr`).
+- Files for new shows are saved locally in a directory called `'hprXXXX'`, based on the show number. A record is kept of the `upload/` sub-directory where the show came from so that the end result can be uploaded to it (in the file `.origin`).
+- For each new show the following steps are carried out:
+  - The raw `shownotes.txt` file is viewed so it can be checked for errors (`do_show`). This is the point at which errors like misspellings in the title, summary or tags can be determined. These are corrected by editing the raw file and re-uploading it. The script used to perform such edits is `do_repair`.
+  - The `shownotes.txt` file is parsed. The declared note format is saved for future reference in a file called `.format`, and the notes themselves are stored in a file called `'hprXXXX.out'`. Parsing is controlled by `do_parse` which calls a Perl script `parse_shownotes`. Any obvious anomalies such as missing media, summary or tags are flagged at this stage by the Perl script. This script also tries to determine whether the declared format (e.g. HTML5) matches the actual note content, and flags any apparent errors.
+  - It is possible to change the declared format at this stage if it seems appropriate (e.g. it's **not** HTML, just plain text) using script `do_change_format`.
+  - The show notes can then be edited. This is done with `do_vim`. This script passes the declared file format to Vim in order to enable the relevant syntax. If the format is 'HTML5' a validator is run on the notes (script `validate_html`). If there are errors these are passed to Vim so that the problems can be found and corrected. For other formats an external script (`make_markdown`) can be used to convert selected parts or the whole file to Markdown when desired.
+  - Having produced suitable Markdown (as appropriate) or other format, the notes can be converted to HTML5 using `pandoc`. This stage is skipped if the notes are already HTML5. The script used is called `do_pandoc`. Two types of files are generated by this script: the first is the HTML fragment destined for the HPR database, called `'hprXXXX.html'`; the second is for local consumption and is a full standalone file which uses the HPR CSS, called `'hprXXXX_full.html'`. The `do_pandoc` script passes `pandoc` various settings according to the format of the input file and the desired output file.
+  - The HTML is viewed with the script `do_browser` (which is actually a soft link to another script tailored for the particular preferred browser; currently the browser is `brave` and the script is `do_brave`).
+  - There is usually an iteration between editing, running `pandoc` and viewing in the browser before the notes are accepted.
+  - The final stage is to run `do_upload` which copies the HTML fragment to the HPR site under the appropriate `upload/` sub-directory.
+- The saved show files are deleted by a cron job according to their age. Currently they are stored for 6 months.

Show_Submission/author_title.pl

@@ -0,0 +1,114 @@
+#!/usr/bin/env perl
+#===============================================================================
+#
+#         FILE: author_title.pl
+#
+#        USAGE: ./author_title.pl JSON_file YAML_file
+#
+#  DESCRIPTION: Reads the JSON file generated as a new HPR show is uploaded
+#               via the submission form. Converts certain elements to a YAML
+#               structure which is written to a file for use by Pandoc when it
+#               generates a version of the notes used for local proof-reading.
+#
+#               This function has been performed by 'jq' in the past, but with
+#               mixed success. Also 'yq' has been used experimentally with
+#               little success because of the wide variety of the input JSON
+#               data. This script has been written to try and achieve parsing,
+#               data extraction and YAML generation in a more controllable
+#               way.
+#
+#      OPTIONS: ---
+# REQUIREMENTS: ---
+#         BUGS: ---
+#        NOTES: 2024-10-18 Modified the YAML to include the field Pandoc is
+#               looking for: 'subtitle', This adds another header to the full
+#               notes, which allows another check to be carried out. Also
+#               using Pandoc template 'hpr_dev.html' which omits the '<base>'
+#               tag.
+#       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
+#      VERSION: 0.0.3
+#      CREATED: 2023-10-13 22:37:36
+#     REVISION: 2024-10-18 16:17:14
+#
+#===============================================================================
+
+use v5.16;
+use strict;
+use warnings;
+use feature qw{ postderef say signatures state try };
+no warnings
+    qw{ experimental::postderef experimental::signatures experimental::try };
+
+use JSON;
+use YAML::Tiny;
+use Data::Dumper;
+
+#
+# Version number (Incremented by Vim)
+#
+our $VERSION = '0.0.3';
+
+#
+# Script and directory names
+#
+( my $PROG = $0 ) =~ s|.*/||mx;
+( my $DIR  = $0 ) =~ s|/?[^/]*$||mx;
+$DIR = '.' unless $DIR;
+
+#-------------------------------------------------------------------------------
+# Declarations
+#-------------------------------------------------------------------------------
+#
+# Constants and other declarations
+#
+my $basedir = "$ENV{HOME}/HPR/Show_Submission";
+
+#
+# Enable Unicode mode
+#
+binmode STDOUT, ":encoding(UTF-8)";
+binmode STDERR, ":encoding(UTF-8)";
+
+#
+# Check arguments
+#
+die "Usage: $PROG JSON_file YAML_file\n" if ( scalar(@ARGV) != 2 );
+my $json_file = shift;
+my $yaml_file = shift;
+
+die "Unable to find $json_file\n" unless ( -e $json_file );
+
+#
+# Read and parse the JSON
+#
+my $json = JSON->new->utf8;
+
+open( my $jfile, "<:encoding(UTF-8)", $json_file )
+    or die "Unable to open $json_file\n";
+my $json_text = <$jfile>;
+close($jfile);
+
+my $content = decode_json($json_text);
+
+#
+# Construct the YAML (is 'subtitle' better?)
+#
+my $yaml = YAML::Tiny->new(
+    {   author   => $content->{host}->{Host_Name},
+        title    => $content->{episode}->{Title},
+        subtitle => $content->{episode}->{Summary},
+        summary  => $content->{episode}->{Summary},
+    }
+);
+#say Dumper(\$yaml);
+
+#
+# Output the YAML
+#
+unless ( $yaml->write($yaml_file) ) {
+    warn "Problem writing the YAML to $yaml_file\n";
+}
+
+exit;
+
+# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker

Show_Submission/copy_shownotes

@@ -18,7 +18,7 @@
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
 #      VERSION: 0.0.10
 #      CREATED: 2015-09-16 21:51:15
-#     REVISION: 2023-07-01 22:48:53
+#     REVISION: 2024-06-18 20:42:17
 #
 #===============================================================================
 
@@ -29,9 +29,9 @@ SCRIPT=${0##*/}
 #
 # Load library functions
 #
-LIB="$HOME/bin/function_lib.sh"
+LIB="$HOME/HPR/function_lib.sh"
 [ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
-# shellcheck source=/home/cendjm/bin/function_lib.sh
+# shellcheck source=/home/cendjm/HPR/function_lib.sh
 source "$LIB"
 
 #

Show_Submission/do_brave

@@ -19,10 +19,17 @@
 #               2022-12-22: We now write state changes to the file .status in
 #               the show directory, so we need to do that here too. Also
 #               changed to using the function library for cleanup_temp.
+#               2024-10-17: Changed the logic around 'hpr????_full.html' by
+#               using a TT² template to enclose the usual HTML fragment in
+#               enough HTML to make it standalone. We have to get values from
+#               'shownotes.json' and paass them into 'tpage' to do this, and
+#               the end result is not the same as the one generated by
+#               'do_pandoc'. The result looks better than using the HTML
+#               fragment.
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
-#      VERSION: 0.0.5
+#      VERSION: 0.0.6
 #      CREATED: 2016-03-20 15:22:29
-#     REVISION: 2022-12-22 17:28:12
+#     REVISION: 2024-10-17 19:33:42
 #
 #===============================================================================
 
@@ -47,18 +54,29 @@ if [[ $# -ne 1 ]]; then
     exit 1
 fi
 
+epno="${1}"
+
 #
 # Directories and files
 #
 BASENAME="$HOME/HPR/Show_Submission"
-SHOWDIR="$BASENAME/shownotes/hpr${1}"
-RAWNOTES="$SHOWDIR/hpr${1}.out"
-HTML="$SHOWDIR/hpr${1}.html"
-FULLHTML="$SHOWDIR/hpr${1}_full.html"
+SHOWDIR="$BASENAME/shownotes/hpr${epno}"
+RAWNOTES="$SHOWDIR/hpr${epno}.out"
+HTML="$SHOWDIR/hpr${epno}.html"
+FULLHTML="$SHOWDIR/hpr${epno}_full.html"
+SHOWNOTES="$SHOWDIR/shownotes.json"
 STATUSFILE="$SHOWDIR/.status"
+FORMATFILE="$SHOWDIR/.format"
+FORMAT="$(cat "$FORMATFILE")"
 
 HTMLFILE="$FULLHTML"
 
+#
+# Check we have this template
+#
+FULLTPL="$BASENAME/make_fullnotes.tpl"
+[[ -e $FULLTPL ]] || { echo "Unable to find template $FULLTPL"; exit 1; }
+
 #
 # Check we have this browser
 #
@@ -66,28 +84,57 @@ BRAVE=$(command -v brave-browser)
 [[ -v BRAVE ]] || { echo "Problem finding the Brave browser"; exit 1; }
 
 #
-# We prefer to view the 'full' html which we do by default. If not found
-# (because the host sent in HTML themselves) we look for hpr????.html, which
-# is a link to the notes from the form (hpr????.out), and view that. If the
-# link didn't get created (not sure why) we copy the "raw" notes to
-# a temporary file with an '.html' extension (TODO: we could just make a link
-# here). Otherwise we found nothing viewable.
+# Taking a different approach with the 'full' html. If the format is known to
+# be 'html5' there will not be one the first time we run this script. We will
+# make 'hpr????_full.html' by enclosing the HTML "fragment" in 'hpr????.html'
+# in a TT² template with HTML header and footer.
 #
-if [[ ! -e $FULLHTML ]]; then
-    if [[ -e $HTML ]]; then
-        echo "No full HTML found, viewing $HTML instead"
-        HTMLFILE="$HTML"
-    elif [[ -e $RAWNOTES ]]; then
-        echo "No files with ''.HTML' suffix, viewing raw notes"
+if [[ $FORMAT = 'html5' ]]; then
+    #
+    # Extract the fields we want from the JSON and make them Bash variables
+    #
+    declare _author _title _summary # Declare them for shellcheck
+    jqprog='@sh "_author=\(.host.Host_Name) '
+    jqprog+='_title=\(.episode.Title) '
+    jqprog+='_summary=\(.episode.Summary)"'
+    eval "$(jq -r "$jqprog" "$SHOWNOTES")"
 
-        TMP1=$(mktemp '/tmp/notes_XXXXXX.html') || { echo "$SCRIPT: creation of temporary file failed!"; exit 1; }
-        trap 'cleanup_temp $TMP1' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
+    #
+    # Feed the variables to the template to make the full HTML
+    #
+    tpage --define author="$_author" \
+        --define title="$_title" \
+        --define summary="$_summary" \
+        --define body="$HTML" \
+        "$FULLTPL" > "$FULLHTML"
+else
+    #
+    # See above for how we make the 'full' notes if the host sent in HTML
+    # notes. Here we have received notes that use one of the excepted markup
+    # formats or are plain text (≡ Pandoc Markdown).
+    #
+    # If the 'full' HTML is not found (for unknown reasons) we look for
+    # hpr????.html, which is a link to the notes from the form (hpr????.out),
+    # and view that. If the link didn't get created (not sure why) we copy the
+    # "raw" notes to a temporary file with an '.html' extension (TODO: we
+    # could just make a link here). Otherwise we found nothing viewable.
+    #
+    if [[ ! -e $FULLHTML ]]; then
+        if [[ -e $HTML ]]; then
+            echo "No full HTML found, viewing $HTML instead"
+            HTMLFILE="$HTML"
+        elif [[ -e $RAWNOTES ]]; then
+            echo "No files with ''.HTML' suffix, viewing raw notes"
 
-        cp "$RAWNOTES" "$TMP1"
-        HTMLFILE="$TMP1"
-    else
-        echo "Nothing to view, giving up"
-        exit
+            TMP1=$(mktemp '/tmp/notes_XXXXXX.html') || { echo "$SCRIPT: creation of temporary file failed!"; exit 1; }
+            trap 'cleanup_temp $TMP1' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
+
+            cp "$RAWNOTES" "$TMP1"
+            HTMLFILE="$TMP1"
+        else
+            echo "Nothing to view, giving up"
+            exit
+        fi
     fi
 fi
 

Show_Submission/do_index

@@ -1,4 +1,4 @@
-#!/bin/bash - 
+#!/bin/bash -
 #===============================================================================
 #
 #         FILE: do_index
@@ -11,11 +11,11 @@
 #      OPTIONS: ---
 # REQUIREMENTS: ---
 #         BUGS: ---
-#        NOTES: ---
+#        NOTES: [[ Probably obsolete! ]]
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
 #      VERSION: 0.0.5
 #      CREATED: 2022-10-30 15:39:28
-#     REVISION: 2022-12-17 17:38:00
+#     REVISION: 2024-06-18 20:28:55
 #
 #===============================================================================
 
@@ -31,9 +31,9 @@ STDOUT="/dev/fd/2"
 #
 # Load library functions (make_file_list, range_parse, cleanup_temp)
 #
-LIB="$HOME/bin/function_lib.sh"
+LIB="$HOME/HPR/function_lib.sh"
 [ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
-# shellcheck source=/home/cendjm/bin/function_lib.sh
+# shellcheck source=/home/cendjm/HPR/function_lib.sh
 source "$LIB"
 
 #

Show_Submission/do_pandoc

@@ -37,9 +37,9 @@
 #               careful about collisions.
 #
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
-#      VERSION: 0.2.10
+#      VERSION: 0.2.11
 #      CREATED: 2016-08-16 15:34:30
-#     REVISION: 2024-02-18 13:27:40
+#     REVISION: 2024-10-18 23:03:25
 #
 #===============================================================================
 
@@ -48,16 +48,16 @@ set -o nounset                              # Treat unset variables as an error
 SCRIPT=${0##*/}
 #DIR=${0%/*}
 
-VERSION='0.2.10'
+VERSION='0.2.11'
 
 STDOUT="/dev/fd/2"
 
 #
 # Load library functions
 #
-LIB="$HOME/bin/function_lib.sh"
+LIB="$HOME/HPR/function_lib.sh"
 [ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
-# shellcheck source=/home/cendjm/bin/function_lib.sh
+# shellcheck source=/home/cendjm/HPR/function_lib.sh
 source "$LIB"
 
 #
@@ -65,7 +65,7 @@ source "$LIB"
 #
 define_colours
 
-# {{{ Functions: -- _usage -- _DEBUG --
+# {{{ Functions: -- _usage --
 #===  FUNCTION  ================================================================
 #         NAME: _usage
 #  DESCRIPTION: Report usage
@@ -102,19 +102,6 @@ Examples
 endusage
     exit
 }
-
-#===  FUNCTION  ================================================================
-#         NAME: _DEBUG
-#  DESCRIPTION: Writes one or more message lines if in DEBUG mode
-#   PARAMETERS: List of messages
-#      RETURNS: Nothing
-#===============================================================================
-_DEBUG () {
-    [ "$DEBUG" == 0 ] && return
-    for msg in "$@"; do
-        printf 'D> %s\n' "$msg"
-    done
-}
 # }}}
 
 #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -168,8 +155,8 @@ options[txt2tags]=''
 #
 # Sanity checks
 #
-JQ=$(command -v jq)
-[ -n "$JQ" ] || { echo "Program 'jq' was not found"; exit 1; }
+# JQ=$(command -v jq)
+# [ -n "$JQ" ] || { echo "Program 'jq' was not found"; exit 1; }
 # YQ=$(command -v yq)
 # [ -n "$YQ" ] || { echo "Program 'yq' was not found"; exit 1; }
 
@@ -235,7 +222,7 @@ trap 'cleanup_temp $TMP1 $TMP2 $TMP3' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
 # Main directory
 BASENAME="$HOME/HPR/Show_Submission"
 
-# JSON to YAML Perl script
+# JSON to YAML Perl script - sanity check
 J2Y="$BASENAME/author_title.pl"
 [ -e "$J2Y" ] || { echo "Program '$J2Y' was not found"; exit 1; }
 
@@ -303,19 +290,23 @@ BASEURL='https://hackerpublicradio.org/eps/'
 # needed for Pandoc
 #
 # Non-YAML alternative - not chosen
-#jqprog="@text \"author: \(.host.Host_Name)\ntitle: \(.episode.Title)\""
+# jqprog="@text \"author: \(.host.Host_Name)\ntitle: \(.episode.Title)\""
 #
 # Testing another formatter (Journal 2023-03-03)
-#jqprog="@sh \"---\nauthor: \(.host.Host_Name)\ntitle: \(.episode.Title)\n---\""
+# jqprog="@sh \"---\nauthor: \(.host.Host_Name)\ntitle: \(.episode.Title)\n---\""
 # Added quotes around the generated strings (2023-03-31)
 # jqprog="@text \"---\nauthor: \(.host.Host_Name)\ntitle: \(.episode.Title)\n---\""
+#
 # Moved to 'yq' 2023-04-01
 # jqprog="@text \"---\nauthor: '\(.host.Host_Name)'\ntitle: '\(.episode.Title)'\n---\""
 # jq -r "$jqprog" "$JSONFILE" > "$TMP1"
+#
 # On 2023-10-01 wrote a Perl JSON to YAML generator just for these two
 # elements. It's called 'author_title.pl'
+#
 # yqprog='{author:.host.Host_Name,title:.episode.Title}'
 # ( echo "---"; $YQ -y "$yqprog" "$JSONFILE"; echo "---"; ) > "$TMP1"
+#
 $J2Y "$JSONFILE" "$TMP1"
 _DEBUG "YAML:" "$(cat "$TMP1")"
 
@@ -497,7 +488,7 @@ if [[ $DRYRUN -eq 0 ]]; then
     #
     # shellcheck disable=SC2086
     pandoc -f ${FROM}${POPTIONS} -t html5 --ascii \
-        --standalone --template=hpr.html5 --no-highlight \
+        --standalone --template=hpr_dev.html5 --no-highlight \
         -c https://hackerpublicradio.org/css/hpr.css \
         --metadata-file="$TMP1" -o "$FULLHTML" "$TMP3"
     RES=$?

Show_Submission/do_pandoc_assets

@@ -0,0 +1,189 @@
+#!/bin/bash -
+#===============================================================================
+#
+#         FILE: do_pandoc_assets
+#
+#        USAGE: ./do_pandoc_assets shownumber
+#
+#  DESCRIPTION: Runs Pandoc on any assets that need it. The assets are in the
+#               directory ~/HPR/Show_Submission/shownotes/hpr1234/ and the
+#               result is written to the 'uploads' directory to go to the HPR
+#               server.
+#
+#      OPTIONS: ---
+# REQUIREMENTS: ---
+#         BUGS: ---
+#        NOTES: ---
+#       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
+#      VERSION: 0.0.1
+#      CREATED: 2024-08-14 18:54:21
+#     REVISION: 2024-08-14 23:15:48
+#
+#===============================================================================
+
+set -o nounset                              # Treat unset variables as an error
+
+SCRIPT=${0##*/}
+# DIR=${0%/*}
+
+VERSION='0.0.1'
+
+STDOUT="/dev/fd/2"
+
+#
+# Load library functions
+#
+LIB="$HOME/HPR/function_lib.sh"
+[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
+# shellcheck source=/home/cendjm/HPR/function_lib.sh
+source "$LIB"
+
+#
+# Colour codes
+#
+define_colours
+
+# {{{ Functions: -- _usage --
+#===  FUNCTION  ================================================================
+#         NAME: _usage
+#  DESCRIPTION: Report usage
+#   PARAMETERS: None
+#      RETURNS: Nothing
+#===============================================================================
+_usage () {
+    cat >$STDOUT <<-endusage
+Usage: ./${SCRIPT} [-h] [-d] shownumber
+
+Version: $VERSION
+
+Runs Pandoc against assets for a particular show, catering for the case when
+a host sends in external notes (referenced from the main notes) in Markdown
+format rather than HTML.
+
+For the moment the only format catered for is Markdown, in files with the
+suffix 'md' or 'mkd'
+
+Options:
+  -h                    Print this help
+  -d                    Select dry run mode
+
+Arguments:
+    shownumber
+
+Examples
+  ./${SCRIPT} -h
+  ./${SCRIPT} -d 2240
+
+endusage
+    exit
+}
+# }}}
+
+#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+#
+# Sanity checks
+#
+JQ=$(command -v jq)
+[ -n "$JQ" ] || { echo "Program 'jq' was not found"; exit 1; }
+
+#
+# Process options first
+#
+while getopts :dDh opt
+do
+    case "${opt}" in
+        d) DRYRUN=1;;
+        h) _usage;;
+        ?) echo "$SCRIPT: Invalid option; aborting"; exit 1;;
+    esac
+done
+shift $((OPTIND - 1))
+
+#
+# Default options if not provided
+#
+DRYRUN=${DRYRUN:-0}
+
+#
+# Check there's an argument after removing any options. Abort if not
+#
+if [[ $# -ne 1 ]]; then
+    _usage
+fi
+
+#
+# Make the explicit show id, catering for leading zeroes (belt & braces)
+#
+# TODO: cater for 'hpr1234' as well as a plain number
+printf -v SHOWID 'hpr%04d' "$1"
+
+#
+# Paths to files
+#
+# ------------------------------------------------------------------------------
+# Main directory
+BASENAME="$HOME/HPR/Show_Submission"
+
+# JSON to YAML Perl script
+# J2Y="$BASENAME/author_title.pl"
+# [ -e "$J2Y" ] || { echo "Program '$J2Y' was not found"; exit 1; }
+
+# The notes for all shows are here
+SHOWNOTES="$BASENAME/shownotes"
+
+# Notes for this show are here
+SHOWDIR="$SHOWNOTES/$SHOWID"
+
+# Assets which need Pandoc are here
+ASSETDIR="$SHOWDIR/uploads"
+
+# The incoming JSON from the upload form
+JSONFILE="$SHOWDIR/shownotes.json"
+
+#
+# Check there are Markdown files in the asset directory and collect their
+# names
+#
+declare -a MARKDOWN
+mapfile -t MARKDOWN < <(find "$ASSETDIR" -type f -regextype posix-extended -regex '.*\.mk?d')
+
+#
+# Nothing found?
+#
+if [[ ${#MARKDOWN[@]} -eq 0 ]]; then
+    coloured 'red' "No Markdown files found"
+    exit 1
+fi
+
+# coloured 'cyan' "Markdown files found:"
+# printf '%s\n' "${MARKDOWN[@]}"
+
+#
+# Process whatever markup we have, only catering for Markdown at present
+#
+for file in "${MARKDOWN[@]}"; do
+    coloured 'yellow' "Processing: $file"
+    OUTFILE="${file%.*}.html"
+
+    if [[ $DRYRUN -eq 0 ]]; then
+        #
+        # Build nice HTML, disabling the smart quotes and stuff, but using
+        # a local format definition and the HPR CSS. For now we just add
+        # a title from the JSON. We could give Pandoc a block of YAML with
+        # more attributes if we wished.
+        #
+        pandoc -f markdown-smart -t html5  --standalone \
+            --template=hpr.html5 -c https://hackerpublicradio.org/css/hpr.css \
+            --metadata title="$(jq -r '.episode.Title' "$JSONFILE")" --strip-comments \
+            -o "$OUTFILE" "${file}"
+        coloured 'green' "Wrote: $OUTFILE"
+    else
+        coloured 'yellow' "Dry run mode: Would have written $OUTFILE"
+    fi
+done
+
+exit
+
+# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker
+

Show_Submission/do_parse

@@ -124,6 +124,7 @@ fi
 #
 # $PARSER -novalid -ep "${show}" -in "$FROM" -show "$TOTPL" \
 #     -json "$JSONTPL" -format="$FMT"
+#
 $PARSER -ep "${show}" -in "$FROM" -show "$TOTPL" \
     -format="$FMT" -release="$REL" -pictures="$PICTURES" -assets="$ASSETS" \
     -zip="$ZIP"

Show_Submission/do_pictures

@@ -60,9 +60,9 @@
 #         BUGS: ---
 #        NOTES: ---
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
-#      VERSION: 0.2.3
+#      VERSION: 0.2.4
 #      CREATED: 2020-05-25 13:20:18
-#     REVISION: 2024-02-22 14:32:03
+#     REVISION: 2024-07-25 21:39:16
 #
 #===============================================================================
 
@@ -71,16 +71,16 @@ set -o nounset                              # Treat unset variables as an error
 SCRIPT=${0##*/}
 # DIR=${0%/*}
 
-VERSION="0.2.3"
+VERSION="0.2.4"
 
 STDOUT="/dev/fd/2"
 
 #
 # Load library functions (make_file_list, range_parse, cleanup_temp)
 #
-LIB="$HOME/bin/function_lib.sh"
+LIB="$HOME/HPR/function_lib.sh"
 [ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
-# shellcheck source=/home/cendjm/bin/function_lib.sh
+# shellcheck source=/home/cendjm/HPR/function_lib.sh
 source "$LIB"
 
 #
@@ -88,7 +88,7 @@ source "$LIB"
 #
 define_colours
 
-# {{{ -- _usage --
+# {{{ -- _usage - _silent -- _dryrun --
 #===  FUNCTION  ================================================================
 #         NAME: _usage
 #  DESCRIPTION: Report usage
@@ -150,9 +150,7 @@ Examples
 endusage
     exit
 }
-# }}}
 
-# {{{ -- _silent -- _dryrun -- _DEBUG --
 #===  FUNCTION  ================================================================
 #         NAME: _silent
 #  DESCRIPTION: Output a message unless we're being silent
@@ -179,19 +177,6 @@ _dryrun () {
         prefix=
     done
 }
-
-#===  FUNCTION  ================================================================
-#         NAME: _DEBUG
-#  DESCRIPTION: Writes one or more message lines if in DEBUG mode
-#   PARAMETERS: List of messages
-#      RETURNS: Nothing
-#===============================================================================
-_DEBUG () {
-    [ "$DEBUG" == 0 ] && return
-    for msg in "$@"; do
-        printf 'D> %s\n' "$msg"
-    done
-}
 # }}}
 
 #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -229,12 +214,25 @@ if [[ $# -ne 1 ]]; then
     echo "${red}Missing shownumber argument${reset}"
     _usage
 fi
+epno="${1}"
+
+#
+# Ensure item spec is correctly formatted. Have to cater for leading zeroes
+# being interpreted as octal.
+#
+if [[ $epno =~ ^(hpr)?([0-9]+)$ ]]; then
+    epno="$((10#${BASH_REMATCH[-1]}))"
+else
+    coloured 'red' "Incorrect show specification: $epno"
+    coloured 'yellow' "Use 'hpr9999' or '9999' formats"
+    exit 1
+fi
 
 #
 # Paths to files
 #
 BASEDIR="$HOME/HPR/Show_Submission"
-SHOWDIR="$BASEDIR/shownotes/hpr${1}"
+SHOWDIR="$BASEDIR/shownotes/hpr${epno}"
 # RAWFILE="$SHOWDIR/shownotes.txt"
 JSONFILE="$SHOWDIR/shownotes.json"
 PICDIR="$SHOWDIR/uploads"

Show_Submission/do_report

@@ -15,7 +15,7 @@
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
 #      VERSION: 0.0.8
 #      CREATED: 2022-09-07 15:27:29
-#     REVISION: 2023-06-01 17:58:09
+#     REVISION: 2024-06-18 20:13:46
 #
 #===============================================================================
 
@@ -32,9 +32,9 @@ STDOUT="/dev/fd/2"
 #
 # Load library functions
 #
-LIB="$HOME/bin/function_lib.sh"
+LIB="$HOME/HPR/function_lib.sh"
 [ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
-# shellcheck source=/home/cendjm/bin/function_lib.sh
+# shellcheck source=/home/cendjm/HPR/function_lib.sh
 source "$LIB"
 
 #
@@ -42,7 +42,7 @@ source "$LIB"
 #
 define_colours
 
-#{{{ Functions: --- _usage --- _DEBUG --- _verbose --- _silent ---
+#{{{ Functions: --- _usage --- _verbose --- _silent ---
 #===  FUNCTION  ================================================================
 #         NAME: _usage
 #  DESCRIPTION: Report usage
@@ -77,22 +77,6 @@ endusage
     exit
 }
 
-#===  FUNCTION  ================================================================
-#         NAME: _DEBUG
-#  DESCRIPTION: Writes one or more messages if in DEBUG mode. Each argument is
-#               seen as a message and is written on a separate line.
-#               References the global variable 'DEBUG' which is expected to be
-#               True if debug output is wanted.
-#   PARAMETERS: List of messages
-#      RETURNS: Nothing
-#===============================================================================
-_DEBUG () {
-    [ "$DEBUG" == 0 ] && return
-    for msg in "$@"; do
-        printf 'D> %s\n' "$msg"
-    done
-}
-
 #===  FUNCTION  ================================================================
 #         NAME: _verbose
 #  DESCRIPTION: Writes a message in verbose mode

Show_Submission/do_update_reservations

@@ -15,7 +15,7 @@
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
 #      VERSION: 0.0.6
 #      CREATED: 2022-04-11 09:36:21
-#     REVISION: 2023-06-14 23:24:42
+#     REVISION: 2024-06-18 20:16:19
 #
 #===============================================================================
 
@@ -32,9 +32,9 @@ STDOUT="/dev/fd/2"
 #
 # Load library functions
 #
-LIB="$HOME/bin/function_lib.sh"
+LIB="$HOME/HPR/function_lib.sh"
 [ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
-# shellcheck source=/home/cendjm/bin/function_lib.sh
+# shellcheck source=/home/cendjm/HPR/function_lib.sh
 source "$LIB"
 
 #

Show_Submission/extract_images

@@ -0,0 +1,739 @@
+#!/usr/bin/env perl
+#===============================================================================
+#
+#         FILE: extract_images
+#
+#        USAGE: ./extract_images [--help] [--documentation|man]
+#               [--prefix=STRING] [--[no-]backup] [--force] [--[no]silent]
+#               HTML_file [ [HTML_file_2] [HTML_file_3] ... ]
+#
+#  DESCRIPTION: Processes HTML files which may have 'data' scheme URIs containing
+#               images, and extracts these images into files in the same
+#               directory.  The 'data' scheme links are converted to 'https'
+#               and reference the extracted files. The modified HTML is
+#               output, and the original will be saved as a backup if
+#               requested.
+#
+#               The 'data' URI scheme is specified in RFC 2397.
+#
+#      OPTIONS: ---
+# REQUIREMENTS: ---
+#         BUGS: ---
+#        NOTES: ---
+#       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
+#      VERSION: 0.0.3
+#      CREATED: 2024-12-25 10:53:15
+#     REVISION: 2024-12-31 20:56:50
+#
+#===============================================================================
+
+use v5.36;
+use strict;
+use warnings;
+use feature qw{ say state try };
+no warnings qw{ experimental::try };
+
+use open ':std', ':encoding(UTF-8)'; # Make all IO UTF-8
+
+use Getopt::Long;
+use Pod::Usage;
+
+use HTML::TreeBuilder 5 -weak;
+use URI;
+use MIME::Types;
+
+use Path::Tiny;
+use File::Copy;
+
+use Data::Dumper;
+
+#
+# Version number (Incremented by Vim)
+#
+our $VERSION = '0.0.3';
+
+#
+# Script and directory names
+#
+( my $PROG = $0 ) =~ s|.*/||mx;
+
+#-------------------------------------------------------------------------------
+# Declarations
+#-------------------------------------------------------------------------------
+my ( $notes,   $typename, $uri );
+my ( $fcount,  $basename, $filename, $suffix, $fh );
+my ( $updates, $bsuffix,  $newURL );
+
+my $backupcount = 5;
+
+#-------------------------------------------------------------------------------
+# Options and arguments
+#-------------------------------------------------------------------------------
+# {{{
+#
+# Option defaults
+#
+my $DEFDEBUG = 0;
+my $DEFPREFIX = 'image';
+
+my %options;
+Options( \%options );
+
+#
+# Default help shows minimal information
+#
+pod2usage( -msg => "$PROG version $VERSION\n", -exitval => 1, -verbose => 0 )
+    if ( $options{'help'} );
+
+#
+# The -documentation or -man option shows the full POD documentation through
+# a pager for convenience
+#
+pod2usage( -msg => "$PROG version $VERSION\n", -exitval => 1, -verbose => 2 )
+    if ( $options{'documentation'} );
+
+#
+# Collect options
+#
+my $DEBUG  = ( defined( $options{debug} )  ? $options{debug}  : $DEFDEBUG );
+my $silent = ( defined( $options{silent} ) ? $options{silent} : 0 );
+my $prefix = ( defined( $options{prefix} ) ? $options{prefix} : $DEFPREFIX );
+my $backup = ( defined( $options{backup} ) ? $options{backup} : 1 );
+my $force  = ( defined( $options{force} )  ? $options{force}  : 0 );
+
+#
+# Check we have arguments
+#
+pod2usage(
+    -msg     => "Missing arguments. One or more HTML file names are needed\n",
+    -exitval => 1,
+    -verbose => 0
+    )
+unless ( scalar(@ARGV) > 0 );
+
+#
+# Clean up the prefix
+#
+$prefix = ($prefix =~ /(.*)_$/ ? $1 : $prefix);
+
+#
+# Backup suffix (the dot gets added later)
+#
+$bsuffix = 'bak';
+
+#
+# Debug the options
+#
+_debug(
+    $DEBUG > 1,
+    '$silent = ' . $silent,
+    '$prefix = ' . $prefix,
+    '$backup = ' . $backup,
+    '$force = ' . $force,
+);
+
+# }}}
+
+#-------------------------------------------------------------------------------
+# Prepare a MIME::Types object
+#-------------------------------------------------------------------------------
+my $mt = MIME::Types->new;
+
+#-------------------------------------------------------------------------------
+# Loop through the arguments
+#-------------------------------------------------------------------------------
+foreach my $notesfile (@ARGV) {
+
+    #
+    # Get the absolute path of the argument
+    #
+    my $abs_nf = path($notesfile)->absolute;
+
+    #
+    # Get the 'dirname' from the absolute path
+    #
+    my $dirname = path($abs_nf)->parent->stringify;
+
+    unless (-e $abs_nf) {
+        warn "Unable to find $notesfile\n";
+        next;
+    }
+
+    #
+    # Force the MIME type of $notesfile to a string
+    #
+    $typename = "" . coalesce( $mt->mimeTypeOf("$abs_nf"), '' );
+
+    #
+    # Check the MIME type and reject non-HTML
+    #
+    unless ($typename eq 'text/html') {
+        warn "File $abs_nf is not HTML\n";
+        next
+    }
+
+    say "Reading from $abs_nf\n" unless $silent;
+
+    #
+    # Get HTML file basename (no path) without the suffix for building filenames
+    #
+    $basename = path($abs_nf)->basename('.html');
+
+    #
+    # Read the HTML
+    #
+    open (my $nfh, '<', $notesfile) or die "Unable to open $notesfile\n";
+    $notes = <$nfh>;
+    close($nfh);
+
+    #
+    # Image files are to have an index/sequence number
+    #
+    $fcount = 0;
+
+    #
+    # Keep a note of HTML updates
+    #
+    $updates = 0;
+
+    #
+    # Initialise the TreeBuilder
+    #
+    my $tree = HTML::TreeBuilder->new;
+    $tree->ignore_unknown(0);
+    $tree->no_expand_entities(1);
+    $tree->p_strict(1);
+    $tree->store_comments(1);
+    $tree->warn(1);
+
+    #
+    # Load the HTML obtained from the file into the TreeBuilder
+    #
+    $tree->parse_content($notes)
+        or die "HTML::TreeBuilder failed to parse notes: $!\n";
+
+    #
+    # Loop through the tree looking for 'data' scheme images
+    #
+    for ( @{ $tree->extract_links('img') } ) {
+        my ( $link, $element, $attr, $tag ) = @$_;
+
+        $uri = URI->new($link);
+        unless ($silent) {
+            say "Scheme: ",     $uri->scheme;
+            say "Media type: ", $uri->media_type;
+        }
+
+        #
+        # We only care about 'data' scheme stuff - for now anyway, and only
+        # images within this set
+        #
+        if ( $uri->scheme eq 'data' ) {
+            #
+            # Only images
+            #
+            if ( $uri->media_type =~ /^image/ ) {
+              #
+              # Extract the file name suffix from the MIME string, and give it
+              # a leading '.'
+              #
+                ( $suffix = $uri->media_type ) =~ s{^.*/}{.};
+
+                #
+                # Construct the filename for this image making sure it's in
+                # the directory where the HTML is located.
+                #
+                # ${fileprefix}_${prefix}_${increment}.${extension}
+                #
+                $fcount++;
+                $filename
+                    = "$dirname/${basename}_${prefix}_${fcount}${suffix}";
+                say "Writing to: $filename" unless $silent;
+
+                say '-' x 40 unless $silent;
+
+                #
+                # If the file exists and --force is not active, warn and skip.
+                # Otherwise write the file.
+                #
+                if ( -e $filename && !$force ) {
+                    warn "File $filename exists; not overwriting\n";
+                }
+                else {
+                    #
+                    # Write the data to the file in binary format. The URI
+                    # module does the conversion so it's already binary.
+                    #
+                    $fh = path($filename)->openw_raw;
+                    print $fh $uri->data;
+                    close($fh);
+                }
+
+                #
+                # Update the HTML with a link to the file we created (or that
+                # was already there from an earlier time).
+                #
+                $updates++;
+                $newURL = path($filename)->basename;
+                $element->attr( $attr, $newURL );
+            }
+        }
+
+    }    # extract_links loop
+
+    #
+    # Output the changed HTML turning what became standalone back into
+    # a <body> fragment.
+    #
+    if ($updates > 0) {
+        my $body = $tree->look_down( _tag => 'body' );
+        ( my $result = $body->as_HTML( undef, ' ', {} ) )
+            =~ s{(^<body[^>]*>|</body>$)}{}gi;
+
+        #$notesfile = path($notesfile)->basename;
+        if ($backup) {
+            if (_backup( $abs_nf, $bsuffix, $backupcount )) {
+                say "$notesfile backed up" unless $silent;
+            }
+        }
+        else {
+            say "$notesfile not backed up" unless $silent;
+        }
+
+        $fh = path($notesfile)->openw;
+        say $fh $result;
+        close($fh);
+
+        say "$updates images converted, $notesfile updated" unless $silent;
+    }
+    else {
+        say "No images found, no changes made to $notesfile" unless $silent;
+    }
+
+} # $notesfile loop
+
+exit;
+
+#===  FUNCTION  ================================================================
+#         NAME: _backup
+#      PURPOSE: Given a file, make a backup of it by appending $suffix, and
+#               handle previous backups
+#   PARAMETERS: $filename               path of file to backup
+#               $suffix                 suffix to use, default 'bck'
+#               $limit                  number of backups to keep
+#      RETURNS: True or false depending on success
+#  DESCRIPTION: Checks that the file exists and returns FALSE if it doesn't.
+#               Rationalises the $limit to avoid it being 0 or less.
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub _backup {
+    my ( $filename, $suffix, $limit ) = @_;
+
+    my ( $backupname, $limitname );
+
+    #
+    # Check the target file exists
+    #
+    unless ( -e $filename ) {
+        warn "Unable to find $filename to backup\n";
+        return 0;
+    }
+
+    #
+    # Handle invalid $limit values
+    #
+    $limit = 1 unless ( $limit >= 1 );
+
+    #
+    # Defaults
+    #
+    $suffix     = 'bck' unless $suffix;
+    $limitname  = "$filename.$suffix.$limit";
+    $backupname = "$filename.$suffix";
+
+    #
+    # Look for existing backups
+    #
+    if ( -e $backupname ) {
+
+        #
+        # If maximum version exists delete it
+        #
+        if ( -e $limitname ) {
+            unlink($limitname);
+        }
+
+        #
+        # Go backwards through the versions incrementing their version numbers
+        #
+        for ( my $vsn = $limit - 1; $vsn > 0; $vsn-- ) {
+            if ( -e "$filename.$suffix.$vsn" ) {
+                move( "$filename.$suffix.$vsn",
+                    sprintf( '%s.%s.%s', $filename, $suffix, $vsn + 1 ) );
+            }
+        }
+
+        #
+        # Make $filename.bck into $filename.bck.1
+        #
+        move( "$filename.$suffix", "$filename.$suffix.1" );
+
+    }
+
+    #
+    # Finally save the $filename as $filename.bck
+    #
+    move( $filename, "$filename.$suffix" );
+
+    return 1;
+}
+
+#===  FUNCTION  ================================================================
+#         NAME: coalesce
+#      PURPOSE: To find the first defined argument and return it
+#   PARAMETERS: Arbitrary number of arguments
+#      RETURNS: The first defined argument or undef if there are none
+#  DESCRIPTION: Just a simple way of ensuring an 'undef' value is never
+#               returned when doing so might be a problem.
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub coalesce {
+    foreach (@_) {
+        return $_ if defined($_);
+    }
+    return undef;    ## no critic
+}
+
+#===  FUNCTION  ================================================================
+#         NAME: _debug
+#      PURPOSE: Prints debug reports
+#   PARAMETERS: $active         Boolean: 1 for print, 0 for no print
+#               $messages...    Arbitrary list of messages to print
+#      RETURNS: Nothing
+#  DESCRIPTION: Outputs messages if $active is true. It removes any trailing
+#               newline from each one and then adds one in the 'print' to the
+#               caller doesn't have to bother. Prepends each message with 'D>'
+#               to show it's a debug message.
+#       THROWS: No exceptions
+#     COMMENTS: Differs from other functions of the same name
+#     SEE ALSO: N/A
+#===============================================================================
+sub _debug {
+    my $active = shift;
+
+    my $message;
+    return unless $active;
+
+    while ($message = shift) {
+        chomp($message);
+        print STDERR "D> $message\n";
+    }
+}
+
+#===  FUNCTION  ================================================================
+#         NAME: Options
+#      PURPOSE: Processes command-line options
+#   PARAMETERS: $optref     Hash reference to hold the options
+#      RETURNS: Undef
+#  DESCRIPTION: Process the options we want to offer. See the documentation
+#               for details
+#       THROWS: no exceptions
+#     COMMENTS: none
+#     SEE ALSO: n/a
+#===============================================================================
+sub Options {
+    my ($optref) = @_;
+
+    my @options = (
+        "help",     "documentation|man", "debug=i", "silent!",
+        "prefix=s", "backup!",           "force!",
+        # "config=s",
+    );
+
+    if ( !GetOptions( $optref, @options ) ) {
+        pod2usage(
+            -msg     => "$PROG version $VERSION\n",
+            -exitval => 1,
+            -verbose => 0
+        );
+    }
+
+    return;
+}
+
+__END__
+
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+#  Application Documentation
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+#{{{
+
+=head1 NAME
+
+extract_images - extract embedded images from HTML and save as files
+
+=head1 VERSION
+
+This documentation refers to extract_images version 0.0.3
+
+=head1 USAGE
+
+    extract_images
+        [--help] [--documentation|man] [-debug=N]
+        [--prefix=STRING] [--[no]backup] [--[no]force] [--[no]silent]
+        HTML_file [ [HTML_file_2] [ [HTML_file_3] ... ] ]
+
+    Examples:
+
+    extract_images --prefix=picture_ --backup index.html
+
+    extract_images --silent --nobackup shownotes.html
+
+=head1 REQUIRED ARGUMENTS
+
+The script requires the names of one or more HTML files which will be scanned
+for embedded images.
+
+=head1 OPTIONS
+
+Note that all on/off options can be written as B<--silent>, B<--nosilent> or
+B<--no-silent>. A single hyphen can be used at the start or a pair.
+
+=over 8
+
+=item B<--help>
+
+Reports brief information about how to use the script and exits. To see the
+full documentation use the option B<--documentation> or B<--man>. Alternatively,
+to generate a PDF version use the I<pod2pdf> tool from
+I<http://search.cpan.org/~jonallen/pod2pdf-0.42/bin/pod2pdf>. This can be
+installed with the cpan tool as App::pod2pdf.
+
+=item B<--documentation> or B<--man>
+
+Reports full information about how to use the script and exits. Alternatively,
+to generate a PDF version use the I<pod2pdf> tool from
+I<http://search.cpan.org/~jonallen/pod2pdf-0.42/bin/pod2pdf>. This can be
+installed with the cpan tool as App::pod2pdf.
+
+=item B<--debug=N>
+
+Run in debug mode at the level specified by I<N>. Possible values are:
+
+=over 4
+
+=item B<0>
+
+No debugging (the default).
+
+=item B<1>
+
+TBA
+
+=item B<2>
+
+Displays the internal variables used to store the options.
+
+=item B<3>
+
+TBA
+
+=back
+
+=item B<--prefix=STRING>
+
+Since embedded images have no names, when they are written to image files
+names are generated for them. These names are built from the following
+components:
+
+=over 4
+
+=item B<HTML filename>
+
+The name of the HTML file without a suffix. So, if the name is 'index.html',
+the 'index' part will be used.
+
+=item B<prefix>
+
+The prefix string provided by this option, or 'image' if not specified.
+
+=item B<counter>
+
+The images found in the HTML are counted, starting from 1, and this number is
+used here.
+
+=item B<extension>
+
+The image format extension, taken from the embedded image information.
+Examples would be 'jpg', 'jpeg' and 'png'.
+
+=back
+
+The B<HTML filename>, B<prefix>, B<counter> and B<extension> are joined
+together to make the file name as follows:
+
+    <HTML filename>_<prefix>_<counter>.<extension>
+
+So the following run of the script would generate the first file shown below
+assuming the first embedded picture was in 'PNG' format:
+
+    extract_images --prefix=picture hpr4283.html
+
+    hpr4283_picture_1.png
+
+=item B<--[no]backup>
+
+The HTML is modified to leave placeholders referring to any embedded images
+found as it it processed. The new HTML is written to the original file if
+B<--nobackup> is specified, or a backup of the original file is made, before
+writing the new HTML. Making a backup is the default behaviour.
+
+=item B<--[no]force>
+
+Images are written to the current directory. If an image already exists, and
+B<--force> has not been specified, the script will stop processing and exit.
+The default setting of this option is B<--noforce>.
+
+=item B<--[no]silent>
+
+By default the script reports its progress as it processes an HTML file, but
+this can be turned off by using this option.
+
+=back
+
+=head1 DESCRIPTION
+
+This Perl script B<extract_images> parses HTML looking for embedded images.
+Such images use a URI with the scheme 'data' followed by image details
+including the encoded binary contents. See the Wikipedia article
+https://en.wikipedia.org/wiki/Data_URI_scheme for details.
+
+When such images are found they are written as files (with generated names),
+and the HTML is updated with these names replacing the original URI. Further
+work is likely to be required to build full image links, but the majority of
+work will have been done.
+
+The script will not overwrite image files unless the B<--force> option is
+used, but will overwrite the original HTML file unless B<--backup> is
+specified.
+
+By default details of progress are written to standard output, but this can be
+preveneted by using the B<--silent> option.
+
+=head2 GENERATED IMAGE FILE NAMES
+
+These names are built from various elements:
+
+    <HTML filename>_<prefix>_<counter>.<extension>
+
+Where <HTML filename> is the base name of the HTML input file, <prefix> is the
+string provided with the B<--prefix=STRING> option (or B<image> by default),
+<counter> is a count of image files found during the scan of the HTML, and
+<extension> is the appropriate file extension for the type of image being
+converted.
+
+See the option B<--prefix=STRING> for details of file name generation.
+
+=head1 DIAGNOSTICS
+
+=over 4
+
+=item B<Unable to find ...>
+
+Type: warning
+
+The script is attempting to find a file presented to it as an argument, but
+cannot. It will skip to the next argument and continue.
+
+=item B<File ... is not HTML>
+
+Type: warning
+
+The script is checking files presented to it as arguments. The named file has
+been checked to see if it contains HTML, and it appears it does not.  It will
+skip to the next argument and continue.
+
+=item B<Unable to open ...>
+
+Type: fatal
+
+The script is attempting to open a file presented to it as an argument. Its
+existence has already been checked but it cannot be opened, possibly due to
+a permissions issue.
+
+=item B<HTML::TreeBuilder failed to parse notes: ...>
+
+Type: fatal
+
+The script has attempted to parse the HTML in a file presented to it. This has
+failed. The error message includes a report from the module used to do this.
+
+=item B<File ... exists; not overwriting>
+
+Type: warning
+
+The script is attempting to write an image file copied from the HTML, but has
+found it already exists. Using the option B<--force> would cause it to be
+overwritten, but this option is not enabled.
+
+The script will not write the file, however, it will still modify the HTML to
+reference the existing image file.
+
+=item B<Unable to find ... to backup>
+
+Type: warning
+
+The script is attempting to make a backup of one of the HTML file arguments,
+but is unable to find it. Since this is a rather unusual circumstance (has the
+file been deleted?), it is assumed it can be regenerated if needed, so the
+writing of the modified HTML is continued.
+
+=head1 CONFIGURATION AND ENVIRONMENT
+
+TBA
+
+=head1 DEPENDENCIES
+
+    Data::Dumper
+    File::Copy
+    Getopt::Long
+    HTML::TreeBuilder
+    MIME::Types
+    Path::Tiny
+    Pod::Usage
+    URI
+
+=head1 BUGS AND LIMITATIONS
+
+There are no known bugs in this module.
+Please report problems to Dave Morriss (Dave.Morriss@gmail.com)
+Patches are welcome.
+
+=head1 AUTHOR
+
+Dave Morriss  (Dave.Morriss@gmail.com)
+
+=head1 LICENCE AND COPYRIGHT
+
+Copyright (c) 2024 Dave Morriss (Dave.Morriss@gmail.com).
+All rights reserved.
+
+This module is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself. See perldoc perlartistic.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+=cut
+
+#}}}
+
+# [zo to open fold, zc to close]
+
+# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker
+

Show_Submission/fix_relative_links

@@ -25,8 +25,7 @@ use v5.16;
 use strict;
 use warnings;
 use utf8;
-use feature qw{ postderef say signatures state };
-no warnings qw{ experimental::postderef experimental::signatures };
+use feature qw{ say state };
 
 use Carp;
 use Getopt::Long;
@@ -605,7 +604,7 @@ base URL described below.
 
 =item B<-baseURL=URL>
 
-This option will default to the foillowing URL if not provided:
+This option will default to the following URL if not provided:
 
     https://hackerpublicradio.org/eps/hpr${show}/
 

Show_Submission/parse_JSON

@@ -21,14 +21,17 @@
 #         BUGS: ---
 #        NOTES: ---
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
-#      VERSION: 0.0.14
+#      VERSION: 0.0.17
 #      CREATED: 2020-11-28 10:52:02
-#     REVISION: 2024-03-09 20:34:54
+#     REVISION: 2024-10-04 18:37:29
 #
 #===============================================================================
 
-use 5.30.0;
+use v5.36;
 use utf8;
+use feature qw{ postderef say signatures state try };
+no warnings
+    qw{ experimental::postderef experimental::signatures experimental::try };
 
 use Getopt::Long;
 use Pod::Usage;
@@ -57,7 +60,7 @@ use Data::Dumper;
 #
 # Version number (manually incremented)
 #
-our $VERSION = '0.0.14';
+our $VERSION = '0.0.17';
 
 #
 # Script and directory names
@@ -79,7 +82,12 @@ my $logfile = "$logdir/${PROG}.log";
 my ( $snlevel, $showno, $summarylength, $notelength );
 
 our $MARKUP_DETECTED = 0;
+
+#
+# Maximum and minimum number of characters (bytes)
+#
 my $MAXNOTELEN = 4000;
+my $MINNOTELEN = 10;
 
 #
 # Printing: general output format
@@ -223,13 +231,14 @@ my @markup_types = (
 my @markup_found;
 my $markup_choice;
 
-my ( $showdir,     $assetdir , $statusfile );
+my ( $showdir,     $assetdir, $statusfile );
 my ( %media_files, $media_files );
 my ( %assets,      @assets,    @audio, $sanitised );
 my ( $has_audio,   $has_extra, $has_archives );
 my ( @archives,    @extracted );
 my ( $astate,      $has_pictures, @pictures, $pictures );
-my ( %spellchecks );
+my ( $has_markup,  @markup,       $markup );
+my ( $json_change, @json_changes, %spellchecks );
 my @pstates = (
     'No pictures found',                # 0
     'Pictures that need management',    # 1
@@ -347,8 +356,14 @@ open( my $fh, '<:encoding(UTF-8)', $infile );
 my $json_text = <$fh>;
 close($fh);
 
-# TODO: bad JSON can crash the script here!
-my $content = decode_json($json_text);
+my $content;
+try {
+    $content = decode_json($json_text);
+}
+catch ($e) {
+    warn colored( "Failed to decode the JSON in $infile", 'red' ) . "\n";
+    die "Error was: $e\n";
+}
 
 $log->info( $showno, "[$VERSION] Processing $infile" );
 
@@ -372,6 +387,19 @@ print STDERR '-' x 80, "\n";
 printf STDERR $ofmt, "Show:", $content->{metadata}{Episode_Number};
 printf STDERR $ofmt, "Date:", $content->{metadata}{Episode_Date};
 
+#
+# Trim off leading and trailing spaces in these fields
+#
+$json_change = 0;
+for my $key ( 'Title', 'Summary', 'Tags' ) {
+    my $str = trim($content->{episode}{$key});
+    if ($str ne $content->{episode}{$key}) {
+        $content->{episode}{$key} = $str;
+        $json_change = 1;
+        push(@json_changes,$key);
+    }
+}
+
 #
 # Detect Unicode in the Title, Summary or Tags and flag their presence.
 #
@@ -386,6 +414,9 @@ for my $key ( 'Title', 'Summary', 'Tags' ) {
         );
 }
 
+alert( $ofmt, "JSON to be updated; changes to: " .
+    join(',',@json_changes) ) if $json_change;
+
 #
 # Check summary length. The field might be filled and something might have
 # been lost.
@@ -429,7 +460,12 @@ printf STDERR $ofmt, "Notes:", $snlevel . " HTML start/end tags found";
 $notelength = length( $content->{episode}{Show_Notes} );
 if ( $notelength > $MAXNOTELEN ) {
     printf STDERR $ofmt, "Notes:",
-        colored( "Notes are longer than $MAXNOTELEN ($notelength)",
+        colored( "Notes are longer than $MAXNOTELEN bytes ($notelength)",
+        'bold yellow on_magenta' );
+}
+elsif ( $notelength <= $MINNOTELEN ) {
+    printf STDERR $ofmt, "Notes:",
+        colored( "Notes are shorter than $MINNOTELEN bytes ($notelength)",
         'bold yellow on_magenta' );
 }
 
@@ -651,6 +687,17 @@ if (%media_files) {
             _debug( $DEBUG > 2, '@assets: ' . Dumper( \@assets ) );
         }
 
+        #-------------------------------------------------------------------------------
+        # Look for un-processed markup in the assets hash. For the moment we
+        # only look for Markdown.
+        #-------------------------------------------------------------------------------
+        $has_markup = @markup
+            = grep { $assets{$_}->{type} =~ m{^text/markdown$} } keys(%assets);
+        $markup = join( ', ', @markup );
+        if (@markup) {
+            _debug( $DEBUG > 2, '@markup ' . Dumper( \@markup ) );
+        }
+
         #-------------------------------------------------------------------------------
         # Look for archive files in the assets
         #-------------------------------------------------------------------------------
@@ -690,6 +737,11 @@ if (%media_files) {
         @assets = array_difference(\@assets,\@archives);
         push(@assets, @extracted);
 
+        #
+        # Remove markup files from the assets so we don't upload them
+        #
+        @assets = array_difference(\@assets,\@markup);
+
         #
         # Remove directory stuff from @assets elements and %assets keys
         #
@@ -745,6 +797,8 @@ if (%media_files) {
         $log->info( $showno, "Media files: $media_files" ) if ($media_files);
         $log->info( $showno, "Pictures: " . join( ', ', @pictures ) )
             if (@pictures);
+        $log->info( $showno, "Markup " . join( ', ', @markup ) )
+            if (@markup);
         $log->info( $showno, "Assets: " . join( ', ', @assets ) )
             if (@assets);
 
@@ -849,7 +903,6 @@ if ( $snlevel > 0 && $content->{metadata}{Shownotes_Format} =~ /html5/i ) {
             "Apparently incorrect URLs detected in the notes", 'red'
             ), "\n";
     }
-
 }
 
 #
@@ -866,6 +919,15 @@ if (   $markup_choice eq 'html5'
         "Declared format 'plain_text' but notes seem to be HTML5!" );
 }
 
+#
+# 7. The host has sent in markup version(s) of their external notes, so we
+#    need to take special action.
+#
+if ($markup) {
+    printf STDERR "%s\n",
+        textFormat( $markup, 'Markup files:', 'L', 18, 80 );
+}
+
 #-------------------------------------------------------------------------------
 # Determine the picture asset state
 #-------------------------------------------------------------------------------
@@ -1709,6 +1771,23 @@ sub find_Unicode {
     return ( $string =~ /[^\x{00}-\x{7F}]/ );
 }
 
+#===  FUNCTION  ================================================================
+#         NAME: trim
+#      PURPOSE: Trims leading and trailing spaces from a string
+#   PARAMETERS: string          string to trim
+#      RETURNS: Trimmed string
+#  DESCRIPTION: 
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub trim {
+    my ($str) = @_;
+
+    $str =~ s/^\s+|\s+$//g;
+    return $str;
+}
+
 #===  FUNCTION  ================================================================
 #         NAME: output_file_name
 #      PURPOSE: Generate an output file name with three choices
@@ -1969,7 +2048,6 @@ sub alert {
     my ( $fmt, $message ) = @_;
 
     $fmt = "%-16s %s\n" unless $fmt;
-#    print STDERR "$bold$red", sprintf( $fmt, "** ALERT **:", $message ), "$reset";
     print STDERR colored( sprintf( $fmt, "** ALERT **:", $message ),
         'bold red' );
 }
@@ -2082,7 +2160,7 @@ parse_JSON - parse the JSON output file from the HPR show submission form
 
 =head1 VERSION
 
-This documentation refers to parse_JSON version 0.0.14
+This documentation refers to parse_JSON version 0.0.17
 
 
 =head1 USAGE

Show_Submission/query2csv

@@ -1 +0,0 @@
-/home/cendjm/HPR/Database/query2csv

Show_Submission/query2csv

@@ -0,0 +1,137 @@
+#!/usr/bin/env perl
+#===============================================================================
+#
+#         FILE: query2csv
+#
+#        USAGE: ./query2csv query
+#
+#  DESCRIPTION: Runs a query given as the only argument. Caution is needed
+#               since *any* query will be run. The result of the query is
+#               output in CSV form on STDOUT. The CSV is always quoted to
+#               cater for the more simplistic consumers.
+#
+#      OPTIONS: ---
+# REQUIREMENTS: ---
+#         BUGS: ---
+#        NOTES: Had to revert to MySQL because of a problem with DBD::MariaDB
+#       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
+#      VERSION: 0.0.2
+#      CREATED: 2015-07-11 15:53:01
+#     REVISION: 2022-02-16 23:17:16
+#
+#===============================================================================
+
+use 5.010;
+use strict;
+use warnings;
+use utf8;
+
+use Config::General;
+use Text::CSV_XS;
+use DBI;
+
+use Data::Dumper;
+
+#
+# Version number (manually incremented)
+#
+our $VERSION = '0.0.2';
+
+#
+# Script and directory names
+#
+( my $PROG = $0 ) =~ s|.*/||mx;
+( my $DIR  = $0 ) =~ s|/?[^/]*$||mx;
+$DIR = '.' unless $DIR;
+
+#-------------------------------------------------------------------------------
+# Declarations
+#-------------------------------------------------------------------------------
+#
+# Constants and other declarations
+#
+my $basedir    = "$ENV{HOME}/HPR/Database";
+my $configfile = "$basedir/.hpr_livedb.cfg";
+
+my ( $dbh, $sth1, $aref1 );
+my ( $query, $csv );
+
+#
+# Enable Unicode mode
+#
+binmode STDOUT, ":encoding(UTF-8)";
+binmode STDERR, ":encoding(UTF-8)";
+
+#
+# Load database configuration data
+#
+my $conf = Config::General->new(
+    -ConfigFile      => $configfile,
+    -InterPolateVars => 1,
+    -ExtendedAccess  => 1
+);
+my %config = $conf->getall();
+
+#-------------------------------------------------------------------------------
+# Options and arguments
+#-------------------------------------------------------------------------------
+$query = shift;
+die "Usage: $PROG query\n" unless $query;
+
+#-------------------------------------------------------------------------------
+# Connect to the database
+#-------------------------------------------------------------------------------
+my $dbhost = $config{database}->{host} // '127.0.0.1';
+my $dbport = $config{database}->{port} // 3306;
+my $dbname = $config{database}->{name};
+my $dbuser = $config{database}->{user};
+my $dbpwd  = $config{database}->{password};
+#$dbh = DBI->connect( "DBI:MariaDB:host=$dbhost;port=$dbport;database=$dbname",
+#    $dbuser, $dbpwd, { AutoCommit => 1 } )
+#    or die $DBI::errstr;
+
+$dbh = DBI->connect( "dbi:mysql:host=$dbhost;port=$dbport;database=$dbname",
+    $dbuser, $dbpwd, { AutoCommit => 1 } )
+    or die $DBI::errstr;
+
+#
+# Enable client-side UTF8
+#
+$dbh->{mysql_enable_utf8} = 1;
+
+#
+# Set up the query
+#
+$sth1 = $dbh->prepare($query) or die $DBI::errstr;
+if ( $dbh->err ) {
+    warn $dbh->errstr;
+}
+
+#
+# Perform the query
+#
+$sth1->execute;
+if ( $dbh->err ) {
+    warn $dbh->errstr;
+}
+
+#
+# Prepare to make CSV. Not sure if always quoting is the best idea though
+#
+$csv = Text::CSV_XS->new(
+#    { always_quote => 1 }
+);
+
+#
+# Loop through the returned rows making and printing CSV. Each row is returned
+# as an arrayref to make it easy to join everything.
+#
+while ( $aref1 = $sth1->fetchrow_arrayref ) {
+    $csv->combine(@$aref1);
+    print $csv->string(), "\n";
+}
+
+exit;
+
+# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker
+

Show_Submission/query2json

@@ -1 +0,0 @@
-/home/cendjm/HPR/Database/query2json

Show_Submission/query2json

@@ -0,0 +1,134 @@
+#!/usr/bin/env perl
+#===============================================================================
+#
+#         FILE: query2json
+#
+#        USAGE: ./query2json query
+#
+#  DESCRIPTION: Runs a query given as the only argument. Caution is needed
+#               since *any* query will be run. The result of the query is
+#               output in JSON form on STDOUT.
+#
+#      OPTIONS: ---
+# REQUIREMENTS: ---
+#         BUGS: ---
+#        NOTES: Had to revert to MySQL because of a problem with DBD::MariaDB
+#       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
+#      VERSION: 0.0.2
+#      CREATED: 2021-06-18 13:24:49
+#     REVISION: 2023-01-05 16:17:24
+#
+#===============================================================================
+
+use 5.010;
+use strict;
+use warnings;
+use utf8;
+
+use Config::General;
+use JSON;
+use DBI;
+
+use Data::Dumper;
+
+#
+# Version number (manually incremented)
+#
+our $VERSION = '0.0.2';
+
+#
+# Script and directory names
+#
+( my $PROG = $0 ) =~ s|.*/||mx;
+( my $DIR  = $0 ) =~ s|/?[^/]*$||mx;
+$DIR = '.' unless $DIR;
+
+#-------------------------------------------------------------------------------
+# Declarations
+#-------------------------------------------------------------------------------
+#
+# Constants and other declarations
+#
+my $basedir    = "$ENV{HOME}/HPR/Database";
+my $configfile = "$basedir/.hpr_livedb.cfg";
+
+my ( $dbh, $sth1, $aref1 );
+my ( $query, $result, $json );
+
+#
+# Enable Unicode mode
+#
+binmode STDOUT, ":encoding(UTF-8)";
+binmode STDERR, ":encoding(UTF-8)";
+
+#
+# Load database configuration data
+#
+my $conf = Config::General->new(
+    -ConfigFile      => $configfile,
+    -InterPolateVars => 1,
+    -ExtendedAccess  => 1
+);
+my %config = $conf->getall();
+
+#-------------------------------------------------------------------------------
+# Options and arguments
+#-------------------------------------------------------------------------------
+$query = shift;
+die "Usage: $PROG query\n" unless $query;
+
+#-------------------------------------------------------------------------------
+# Connect to the database
+#-------------------------------------------------------------------------------
+my $dbhost = $config{database}->{host} // '127.0.0.1';
+my $dbport = $config{database}->{port} // 3306;
+my $dbname = $config{database}->{name};
+my $dbuser = $config{database}->{user};
+my $dbpwd  = $config{database}->{password};
+#$dbh = DBI->connect( "DBI:MariaDB:host=$dbhost;port=$dbport;database=$dbname",
+#    $dbuser, $dbpwd, { AutoCommit => 1 } )
+#    or die $DBI::errstr;
+
+$dbh = DBI->connect( "dbi:mysql:host=$dbhost;port=$dbport;database=$dbname",
+    $dbuser, $dbpwd, { AutoCommit => 1 } )
+    or die $DBI::errstr;
+
+#
+# Enable client-side UTF8
+#
+$dbh->{mysql_enable_utf8} = 1;
+
+#
+# Set up the query
+#
+$sth1 = $dbh->prepare($query) or die $DBI::errstr;
+if ( $dbh->err ) {
+    warn $dbh->errstr;
+}
+
+#
+# Perform the query
+#
+$sth1->execute;
+if ( $dbh->err ) {
+    warn $dbh->errstr;
+}
+
+#
+# Grab everything as an arrayref of hashrefs
+#
+$result = $sth1->fetchall_arrayref( {} );
+
+#
+# Prepare for JSON, forcing object key sorting (expensive)
+#
+$json = JSON->new->utf8->canonical;
+
+#
+# Encode the Perl structure to JSON
+#
+print $json->encode($result), "\n";
+
+exit;
+
+# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker

hpr_container_email_shownotes.sh

@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+#License: GPL v3
+#see <https://www.gnu.org/licenses/>.
+
+#Name: button_hpr_container.sh
+#Purpose: build/run HPR Container.
+#Version: beta 0.01
+#Author: SGOTI (Some Guy On The Internet)
+#Email: Lyunpaw@gmail.com
+#Date: 2025-04-19
+
+#declaration:
+declare bindir="/usr/bin/"
+declare podman="${bindir}podman"
+declare echo="builtin echo -e"
+declare unprivilegedUser="janitor"
+declare date="${bindir}date"
+declare flags
+declare OPTIND
+declare -A containerBulidProperties
+
+declare currentMonth
+declare nextMonth
+declare currentYear
+
+#start:
+currentMonth=$(${date} +%m)
+currentYear=$(${date} +%Y)
+
+if [[ ${currentmonth} -gt 0 ]] && [[ ${currentmonth} -le 11 ]]; then
+    nextMonth="(($(${date} +%m)+01))" #Incomplete: Can return single-digit integer; must be a double-digit integer.
+else
+    nextMonth="01"
+fi
+
+containerBulidProperties=(
+    "containerFile" "/path/to/Containerfile"
+    "hostMountDir01" "/path/to/project/directory"
+    "hostMountDir02" "/tmp/"
+    "containerMountDir01" "/opt/hpr/"
+    "containerMountDir02" "/tmp/hpr/"
+    "containerImageTag" "hpr_image:5.40.1"
+    "pullNewImage" "podman pull docker.io/library/perl"
+    "recordingDate" "$(${date} -d "${currentYear}/${currentMonth}/01")"
+    "recordingTimeStart" "15:00" #TZ: UTC
+    "recordingTimeEnd" "17:00" #TZ: UTC
+)
+
+function runHPRContainer () {
+    local makeEmail #Incomplete: 
+    makeEmail="./make_email -month=${containerBulidProperties[recordingDate]} -start=15:00 -end=17:00 -out=/tmp/hpr/%semail.txt"
+    local makeShownotes #Incomplete: 
+    makeShownotes="/opt/hpr/src/hpr-tools/Community_News/make_shownotes -from=${containerBulidProperties[recordingDate]} -full=/tmp/hpr/%sfull_shownotes.html -mail -comments"
+
+    ${podman} run \
+        --mount type=bind,src=${containerBulidProperties[hostMountDir02]},dst=${containerBulidProperties[containerMountDir02]},rw=true \
+        --label="Project"="HPR" \
+        --label="Forge"="https://repo.anhonesthost.net/HPR/" \
+        --interactive \
+        --tty \
+        --rm=true \
+        --privileged=false \
+        --pull="never" \
+        --cgroups=enabled \
+        --cgroupns=private \
+        --uts=private \
+        --pid=private \
+        --memory="32m" \
+        --memory-reservation="16m" \
+        --hostname="hpr" \
+        --name="hpr_project" \
+        --user="${unprivilegedUser}" \
+        ${containerBulidProperties[containerImageTag]} bash;
+    return;
+}
+
+function buildNewContainerImage () {
+    if [[ -f "${containerBulidProperties[containerFile]}" ]]; then
+        ${echo} "Building new container image...\nThis may take several minutes.\
+            \nThis is a non interactive build process, so you can return when \
+            it's completed.";
+        ${podman} build --file="${containerBulidProperties[containerFile]}" --tag="${containerBulidProperties[containerImageTag]}";
+    else
+        ${echo} 'Dont forget to assign the ${containerBulidProperties[containerFile]} ;). The Containerfile is needed to build the container image.'
+    fi
+    return;
+}
+
+function help () {
+    ${echo} "$0 [-hbpq]\n\t[-h] help\n\t[-b] build new container\n\t[-p] pull new perl image\n\t[-q] quit";
+    return;
+}
+
+while getopts 'hbpq' flags; do
+    case "${flags}" in
+        h) help; exit 0;
+        ;;
+
+        i) ${echo} "Work in progress... :D"; break;
+        ;;
+
+        b) buildNewContainerImage;
+        ;;
+
+        p) ${containerBulidProperties[pullNewImage]}; exit 0;
+        ;;
+
+        e) ${echo} "Work in progress... :D"; break;
+        ;;
+
+        s) ${echo} "Work in progress... :D"; break;
+        ;;
+
+        q)
+        ${echo} "Quitting script."; break;
+        ;;
+
+        *)
+        ${echo} "Good Heavens! Wrong input."; help; exit 1
+        ;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [[ -z ${1} ]]; then help; runHPRContainer; fi;
+
+unset echo
+unset flags
+unset podman
+unset OPTIND
+unset currentMonth
+unset currentYear
+unset containerBulidProperties
+exit 0
+

workflow/audio2image.bash

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
+#============================================================
+
+#
+# Copied from a 'history' file on archive.org and turned into a script
+#
+
+set -o nounset                              # Treat unset variables as an error
+
+SCRIPT=${0##*/}
+
+#===  FUNCTION  ================================================================
+#          NAME:  cleanup_temp
+#   DESCRIPTION:  Cleanup temporary files in case of a keyboard interrupt
+#                 (SIGINT) or a termination signal (SIGTERM) and at script
+#                 exit
+#    PARAMETERS:  * - names of temporary files to delete
+#       RETURNS:  Nothing
+#===============================================================================
+function cleanup_temp {
+    for tmp in "$@"; do
+        [ -e "$tmp" ] && rm --force "$tmp"
+    done
+    exit 0
+}
+
+#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+if [[ $# -eq 0 ]]; then
+    echo "Usage: $SCRIPT audio_file"
+    exit 1
+fi
+
+AUDIO="$1"
+IMAGE="${AUDIO%.*}.png"
+
+if [[ ! -e $AUDIO ]]; then
+    echo "$SCRIPT: audio file $AUDIO not found"
+    exit 1
+fi
+
+#
+# Make  a temporary file (called /tmp/XXXX.png) and set traps to delete them
+#
+TMP1=$(mktemp -p /tmp XXXXXXXX.png) || {
+    echo "$SCRIPT: creation of temporary file failed!"
+    exit 1
+}
+trap 'cleanup_temp $TMP1' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
+
+#
+# Make a temporary PNG file from the audio. The original uses a 'timeout' call
+# but I can't get it to work for me.
+#
+ffmpeg -v 0 -analyzeduration 900000000000 -probesize 200M -threads 2 -i "$AUDIO" \
+    -filter_complex aformat=channel_layouts=mono,showwavespic=colors=white:s=3200x800 \
+    -frames:v 1 -f apng - 2>/dev/null | \
+    magick - -background black -alpha remove -alpha off "$TMP1" 2>/dev/null | cat > /dev/null 2>&1
+
+#
+# Not clear what's being done to the image here, but it produces a useful end
+# result, or seems to!
+#
+magick -background black "$TMP1" -gravity center -background black -transparent white - | \
+    magick - -gravity South -background white -splice 0x5 -background black -splice 0x5 - | \
+    magick - -trim - | \
+    magick - -gravity South -chop 0x5 - | \
+    magick - -gravity North -background white -splice 0x5 -background black -splice 0x5 - | \
+    magick - -trim - | \
+    magick - -gravity North -chop 0x5 - | \
+    magick - -resize '800x200!' "$IMAGE"
+
+#
+# The original renamed the temporary file to be the target image file, but
+# that's probably becausae part of this is done in a Docker container. Doesn't
+# seem appropriate here
+#
+
+# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
+

workflow/check_assets.bash

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
+#============================================================
+
+assets_db="assets_202410181601.csv"
+total="$( wc -l assets_202410181601.csv | awk '{print $1}' )"
+count=1
+awk '{print $2}' "${assets_db}" | grep -E 'ogg|spx|mp3' | while read this_asset_name
+do
+  this_asset="/mnt/data/HPR/${this_asset_name}"
+  #echo -ne "\r$(\date "+%Y-%m-%d %H:%M:%S") The scanner lid is CLOSED.                        \r"
+  if [ ! -s "${this_asset}" ]
+  then
+    echo "ERROR: \"${this_asset}\" missing."
+  else
+    this_size="$( ls -al "${this_asset}" | awk '{print $5}' )"
+    this_sha1sum="$( sha1sum "${this_asset}" | awk '{print $1}' )"
+    if [ "$( grep -cE "${this_asset_name}.*${this_size}.*${this_sha1sum}" "${assets_db}" )" -ne 1 ]
+    then
+      echo "ERROR: \"${this_asset}\" incorrect properties."
+    fi
+  fi
+  echo -ne "\rProcessing ${count} of ${total}"
+  count=$((count+1))
+done
+
+exit

workflow/duration.bash

@@ -1,20 +1,17 @@
-#!/bin/bash
-# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ ]
+#!/usr/bin/env bash
+# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
+#============================================================
 
+search_dir="${HOME}/processing/"
 
-if [ -f "${1}" ]
+if [ -d "${1}" ]
 then
-  mediainfo --Output=XML --Full "${1}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | sed 's/0$//g'
-  exit 0
+  search_dir="${1}"
 fi
 
-find ./ -type f | grep -E -v '/sponsor-anhonesthost.com-hpr15.flac|/outro.flac|/intro.flac|/sponsor-archive.org.flac' | while read mediafile
-do
-  if [ "$( file "${mediafile}" | grep -c audio )" == "0" ]
-  then
-    continue
-  fi
-  duration=$( mediainfo --full --Output=XML "${mediafile}" | xmlstarlet sel -T -t -m "_:MediaInfo/_:media/_:track[@type='Audio']/_:Duration[1]" -v "." -n - | sed 's/0$//g')
+find ${search_dir} -type f | grep -vP '/sponsor-anhonesthost.com-hpr15.flac|/outro.flac|/intro.flac|/sponsor-archive.org.flac' | while read mediafile
+do 
+  duration=$( mediainfo --full --Output=XML "${mediafile}" | xmlstarlet sel -T -t -m "_:MediaInfo/_:media/_:track[@type='Audio']/_:Duration[1]" -v "." -n - | awk -F '.' '{print $1}' )
   if [ "${duration}" != "" ]
   then
     echo "${mediafile}: ${duration}"
@@ -27,4 +24,3 @@ do
     continue
   fi
 done
-

workflow/hpr-assets.bash

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
+#============================================================
+
+file_dir="/var/IA/uploads"
+
+#$allowed_extensions = "flac", "opus", "ogg", "spx", "mp3", "jpg", "png", "json", "srt", "tsv", "txt", "vtt";
+
+if [ -d "${file_dir}" ]
+then
+  cd "${file_dir}"
+  echo '"episode_id","filename","extension","size", "sha1sum", "mime_type", "file_type"' | tee "${file_dir}/assets.csv"
+  find "${file_dir}" -type f \( -iname "hpr*.wav" -o -iname "hpr*.flac" -o -iname "hpr*.opus" -o -iname "hpr*.ogg" -o -iname "hpr*.spx" -o -iname "hpr*.mp3" -o -iname "hpr*.jpg" -o -iname "hpr*.png" -o -iname "hpr*.json" -o -iname "hpr*.srt" -o -iname "hpr*.tsv" -o -iname "hpr*.txt" -o -iname "hpr*.vtt" \) | while read this_file
+  do
+    this_file_basename=$( basename "${this_file}" )
+    this_file_dirname=$( dirname "${this_file}" )
+    this_file_prefix="${this_file_basename%.*}"
+    this_file_extension="${this_file_basename##*.}"
+    if [ "$( echo "${this_file_prefix}" | grep -Ec '^hpr[0-9]{4}' )" -ne "1" ]
+    then
+      continue
+    fi
+    
+    this_file_episode_id="$( echo "${this_file_prefix}" | sed 's/hpr//g' | awk -F '_' '{print $1}' )"
+    if [ "${this_file_episode_id}" -lt "1" ]
+    then
+      continue
+    fi
+    
+    if [ "${this_file_episode_id}" -gt "99999" ]
+    then
+      continue
+    fi
+
+    this_file_size="$( ls -al "${this_file}" | awk '{print $5}' )"
+    this_file_sha1sum="$( sha1sum "${this_file}" | awk '{print $1}' )"
+    this_file_mime_type=$( file --dereference --brief --mime "${this_file}" )
+    this_file_file_type=$( file --dereference --brief "${this_file}" )
+    
+    echo "${this_file_episode_id},\"${this_file_basename}\",\"${this_file_extension}\",\"${this_file_size}\",\"${this_file_sha1sum}\",\"${this_file_mime_type}\",\"${this_file_file_type}\"" | tee --append "${file_dir}/assets.csv"
+    
+  done
+fi
+
+if [ -s "${file_dir}/assets.csv" ]
+then
+  cat "${file_dir}/assets.csv" | csvtojson | jq '{"assets":[.[]]}' | tee "${file_dir}/assets.json"
+fi
+
+if [ -s "${file_dir}/assets.json" ]
+then
+  curl --netrc-file $HOME/.netrc --verbose --request POST https://hub.hackerpublicradio.org/cms/assets.php --data-ascii @"${file_dir}/assets.json" --header "Content-Type: application/json"
+fi

workflow/hprid

@@ -1,268 +0,0 @@
-#!/bin/bash
-# hprid
-
-################################################################################
-#
-# script to prepare audio files for HPR shows
-#
-# input: mp3 or ogg file
-# result: mp3, ogg in 44100 Hz, spx files 16000Hz with intro and outro
-# provides 3 interactive checks for audio quality, intro and outro
-#
-################################################################################
-# This file is part of the HPR Tool set
-# 
-# HPR Tool set is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# HPR Tool set is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-# 
-# You should have received a copy of the GNU Affero General Public License
-# along with HPR Tool set.  If not, see <http://www.gnu.org/licenses/>.
-# http://www.gnu.org/licenses/agpl-3.0.html
-# #########################################################################
-
-################################################################################
-#
-# PREREQUISITS
-# - current folder has to be writable
-# - there should NOT be a temp.ogg or temp.mp3 file
-# - intro.mp3 and outro.mp3 have to be present
-# - IMPORTANT: sox compiled with mp3 support
-#                see http://a0u.xanga.com/700438974/howto-soc-installation
-#
-#
-# IMPORTANT
-# Backup the files before feeding them to this script, no guarantees here
-# Handling of .wav not yet tested, but it should work 
-#
-# code.cruncher, May 2011
-# 
-################################################################################
-
-################################################################################
-#
-# TODO
-# 
-# test handling of wav files
-# add play final files or open them in specific player(s)
-# add handling of ID3 tags
-# create html interface for standardized info gathering 
-#
-################################################################################
-
-
-#============================================================
-# Check input
-usage="usage: $(basename $0 ) [ -i ] [ -o ] <fname>, -i to add intro and -o outro, fname is a file with audio for HPR"
-
-CHANNELS="1"
-ADDINTRO="n"
-ADDOUTRO="n"
-
-
-while getopts "io" opt; do
-    case $opt in
-        i )
-            ADDINTRO="y"
-            ;;
-        o )
-            ADDOUTRO="y"
-            ;;
-    esac
-done
-shift $(($OPTIND - 1))
-
-# if not ${mediafile} return usage
-if [ $# -lt 1 ]; then
-  echo $usage
-  exit 1
-fi
-
-mediafile=${1}
-
-# test if file exists 
-if [ ! -f "intro.flac" ]; then
-  echo "sorry, file \"intro.flac\" does not exist"
-  echo "To download it run the command:"
-  echo "   wget http://hackerpublicradio.org/media/theme-music/intro.flac"
-  exit 1
-fi
-
-if [ ! -f "outro.flac" ]; then
-  echo "sorry, file \"outro.flac\" does not exist"
-  echo "To download it run the command:"
-  echo "   wget http://hackerpublicradio.org/media/theme-music/outro.flac"
-  exit 1
-fi
-
-for mediafile in "$@"
-do
-    echo $var   
-
-    if [ ! -f "${mediafile}" ]; then
-        echo "sorry, file \"${mediafile}\" does not exist"
-        continue
-    fi
-
-    # test if file exists 
-    if [ ! -r "${mediafile}" ]; then
-        echo "sorry, file \"${mediafile}\" is not readable"
-        continue
-    fi
-
-    if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ]; then
-        echo "sorry, file \"${mediafile}\" has no audio track"
-        continue
-    fi
-
-    # extract file name and extension
-    fname=${mediafile%.*}
-    ext=${mediafile/*./}
-
-    #Make a backup
-    # mediafilebackup=${mediafile}_$(md5sum ${mediafile} | cut -c -32 )_orig.${ext}
-    # cp -v ${mediafile} ${mediafilebackup}
-    # 
-    # if [[ ! -e ${mediafilebackup} ]]; then
-    #   echo "Backup not made: ${mediafilebackup}"
-    #   exit
-    # fi
-
-    # check audio quality
-    dur=7			# playtime of sample in seconds
-    go=1			# variable to repeat playing of sample
-    from=180                # start sample at 3 minutes in
-
-    #============================================================
-    # Question Time
-
-    # # # while [ $go -ne 0 ]
-    # # # do
-    # # #   echo
-    # # #   echo "--------------------------------------------------------------------------------"
-    # # #   echo "1/4 AUDIO TEST: check audio quality: ... playing $dur seconds ..." 
-    # # #   play "${mediafile}" trim $from $dur
-    # # #   ((from+=180))           # next sample will be 3 minutes later
-    # # #   read -s -n1 -p "sound quality ok?[y,n] ... or play another sample[a] ... [y,n,a]"
-    # # #   echo
-    # # #   case "$REPLY" in
-    # # #     n) echo "aborting ... get better quality sound file ... good bye!"; exit 0;;
-    # # #     y) go=0;;
-    # # #   esac
-    # # # done
-    # # # 
-    # # # # Check for intro
-    # # # echo
-    # # # echo "--------------------------------------------------------------------------------"
-    # # # echo "2/4 ADDINTRO TEST: Is the intro playing? "
-    # # # play "${mediafile}" trim 1 5	# play 5 seconds at beginning of file
-    # # # read -s -n1 -p "Is there a intro? [y, n]" -i "n"; echo
-    # # # if [ "$REPLY" = 'y' ]; then
-    # # #   echo "Will add the intro"
-    # # #   ADDINTRO="y"
-    # # # fi
-    # # # 
-    # # # # Check for outro
-    # # # echo 
-    # # # echo "--------------------------------------------------------------------------------"
-    # # # echo "3/4 ADDOUTRO TEST: Is the outro playing? "
-    # # # len=$(eval "soxi -D \"${mediafile}\"")
-    # # # len=$(echo "scale=0; $len - 50" | bc)
-    # # # play "${mediafile}" trim $len 5
-    # # # read -s -n1 -p "Is there a outro ? [y, n]" -i "n"; echo
-    # # # if [ "$REPLY" = 'y' ]; then
-    # # #   echo "Will add the outro"
-    # # #   ADDOUTRO="y"
-    # # # fi
-    # # # 
-    # # # 
-    # # # echo
-    # # # echo "--------------------------------------------------------------------------------"
-    # # # echo "4/4 STEREO TEST: Should this be mono or stereo [m,s] ? "
-    # # # CHANNELS="1"
-    # # # read -s -n1 -p "intro ok? [m, s]" -i "m" ; echo
-    # # # if [ "$REPLY" = 's' ]; then
-    # # #   echo "Will convert to stereo"
-    # # #   CHANNELS="2"
-    # # # fi
-
-    #============================================================
-    # Preprocess the source file
-
-    echo "Convert from ${mediafile} to known wav format ${fname}_tmp.wav"
-    ffmpeg -i ${mediafile} -ar 44100 -ac $CHANNELS ${fname}_tmp.wav > ${fname}_tmp.log 2>&1
-
-    # echo "Normalising the audio"
-    # normalize -a 0.5 ${fname}_tmp.wav >> ${fname}_tmp.log 2>&1
-
-    # TODO Compressor !
-
-    # TODO add a little speed up
-
-    # TODO little overlap in fade in of intro
-
-    # echo "Truncating the silence"
-    # sox ${fname}_tmp.wav ${fname}_sox.wav silence -l 1 0.1 1.6% -1 0.6 1.6% >> ${fname}_tmp.log 2>&1
-    cp -v  ${fname}_tmp.wav ${fname}_sox.wav
-
-    echo "Add the intro if it is missing and add it to the temp pcm file"
-    if [ "$ADDINTRO" = 'y' ]; then
-    ffmpeg -i intro.flac -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - > ${fname}_tmp.pcm 2>> ${fname}_tmp.log 
-    fi
-
-    echo "convert the uploaded episode and add it to the temp pcm file"
-    ffmpeg -i ${fname}_sox.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp.pcm 2>> ${fname}_tmp.log 
-
-    echo "Add the outro if it is missing and add it to the temp pcm file"
-    if [ "$ADDOUTRO" = 'y' ]; then
-    ffmpeg -i outro.flac -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp.pcm 2>> ${fname}_tmp.log 
-    fi
-
-    echo "Convert the pcm file to a know wav format"
-    ffmpeg -f s16le -ar 44100 -ac 1 -acodec pcm_s16le -i ${fname}_tmp.pcm ${fname}_mez.wav 2>> ${fname}_tmp.log 
-
-    # echo "Get an image of the converted audio"
-    # sox ${fname}_mez.wav -n  spectrogram -x 800 -y 100 -o ${fname}_mez.png
-
-    echo "--------------------------------------------------------------------------------"
-    echo "File information"
-    ffprobe ${fname}_mez.wav 2>&1 | grep Audio:
-    mediainfo ${fname}_mez.wav
-
-    echo "--------------------------------------------------------------------------------"
-    # display ${fname}_mez.png &
-    # read -s -n1 -p "Spectrogram check: Everything look ok [y,n] ? " -i "y" ; echo
-    # if [ "$REPLY" = 'n' ]; then
-    #   echo "Something went w  rong. Aborting."
-    #   exit
-    # fi
-
-    # echo "--------------------------------------------------------------------------------"
-    # vlc ${fname}_mez.wav >> ${fname}_tmp.log 2>&1 &
-    # read -s -n1 -p "VLC check: Everything look ok [y,n] ? " -i "y" ; echo
-    # if [ "$REPLY" = 'n' ]; then
-    #   echo "Something went wrong. Aborting."
-    #   exit
-    # fi
-
-    echo "Remove temp files"
-    rm -v ${fname}_tmp.wav ${fname}_sox.wav ${fname}_tmp.pcm ${fname}_tmp.log
-
-    echo "Convert to mp3" # TODO and add tags"
-    sox -S ${fname}_mez.wav ${fname}_mez.mp3 
-
-    echo "Convert to ogg" # TODO and add tags"
-    sox -S ${fname}_mez.wav ${fname}_mez.ogg
-
-    echo "Convert to spx" # TODO and add tags"
-    sox -S ${fname}_mez.wav -c 1 -r 16000 -t wav  - | speexenc - ${fname}_mez.spx
-
-    echo "Changing the file dates to the time of upload"
-    touch -r ${mediafile} ${fname}*
-done

workflow/hprid.sh

@@ -1,268 +0,0 @@
-#!/bin/bash
-# hprid
-
-################################################################################
-#
-# script to prepare audio files for HPR shows
-#
-# input: mp3 or ogg file
-# result: mp3, ogg in 44100 Hz, spx files 16000Hz with intro and outro
-# provides 3 interactive checks for audio quality, intro and outro
-#
-################################################################################
-# This file is part of the HPR Tool set
-# 
-# HPR Tool set is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# HPR Tool set is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-# 
-# You should have received a copy of the GNU Affero General Public License
-# along with HPR Tool set.  If not, see <http://www.gnu.org/licenses/>.
-# http://www.gnu.org/licenses/agpl-3.0.html
-# #########################################################################
-
-################################################################################
-#
-# PREREQUISITS
-# - current folder has to be writable
-# - there should NOT be a temp.ogg or temp.mp3 file
-# - intro.mp3 and outro.mp3 have to be present
-# - IMPORTANT: sox compiled with mp3 support
-#                see http://a0u.xanga.com/700438974/howto-soc-installation
-#
-#
-# IMPORTANT
-# Backup the files before feeding them to this script, no guarantees here
-# Handling of .wav not yet tested, but it should work 
-#
-# code.cruncher, May 2011
-# 
-################################################################################
-
-################################################################################
-#
-# TODO
-# 
-# test handling of wav files
-# add play final files or open them in specific player(s)
-# add handling of ID3 tags
-# create html interface for standardized info gathering 
-#
-################################################################################
-
-
-#============================================================
-# Check input
-usage="usage: $(basename $0 ) [ -i ] [ -o ] <fname>, -i to add intro and -o outro, fname is a file with audio for HPR"
-
-CHANNELS="1"
-ADDINTRO="n"
-ADDOUTRO="n"
-
-
-while getopts "io" opt; do
-    case $opt in
-        i )
-            ADDINTRO="y"
-            ;;
-        o )
-            ADDOUTRO="y"
-            ;;
-    esac
-done
-shift $(($OPTIND - 1))
-
-# if not ${mediafile} return usage
-if [ $# -lt 1 ]; then
-  echo $usage
-  exit 1
-fi
-
-mediafile=${1}
-
-# test if file exists 
-if [ ! -f "intro.flac" ]; then
-  echo "sorry, file \"intro.flac\" does not exist"
-  echo "To download it run the command:"
-  echo "   wget http://hackerpublicradio.org/media/theme-music/intro.flac"
-  exit 1
-fi
-
-if [ ! -f "outro.flac" ]; then
-  echo "sorry, file \"outro.flac\" does not exist"
-  echo "To download it run the command:"
-  echo "   wget http://hackerpublicradio.org/media/theme-music/outro.flac"
-  exit 1
-fi
-
-for mediafile in "$@"
-do
-    echo $var   
-
-    if [ ! -f "${mediafile}" ]; then
-        echo "sorry, file \"${mediafile}\" does not exist"
-        continue
-    fi
-
-    # test if file exists 
-    if [ ! -r "${mediafile}" ]; then
-        echo "sorry, file \"${mediafile}\" is not readable"
-        continue
-    fi
-
-    if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ]; then
-        echo "sorry, file \"${mediafile}\" has no audio track"
-        continue
-    fi
-
-    # extract file name and extension
-    fname=${mediafile%.*}
-    ext=${mediafile/*./}
-
-    #Make a backup
-    # mediafilebackup=${mediafile}_$(md5sum ${mediafile} | cut -c -32 )_orig.${ext}
-    # cp -v ${mediafile} ${mediafilebackup}
-    # 
-    # if [[ ! -e ${mediafilebackup} ]]; then
-    #   echo "Backup not made: ${mediafilebackup}"
-    #   exit
-    # fi
-
-    # check audio quality
-    dur=7			# playtime of sample in seconds
-    go=1			# variable to repeat playing of sample
-    from=180                # start sample at 3 minutes in
-
-    #============================================================
-    # Question Time
-
-    # # # while [ $go -ne 0 ]
-    # # # do
-    # # #   echo
-    # # #   echo "--------------------------------------------------------------------------------"
-    # # #   echo "1/4 AUDIO TEST: check audio quality: ... playing $dur seconds ..." 
-    # # #   play "${mediafile}" trim $from $dur
-    # # #   ((from+=180))           # next sample will be 3 minutes later
-    # # #   read -s -n1 -p "sound quality ok?[y,n] ... or play another sample[a] ... [y,n,a]"
-    # # #   echo
-    # # #   case "$REPLY" in
-    # # #     n) echo "aborting ... get better quality sound file ... good bye!"; exit 0;;
-    # # #     y) go=0;;
-    # # #   esac
-    # # # done
-    # # # 
-    # # # # Check for intro
-    # # # echo
-    # # # echo "--------------------------------------------------------------------------------"
-    # # # echo "2/4 ADDINTRO TEST: Is the intro playing? "
-    # # # play "${mediafile}" trim 1 5	# play 5 seconds at beginning of file
-    # # # read -s -n1 -p "Is there a intro? [y, n]" -i "n"; echo
-    # # # if [ "$REPLY" = 'y' ]; then
-    # # #   echo "Will add the intro"
-    # # #   ADDINTRO="y"
-    # # # fi
-    # # # 
-    # # # # Check for outro
-    # # # echo 
-    # # # echo "--------------------------------------------------------------------------------"
-    # # # echo "3/4 ADDOUTRO TEST: Is the outro playing? "
-    # # # len=$(eval "soxi -D \"${mediafile}\"")
-    # # # len=$(echo "scale=0; $len - 50" | bc)
-    # # # play "${mediafile}" trim $len 5
-    # # # read -s -n1 -p "Is there a outro ? [y, n]" -i "n"; echo
-    # # # if [ "$REPLY" = 'y' ]; then
-    # # #   echo "Will add the outro"
-    # # #   ADDOUTRO="y"
-    # # # fi
-    # # # 
-    # # # 
-    # # # echo
-    # # # echo "--------------------------------------------------------------------------------"
-    # # # echo "4/4 STEREO TEST: Should this be mono or stereo [m,s] ? "
-    # # # CHANNELS="1"
-    # # # read -s -n1 -p "intro ok? [m, s]" -i "m" ; echo
-    # # # if [ "$REPLY" = 's' ]; then
-    # # #   echo "Will convert to stereo"
-    # # #   CHANNELS="2"
-    # # # fi
-
-    #============================================================
-    # Preprocess the source file
-
-    echo "Convert from ${mediafile} to known wav format ${fname}_tmp.wav"
-    ffmpeg -i ${mediafile} -ar 44100 -ac $CHANNELS ${fname}_tmp.wav > ${fname}_tmp.log 2>&1
-
-    # echo "Normalising the audio"
-    # normalize -a 0.5 ${fname}_tmp.wav >> ${fname}_tmp.log 2>&1
-
-    # TODO Compressor !
-
-    # TODO add a little speed up
-
-    # TODO little overlap in fade in of intro
-
-    # echo "Truncating the silence"
-    # sox ${fname}_tmp.wav ${fname}_sox.wav silence -l 1 0.1 1.6% -1 0.6 1.6% >> ${fname}_tmp.log 2>&1
-    cp -v  ${fname}_tmp.wav ${fname}_sox.wav
-
-    echo "Add the intro if it is missing and add it to the temp pcm file"
-    if [ "$ADDINTRO" = 'y' ]; then
-    ffmpeg -i intro.flac -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - > ${fname}_tmp.pcm 2>> ${fname}_tmp.log 
-    fi
-
-    echo "convert the uploaded episode and add it to the temp pcm file"
-    ffmpeg -i ${fname}_sox.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp.pcm 2>> ${fname}_tmp.log 
-
-    echo "Add the outro if it is missing and add it to the temp pcm file"
-    if [ "$ADDOUTRO" = 'y' ]; then
-    ffmpeg -i outro.flac -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp.pcm 2>> ${fname}_tmp.log 
-    fi
-
-    echo "Convert the pcm file to a know wav format"
-    ffmpeg -f s16le -ar 44100 -ac 1 -acodec pcm_s16le -i ${fname}_tmp.pcm ${fname}_mez.wav 2>> ${fname}_tmp.log 
-
-    # echo "Get an image of the converted audio"
-    # sox ${fname}_mez.wav -n  spectrogram -x 800 -y 100 -o ${fname}_mez.png
-
-    echo "--------------------------------------------------------------------------------"
-    echo "File information"
-    ffprobe ${fname}_mez.wav 2>&1 | grep Audio:
-    mediainfo ${fname}_mez.wav
-
-    echo "--------------------------------------------------------------------------------"
-    # display ${fname}_mez.png &
-    # read -s -n1 -p "Spectrogram check: Everything look ok [y,n] ? " -i "y" ; echo
-    # if [ "$REPLY" = 'n' ]; then
-    #   echo "Something went w  rong. Aborting."
-    #   exit
-    # fi
-
-    # echo "--------------------------------------------------------------------------------"
-    # vlc ${fname}_mez.wav >> ${fname}_tmp.log 2>&1 &
-    # read -s -n1 -p "VLC check: Everything look ok [y,n] ? " -i "y" ; echo
-    # if [ "$REPLY" = 'n' ]; then
-    #   echo "Something went wrong. Aborting."
-    #   exit
-    # fi
-
-    echo "Remove temp files"
-    rm -v ${fname}_tmp.wav ${fname}_sox.wav ${fname}_tmp.pcm ${fname}_tmp.log
-
-    echo "Convert to mp3" # TODO and add tags"
-    sox -S ${fname}_mez.wav ${fname}_mez.mp3 
-
-    echo "Convert to ogg" # TODO and add tags"
-    sox -S ${fname}_mez.wav ${fname}_mez.ogg
-
-    echo "Convert to spx" # TODO and add tags"
-    sox -S ${fname}_mez.wav -c 1 -r 16000 -t wav  - | speexenc - ${fname}_mez.spx
-
-    echo "Changing the file dates to the time of upload"
-    touch -r ${mediafile} ${fname}*
-done

workflow/hprtranscode-archive.bash

@@ -1,422 +0,0 @@
-#!/bin/bash
-# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
-#============================================================
-# Check input
-usage="usage: $(basename $0 ) [ -i ] [ -o ] <fname>, -i to add intro and -o outro, fname is a file with audio for HPR"
-
-# Argument = -t test -r server -p password -v
-
-echoerr() 
-{ 
-echo "$@" 1>&2; 
-}
-
-usage()
-{
-cat << EOF
-usage: $0 [options] {media file to encode} {episode number}
-
-OPTIONS:
-   -h      Show this message
-   -p      add a promo
-   -i      add the intro
-   -s      Automatically generate the summary
-   -o      add the outro
-   -b      add the intro and outro ( and summary default)
-   -2      encode to 2 channels
-   -n      no audio normalization
-   -x      eXclude sponsor mention
-   
-EOF
-}
-
-TEMP_DIR="/var/tmp/"
-CHANNELS="1"
-FIXAUDIO="1"
-ADDSUMMARY="n"
-ADDINTRO="n"
-ADDPROMO="n"
-ADDOUTRO="n"
-ADDSPONSOR="y"
-ARTIST="EMPTY"
-TITLE="EMPTY"
-YEAR="EMPTY"
-SLOT="EMPTY"
-basedir="/var/IA"
-intro="${basedir}/intro.flac"
-promo="${basedir}/promo.flac"
-outro="${basedir}/outro.flac"
-anhonesthost="${basedir}/sponsor-anhonesthost.com-hpr15.flac"
-internetarchive="${basedir}/sponsor-archive.org.flac"
-
-while getopts "hipsob2nx" OPTION
-do
-     case $OPTION in
-         h)
-             usage
-             exit 1
-             ;;
-         s)
-             ADDSUMMARY="y"
-             ;;
-         i)
-             ADDINTRO="y"
-             ;;
-         p)
-             ADDPROMO="y"
-             ;;
-         o)
-             ADDOUTRO="y"
-             ;;
-         b)
-             ADDINTRO="y"
-             ADDOUTRO="y"
-             ;;
-         2) 
-             CHANNELS="2"
-             ;;
-         n)
-             FIXAUDIO="0"
-             ;;         
-         x)
-             ADDSPONSOR="n"
-             ;;         
-         ?)
-             usage
-             exit
-             ;;
-     esac
-done
-
-shift $(($OPTIND - 1))
-
-if [ "$#" -ne 2 ]; then
-    echoerr "Please enter the source file and episode number"
-    exit
-fi
-
-mediafile=${1}
-ep_num=${2}
-
-ep_num=$(echo $ep_num | sed 's/hpr//g')
-re='^[0-9]+$'
-if ! [[ $ep_num =~ $re ]] ; then
-   echoerr "error: episode \"${ep_num}\" is not a number"
-   exit 1
-fi
-
-if [ ! -f "${anhonesthost}" ]; then
-  echoerr "sorry, file \"${anhonesthost}\" does not exist"
-  exit 1
-fi
-
-if [ ! -f "${internetarchive}" ]; then
-  echoerr "sorry, file \"${internetarchive}\" does not exist"
-  exit 1
-fi
-
-if [ ! -f "${intro}" ]; then
-  echoerr "sorry, file \"intro.flac\" does not exist"
-  exit 1
-fi
-
-if [ "$ADDPROMO" = 'y' ]; then
-  if [ ! -f "${promo}" ]; then
-    echoerr "sorry, file \"promo.flac\" does not exist"
-    exit 1
-  fi
-fi
-
-if [ ! -f "${outro}" ]; then
-  echoerr "sorry, file \"outro.flac\" does not exist"
-  exit 1
-fi
-
-if [ ! -f "${mediafile}" ]; then
-  echoerr "sorry, media file \"${mediafile}\" does not exist"
-  exit
-fi
-
-if [ ! -r "${mediafile}" ]; then
-  echoerr "sorry, media file \"${mediafile}\" is not readable"
-  exit
-fi
-
-if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ]; then
-  echoerr "sorry, media file \"${mediafile}\" has no audio track"
-  exit
-fi
-
-# extract file name and extension
-media_dir=$(dirname ${mediafile})
-fname=${mediafile%.*}
-ext=${mediafile/*./}
-
-if [ "$ADDSUMMARY" = 'n' ]; then
-  if [ ! -e "${media_dir}/summary.wav" ]
-  then
-    echoerr "ERROR: Can not find the summary file \"${media_dir}/summary.wav\""
-    wget -O- --timeout=10 --tries=1 --quiet http://hackerpublicradio.org/say.php?id=${ep_num} | grep HPR_summary
-    exit 1
-  fi
-fi
-
-if [[ -e hpr${ep_num}.wav ]] || [[ -e hpr${ep_num}.mp3 ]] || [[ -e hpr${ep_num}.ogg ]] || [[ -e hpr${ep_num}.spx ]] || [[ -e hpr${ep_num}_summary.wav ]] || [[ -e ${fname}_mez_norm.wav ]] || [[ -e ${fname}_mez.wav ]] || [[ -e ${fname}_sox_norm.wav ]] || [[ -e ${fname}_sox.wav ]] || [[ -e ${fname}_tmp_hh.pcm ]] || [[ -e ${fname}_tmp_ia.pcm ]] || [[ -e ${fname}_tmp.log ]]
-then
-  echoerr "Files for this episode already exist."
-  ls -1 hpr${ep_num}* ${fname}_* 2>/dev/null
-  exit 1
-fi
-
-echo "--------------------------------------------------------------------------------"
-echo "Geting metadata for hpr${ep_num}"
-
-while read -r line
-do
-  field=$(echo $line | awk -F ':' '{print $1}')
-  case $field in
-  "HPR_summary")
-      HPR_summary=$(echo $line | grep "HPR_summary: " | cut -c 14- )
-      continue
-      ;;
-  "HPR_album")
-     HPR_album=$(echo $line | grep "HPR_album: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_artist")
-    HPR_artist=$(echo $line | grep "HPR_artist: "  | cut -c 13- )
-      continue
-      ;;
-  "HPR_comment")
-    HPR_comment=$(echo $line | grep "HPR_comment: " | cut -c 14- )
-      continue
-      ;;
-  "HPR_genre")
-    HPR_genre=$(echo $line | grep "HPR_genre: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_title")
-    HPR_title=$(echo $line | grep "HPR_title: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_track")
-    HPR_track=$(echo $line | grep "HPR_track: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_year")
-    HPR_year=$(echo $line | grep "HPR_year: " | cut -c 11- )
-      continue
-      ;;
-  "HPR_duration")
-    HPR_duration=$(echo $line | grep "HPR_duration: " | cut -c 15- )
-      continue
-      ;;
-  "HPR_explicit")
-    HPR_explicit=$(echo $line | grep "HPR_explicit: " | cut -c 15- )
-      continue
-      ;;
-  "HPR_license")
-    HPR_license=$(echo $line | grep "HPR_license: " | cut -c 14- )
-      continue
-      ;;
-  esac
-done < <( wget --timeout=10 --tries=1 --quiet http://hackerpublicradio.org/say.php?id=${ep_num} -O - )
-
-if [[ -z "$ADDINTRO" || -z "$ADDPROMO" || -z "$ADDSUMMARY" || -z "$ADDOUTRO" || -z "$HPR_album" || -z "$HPR_artist" || -z "$HPR_comment" || -z "$HPR_genre" || -z "$HPR_title" || -z "$HPR_track" || -z "$HPR_year" || -z "$HPR_summary" || -z "$HPR_duration" || -z "$HPR_explicit" || -z "$HPR_license" ]]
-then
-  echoerr "Could not find information on ${ep_num}. Has the show been posted ?"
-  exit;
-fi
-echo "--------------------------------------------------------------------------------"
-echo "Add intro   : $ADDINTRO"
-echo "Add promo   : $ADDPROMO"
-echo "Add Summary : $ADDSUMMARY"
-echo "Add outro   : $ADDOUTRO"
-echo "album       : $HPR_album"
-echo "artist      : $HPR_artist"
-echo "comment     : $HPR_comment"
-echo "genre       : $HPR_genre"
-echo "title       : $HPR_title"
-echo "track       : $HPR_track"
-echo "year        : $HPR_year"
-echo "summary     : $HPR_summary"
-echo "duration    : $HPR_duration"
-echo "explicit    : $HPR_explicit"
-echo "license     : $HPR_license"
-
-if [[ $HPR_duration == "0" ]]
-then
-  echoerr "The duration is set to 0. Please update the show with the correct time."
-  exit;
-fi
-
-#============================================================
-# Preproc`s the source file
-
-echo "--------------------------------------------------------------------------------"
-echo "Prepare mezzanine file"
-ffmpeg -i ${mediafile} -ar 44100 -ac $CHANNELS ${fname}_sox.wav > ${fname}_tmp.log 2>&1
-
-if [ "$FIXAUDIO" = "1" ];then 
-	echo "normalizing audio"
-	sox --temp "${TEMP_DIR}" --norm ${fname}_sox.wav ${fname}_sox_norm.wav
-	mv -v ${fname}_sox_norm.wav ${fname}_sox.wav >> ${fname}_tmp.log 2>&1
-#	normalize -a 0.5 ${fname}_sox.wav >> ${fname}_tmp.log 2>&1
-fi
-
-echo "--------------------------------------------------------------------------------"
-echo "Add HPR Branding"
-
-if [ "$ADDSUMMARY" = 'y' ]; then
-  echo "Creating the summary"
-  #echo "$HPR_summary" - | espeak -w hpr${ep_num}_summary.wav
-  echo "$HPR_summary" - | text2wave - -o hpr${ep_num}_summary.wav #festival --tts
-  #echo "${HPR_summary}" | gtts-cli - --output hpr${ep_num}_summary.mp3
-  #ffmpeg -i hpr${ep_num}_summary.mp3 hpr${ep_num}_summary.wav
-  #rm -v hpr${ep_num}_summary.mp3
-else
-  echo "Copying the supplied summary"
-  if [ ! -e "${media_dir}/summary.wav" ]
-  then
-    echoerr "ERROR: Can not find the summary file \"${media_dir}/summary.wav\"" 
-    exit 1
-  fi
-  cp -v "${media_dir}/summary.wav" hpr${ep_num}_summary.wav
-fi
-
-ffmpeg -i hpr${ep_num}_summary.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log
-ffmpeg -i hpr${ep_num}_summary.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log
-rm hpr${ep_num}_summary.wav
-
-if [ "$ADDSPONSOR" = 'y' ]; then
-  echo "Adding the sponsor"
-  ffmpeg -i "$anhonesthost" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log
-  ffmpeg -i "$internetarchive" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log
-else
-  echo "NOT Adding the sponsor"
-fi
-
-if [ "$ADDPROMO" = 'y' ]; then
-  echo "Adding the promo"
-  ffmpeg -i "$promo" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log 
-  ffmpeg -i "$promo" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log 
-fi
-
-if [ "$ADDINTRO" = 'y' ]; then
-  echo "Adding the intro"
-  ffmpeg -i "$intro" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log 
-  ffmpeg -i "$intro" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log 
-fi
-
-echo "convert the uploaded episode and add it to the temp pcm file"
-ffmpeg -i ${fname}_sox.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log 
-ffmpeg -i ${fname}_sox.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log 
-
-if [ "$ADDOUTRO" = 'y' ]; then
-  echo "Adding the outro"
-  ffmpeg -i "$outro" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log 
-  ffmpeg -i "$outro" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log 
-fi
-
-echo "Convert the pcm file to a know wav format"
-ffmpeg -f s16le -ar 44100 -ac $CHANNELS -acodec pcm_s16le -i ${fname}_tmp_hh.pcm ${fname}_mez.wav 2>> ${fname}_tmp.log 
-ffmpeg -f s16le -ar 44100 -ac $CHANNELS -acodec pcm_s16le -i ${fname}_tmp_ia.pcm hpr${ep_num}.wav 2>> ${fname}_tmp.log 
-
-echo "Normalizing the wav files"
-sox --temp "${TEMP_DIR}" --norm ${fname}_mez.wav ${fname}_mez_norm.wav
-mv -v ${fname}_mez_norm.wav ${fname}_mez.wav >> ${fname}_tmp.log 2>&1
-# normalize -a 0.5 ${fname}_mez.wav >> ${fname}_tmp.log 2>&1
-sox --temp "${TEMP_DIR}" --norm hpr${ep_num}.wav hpr${ep_num}_norm.wav
-mv -v hpr${ep_num}_norm.wav /var/IA/uploads/hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1
-# normalize -a 0.5 hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1
-
-echo "--------------------------------------------------------------------------------"
-echo "File information"
-ffprobe ${fname}_mez.wav 2>&1 | grep Audio:
-mediainfo ${fname}_mez.wav
-
-echo "--------------------------------------------------------------------------------"
-echo "Convert to opus for IA"
-opusenc hpr${ep_num}.wav /var/IA/uploads/hpr${ep_num}.opus
-
-echo "--------------------------------------------------------------------------------"
-echo "Convert to flac for IA"
-sox --temp "${TEMP_DIR}" -S hpr${ep_num}.wav /var/IA/uploads/hpr${ep_num}.flac
-
-echo "--------------------------------------------------------------------------------"
-echo "Convert to mp3 for HPR"
-sox --temp "${TEMP_DIR}" -S ${fname}_mez.wav hpr${ep_num}.mp3
-echo "Convert to mp3 for IA"
-sox --temp "${TEMP_DIR}" -S hpr${ep_num}.wav /var/IA/uploads/hpr${ep_num}.mp3
-
-echo "--------------------------------------------------------------------------------"
-echo "Convert to ogg for HPR"
-sox --temp "${TEMP_DIR}" -S ${fname}_mez.wav hpr${ep_num}.ogg
-echo "Convert to ogg for IA"
-sox --temp "${TEMP_DIR}" -S hpr${ep_num}.wav /var/IA/uploads/hpr${ep_num}.ogg
-
-echo "--------------------------------------------------------------------------------"
-echo "Convert to spx for HPR"
-sox --temp "${TEMP_DIR}" -S ${fname}_mez.wav -c 1 -r 16000 -t wav  - | speexenc - hpr${ep_num}.spx
-echo "Convert to spx for IA"
-sox --temp "${TEMP_DIR}" -S hpr${ep_num}.wav -c 1 -r 16000 -t wav  - | speexenc - /var/IA/uploads/hpr${ep_num}.spx
-
-
-if [[ ! -s /var/IA/uploads/hpr${ep_num}.wav ]] || [[ ! -s /var/IA/uploads/hpr${ep_num}.mp3 ]] || [[ ! -s /var/IA/uploads/hpr${ep_num}.ogg ]] || [[ ! -s /var/IA/uploads/hpr${ep_num}.spx ]] || [[ ! -s hpr${ep_num}.mp3 ]] || [[ ! -s hpr${ep_num}.ogg ]] || [[ ! -s hpr${ep_num}.spx ]]
-then
-  echoerr "ERROR: Something went wrong encoding the files"
-  exit 1
-fi
-
-fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="${HPR_comment} The license is ${HPR_license}" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2
-fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="$HPR_comment" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" /var/IA/uploads/hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2
-fix_tags hpr${ep_num}*
-fix_tags /var/IA/uploads/hpr${ep_num}*
-
-#echo "Changing the file dates to the time of upload"
-touch -r ${mediafile} hpr${ep_num}*
-touch -r ${mediafile} /var/IA/uploads/hpr${ep_num}*
-
-
-rsync -ave ssh --partial --progress --ignore-existing hpr${ep_num}.mp3 hpr${ep_num}.ogg hpr${ep_num}.spx hpr:www/eps/
-
-firefox http://hackerpublicradio.org/local/hpr${ep_num}.mp3
-firefox http://hackerpublicradio.org/local/hpr${ep_num}.ogg
-firefox file:///var/IA/uploads/hpr${ep_num}.mp3
-firefox file:///var/IA/uploads/hpr${ep_num}.ogg
-
-echo "Source: $( mediainfo --Output=XML --Full "${mediafile}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
-mediainfo --Output=XML --Full hpr${ep_num}* /var/IA/uploads/hpr${ep_num}* | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' | sort | uniq -c
-
-read -p "Remove files for \"${fname}\" (y|N) ? " -n 1 -r
-echo    # (optional) move to a new line
-if [[ $REPLY =~ ^[Yy]$ ]]
-then
-  mediafilename=$(basename "${mediafile}")
-  mediaextension="${mediafilename##*.}"
-
-  ssh hpr -t "mkdir /home/hpr/www/eps/hpr${ep_num}" >/dev/null 2>&1
-  rsync -ave ssh --partial --progress --ignore-existing "${mediafile}" hpr:www/eps/hpr${ep_num}/hpr${ep_num}_source.${mediaextension}
-  ssh hpr -t "ls -al /home/hpr/www/eps/hpr${ep_num}*" 
-  cp -v "${mediafile}" "/var/IA/uploads/hpr${ep_num}_source.${mediaextension}"
-
-  #echo "Remove temp files"
-  rm -v ${fname}_sox.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav
-  mv -v ${fname}* hpr${ep_num}*  *_${ep_num}_* /var/IA/done/
-  wget --timeout=0 -q "http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1" -O - | xmlstarlet val --err -
-  wget --timeout=0 -q "http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1" -O - | xmlstarlet val --err -
-  wget --timeout=0 -q "http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1" -O - | xmlstarlet val --err -
-
-  #rsync -ave ssh --partial --progress /var/IA/uploads/ hpr:/home/hpr/upload/processed/
-  rsync -ave ssh --partial --progress /var/IA/uploads/ borg:/data/IA/uploads/
-  find /var/IA/done/ -empty -delete
-else
-  echo "skipping...."
-  echo "cp -v \"${mediafile}\" \"/var/IA/uploads/hpr${ep_num}_source.${mediaextension}\""
-  echo "rm -v ${fname}_sox.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav"
-  echo "mv -v ${fname}* hpr${ep_num}*  *_${ep_num}_* /var/IA/done/"
-  echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
-  echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
-  echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
-  echo "rsync -ave ssh --partial --progress /var/IA/uploads/ borg:/data/IA/uploads/"
-fi

workflow/hprtranscode-just-transcode.bash

@@ -1,238 +0,0 @@
-#!/bin/bash
-# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
-#============================================================
-
-echoerr() 
-{ 
-echo "$@" 1>&2; 
-}
-
-TEMP_DIR="/var/tmp/"
-CHANNELS="1"
-FIXAUDIO="1"
-ARTIST="EMPTY"
-TITLE="EMPTY"
-YEAR="EMPTY"
-SLOT="EMPTY"
-basedir="/var/IA"
-upload_dir="${basedir}/uploads"
-intro="${basedir}/short.flac"
-
-
-if [ "$#" -ne 2 ]; then
-    echoerr "Please enter the source file and episode number"
-    exit
-fi
-
-mediafile=${1}
-ep_num=${2}
-
-ep_num=$(echo $ep_num | sed 's/hpr//g')
-re='^[0-9]+$'
-if ! [[ $ep_num =~ $re ]] ; then
-   echoerr "error: episode \"${ep_num}\" is not a number"
-   exit 1
-fi
-
-if [ ! -f "${mediafile}" ]; then
-  echoerr "sorry, media file \"${mediafile}\" does not exist"
-  exit
-fi
-
-if [ ! -r "${mediafile}" ]; then
-  echoerr "sorry, media file \"${mediafile}\" is not readable"
-  exit
-fi
-
-if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ]; then
-  echoerr "sorry, media file \"${mediafile}\" has no audio track"
-  exit
-fi
-
-# extract file name and extension
-media_dir=$(dirname ${mediafile})
-fname=${mediafile%.*}
-ext=${mediafile/*./}
-
-if [[ -e hpr${ep_num}.wav ]] || [[ -e hpr${ep_num}.mp3 ]] || [[ -e hpr${ep_num}.ogg ]] || [[ -e hpr${ep_num}.spx ]] || [[ -e hpr${ep_num}_summary.wav ]] || [[ -e hpr${ep_num}.wav ]] || [[ -e ${fname}_mez.wav ]] || [[ -e ${fname}_sox_norm.wav ]] || [[ -e ${fname}_mezzanine.wav ]] || [[ -e ${fname}_tmp.pcm ]] || [[ -e ${fname}_tmp.log ]]
-then
-  echoerr "Files for this episode already exist."
-  ls -1 hpr${ep_num}* ${fname}_* 2>/dev/null
-  exit 1
-fi
-
-echo "--------------------------------------------------------------------------------"
-echo "Geting metadata for hpr${ep_num}"
-
-while read -r line
-do
-  field=$(echo $line | awk -F ':' '{print $1}')
-  case $field in
-  "HPR_summary")
-      HPR_summary=$(echo $line | grep "HPR_summary: " | cut -c 14- )
-      continue
-      ;;
-  "HPR_album")
-     HPR_album=$(echo $line | grep "HPR_album: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_artist")
-    HPR_artist=$(echo $line | grep "HPR_artist: "  | cut -c 13- )
-      continue
-      ;;
-  "HPR_comment")
-    HPR_comment=$(echo $line | grep "HPR_comment: " | cut -c 14- )
-      continue
-      ;;
-  "HPR_genre")
-    HPR_genre=$(echo $line | grep "HPR_genre: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_title")
-    HPR_title=$(echo $line | grep "HPR_title: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_track")
-    HPR_track=$(echo $line | grep "HPR_track: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_year")
-    HPR_year=$(echo $line | grep "HPR_year: " | cut -c 11- )
-      continue
-      ;;
-  "HPR_duration")
-    HPR_duration=$(echo $line | grep "HPR_duration: " | cut -c 15- )
-      continue
-      ;;
-  "HPR_explicit")
-    HPR_explicit=$(echo $line | grep "HPR_explicit: " | cut -c 15- )
-      continue
-      ;;
-  "HPR_license")
-    HPR_license=$(echo $line | grep "HPR_license: " | cut -c 14- )
-      continue
-      ;;
-  esac
-done < <( wget --timeout=10 --tries=1 --quiet http://hackerpublicradio.org/say.php?id=${ep_num} -O - )
-
-if [[ -z "$HPR_album" || -z "$HPR_artist" || -z "$HPR_comment" || -z "$HPR_genre" || -z "$HPR_title" || -z "$HPR_track" || -z "$HPR_year" || -z "$HPR_summary" || -z "$HPR_duration" || -z "$HPR_explicit" || -z "$HPR_license" ]]
-then
-  echoerr "Could not find information on ${ep_num}. Has the show been posted ?"
-  exit;
-fi
-echo "--------------------------------------------------------------------------------"
-echo "album       : $HPR_album"
-echo "artist      : $HPR_artist"
-echo "comment     : $HPR_comment"
-echo "genre       : $HPR_genre"
-echo "title       : $HPR_title"
-echo "track       : $HPR_track"
-echo "year        : $HPR_year"
-echo "summary     : $HPR_summary"
-echo "duration    : $HPR_duration"
-echo "explicit    : $HPR_explicit"
-echo "license     : $HPR_license"
-
-if [[ $HPR_duration == "0" ]]
-then
-  echoerr "The duration is set to 0. Please update the show with the correct time."
-  exit;
-fi
-
-#============================================================
-# Preproc`s the source file
-
-echo "--------------------------------------------------------------------------------"
-echo "Prepare mezzanine file"
-ffmpeg -i ${mediafile} -ar 44100 -ac $CHANNELS hpr${ep_num}.wav > ${fname}_tmp.log 2>&1
-
-echo "Normalizing the wav files"
-sox --temp "${TEMP_DIR}" --norm hpr${ep_num}.wav hpr${ep_num}_norm.wav
-mv -v hpr${ep_num}_norm.wav ${upload_dir}/hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1
-
-echo "--------------------------------------------------------------------------------"
-echo "File information"
-ffprobe ${upload_dir}/hpr${ep_num}.wav 2>&1 | grep Audio:
-mediainfo ${upload_dir}/hpr${ep_num}.wav
-
-echo "--------------------------------------------------------------------------------"
-echo "Convert to opus"
-opusenc ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.opus 2>> ${fname}_tmp.log 1>&2
-
-echo "--------------------------------------------------------------------------------"
-echo "Convert to flac"
-sox --temp "${TEMP_DIR}" -S ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.flac 2>> ${fname}_tmp.log 1>&2
-
-echo "--------------------------------------------------------------------------------"
-echo "Convert to mp3"
-sox --temp "${TEMP_DIR}" -S ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.mp3 2>> ${fname}_tmp.log 1>&2
-
-echo "--------------------------------------------------------------------------------"
-echo "Convert to ogg"
-sox --temp "${TEMP_DIR}" -S ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.ogg 2>> ${fname}_tmp.log 1>&2
-
-echo "--------------------------------------------------------------------------------"
-echo "Convert to spx"
-sox --temp "${TEMP_DIR}" -S ${upload_dir}/hpr${ep_num}.wav -c 1 -r 16000 -t wav  - | speexenc - ${upload_dir}/hpr${ep_num}.spx 2>> ${fname}_tmp.log 1>&2
-
-if [[ ! -s ${upload_dir}/hpr${ep_num}.wav ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.opus ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.flac ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.mp3 ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.ogg ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.spx ]] 
-then
-  echoerr "ERROR: Something went wrong encoding the files"
-  exit 1
-fi
-
-fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="${HPR_comment} The license is ${HPR_license}" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" ${upload_dir}/hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2
-
-fix_tags ${upload_dir}/hpr${ep_num}*
-
-#echo "Changing the file dates to the time of upload"
-touch -r ${mediafile} hpr${ep_num}*
-touch -r ${mediafile} ${upload_dir}/hpr${ep_num}*
-
-ls -al hpr${ep_num}* ${upload_dir}/hpr${ep_num}*
-
-rsync -ave ssh --partial --progress --ignore-existing ${upload_dir}/hpr${ep_num}.mp3 ${upload_dir}/hpr${ep_num}.ogg ${upload_dir}/hpr${ep_num}.spx hpr:www/eps/
-
-firefox http://hackerpublicradio.org/local/hpr${ep_num}.mp3
-firefox http://hackerpublicradio.org/local/hpr${ep_num}.ogg
-firefox file://${upload_dir}/hpr${ep_num}.mp3
-firefox file://${upload_dir}/hpr${ep_num}.ogg
-mpv "http://hackerpublicradio.org/local/hpr${ep_num}.spx" "http://hackerpublicradio.org/local/hpr${ep_num}.ogg" "http://hackerpublicradio.org/local/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.spx" "file://${upload_dir}/hpr${ep_num}.opus" "file://${upload_dir}/hpr${ep_num}.ogg" "file://${upload_dir}/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.flac"
-
-echo "Source: $( mediainfo --Output=XML --Full "${mediafile}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
-mediainfo --Output=XML --Full hpr${ep_num}* ${upload_dir}/hpr${ep_num}* | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' | sort | uniq -c
-
-read -p "Remove files for \"${fname}\" (y|N) ? " -n 1 -r
-echo    # (optional) move to a new line
-if [[ $REPLY =~ ^[Yy]$ ]]
-then
-  mediafilename=$(basename "${mediafile}")
-  mediaextension="${mediafilename##*.}"
-
-  ssh hpr -t "mkdir /home/hpr/www/eps/hpr${ep_num}" >/dev/null 2>&1
-  rsync -ave ssh --partial --progress --ignore-existing "${mediafile}" hpr:www/eps/hpr${ep_num}/hpr${ep_num}_source.${mediaextension}
-  ssh hpr -t "ls -al /home/hpr/www/eps/hpr${ep_num}*" 
-  cp -v "${mediafile}" "${upload_dir}/hpr${ep_num}_source.${mediaextension}"
-
-  #echo "Remove temp files"
-  rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav
-  mv -v ${fname}* hpr${ep_num}*  *_${ep_num}_* /var/IA/done/
-  wget --timeout=0 -q "http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1" -O - | xmlstarlet val --err -
-  wget --timeout=0 -q "http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1" -O - | xmlstarlet val --err -
-  wget --timeout=0 -q "http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1" -O - | xmlstarlet val --err -
-  wget --timeout=0 -q "http://hackerpublicradio.org/rss-future.php" -O - | xmlstarlet val --err -
-
-  #rsync -ave ssh --partial --progress ${upload_dir}/ hpr:/home/hpr/upload/processed/
-  rsync -ave ssh --partial --progress ${upload_dir}/ borg:/data/IA/uploads/
-  echo "$( ssh borg -t "ls -al /data/IA/uploads/hpr${ep_num}*" ; ls -al ${upload_dir}/hpr${ep_num}* )" | grep "hpr${ep_num}" | awk '{print $5, $NF}' | sort
-  find /var/IA/done/ -empty -delete
-else
-  echo "skipping...."
-  echo "cp -v \"${mediafile}\" \"${upload_dir}/hpr${ep_num}_source.${mediaextension}\""
-  echo "rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav"
-  echo "mv -v ${fname}* hpr${ep_num}*  *_${ep_num}_* /var/IA/done/"
-  echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
-  echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
-  echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
-  echo "rsync -ave ssh --partial --progress ${upload_dir}/ borg:/data/IA/uploads/"
-fi

workflow/hprtranscode-simple.bash

@@ -1,603 +0,0 @@
-#!/usr/bin/env bash
-# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
-#============================================================
-TEMP_DIR="/var/tmp/"
-CHANNELS="1"
-FIXAUDIO="1"
-ARTIST="EMPTY"
-TITLE="EMPTY"
-YEAR="EMPTY"
-SLOT="EMPTY"
-basedir="/var/IA"
-upload_dir="${basedir}/uploads"
-theme="${basedir}/theme.wav"
-outro="${basedir}/2022-03-07-outro.wav"
-ttsserver="http://localhost:5500"
-processing_dir="/home/ken/tmp/hpr/processing"
-git_image_dir="/home/ken/sourcecode/hpr/HPR_Public_Code/www/images/hosts"
-acceptable_duration_difference="2" # Seconds
-thedate=$(/usr/bin/date -u +%Y-%m-%d_%H-%M-%SZ_%A)
-echo "Processing the next HPR Show in the queue"
-
-
-if [ $( curl -s -o /dev/null -w "%{http_code}" -X 'GET' "${ttsserver}/api/voices" -H 'accept: */*' ) != "200" ]
-then
-  echo "Please start the tts-server \"podman run -it -p 5500:5500 synesthesiam/opentts:en\""
-  exit
-fi
-
-function abs_diff {
-  echo $(($1 >= $2 ? $1 - $2 : $2 - $1))
-}
-
-function create_tts_summary {
-  HPR_summary="$( cat "hpr${ep_num}_summary.txt" )"
-  echo "INFO: Converting text \"${HPR_summary}\" to speech."
-  curl -X 'GET' -G --data-urlencode "voice=coqui-tts:en_ljspeech" --data-urlencode "text=${HPR_summary}" --data-urlencode "vocoder=high" --data-urlencode "denoiserStrength=0.03" --data-urlencode "cache=false" ${ttsserver}/api/tts -H 'accept: */*' --output ~hpr${ep_num}_summary.wav
-}
-
-###################
-# Locate media directory
-#
-# 
-show_type=""
-if [[ -f "${1}" && -n "${2}" ]]
-then
-  show_type="local"
-  mediafile="${1}"
-  media_dir="$( dirname "${mediafile}" )"
-  ep_num="${2}"
-  echo "The duration is \"$( \date -ud "1970-01-01 $( ffprobe -i "${mediafile}" 2>&1| awk -F ': |, ' '/Duration:/ { print $2 }' )" +%s )\"."
-else
-  show_type="remote"
-  response=$( curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | \
-  grep 'SHOW_POSTED' | \
-  head -1 | \
-  sed 's/,/ /g' )
-
-  if [ -z "${response}" ]
-  then
-    echo "INFO: There appear to be no more shows with the status \"SHOW_POSTED\"."
-    echo "Getting a list of all the reservations."
-    curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php"
-    exit 3
-  fi
-
-  timestamp_epoc="$( echo ${response} | awk '{print $1}' )"
-  ep_num="$( echo ${response} | awk '{print $2}' )"
-  ep_date="$( echo ${response} | awk '{print $3}' )"
-  key="$( echo ${response} | awk '{print $4}' )"
-  status="$( echo ${response} | awk '{print $5}' )"
-  email="$( echo ${response} | awk '{print $6}' )"
-
-  #source_dir="hpr:/home/hpr/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
-  dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
-  media_dir="${processing_dir}/${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
-fi
-
-if [ -z "${show_type}" ]
-then
-  echo "sorry, variable \"show_type\" is not set."
-  exit
-fi
-
-if [ -z "${media_dir}" ]
-then
-  echo "sorry, variable \"media_dir\" is not set."
-  exit
-fi
-
-if [ ! -d "${media_dir}" ]
-then
-  echo "sorry, directory \"media_dir: ${media_dir}\" does not exist"
-  exit 1
-fi
-
-echo detox -v "${media_dir}/"
-detox -vr "${media_dir}/"
-
-if [[ "$( find "${media_dir}" \( -iname "hpr${ep_num}.wav" -o -iname "hpr${ep_num}.mp3" -o -iname "hpr${ep_num}.ogg" -o -iname "hpr${ep_num}.spx" -o -iname "hpr${ep_num}_summary.txt" -o -iname "hpr${ep_num}_summary.wav" -o -iname "hpr${ep_num}.wav" -o -iname "*_mez.wav" -o -iname "*_sox_norm.wav" -o -iname "*_mezzanine.wav" -o -iname "*_tmp.pcm" -o -iname "hpr${ep_num}_intro.wav" -o -iname "~hpr${ep_num}_summary.wav" -o -iname "~~hpr${ep_num}_summary.wav" -o -iname "*_tmp.log" -o -iname "silence.wav" -o -iname "hpr${ep_num}_norm.wav" \) | wc -l )" -ne 0 ]] 
-then
-  echo "Files for this episode already exist."
-  find "${media_dir}" \( -iname "hpr${ep_num}.wav" -o -iname "hpr${ep_num}.mp3" -o -iname "hpr${ep_num}.ogg" -o -iname "hpr${ep_num}.spx" -o -iname "hpr${ep_num}_summary.txt" -o -iname "hpr${ep_num}_summary.wav" -o -iname "hpr${ep_num}.wav" -o -iname "*_mez.wav" -o -iname "*_sox_norm.wav" -o -iname "*_mezzanine.wav" -o -iname "*_tmp.pcm" -o -iname "hpr${ep_num}_intro.wav" -o -iname "~hpr${ep_num}_summary.wav" -o -iname "~~hpr${ep_num}_summary.wav" -o -iname "*_tmp.log" -o -iname "silence.wav" -o -iname "hpr${ep_num}_norm.wav" \)
-  read -p "Shall I remove them ? (N|y) ? " -n 1 -r
-  echo    # (optional) move to a new line
-  if [[ ! $REPLY =~ ^[yY]$ ]]
-  then
-    echo "skipping...."
-    exit
-  else
-    echo "Removing old files ...."
-    find "${media_dir}" \( -iname "hpr${ep_num}.wav" -o -iname "hpr${ep_num}.mp3" -o -iname "hpr${ep_num}.ogg" -o -iname "hpr${ep_num}.spx" -o -iname "hpr${ep_num}_summary.txt" -o -iname "hpr${ep_num}_summary.wav" -o -iname "hpr${ep_num}.wav" -o -iname "*_mez.wav" -o -iname "*_sox_norm.wav" -o -iname "*_mezzanine.wav" -o -iname "*_tmp.pcm" -o -iname "hpr${ep_num}_intro.wav" -o -iname "~hpr${ep_num}_summary.wav" -o -iname "~~hpr${ep_num}_summary.wav" -o -iname "*_tmp.log" -o -iname "silence.wav" -o -iname "hpr${ep_num}_norm.wav" \) -delete -print
-  fi
-fi
-
-
-#####################################################
-# Process media
-
-media=$( find "${media_dir}" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' )
-if [ -z "${media}" ]
-then
-  echo "ERROR: Can't find any media in \"${media_dir}/\""
-  find "${media_dir}/" -type f
-  exit 1
-fi
-
-mediafile=""
-
-echo "Found more than one media file. Please select which one to use ?"
-if [ "$( echo "${media}" | wc -l )" -ne 1 ]
-then
-  select this_media in $( echo "${media}" )
-  do 
-    echo "INFO: You selected \"${this_media}\"."
-    ls -al "${this_media}"
-    mediafile="${this_media}"
-    break
-  done
-else
-  echo "INFO: Selecting media as \"${media}\"."
-  mediafile="${media}"
-fi
-
-# extract file name and extension
-fname="$( basename "${mediafile%.*}" )"
-ext="${mediafile/*./}"
-
-cd "${media_dir}/"
-pwd
-
-echo "INFO: Processing hpr${ep_num} from ${email}"
-echo "INFO: Working directory is \"${media_dir}/\""
-echo ""
-
-
-if [ -z "${mediafile}" ]
-then
-  echo "sorry, variable \"mediafile\" is not set."
-  exit
-fi
-
-if [ -z "${fname}" ]
-then
-  echo "sorry, variable \"fname\" is not set."
-  exit
-fi
-
-if [ -z "${ext}" ]
-then
-  echo "sorry, variable \"ext\" is not set."
-  exit
-fi
-
-if [ ! -f "${mediafile}" ]
-then
-  echo "sorry, media file \"${mediafile}\" does not exist"
-  exit
-fi
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "File information for \"${mediafile}\"" | tee -a ${fname}_tmp.log
-ffprobe ${mediafile} 2>&1 | grep Audio:
-mediainfo ${mediafile}
-audio2image.bash ${mediafile}
-xdg-open ${mediafile%.*}.png >/dev/null 2>&1 &
-unset REPLY
-until [[ $REPLY =~ ^[yYnN]$ ]]
-do
-  read -p "Source Waveform look ok ? (N|y) ? " -n 1 -r
-  echo ""
-done
-
-if [[ ! $REPLY =~ ^[yY]$ ]]
-then
-  echo "skipping.... $REPLY"
-  exit
-fi
-
-
-re='^[0-9]+$'
-if ! [[ $ep_num =~ $re ]] 
-then
-   echo "error: episode \"${ep_num}\" is not a number"
-   exit 1
-fi
-
-if [ ! -f "${outro}" ]
-then
-  echo "sorry, file \"${outro}\" does not exist"
-  exit 1
-fi
-
-if [ ! -f "${theme}" ]
-then
-  echo "sorry, file \"${theme}\" does not exist"
-  exit 1
-fi
-
-if [ ! -r "${mediafile}" ]
-then
-  echo "sorry, media file \"${mediafile}\" is not readable"
-  exit
-fi
-
-if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ]
-then
-  echo "sorry, media file \"${mediafile}\" has no audio track"
-  exit
-fi
-
-xdg-open "https://hackerpublicradio.org/eps/hpr${ep_num}/index.html" >/dev/null 2>&1 &
-
-mpv -vo=null "${mediafile}"
-
-unset REPLY
-until [[ $REPLY =~ ^[yYnN]$ ]]
-do
-  read -p "Is the audio ok (n|Y) ? " -n 1 -r 
-  echo ""
-done
-
-if [[ ! $REPLY =~ ^[yY]$ ]]
-then
-  echo "skipping.... $REPLY"
-  exit
-fi
-
-echo "--------------------------------------------------------------------------------"
-echo "Geting metadata for hpr${ep_num}"
-
-while read -r line
-do
-  field=$(echo $line | awk -F ':' '{print $1}')
-  case $field in
-  "HPR_summary")
-      HPR_summary=$(echo $line | grep "HPR_summary: " | cut -c 14- )
-      continue
-      ;;
-  "HPR_album")
-     HPR_album=$(echo $line | grep "HPR_album: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_artist")
-    HPR_artist=$(echo $line | grep "HPR_artist: "  | cut -c 13- )
-      continue
-      ;;
-  "HPR_comment")
-    HPR_comment=$(echo $line | grep "HPR_comment: " | cut -c 14- )
-      continue
-      ;;
-  "HPR_genre")
-    HPR_genre=$(echo $line | grep "HPR_genre: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_title")
-    HPR_title=$(echo $line | grep "HPR_title: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_track")
-    HPR_track=$(echo $line | grep "HPR_track: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_year")
-    HPR_year=$(echo $line | grep "HPR_year: " | cut -c 11- )
-      continue
-      ;;
-  "HPR_duration")
-    HPR_duration=$(echo $line | grep "HPR_duration: " | cut -c 15- )
-      continue
-      ;;
-  "HPR_explicit")
-    HPR_explicit=$(echo $line | grep "HPR_explicit: " | cut -c 15- )
-      continue
-      ;;
-  "HPR_license")
-    HPR_license=$(echo $line | grep "HPR_license: " | cut -c 14- )
-      continue
-      ;;
-  esac
-done < <( curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num}" )
-
-if [[ -z "$HPR_album" || -z "$HPR_artist" || -z "$HPR_comment" || -z "$HPR_genre" || -z "$HPR_title" || -z "$HPR_track" || -z "$HPR_year" || -z "$HPR_summary" || -z "$HPR_duration" || -z "$HPR_explicit" || -z "$HPR_license" ]]
-then
-  echo "Could not find information on ${ep_num}. Has the show been posted ?"
-  exit;
-fi
-echo "--------------------------------------------------------------------------------"
-echo "album       : $HPR_album"
-echo "artist      : $HPR_artist"
-echo "comment     : $HPR_comment"
-echo "genre       : $HPR_genre"
-echo "title       : $HPR_title"
-echo "track       : $HPR_track"
-echo "year        : $HPR_year"
-echo "summary     : $HPR_summary"
-echo "duration    : $HPR_duration"
-echo "explicit    : $HPR_explicit"
-echo "license     : $HPR_license"
-echo "media_dir   : ${media_dir}"
-echo "mediafile   : ${mediafile}"
-echo "fname       : ${fname}"
-echo "ext         : ${ext}"
-
-if [[ $HPR_duration == "0" ]]
-then
-  echo "The duration is set to 0. Please update the show with the correct time."
-  exit;
-fi
-
-#============================================================
-# Preproc`s the source file
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Prepare mezzanine file" | tee -a ${fname}_tmp.log
-ffmpeg -y -i ${mediafile} -ar 44100 -ac $CHANNELS ${fname}_mezzanine.wav > ${fname}_tmp.log 2>&1
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Add HPR Branding" | tee -a ${fname}_tmp.log
-
-echo "Creating the summary" | tee -a ${fname}_tmp.log
-#echo "$HPR_summary" - | text2wave -F 44100 - -o hpr${ep_num}_summary.wav #festival --tts
-#echo "$HPR_summary" - | espeak -w ~hpr${ep_num}_summary.wav
-
-echo "$HPR_summary" > "hpr${ep_num}_summary.txt"
-
-create_tts_summary_ok="not-ok"
-
-while [ "${create_tts_summary_ok}" != "OK" ]
-do
-  create_tts_summary
-
-  xdg-open "hpr${ep_num}_summary.txt" 2>&1 &
-  mpv --speed=1.8 ~hpr${ep_num}_summary.wav
-
-  read -p "Is the text to speech correct (y|N) ? " -n 1 -r
-  echo    # (optional) move to a new line
-  if [[ $REPLY =~ ^[Yy]$ ]]
-  then
-    create_tts_summary_ok="OK"
-  else
-    echo "WARN: Please correct the text and try again."
-    inotifywait --event modify "hpr${ep_num}_summary.txt"
-  fi
-done
-echo "INFO: TTS complete."
-
-ffmpeg -y -i ~hpr${ep_num}_summary.wav -ar 44100 ~~hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1
-
-sox -V2 -n -r 44100 -c 1 silence.wav trim 0.0 6.0 >> ${fname}_tmp.log 2>&1
-
-sox -V2 silence.wav ~~hpr${ep_num}_summary.wav hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1
-
-echo "Adding the theme" | tee -a ${fname}_tmp.log
-sox -V2 -m "hpr${ep_num}_summary.wav" "${theme}" "hpr${ep_num}_intro.wav" >> ${fname}_tmp.log 2>&1
-
-echo "Creating the sandwitch: \"hpr${ep_num}_intro.wav\" \"${fname}_mezzanine.wav\" \"${outro}\" \"hpr${ep_num}.wav\" " | tee -a ${fname}_tmp.log
-
-sox -V2 "hpr${ep_num}_intro.wav" "${fname}_mezzanine.wav" "${outro}" "hpr${ep_num}.wav" >> ${fname}_tmp.log 2>&1
-
-echo "Normalizing the wav files" | tee -a ${fname}_tmp.log
-#sox --temp "${TEMP_DIR}" --norm hpr${ep_num}.wav hpr${ep_num}_norm.wav
-ffmpeg -y -i hpr${ep_num}.wav -af loudnorm=I=-16:LRA=11:TP=-1.5  hpr${ep_num}_norm.wav  >> ${fname}_tmp.log 2>&1
-
-mv -v hpr${ep_num}_norm.wav ${upload_dir}/hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "File information" | tee -a ${fname}_tmp.log
-ffprobe ${upload_dir}/hpr${ep_num}.wav 2>&1 | grep Audio:
-mediainfo ${upload_dir}/hpr${ep_num}.wav
-audio2image.bash ${upload_dir}/hpr${ep_num}.wav
-xdg-open ${upload_dir}/hpr${ep_num}.png >/dev/null 2>&1 &
-read -p "Processed Waveform look ok ? (N|y) ? " -n 1 -r
-echo    # (optional) move to a new line
-if [[ ! $REPLY =~ ^[yY]$ ]]
-then
-  echo "skipping...."
-  exit
-fi
-rm -v ${upload_dir}/hpr${ep_num}.png
-
-
-echo "--------------------------------------------------------------------------------"
-echo "Geting transcript of hpr${ep_num}"
-
-whisper --model tiny --language en --output_dir "${media_dir}" "${upload_dir}/hpr${ep_num}.wav" | tee -a ${fname}_tmp.log
-
-ls -al "${media_dir}/hpr${ep_num}".*
-xdg-open "${media_dir}/hpr${ep_num}".txt 2>&1 &
-echo mpv --no-audio-display --audio-channels=stereo --speed="3.5" "${media_dir}/hpr${ep_num}".txt
-unset REPLY
-until [[ $REPLY =~ ^[yYnN]$ ]]
-do
-  read -p "Processed transcript look ok ? (N|y) ? " -n 1 -r
-  echo ""
-done
-
-if [[ ! $REPLY =~ ^[yY]$ ]]
-then
-  echo "skipping.... $REPLY"
-  exit
-fi
-
-mkdir "${upload_dir}/hpr${ep_num}" >/dev/null 2>&1
-
-cp -v "${media_dir}/hpr${ep_num}".vtt "${upload_dir}/hpr${ep_num}/hpr${ep_num}.vtt" | tee -a ${fname}_tmp.log
-cp -v "${media_dir}/hpr${ep_num}".srt "${upload_dir}/hpr${ep_num}/hpr${ep_num}.srt" | tee -a ${fname}_tmp.log
-cp -v "${media_dir}/hpr${ep_num}".txt "${upload_dir}/hpr${ep_num}/hpr${ep_num}.txt" | tee -a ${fname}_tmp.log
-cp -v "${media_dir}/hpr${ep_num}".tsv "${upload_dir}/hpr${ep_num}/hpr${ep_num}.tsv" | tee -a ${fname}_tmp.log
-cp -v "${media_dir}/hpr${ep_num}".json "${upload_dir}/hpr${ep_num}/hpr${ep_num}.json" | tee -a ${fname}_tmp.log
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Convert to opus" | tee -a ${fname}_tmp.log
-ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.opus 2>> ${fname}_tmp.log 1>&2
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Convert to flac" | tee -a ${fname}_tmp.log
-ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.flac 2>> ${fname}_tmp.log 1>&2
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Convert to mp3" | tee -a ${fname}_tmp.log
-ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.mp3 2>> ${fname}_tmp.log 1>&2
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Convert to ogg" | tee -a ${fname}_tmp.log
-#ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.ogg 2>> ${fname}_tmp.log 1>&2
-ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav -acodec libopus -f ogg ${upload_dir}/hpr${ep_num}.ogg 2>> ${fname}_tmp.log 1>&2
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Convert to spx" | tee -a ${fname}_tmp.log
-ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.spx 2>> ${fname}_tmp.log 1>&2
-
-### End conversion
-
-intro_duration="$( mediainfo --Output=XML --Full "hpr${ep_num}_intro.wav" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
-outro_duration="$( mediainfo --Output=XML --Full "${outro}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
-source_duration="$( mediainfo --Output=XML --Full "${mediafile}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
-expected_duration=$(( ${intro_duration} + ${HPR_duration} + ${outro_duration} ))
-
-echo "Intro(${intro_duration}) + Show(${HPR_duration}) + Outro(${outro_duration}) = ${expected_duration}"
-
-media_error="0"
-
-for check_file in \
-${upload_dir}/hpr${ep_num}.wav \
-${upload_dir}/hpr${ep_num}.opus \
-${upload_dir}/hpr${ep_num}.flac \
-${upload_dir}/hpr${ep_num}.mp3 \
-${upload_dir}/hpr${ep_num}.spx \
-${upload_dir}/hpr${ep_num}.ogg
-do
-  # ${upload_dir}/hpr${ep_num}.spx
-  echo "INFO: Processing the file \"${check_file}\""
-  if [[ ! -s "${check_file}" ]]
-  then
-    echo "ERROR: Something went wrong encoding of the file \"${check_file}\""
-    exit
-  else
-    mediainfo --Output=XML --Full "${check_file}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}'
-    this_duration=$( mediainfo --Output=XML --Full "${check_file}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )
-    if [[ $(abs_diff "${this_duration}" "${expected_duration}" ) -le "${acceptable_duration_difference}" ]]
-    then
-      echo "INFO: The file \"${check_file}\" duration of ${this_duration} () is close enough to ${expected_duration}"
-    else
-      echo "ERROR: The file \"${check_file}\" actual duration of ${this_duration} is not close enough to posted duration of ${expected_duration}."
-      echo "       Fix or update the posted duration to ${source_duration}."
-      media_error="1"
-    fi
-    #${expected_duration}
-  fi
-done
-echo "Source: ${source_duration}"
-
-if [[ "${media_error}" -eq "1" ]]
-then
-  echo "ERROR: Media is not encoded correctly"
-  exit
-else
-  echo "INFO: Media duration is correct"
-fi
-
-
-if [[ ! -s ${upload_dir}/hpr${ep_num}.wav ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.opus ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.flac ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.mp3 ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.ogg ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.spx ]] 
-then
-  echo "ERROR: Something went wrong encoding the files"
-  exit 1
-fi
-
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Fixing Tags" | tee -a ${fname}_tmp.log
-
-fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="${HPR_comment} The license is ${HPR_license}" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" ${upload_dir}/hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2
-
-fix_tags ${upload_dir}/hpr${ep_num}*
-
-#echo "Changing the file dates to the time of upload"
-touch -r ${mediafile} hpr${ep_num}*
-touch -r ${mediafile} ${upload_dir}/hpr${ep_num}*
-
-ls -al hpr${ep_num}* ${upload_dir}/hpr${ep_num}* | tee -a ${fname}_tmp.log
-
-# # echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-# # echo "Getting info for the asset table" | tee -a ${fname}_tmp.log
-# # 
-# # echo "INSERT INTO assets (episode_id,filename,extension,size,sha1sum,mime_type,file_type) VALUES"
-# # 
-# # for asset_file in \
-# # ${upload_dir}/hpr${ep_num}.wav \
-# # ${upload_dir}/hpr${ep_num}.opus \
-# # ${upload_dir}/hpr${ep_num}.flac \
-# # ${upload_dir}/hpr${ep_num}.mp3 \
-# # ${upload_dir}/hpr${ep_num}.spx \
-# # ${upload_dir}/hpr${ep_num}.ogg
-# # do
-# #   size="$( ls -al "${asset_file}" | awk '{print $5}' )"
-# #   sha1sum="$( sha1sum "${asset_file}" | awk '{print $1}' )"
-# #   mime_type=$( file --dereference --brief --mime "${asset_file}" )
-# #   file_type=$( file --dereference --brief "${asset_file}" )
-# #   echo "(${ep_num},'${filename}','${extension}','${size}','${sha1sum}','${mime_type}','${file_type}'),"
-# # done
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Transferring files to Servers" | tee -a ${fname}_tmp.log
-
-
-#rsync -ave ssh --partial --progress --ignore-existing ${upload_dir}/hpr${ep_num}.mp3 ${upload_dir}/hpr${ep_num}.ogg ${upload_dir}/hpr${ep_num}.spx hpr:www/eps/
-
-#firefox "https://hackerpublicradio.org/local/hpr${ep_num}.mp3" >/dev/null 2>&1 &
-#firefox "https://hackerpublicradio.org/local/hpr${ep_num}.ogg" >/dev/null 2>&1 &
-firefox "file://${upload_dir}/hpr${ep_num}.mp3" >/dev/null 2>&1 &
-firefox "file://${upload_dir}/hpr${ep_num}.ogg" >/dev/null 2>&1 &
-#firefox "https://hackerpublicradio.org/eps.php?id=${ep_num}" >/dev/null 2>&1 &
-
-#mpv "http://hackerpublicradio.org/local/hpr${ep_num}.spx" "http://hackerpublicradio.org/local/hpr${ep_num}.ogg" "http://hackerpublicradio.org/local/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.spx" "file://${upload_dir}/hpr${ep_num}.opus" "file://${upload_dir}/hpr${ep_num}.ogg" "file://${upload_dir}/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.flac"
-mpv "file://${upload_dir}/hpr${ep_num}.spx" "file://${upload_dir}/hpr${ep_num}.opus" "file://${upload_dir}/hpr${ep_num}.ogg" "file://${upload_dir}/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.flac"
-
-read -p "Remove files for \"${fname}\" (y|N) ? " -n 1 -r
-echo    # (optional) move to a new line
-if [[ $REPLY =~ ^[Yy]$ ]]
-then
-  mediafilename=$(basename "${mediafile}")
-  mediaextension="${mediafilename##*.}"                                             # "
-#   ssh hpr -t "mkdir /home/hpr/www/eps/hpr${ep_num}" >/dev/null 2>&1
-#   rsync -ave ssh --partial --progress --ignore-existing "${mediafile}" hpr:www/eps/hpr${ep_num}/hpr${ep_num}_source.${mediaextension} | tee -a ${fname}_tmp.log
-#   
-#   rsync -ave ssh --partial --progress --ignore-existing "${media_dir}/hpr${ep_num}.wav".vtt hpr:www/eps/hpr${ep_num}/hpr${ep_num}.vtt | tee -a ${fname}_tmp.log
-#   rsync -ave ssh --partial --progress --ignore-existing "${media_dir}/hpr${ep_num}.wav".srt hpr:www/eps/hpr${ep_num}/hpr${ep_num}.srt | tee -a ${fname}_tmp.log
-#   rsync -ave ssh --partial --progress --ignore-existing "${media_dir}/hpr${ep_num}.wav".txt hpr:www/eps/hpr${ep_num}/hpr${ep_num}.txt | tee -a ${fname}_tmp.log
-#   
-#   ssh hpr -t "ls -al /home/hpr/www/eps/hpr${ep_num}*" 
-#   cp -v "${mediafile}" "${upload_dir}/hpr${ep_num}_source.${mediaextension}"
-
-  #echo "Remove temp files"
-  rm -v ~hpr${ep_num}_summary.wav ~~hpr${ep_num}_summary.wav silence.wav 
-  rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav
-  #mv -v ${fname}* hpr${ep_num}*  *_${ep_num}_* /var/IA/done/
-  
-#   wget --timeout=0 -q "http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1" -O - | xmlstarlet val --err -
-#   wget --timeout=0 -q "http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1" -O - | xmlstarlet val --err -
-#   wget --timeout=0 -q "http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1" -O - | xmlstarlet val --err -
-#   wget --timeout=0 -q "http://hackerpublicradio.org/rss-future.php" -O - | xmlstarlet val --err -
-
-  echo "INFO: rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* borg:/data/IA/uploads/"
-  rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* borg:/data/IA/uploads/
-  echo "$( ssh borg -t "ls -al /data/IA/uploads/hpr${ep_num}*" ; ls -al ${upload_dir}/hpr${ep_num}* )" | grep "hpr${ep_num}" | awk '{print $5, $NF}' | sort
-  
-  if [ ${show_type} == "remote" ]
-  then
-    echo "INFO: Setting the status"
-    # SHOW_SUBMITTED → METADATA_PROCESSED → SHOW_POSTED → MEDIA_TRANSCODED → UPLOADED_TO_IA → UPLOADED_TO_RSYNC_NET
-    curl --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php?ep_num=${ep_num}&status=MEDIA_TRANSCODED"
-    curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" 
-  fi
-else
-  echo "skipping...."
-  echo "cp -v \"${mediafile}\" \"${upload_dir}/hpr${ep_num}_source.${mediaextension}\""
-  echo "rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav"
-  #echo "mv -v ${fname}* hpr${ep_num}*  *_${ep_num}_* /var/IA/done/"
-  echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
-  echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
-  echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
-  echo "rsync -ave ssh --partial --progress ${upload_dir}/ borg:/data/IA/uploads/"
-fi

workflow/hprtranscode-simple.bash.2022-07-31

@@ -1,397 +0,0 @@
-#!/usr/bin/env bash
-# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
-#============================================================
-TEMP_DIR="/var/tmp/"
-CHANNELS="1"
-FIXAUDIO="1"
-ARTIST="EMPTY"
-TITLE="EMPTY"
-YEAR="EMPTY"
-SLOT="EMPTY"
-basedir="/var/IA"
-upload_dir="${basedir}/uploads"
-theme="${basedir}/theme.wav"
-outro="${basedir}/2022-03-07-outro.wav"
-ttsserver="http://localhost:5500"
-processing_dir="/home/ken/tmp/hpr/processing"
-git_image_dir="/home/ken/sourcecode/hpr/HPR_Public_Code/www/images/hosts"
-thedate=$(/usr/bin/date -u +%Y-%m-%d_%H-%M-%SZ_%A)
-echo "Processing the next HPR Show in the queue"
-
-
-if [ $( curl -s -o /dev/null -w "%{http_code}" -X 'GET' "${ttsserver}/api/voices" -H 'accept: */*' ) != "200" ]
-then
-  echo "Please start the tts-server \"podman run -it -p 5500:5500 synesthesiam/opentts:en\""
-  exit
-fi
-
-###################
-# Get the show
-#
-# 
-if [[ -f "${1}" && -n "${2}" ]]
-then
-  mediafile="${1}"
-  ep_num="${2}"
-  echo "The duration is \"$( \date -ud "1970-01-01 $( ffprobe -i "${mediafile}" 2>&1| awk -F ': |, ' '/Duration:/ { print $2 }' )" +%s )\"."
-else
-  response=$( curl --silent --netrc-file ${HOME}/.netrc "https://hackerpublicradio.org/cms/status.php" | \
-  grep 'SHOW_POSTED' | \
-  head -1 | \
-  sed 's/,/ /g' )
-
-  if [ -z "${response}" ]
-  then
-    echo "INFO: There appear to be no more shows with the status \"SHOW_POSTED\"."
-    echo "Getting a list of all the reservations."
-    curl --silent --netrc-file ${HOME}/.netrc "https://hackerpublicradio.org/cms/status.php"
-    exit 3
-  fi
-
-  timestamp_epoc="$( echo ${response} | awk '{print $1}' )"
-  ep_num="$( echo ${response} | awk '{print $2}' )"
-  ep_date="$( echo ${response} | awk '{print $3}' )"
-  key="$( echo ${response} | awk '{print $4}' )"
-  status="$( echo ${response} | awk '{print $5}' )"
-  email="$( echo ${response} | awk '{print $6}' )"
-
-  #source_dir="hpr:/home/hpr/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
-  dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
-  echo detox -v "${processing_dir}/${dest_dir}/"
-  detox -vr "${processing_dir}/${dest_dir}/"
-  cd "${processing_dir}/${dest_dir}/"
-  pwd
-
-  echo "INFO: Processing hpr${ep_num} from ${email}"
-  echo "INFO: Working directory is \"${processing_dir}/${dest_dir}/\""
-  echo ""
-
-  media=$( find "${processing_dir}/${timestamp_epoc}_${ep_num}_${ep_date}_${key}" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' )
-  if [ -z "${media}" ]
-  then
-    echo "ERROR: Can't find any media in \"${processing_dir}/${dest_dir}/\""
-    find "${processing_dir}/${dest_dir}/" -type f
-    exit 1
-  fi
-
-  mediafile=""
-
-  if [ "$( echo "${media}" | wc -l )" -ne 1 ]
-  then
-    select this_media in $( echo "${media}" )
-    do 
-      echo "INFO: You selected \"${this_media}\"."
-      ls -al "${this_media}"
-      mediafile="${this_media}"
-      break
-    done
-  else
-    echo "INFO: Selecting media as \"${media}\"."
-    mediafile="${media}"
-  fi
-  echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-  echo "File information for \"${mediafile}\"" | tee -a ${fname}_tmp.log
-  ffprobe ${mediafile} 2>&1 | grep Audio:
-  mediainfo ${mediafile}
-  audio2image.bash ${mediafile}
-  xdg-open ${mediafile%.*}.png >/dev/null 2>&1 &
-  read -p "Source Waveform look ok ? (N|y) ? " -n 1 -r
-  echo    # (optional) move to a new line
-  if [[ ! $REPLY =~ ^[yY]$ ]]
-  then
-    echo "skipping...."
-    exit
-  fi
-
-fi
-
-re='^[0-9]+$'
-if ! [[ $ep_num =~ $re ]] 
-then
-   echo "error: episode \"${ep_num}\" is not a number"
-   exit 1
-fi
-
-if [ ! -f "${outro}" ]
-then
-  echo "sorry, file \"${outro}\" does not exist"
-  exit 1
-fi
-
-if [ ! -f "${theme}" ]
-then
-  echo "sorry, file \"${theme}\" does not exist"
-  exit 1
-fi
-
-if [ ! -f "${mediafile}" ]
-then
-  echo "sorry, media file \"${mediafile}\" does not exist"
-  exit
-fi
-
-if [ ! -r "${mediafile}" ]
-then
-  echo "sorry, media file \"${mediafile}\" is not readable"
-  exit
-fi
-
-if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ]
-then
-  echo "sorry, media file \"${mediafile}\" has no audio track"
-  exit
-fi
-
-mpv -vo=null "${mediafile}"
-read -p "Is there any problems with the audio (N|y) ? " -n 1 -r
-echo    # (optional) move to a new line
-if [[ $REPLY =~ ^[yY]$ ]]
-then
-  echo "skipping...."
-  exit 1
-fi
-
-# extract file name and extension
-media_dir=$(dirname ${mediafile})
-fname=${mediafile%.*}
-ext=${mediafile/*./}
-
-if [[ -e hpr${ep_num}.wav ]] || [[ -e hpr${ep_num}.mp3 ]] || [[ -e hpr${ep_num}.ogg ]] || [[ -e hpr${ep_num}.spx ]] || [[ -e hpr${ep_num}_summary.wav ]] || [[ -e hpr${ep_num}.wav ]] || [[ -e ${fname}_mez.wav ]] || [[ -e ${fname}_sox_norm.wav ]] || [[ -e ${fname}_mezzanine.wav ]] || [[ -e ${fname}_tmp.pcm ]] || [[ -e ${fname}_tmp.log ]] || [[ -e hpr${ep_num}_intro.wav ]] || [[ -e ~hpr${ep_num}_summary.wav ]] || [[ -e ~~hpr${ep_num}_summary.wav ]] 
-then
-  echo "Files for this episode already exist."
-  ls -1 *hpr${ep_num}* ${fname}_* 2>/dev/null
-  exit 1
-fi
-
-echo "--------------------------------------------------------------------------------"
-echo "Geting metadata for hpr${ep_num}"
-
-while read -r line
-do
-  field=$(echo $line | awk -F ':' '{print $1}')
-  case $field in
-  "HPR_summary")
-      HPR_summary=$(echo $line | grep "HPR_summary: " | cut -c 14- )
-      continue
-      ;;
-  "HPR_album")
-     HPR_album=$(echo $line | grep "HPR_album: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_artist")
-    HPR_artist=$(echo $line | grep "HPR_artist: "  | cut -c 13- )
-      continue
-      ;;
-  "HPR_comment")
-    HPR_comment=$(echo $line | grep "HPR_comment: " | cut -c 14- )
-      continue
-      ;;
-  "HPR_genre")
-    HPR_genre=$(echo $line | grep "HPR_genre: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_title")
-    HPR_title=$(echo $line | grep "HPR_title: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_track")
-    HPR_track=$(echo $line | grep "HPR_track: " | cut -c 12- )
-      continue
-      ;;
-  "HPR_year")
-    HPR_year=$(echo $line | grep "HPR_year: " | cut -c 11- )
-      continue
-      ;;
-  "HPR_duration")
-    HPR_duration=$(echo $line | grep "HPR_duration: " | cut -c 15- )
-      continue
-      ;;
-  "HPR_explicit")
-    HPR_explicit=$(echo $line | grep "HPR_explicit: " | cut -c 15- )
-      continue
-      ;;
-  "HPR_license")
-    HPR_license=$(echo $line | grep "HPR_license: " | cut -c 14- )
-      continue
-      ;;
-  esac
-done < <( wget --timeout=10 --tries=1 --quiet http://hackerpublicradio.org/say.php?id=${ep_num} -O - )
-
-if [[ -z "$HPR_album" || -z "$HPR_artist" || -z "$HPR_comment" || -z "$HPR_genre" || -z "$HPR_title" || -z "$HPR_track" || -z "$HPR_year" || -z "$HPR_summary" || -z "$HPR_duration" || -z "$HPR_explicit" || -z "$HPR_license" ]]
-then
-  echo "Could not find information on ${ep_num}. Has the show been posted ?"
-  exit;
-fi
-echo "--------------------------------------------------------------------------------"
-echo "album       : $HPR_album"
-echo "artist      : $HPR_artist"
-echo "comment     : $HPR_comment"
-echo "genre       : $HPR_genre"
-echo "title       : $HPR_title"
-echo "track       : $HPR_track"
-echo "year        : $HPR_year"
-echo "summary     : $HPR_summary"
-echo "duration    : $HPR_duration"
-echo "explicit    : $HPR_explicit"
-echo "license     : $HPR_license"
-
-if [[ $HPR_duration == "0" ]]
-then
-  echo "The duration is set to 0. Please update the show with the correct time."
-  exit;
-fi
-
-#============================================================
-# Preproc`s the source file
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Prepare mezzanine file" | tee -a ${fname}_tmp.log
-ffmpeg -i ${mediafile} -ar 44100 -ac $CHANNELS ${fname}_mezzanine.wav > ${fname}_tmp.log 2>&1
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Add HPR Branding" | tee -a ${fname}_tmp.log
-
-echo "Creating the summary" | tee -a ${fname}_tmp.log
-#echo "$HPR_summary" - | text2wave -F 44100 - -o hpr${ep_num}_summary.wav #festival --tts
-#echo "$HPR_summary" - | espeak -w ~hpr${ep_num}_summary.wav
-echo "$HPR_summary" - | curl -X 'GET' -G --data-urlencode "voice=coqui-tts:en_ljspeech" --data-urlencode "text=${HPR_summary}" --data-urlencode "vocoder=high" --data-urlencode "denoiserStrength=0.03" --data-urlencode "cache=false" ${ttsserver}/api/tts -H 'accept: */*' --output ~hpr${ep_num}_summary.wav
-
-ffmpeg -i ~hpr${ep_num}_summary.wav -ar 44100 ~~hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1
-
-sox -n -r 44100 -c 1 silence.wav trim 0.0 6.0 >> ${fname}_tmp.log 2>&1
-
-sox silence.wav ~~hpr${ep_num}_summary.wav hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1
-
-echo "Adding the theme" | tee -a ${fname}_tmp.log
-sox -m "hpr${ep_num}_summary.wav" "${theme}" "hpr${ep_num}_intro.wav" >> ${fname}_tmp.log 2>&1
-
-echo "Creating the sandwitch" | tee -a ${fname}_tmp.log
-
-sox "hpr${ep_num}_intro.wav" "${fname}_mezzanine.wav" "${outro}" "hpr${ep_num}.wav" >> ${fname}_tmp.log 2>&1
-
-echo "Normalizing the wav files" | tee -a ${fname}_tmp.log
-#sox --temp "${TEMP_DIR}" --norm hpr${ep_num}.wav hpr${ep_num}_norm.wav
-ffmpeg -i hpr${ep_num}.wav -af loudnorm=I=-16:LRA=11:TP=-1.5  hpr${ep_num}_norm.wav  >> ${fname}_tmp.log 2>&1
-
-mv -v hpr${ep_num}_norm.wav ${upload_dir}/hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "File information" | tee -a ${fname}_tmp.log
-ffprobe ${upload_dir}/hpr${ep_num}.wav 2>&1 | grep Audio:
-mediainfo ${upload_dir}/hpr${ep_num}.wav
-audio2image.bash ${upload_dir}/hpr${ep_num}.wav
-xdg-open ${upload_dir}/hpr${ep_num}.png >/dev/null 2>&1 &
-read -p "Processed Waveform look ok ? (N|y) ? " -n 1 -r
-echo    # (optional) move to a new line
-if [[ ! $REPLY =~ ^[yY]$ ]]
-then
-  echo "skipping...."
-  exit
-fi
-rm -v ${upload_dir}/hpr${ep_num}.png
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Convert to opus" | tee -a ${fname}_tmp.log
-ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.opus 2>> ${fname}_tmp.log 1>&2
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Convert to flac" | tee -a ${fname}_tmp.log
-sox --temp "${TEMP_DIR}" -S ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.flac 2>> ${fname}_tmp.log 1>&2
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Convert to mp3" | tee -a ${fname}_tmp.log
-sox --temp "${TEMP_DIR}" -S ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.mp3 2>> ${fname}_tmp.log 1>&2
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Convert to ogg" | tee -a ${fname}_tmp.log
-ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.ogg 2>> ${fname}_tmp.log 1>&2
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Convert to spx" | tee -a ${fname}_tmp.log
-ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.spx 2>> ${fname}_tmp.log 1>&2
-
-for check_file in \
-${upload_dir}/hpr${ep_num}.wav \
-${upload_dir}/hpr${ep_num}.opus \
-${upload_dir}/hpr${ep_num}.flac \
-${upload_dir}/hpr${ep_num}.mp3 \
-${upload_dir}/hpr${ep_num}.ogg \
-${upload_dir}/hpr${ep_num}.spx
-do
-  if [[ ! -s "${check_file}" ]]
-  then
-    echo "ERROR: Something went wrong encoding of the file \"${check_file}\""
-  fi
-done
-
-if [[ ! -s ${upload_dir}/hpr${ep_num}.wav ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.opus ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.flac ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.mp3 ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.ogg ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.spx ]] 
-then
-  echo "ERROR: Something went wrong encoding the files"
-  exit 1
-fi
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Fixing Tags" | tee -a ${fname}_tmp.log
-
-fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="${HPR_comment} The license is ${HPR_license}" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" ${upload_dir}/hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2
-
-fix_tags ${upload_dir}/hpr${ep_num}*
-
-#echo "Changing the file dates to the time of upload"
-touch -r ${mediafile} hpr${ep_num}*
-touch -r ${mediafile} ${upload_dir}/hpr${ep_num}*
-
-ls -al hpr${ep_num}* ${upload_dir}/hpr${ep_num}* | tee -a ${fname}_tmp.log
-
-
-echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
-echo "Transferring files to Servers" | tee -a ${fname}_tmp.log
-
-
-rsync -ave ssh --partial --progress --ignore-existing ${upload_dir}/hpr${ep_num}.mp3 ${upload_dir}/hpr${ep_num}.ogg ${upload_dir}/hpr${ep_num}.spx hpr:www/eps/
-
-firefox "https://hackerpublicradio.org/local/hpr${ep_num}.mp3" >/dev/null 2>&1 &
-firefox "https://hackerpublicradio.org/local/hpr${ep_num}.ogg" >/dev/null 2>&1 &
-firefox "file://${upload_dir}/hpr${ep_num}.mp3" >/dev/null 2>&1 &
-firefox "file://${upload_dir}/hpr${ep_num}.ogg" >/dev/null 2>&1 &
-firefox "https://hackerpublicradio.org/eps.php?id=${ep_num}" >/dev/null 2>&1 &
-
-mpv "http://hackerpublicradio.org/local/hpr${ep_num}.spx" "http://hackerpublicradio.org/local/hpr${ep_num}.ogg" "http://hackerpublicradio.org/local/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.spx" "file://${upload_dir}/hpr${ep_num}.opus" "file://${upload_dir}/hpr${ep_num}.ogg" "file://${upload_dir}/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.flac"
-
-echo "Source: $( mediainfo --Output=XML --Full "${mediafile}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
-mediainfo --Output=XML --Full hpr${ep_num}* ${upload_dir}/hpr${ep_num}* | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' | sort | uniq -c
-
-read -p "Remove files for \"${fname}\" (y|N) ? " -n 1 -r
-echo    # (optional) move to a new line
-if [[ $REPLY =~ ^[Yy]$ ]]
-then
-  mediafilename=$(basename "${mediafile}")
-  mediaextension="${mediafilename##*.}"                                             # "
-  ssh hpr -t "mkdir /home/hpr/www/eps/hpr${ep_num}" >/dev/null 2>&1
-  rsync -ave ssh --partial --progress --ignore-existing "${mediafile}" hpr:www/eps/hpr${ep_num}/hpr${ep_num}_source.${mediaextension}
-  ssh hpr -t "ls -al /home/hpr/www/eps/hpr${ep_num}*" 
-  cp -v "${mediafile}" "${upload_dir}/hpr${ep_num}_source.${mediaextension}"
-
-  #echo "Remove temp files"
-  rm -v ~hpr${ep_num}_summary.wav ~~hpr${ep_num}_summary.wav silence.wav 
-  rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav
-  #mv -v ${fname}* hpr${ep_num}*  *_${ep_num}_* /var/IA/done/
-  wget --timeout=0 -q "http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1" -O - | xmlstarlet val --err -
-  wget --timeout=0 -q "http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1" -O - | xmlstarlet val --err -
-  wget --timeout=0 -q "http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1" -O - | xmlstarlet val --err -
-  wget --timeout=0 -q "http://hackerpublicradio.org/rss-future.php" -O - | xmlstarlet val --err -
-
-  rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* borg:/data/IA/uploads/
-  echo "$( ssh borg -t "ls -al /data/IA/uploads/hpr${ep_num}*" ; ls -al ${upload_dir}/hpr${ep_num}* )" | grep "hpr${ep_num}" | awk '{print $5, $NF}' | sort
-  # SHOW_SUBMITTED → METADATA_PROCESSED → SHOW_POSTED → MEDIA_TRANSCODED → UPLOADED_TO_IA → UPLOADED_TO_RSYNC_NET
-  curl --netrc-file ${HOME}/.netrc "https://hackerpublicradio.org/cms/status.php?ep_num=${ep_num}&status=MEDIA_TRANSCODED"
-else
-  echo "skipping...."
-  echo "cp -v \"${mediafile}\" \"${upload_dir}/hpr${ep_num}_source.${mediaextension}\""
-  echo "rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav"
-  #echo "mv -v ${fname}* hpr${ep_num}*  *_${ep_num}_* /var/IA/done/"
-  echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
-  echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
-  echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
-  echo "rsync -ave ssh --partial --progress ${upload_dir}/ borg:/data/IA/uploads/"
-fi

workflow/intro.srt

@@ -0,0 +1,20 @@
+1
+00:00:00,000 --> 00:00:13,200
+REPLACE_LINE_1
+
+2
+00:00:13,200 --> 00:00:16,400
+REPLACE_LINE_2
+
+3
+00:00:16,400 --> 00:00:20,040
+REPLACE_LINE_3
+
+4
+00:00:20,040 --> 00:00:22,080
+REPLACE_LINE_4
+
+5
+00:00:22,080 --> 00:00:27,320
+REPLACE_LINE_5
+

workflow/outro.srt

@@ -0,0 +1,31 @@
+1
+00:00:00,000 --> 00:00:07,720
+You have been listening to Hacker Public Radio at Hacker Public Radio.org.
+
+2
+00:00:07,720 --> 00:00:11,520
+Today's show was contributed by a HPR listener like yourself.
+
+3
+00:00:11,520 --> 00:00:18,000
+If you ever thought of recording podcast, click on our upload link
+
+4
+00:00:18,000 --> 00:00:19,000
+ to find out how easy it is.
+
+5
+00:00:19,000 --> 00:00:26,480
+Hosting for HPR has been kindly provided by an AnHonestHost.com, the Internet Archive,
+
+6
+00:00:26,480 --> 00:00:27,480
+rsync.net, and our mirror network.
+
+7
+00:00:27,480 --> 00:00:33,480
+Unless otherwise stated, today's show is released under a Creative Commons
+
+8
+00:00:33,480 --> 00:00:36,480
+Attribution-ShareAlike 4.0 International (CC BY-SA 4.0) license.

workflow/remove-image.pl

@@ -0,0 +1,12 @@
+#!/usr/bin/env perl
+
+use strict;
+
+while (<>) {
+    s/(<img.*src.*data:image\/[^;]*;base64,[a-zA-Z0-9+\/=]*)/<img src="LOCAL_IMAGE_REMOVED/g;
+    s/(<img.*src.*http.*>)/<img src="REMOTE_IMAGE_REMOVED" \/>/g;
+    print;
+}
+
+exit
+# <img src="data:image/jpeg;base64,/9j/4QnuRXhpZgAATU

workflow/rename-reserve.bash

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
+#============================================================
+
+upload_dir="/home/hpr/upload"
+reserve_dir="/home/hpr/reserve"
+hub_dir="/home/hpr/hub"
+reserve_file="${hub_dir}/reserve.txt"
+
+while read reserve_show_dir
+do
+  echo "Processing \"${reserve_show_dir}\""
+  shownotes="${reserve_show_dir}/shownotes.json"
+  if [ ! -s "${shownotes}" ]
+  then
+    echo "ERROR: \"${shownotes}\" not found"
+    exit 1
+  fi
+  host_id="$( jq --raw-output '.host.Host_ID' "${shownotes}" )"
+  if [ "${host_id}" -eq "0" ]
+  then
+    echo "ERROR: New host detected \"${host_id}\" post the first show to the regular queue."
+    exit 1
+  fi
+  
+  metadata_url="$( jq --raw-output '.metadata.url' "${shownotes}" )"
+  if [ -n "${metadata_url}" ]
+  then
+    save_name=$( basename "${metadata_url}" | sed -e 's/[^A-Za-z0-9.]/_/g' -e 's/__/_/g' )
+    wget "${metadata_url}" -O "${reserve_show_dir}/${save_name}"
+    if [ ! -s "${reserve_show_dir}/${save_name}" ]
+    then
+      echo "ERROR: \"${metadata_url}\" needs to be downloaded as \"${reserve_show_dir}/${save_name}\""
+      exit 1
+    fi    
+  fi
+  Host_ID="$( jq --raw-output '.host.Host_ID' "${shownotes}" )"
+  Host_Name="$( jq --raw-output '.host.Host_Name' "${shownotes}" | sed -e 's/[^A-Za-z0-9]/_/g' -e 's/__/_/g' )"
+  Key="$( jq --raw-output '.metadata.Key' "${shownotes}" )"
+  Timestamp="$( jq --raw-output '.metadata.Timestamp' "${shownotes}" )"
+  Title="$( jq --raw-output '.episode.Title' "${shownotes}" | sed -e 's/[^A-Za-z0-9]/_/g' -e 's/__/_/g')"
+  Timestamp_Epoch="$( \date -u +%s -d "${Timestamp}" )"
+  mv -v "${reserve_show_dir}" "${reserve_dir}/${Timestamp_Epoch}_${Host_ID}_${Key}_${Host_Name}_${Title}"
+done  < <( find "${upload_dir}" -type d -iname "*_9999_*" )
+
+detox -r -v "${reserve_dir}"
+
+ls -1 "${reserve_dir}" | awk -F '_' '{ $2=""; $3=""; print}' | while read line
+do
+  upload_epoch="$( echo "${line}" | awk '{print $1}' )"
+  upload_iso8601="$( \date -d "@${upload_epoch}" +%Y-%m-%d )"
+  echo "${upload_iso8601} $( echo ${line} | awk '{ $1=""; print}' )"
+done > "${reserve_file}"
+nl "${reserve_file}"

workflow/show2youtube.bash

@@ -1,60 +0,0 @@
-#!/bin/bash
-# http://eddmann.com/posts/uploading-podcast-audio-to-youtube/
-# http://cutycapt.sourceforge.net/ xvfb-run
-# https://el-tramo.be/blog/ken-burns-ffmpeg/
-
-#hpr${ep_num}.wav
-
-ep_num=2463
-
-xvfb-run --server-args="-screen 0, 1024x768x24" CutyCapt --url="http://hackerpublicradio.org/eps.php?id=${ep_num}" --out=${ep_num}.png --insecure 
-
-ffmpeg -i input.wav -filter_complex "[0:a]showwaves=s=800x600:mode=line:rate=25,format=yuv420p[v]" -map "[v]" -map 0:a output-showwaves.mp4
-
-
-# ffmpeg -start_number n -i test_%d.jpg -vcodec mpeg4 test.avi
-# ffmpeg -loop 1 -r 2 -i image.jpg -i input.mp3 -vf scale=-1:380 -c:v libx264 -preset slow -tune stillimage -crf 18 -c:a copy -shortest -pix_fmt yuv420p -threads 0 output.mkv
-
-# https://video.stackexchange.com/questions/9644/how-do-i-turn-audio-into-video-that-is-show-the-waveforms-in-a-video
-
-#this
-# 
-# rm out.mp4 output.mp4 ; ffmpeg -i 2463.png -filter_complex "pad=w=9600:h=6000:x='(ow-iw)/2':y='(oh-ih)/2',zoompan=x='(iw-0.625*ih)/2':y='(1-on/(25*4))*(ih-ih/zoom)':z='if(eq(on,1),2.56,zoom+0.002)':d=25*4:s=1280x800" -pix_fmt yuv420p -c:v libx264 out.mp4
-# 
-# rm out.mp4 output.mp4; ffmpeg -i 2463.png -filter_complex "pad=w=9600:h=6000:x='(ow-iw)/2':y='(oh-ih)/2',zoompan=z='zoom+0':d=25*4:s=1280x2048,crop=w=1280:h=800:x='(iw-ow)/2':y='(ih-oh)/2' " -pix_fmt yuv420p -c:v libx264 out.mp4
-# 
-# ffmpeg -i in.jpg 
-#   -filter_complex 
-#     "zoompan=z='zoom+0.002':d=25*4:s=1280x800" 
-#   -pix_fmt yuv420p -c:v libx264 out.mp4
-#   
-#   
-#   
-# ffmpeg -i input.flac -filter_complex "[0:a]ahistogram,format=yuv420p[v]" -map "[v]" -map 0:a output.mp4
-# 
-# 
-# ffmpeg -i input -i background.png -filter_complex "[0:a]showwavespic=s=640x240[fg];[1:v][top]overlay=format=auto" -frames:v 1 output.png
-
-
-ffmpeg -loop 1 -i background.png -i video1.mp4 -i video2.mp4 -filter_complex \
-"[1:v]scale=(iw/2)-20:-1[a]; \
- [2:v]scale=(iw/2)-20:-1[b]; \
- [0:v][a]overlay=10:(main_h/2)-(overlay_h/2):shortest=1[c]; \
- [c][b]overlay=main_w-overlay_w-10:(main_h/2)-(overlay_h/2)[video]" \
--map "[video]" output.mkv
-
-
-https://stackoverflow.com/questions/13390714/superimposing-two-videos-onto-a-static-image
-https://video.stackexchange.com/questions/14519/add-image-under-the-video-with-ffmepg
-
-
-ffmpeg \
-    -loop 1 -i hprback.png \
-    -i output-showwaves-320x240.mp4 \
-    -filter_complex "overlay=0:0:shortest=1" \
-    out.m4v
-    
-    
-# ffmpeg -i input.wav -filter_complex "[0:a]showfreqs=s=320x266:mode=line:fscale=log,format=yuv420p[v]" -map "[v]" -map 0:a output-showfreqs-319x266.mp4
-# ffmpeg -loop 1 -i hprback.png -i output-showfreqs-319x266.mp4 -filter_complex "overlay=663:143:shortest=1"  out1.m4v
-

workflow/showstatus.bash

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
+#============================================================
+
+curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | sort -t ',' -k 5 -r

workflow/status_lighttower.py

@@ -1,233 +0,0 @@
-#!/usr/bin/env python3
-# 
-# Copyright (C) 2017 Ken Fallon ken@fallon.ie
-# 
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-# 
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# UTF-8 Test >> ÇirçösÚáóíéőöÓÁśł <<
-# 213.46.252.136 gateway
-
-import sys, getopt, json, os, logging, sys, requests, datetime, re, dateutil.parser, base64, xmltodict, random, psycopg2, time, automationhat
-
-from pprint import pprint
-#import traceback
-logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
-#logging.disable(logging.DEBUG)
-settings = dict()
-
-def usage(exitcode, message = None):
-    '''
-    prints usage and exits
-    '''
-    if message is not None:
-        logging.error( message )
-    print( os.path.basename(sys.argv[0]) + ' [--help -l <lab> -c <config> ]')
-    sys.exit(exitcode)
-    
-def get_lab_settings(settings):
-    '''
-    Get configuration for the lab, usually from     http://172.30.218.244/autotest/settings.json
-    '''
-    if os.path.isfile( settings[ 'config_url' ] ):
-        with open( settings[ 'config_url' ] ) as json_file:
-            info_dict = json.load(json_file)
-            return info_dict
-    else:
-        session = requests.session()
-        resp = session.get(url=settings[ 'config_url' ], headers={'Accept-Encoding': 'gzip'})
-        if resp.status_code == 200:
-            info_dict = json.loads(resp.text)
-            return info_dict
-    return None
-    
-def argumentTest():
-    '''
-    Gets the required paramaters that the program needs in order to run. These can either be provided on the command line, or as environemental variables, or both.
-
-    export LAB="lab4b"
-    export config_url="http://172.30.218.244/autotest/settings.json"
-    export service="purcha#!/usr/bin/env python
-
-import automationhat
-import time
-import os
-import sys
-
-counter = -1
-lightPattern = [
-    [1,0,0],
-    [0,1,0],
-    [0,0,1],
-    [0,1,1],
-    [0,1,0],
-    [1,1,0],
-    [1,0,0],
-    [1,1,0],
-    [0,1,0],
-    [0,1,1],
-    [0,0,1],
-    [0,0,0],
-    [1,0,1],
-    [0,1,0],
-    [0,0,0],
-]
-
-def increment():
-    global counter
-    counter+=1
-    if counter==len(lightPattern):
-        counter = 0
-
-def doCurrentCycle():
-    currentCycle = lightPattern[counter]
-    print currentCycle
-    if (currentCycle[0]==1) :
-        automationhat.output.one.on()
-    else:
-        automationhat.output.one.off()
-
-    if (currentCycle[1] == 1):
-        automationhat.output.two.on()
-    else:
-        automationhat.output.two.off()
-
-    if (currentCycle[2] == 1):
-        automationhat.output.three.on()
-    else:
-        automationhat.output.three.off()
-
-    time.sleep(2)
-
-
-
-while True:
-    increment()
-    doCurrentCycle()se-service"
-    export customers="customers.json"
-
-    '''
-    lab = str()
-    config_url = str()
-    service = str()
-    settings = dict()
-    
-    for evariable in [ 'LAB', 'config_url' ]:
-        if os.getenv( evariable ) is not None:
-            settings[ evariable ] = os.environ[ evariable ]
-            logging.debug( "Setting \"" + evariable + "\" set to \"" + settings[ evariable ] + "\" from an enviroment variable.")
-
-    try:
-        options, remainder = getopt.getopt(sys.argv[1:], 'hl:c:', [ 'help', 'lab=', 'config=' ])
-
-    except getopt.GetoptError:
-        usage(1, "Unrecognized option was found or missing argument." )
-
-    for opt, arg in options:
-        if opt in ('-h', '--help'):
-            usage(2)
-        elif opt in ('-l', '--lab'):
-            logging.debug( "Setting \"LAB\" set to \"" + arg + "\" from the \"" + opt + "\" command line argument.")
-            settings[ 'LAB' ] = arg
-        elif opt in ('-c', '--config'):
-            logging.debug( "Setting \"config_url\" set to \"" + arg + "\" from the \"" + opt + "\" command line argument.")
-            settings[ 'config_url' ] = arg
-
-    for param in [ 'LAB', 'config_url' ]:
-        if not param in settings:
-            usage( 3, "Cant find value for \"" + param +"\"")
-
-    try:
-        settings[ 'config' ] = get_lab_settings( settings )[ settings[ 'LAB' ] ]
-        if 'jenkins' in settings.keys():
-            logging.debug('Updating TraxIS Customer file to include the Purchase Service status.')
-
-    except Exception as e:
-        print('Error: Could not get Lab Configuration for "%s".' % settings[ 'LAB' ] )
-        traceback.print_exc()   
-        exit(1)
-
-    return settings
-
-
-#def get_server_instance():
-    #jenkins_url = 'http://172.22.137.160:8080'
-    #server = Jenkins(jenkins_url, username='autotest', password='autotest')
-    #return server
-
-def get_server_instance(settings):
-    jenkins_url = 'http://%s:%s' % ( settings[ 'config' ][ 'jenkins' ][ 'host' ] , settings[ 'config' ][ 'jenkins' ][ 'port' ] )
-    server = Jenkins(jenkins_url, username = settings[ 'config' ][ 'jenkins' ][ 'user' ], password = settings[ 'config' ][ 'jenkins' ][ 'password' ])
-    return server
-
-def alloff():
-    automationhat.relay.one.off()
-    automationhat.output.one.off()
-    automationhat.output.two.off()
-    automationhat.output.three.off()
-
-# Set according to status
-def setstatus(status):
-    if status != "":
-        alloff()
-        if status == "SUCCESS":
-            automationhat.output.three.on()
-            print( "SUCCESS" )
-
-        if status == "UNSTABLE":
-            automationhat.output.two.on()
-            print( "UNSTABLE" )
-
-        if status == "FAILURE":
-            automationhat.output.one.on()
-            print( "FAILURE" )
-    return
-
-def get_job_details( settings ):
-    # Refer Example #1 for definition of function 'get_server_instance'
-    server = get_server_instance( settings )
-    settings[ 'teststatus' ] = str()
-    for job in settings[ 'config' ][ 'jenkins' ][ 'jobs' ]:
-        logging.debug( "Checking job %s" % job )
-        job_instance = server.get_job( job )
-        this_status = job_instance.get_last_build().get_status()
-        if not ( settings[ 'teststatus' ] == "FAILURE" or (settings[ 'teststatus' ] == "UNSTABLE" and this_status == "SUCCESS") ):
-            settings[ 'teststatus' ] = this_status
-    return settings
-
-if __name__ == "__main__":
-    logging.debug( "start" )
-    try:
-        while True:
-            alloff()
-            automationhat.relay.one.on()
-            settings = argumentTest()
-            #pprint( settings[ 'config' ][ 'jenkins' ] )
-            #print( "Hello" )
-            #pprint ( get_server_instance( settings ).version )
-            settings = get_job_details( settings )
-            status = settings[ 'teststatus' ]
-            setstatus(status)
-            #print( get_server_instance().version )
-            time.sleep(300)
-
-    except Exception as e:
-        print("Error: " + str(e))
-        traceback.print_exc()
-        exit(1)
-    logging.debug('done')
-
-'''
-export LAB="lab4b"
-export config_url="http://172.30.218.244/autotest/settings.json"
-'''

workflow/this_duration.bash

@@ -1,18 +0,0 @@
-#!/bin/bash
-# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
-find ${pwd} -type f | while read mediafile
-do 
-  duration=$( mediainfo --full --Output=XML "${mediafile}" | xmlstarlet sel -T -t -m "_:MediaInfo/_:media/_:track[@type='Audio']/_:Duration[1]" -v "." -n - | awk -F '.' '{print $1}' )
-  if [ "${duration}" != "" ]
-  then
-    echo "${duration} ${mediafile}"
-    continue
-  fi
-  duration=$( /bin/date -ud "1970-01-01 $( ffprobe -i "${mediafile}" 2>&1| awk -F ': |, ' '/Duration:/ { print $2 }' )" +%s )
-  if [ "${duration}" != 0 ]
-  then
-    echo "${duration} ${mediafile}"
-    continue
-  fi
-done
-find -type f -exec file {} \; | grep -vEi 'audio|mpeg|video' 

workflow/unpack.bash

@@ -1,44 +0,0 @@
-#!/bin/bash
-# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
-intro_clip_end="00:01:30"
-quality_clip_duration="90"
-outro_clip="90"
-#DEBUG="echo "
-cd /mnt/DUMP/hpr-for-archive.org/todo/
-for FILEX in *
-do 
-	if [[ "$(file "$FILEX" | grep -i audio | wc -l )" -ne 1 && "$(mediainfo "$FILEX" | grep -i audio | wc -l )" -ne 1 ]]
-	then
-		echo "$FILEX is not an audio file"
-	else
-		echo "Processing $FILEX"
-		fname=${FILEX%.*}
-		ext=${FILEX/*./}
-		# We get the druation from mediainfo in miliseconds and remove the last three digits to convert it to seconds
-		# Then calculate the times of the end clip working back from the end of the file
-		# We then pick a random segment between the intro and outro. 
-		# Finally we convert to hour minute second format and the date command is the easiest way to do that
-		duration=$( mediainfo --full "${fname}.${ext}" | grep Duration | egrep -v ".*:.*:|ms|s|mn" | head -1 | awk -F ': ' '{print $2}' )
-		duration="${duration:0:${#duration}-3}"
-		outro_clip_start="$(($duration-$outro_clip))"
-		quality_clip_end="$(($outro_clip_start-$quality_clip_duration))"
-		quality_clip_start=$(shuf -i ${intro_clip_end}-${quality_clip_end} -n 1)
-		quality_clip_end="$(($quality_clip_start+$quality_clip_duration))"
-		duration=$(\date -d@${duration} -u +%H:%M:%S)
-		quality_clip_start=$(\date -d@${quality_clip_start} -u +%H:%M:%S)
-		quality_clip_end=$(\date -d@${quality_clip_end} -u +%H:%M:%S)
-		outro_clip_start=$(\date -d@${outro_clip_start} -u +%H:%M:%S)
-		echo -e "\tExtracting spectrogram"
-		${DEBUG} sox "${fname}.${ext}" -n spectrogram -Y 130 -l -r -t "${fname}.${ext}" -o "${fname}_spectrogram.png"
-		echo -e "\tExtracting intro clip from 0 to ${intro_clip_end}"
-		${DEBUG} ffmpeg -y -ss 0 -t ${intro_clip_end} -i "${fname}.${ext}" "${fname}_intro_clip".wav 2>/dev/null
-		echo -e "\tExtracting quality clip from ${quality_clip_start} to ${quality_clip_end}"
-		${DEBUG} ffmpeg -y -ss ${quality_clip_start} -t ${quality_clip_end} -i "${fname}.${ext}" "${fname}_quality_clip".wav 2>/dev/null
-		echo -e "\tExtracting outro clip from ${outro_clip_start}  ${duration}"
-		${DEBUG} ffmpeg -y -ss ${outro_clip_start} -t ${duration} -i "${fname}.${ext}" "${fname}_outro_clip".wav 2>/dev/null
-		echo -e "\tCreating low fidelity version"
-		${DEBUG} ffmpeg -y -i "${fname}.${ext}" temp.wav  2>/dev/null
-		${DEBUG} sox "temp.wav" -c 1 -r 16000 -t wav  - 2>/dev/null | speexenc - "${fname}_low_fidelity.spx" 2>/dev/null
-		${DEBUG} rm temp.wav
-	fi
-done

workflow/x.bash

@@ -1,50 +0,0 @@
-#cd /home/ken/processing/1659905303_3664_2022-08-18_037b55af6b77191b2610f5cedb5d2bd962f0251720bf0
-ttsserver="http://localhost:5500"
-ep_num="3664"
-
-echo "This is Hacker Public Radio " > "hpr${ep_num}_summary.txt"
-
-## Jump to encoding
-
-function create_tts_summary {
-  HPR_summary="$( cat "hpr${ep_num}_summary.txt" )"
-  echo "INFO: Converting text \"${HPR_summary}\" to speech."
-  curl -X 'GET' -G --data-urlencode "voice=coqui-tts:en_ljspeech" --data-urlencode "text=${HPR_summary}" --data-urlencode "vocoder=high" --data-urlencode "denoiserStrength=0.03" --data-urlencode "cache=false" ${ttsserver}/api/tts -H 'accept: */*' --output ~hpr${ep_num}_summary.wav
-}
-
-create_tts_summary_ok="not-ok"
-
-while [ "${create_tts_summary_ok}" != "OK" ]
-do
-  create_tts_summary
-
-  mpv ~hpr${ep_num}_summary.wav
-
-  read -p "Is the text to speech correct (y|N) ? " -n 1 -r
-  echo    # (optional) move to a new line
-  if [[ $REPLY =~ ^[Yy]$ ]]
-  then
-    create_tts_summary_ok="OK"
-  else
-    echo "WARN: Please correct the text and try again."
-    xdg-open "hpr${ep_num}_summary.txt" 2>&1 &
-    inotifywait --event modify "hpr${ep_num}_summary.txt"
-  fi
-done
-echo "INFO: TTS complete."
-# 
-# 
-# 
-# 
-# 
-# 
-# 
-# 
-# 
-# 
-# 
-# 
-# 
-# 
-# 
-# exit 9999