paulj/hpr-tools
1
0
Fork 0
forked from HPR/hpr-tools
Code Pull Requests Activity

Moved project directories and files to an empty local repo

Browse Source
This commit is contained in:
Dave Morriss
2024-06-04 16:35:44 +01:00
parent 2d2b937a9b
commit 38abbcdd39
271 changed files with 55348 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
Show_Submission/.rsync_hpr_upload Normal file
View File

@@ -0,0 +1,56 @@
#-------------------------------------------------------------------------------
# Last change: 2023-07-03 16:03:16
#
# rsync rules for copying files from the upload directory on
# the hackerpublicradio.org server. We want to visit sub-directories and
# collect the file 'shownotes.txt' as well as any ancillary files,
# particularly pictures. We do not want the audio, nor do we want the
# 'shownotes.html' which we upload after processing. The local directory is
# kept in step with the remote one, including deleting files.
#
# 2019-12-11: The upper-case versions of some file extensions weren't
# originally catered for and have been added.
#
# 2021-11-24: Ken changed a show number (3487->3482) and left its original
# directory, then (since the slot was now open) somebody else sent in 3487 but
# of course got a different directory name because it contains a random key.
# I added an underscore to the start of the directory name to ignore it, and
# have catered for this below.
#
# 2022-12-22: Removed shownotes.txt from the list of files to collect and
# added it to the list _not_ to collect!
#
# 2023-04-07: Added '*.oga' and '*.OGA' to the list of audio types to ignore
#
# 2023-06-22: Added directories containing the string '_9999_' to the ignore
# list. This is apparently used for shows destined for the reserve queue. This
# might be a temporary measure because we might want to process such shows in
# this workflow.
#-------------------------------------------------------------------------------
- *.swp
- *~
- *.aiff
- *.AIFF
- *.flac
- *.FLAC
- *.oga
- *.OGA
- *.ogg
- *.OGG
- *.mp3
- *.MP3
- *.wav
- *.WAV
- *.m4a
- *.M4A
- *.lzma
- *.LZMA
- *.diff
- shownotes.txt
- shownotes.html
- photo
- processed/
- _*/
- *_9999_*/
+ */
+ shownotes.json

306
Show_Submission/check_reservations Executable file
View File

@@ -0,0 +1,306 @@
#!/usr/bin/env perl
#===============================================================================
#
# FILE: check_reservations
#
# USAGE: ./check_reservations
#
# DESCRIPTION: Interrogate the 'reservations' table in the live HPR database
# through an SSH tunnel. The result is used to look at and
# report the state of processing on the local system.
#
# The original version of this script ran a remote command on
# the VPS, but was very vulnerable to VPS and tunnel failure.
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.8
# CREATED: 2019-01-07 12:29:06
# REVISION: 2023-07-01 23:04:16
#
#===============================================================================
use strict;
use warnings;
use utf8;
use feature qw{ postderef say signatures state };
no warnings qw{ experimental::postderef experimental::signatures };
use Carp;
use Getopt::Long;
use Config::General;
use DBI;
#
# Version number (manually incremented)
#
our $VERSION = '0.0.8';
#
# Script and directory names
#
( my $PROG = $0 ) =~ s|.*/||mx;
( my $DIR = $0 ) =~ s|/?[^/]*$||mx;
$DIR = '.' unless $DIR;
#-------------------------------------------------------------------------------
# Declarations
#-------------------------------------------------------------------------------
#
# Constants and other declarations
#
my $basedir = "$ENV{HOME}/HPR/Show_Submission";
my $cache = "$basedir/shownotes";
my $configfile = "$basedir/.hpr_db.cfg";
my $reserved = '9999';
my ( $dbh, $sth1, $h1 );
my ( $show, @res);
my $count = 0;
#
# Interpretations of the new status values in the database
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# The sequence seems to be:
# 1. User selects a free slot (request.php), status becomes
# REQUEST_UNVERIFIED, verified is 0, no email address recorded yet, IP
# address recorded
# 2. User enters the email address to receive the link to the form and presses
# 'Next'. An email is sent with the link and status changes to
# REQUEST_EMAIL_SENT, now the email address is filled out but verified
# is 0 still.
# 3. User clicks the link in the email they received which takes them to the
# form, now the state becomes EMAIL_LINK_CLICKED and verified is 1. We see
# this as 'pending', the first status we take account of since we exclude
# all records in the 'reservations' table where verified is 0.
# 4. The user completes the form and hits the 'Send' button. When all files
# have been uploaded the status changes to SHOW_SUBMITTED.
#
# 2022-04-07 New values added:
#
# 5. METADATA_PROCESSED signals that the processing of notes and related
# things is complete
# 6. MEDIA_TRANSCODED indicates that Ken has done the transcoding and posted
# the show.
# 7. UPLOADED_TO_IA indicates that the IA upload has been done (by me usually)
# 8. UPLOADED_TO_RSYNC_NET final step shows that the files (audio & assets)
# have been archived/backed up
#
# 2023-07-02 New value added for reserve shows
#
# 9. RESERVE_SHOW_SUBMITTED indication that a reserve show has been uploaded
# and stashed away ready for adding to a slot at a later time.
#
# I don't think there's a way of knowing what has happened between
# EMAIL_LINK_CLICKED and SHOW_SUBMITTED.
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
my %status = (
REQUEST_UNVERIFIED => 'unverified', # shouldn't be returned
REQUEST_EMAIL_SENT => 'email sent', # been sent the email with a link
EMAIL_LINK_CLICKED => 'pending', # filling in the form/sending the show
RESERVE_SHOW_SUBMITTED => 'reserved show', # reserve show received
SHOW_SUBMITTED => 'uploaded', # all done
METADATA_PROCESSED => 'metadata processed', # notes processed, etc
SHOW_POSTED => 'in the database', # awaiting audio transcoding
MEDIA_TRANSCODED => 'transcoded', # audio transcoded
UPLOADED_TO_IA => 'uploaded to IA', # uploaded to IA
UPLOADED_TO_RSYNC_NET => 'archived', # archived on rsync.net
);
#
# Enable Unicode mode
#
binmode STDOUT, ":encoding(UTF-8)";
binmode STDERR, ":encoding(UTF-8)";
#-------------------------------------------------------------------------------
# Options and arguments
#-------------------------------------------------------------------------------
#
# Process options
#
my %options;
Options( \%options );
#
# Default help
#
Usage() if ( $options{'help'} );
#
# Collect options
#
my $cfgfile
= ( defined( $options{config} ) ? $options{config} : $configfile );
#
# Sanity checks
#
die "Unable to find $cfgfile\n" unless ( -e $cfgfile );
#-------------------------------------------------------------------------------
# Configuration file - load data
#-------------------------------------------------------------------------------
my $conf = Config::General->new(
-ConfigFile => $cfgfile,
-InterPolateVars => 1,
-ExtendedAccess => 1,
);
my %config = $conf->getall();
#-------------------------------------------------------------------------------
# Connect to the database
#-------------------------------------------------------------------------------
my $dbhost = $config{database}->{host} // '127.0.0.1';
my $dbport = $config{database}->{port} // 3306;
my $dbname = $config{database}->{name};
my $dbuser = $config{database}->{user};
my $dbpwd = $config{database}->{password};
$dbh = DBI->connect( "dbi:mysql:host=$dbhost;port=$dbport;database=$dbname",
$dbuser, $dbpwd, { AutoCommit => 1 } )
or croak $DBI::errstr;
#
# Enable client-side UTF8
#
$dbh->{mysql_enable_utf8} = 1;
#
# Set the local timezone to UTC for this connection
#
$dbh->do("set time_zone = '+00:00'") or carp $dbh->errstr;
#
# Query the reservations table for shows which are more or less kosher.
# 2023-07-01 the episode number 9999 is currently a marker that the show is
# for the reserve queue, so we omit it
#
$sth1 = $dbh->prepare(
q{SELECT * FROM reservations WHERE ep_num > 0 ORDER BY timestamp});
$sth1->execute();
if ( $dbh->err ) {
carp $dbh->errstr;
}
#
# Collect details of all the verified reservations found, with an
# interpretation of their state and the email of the sender. For each show
# look at its local state and report it.
#
while ( $h1 = $sth1->fetchrow_hashref() ) {
$show = $h1->{ep_num};
if ($show == $reserved) {
push(@res,$h1);
next;
}
$count++;
my @atts;
push( @atts, "+dir" ) if ( -e "$cache/hpr${show}" );
push( @atts, "+shownotes" ) if ( -s "$cache/hpr${show}/shownotes.txt" );
push( @atts, "+processed" ) if ( -e "$cache/hpr${show}/hpr${show}.html" );
push( @atts, "+uploaded" ) if ( -e "$cache/hpr${show}/.uploaded" );
printf "[%02d] %04d: %-18s (%s) %s\n", $count, $show,
(
exists( $status{ $h1->{status} } )
? $status{ $h1->{status} }
: 'unknown'
),
$h1->{email},
join( "; ", @atts );
}
#
# If for some reason there aren't any reservations tell the caller
#
say "No show reservations" if ( $count == 0 );
if (@res) {
say " ";
say "Reserve queue entries";
$count = 0;
for my $r (@res) {
$count++;
printf "[%02d] %-24s %s (%s)\n", $count,
(
exists( $status{ $r->{status} } )
? $status{ $r->{status} }
: 'unknown'
),
$r->{timestamp},
$r->{email};
}
}
exit;
#=== FUNCTION ================================================================
# NAME: Usage
# PURPOSE: Display a usage message and exit
# PARAMETERS: None
# RETURNS: To command line level with exit value 1
# DESCRIPTION: Builds the usage message using global values
# THROWS: no exceptions
# COMMENTS: none
# SEE ALSO: n/a
#===============================================================================
sub Usage {
print STDERR <<EOD;
$PROG v$VERSION
Usage: $PROG [options]
Scans the HPR database table 'reservations' and reports what new shows are
indicated there and what state they are in.
Options:
-help Display this information
-config=FILE This option allows an alternative configuration file
to be used. This file defines the location of the
database, its port, its name and the username and
password to be used to access it. This feature was
added to allow the script to access alternative
databases or the live database over an SSH tunnel.
If the option is omitted the default file is used:
.hpr_db.cfg
EOD
exit(1);
}
#=== FUNCTION ================================================================
# NAME: Options
# PURPOSE: Processes command-line options
# PARAMETERS: $optref Hash reference to hold the options
# RETURNS: Undef
# DESCRIPTION:
# THROWS: no exceptions
# COMMENTS: none
# SEE ALSO: n/a
#===============================================================================
sub Options {
my ($optref) = @_;
my @options = ( "help", "config=s", );
Usage() if ( !GetOptions( $optref, @options ) );
return;
}
# vim: syntax=perl:ts=8:sw=4:et:ai:tw=100:fo=tcrqn21:fdm=marker

116
Show_Submission/copy_shownotes Executable file
View File

@@ -0,0 +1,116 @@
#!/usr/bin/env bash
#===============================================================================
#
# FILE: copy_shownotes
#
# USAGE: ./copy_shownotes
#
# DESCRIPTION: Copies the shownotes (and related files) downloaded from the
# HPR server. This happens after 'sync_hpr' has been run to
# collect updates from the ~hpr/upload/ directory on the server
# and store the result in the local upload/ directory.
# 2022-12-17: Converted to shownotes.json.
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.10
# CREATED: 2015-09-16 21:51:15
# REVISION: 2023-07-01 22:48:53
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
#
# Load library functions
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
# shellcheck source=/home/cendjm/bin/function_lib.sh
source "$LIB"
#
# Colour codes
#
define_colours
#
# Directories
#
BASEDIR="$HOME/HPR/Show_Submission"
LOGS="$BASEDIR/logs"
UPLOAD="$BASEDIR/upload"
CACHE="$BASEDIR/shownotes"
#
# Filenames
#
LOGFILE="$LOGS/${SCRIPT}.log"
#SHOWNOTES='shownotes.txt'
SHOWNOTES='shownotes.json'
ORIGIN='.origin'
STATUS='.status'
DUMMY='.dummy'
#
# Loop through everything in the $UPLOAD directory using a regular expression
# to find sub-directories. These look like:
# 1445633350_1906_2015-11-23_f348faf9125c129c1ebe0dd0edd721a0562a9d464bbbf
#
# The regex used in find takes into account that it needs to match the full
# path.
#
count=0
target=".*/[0-9]+_[0-9]{4}_[0-9]{4}-[0-9]{2}-[0-9]{2}_.+$"
while read -r d
do
#
# Parse out the show number
#
show="$(echo "$d" | cut -f2 -d_)"
from="$UPLOAD/$d/"
dir="$CACHE/hpr${show}"
to="$dir/$SHOWNOTES"
origin="$dir/$ORIGIN"
status="$dir/$STATUS"
dummy="$dir/$DUMMY"
#
# Make the receiving directory if it doesn't exist
#
if [[ ! -e $dir ]]; then
mkdir "$dir"
fi
#
# Copy files if there are no shownotes or the file exists and is empty.
#
# 2022-12-17: We're soon not using shownotes.txt any more. The data is in
# shownotes.json instead. Also, dummy shows have a '.dummy' file rather
# than empty notes.
#
# if [[ ! -e $to || ! -s $to ]]; then
if [[ ! -e $to || -e $dummy ]]; then
rsync -vaP "$from" "${dir}/"
echo "$d" > "$origin"
echo "copied" > "$status"
((count++))
printf '%(%F %T)T %s\n' -1 "$dir" >> "$LOGFILE"
echo "${green}Copied notes for show $show${reset}"
fi
done < <(find "$UPLOAD" -maxdepth 1 -regextype posix-egrep -regex "$target" -type d -printf '%f\n')
if [[ $count -eq 0 ]]; then
echo "${yellow}Nothing to do${reset}"
else
echo "${green}Notes copied: $count${reset}"
fi
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21

133
Show_Submission/do_brave Executable file
View File

@@ -0,0 +1,133 @@
#!/bin/bash -
#===============================================================================
#
# FILE: do_brave
#
# USAGE: ./do_brave <epno>
#
# DESCRIPTION: Run the Brave browser to view completed notes
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: 2020-06-01: New version of Brave (now called 'brave-browser')
# necessitates changes in how this script was originally
# designed. The softlink between hpr????.out and hpr????.html
# has been converted to a hard link.
# NOTES: We use a link 'do_browser' to point to whichever script runs
# the preferred browser. It's been Brave for several years now,
# but we haven't changed this!
# 2022-12-22: We now write state changes to the file .status in
# the show directory, so we need to do that here too. Also
# changed to using the function library for cleanup_temp.
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.5
# CREATED: 2016-03-20 15:22:29
# REVISION: 2022-12-22 17:28:12
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
#DIR=${0%/*}
#
# Load library functions
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
# shellcheck source=/home/cendjm/bin/function_lib.sh
source "$LIB"
#
# Basic validation
#
if [[ $# -ne 1 ]]; then
echo "Usage $SCRIPT shownumber"
exit 1
fi
#
# Directories and files
#
BASENAME="$HOME/HPR/Show_Submission"
SHOWDIR="$BASENAME/shownotes/hpr${1}"
RAWNOTES="$SHOWDIR/hpr${1}.out"
HTML="$SHOWDIR/hpr${1}.html"
FULLHTML="$SHOWDIR/hpr${1}_full.html"
STATUSFILE="$SHOWDIR/.status"
HTMLFILE="$FULLHTML"
#
# Check we have this browser
#
BRAVE=$(command -v brave-browser)
[[ -v BRAVE ]] || { echo "Problem finding the Brave browser"; exit 1; }
#
# We prefer to view the 'full' html which we do by default. If not found
# (because the host sent in HTML themselves) we look for hpr????.html, which
# is a link to the notes from the form (hpr????.out), and view that. If the
# link didn't get created (not sure why) we copy the "raw" notes to
# a temporary file with an '.html' extension (TODO: we could just make a link
# here). Otherwise we found nothing viewable.
#
if [[ ! -e $FULLHTML ]]; then
if [[ -e $HTML ]]; then
echo "No full HTML found, viewing $HTML instead"
HTMLFILE="$HTML"
elif [[ -e $RAWNOTES ]]; then
echo "No files with ''.HTML' suffix, viewing raw notes"
TMP1=$(mktemp '/tmp/notes_XXXXXX.html') || { echo "$SCRIPT: creation of temporary file failed!"; exit 1; }
trap 'cleanup_temp $TMP1' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
cp "$RAWNOTES" "$TMP1"
HTMLFILE="$TMP1"
else
echo "Nothing to view, giving up"
exit
fi
fi
#
# Open a parent instance of Brave (in the background), then open the HTML
# notes after a short delay, thereby ensuring they open in a tab rather than
# in another window. Brave has great potential but documentation is a bit
# sparse.
# NOTE: We're using debug statements for the moment until this method is shown
# to be a good one.
#
# if [[ $(pgrep -u "$USER" -f '/usr/bin/brave-browser' -c) -eq 0 ]]; then
# echo "D> Starting parent browser"
# $BRAVE > /dev/null 2>&1 &
# echo "D> Delaying ..."
# sleep 3
# fi
#
# 2020-11-29 Looks like the parent + child model doesn't work any more (they
# keep changing this browser!). Also, just running the browser doesn't return
# to the command line any more so it seems to need to be in the background.
#
# echo "D> Starting browser tab"
echo "D> Starting browser itself"
$BRAVE "${HTMLFILE}" > /dev/null 2>&1 &
# $BRAVE "${HTMLFILE}&"
# $BRAVE "${HTMLFILE}"
RES=$?
if [[ $RES -eq 0 ]]; then
#
# Update the status file
#
echo "rendered" >> "$STATUSFILE" || \
{ echo "Failed to update $STATUSFILE"; exit 1; }
else
echo "Oops! Something went wrong!"
exit 1
fi
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21

1
Show_Submission/do_browser Symbolic link
View File

@@ -0,0 +1 @@
do_brave

174
Show_Submission/do_change_format Executable file
View File

@@ -0,0 +1,174 @@
#!/bin/bash -
#===============================================================================
#
# FILE: do_change_format
#
# USAGE: ./do_change_format <epno>
#
# DESCRIPTION: Changes the declared format of a show. Mainly useful to change
# 'markdown_standard' (largely useless) to 'Markdown_Pandoc',
# but can be used to override notes declared as 'html5' when
# they are 'plain_text'.
# 2022-12-22: Now using only shownotes.json, writing the new
# value to the .format file and writing status changes to the
# .status file. DOES NOT update the JSON.
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.4
# CREATED: 2018-12-06 11:11:30
# REVISION: 2024-02-13 20:33:42
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
#DIR=${0%/*}
VERSION='0.0.4'
#
# Load library functions (make_file_list, range_parse, cleanup_temp)
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
# shellcheck source=/home/cendjm/bin/function_lib.sh
source "$LIB"
#
# Check the argument after any options
#
if [[ $# -ne 1 ]]; then
echo "$SCRIPT ($VERSION): Usage $SCRIPT shownumber"
exit
fi
#
# Paths to files
#
BASENAME="$HOME/HPR/Show_Submission"
SHOWDIR="$BASENAME/shownotes/hpr${1}"
FORMAT="$SHOWDIR/.format"
FROM="$SHOWDIR/shownotes.json"
STATUSFILE="$SHOWDIR/.status"
JSONNOTES="$SHOWDIR/shownotes.json"
JSONORIG="$SHOWDIR/shownotes.json.orig"
RELEASE="$SHOWDIR/.release"
ORIGIN="$SHOWDIR/.origin"
DEST="hpr@hackerpublicradio.org:upload"
SSHPORT=22
#
# Collect the place to write results and the release date
#
upload_dir=$(cat "$ORIGIN")
release_date=$(cat "$RELEASE")
#
# The permitted formats as defined in the web form
#
declare -A formats
formats[plain_text]='FALSE'
formats[html5]='FALSE'
formats[markdown_standard]='FALSE'
formats[Markdown_GitHub]='FALSE'
formats[Markdown_Pandoc]='FALSE'
formats[restructured_text]='FALSE'
formats[txt2tags]='FALSE'
#
# Check that we actually have notes for this show, and they aren't empty
#
if [[ -e $FROM ]]; then
if [[ ! -s $FROM ]]; then
echo "File $FROM is empty"
exit 1
fi
else
echo "$SCRIPT: File not found: $FROM"
exit 1
fi
#
# Make temporary files and set traps to delete them.
#
# TMP1 - HTML header
#
TMP1=$(mktemp) || {
echo "$SCRIPT: creation of temporary file failed!"
exit 1
}
trap 'cleanup_temp $TMP1' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
#
# Record the current format
#
FFORMAT="$(cat "$FORMAT")"
formats[$FFORMAT]='TRUE'
# for k in $(printf '%s\n' "${!formats[@]}" | sort); do
# printf 'D> %18s: %s\n' "$k" "${formats[$k]}"
# done
#
# Generate a Zenity list box with radio buttons. Show current setting and
# allow any other to be selected to replace it.
#
newfmt=$(for k in $(printf '%s\n' "${!formats[@]}" | sort); do
printf '%s\n%s\n' "${formats[$k]}" "$k"; done |\
zenity --list --radiolist --height=300 --width=300 \
--column=Choice --title="Change format of notes" \
--text="Choose format" --column=Format 2> /dev/null) ||\
{ echo "Cancelled"; exit; }
# echo "D> $newfmt"
#
# If there was a change save the new value
#
if [[ $newfmt != "$FFORMAT" ]]; then
echo "Changing format to '$newfmt'"
#
# Update .format
#
echo "$newfmt" > "$FORMAT"
#
# Update the JSON
#
cp "$FROM" "${FROM}.orig"
cp "$FROM" "$TMP1"
jq -c --arg new "$newfmt" \
'(.metadata.POST.shownotes_format,.metadata.Shownotes_Format) |= $new' \
"$TMP1" > "$FROM"
#
# Upload updated JSON to the server, but only if the release date is in
# the future
#
rd=$(date -d "$release_date" +%s)
now=$(date +%s)
if [[ $((rd - now)) -gt 0 ]]; then
echo "Copying $JSONNOTES to upload/$upload_dir/shownotes.json on the HPR server"
echo "Copying $JSONORIG to upload/$upload_dir/shownotes.json.orig on the HPR server"
scp -P $SSHPORT "$JSONNOTES" "$JSONORIG" "$DEST/$upload_dir/"
else
echo "JSON files not uploaded to the server"
echo "The release date $release_date is in the past!"
fi
#
# Update the status file
#
echo "format" >> "$STATUSFILE" || \
{ echo "Failed to update $STATUSFILE"; exit 1; }
fi
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21

1
Show_Submission/do_edit Symbolic link
View File

@@ -0,0 +1 @@
do_vim

114
Show_Submission/do_edit_shownotes Executable file
View File

@@ -0,0 +1,114 @@
#!/bin/bash -
#===============================================================================
#
# FILE: do_edit_shownotes
#
# USAGE: ./do_edit_shownotes <field> <epno>
#
# DESCRIPTION: Edit one of the fields often needing work in
# shownotes.{json,txt}, writing the updates back to the HPR
# server in case they are needed there.
#
# ** Under development, not properly tested **
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: 2022-12-20: The shownotes.txt file is now obsolete
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.3
# CREATED: 2022-12-09 21:52:29
# REVISION: 2023-07-29 18:26:04
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
# DIR=${0%/*}
VERSION="0.0.3"
if [[ $# -ne 2 ]]; then
echo "[${SCRIPT} ${VERSION}]: Usage: $SCRIPT field shownumber"
exit
fi
#
# Collect arguments and validate them
#
field="${1,,}"
showno="$2"
field="${field// /}"
showno="${showno// /}"
if ! [[ $field =~ ^(tags|title|summary) ]]; then
echo "Invalid field: $field"
exit 1
fi
if ! [[ $showno =~ ^[0-9]+$ ]]; then
echo "Invalid show number: $showno"
exit 1
fi
#
# Constants, paths and files
#
BASENAME="$HOME/HPR/Show_Submission"
SHOWDIR="$BASENAME/shownotes/hpr${showno}"
# SHOWNOTES="$SHOWDIR/shownotes.txt"
# SNORIG="$SHOWDIR/shownotes.txt.orig"
JSONNOTES="$SHOWDIR/shownotes.json"
JSONORIG="$SHOWDIR/shownotes.json.orig"
ORIGIN="$SHOWDIR/.origin"
EDITSN="$BASENAME/edit_shownotes"
STATUSFILE="$SHOWDIR/.status"
DEST="hpr@hackerpublicradio.org:upload"
SSHPORT=22
#
# Collect the place to write results and the current MD5 sum of the JSON file
#
upload_dir=$(cat "$ORIGIN")
MD5_1=$(md5sum "$JSONNOTES")
#
# Run the editing program
#
echo "Editing $field for show $showno"
$EDITSN -field="$field" -episode="$showno"
RES=$?
#
# Normal exit (0) means something was done. Anything else either means the
# program aborted in a controlled way or there was an error.
#
if [[ $RES -eq 0 ]]; then
echo "Edited show notes ok"
MD5_2=$(md5sum "$JSONNOTES")
if [[ $MD5_1 = "$MD5_2" ]]; then
echo "The files were not changed"
exit
else
echo "Copying $JSONNOTES to upload/$upload_dir/shownotes.json on the HPR server"
echo "Copying $JSONORIG to upload/$upload_dir/shownotes.json.orig on the HPR server"
# scp -P 22074 "$JSONNOTES" "$JSONORIG" "hpr@hackerpublicradio.org:upload/$upload_dir/"
scp -P $SSHPORT "$JSONNOTES" "$JSONORIG" "$DEST/$upload_dir/"
#
# Update the status file
#
echo "metadata" >> "$STATUSFILE" || \
{ echo "Failed to update $STATUSFILE"; exit 1; }
fi
else
echo "Editing script ended prematurely"
fi
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21

313
Show_Submission/do_index Executable file
View File

@@ -0,0 +1,313 @@
#!/bin/bash -
#===============================================================================
#
# FILE: do_index
#
# USAGE: ./do_index <epno>
#
# DESCRIPTION: Makes an index.html file in the uploads directory of a show
# for upload to the show's asset directory on the HPR server.
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.5
# CREATED: 2022-10-30 15:39:28
# REVISION: 2022-12-17 17:38:00
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
# DIR=${0%/*}
VERSION="0.0.5"
STDOUT="/dev/fd/2"
#
# Load library functions (make_file_list, range_parse, cleanup_temp)
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
# shellcheck source=/home/cendjm/bin/function_lib.sh
source "$LIB"
#
# Colour codes
#
define_colours
# {{{ -- _usage -- _dryrun --
#=== FUNCTION ================================================================
# NAME: _usage
# DESCRIPTION: Report usage
# PARAMETERS: None
# RETURNS: Nothing
#===============================================================================
_usage () {
cat >$STDOUT <<-endusage
Usage: ./${SCRIPT} [-h] [-d] shownumber
Version: $VERSION
Generates an 'index.html' file for a show with assets
Options:
-h Print this help
-d Select dry run mode
Arguments:
shownumber
Examples
./${SCRIPT} 3112
endusage
exit
}
#=== FUNCTION ================================================================
# NAME: _dryrun
# DESCRIPTION: Output a dry run message
# PARAMETERS: List of messages
# RETURNS: Nothing
#===============================================================================
_dryrun () {
local prefix="Dry run: "
for msg in "$@"; do
printf '%9s%s\n' "$prefix" "$msg"
prefix=
done
}
# }}}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Option defaults
#
DRYRUN=0 # not dry run mode by default
#
# Process options
#
while getopts :hd opt
do
case "${opt}" in
h) _usage;;
d) DRYRUN=1;;
?) echo "$SCRIPT: Invalid option; aborting"; exit 1;;
esac
done
shift $((OPTIND - 1))
#
# Check there's an argument after removing any options. Abort if not
#
if [[ $# -ne 1 ]]; then
echo "${red}Missing shownumber argument${reset}"
_usage
fi
ep="$1"
printf -v show 'hpr%04d' "$ep"
#
# Paths to files
#
BASEDIR="$HOME/HPR/Show_Submission"
SHOWDIR="$BASEDIR/shownotes/hpr${ep}"
RAWFILE="$SHOWDIR/shownotes.txt"
JSONFILE="$SHOWDIR/shownotes.json"
PICDIR="$SHOWDIR/uploads"
INDEXFILE="$PICDIR/index.html"
ASSETS="$SHOWDIR/.assets" # Created in parse_JSON
PICLIST="$SHOWDIR/.pictures" # Created in parse_JSON
STATUS="$SHOWDIR/.status"
#
# Check the show directory exists
#
if [[ ! -d $SHOWDIR ]]; then
echo "$SCRIPT: ${red}Directory $SHOWDIR not found${reset}"
exit 1
fi
#
# If no assets then don't continue
#
if [[ ! -e $ASSETS ]]; then
echo "$SCRIPT: ${red}Asset file (.assets) not found in $SHOWDIR${reset}"
exit 1
fi
#
# Do we already have an 'index.html'?
#
if [[ -e $INDEXFILE ]]; then
echo "${red}There's already an 'index.html'${reset}"
if [[ $DRYRUN -eq 0 ]] && ! yes_no 'Overwrite this file? %s ' 'No'; then
echo "${red}Aborting!${reset}"
exit 1
fi
fi
#
# Make temporary files and set traps to delete them.
#
# TMP1 - HTML header
# TMP2 - HTML footer
#
TMP1=$(mktemp) || {
echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
exit 1
}
TMP2=$(mktemp) || {
echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
exit 1
}
trap 'cleanup_temp $TMP1 $TMP2' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
#
# Parse these fields from the raw input file (shownotes.txt) into Bash
# variables so we can make an index.html later.
#
# Title='' Summary='' Host_Name=''
# eval \
# "$(sed -n '/^\(Title\|Summary\|Host_Name\):/{s/^\([^:]\+\):\t/\1="/;s/$/"/;p}' "$RAWFILE")"
#
# Parse these fields from the JSON input file (shownotes.json) into Bash
# variables so we can make an index.html later.
#
Title='' Summary='' Host_Name=''
jqprog="@sh \"Host_Name=\(.host.Host_Name) Title=\(.episode.Title) Summary=\(.episode.Summary)\""
command=$(jq -r "$jqprog" "$JSONFILE")
eval "${command}"
#
# Make two HTML templates and two TT² templates. We do this in dry-run mode
# too then throw them away!
#
# {{{
cat > "$TMP1" <<ENDHTML1
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="generator" content="pandoc">
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
<meta name="author" content="$Host_Name">
<title>$Title (HPR Show $ep)</title>
<style type="text/css">code{white-space: pre;}</style>
<!--[if lt IE 9]>
<script src="https://html5shim.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->
<link rel="stylesheet" href="https://hackerpublicradio.org/css/hpr.css">
</head>
<body id="home">
<div id="container" class="shadow">
<header>
<h1 class="title">$Title (HPR Show $ep)</h1>
<h3 class="summary">$Summary</h3>
<h3 class="author">$Host_Name</h3>
<hr/>
</header>
<main id="maincontent">
<article>
<h3 id="index">Index</h3>
<ul>
ENDHTML1
cat > "$TMP2" <<ENDHTML2
</ul>
</article>
</main>
</div>
</body>
</html>
ENDHTML2
# }}}
#
# Make a simple index.html file. We use the .assets file and the .pictures
# list (if there is one) and read them into an aray and a hash. We can then
# generate HTML appropriate to their type.
#
if [[ $DRYRUN -eq 0 ]]; then
#
# Make an array of assets
#
if [[ -e $ASSETS ]]; then
declare -a assets
mapfile -t assets < "$ASSETS"
else
echo "$SCRIPT: ${red}No assets found!${reset}"
echo "$SCRIPT: ${red}Show has no $ASSETS file!${reset}"
exit 1
fi
#
# Make a hash of pictures. It's not an error if there are none; we still
# need to process assets.
#
if [[ -e $PICLIST ]]; then
declare -A pix
while read -er pic; do
pix[$pic]=1
done < "$PICLIST"
else
echo "$SCRIPT: ${red}No pictures found!${reset}"
echo "$SCRIPT: ${blue}Processing $ASSETS only${reset}"
fi
#
# Assemble the index from header, footer and a body of file/picture
# references. If an asset is in the 'pix' hash then it's a picture, so we
# generate different HTML than in the other case.
# TODO: Do a better job with displaying the pictures! Maybe limit the
# size? Who's going to be looking at this anyway? Could the image URL
# contain a size I wonder?
#
(
cat "$TMP1"
for asset in "${assets[@]}"; do
# Don't let the index point to itself!
if [[ $asset == 'index.html' ]]; then
continue
fi
if [ -e "$PICLIST" ] && [ "${pix[$asset]+set}" == 'set' ]; then
printf '<li><img src="https://hackerpublicradio.org/eps/%s/%s" alt="%s"></li>\n' \
"$show" "$asset" "$asset"
else
printf '<li><a href="https://hackerpublicradio.org/eps/%s/%s">%s</a></li>\n' \
"$show" "$asset" "$asset"
fi
done
cat "$TMP2"
) > "$INDEXFILE"
#
# Report what happened
#
if [[ -e $INDEXFILE ]]; then
echo "${green}Generated index in $INDEXFILE${reset}"
else
echo "${red}Something went wrong; can't find $INDEXFILE${reset}"
fi
#
# Set the status for this show
#
echo "index" >> "$STATUS"
else
_dryrun "Would have generated index file ${yellow}$INDEXFILE${reset}"
fi
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker

524
Show_Submission/do_pandoc Executable file
View File

@@ -0,0 +1,524 @@
#!/bin/bash -
#===============================================================================
#
# FILE: do_pandoc
#
# USAGE: ./do_pandoc [option] <epno>
#
# DESCRIPTION: Performs an "intelligent" Pandoc run on various types of show
# notes. Converts various markup formats into HTML. Treate plain
# text as Markdown, though this depends on the prior editing
# step doing the right thing. Handles pictures and other assets
# in plain text shows that have them - this is done by the edit
# phase adding TT² macros and this script processing them with
# 'tpage'.
# Version 0:2:5 (released 2022-12-04) has not yet been fully
# tested. Seems reliable 2023-03-03.
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: 2021-04-03: removed the TOC option
# 2021-11-07: Added --strip-comments to the HTML snippet
# generation stage
# 2022-11-01: Big rewrite over the month. Refer to
# do_pandoc_0.1.6 for the previous version since there have been
# some big changes.
# 2022-12-17: Ending reliance on shownotes.txt, and using
# shownotes.json instead into the future. Some massive tidying
# and rationalisation are still required.
# 2023-03-03: If the title contained a quote then the previous
# algorithm made bad YAML which caused the 'full' html to fail.
# Fixed.
# 2023-11-15: The 'prefix' setting is wrong. It needs another
# 'hpr1234' directory level. This is needed because the
# top-level stuff for the show is already in such a directory,
# and while we have assets in a sub-directory we need to be
# careful about collisions.
#
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.2.10
# CREATED: 2016-08-16 15:34:30
# REVISION: 2024-02-18 13:27:40
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
#DIR=${0%/*}
VERSION='0.2.10'
STDOUT="/dev/fd/2"
#
# Load library functions
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
# shellcheck source=/home/cendjm/bin/function_lib.sh
source "$LIB"
#
# Colour codes
#
define_colours
# {{{ Functions: -- _usage -- _DEBUG --
#=== FUNCTION ================================================================
# NAME: _usage
# DESCRIPTION: Report usage
# PARAMETERS: None
# RETURNS: Nothing
#===============================================================================
_usage () {
cat >$STDOUT <<-endusage
Usage: ./${SCRIPT} [-h] [-d] [-D] shownumber
Version: $VERSION
Runs Pandoc against a particular show, choosing a format as
defined by the declared format (in the file '.format').
(In this version there is no method to force an explicit input format)
Options:
-h Print this help
-d Select dry run mode
-D Turn on debug mode with lots of extra output
The default behaviour is now to access the '.format' file in the show
directory.
Arguments:
shownumber
Examples
./${SCRIPT} -h
./${SCRIPT} -d 2240
./${SCRIPT} -D 2250
endusage
exit
}
#=== FUNCTION ================================================================
# NAME: _DEBUG
# DESCRIPTION: Writes one or more message lines if in DEBUG mode
# PARAMETERS: List of messages
# RETURNS: Nothing
#===============================================================================
_DEBUG () {
[ "$DEBUG" == 0 ] && return
for msg in "$@"; do
printf 'D> %s\n' "$msg"
done
}
# }}}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Formats offered by the web form on the HPR site:
#
# Keyword Explanation
# ------- -----------
# plain_text Plain text
# html5 HTML5 (preferred)
# markdown_standard Markdown (standard)
# Markdown_GitHub Markdown (GitHub flavoured)
# Markdown_Pandoc Markdown (Pandoc flavoured)
# restructured_text RestructuredText
# txt2tags txt2tags
#
#
# Hash to perform translation from declared format to Pandoc "-from" value
#
# {{{ -- 'lookup' hash --
declare -A lookup
lookup[plain_text]='markdown-implicit_figures'
lookup[html5]='html'
lookup[markdown_standard]='markdown_strict'
lookup[Markdown_GitHub]='gfm' # Extensions are limited
lookup[Markdown_Pandoc]='markdown-implicit_figures'
lookup[restructured_text]='rst'
lookup[txt2tags]='t2t'
# }}}
#
# Hash for options when generating standalone readable HTML
#
# (The 'smart' extension is only applicable to markdown, commonmark, latex,
# mediawiki, org, rst, twiki; we want to turn it off to remove smart quotes)
#
# {{{ -- 'options' hash --
declare -A options
options[plain_text]='-smart'
options[html5]=''
options[markdown_standard]='+yaml_metadata_block'
options[Markdown_GitHub]=''
options[Markdown_Pandoc]='-smart+yaml_metadata_block'
options[restructured_text]='-smart'
options[txt2tags]=''
# }}}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Sanity checks
#
JQ=$(command -v jq)
[ -n "$JQ" ] || { echo "Program 'jq' was not found"; exit 1; }
# YQ=$(command -v yq)
# [ -n "$YQ" ] || { echo "Program 'yq' was not found"; exit 1; }
#
# Process options first
#
while getopts :dDh opt
do
case "${opt}" in
d) DRYRUN=1;;
D) DEBUG=1;;
h) _usage;;
?) echo "$SCRIPT: Invalid option; aborting"; exit 1;;
esac
done
shift $((OPTIND - 1))
#
# Default options if not provided
#
DEBUG=${DEBUG:-0}
DRYRUN=${DRYRUN:-0}
#
# Check there's an argument after removing any options. Abort if not
#
if [[ $# -ne 1 ]]; then
_usage
fi
#
# Declare variables for later
# TODO: Is this necessary?
#
declare SHOWID FROM POPTIONS
#
# Make the explicit show id, catering for leading zeroes (belt & braces)
#
printf -v SHOWID 'hpr%04d' "$1"
#
# Make temporary files and set traps to delete them
#
TMP1=$(mktemp) || {
echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
exit 1
}
TMP2=$(mktemp) || {
echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
exit 1
}
TMP3=$(mktemp) || {
echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
exit 1
}
trap 'cleanup_temp $TMP1 $TMP2 $TMP3' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
#
# Paths to files
#
# ------------------------------------------------------------------------------
# Main directory
BASENAME="$HOME/HPR/Show_Submission"
# JSON to YAML Perl script
J2Y="$BASENAME/author_title.pl"
[ -e "$J2Y" ] || { echo "Program '$J2Y' was not found"; exit 1; }
# The notes for all shows are here
SHOWNOTES="$BASENAME/shownotes"
# Notes for this show are here
SHOWDIR="$SHOWNOTES/$SHOWID"
# Paths to all files already created or being created here
#RAWFILE="$SHOWDIR/shownotes.txt"
JSONFILE="$SHOWDIR/shownotes.json"
FMTFILE="$SHOWDIR/.format"
PICLIST="$SHOWDIR/.pictures"
STATUS="$SHOWDIR/.status"
EXTRACT="$SHOWDIR/${SHOWID}.out"
FULLHTML="$SHOWDIR/${SHOWID}_full.html"
HTML="$SHOWDIR/${SHOWID}.html"
# TT² macros and paths for adding pictures
PICTPL="$BASENAME/pic_definitions.tpl"
MANIFEST="$SHOWDIR/.pictures.mf" # From do_pictures
# ------------------------------------------------------------------------------
#
# The partial URL for pictures on the HPR site
#
BASEURL='https://hackerpublicradio.org/eps/'
#{{{ --- Obsolete 2022-12-17 ---
#
# Make a metadata file by parsing the raw data file fields with awk. Save it
# in a temporary file.
#
# See the original do_pandoc_0.1.6 for the extended comments that led to this
# design. We were trying to make acceptable YAML, but ended up extracting
# metadata from the result.
# ----
# TODO: 2022-12-01 Rationalise all of this; it's full of debris from previous
# attempts to solve the problem of passing metadata to Pandoc.
# ----
#
# awk -f - "$RAWFILE" > "$TMP1" <<'ENDAWK'
# BEGIN {print "---"}
# /^Title:/ && got_title == 0 {
# sub(/^Title:\s+/,"")
# printf "#title: %s\n",$0
# gsub(/'/,"''")
# printf "title: '%s'\n",$0
# got_title = 1
# }
# /^Host_Name:/ && got_author == 0 {
# sub(/^Host_Name:\s+/,"")
# printf "#author: %s\n",$0
# gsub(/'/,"''")
# printf "author: '%s'\n",$0
# got_author = 1
# }
# END {print "---"}
# ENDAWK
#}}}
#
# Use 'jq' to parse the JSON and make the metadata (in the form of YAML)
# needed for Pandoc
#
# Non-YAML alternative - not chosen
#jqprog="@text \"author: \(.host.Host_Name)\ntitle: \(.episode.Title)\""
#
# Testing another formatter (Journal 2023-03-03)
#jqprog="@sh \"---\nauthor: \(.host.Host_Name)\ntitle: \(.episode.Title)\n---\""
# Added quotes around the generated strings (2023-03-31)
# jqprog="@text \"---\nauthor: \(.host.Host_Name)\ntitle: \(.episode.Title)\n---\""
# Moved to 'yq' 2023-04-01
# jqprog="@text \"---\nauthor: '\(.host.Host_Name)'\ntitle: '\(.episode.Title)'\n---\""
# jq -r "$jqprog" "$JSONFILE" > "$TMP1"
# On 2023-10-01 wrote a Perl JSON to YAML generator just for these two
# elements. It's called 'author_title.pl'
# yqprog='{author:.host.Host_Name,title:.episode.Title}'
# ( echo "---"; $YQ -y "$yqprog" "$JSONFILE"; echo "---"; ) > "$TMP1"
$J2Y "$JSONFILE" "$TMP1"
_DEBUG "YAML:" "$(cat "$TMP1")"
#
# Check the main output file from do_parse exists
#
if [[ ! -e $EXTRACT ]]; then
echo "$SCRIPT: ${red}File not found: $EXTRACT${reset}"
exit 1
fi
#
# Get the format or fail with an error
#
if [[ -e $FMTFILE ]]; then
FORMAT=$(cat "$FMTFILE")
else
# TODO: Should we default to something rather than abort?
echo "$SCRIPT: ${red}Could not find declared format (.format file)${reset}"
echo "${yellow}Has do_parse been run? If so try and fix the .format file.${reset}"
exit
fi
#
# Need to match plain text and Markdown variants when deciding to use the
# manifest file and 'tpage'
#
FMTRE='^(plain_text|[Mm]arkdown_)'
#
# Determine if there are pictures
#
if [[ -e $PICLIST ]]; then
hasPictures=1
else
hasPictures=0
fi
#
# Here we use the declared format to determine what to do.
#
# This code now blocks HTML->HTML processing since it just confuses matters.
# The files hprNNNN.out and hprNNNN.html are linked to one another, so editing
# the former will edit the latter in preparation for uploading if there are
# changes that need to be made. See the journal discussion for 2018-05-24.
# ---
# TODO: 2022-12-01 If the notes are HTML but declared as something else then
# this check will not work. Trouble is, parse_JSON will have worked this out
# but not saved it so we can't avoid running Pandoc on HTML in this case.
# ---
# TODO: 2022-12-01 Look at resolving this in parse_JSON?
# ---
#
if [[ $FORMAT == 'html5' ]]; then
echo "${red}Running Pandoc on HTML is not allowed${reset}"
echo "${yellow}Run do_edit to edit as necessary and do_browser to view${reset}"
exit
elif [[ $FORMAT == 'plain_text' ]]; then
echo "${yellow}Format chosen is plain text${reset}"
if [[ $hasPictures -eq 0 ]]; then
echo "${yellow}This will be treated as Markdown${reset}"
else
echo "${yellow}This will be treated as Markdown and will need special action${reset}"
echo "${yellow}since there are pictures${reset}"
fi
fi
FROM=${lookup[$FORMAT]}
POPTIONS=${options[$FORMAT]}
echo "${green}Will process $FORMAT with 'pandoc -f ${FROM}'${reset}"
echo "${yellow}Options chosen for --standalone are '${POPTIONS}'${reset}"
#
# Only for plain text notes, process pictures for the HTML we'll be adding to
# the database.
#
# We need the following things:
# - The notes to be plain text format
# - Some pictures
# - Files written by do_parse and do_pictures:
# - .assets (not used here)
# - .pictures (were previously used here, but no longer)
# - .pictures.mf (needed by the TT² macros)
# - .pictures.tt2 (the TT² macro calls - already edited into the notes)
# - pic_definitions.tpl (macro definitions, common to all shows)
# - Not to be in dry run mode; if we are we just report intentions
#
if [[ $DRYRUN -eq 0 ]]; then
# if [[ $FORMAT == 'plain_text' && -e $MANIFEST ]]; then
if [[ $FORMAT =~ $FMTRE && -e $MANIFEST ]]; then
#
# Deal with pictures using the TT² macros
#
_DEBUG "Processing TT² inclusions"
# Make a picture manifest with a header
awk 'BEGIN{print "file : thumb"}{p1=$0; getline p2; printf "%s : %s\n",p1,p2}' \
"$MANIFEST" > "$TMP2"
_DEBUG "Picture list:" "$(cat "$TMP2")" "---" \
"BASEURL=${BASEURL}${SHOWID}/" \
"EXTRACT=$EXTRACT" \
"Extract file contents:" "$(cat "$EXTRACT")" "---"
# Run the macros on the notes to make $TMP3 for Pandoc
tpage --pre_process="$PICTPL" \
--define "piclist=$TMP2" \
--define "prefix=${BASEURL}${SHOWID}/${SHOWID}/" \
"$EXTRACT" > "$TMP3"
_DEBUG "Processed by tpage" "$(cat "$TMP3")" "---"
else
# Not plain text or a Markdown variant and no pictures, so put the
# notes in $TMP3 where Pandoc will look for them
cat "$EXTRACT" > "$TMP3"
fi
else
#
# We would not have used TT² unless the notes were plain text (or
# a Markdown variant) and there were pictures
#
# if [[ $FORMAT == 'plain_text' && -e $MANIFEST ]]; then
if [[ $FORMAT =~ $FMTRE && -e $MANIFEST ]]; then
echo "${yellow}Would have prepared TT² code for pandoc${reset}"
fi
fi
#
# Generate an HTML snippet for adding to the database.
# (Note 2021-11-24: Added --ascii option.)
#
if [[ $DRYRUN -eq 0 ]]; then
# shellcheck disable=SC2086
pandoc -f "$FROM"-smart -t html5 --ascii --no-highlight --strip-comments \
"$TMP3" -o "$HTML" # $EXTRAS
RES=$?
if [[ $RES -eq 0 ]]; then
echo "$SCRIPT: ${green}Created shownotes/$SHOWID/${HTML##*/}${reset}"
else
echo "$SCRIPT: ${red}Oops! Something went wrong! (line $LINENO)${reset}"
echo "${yellow}$SCRIPT: Aborting now${reset}"
exit 1
fi
else
echo "${yellow}Would have run pandoc to make HTML for upload${reset}"
fi
#
# Make HTML for proof reading. All pictures referenced are now on the HPR
# server (we ran 'do_asset_upload'), so we want to refer to them here.
#
# File $TMP2 contains the .pictures.mf contents with a header line; and it
# contains data for the macros. It was created when we prepared the main HTML
# for the database. We use $BASEURL again here because we want to reference
# the pictures on the server.
#
# We use the awk-formatted file (now yq-formatted) in $TMP1 from earlier to do
# this. At the end TMP3 contains Markdown for Pandoc.
#
if [[ $DRYRUN -eq 0 ]]; then
# if [[ $FORMAT == 'plain_text' && -e $MANIFEST ]]; then
if [[ $FORMAT =~ $FMTRE && -e $MANIFEST ]]; then
tpage --pre_process="$PICTPL" \
--define "piclist=$TMP2" \
--define "prefix=${BASEURL}${SHOWID}/${SHOWID}/" \
"$EXTRACT" > "$TMP3"
else
cat "$EXTRACT" > "$TMP3"
fi
#
# Generate complete HTML that we can proofread. We need metadata for this
# stand-alone HTML which is in the form of YAML in this version.
#
# ----------------------------------------------------------------------
# Original options below when using 'awk' to parse shownotes.txt:
# --metadata="$(sed -n '/^#author:/{s/#//;p}' "$TMP1")" \
# --metadata="$(sed -n '/^#title:/{s/#//;p}' "$TMP1")" \
#
# shellcheck disable=SC2086
pandoc -f ${FROM}${POPTIONS} -t html5 --ascii \
--standalone --template=hpr.html5 --no-highlight \
-c https://hackerpublicradio.org/css/hpr.css \
--metadata-file="$TMP1" -o "$FULLHTML" "$TMP3"
RES=$?
if [[ $RES -eq 0 ]]; then
echo "$SCRIPT: ${green}Created shownotes/$SHOWID/${FULLHTML##*/}${reset}"
else
echo "$SCRIPT: ${red}Oops! Something went wrong making the full HTML! (line $LINENO)${reset}"
fi
else
# Dry run
echo "${yellow}Would have run pandoc to make HTML for proof reading${reset}"
fi
#
# Set the status for this show
#
if [[ $DRYRUN -eq 0 ]]; then
echo "converted" >> "$STATUS"
fi
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker

149
Show_Submission/do_parse Executable file
View File

@@ -0,0 +1,149 @@
#!/bin/bash -
#===============================================================================
#
# FILE: do_parse
#
# USAGE: ./do_parse <epno>
#
# DESCRIPTION: Run 'parse_JSON' on a given show
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.15
# CREATED: 2016-05-14 14:21:34
# REVISION: 2022-10-01 21:49:32
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
#DIR=${0%/*}
VERSION="0.0.15"
#
# Load library functions
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
# shellcheck source=/home/cendjm/bin/function_lib.sh
source "$LIB"
#
# Colour codes
#
define_colours
#
# Process options
#
while getopts :fh opt
do
case "${opt}" in
h) echo "Usage: $SCRIPT [-h] [-f] shownumber"; exit 0;;
f) FORCE=1;;
?) echo "$SCRIPT: Invalid option; aborting"; exit 1;;
esac
done
shift $((OPTIND - 1))
FORCE="${FORCE:-0}"
if [[ $# -ne 1 ]]; then
echo "[${SCRIPT} ${VERSION}] Usage: ${red}$SCRIPT shownumber${reset}"
exit
fi
show="$1"
BASENAME="$HOME/HPR/Show_Submission"
SHOWDIR="$BASENAME/shownotes/hpr${show}"
LOGDIR="$BASENAME/logs"
#PARSER="$BASENAME/parse_shownotes"
PARSER="$BASENAME/parse_JSON"
PARSELOG="$LOGDIR/${PARSER##*/}.log"
#FROM="$BASENAME/shownotes/hpr${show}/hpr${show}_shownotes.txt"
#FROM="$BASENAME/shownotes/hpr${show}/shownotes.txt"
FROM="$SHOWDIR/shownotes.json"
TO="$SHOWDIR/hpr${show}.out"
TOTPL="$SHOWDIR/hpr%d.out"
SLINK="$SHOWDIR/hpr${show}.html"
FMT="$SHOWDIR/.format"
REL="$SHOWDIR/.release"
PICTURES="$SHOWDIR/.pictures"
ASSETS="$SHOWDIR/.assets"
ZIP="$SHOWDIR/.backup.zip"
#JSONTPL="$SHOWDIR/hpr%d.json"
SHOWLOG="$SHOWDIR/error.log"
#
# Sanity checks
#
[ -e "$PARSER" ] || {
echo "$SCRIPT: ${red}$PARSER not found${reset}"
exit 1
}
if [[ ! -e $FROM ]]; then
echo "$SCRIPT: ${red}File not found: $FROM${reset}"
exit 1
fi
#
# Allow overwriting of the output file if the -f option is given. Also, if
# there's a link delete it so parse_shownotes can re-create it (we use '-ef'
# to check that the files have the same inode).
#
if [[ $FORCE -eq 0 ]]; then
if [[ -e $TO ]]; then
echo "$SCRIPT: ${red}The output file $TO already exists${reset}"
exit 1
fi
else
if [[ $TO -ef $SLINK ]]; then
rm -f "$SLINK"
fi
fi
if [[ ! -s $FROM ]]; then
echo "$SCRIPT: ${red}Input file $FROM is empty${reset}"
exit 1
fi
#
# Run the parser, don't validate, and use the show number argument. Read the
# input file 'shownotes.txt' and generate an output file derived from the
# template '$TOTPL'. Write a JSON version of the output and write the declared
# format to a file for future reference.
#
# Updated 2021-02-23: Now we parse the incoming JSON file .shownotes.json'.
# There's no validation and we don't write JSON output.
#
# $PARSER -novalid -ep "${show}" -in "$FROM" -show "$TOTPL" \
# -json "$JSONTPL" -format="$FMT"
$PARSER -ep "${show}" -in "$FROM" -show "$TOTPL" \
-format="$FMT" -release="$REL" -pictures="$PICTURES" -assets="$ASSETS" \
-zip="$ZIP"
RES=$?
if [[ $RES -ne 0 ]]; then
echo "$SCRIPT: ${red}Oops! Something went wrong!${reset}"
exit 1
fi
#
# Make a reference copy of the output file
#
cp "$TO" "${TO%.out}.orig"
#
# Grep the common log file for error reports relating to this show
#
grep -E "^$(date +%Y/%m/%d).+ \[ERROR\] $show" "$PARSELOG" >> "$SHOWLOG"
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21

699
Show_Submission/do_pictures Executable file
View File

@@ -0,0 +1,699 @@
#!/bin/bash -
#===============================================================================
#
# FILE: do_pictures
#
# USAGE: ./do_pictures [options] <epno>
#
# DESCRIPTION: Process pictures associated with a show.
#
# If there are pictures associated with an episode they will
# have been found amongst the show media, including unpacking
# any archives. Their names will have been written to a file
# which is called '.pictures' in the show directory. The files
# themselves will have been moved to a directory called
# 'uploads' in the show directory.
# {{{ --- lengthy details ---
# We no longer write files to the previous directory 'hpr<epno>'
# because we will not make a tar file any more but will upload
# directly to the server (elsewhere). We need the names because
# we want to process them from the '.pictures' file. Processing
# consists of renaming by replacing spaces with underscores and
# using 'gm convert' to remove EXIF data. If the picture size is
# too large a thumbnail is created, called
# '<name>_thumbnail.<ext>'. If the notes are plain text (which
# will be without links) then we will use `.pictures' as an
# ordered list as we place them into the Markdown we create as
# TT² macros. The list is reordered by asking the user to
# resequence it with a list of reference numbers.
#
# Also for plain text and Markdown variant notes we create
# a file called '.pictures.tt2' which contains the TT² commands
# to add the pictures. When editing the notes this list is
# shown in another window so the contents can be copied and
# pasted. There are two TT² macros, 'pic1' for stand-alone
# pictures and 'pic2' for pictures with thumbnails (clicking the
# thumbnail causes the larger picture to be opened).
#
# Running `do_pandoc` on plain text and Markdown variant notes
# will assemble the TT² macros from 'pic_definitions.tpl', and
# the notes containing the statements in '.pictures.tt2' placed
# as required.
#
# If the notes are not plain text then different paths are
# taken:
#
# Markdown variants: any links will require editing to refer to
# any filename changes, and if the picture needs a thumbnail
# then this will need to be catered for. There will be no need
# for `.pictures', `.pictures.tt2' or 'pic_definitions.tpl'
# unless ther are no links already. The notes will be processed
# with 'do_pandoc' which will cater for these differences.
#
# HTML: again, links might need adjusting, and as for Markdown,
# no supplementary files will be required. The modified notes
# will not be further processed before uploading.
# }}}
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.2.3
# CREATED: 2020-05-25 13:20:18
# REVISION: 2024-02-22 14:32:03
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
# DIR=${0%/*}
VERSION="0.2.3"
STDOUT="/dev/fd/2"
#
# Load library functions (make_file_list, range_parse, cleanup_temp)
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
# shellcheck source=/home/cendjm/bin/function_lib.sh
source "$LIB"
#
# Colour codes
#
define_colours
# {{{ -- _usage --
#=== FUNCTION ================================================================
# NAME: _usage
# DESCRIPTION: Report usage
# PARAMETERS: None
# RETURNS: Nothing
#===============================================================================
_usage () {
cat >$STDOUT <<-endusage
Usage: ./${SCRIPT} [-h] [-d] [-D] shownumber
Version: $VERSION
Processes pictures associated with an HPR show.
The script detects any pictures downloaded with an episode, looking for JPEG
and PNG. It copies them to a sub-directory called 'hpr<epno>' (since that's
the name of the directory where where they'll go on the server).
As it copies the files the script renames the copies if necessary, mainly by
replacing spaces with underscores. The copying process uses 'gm convert' and
removes EXIF data along the way. It also generates a thumbnail for each
picture if the size warrants it, called 'NAME_thumbnail.ext'.
The script also makes a list of the newly named pictures in a file called
'.pictures' which is used to refer to them in the notes (by position in the
list). The order in which the pictures are processed and the list built up can
be controlled by the user by the ordering of the presented enumerated list.
A file called '.pictures.tt2' contains the TT² commands to add the pictures.
When editing the notes this list is shown in another window so the contents
can be copied and pasted. There are two TT² macros, 'pic1' for stand-alone
pictures and 'pic2' for pictures with thumbnails (clicking the thumbnail
causes the larger picture to be opened).
Running 'do_pandoc' on the notes will assemble the TT² macros from
'pic_definitions.tpl', and the notes containing the statements in
'.pictures.tt2' which have been placed as required.
An 'index.html' file is created to allow the images to be visualised outside
the notes.
Options:
-h Print this help
-d Select dry run mode
-D Select debug mode (works the same; more output)
-s Silent mode; suppress as many output messages as
possible
Arguments:
shownumber
Examples
./${SCRIPT} -h
./${SCRIPT} -d 3112
./${SCRIPT} -D -d 3112
./${SCRIPT} 3112
./${SCRIPT} -s 3112
endusage
exit
}
# }}}
# {{{ -- _silent -- _dryrun -- _DEBUG --
#=== FUNCTION ================================================================
# NAME: _silent
# DESCRIPTION: Output a message unless we're being silent
# PARAMETERS: List of messages
# RETURNS: Nothing
#===============================================================================
_silent () {
[ "$SILENT" == 1 ] && return
for msg in "$@"; do
printf '%s\n' "$msg"
done
}
#=== FUNCTION ================================================================
# NAME: _dryrun
# DESCRIPTION: Output a dry run message
# PARAMETERS: List of messages
# RETURNS: Nothing
#===============================================================================
_dryrun () {
local prefix="Dry run: "
for msg in "$@"; do
printf '%9s%s\n' "$prefix" "$msg"
prefix=
done
}
#=== FUNCTION ================================================================
# NAME: _DEBUG
# DESCRIPTION: Writes one or more message lines if in DEBUG mode
# PARAMETERS: List of messages
# RETURNS: Nothing
#===============================================================================
_DEBUG () {
[ "$DEBUG" == 0 ] && return
for msg in "$@"; do
printf 'D> %s\n' "$msg"
done
}
# }}}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Debug mode, default off
#
DEBUG=0
#
# Option defaults
#
DRYRUN=0 # not dry run mode by default
SILENT=0 # not silent by default
#
# Process options
#
while getopts :hdDs opt
do
case "${opt}" in
h) _usage;;
d) DRYRUN=1;;
D) DEBUG=1;;
s) SILENT=1;;
?) echo "$SCRIPT: Invalid option; aborting"; exit 1;;
esac
done
shift $((OPTIND - 1))
#
# Check there's an argument after removing any options. Abort if not
#
if [[ $# -ne 1 ]]; then
echo "${red}Missing shownumber argument${reset}"
_usage
fi
#
# Paths to files
#
BASEDIR="$HOME/HPR/Show_Submission"
SHOWDIR="$BASEDIR/shownotes/hpr${1}"
# RAWFILE="$SHOWDIR/shownotes.txt"
JSONFILE="$SHOWDIR/shownotes.json"
PICDIR="$SHOWDIR/uploads"
# INDEXFILE="$PICDIR/index.html"
FORMAT="$SHOWDIR/.format"
ASSETS="$SHOWDIR/.assets" # Created in parse_JSON
PICLIST="$SHOWDIR/.pictures" # Created in parse_JSON
STATUS="$SHOWDIR/.status"
# ----
# Files to be used by the TT² macros that build Markdown from plain
# text notes.
# ----
# MANIFEST contains a list of pictures and thumbnails, with blank lines if no
# thumbnail.
MANIFEST="$SHOWDIR/.pictures.mf"
# PICTPL contains TT² macro calls which can be cut and pasted into the plain
# text notes
PICTPL="$SHOWDIR/.pictures.tt2"
# ----
#
# Constants
#
TNSIZE=800 # Thumbnail size
TNW=$TNSIZE # Maximum width
TNH=$((TNSIZE*2)) # Maximum height
#
# Check the show directory exists
#
if [[ ! -d $SHOWDIR ]]; then
echo "$SCRIPT: ${red}Directory $SHOWDIR not found${reset}"
exit 1
fi
#
# Make temporary files and set traps to delete them.
#
# TMP1 - HTML header
# TMP2 - HTML footer
# TMP3 - call to TT² macro 'pic1' (make a plain picture link)
# TMP4 - call to TT² macro 'pic2' (make a thumbnail link)
#
TMP1=$(mktemp) || {
echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
exit 1
}
TMP2=$(mktemp) || {
echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
exit 1
}
TMP3=$(mktemp) || {
echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
exit 1
}
TMP4=$(mktemp) || {
echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
exit 1
}
trap 'cleanup_temp $TMP1 $TMP2 $TMP3 $TMP4' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
#
# Parse these fields from the raw input file (shownotes.txt) into Bash
# variables so we can make an index.html later.
#
# ----
# NOTE: We're moving to the JSON file for show info. This could be parsed with
# 'jq'. This filter seems reasonably close to our needs:
#
# command=$(jq -r '@sh "hostname=\(.host.Host_Name) title=\(.episode.Title) summary=\(.episode.Summary)"' $RAWFILE)
# eval ${command:1:-1}
#
# The '@sh' format thing ensures the output is OK for the shell.
# It seems 'jq' quotes the returned text (it's a string) which 'eval' doesn't
# like, but giving 'eval' the string without the quotes (${command:1:-1})
# makes this work.
# Update: adding the '-r' option removes the enclosing double quotes
# Use the '-f FILE' option to read the filter from a file.
# ----
#
# Title='' Summary='' Host_Name=''
# eval \
# "$(sed -n '/^\(Title\|Summary\|Host_Name\):/{s/^\([^:]\+\):\t/\1="/;s/$/"/;p}' "$RAWFILE")"
#
Title='' Summary='' Host_Name=''
jqprog="@sh \"Host_Name=\(.host.Host_Name) Title=\(.episode.Title) Summary=\(.episode.Summary)\""
command=$(jq -r "$jqprog" "$JSONFILE")
eval "${command}"
#
# Make two HTML templates and two TT² templates. We do this in dry-run mode
# too then throw them away!
#
# {{{
cat > "$TMP1" <<ENDHTML1
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="generator" content="pandoc">
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
<meta name="author" content="$Host_Name">
<title>$Title (HPR Show $1)</title>
<style type="text/css">code{white-space: pre;}</style>
<!--[if lt IE 9]>
<script src="https://html5shim.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->
<link rel="stylesheet" href="https://hackerpublicradio.org/css/hpr.css">
</head>
<body id="home">
<div id="container" class="shadow">
<header>
<h1 class="title">$Title (HPR Show $1)</h1>
<h3 class="summary">$Summary</h3>
<h3 class="author">$Host_Name</h3>
<hr/>
</header>
<main id="maincontent">
<article>
<h3 id="index">Index</h3>
<ul>
ENDHTML1
cat > "$TMP2" <<ENDHTML2
</ul>
</article>
</main>
</div>
</body>
</html>
ENDHTML2
cat > "$TMP3" <<'ENDPIC1'
[% row = piclist.$ind -%]
[% pic1("$prefix",'*Title*',"$row.file") -%]
ENDPIC1
cat > "$TMP4" <<'ENDPIC2'
[% row = piclist.$ind -%]
[% pic2("$prefix",'*Title*',"$row.thumb","$row.file") -%]
ENDPIC2
# }}}
#
# Read the previously created list of pictures into array 'pix'
#
if [[ -e $PICLIST ]]; then
declare -a pix
mapfile -t pix < "$PICLIST"
else
echo "$SCRIPT: ${red}No pictures found!${reset}"
echo "$SCRIPT: ${red}Show has no $PICLIST file!${reset}"
exit 1
fi
_DEBUG 'Pictures:' "${pix[@]}"
#
# The assets directory doesn't exist - something's wrong
#
if [[ ! -d $PICDIR ]]; then
echo "$SCRIPT: ${red}No picture/asset directory found!${reset}"
echo "$SCRIPT: ${red}Expected $PICDIR${reset}"
exit 1
fi
#
# Look for the show note format file
#
# Formats offered by the web form on the HPR site:
#
# Keyword Explanation
# ------- -----------
# plain_text Plain text
# html5 HTML5 (preferred)
# markdown_standard Markdown (standard)
# Markdown_GitHub Markdown (GitHub flavoured)
# Markdown_Pandoc Markdown (Pandoc flavoured)
# restructured_text RestructuredText
# txt2tags txt2tags
#
NOTEFORMAT=
if [[ -e $FORMAT ]]; then
NOTEFORMAT="$(cat "$FORMAT")"
else
echo "$SCRIPT: ${red}No format file found!${reset}"
echo "$SCRIPT: ${red}Expected $FORMAT${reset}"
exit 1
fi
#
# Regex to match 'plain_text' or any Markdown variant
#
FMTRE='^(plain_text|[Mm]arkdown_)'
_DEBUG "Format: $NOTEFORMAT"
# ----
# Check whether we're trying to run this script a second time. It will
# potentially try and make thumbnails of thumbnails if we do!
# ----
if [[ -e $MANIFEST || -e $PICTPL || $(grep -c pictures "$STATUS") -gt 0 ]]; then
echo "$SCRIPT: ${red}This doesn't seem to be the first run of this script!${reset}"
echo "$SCRIPT: ${yellow}Delete $MANIFEST, $PICTPL and any thumbnails " \
"and run it again - if you are certain about this!${reset}" | fmt -tu
echo "$SCRIPT: ${red}Exiting...${reset}"
exit 1
fi
#
# Start the TT² template and order the files (only for plain text and Markdown
# variant shows)
#
declare -a opix
if [[ $NOTEFORMAT =~ $FMTRE ]]; then
if [[ $DRYRUN -eq 0 ]]; then
echo "[%# Place at the top of the show notes -%]" > "$PICTPL"
echo -e "[% ind = 0 -%]\n" >> "$PICTPL"
else
_dryrun "would have initialised $PICTPL"
fi
#
# Get the order of the picture files, assuming they aren't in a usable sort
# order, and store them in 'opix'. The user can just type 1-10 (or whatever
# the count is) to use them in the presented order. The default prompt
# offers such an order for speed. They can abort the script at this point
# if they type CTRL-D.
#
_silent "${blue}Building ordered picture list${reset}" " "
if ! make_file_list pix opix; then
echo "Script aborted"
exit 1
fi
else
#
# Fill the 'opix' array, but we don't care about ordering
#
mapfile -t -d ' ' opix < <(echo -n "${pix[@]}")
fi
_DEBUG '(Possibly) re-ordered pictures:' "${opix[@]}"
#
# Convert the original pictures and make thumbnails if needed. Thumbnails get
# a name in the form 'NAME_thumbnail.EXT'. We record any thumbnails we create.
# In plain text mode we write details to a new file called '.pictures.mf'
# which is to be a manifest used by 'do_pandoc' to simplify the addition of
# Markdown picture and thumbnail links to the notes.
#
pcount=1
for pic in "${opix[@]}"; do
#
# The extension and name of the file we'll convert the original picture
# into. No path here. We replace internal spaces with underscores.
# NOTE: Space → underscore conversion has been done in parse_JSON
#
ext="${pic#*.}"
newpic="${pic##*/}"
# newpic="${newpic// /_}"
#
# What is this picture and does it need a thumbnail? Collect the type,
# width and height.
#
if [[ -e "$PICDIR/$pic" ]]; then
geo=$(gm identify -format "%m:%w:%h/" "$PICDIR/$pic")
RES=$?
[ $RES -ne 0 ] && { echo "Aborting!"; exit $RES; }
fi
#
# Remove repetitions for GIFs and extract the type
#
geo="${geo%%/*}"
type="${geo%%:*}"
#
# Special actions for GIFs, only needed if the notes are plain text or
# a Markdown variant
#
if [[ $type == 'GIF' ]]; then
echo "GIF detected"
# Cater for plain text and Markdown variant format notes
if [[ $NOTEFORMAT =~ $FMTRE ]]; then
if [[ $DRYRUN -eq 0 ]]; then
echo "$PICDIR/$newpic" >> "$MANIFEST"
echo "" >> "$MANIFEST"
if [[ $pcount -gt 1 ]]; then
echo "[% ind = ind + 1 -%]" >> "$PICTPL"
fi
# Add 'pic1' macro call
cat "$TMP3" >> "$PICTPL"
else
_dryrun "Would have added ${yellow}$PICDIR/$newpic${reset} to" \
"${yellow}$MANIFEST${reset}"
_dryrun "would have added a blank line to ${yellow}$MANIFEST${reset}"
_dryrun "Would have added the ${yellow}pic1${reset} macro to" \
"${yellow}$PICTPL${reset}"
fi
fi
#
# Nothing else needs to be done, so skip the rest of the loop
#
_silent "No image processing performed for $type file $pic"
echo "----"
((pcount++))
continue
else
#
# A simple image, get its size
#
geo="${geo#*:}"
pw="${geo%:*}"
ph="${geo#*:}"
if [[ $pw -gt $TNW || $ph -gt $TNH ]]; then
THUMB=1
tn="${newpic%.*}_thumbnail.$ext"
else
THUMB=0
tn=
fi
fi
#
# Include thumbnails or omit them
#
if [[ $THUMB -eq 1 ]]; then
_DEBUG 'Thumbnail:' "${newpic}" "${ext}" "${tn}"
#
# Convert originals to new path/file (or document it in dry-run mode)
#
if [[ $DRYRUN -eq 0 ]]; then
gm convert "$PICDIR/$pic" -strip -auto-orient "$PICDIR/$newpic"
gm convert "$PICDIR/$pic" -strip -auto-orient -resize $TNSIZE "$PICDIR/$tn"
#
# Only make a (new) picture list if we have plain text or
# a Markdown variant
#
if [[ $NOTEFORMAT =~ $FMTRE ]]; then
echo "$PICDIR/$newpic" >> "$MANIFEST"
echo "$PICDIR/$tn" >> "$MANIFEST"
if [[ $pcount -gt 1 ]]; then
echo "[% ind = ind + 1 -%]" >> "$PICTPL"
fi
# Add 'pic2' macro call
cat "$TMP4" >> "$PICTPL"
fi
#
# All thumbnails are new so need to be recorded
#
echo "$tn" >> "$PICLIST"
echo "$tn" >> "$ASSETS"
_silent "Converted picture ${yellow}$pic${reset} with thumbnail"
else
_dryrun \
"Would have converted ${yellow}$pic${reset} to " \
"${yellow}$newpic${reset} and ${yellow}$tn${reset} in " \
"${yellow}$(trimpath "$PICDIR" 3)${reset}"
#
# Only make a (new) picture list if we have plain text or
# a Markdown variant
#
if [[ $NOTEFORMAT =~ $FMTRE ]]; then
_dryrun "Would have added ${yellow}$PICDIR/$newpic${reset} to" \
"${yellow}$MANIFEST${reset}"
_dryrun "would have added ${yellow}$PICDIR/$tn${reset} to" \
"${yellow}$MANIFEST${reset}"
_dryrun "Would have added the ${yellow}pic2${reset} macro to" \
"${yellow}$PICTPL${reset}"
fi
echo "----"
fi
else
_DEBUG 'No thumbnail:' "${newpic}" "${ext}"
#
# Convert originals to new path/file (or document it)
#
if [[ $DRYRUN -eq 0 ]]; then
gm convert "$PICDIR/$pic" -strip -auto-orient "$PICDIR/$newpic"
#
# Only make a (new) picture list if we have plain text or
# a Markdown variant
#
if [[ $NOTEFORMAT =~ $FMTRE ]]; then
echo "$PICDIR/$newpic" >> "$MANIFEST"
echo "" >> "$MANIFEST"
if [[ $pcount -gt 1 ]]; then
echo "[% ind = ind + 1 -%]" >> "$PICTPL"
fi
# Add 'pic1' macro call
cat "$TMP3" >> "$PICTPL"
fi
_silent "Converted picture ${yellow}$pic${reset} without thumbnail"
else
_dryrun "Would have converted ${yellow}$pic${reset} to " \
"${yellow}$newpic${reset}"
#
# Only make a (new) picture list if we have plain text or
# a Markdown variant
#
if [[ $NOTEFORMAT =~ $FMTRE ]]; then
_dryrun "Would have added ${yellow}$PICDIR/$newpic${reset} to" \
"${yellow}$MANIFEST${reset}"
_dryrun "would have added a blank line to ${yellow}$MANIFEST${reset}"
_dryrun "Would have added the ${yellow}pic1${reset} macro to" \
"${yellow}$PICTPL${reset}"
fi
echo "----"
fi
fi
((pcount++))
done
#
# Final steps - not wanted in dry run mode
#
if [[ $DRYRUN -eq 0 ]]; then
#
# If we made thumbnails the .assets and .pictures files will have been
# updated. Clean these up here.
#
perl -i.BAK -ne 'print if ! $x{$_}++' "$PICLIST"
perl -i.BAK -ne 'print if ! $x{$_}++' "$ASSETS"
#
# Set the status for this show
#
echo "pictures" >> "$STATUS"
fi
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker

73
Show_Submission/do_repair Executable file
View File

@@ -0,0 +1,73 @@
#!/bin/bash -
#===============================================================================
#
# FILE: do_repair
#
# USAGE: ./do_repair <epno>
#
# DESCRIPTION: Run vim on the raw shownotes.txt file for a show and offer to
# upload it to the server
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.1
# CREATED: 2019-04-28 11:06:45
# REVISION: 2019-04-28 11:26:09
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
#DIR=${0%/*}
VERSION="0.0.1"
if [[ $# -ne 1 ]]; then
echo "[${SCRIPT} ${VERSION}]: Usage $SCRIPT shownumber"
exit
fi
BASENAME="/home/cendjm/HPR/Show_Submission"
SHOWDIR="$BASENAME/shownotes/hpr${1}"
SHOWNOTES="$SHOWDIR/shownotes.txt"
SNORIG="$SHOWDIR/shownotes.txt.orig"
ORIGIN="$SHOWDIR/.origin"
#
# Backup the original file
#
if [[ ! -e $SNORIG ]]; then
cp "$SHOWNOTES" "$SNORIG"
fi
upload_dir=$(cat "$ORIGIN")
MD5_1=$(md5sum "$SHOWNOTES")
#
# Edit the data from the form
#
vim "$SHOWNOTES"
RES=$?
if [[ $RES -eq 0 ]]; then
echo "Edited $SHOWNOTES ok"
MD5_2=$(md5sum "$SHOWNOTES")
if [[ $MD5_1 = "$MD5_2" ]]; then
echo "The file was not changed"
exit
else
echo "Copying $SHOWNOTES to upload/$upload_dir/shownotes.txt on the HPR server"
echo "Copying $SNORIG to upload/$upload_dir/shownotes.txt.orig on the HPR server"
scp -P 22074 "$SHOWNOTES" "$SNORIG" "hpr@hackerpublicradio.org:upload/$upload_dir/"
fi
else
echo "Oops! Something went wrong!"
fi
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21

308
Show_Submission/do_report Executable file
View File

@@ -0,0 +1,308 @@
#!/bin/bash -
# shellcheck disable=SC2317
#===============================================================================
#
# FILE: do_report
#
# USAGE: ./do_report [-h] [-D] [-m] [-s] [-Y] epno path_to_shownotes.json
#
# DESCRIPTION: Script to be invoked after a show has been processed to make
# a Matrix report.
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.8
# CREATED: 2022-09-07 15:27:29
# REVISION: 2023-06-01 17:58:09
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
#DIR=${0%/*}
# shellcheck disable=SC2034
VERSION="0.0.8"
STDOUT="/dev/fd/2"
#
# Load library functions
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
# shellcheck source=/home/cendjm/bin/function_lib.sh
source "$LIB"
#
# Colour codes
#
define_colours
#{{{ Functions: --- _usage --- _DEBUG --- _verbose --- _silent ---
#=== FUNCTION ================================================================
# NAME: _usage
# DESCRIPTION: Report usage
# PARAMETERS: None
# RETURNS: Nothing
#===============================================================================
_usage () {
cat >$STDOUT <<-endusage
Usage: ./${SCRIPT} [-h] [-D] [-m] [-s] [-Y] shownumber
Version: $VERSION
Script to create and send a Matrix message about the processing of the show
Options:
-h Print this help
-D Run in debug mode where a lot more information is
reported
-m Monochrome mode - no colours
-s Silent mode, output less text about actions
Arguments:
shownumber
Examples
./${SCRIPT} -h
./${SCRIPT} -m 3112
./${SCRIPT} -D 3112
./${SCRIPT} 3112
endusage
exit
}
#=== FUNCTION ================================================================
# NAME: _DEBUG
# DESCRIPTION: Writes one or more messages if in DEBUG mode. Each argument is
# seen as a message and is written on a separate line.
# References the global variable 'DEBUG' which is expected to be
# True if debug output is wanted.
# PARAMETERS: List of messages
# RETURNS: Nothing
#===============================================================================
_DEBUG () {
[ "$DEBUG" == 0 ] && return
for msg in "$@"; do
printf 'D> %s\n' "$msg"
done
}
#=== FUNCTION ================================================================
# NAME: _verbose
# DESCRIPTION: Writes a message in verbose mode
# PARAMETERS: $1 message
# RETURNS: Nothing
#===============================================================================
_verbose () {
[ "$VERBOSE" -eq 0 ] && return
for msg in "$@"; do
printf '%s\n' "$msg"
done
}
#=== FUNCTION ================================================================
# NAME: _silent
# DESCRIPTION: Writes a message unless in silent mode
# PARAMETERS: $1 message
# RETURNS: Nothing
#===============================================================================
_silent () {
[ "$SILENT" -eq 1 ] && return
for msg in "$@"; do
printf '%s\n' "$msg"
done
}
#}}}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Paths to files
#
BASENAME="$HOME/HPR/Show_Submission"
TPL="$BASENAME/shownotes/hpr%d/%s"
Q2CSV="$BASENAME/query2csv"
[ -e "$Q2CSV" ] || { echo "Unable to find '$Q2CSV'; aborting"; exit 1; }
#
# Option defaults
#
COLOUR=1 # use colours by default
SILENT=0 # not silent by default
#
# Process options
#
while getopts :hDmsY opt
do
case "${opt}" in
h) _usage;;
D) DEBUG=1;;
m) COLOUR=0;;
s) SILENT=1;;
?) echo "$SCRIPT: Invalid option; aborting"; exit 1;;
esac
done
shift $((OPTIND - 1))
DEBUG=${DEBUG:-0}
SILENT=${SILENT:-0}
#
# Cancel colours if requested
#
if [[ $COLOUR -eq 0 ]]; then
undefine_colours
fi
#
# Check the argument after any options
#
if [[ $# -ne 1 ]]; then
echo "$SCRIPT: ${red}Usage: $SCRIPT shownumber${reset}"
exit 1
fi
show="$1"
# json="$2"
#
# Compute paths to show-specific files
#
# shellcheck disable=SC2059
{
printf -v json "$TPL" "$show" "shownotes.json"
printf -v assetfile "$TPL" "$show" ".assets"
printf -v statusfile "$TPL" "$show" ".status"
_DEBUG "Path to json = $json"
}
#
# Simplify checks
#
if [[ ! -e $json ]]; then
echo "$SCRIPT: ${red}Unable to find $json${reset}"
exit 1
fi
#-------------------------------------------------------------------------------
# Get the show details
#-------------------------------------------------------------------------------
#
# Extract JSON data and make Bash assignments which are then processed
# with 'eval'.
# Have to declare variables to avoid upsetting Shellcheck
#
declare host hostid email format
jqscript='host=\(.host.Host_Name) hostid=\(.host.Host_ID) '
jqscript+='email=\(.host.Host_Email) format=\(.metadata.POST.shownotes_format)'
commands=$(jq -r "@sh \"$jqscript\"" "$json")
eval "${commands}"
#
# The zero hostid needs checking
#
if [ "$hostid" -eq 0 ]; then
_silent "${yellow}Checking host id 0 is valid${reset}"
# Look up in database
hid=$($Q2CSV "select hostid from hosts where host like '%${host}%'")
# Use the host id we found if the zero id is wrong
if [[ -n $hid ]]; then
_silent "${yellow}Found the host name $host with id $hid${reset}"
hostid=$hid
newhost=""
email=" (using $email)"
else
newhost="new host "
email=" ($email)"
fi
else
newhost=""
email=""
fi
#
# If the hostid is zero the email wasn't known (so maybe a new host) and we
# didn't find the name in the database (so treat them as new). We only report
# the email if it's a known host (by name) using a new address or if it's
# a new host.
#
# if [ "$hostid" -eq 0 ]; then
# newhost="new host "
# email=" ($email)"
# else
# newhost=""
# email=""
# fi
#
# If there are assets collect their names
# NOTE: now not used except as a non-blank string
#
if [[ -e $assetfile ]]; then
# The sed expression confuses ShellCheck
# shellcheck disable=SC2016
assets="$(sort "$assetfile" | sed -ne 'H;${x;s/\n//;s/\n/, /g;p}')"
else
assets=
fi
#
# Report the settings in debug mode
#
_DEBUG "Show number = $show" \
"Host name = $host" \
"Host ID = $hostid" \
"Host email = $email" \
"Assets = $assets"
#
# Handle backticks in the host string (Rho`n/Roan made me do it!)
#
if grep -q -E '`' <<<"$host"; then
# shellcheck disable=SC2001 disable=SC2016
host=$(sed -e 's/^\([0-9A-Za-z_`-]\+\)$/`\1`/' <<<"$host")
fi
#
# Generate the message we want to send
#
# shellcheck disable=SC2016
printf -v message 'Processed %s from %s`%s`%s. Format is *%s*.' \
"$show" "$newhost" "$host" "$email" "$format"
if [[ -n $assets ]]; then
# We have assets but were they sent?
if grep -q -E '^assets' "$statusfile"; then
message+=" Assets uploaded"
else
_silent "${yellow}Note: assets were found but not uploaded${reset}"
fi
fi
#
# Send it, after checking
#
echo "Will run the following command:"
echo "${green}matrix-commander -z -m '$message'${reset}"
if yes_no 'OK to proceed? %s ' 'No'; then
command="matrix-commander -z -m '$message'"
eval "$command" || \
{ echo "Failed to invoke the command!"; exit 1; }
#
# Change state/log what we did, but only if we actually did it
#
echo "reported" >> "$statusfile" || \
{ echo "Failed to update $statusfile"; exit 1; }
fi
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker

1
Show_Submission/do_reserve Symbolic link
View File

@@ -0,0 +1 @@
do_state_change

87
Show_Submission/do_show Executable file
View File

@@ -0,0 +1,87 @@
#!/bin/bash -
#===============================================================================
#
# FILE: do_show
#
# USAGE: ./do_show <epno>
#
# DESCRIPTION: Displays the raw show details of a given show. To be run after
# 'sync_hpr' and 'copy_shownotes'. The resulting file will be
# called 'shownotes.json' in a directory called 'hpr${epno}'.
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: The target file used to be called 'hpr${epno}_shownotes.txt'
# but we simplified it so we can use 'rsync' for copying files.
# 2022-12-17: stoped referring to this old file, since now we
# only use shownotes.json.
# 2023-01-07: Refactored to avoid nested tests (though I don't
# find them confusing in this script).
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.6
# CREATED: 2016-12-15 09:51:09
# REVISION: 2023-01-07 14:51:41
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
#DIR=${0%/*}
VERSION='0.0.6'
#
# Check the argument after any options
#
if [[ $# -ne 1 ]]; then
echo "$SCRIPT ($VERSION); Usage: $SCRIPT shownumber"
exit
fi
#
# Paths to files
#
BASENAME="$HOME/HPR/Show_Submission"
SHOWDIR="$BASENAME/shownotes/hpr${1}"
FROM="$SHOWDIR/shownotes.json"
PLACEHOLDER="$SHOWDIR/.dummy"
#
# Not a show, just a placeholder
#
if [[ -e $PLACEHOLDER ]]; then
echo "$SCRIPT: This directory contains a placeholder only; aborting"
exit 1
fi
#
# The JSON file isn't there
#
if [[ ! -e $FROM ]]; then
echo "$SCRIPT: File not found: $FROM"
exit 1
fi
#
# File is there but empty!
#
if [[ ! -s $FROM ]]; then
echo "$SCRIPT: File $FROM is empty"
exit 1
fi
#
# Display a subset of the JSON
#
jqprog="{ host: .host, episode: .episode, "
jqprog+="format: .metadata.POST.shownotes_format, "
jqprog+="series: .metadata.POST.series, "
jqprog+="media: .metadata.FILES.media_files.name }"
jq -C "$jqprog" "$FROM" | less -R
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21

302
Show_Submission/do_state_change Executable file
View File

@@ -0,0 +1,302 @@
#!/bin/bash -
#===============================================================================
#
# FILE: do_state_change
#
# USAGE: ./do_state_change [-h] [-D] [-m] [-s] [-Y] <epno>
#
# DESCRIPTION: Script to be invoked via symlinks to perform related tasks
# relating to show states.
#
# - When called as 'do_reserve' toggles a '.reserved' marker.
# - When called as 'do_block' creates the directory if necessary
# and makes a dummy 'shownotes.json' as well as a file called
# '.dummy'. Doesn't delete any of this since other software
# will tidy things.
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.6
# CREATED: 2021-06-05 22:04:26
# REVISION: 2024-02-06 15:36:02
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
#DIR=${0%/*}
# shellcheck disable=SC2034
VERSION="0.0.6"
STDOUT="/dev/fd/2"
#
# Load library functions
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
# shellcheck source=/home/cendjm/bin/function_lib.sh
source "$LIB"
#
# Colour codes
#
define_colours
#=== FUNCTION ================================================================
# NAME: _usage
# DESCRIPTION: Report usage
# PARAMETERS: None
# RETURNS: Nothing
#===============================================================================
_usage () {
cat >$STDOUT <<-endusage
Usage: ./${SCRIPT} [-h] [-D] [-m] [-s] [-Y] shownumber
Version: $VERSION
Script to be invoked via symlinks to perform related tasks relating to show states
Options:
-h Print this help
-D Run in debug mode where a lot more information is
reported
-m Monochrome mode - no colours
-s Silent mode, output less text about actions
-Y Assume 'Yes' in answer to the prompt
Arguments:
shownumber
Examples
./${SCRIPT} -h
./${SCRIPT} -m 3112
./${SCRIPT} -D 3112
./${SCRIPT} 3112
./${SCRIPT} -Y 3112
endusage
exit
}
#=== FUNCTION ================================================================
# NAME: exists
# DESCRIPTION: Determines wheher all paths given as arguments exist
# PARAMETERS: List of paths
# RETURNS: True if they all exist, otherwise false
#===============================================================================
exists () {
for path in "$@"; do
if [[ ! -e $path ]]; then
return 1
fi
done
return
}
#=== FUNCTION ================================================================
# NAME: _DEBUG
# DESCRIPTION: Writes one or more messages if in DEBUG mode. Each argument is
# seen as a message and is written on a separate line.
# References the global variable 'DEBUG' which is expected to be
# True if debug output is wanted.
# PARAMETERS: List of messages
# RETURNS: Nothing
#===============================================================================
_DEBUG () {
[ "$DEBUG" == 0 ] && return
for msg in "$@"; do
printf 'D> %s\n' "$msg"
done
}
#=== FUNCTION ================================================================
# NAME: _verbose
# DESCRIPTION: Writes a message in verbose mode
# PARAMETERS: $1 message
# RETURNS: Nothing
#===============================================================================
_verbose () {
[ "$VERBOSE" -eq 0 ] && return
for msg in "$@"; do
printf '%s\n' "$msg"
done
}
#=== FUNCTION ================================================================
# NAME: _silent
# DESCRIPTION: Writes a message unless in silent mode
# PARAMETERS: $1 message
# RETURNS: Nothing
#===============================================================================
_silent () {
[ "$SILENT" -eq 1 ] && return
for msg in "$@"; do
printf '%s\n' "$msg"
done
}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Paths to files
#
BASENAME="$HOME/HPR/Show_Submission"
TPL="$BASENAME/shownotes/hpr%d/%s"
#
# Option defaults
#
COLOUR=1 # use colours by default
YES=0 # prompt for yes/no by default
TOGGLE=0 # marker can't be removed by default
#
# Process options
#
while getopts :hDmsY opt
do
case "${opt}" in
h) _usage;;
D) DEBUG=1;;
m) COLOUR=0;;
s) SILENT=1;;
Y) YES=1;;
?) echo "$SCRIPT: Invalid option; aborting"; exit 1;;
esac
done
shift $((OPTIND - 1))
DEBUG=${DEBUG:-0}
SILENT=${SILENT:-0}
#
# Cancel colours if requested
#
if [[ $COLOUR -eq 0 ]]; then
undefine_colours
fi
#
# Determine actions depending on how the script was called. Save filenames in
# an array.
#
declare -a MK MFILES
case "${SCRIPT#*/}" in
do_reserve)
MK+=('.reserved')
state='reserved'
TOGGLE=1
;;
do_block)
echo "Blocking of pending shows is not needed any more."
echo "See Journal for details (2024-02-06)"
exit
# MK+=('shownotes.json')
# MK+=('.dummy')
# state='blocked'
# TOGGLE=0
;;
*)
echo "Don't call this script directly; use one of its soft links"
echo "D> $0 $*"
exit 1
;;
esac
#
# Check the argument after any options
#
if [[ $# -ne 1 ]]; then
echo "$SCRIPT: ${red}Usage: $SCRIPT shownumber${reset}"
exit
fi
show="$1"
#
# I used 'hpr4065' by mistake today, so we need to check the show specification
#
if [[ $show =~ ^(hpr)?([0-9]+)$ ]]; then
printf -v show '%04d' "${BASH_REMATCH[2]}"
else
echo "$SCRIPT: ${red}Invalid show specification: $show${reset}"
exit
fi
#
# Build the path(s) to the marker(s) in an array
#
# shellcheck disable=SC2059
for fn in "${MK[@]}"; do
printf -v path "$TPL" "$show" "$fn"
MFILES+=("$path")
done
#
# Path to the show directory
#
SHOWDIR="$BASENAME/shownotes/hpr${show}"
#
# Report the settings in debug mode
#
_DEBUG "Called as = $0" \
"Show number = $show" \
"MK = ${MK[*]}" \
"state = $state" \
"TOGGLE = $TOGGLE" \
"MFILES = ${MFILES[*]}" \
"SHOWDIR = $SHOWDIR"
#
# We need a show directory. If it doesn't exist then we'll create it because
# other scripts will use it.
#
if [[ ! -d $SHOWDIR ]]; then
echo "${red}There is no directory for show $show${reset}"
#
# If the -Y option was not chosen ask with 'yes_no'. It -Y was chosen
# we're to go ahead regardless. This relies on the fact that Bash
# "short-circuits" logical expressions like this.
#
if [[ $YES -eq 1 ]] || yes_no 'Create directory? %s ' 'N'; then
mkdir "$SHOWDIR"
_silent "${green}Directory created for show $show${reset}"
else
_silent "${yellow}Not changed${reset}"
fi
fi
#
# If the marker exists and we can toggle it, we'll do so. If no toggle, we
# report the marker presence or set it as appropriate.
#
if exists "${MFILES[@]}"; then
if [[ $TOGGLE -eq 1 ]]; then
_silent "${yellow}Show $show has a '$state' marker${reset}"
if [[ $YES -eq 1 ]] || yes_no 'Remove marker? %s ' 'N'; then
rm -f "${MFILES[@]}"
_silent "${green}Removed '$state' marker for show $show${reset}"
else
_silent "${yellow}Not changed${reset}"
fi
else
echo "${red}Show $show is already marked '$state'${reset}"
fi
else
_silent "${yellow}Show $show has no '$state' marker${reset}"
touch "${MFILES[@]}"
_silent "${green}Marked show $show as '$state'${reset}"
fi
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21

227
Show_Submission/do_update_reservations Executable file
View File

@@ -0,0 +1,227 @@
#!/bin/bash -
#===============================================================================
#
# FILE: do_update_reservations
#
# USAGE: ./do_update_reservations <epno>
#
# DESCRIPTION: Script to update the status in the 'reservations' table after
# a show has been processed.
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.6
# CREATED: 2022-04-11 09:36:21
# REVISION: 2023-06-14 23:24:42
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
#DIR=${0%/*}
# shellcheck disable=SC2034
VERSION="0.0.6"
STDOUT="/dev/fd/2"
#
# Load library functions
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
# shellcheck source=/home/cendjm/bin/function_lib.sh
source "$LIB"
#
# Colour codes
#
define_colours
#{{{ Functions: --- _usage ---
#=== FUNCTION ================================================================
# NAME: _usage
# DESCRIPTION: Report usage
# PARAMETERS: None
# RETURNS: Nothing
#===============================================================================
_usage () {
cat >$STDOUT <<-endusage
Usage: ./${SCRIPT} [-h] [-d] [-m] shownumber
Version: $VERSION
Script to update the status in the 'reservations' table after a show has been
processed.
Options:
-h Print this help
-d Dry-run mode. Reports what it will do but doesn't do it
-m Monochrome mode - no colours
Arguments:
shownumber
Examples
./${SCRIPT} -h
./${SCRIPT} -m 3112
./${SCRIPT} -d 3112
./${SCRIPT} -dm 3112
./${SCRIPT} 3112
endusage
exit
}
#}}}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Paths to files
#
BASENAME="$HOME/HPR/Show_Submission"
#
# Tools
#
BASECOM='curl -K ./.hpradmin_curlrc -s'
URL="https://hub.hackerpublicradio.org/cms/status.php"
QUERY="${BASECOM} ${URL}"
#
# Option defaults
#
COLOUR=1 # use colours by default
DRYRUN=0 # live mode by default
#
# Process options
#
while getopts :hdm opt
do
case "${opt}" in
h) _usage;;
d) DRYRUN=1;;
m) COLOUR=0;;
?) echo "$SCRIPT: Invalid option; aborting"; exit 1;;
esac
done
shift $((OPTIND - 1))
#
# Cancel colours if requested
#
if [[ $COLOUR -eq 0 ]]; then
undefine_colours
fi
#
# Check the argument after any options
#
if [[ $# -ne 1 ]]; then
echo "$SCRIPT: ${red}Usage: $SCRIPT shownumber${reset}"
exit
fi
epno="$1"
SHOWDIR="$BASENAME/shownotes/hpr${epno}"
STATUSFILE="$SHOWDIR/.status"
#
# Check the show in question exists
#
if [[ ! -d $SHOWDIR ]]; then
echo "$SHOWDIR not found, can't continue"
exit 1
fi
#
# Check the show is in the correct state of local processing
#
declare -a states
states+=( '+dir' )
if [[ -e "$SHOWDIR/shownotes.json" ]]; then
states+=( '+shownotes' )
fi
if [[ -e "$SHOWDIR/hpr${epno}.html" ]]; then
states+=( '+processed' )
fi
if [[ -e "$SHOWDIR/.uploaded" ]]; then
states+=( '+uploaded' )
fi
echo "${green}Current state: ${states[*]}${reset}"
if ! grep -q -E '\+uploaded' <<<"${states[@]}"; then
echo "The show is not in the +uploaded state"
exit 1
fi
#
# Run the query with 'curl' and filter out this episode
#
reply="$(${QUERY} | grep -E "^[^,]+,$epno" )"
# echo "D> $reply"
#
# If we found the episode in the list we need to test further
#
if [[ -n $reply ]]; then
echo "Found $epno in 'reservations'"
#
# Get the current state in the database by parsing the line returned.
# Since Ken changed this out of the blue we now have:
# 1. Epoch timestamp
# 2. Episode number
# 3. Episode date
# 4. Key
# 5. Status
# 6. Email
#
if [[ $reply =~ ^([^,]+),([^,]+),([^,]+),([^,]+),([^,]+),.*$ ]]; then
state="${BASH_REMATCH[5]}"
#
# If it's aready in the desired state we're finished, otherwise we'll
# set the new state
#
if [[ $state == 'METADATA_PROCESSED' ]]; then
echo "Already marked ${blue}${state}${reset}"
else
echo "Ready for marking"
command="${BASECOM} ${URL}?ep_num=${epno}&status=METADATA_PROCESSED"
if [[ $DRYRUN -eq 1 ]]; then
echo -e "Dry-run: would have run\n${yellow}$command${reset}"
else
echo "${yellow}$command${reset}"
$command
#
# Change state/log what we did
#
echo "database" >> "$STATUSFILE" || \
{ echo "Failed to update $STATUSFILE"; exit 1; }
fi
fi
else
echo "Couldn't parse '$reply'; aborting"
exit 1
fi
else
#
# We have no record of this show locally
#
echo "Not found $epno"
fi
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker

390
Show_Submission/do_upload Executable file
View File

@@ -0,0 +1,390 @@
#!/bin/bash -
#===============================================================================
#
# FILE: do_upload
#
# USAGE: ./do_upload <epno> [files...]
#
# DESCRIPTION: Uploads the processed HTML back to the HPR server. Also
# capable of uploading supplementary files such as a missed
# audio file, pictures, etc.
#
# There are two main types of uploads:
# 1) Extracted and edited show notes to go back to the server as
# HTML. Possiblly altered 'shotnotes.txt' as a means of
# fixing errors in the title, summary or tags.
# 2) Assets collected by 'parse_JSON' and possibly changed by
# 'do_pictures' if there are pictures provided and referenced
# in the notes. Any archives will have had files extracted so
# that there is much less to do on the server.
#
# Assets are sent to a directory under '~hpr/www/eps/' named
# 'hprNNNN'. This is created here and files copied with 'rsync'.
# There is a local directory in the show directory which is
# called 'uploads'. Unless the host has provided archives to be
# downloaded with their show we will not send such files to the
# server. We ask that such files be enclosed in an archive when
# sent - so we never extract the archives we have extracted from
# the sent archives!
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: 2021-03-15: Added code to place the "assets" in the final
# directory on the server. This will be 'www/eps/hprNNNN/'.
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.1.2
# CREATED: 2017-03-06 19:11:51
# REVISION: 2023-06-14 23:12:04
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
# DIR=${0%/*}
VERSION="0.1.2"
STDOUT="/dev/fd/2"
#
# Load library functions
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "$SCRIPT: ${red}Unable to source functions${reset}"; exit 1; }
# shellcheck source=/home/cendjm/bin/function_lib.sh
source "$LIB"
#
# Colour codes
#
define_colours
# {{{ --- Functions: usage, dryrun ---
#=== FUNCTION ================================================================
# NAME: _usage
# DESCRIPTION: Report usage
# PARAMETERS: None
# RETURNS: Nothing
#===============================================================================
_usage () {
cat >$STDOUT <<-endusage
Usage: ./${SCRIPT} [-h] [-a] [-d] [-D] shownumber [file1 file2...]
Version: $VERSION
Uploads the processed HTML back to the HPR server. Also
capable of uploading supplementary files such as a missed
audio file.
Options:
-h Print this help
-a Send any assets to the show sub-directory on the HPR
server
-d Select dry run mode
-D Select debug mode (works the same; more output)
Arguments:
shownumber
list of files to be uploaded in addition
Examples
./${SCRIPT} -h
./${SCRIPT} -d 3099
./${SCRIPT} -d 3123 otherfile1.txt otherfile2.dat
endusage
exit
}
#=== FUNCTION ================================================================
# NAME: _dryrun
# DESCRIPTION: Output a dry run message
# PARAMETERS: List of messages
# RETURNS: Nothing
#===============================================================================
_dryrun () {
for msg in "$@"; do
printf 'Dry run: %s\n' "$msg"
done
}
#=== FUNCTION ================================================================
# NAME: trimpath
# DESCRIPTION: Trim a file path to its last few components for display
# PARAMETERS: $1 - path to trim
# $2 - number of elements to retain
# RETURNS: Trimmed path
#===============================================================================
# trimpath () {
# local path=${1:?Usage: mpath path elements}
# local elements=${2:?Usage: mpath path elements}
# local -a arr1 arr2
#
# # Split the path
# # IFS='/' arr1=($path)
# IFS='/' mapfile -d'/' -t arr1 <<<"$path"
#
# if [[ ${#arr1[@]} -gt $elements ]]; then
# # Put the last elements in another array
# # arr2=(${arr1[@]: -$elements})
# mapfile -t arr2 < <(printf '%s\n' "${arr1[@]: -$elements}")
#
# # return the second array interleaved with '/'
# echo "${arr2[*]/#/}"
# else
# echo "$path"
# fi
# }
# }}}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Option defaults
#
ASSETUPLOAD=0
DEBUG=0
DRYRUN=0
#
# Process options
#
while getopts :ahdD opt
do
case "${opt}" in
a) ASSETUPLOAD=1;;
h) _usage;;
d) DRYRUN=1;;
D) DEBUG=1;;
?) echo "$SCRIPT: Invalid option; aborting"; exit 1;;
esac
done
shift $((OPTIND - 1))
#
# Check there's at least one argument after removing any options. Abort if not
#
if [[ $# -eq 0 ]]; then
echo "Missing argument(s)"
_usage
fi
#
# Paths to files
#
# 2021-02-12 Not using the .format file any more
BASEDIR="$HOME/HPR/Show_Submission"
LOCAL_UPLOAD="$BASEDIR/upload"
SHOWDIR="$BASEDIR/shownotes/hpr${1}"
FROM="$SHOWDIR/hpr${1}.html"
# FORMAT="$SHOWDIR/.format"
ORIGIN="$SHOWDIR/.origin"
UPLOADED="$SHOWDIR/.uploaded"
STATUSFILE="$SHOWDIR/.status"
# If there are HTML errors
REPORT="$SHOWDIR/errors.txt"
# For differences
# 2021-02-12 $RE removed since we find differences for all formats
RAWNOTES="$SHOWDIR/hpr${1}.orig"
PROCESSED="$SHOWDIR/hpr${1}.out"
# RE='plain_text|markdown_standard|Markdown_GitHub|Markdown_Pandoc|restructured_text|txt2tags'
DIFFFILE="$SHOWDIR/hpr${1}.diff"
# PORT=22074
PORT=22
#
# Asset locations for upload
#
# If there are supplementary files
# SUPPDIR="$SHOWDIR/hpr${1}"
TARFILE="$SHOWDIR/hpr${1}.tgz"
PICLIST="$SHOWDIR/.pictures"
ASSETLIST="$SHOWDIR/.assets"
# Where any "assets" (pictures, scripts, etc) come from and will go on the server
LOCAL_ASSETDIR="$SHOWDIR/uploads"
REMOTE_ASSETDIR="www/eps/hpr${1}"
CMDTPL='ssh hpr@hackerpublicradio.org -p %d %s'
# Use to build the rsync command, assuming the tunnel is open:
# rsync -a -e 'ssh -p 22' $SHOWDIR/hpr3656/ hpr@hpr:www/eps/hpr3656/
#
RSYNCTPL="rsync -a -e 'ssh -p %d' %s hpr@hpr:www/eps/%s"
# Did the following by hand for uploading
# ssh -p $PORT hpr@hpr 'mkdir ~/www/eps/hpr3685'
# scp -P $PORT shownotes/hpr3685/Budget_sample_2022-9-9.ods hpr@hpr:www/eps/hpr3685/
EXTRA=0
#
# Is there a marker to indicate the show has already been uploaded? This
# isn't an error, but a warning is in order.
#
if [[ -e $UPLOADED ]]; then
echo "$SCRIPT: ${yellow}Warning - the notes have already been uploaded${reset}"
fi
#
# Check we actually have a file to upload
#
if [[ ! -e $FROM ]]; then
echo "$SCRIPT: ${red}File not found: ${FROM}${reset}"
exit 1
fi
#
# If there's an error report then add it to the list of supplementary files
# (which may be none)
#
if [[ -e $REPORT ]]; then
_DEBUG "Error report being added"
set -- "$@" "${REPORT##*/}"
fi
#
# Deal with the raw notes extracted by 'parse_shownotes' and saved in
# hprXXXX.orig and the edited notes in hprXXXX.out
#
# 2021-02-12 Now using the same code for all formats
#
# if [[ -e $RAWNOTES && -e $PROCESSED ]]; then
# FMT="$(cat "$FORMAT")"
# if [[ $FMT =~ $RE ]]; then
# diff "$RAWNOTES" "$PROCESSED" > "$DIFFFILE"
# if [[ -s $DIFFFILE ]]; then
# set -- "$@" "${DIFFFILE##*/}"
# else
# echo "$SCRIPT: ${red}No differences found${reset}"
# fi
# fi
# fi
if [[ -e $RAWNOTES && -e $PROCESSED ]]; then
_DEBUG "Differences being determined:" "$RAWNOTES vs $PROCESSED"
diff "$RAWNOTES" "$PROCESSED" > "$DIFFFILE"
if [[ -s $DIFFFILE ]]; then
_DEBUG "Differences found"
set -- "$@" "${DIFFFILE##*/}"
else
_DEBUG "Differences file is empty"
echo "$SCRIPT: ${red}No differences found${reset}"
fi
fi
#
# If there's a picture list and a tarfile has been made add the latter to the
# upload list
# TODO: Not needed in next iteration
#
if [[ -e $PICLIST && -e $TARFILE ]]; then
_DEBUG "Tar file being added"
set -- "$@" "${TARFILE##*/}"
fi
#
# Check any supplementary files in the argument list (added by the user or
# this script)
#
if [[ $# -gt 1 ]]; then
EXTRA=1
shift # Delete argument 1, the show number
for arg; do
if [[ ! -e "$SHOWDIR/$arg" ]]; then
echo "$SCRIPT: ${red}File missing: $SHOWDIR/$arg${reset}"
echo "Can't continue"
exit 1
fi
done
echo "$SCRIPT: ${blue}Number of supplementary files to upload: $#${reset}"
fi
#
# Check we have a record of where the files are to go
#
if [[ ! -e $ORIGIN ]]; then
echo "$SCRIPT: ${red}Unable to find the .origin file${reset}"
echo "(This holds the directory on the server where the files are to be sent)"
exit
fi
#
# Check there's a local copy of the upload directory. If not, then the rsync
# run has deleted it because the directory on the server has been deleted.
# This implies the show has been processed on the server.
#
upload_dir="$(cat "$ORIGIN")"
_DEBUG "upload_dir = $upload_dir"
if [[ ! -e "$LOCAL_UPLOAD/$upload_dir" ]]; then
echo "$SCRIPT: ${red}The upload directory seems to have been deleted${reset}"
echo "(This happens after the show is processed on the server)"
exit
fi
#
# Upload the processed show notes
#
if [[ $DRYRUN -eq 0 ]]; then
echo "Copying $FROM to upload/$upload_dir/shownotes.html on the HPR server"
scp -P $PORT "$FROM" "hpr@hackerpublicradio.org:upload/$upload_dir/shownotes.html"
RES=$?
if [[ $RES -eq 0 ]]; then
echo "$SCRIPT: ${green}Uploaded successfully${reset}"
touch "$UPLOADED"
else
echo "$SCRIPT: ${red}Oops! Something went wrong!${reset}"
echo "$SCRIPT: Aborting now"
exit 1
fi
else
_dryrun "would have copied $FROM to the server"
fi
#
# Handle supplementary files if there are any. The remaining arguments are
# these file names without any path information. Variable 'EXTRA' is
# true/false if there are/are not extra arguments. If there are we have
# already deleted the first argument and we've checked the existence of the
# file(s).
#
if [[ $EXTRA -eq 1 ]]; then
_DEBUG "Uploading supplementary files"
for arg; do
FROM="$SHOWDIR/$arg"
if [[ $DRYRUN -eq 0 ]]; then
echo "Copying $FROM to upload/$upload_dir/$arg on the HPR server"
scp -P $PORT "$FROM" "hpr@hackerpublicradio.org:upload/$upload_dir/$arg"
RES=$?
if [[ $RES -eq 0 ]]; then
echo "$SCRIPT: ${green}Uploaded ${arg} successfully${reset}"
else
echo "$SCRIPT: ${red}Oops! Something went wrong with ${arg}!${reset}"
echo "$SCRIPT: Aborting now"
exit 1
fi
else
_dryrun "would have copied $FROM to the server"
fi
done
fi
#
# Update the status file
#
if [[ $DRYRUN -eq 0 ]]; then
echo "uploaded" >> "$STATUSFILE" || \
{ echo "Failed to update $STATUSFILE"; exit 1; }
fi
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker

170
Show_Submission/do_vim Executable file
View File

@@ -0,0 +1,170 @@
#!/bin/bash -
#===============================================================================
#
# FILE: do_vim
#
# USAGE: ./do_vim <epno>
#
# DESCRIPTION: Run vim on notes for a show
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: The validate_html script failed because the HTML::Valid module
# wasn't visible but the edit went ahead regardless. How to
# handle such errors cleanly? Not found an answer yet.
# 2022-12-22: Added code to write to the .status file
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.8
# CREATED: 2016-05-14 14:21:14
# REVISION: 2022-12-22 17:08:22
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
#DIR=${0%/*}
VERSION="0.0.8"
if [[ $# -ne 1 ]]; then
echo "[${SCRIPT} ${VERSION}] Usage: $SCRIPT shownumber"
exit
fi
BASENAME="$HOME/HPR/Show_Submission"
SHOWDIR="$BASENAME/shownotes/hpr${1}"
EXTRACT="$SHOWDIR/hpr${1}.out"
FORMAT="$SHOWDIR/.format"
VALIDATE="$BASENAME/validate_html"
ERRORS="$SHOWDIR/hpr${1}.err"
REPORT="$SHOWDIR/errors.txt"
PICTPL="$SHOWDIR/.pictures.tt2"
STATUSFILE="$SHOWDIR/.status"
#
# Sanity checks
#
[ -e "$VALIDATE" ] || {
echo "$SCRIPT: HTML validator '$VALIDATE' not found"
exit 1
}
if [[ ! -e $EXTRACT ]]; then
echo "$SCRIPT: File not found: $EXTRACT"
exit
fi
fmt=
#
# Set the Vim command from the declared format
#
if [[ ! -e $FORMAT ]]; then
#
# Default
#
vimcom=':set filetype=markdown'
else
#
# Define filetypes that Vim knows based on the names that arrive in the
# form. All need work other than 'txt2tags'
#
fmt="$(cat "$FORMAT")"
case $fmt in
html5)
fmt='html';;
Markdown_GitHub | Markdown_Pandoc | markdown_standard | plain_text)
fmt='markdown';;
restructured_text)
fmt='rst';;
txt2tags)
;; # No action needed
*)
fmt='markdown';;
esac
vimcom=":set filetype=$fmt"
fi
#
# For html5 (now called 'html') run a validator on it. If this generates
# errors run Vim with a Quickfix window on the error file so the errors can be
# spotted and fixed.
#
if [[ $fmt == 'html' ]]; then
#
# The validator exits with False if there are errors
#
if ! $VALIDATE "$EXTRACT" > "$ERRORS"; then
#
# Did the validation script crash?
#
RES=$?
if [[ $RES -ne 0 ]]; then
echo "Problem running $VALIDATE; Edit aborted"
exit
fi
#
# HTML::Valid can put unwanted stuff in the error file. Not sure why.
# We remove it with 'sed'. Only lines beginning with the path of the
# input file are wanted.
#
# shellcheck disable=SC1117
sed -i -ne "\#^$EXTRACT#p" "$ERRORS"
#
# Make an error file for reporting to the host, but only if it doesn't
# already exist. Start with the HTML itself with line numbers and
# follow with the error report with the file paths truncated.
#
if [[ ! -e $REPORT ]]; then
nl -ba -w3 -s': ' "$EXTRACT" > "$REPORT"
sed -e "s:^$BASENAME/::" "$ERRORS" >> "$REPORT"
fi
#
# Run Vim on the errors with the option of running 'make' to
# re-validate after correcting. We force Vim to open the quickfix
# window so errors and warnings can be seen and edited.
#
# shellcheck disable=SC1117
vim --cmd ":set makeprg=${VALIDATE}\ %" -c "$vimcom" \
-c ':setlocal spell' -c:cope -q "$ERRORS"
RES=$?
else
#
# No validation errors
#
vim -c "$vimcom" -c ':setlocal spell' "$EXTRACT"
RES=$?
fi
else
#
# Not html5. If the picture template exists open it in a vertical split
#
if [[ -e $PICTPL ]]; then
vim -c "$vimcom" -c ':setlocal spell' -O "$EXTRACT" "$PICTPL"
RES=$?
else
vim -c "$vimcom" -c ':setlocal spell' "$EXTRACT"
RES=$?
fi
fi
if [[ $RES -eq 0 ]]; then
echo "Edited $EXTRACT ok"
#
# Update the status file
#
echo "edited" >> "$STATUSFILE" || \
{ echo "Failed to update $STATUSFILE"; exit 1; }
else
echo "Oops! Something went wrong!"
fi
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21

881
Show_Submission/edit_shownotes Executable file
View File

@@ -0,0 +1,881 @@
#!/usr/bin/env perl
#===============================================================================
#
# FILE: edit_shownotes
#
# USAGE: ./edit_shownotes [-help] [-debug=N] [-field=NAME] -episode=N
#
# DESCRIPTION: Perform edits on metadata in show notes
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: 2022-12-20: originally written to update the shownotes.txt
# file as well as shownotes.json. Now we are phasing out the
# former and using just the JSON, so the former file is not
# being edited.
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.5
# CREATED: 2022-12-07 13:05:40
# REVISION: 2024-04-27 22:41:24
#
#===============================================================================
use v5.36;
use strict;
use warnings;
use utf8;
use feature qw{ postderef say signatures state try };
no warnings
qw{ experimental::postderef experimental::signatures experimental::try };
use Getopt::Long;
use Pod::Usage qw{pod2usage};
use Term::ANSIColor;
use File::Temp;
use File::Slurper qw{ read_text write_text };
use File::Copy;
use JSON;
use Log::Handler;
use Data::Dumper;
#
# Version number (manually incremented)
#
our $VERSION = '0.0.5';
#
# Script and directory names
#
( my $PROG = $0 ) =~ s|.*/||mx;
( my $DIR = $0 ) =~ s|/?[^/]*$||mx;
$DIR = '.' unless $DIR;
#-------------------------------------------------------------------------------
# Declarations
#-------------------------------------------------------------------------------
#
# Constants and other declarations
#
my $basedir = "$ENV{HOME}/HPR/Show_Submission";
my $cache = "$basedir/shownotes";
my $logdir = "$basedir/logs";
my $logfile = "$logdir/${PROG}.log";
my ( $fh, $showno, $showid );
my ( $jsonfile, $jsonfile_bk, $json_text, $JSON_content );
my ( $before, $field_contents );
#my ( $txtfile, $txtfile_bk, $txtstring, @txtbuffer );
#
# Text colours
#
my $red = color('red');
my $green = color('green');
my $yellow = color('yellow');
my $magenta = color('magenta');
my $bold = color('bold');
my $reset = color('reset');
#
# How to find (and edit) the fields in shownotes.{txt,json}.
# The fields are 'tags', 'title' and 'summary'.
# If the JSON file is being edited the primary field offered to the editor is
# the first in the array (e.g. 'episode.Tags') and the other one simply echoes
# it.
# NOTE: 2022-12-20 - stopped modifying the shownotes.txt file
#
# If the TXT file is being updated (from the primary JSON version edited
# earlier) we find it in the array holding the text file using the 'regex'
# value, and build a new line using the 'label' value.
# This will be cleaned up when the TXT variant is dropped.
#
my %fields = (
'tags' => {
'json' => [ 'episode.Tags', 'metadata.POST.tags' ],
# 'txt' => { regex => qr{^Tags:}, label => 'Tags', },
},
'title' => {
'json' => [ 'episode.Title', 'metadata.POST.title' ],
# 'txt' => { regex => qr{^Title:}, label => 'Title', },
},
'summary' => {
'json' => [ 'episode.Summary', 'metadata.POST.summary' ],
# 'txt' => { regex => qr{^Summary:}, label => 'Summary', },
},
);
#
# Enable Unicode mode
#
binmode STDOUT, ":encoding(UTF-8)";
binmode STDERR, ":encoding(UTF-8)";
#-------------------------------------------------------------------------------
# Options and arguments
#-------------------------------------------------------------------------------
my $DEF_DEBUG = 0;
my $DEF_FIELD = 'tags';
#
# Process options
#
my %options;
Options( \%options );
#
# Default help
#
pod2usage(
-msg => "$PROG version $VERSION\n",
-verbose => 0,
-exitval => 1
) if ( $options{'help'} );
#
# Detailed help
#
pod2usage(
-msg => "$PROG version $VERSION\n",
-verbose => 2,
-exitval => 1,
) if ( $options{'documentation'} );
#
# Collect options
#
my $DEBUG = ( defined( $options{debug} ) ? $options{debug} : $DEF_DEBUG );
my $field = ( defined( $options{field} ) ? $options{field} : $DEF_FIELD );
my $test_dir = $options{test_dir};
$showno = $options{episode};
pod2usage(
-msg => "$PROG version $VERSION\nMissing mandatory option -episode\n",
-verbose => 0,
-exitval => 1
) unless $showno;
#
# Make an id in 'hpr1234' format
#
$showid = sprintf('hpr%04d',$showno);
#
# Check that the -field=FIELD option is valid
#
die "Invalid field specification: $field\n"
unless ($field =~ /^(tags|title|summary)$/);
#
# The files we'll parse and/or edit (and that we'll backup to). Check the
# former exist.
# If we've received a test directory we'll run in test mode
#
if ($test_dir) {
#
# We need a directory here so remove any file component.
#
die "Missing path $test_dir\n" unless ( -e $test_dir );
unless (-d $test_dir) {
#
# Trim a file path to what is likely to be a directory only, and test
# it exists. Add a trailing '/' if none.
#
$test_dir =~ s|/?[^/]*$||mx;
die "Missing directory $test_dir\n" unless ( -d $test_dir );
$test_dir .= '/' unless ($test_dir =~ qr{/$});
}
#
# Look in the test place
#
$jsonfile = "$test_dir/shownotes.json";
# $txtfile = "$test_dir/shownotes.txt";
$jsonfile_bk = "$test_dir/shownotes.json.orig";
# $txtfile_bk = "$test_dir/shownotes.txt.orig";
}
else {
#
# Look in the default place
#
$jsonfile = "$cache/$showid/shownotes.json";
# $txtfile = "$cache/$showid/shownotes.txt";
$jsonfile_bk = "$cache/$showid/shownotes.json.orig";
# $txtfile_bk = "$cache/$showid/shownotes.txt.orig";
}
die colored( "Unable to find JSON file $jsonfile", 'red' ) . "\n"
unless ( -e $jsonfile );
#die colored( "Unable to find text file $txtfile", 'red' ) . "\n"
# unless ( -e $txtfile );
#-------------------------------------------------------------------------------
# Set up logging keeping the default log layout except for the date. The format
# is "%T [%L] %m" where '%T' is the timestamp, '%L' is the log level and '%m is
# the message. We fiddle with this to implement a 'Test' mode.
#-------------------------------------------------------------------------------
my $log = Log::Handler->new();
$log->add(
file => {
timeformat => "%Y/%m/%d %H:%M:%S",
message_layout => (defined($test_dir) ? "%T [%L] TEST %m" : "%T [%L] %m"),
filename => $logfile,
minlevel => 0,
maxlevel => 7,
}
);
#-------------------------------------------------------------------------------
# Read the JSON input file and parse it into $JSON_content (hashref)
#-------------------------------------------------------------------------------
my $json = JSON->new->utf8;
open( $fh, '<:encoding(UTF-8)', $jsonfile );
$json_text = <$fh>;
close($fh);
#
# Careful! Bad JSON can crash the script here!
#
try {
$JSON_content = decode_json($json_text);
}
catch ($e) {
die colored( "Failed to decode the JSON in $jsonfile", 'red' ) . "\n"
}
$log->info( $showno, "Collecting $field from JSON" );
_debug( $DEBUG > 2, Dumper($JSON_content) );
#_debug( $DEBUG > 2, "\$field: $field" );
#_debug( $DEBUG > 2, Dumper(\%fields) );
#-------------------------------------------------------------------------------
# Get the contents of the primary JSON field. The group will all be the same,
# as will the text versions, so we edit the primary and propagate to the
# others.
#-------------------------------------------------------------------------------
$log->info( $showno,
"Collecting from JSON with path '$fields{$field}->{json}->[0]'" );
#
# Use the path in the %fields hash to get the prinmary node in the JSON hash.
# We are using 'get_JSON' rather than going to the field directly because
# we're hunting for different parts of the JSON structure depending on the
# -field=NAME option and using a jq-like path stored in %fields.
#
$field_contents = get_JSON( $JSON_content, $fields{$field}->{json}->[0] );
#say( defined($field_contents) ? $field_contents : 'undefined' );
die colored( "No contents in field '$field'. Can't continue", 'red' ) . "\n"
unless ( defined($field_contents) );
$log->info( $showno, "Contents of field: $field_contents" );
$before = $field_contents;
#
# Run the editor on what we collected
#
if ( $field eq 'tags' ) {
$field_contents
= check_field( $field, run_editor($field_contents), 200, qr{(\n)} );
}
elsif ( $field eq 'title' ) {
$field_contents
= check_field( $field, run_editor($field_contents), 100, qr{(\n)} );
}
elsif ( $field eq 'summary' ) {
$field_contents
= check_field( $field, run_editor($field_contents), 100, qr{(\n)} );
}
die colored( "Nothing was changed. Exiting", 'red' ) . "\n"
if ( $field_contents eq $before );
$log->info( $showno, "New contents of field: $field_contents" );
#-------------------------------------------------------------------------------
# Load the text version of the metadata
# NOTE: 2022-12-20 - stopped modifying the shownotes.txt file
#-------------------------------------------------------------------------------
#open( $fh, '<:encoding(UTF-8)', $txtfile );
#@txtbuffer = <$fh>;
#close($fh);
#-------------------------------------------------------------------------------
# Make the changes to all the relevant points in the two data structures.
# NOTE: 2022-12-20 - stopped modifying the shownotes.txt file
#-------------------------------------------------------------------------------
put_JSON( $JSON_content, $fields{$field}->{json}->[0], $field_contents );
put_JSON( $JSON_content, $fields{$field}->{json}->[1], $field_contents );
#_debug( $DEBUG > 1, Dumper($JSON_content) );
$log->info( $showno, "Updated JSON '$field' fields" );
#@txtbuffer = update_txt( \@txtbuffer, $fields{$field}->{txt}->{regex},
# $fields{$field}->{txt}->{label} . ":\t$field_contents\n" );
#_debug( $DEBUG > 1, "Text: " . Dumper( \@txtbuffer ) );
#$log->info( $showno, "Updated text '$field' fields" );
#-------------------------------------------------------------------------------
# Turn the data structures back into files, backing up the originals and
# writing the new.
#-------------------------------------------------------------------------------
unless ( -e $jsonfile_bk ) {
copy ($jsonfile, $jsonfile_bk) or die "Unable to backup $jsonfile\n";
$log->info( $showno, "JSON backup created" );
}
$json_text = encode_json($JSON_content);
write_text($jsonfile,$json_text);
$log->info( $showno, "JSON file updated" );
#unless ( -e $txtfile_bk ) {
# copy ($txtfile, $txtfile_bk) or die "Unable to backup $txtfile\n";
# $log->info( $showno, "Text backup created" );
#}
#$txtstring = join('',@txtbuffer);
#write_text($txtfile,$txtstring);
#
#$log->info( $showno, "Text file updated" );
exit;
#=== FUNCTION ================================================================
# NAME: get_JSON
# PURPOSE: Traverse a data structure using dotted notation and return the
# value at the end point
# PARAMETERS: $hash hashref to traverse
# $path a path in the form of '.key1.key2.[0]' which
# should walk to the key 'key1', then 'key2'
# within it, and open the array expected to be
# there and get element 0.
# RETURNS: The value found or undef
# DESCRIPTION: Uses 'traverse_JSON' to find the $path in the $hash, returning
# a pointer (reference) to it. This can then be de-referenced to
# return the item being referenced - unless it's undefined in
# which case that value is returned. Being undefined means that
# the path did not match the structure of the hash.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub get_JSON {
my ( $hash, $path ) = @_;
my $pointer = traverse_JSON( $hash, $path );
#
# If undef then need to return undef, not try and deref it!
#
if (defined($pointer)) {
return $$pointer;
}
return;
}
#=== FUNCTION ================================================================
# NAME: put_JSON
# PURPOSE: Traverse a data structure using dotted notation and update the
# value at the end point
# PARAMETERS: $hash hashref to traverse
# $path a path in the form of '.key1.key2.[0]' which
# should walk to the key 'key1', then 'key2'
# within it, and open the array expected to be
# there and rewrite element 0.
# $new_value the new value to place at the point in the
# hash
# RETURNS: The previous value if found, or undef
# DESCRIPTION: Uses 'traverse_JSON' to find the $path in the $hash, returning
# a pointer (reference) to it. This can then be de-referenced to
# return the item being referenced - unless it's undefined in
# which case that value is returned. Being undefined means that
# the path did not match the structure of the hash. Finally, if
# the reference is valid then it's used to save the contents of
# $new_value.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub put_JSON {
my ( $hash, $path, $new_value ) = @_;
my $pointer = traverse_JSON( $hash, $path );
my $old_value;
#
# If $pointer is defined then dereference it as the return value. Also,
# use it to write the new value.
#
if (defined($pointer)) {
$old_value = $$pointer;
$$pointer = $new_value;
}
return $old_value;
}
#=== FUNCTION ================================================================
# NAME: traverse_JSON
# PURPOSE: Traverse a data structure using dotted notation
# PARAMETERS: $hash hashref to traverse
# $path a path in the form of '.key1.key2.[0]' which
# should walk to the key 'key1', then 'key2'
# within it, and open the array expected to be
# there and return a pointer to element 0.
# RETURNS: A reference to the end point of the path, or undef if not
# found.
# DESCRIPTION: Given a path like '.metadata.POST.title' (where the leading
# '.' is optional because it's ignored), this is used to
# traverse a Perl version of a JSON data structure to return
# a *pointer* (reference) to a node.
#
# This is done by processing the path step by step and using
# 'name' elements as hash keys to get to the next level or
# return a terminal hash element. We also cater for '.[0]'
# format path elements which means to use the zeroth member of
# an array. Thus, if the tags were represented as an array rather
# than a scalar CSV string, we'd be able to use a path like:
# 'episode.Tags.[0]' to return the first tag.
#
# The data structure traversal is performed by using a pointer
# to a part of it, and since nested hashes are constructed by
# using hashrefs as hash element values, we get back references
# by default.
#
# Once the terminal node has been reached it may be a scalar
# (very likely in this application), so we make and return
# a reference to it so that the caller can use this reference in
# a standard way to return or change the contents.
#
# This function is meant to be more generic than this
# application requires. The JSON we're dealing with here doesn't
# have arrays (at the moment). This description is
# application-specifc however!
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub traverse_JSON {
my ( $hash, $path ) = @_;
my ( $pointer, $index );
my @elements = split( '\.', $path );
#
# Because $hash is a hashref we can just copy it
#
$pointer = $hash;
#
# Traverse the hash using the path
#
foreach my $element (@elements) {
next unless defined($element);
#
# Plain names are keys or scalar end nodes, bracketed things are
# arrays (which can be scalars of course)
#
unless ( ($index) = ( $element =~ /\[([^]])\]/ ) ) {
# Hash key
if ( exists( $pointer->{$element} ) ) {
if ( ref( $pointer->{$element} ) eq '' ) {
#
# It's not a refererence so make a reference to it
#
$pointer = \$pointer->{$element};
}
else {
#
# It's a reference
#
$pointer = $pointer->{$element};
}
}
else {
#
# Doesn't exist!
#
return; #undef
}
}
else {
# Array
if ( exists( $pointer->[$index] ) ) {
if ( ref( $pointer->[$index] ) eq '' ) {
#
# It's not a refererence so make a reference to it
#
$pointer = \$pointer->[$index];
}
else {
#
# It's a reference
#
$pointer = $pointer->[$index];
}
}
else {
#
# Doesn't exist!
#
return; # undef
}
}
}
return $pointer;
}
#=== FUNCTION ================================================================
# NAME: update_txt
# PURPOSE: Update the data structure from 'shownotes.txt' in the light of
# requested changes.
# PARAMETERS: $array arrayref to the contents of the shownotes.txt
# file
# $regex regular expression to find the line that needs
# to be changed (there's just one main one).
# $new_value new value to replace the original line
# RETURNS: Modified array
# DESCRIPTION: Finds the element in the arrayref ($array) which matches the
# regular expression ($regex). When found it is replaced by the
# new value ($new_value). The modified buffer is returned as
# a list. Uses $result to determine the value of each line since
# it is not permitted to change the $_ variable in the 'map'
# statement.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub update_txt {
my ( $array, $regex, $new_value ) = @_;
my $result;
my @newbuffer
= map { $result = ( $_ =~ /$regex/ ? $new_value : $_ ); $result }
@$array;
return @newbuffer;
}
#=== FUNCTION ================================================================
# NAME: run_editor
# PURPOSE: Run an interactive vim editor on a string
# PARAMETERS: $string String to edit
# $options An arrayref containing options for vim
# (optional) Example '+set paste'. Each option
# (such as '-c startinsert') needs to be
# a separate array element.
# RETURNS: Edited string
# DESCRIPTION: Makes a temporary file with File::Temp ensuring that the file
# is in utf8 mode. Writes the edit string to the file and invokes
# the 'vim' editor on it. The resulting file is then read back
# into a string and returned to the caller, again taking care to
# retain utf8 mode.
# THROWS: No exceptions
# COMMENTS: File::Slurp and UTF-8 don't go well together. Moved to
# File::Slurper instead
# SEE ALSO: N/A
#===============================================================================
sub run_editor {
my ( $string, $options ) = @_;
#
# Build an arguments array for 'system'
#
my @args;
push( @args, @$options ) if $options;
#
# Make a temporary file
#
my $tfh = File::Temp->new;
binmode $tfh, ":encoding(UTF-8)";
my $tfn = $tfh->filename;
print $tfh $string if $string;
$tfh->close;
#
# Add the filename to the arguments
#
push( @args, $tfn );
die "Edit failed\n"
unless ( system( ( 'vim', @args ) ) == 0 );
return read_text($tfn);
}
#=== FUNCTION ================================================================
# NAME: check_field
# PURPOSE: Checks the a field is not too long and doesn't contain certain
# characters
# PARAMETERS: $name name of field
# $field string to be checked
# $maxlen maximum string length
# $regex regex containing illegal characters to be removed
# RETURNS: The input string truncated and with any illegal characters
# removed.
# DESCRIPTION: Runs a substitution on the string then truncates the result if
# it is too long.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub check_field {
my ( $name, $field, $maxlen, $regex ) = @_;
return unless $field;
$field =~ s/$regex//g;
if ( length($field) > $maxlen ) {
warn "Field '$name' too long ("
. length($field)
. "); truncated to "
. $maxlen . "\n";
$field = substr( $field, 0, $maxlen );
}
return $field;
}
#=== FUNCTION ================================================================
# NAME: coalesce
# PURPOSE: To find the first defined argument and return it
# PARAMETERS: Arbitrary number of arguments
# RETURNS: The first defined argument or undef if there are none
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub coalesce {
foreach (@_) {
return $_ if defined($_);
}
return undef; ## no critic
}
#=== FUNCTION ================================================================
# NAME: _debug
# PURPOSE: Prints debug reports
# PARAMETERS: $active Boolean: 1 for print, 0 for no print
# $message Message to print
# RETURNS: Nothing
# DESCRIPTION: Outputs a message if $active is true. It removes any trailing
# newline and then adds one in the 'print' to the caller doesn't
# have to bother. Prepends the message with 'D> ' to show it's
# a debug message.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub _debug {
my ( $active, $message ) = @_;
chomp($message);
print STDERR "D> $message\n" if $active;
}
#=== FUNCTION ================================================================
# NAME: Options
# PURPOSE: Processes command-line options
# PARAMETERS: $optref Hash reference to hold the options
# RETURNS: Undef
# DESCRIPTION:
# THROWS: no exceptions
# COMMENTS: none
# SEE ALSO: n/a
#===============================================================================
sub Options {
my ($optref) = @_;
my @options = (
"help", "documentation|manpage",
"debug=i", "field=s",
"episode=i", "test_dir=s",
);
if ( !GetOptions( $optref, @options ) ) {
pod2usage(
-msg => "$PROG version $VERSION\n",
-verbose => 0,
-exitval => 1
);
}
return;
}
__END__
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# Application Documentation
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#{{{
=head1 NAME
edit_shownotes - edit JSON and text shownote metadata
=head1 VERSION
This documentation refers to edit_shownotes version 0.0.5
=head1 USAGE
./edit_shownotes [-help] [-documentation|manpage] [-debug=N]
[-field=FIELD] -episode=N
./edit_shownotes -field=tags -episode=3754
./edit_shownotes -field=summary -episode=3762 \
-test_dir=$PWD/tests/shownotes/hpr3762
=head1 OPTIONS
=over 8
=item B<-help>
Prints a brief help message describing the usage of the program, and then exits.
=item B<-manpage> or B<-documentation>
Prints the entire documentation for the script.
=item B<-field=FIELD>
This optional parameter defaults to B<-field=tags>. The only permitted fields
are 'tags', 'title' and 'summary'.
=item B<-episode=N>
This mandatory option specifies the number of the HPR episode that is being
parsed. It needs to be numeric even though it is often used with an 'hpr'
prefix internally. Leading zeroes are not required (when these are
appropriate).
=item B<-test_dir=PATH>
This option is for testing. Normally the script determines the path to the
file(s) it will parse and update. It actually finds the field specified by the
B<-field-FIELD> option in the JSON file B<shownotes.json> and calls the editor
to change this, then propagates any changes to the other JSON instance and the
one in B<shownotes.txt>.
This option has been provided to make it simpler to make copies of the files
relating to an incoming show and use them to test changes to the script.
The current workflow expects the "live" files in
B<$HOME/HPR/Show_Submission/shownotes/hprSHOW>, but there is another area
where show files are placed for testing:
B<$HOME/HPR/Show_Submission/tests/shownotes/hprSHOW>.
It is simply a matter of making a copy and of referencing the directory path
to this option. Changes will be made to this copy and log records written, but
these will contain the word 'TEST' to show that the script is being run in
this mode.
=item B<-debug=N>
Causes certain debugging information to be displayed.
0 (the default) no debug output
1 N/A
2 TBA
3 TBA
=back
=head1 DESCRIPTION
This script is meant to be a component in a workflow dealing with the notes
submitted by HPR hosts with the shows they are uploading.
The data from the web form which is used by hosts submitting shows is received
in two formats, with fixed names.
The file B<shownotes.txt> contains the form data in a label/value format, but
also uses PHP dumps to show some of the metadata. It is no longer used by the
current workflow, but is retained for historical reasons.
The alternative file format is JSON, which is stored in B<shownotes.json>.
This contains the same data as the other file but in a more standardised
format.
This script is designed to allow the editing of three specific fields in these
two files. This is because these fields are not used in the show note
preparation workflow (though they are referenced), but any changes can be
propagated back to the HPR server to allow workflows there to use the
corrected version(s).
The fields which can be edited are duplicated in the two files. One of these
is edited, and if appropriate, is propagated to the other duplicates. This
simplifies the editing of the text format, and the JSON format, which is not
simple to edit in a conventional way.
=head1 DIAGNOSTICS
A list of every error and warning message that the application can generate
(even the ones that will "never happen"), with a full explanation of each
problem, one or more likely causes, and any suggested remedies. If the
application generates exit status codes (e.g. under Unix) then list the exit
status associated with each error.
=head1 CONFIGURATION AND ENVIRONMENT
A full explanation of any configuration system(s) used by the application,
including the names and locations of any configuration files, and the
meaning of any environment variables or properties that can be set. These
descriptions must also include details of any configuration language used
=head1 DEPENDENCIES
Data::Dumper
File::Slurper
File::Temp
Getopt::Long
JSON
Log::Handler
Pod::Usage
Term::ANSIColor
=head1 BUGS AND LIMITATIONS
There are no known bugs in this module.
Please report problems to Dave Morriss (Dave.Morriss@gmail.com)
Patches are welcome.
=head1 AUTHOR
Dave Morriss (Dave.Morriss@gmail.com)
=head1 LICENCE AND COPYRIGHT
Copyright (c) 2022 Dave Morriss (Dave.Morriss@gmail.com). All rights reserved.
This module is free software; you can redistribute it and/or
modify it under the same terms as Perl itself. See perldoc perlartistic.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
=cut
#}}}
# [zo to open fold, zc to close]
# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker

751
Show_Submission/fix_relative_links Executable file
View File

@@ -0,0 +1,751 @@
#!/usr/bin/env perl
#===============================================================================
#
# FILE: fix_relative_links
#
# USAGE: ./fix_relative_links [options] -episode=N FILE
#
# DESCRIPTION: Processes an HTML input file, looking for relative URLs. If
# any are found these are made absolute using the -baseURL=URL
# option or a default. The intention is to make them into
# HPR-absolute URLs.
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.3
# CREATED: 2022-10-14 11:56:03
# REVISION: 2022-10-23 22:12:08
#
#===============================================================================
use v5.16;
use strict;
use warnings;
use utf8;
use feature qw{ postderef say signatures state };
no warnings qw{ experimental::postderef experimental::signatures };
use Carp;
use Getopt::Long;
use Pod::Usage;
use File::Basename;
use IO::HTML;
use HTML::TreeBuilder 5 -weak;
use URI;
use Log::Handler;
use Data::Dumper;
#
# Version number (manually incremented)
#
our $VERSION = '0.0.3';
#
# Script and directory names
#
( my $PROG = $0 ) =~ s|.*/||mx;
( my $DIR = $0 ) =~ s|/?[^/]*$||mx;
$DIR = '.' unless $DIR;
#-------------------------------------------------------------------------------
# Declarations
#-------------------------------------------------------------------------------
#
# Constants and other declarations
#
my $basedir = "$ENV{HOME}/HPR/Show_Submission";
my $logdir = "$basedir/logs";
my $logfile = "$logdir/${PROG}.log";
#
# Variables, arrays and hashes
#
my ( $DEBUG, $verbose, $silent, $showno, $base_URL, $fragment, $count_only );
my ( $outfile, $filename, $showdir, $changes, $html );
#
# Enable Unicode mode
#
binmode STDOUT, ":encoding(UTF-8)";
binmode STDERR, ":encoding(UTF-8)";
#-------------------------------------------------------------------------------
# Options and arguments
#-------------------------------------------------------------------------------
#
# Option defaults
#
my $DEFDEBUG = 0;
my %options;
Options( \%options );
#
# Default help shows minimal information
#
pod2usage( -msg => "$PROG version $VERSION\n", -exitval => 1, -verbose => 0 )
if ( $options{'help'} );
#
# The -documentation or -man option shows the full POD documentation through
# a pager for convenience
#
pod2usage( -msg => "$PROG version $VERSION\n", -exitval => 1, -verbose => 2 )
if ( $options{'documentation'} );
#
# Collect options
#
$DEBUG = ( defined( $options{debug} ) ? $options{debug} : $DEFDEBUG );
$showno = $options{episode};
$base_URL = $options{baseURL};
$fragment = ( defined( $options{fragment} ) ? $options{fragment} : 0 );
$count_only = ( defined( $options{count} ) ? $options{count} : 0 );
$outfile = $options{output};
#
# Argument
#
$filename = shift;
#
# Sanity checks
#
pod2usage(
-msg => "$PROG version $VERSION\nShow number missing\n",
-exitval => 1,
-verbose => 0
) unless $showno;
pod2usage(
-msg => "$PROG version $VERSION\nInput file name missing\n",
-exitval => 1,
-verbose => 0
) unless $filename;
#
# Add leading zeroes to the show number if necessary
#
$showno = sprintf( '%04d', $showno );
#
# Directories and files specific to this show
#
$showdir = "$basedir/shownotes/hpr$showno";
#
# Allow the input filename to be a bare name
#
if ( !-e $filename ) {
$filename = "$showdir/$filename";
}
die "Unable to find $filename" unless ( -e $filename );
#
# Work on the output file, allowing defaults and substitution points for
# convenience. If there's no outfile we'll just process the HTML and nothing
# more.
#
if ( defined($outfile) ) {
$outfile = output_file_name( $outfile, $showno, 'hpr%d_new.html' );
$outfile = "$showdir/$outfile" if (dirname($outfile) eq '.');
}
#
# Default base URL
#
unless ($base_URL) {
$base_URL = "https://hackerpublicradio.org/eps/hpr$showno/";
}
#
# Base URL must have a trailing '/'
#
$base_URL .= '/' unless ( $base_URL =~ qr{/$} );
#-------------------------------------------------------------------------------
# Set up logging keeping the default log layout except for the date. The format
# is "%T [%L] %m" where '%T' is the timestamp, '%L' is the log level and '%m is
# the message.
#-------------------------------------------------------------------------------
my $log = Log::Handler->new();
$log->add(
file => {
timeformat => "%Y/%m/%d %H:%M:%S",
filename => $logfile,
minlevel => 0,
maxlevel => 7,
}
);
#
# Log preamble
#
$log->info("Show number: $showno");
$log->info("Processing: $filename");
$log->info("Base: $base_URL");
#
# Find and change any relative URLs returning the number of changes and the
# altered HTML
#
( $changes, $html )
= find_links_in_file( $filename, $base_URL, $fragment, $count_only );
$log->info("Number of changes: $changes");
#
# Exit without writing if we're just counting
#
if ($count_only) {
$log->info("Count only mode");
exit $changes;
}
#
# Exit without writing if there were no changes
#
if ($changes == 0) {
$log->info("No output written");
exit $changes;
}
#
# Write output if an output file was specified
#
if ($outfile) {
write_output( $outfile, $html );
$log->info("Changes applied; written to $outfile");
}
else {
$log->info("No output written");
}
exit $changes;
#=== FUNCTION ================================================================
# NAME: find_links_in_file
# PURPOSE: Finds relative links in an HTML file
# PARAMETERS: $filename the name of the file we're parsing
# $base_URL the part of the full URL we'll replace
# $fragment Boolean signalling whether to treat the HTML
# as a fragment or an entire document
# $count_only Boolean signalling that all we want is the
# count of relative URLs, no action is to be taken
# RETURNS: The number of URLs "repaired".
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub find_links_in_file {
my ( $filename, $base_URL, $fragment, $count_only ) = @_;
my ( $base_uri, $tree, $uri_orig, $uri_new );
my ( $newlink, $linkedits, $result );
#
# Parse the base URL
#
$base_uri = URI->new($base_URL);
#
# Create a tree object
#
$tree = HTML::TreeBuilder->new;
$tree->ignore_unknown(0);
$tree->no_expand_entities(1);
$tree->p_strict(1);
$tree->store_comments(1);
$tree->warn(1);
#
# Parse the file using IO::HTML to grab it. Die if we fail because then we
# know this stuff needs some urgent attention.
#
$tree->parse_file( html_file($filename) )
or die "HTML::TreeBuilder failed to process $filename: $!\n";
$linkedits = 0;
#
# Scan for all anchors and images using the HTML::Element method
# 'extract_links'. The technique used here is from the HTML::Element man
# page.
#
for ( @{ $tree->extract_links( 'a', 'img' ) } ) {
my ( $link, $element, $attr, $tag ) = @$_;
#
# Parse the link
#
$uri_orig = URI->new($link);
#
# A relative link (presumably) doesn't have a scheme
#
unless ( $uri_orig->scheme ) {
#
# Original link
#
say "Relative link: $link";
#
# Make the link absolute
#
$uri_new = make_absolute( $uri_orig, $base_uri );
# $uri_new = URI->new_abs( $link, $base_URL );
$newlink = sprintf( "%s:%s", $uri_new->scheme, $uri_new->opaque );
say "Absolute link: $newlink";
#
# Modify the HTML to make the relative absolute
#
if ( $uri_orig->fragment ) {
# Not sure if we need to cater for URI fragments, but we'll try it
$element->attr( $attr, $newlink . '#' . $uri_orig->fragment );
}
else {
$element->attr( $attr, $newlink );
}
$linkedits++;
}
}
#
# Exit here if we were just asked to count
#
return ( $linkedits, undef ) if $count_only;
#
# In 'HTML fragment' mode generate the body part without the <body> tags.
#
if ($fragment) {
my $body = $tree->look_down( _tag => 'body' );
( $result = $body->as_HTML( undef, ' ', {} ) )
=~ s{(^<body[^>]*>|</body>$)}{}gi;
}
else {
$result = $tree->as_HTML( undef, ' ', {} );
}
return ( $linkedits, $result );
}
#=== FUNCTION ================================================================
# NAME: write_output
# PURPOSE: Write the "repaired" HTML
# PARAMETERS: $outfile name of the output file
# $html the HTML to write out
# RETURNS: Nothing
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub write_output {
my ( $outfile, $html ) = @_;
open( my $out, '>:encoding(UTF-8)', $outfile )
or die "Unable to open $outfile for output: $!\n";
print $out $html;
close($out);
}
#=== FUNCTION ================================================================
# NAME: make_absolute
# PURPOSE: Take a relative URI and a base URI and return the absolute URI
# PARAMETERS: $relative relative URL as a URI object
# $base base URL as a URI object
# RETURNS: Absolute URL as a URI object
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub make_absolute {
my ( $relative, $base ) = @_;
my ( %base_path, @relative_path, $absolute );
#
# Chop up the path from the base and store in a hash
#
%base_path = map { $_ => 1 } split( '/', $base->path );
#
# Chop up the relative path
#
@relative_path = split( '/', $relative->path );
#
# Remove relative path elements if they are in the base
#
@relative_path = grep { !exists( $base_path{$_} ) } @relative_path;
#
# If the relative path is empty we assume it's referring to the
# 'index.html' file.
#
push( @relative_path, 'index.html' ) unless (@relative_path);
#
# Rebuild the relative path
#
$relative->path( join( '/', @relative_path ) );
#
# Return the result of joining relative URL and base URL
#
$absolute = URI->new_abs( $relative->as_string, $base->as_string );
return $absolute;
}
#=== FUNCTION ================================================================
# NAME: output_file_name
# PURPOSE: Generate an output file name with three choices
# PARAMETERS: $optarg the argument to the option choosing the filename
# $showno the show number to add to certain name types
# $template a default 'sprintf' template for the name
# RETURNS: The name of the output file
# DESCRIPTION: If there's a defined output filename then there are three
# options: a null string, a plain filename and a substitution
# string with '%d' sequences. The null string means the user used
# '-option' without a value, so we want to generate a substitution
# string. A string with '%d' requires a check to ensure there's
# the right number, just one. The plain filename needs no more
# work.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub output_file_name {
my ( $optarg, $showno, $template ) = @_;
my ( $filename, $count );
#
# We shouldn't be called with a null option argument
#
return unless defined($optarg);
#
# Does the option have an argument?
#
if ( $optarg =~ /^$/ ) {
#
# No argument; use the show number from the -episode=N option
#
$filename = sprintf( $template, $showno );
}
elsif ( $optarg =~ /%d/ ) {
#
# There's an argument, does it have a '%d' in it?
#
$count = () = $optarg =~ /%d/g;
die "Invalid - too many '%d' sequences in '$optarg'\n"
if ( $count > 1 );
$filename = sprintf( $optarg, $showno );
}
else {
#
# It's a plain filename, just return it
#
$filename = $optarg;
}
return $filename;
}
#=== FUNCTION ================================================================
# NAME: coalesce
# PURPOSE: To find the first defined argument and return it
# PARAMETERS: Arbitrary number of arguments
# RETURNS: The first defined argument or undef if there are none
# DESCRIPTION: Just a simple way of ensuring an 'undef' value is never
# returned when doing so might be a problem.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub coalesce {
foreach (@_) {
return $_ if defined($_);
}
return undef; ## no critic
}
#=== FUNCTION ================================================================
# NAME: _debug
# PURPOSE: Prints debug reports
# PARAMETERS: $active Boolean: 1 for print, 0 for no print
# $message Message to print
# RETURNS: Nothing
# DESCRIPTION: Outputs a message if $active is true. It removes any trailing
# newline and then adds one in the 'print' to the caller doesn't
# have to bother. Prepends the message with 'D> ' to show it's
# a debug message.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub _debug {
my ( $active, $message ) = @_;
chomp($message);
print "D> $message\n" if $active;
}
#=== FUNCTION ================================================================
# NAME: Options
# PURPOSE: Processes command-line options
# PARAMETERS: $optref Hash reference to hold the options
# RETURNS: Undef
# DESCRIPTION: Process the options we want to offer. See the documentation
# for details
# THROWS: no exceptions
# COMMENTS: none
# SEE ALSO: n/a
#===============================================================================
sub Options {
my ($optref) = @_;
my @options = (
"help", "documentation|man", "debug=i", "episode=i",
"baseURL=s", "fragment!", "count!", "output:s",
);
if ( !GetOptions( $optref, @options ) ) {
pod2usage(
-msg => "$PROG version $VERSION\n",
-exitval => 1,
-verbose => 0
);
}
return;
}
__END__
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# Application Documentation
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#{{{
=head1 NAME
fix_relative_links - Repair relative URLs in HTML shownotes
=head1 VERSION
This documentation refers to fix_relative_links version 0.0.3
=head1 USAGE
./fix_relative_links -ep=3705 shownotes/hpr3705/hpr3705.html -fragment
=head1 REQUIRED ARGUMENTS
=over 4
=item B<filename>
The name of the file containing the HTML to be repaired. If no path is given
this will be supplied by the script as:
~/HPR/Show_Submission/shownotes/hpr${show}/
It is probably wiser to be explicit about the path to the HTML file to be
parsed.
=back
=head1 OPTIONS
=over 8
=item B<-help>
Prints a brief help message describing the usage of the program, and then exits.
=item B<-documentation> or B<-man>
Prints the entire documentation for the script in the form of a manual page.
=item B<-debug=N>
Causes certain debugging information to be displayed.
0 (the default) no debug output
1
2
3
=item B<-episode=N>
This option is mandatory and specifies the show number being processed. The
number is used to generate default file names and paths as well as the default
base URL described below.
=item B<-baseURL=URL>
This option will default to the foillowing URL if not provided:
https://hackerpublicradio.org/eps/hpr${show}/
It can be used to define a non-standard URL, such as one at a lower level than
the example above which might contain thumbnail pictures for example.
=item B<-[no]fragment>
This Boolean option defines the HTML being parsed and checked as a fragment or
a complete stand-alone document. By default B<-nofragment> is assumed. It is
necessary to use B<fragment> for the case where the HTML shownotes are being
parsed.
=item B<-[no]count>
This Boolean option defines whether to simply count the necessary changes or
to apply them to the given HTML file. By default B<-nocount> is assumed, and
changes will be applied.
=item B<-output[=FILE]>
This option can be omitted or can be given without the B<FILE> name. If
omitted entirely no output will be written even though the HTML file has been
read and processed. If specified without the output file name the default name
will be B<hpr${show}_new.html>. If no path is specified with the file name
then a default will be generated as:
~/HPR/Show_Submission/shownotes/hpr${show}/hpr${show}_new.html
The output file name can be given in the form of a B<printf> template such as:
hpr%d_new.html
and the B<%d> will be replaced by the show number given through the
B<-episode=N> option described above.
=back
=head1 DESCRIPTION
The script reads a file of HTML which has either been submitted by an HPR host
as it is or has been generated from one of the markup languages accepted in
the upload form. Most often this file will contain the main notes for a show
and will eventually be saved in the HPR database.
It is also possible to use the script to process other HTML files submitted
with an HPR show.
The purpose of the script is to find relative URLs in the HTML and convert
them to absolute ones. The HPR website requests that absolute URLs be used
since then they can be used in the various RSS feeds which are available, but
many hosts forget to follow this request.
The HTML is parsed using B<HTML::TreeBuilder> and is searched for B<a> or
B<img> tags. These are checked to ensure they contain absolute links, and if
not are converted appropriately using a base URL for the HPR website.
A count of changes is returned by the script and the converted HTML is written
out to a file if required. The script can be used to see if any conversions
are necessary before making the changes.
The script is also capable of treating full HTML documents differently from
the HTML fragments that are stored in the HPR database. An option is required
to specify which type of HTML is being processed.
=head1 DIAGNOSTICS
Error and warning messages generated by the script.
=over 4
=item B<Unable to find ...>
Type: fatal
The script was unable to find the specified input file.
=item B<HTML::TreeBuilder failed to process ...: ...>
Type: fatal
The script attempted to use B<HTML::TreeBuilder> to parse the input file
but failed. The message also contains details of the failure.
=item B<Unable to open ... for output: ...>
Type: fatal
The script attempted to open the requested output file but failed. The reason
for the failure is included in the error message.
=item B<Invalid - too many '%d' sequences in '...'>
Type: fatal
The script attempted to generate a name for the requested output file using
the supplied template, but failed because there were too many B<%d> elements
in the template. Only one should be provided, which will be substituted with
the show number.
=back
=head1 DEPENDENCIES
Carp
Data::Dumper
File::Basename
Getopt::Long
HTML::TreeBuilder
IO::HTML
Log::Handler
Pod::Usage
URI
=head1 BUGS AND LIMITATIONS
There are no known bugs in this module.
Please report problems to Dave Morriss (Dave.Morriss@gmail.com)
Patches are welcome.
=head1 AUTHOR
Dave Morriss (Dave.Morriss@gmail.com)
=head1 LICENCE AND COPYRIGHT
Copyright (c) <year> Dave Morriss (Dave.Morriss@gmail.com). All rights reserved.
This module is free software; you can redistribute it and/or
modify it under the same terms as Perl itself. See perldoc perlartistic.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
=cut
#}}}
# [zo to open fold, zc to close]
# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker

393
Show_Submission/make_markdown Executable file
View File

@@ -0,0 +1,393 @@
#!/usr/bin/env perl
#===============================================================================
#
# FILE: make_markdown
#
# USAGE: ./make_markdown [-[no]simple] [-[no]entities] [-[no]stamp] file
#
# DESCRIPTION: Turn plain text to Markdown. Designed to be used as a filter
# in vim.
# Finds all bare URLs, ignoring lines with HTML or Markdown
# links. It checks the path part of the URL to see if it's an
# image. If it is it uses an image link, otherwise it uses
# a plain link. Links are all to HPR standard where the text
# part and the URL part are the same.
# Unless the '-simple' option is present all HTML URLs are
# accumulated, and if the host hasn't provided a 'Links' section
# they are added to one. This happens even if the overall text
# is short, but it's easy to remove if not wanted.
# By default, in -nosimple mode an HTML comment with details of
# this script is added to the output. This can be turned off
# with the -nostamp option.
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.9
# CREATED: 2015-10-07 16:05:21
# REVISION: 2024-01-14 15:59:34
#
#===============================================================================
use 5.010;
use strict;
use warnings;
use utf8;
use version 0.77; # Planning to experiment with this new feature
use open ':encoding(UTF-8)'; # Make all IO UTF-8
use Getopt::Long;
use Regexp::Common qw{URI pattern};
use HTML::Entities;
#use Encoding::FixLatin qw(fix_latin);
use URI::Find;
use URI;
use List::MoreUtils qw{uniq};
#
# Version number (manually incremented)
#
our $VERSION = 'v0.0.9';
#
# Script and directory names
#
( my $PROG = $0 ) =~ s|.*/||mx;
( my $DIR = $0 ) =~ s|/?[^/]*$||mx;
$DIR = '.' unless $DIR;
#-------------------------------------------------------------------------------
# Declarations
#-------------------------------------------------------------------------------
#
# Constants and other declarations
#
my $basedir = "$ENV{HOME}/HPR/Show_Submission";
my @urls;
my $have_links = 0;
my $uri_count;
my $new_url;
#
# Enable Unicode mode
#
#binmode STDIN, ":encoding(UTF-8)";
#binmode STDOUT, ":encoding(UTF-8)";
#binmode STDERR, ":encoding(UTF-8)";
#-------------------------------------------------------------------------------
# Patterns
#-------------------------------------------------------------------------------
#
# 1. Look for a line beginning with 'Links:'
#
pattern
name => ['links'],
create => '^\s*Links:?\s*$',
;
#
# 2. Look for an HTML link (some people embed these in their notes)
#
pattern
name => ['html_link'],
create => '<a[^>]*>',
;
#
# 3. Look for existing Markdown links
#
pattern
name => ['md_link'],
create => '\[([^]]+)\]\(\1\)',
;
#
# 4. Look for mail addresses
#
pattern
name => ['mail_link'],
create => '(?i)[a-z0-9_+.](?:[a-z0-9_+.]+[a-z0-9_+])?\@[a-z0-9.-]+',
;
#-------------------------------------------------------------------------------
# Options and arguments
#-------------------------------------------------------------------------------
#
# Process options
#
my %options;
Options( \%options );
#
# Collect options
#
Usage() if ( $options{'help'} );
my $simple = ( defined( $options{simple} ) ? $options{simple} : 0 );
my $entities = ( defined( $options{entities} ) ? $options{entities} : 0 );
my $stamp = ( defined( $options{stamp} ) ? $options{stamp} : 0 );
#-------------------------------------------------------------------------------
# Process the data on STDIN
#-------------------------------------------------------------------------------
#
# Set up the URI finder to call back to the 'cb_general' routine. We use
# a closure to allow the passing of the $simple variable
#
my $finder = URI::Find->new( sub { cb_general( shift, shift, $simple ) } );
#
# Slurp everything on STDIN
#
my @text = <>;
if ($stamp) {
printf "<!-- Processed by %s %s -->\n", $PROG, $VERSION unless $simple;
}
#
# Process the stuff we got
#
foreach my $line (@text) {
chomp($line);
#
# Look for a user-provided 'Links' section
#
if ( $line =~ /($RE{links})/ ) {
$have_links = 1;
$line = "### $1";
print "$line\n";
next;
}
#
# Check whether the line contains some HTML. If it does we don't want to
# touch it.
#
if ( $line =~ /$RE{html_link}/ ) {
print "$line\n";
next;
}
#
# An existing Markdown link means we'll skip the line
#
if ( $line =~ /$RE{md_link}/ ) {
print "$line\n";
next;
}
#
# Special action for mail addresses.
#
# TODO: Loop through the line extracting each mail address and checking it
# with Email::Valid. If valid turn into a Markdown linkgg of the form
# <mailaddr>, otherwise ignore it.
#
if ( $line =~ /$RE{mail_link}/ ) {
$line =~ s/($RE{mail_link})/<$1>/g;
}
#
# Look for URLs of any sort and process them in the callback 'cb_general'
# defined earlier
#
$uri_count = $finder->find( \$line );
#
# Print the line. It might have been edited by the URI::Find callback or
# might just be as it was.
#
encode_entities($line) if ($entities);
print "$line\n";
}
#
# Unless we're in 'simple' mode generate links if there's data and the host
# hasn't provided them. Make sure they are unique (since there may have been
# multiple references in the notes)
#
unless ($simple) {
if ( @urls && !$have_links ) {
@urls = uniq @urls;
print "\n### Links\n\n";
foreach my $url (@urls) {
#$new_url = uri_reformat($url); # New format is not accepted
print "- [$url]($url)\n";
}
}
}
exit;
#=== FUNCTION ================================================================
# NAME: cb_general
# PURPOSE: Process a URI found by URI::Find containing a link
# PARAMETERS: $uri_obj a URI object
# $uri_text the URI text
# $simple Boolean that controls what type of Markdown
# link is generated. If true (1) we use <URI>
# form, otherwise we use [URI](URI) form
# RETURNS: The modified URI text
# DESCRIPTION: The URI::Find 'find' method calls this callback for every URI
# it finds in the text it discovers. We only care about
# http/https here (for now anyway). If the text is changed and
# returned then the changed item is placed in the original text.
# We differentiate between what look like images and plain HTML
# URLs and generate different markup. We also save the HTML URL
# in a global array for building a list of links later.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub cb_general {
my ( $uri_obj, $uri_text, $simple ) = @_;
my $path;
#my $new_uri = uri_reformat($uri_text);
#
# For http: or https:
#
if ( $uri_obj->scheme =~ /^https?/ ) {
$path = $uri_obj->path;
if ( $path =~ /(?i:\.(jpg|png|gif)$)/ ) {
$uri_text = "![$uri_text]($uri_text)";
}
else {
push( @urls, $uri_text );
if ($simple) {
$uri_text = "<$uri_text>";
}
else {
$uri_text = "[$uri_text]($uri_text)";
}
}
}
return $uri_text;
}
#=== FUNCTION ================================================================
# NAME: uri_reformat
# PURPOSE: Turns a URI into a form which is better for Markdown
# PARAMETERS: $uri_str - the URI string
# RETURNS: The URI string reformatted
# DESCRIPTION: Places the URI string in an URI object to parse it. Copies
# each of the elements to another empty URI object but takes
# advantage of the setting of URI::DEFAULT_QUERY_FORM_DELIMITER
# which forces the '&' delimiters to ';'. Returns the string
# version of this new URI.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub uri_reformat {
my ($uri_str) = @_;
local $URI::DEFAULT_QUERY_FORM_DELIMITER = ';';
my $u1 = URI->new($uri_str);
my $u2 = URI->new;
$u2->scheme($u1->scheme);
$u2->authority($u1->authority);
$u2->path($u1->path);
$u2->query_form($u1->query_form);
$u2->fragment($u1->fragment);
return $u2->as_string;
}
#=== FUNCTION ================================================================
# NAME: coalesce
# PURPOSE: To find the first defined argument and return it
# PARAMETERS: Arbitrary number of arguments
# RETURNS: The first defined argument or undef if there are none
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub coalesce {
foreach (@_) {
return $_ if defined($_);
}
return undef; ## no critic
}
#=== FUNCTION ================================================================
# NAME: Usage
# PURPOSE: Display a usage message and exit
# PARAMETERS: None
# RETURNS: To command line level with exit value 1
# DESCRIPTION: Builds the usage message using global values
# THROWS: no exceptions
# COMMENTS: none
# SEE ALSO: n/a
#===============================================================================
sub Usage {
print STDERR <<EOD;
$PROG v$VERSION
Usage: $PROG [options]
Scans STDIN in the assumption that it is plain text or some kind of hybrid
Markdown looking for URLs, mail links, HTML and Markdown elements.
Options:
-help Display this information
-[no]simple Turns 'simple' mode on or off; it is off by default.
In 'simple' mode links are not accumulated and written
at the end of the data in a 'Links' section. URLs are
converted to a Markdown form though.
-[no]entities Turns 'entities' mode on and off; it is off by
default. When on this mode causes each line to be
scanned for characters best represented as HTML
entities and to change them appropriately. For example
the '<' character becomes '&lt;'.
-[no]stamp Turns 'stamp' mode on and off; it is off by default.
When this mode is on an HTML comment is written at the
start of the output containing the name of the script
and its version number.
EOD
exit(1);
}
#=== FUNCTION ================================================================
# NAME: Options
# PURPOSE: Processes command-line options
# PARAMETERS: $optref Hash reference to hold the options
# RETURNS: Undef
# DESCRIPTION:
# THROWS: no exceptions
# COMMENTS: none
# SEE ALSO: n/a
#===============================================================================
sub Options {
my ($optref) = @_;
my @options = ( "help", "simple!", "entities!", "stamp!", );
if ( !GetOptions( $optref, @options ) ) {
die "Failed to process options\n";
}
return;
}
# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker

222
Show_Submission/move_shownotes Executable file
View File

@@ -0,0 +1,222 @@
#!/bin/bash -
#===============================================================================
#
# FILE: move_shownotes
#
# USAGE: ./move_shownotes original_show new_show
#
# DESCRIPTION: After a show has been moved from one slot to another on the
# HPR server this script ensures that the local versions of
# processed show notes correspond to the change by moving
# directories around and correcting file names and some
# contents.
#
# ** Under development **
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.2
# CREATED: 2022-12-02 22:12:20
# REVISION: 2022-12-06 23:03:12
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
DIR=${0%/*}
VERSION='0.0.2'
STDOUT="/dev/fd/2"
#
# Load library functions
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
# shellcheck source=/home/cendjm/bin/function_lib.sh
source "$LIB"
#
# Colour codes
#
define_colours
#=== FUNCTION ================================================================
# NAME: _usage
# DESCRIPTION: Reports usage; always exits the script after doing so
# PARAMETERS: 1 - the integer to pass to the 'exit' command
# RETURNS: Nothing
#===============================================================================
_usage () {
local -i result=${1:-0}
cat >$STDOUT <<-endusage
Usage: ./${SCRIPT} [-h] [-d] [-D] original_show new_show
Version: $VERSION
When a show has been processed and posted on the server, but needs to be moved
to another slot (after getting the host's permission) this script assists with
the processes involved in ensuring the local shownote caches reflect the
server state.
The need to move a show is rare, but often follows when a host has forgotten
the request to spread their shows out by at least two weeks and has sent them
in to clots too close in time to one another.
Options:
-h Print this help
-d Select dry run mode
-D Turn on debug mode with lots of extra output
Arguments:
original_show
new_show
Both arguments can use either an integer number or a format like 'hpr9999'
and also compensate for the use of 'HPR' rather than 'hpr'.
Examples
./${SCRIPT} -h
./${SCRIPT} -d 3742 3756
./${SCRIPT} -D 3738 3746
endusage
exit "$result"
}
#=== FUNCTION ================================================================
# NAME: _DEBUG
# DESCRIPTION: Writes one or more message lines if in DEBUG mode
# PARAMETERS: List of messages
# RETURNS: Nothing
#===============================================================================
_DEBUG () {
[ "$DEBUG" == 0 ] && return
for msg in "$@"; do
printf 'D> %s\n' "$msg"
done
}
#=== FUNCTION ================================================================
# NAME: coloured
# DESCRIPTION: Write a message with colour codes
# PARAMETERS: 1 - colour name
# 2 - message
# RETURNS: Nothing
#===============================================================================
coloured () {
local colour="${1:-green}"
local message="${2:-no message}"
printf '%s%s%s\n' "${!colour}" "$message" "${reset}"
}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Directories
#
BASEDIR="$HOME/HPR/Show_Submission"
RES="$BASEDIR/reservations"
SN="$BASEDIR/shownotes"
#
# Check arguments
#
if [[ $# -ne 2 ]]; then
_usage 1
fi
#
# Check the arguments and standardise them
#
declare -a ARGS
re="^(hpr|HPR)?([0-9]{1,4})$"
for arg; do
if [[ $arg =~ $re ]]; then
printf -v n '%04d' "${BASH_REMATCH[2]}"
ARGS+=("hpr$n")
else
coloured 'red' "Invalid show specification: $arg"
fi
done
if [[ ${#ARGS[@]} -ne 2 ]]; then
coloured 'red' "Unable to continue with invalid arguments"
exit 1
fi
#
# Save the parsed show details
#
original_show="${ARGS[0]}"
new_show="${ARGS[1]}"
coloured 'blue' "Moving from $original_show to $new_show"
#-------------------------------------------------------------------------------
# Consistency checks
#-------------------------------------------------------------------------------
#
# We expect the original show to have been moved into the reservations/
# directory
#
if [[ ! -e "$RES/$original_show" ]]; then
coloured 'red' "Did not find the moved original show as $RES/$original_show."
#
# Perhaps the original show hasn't been moved yet, otherwise there's an
# error - the wrong show has been specified.
#
if [[ -e "$SN/$original_show" ]]; then
coloured 'red' "Found the original show as $SN/$original_show."
coloured 'yellow' "Double check that this show has been moved on the HPR server."
coloured 'yellow' "If so, wait for 'scan_HPR' to move it locally."
coloured 'yellow' "If not, the show numbers you used are wrong."
else
coloured 'red' "Did not find the original show as $SN/$original_show."
coloured 'yellow' "It looks as if the show numbers you used are wrong."
fi
coloured 'red' "Unable to continue"
exit
fi
#
# Is the destination free? It may have been filled with a dummy show by
# 'scrape_HPR' or there may have been a mistake in the arguments.
#
if [[ -e "$SN/$new_show" ]]; then
coloured 'yellow' "Found a directory at $SN/$new_show"
#
# A dummy show directory will only contain 'shownotes.txt', 'hpr1234.out'
# and 'hpr1234.html' to keep other scripts happy and a file called .dummy
# to be detected by this script. So look for this signature, and if not
# found halt with an error.
#
if [[ ! -e "$SN/$new_show/.dummy" ]]; then
coloured 'red' "The directory seems to hold a live show!"
coloured 'red' "Cannot continue; please check the situation"
exit
fi
fi
#
# Now we should have found the original show has been placed in the
# reservations/ directory and the new slot should have had a dummy show placed
# in it. We now need to move the reserved show into the destination slot after
# removing the dummy show. Once done we need to rename files and change the
# contents of some files.
#
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21

2348
Show_Submission/parse_JSON Executable file
View File

File diff suppressed because it is too large Load Diff

23
Show_Submission/pic_definitions.tpl Normal file
View File

@@ -0,0 +1,23 @@
[%# Template for generating index.html via Markdown. -%]
[%# Last updated: |2022-12-04 12:53:26| -%]
[% USE piclist = datafile(piclist) -%]
[%# >> Single picture, no thumbnail -%]
[% MACRO pic1(prefix,title,fn1) BLOCK -%]
[%# SET fn1 = "$prefix" _ fn1.split('/').slice(-1).join('/') -%]
[% SET fn1 = "$prefix" _ fn1.remove('^.+/uploads/') -%]
[%title%]\
![[%title%]]([%fn1%])
[% END -%]
[%# >> Thumbnail, linking to full-sized picture -%]
[% MACRO pic2(prefix,title,fn1,fn2) BLOCK -%]
[%# SET fn1 = "$prefix" _ fn1.split('/').slice(-1).join('/') -%]
[%# SET fn2 = "$prefix" _ fn2.split('/').slice(-1).join('/') -%]
[% SET fn1 = "$prefix" _ fn1.remove('^.+/uploads/') -%]
[% SET fn2 = "$prefix" _ fn2.remove('^.+/uploads/') -%]
[%title%]\
[![[%title%]]([%fn1%])]([%fn2%])
<br/><small><small><em>Click the thumbnail to see the full-sized image</em></small></small>
[% END -%]
[%#
# vim: syntax=tt2:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
-%]

1
Show_Submission/query2csv Symbolic link
View File

@@ -0,0 +1 @@
/home/cendjm/HPR/Database/query2csv

1
Show_Submission/query2json Symbolic link
View File

@@ -0,0 +1 @@
/home/cendjm/HPR/Database/query2json

316
Show_Submission/resume_workflow Executable file
View File

@@ -0,0 +1,316 @@
#!/usr/bin/env perl
#===============================================================================
#
# FILE: resume_workflow
#
# USAGE: ./resume_workflow epno
#
# DESCRIPTION: For use with an HPR show that may not be fully processed.
# Determines which workflow step is the next that needs to be
# run, displays what it is and the script needed to perform it,
# and passes the script name to the caller.
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.2
# CREATED: 2024-02-10 17:53:09
# REVISION: 2024-03-10 19:15:14
#
#===============================================================================
use v5.16;
use strict;
use warnings;
use feature qw{ postderef say signatures state try };
no warnings
qw{ experimental::postderef experimental::signatures experimental::try };
use Config::General;
use List::Util qw( maxstr minstr );
use List::MoreUtils qw( uniq );
use File::Slurper qw( read_text read_lines );
use JSON;
use Log::Handler;
use Data::Dumper;
#
# Version number (Incremented by Vim)
#
our $VERSION = '0.0.2';
#
# Script and directory names
#
( my $PROG = $0 ) =~ s|.*/||mx;
( my $DIR = $0 ) =~ s|/?[^/]*$||mx;
$DIR = '.' unless $DIR;
#-------------------------------------------------------------------------------
# Declarations
#-------------------------------------------------------------------------------
#
# Constants and other declarations
#
my $basedir = "$ENV{HOME}/HPR/Show_Submission";
my $cache = "$basedir/shownotes";
my $logs = "$basedir/logs";
my $logfile = "$logs/$PROG.log";
my $j_shownotes = 'shownotes.json';
my $formatfile = '.format';
my $statusfile = '.status';
my $assetfile = '.assets';
my $picturefile = '.pictures';
#
# Setting the environment variable 'resume_workflow_DEBUG' to greater than
# 0 enables debug output. Different values control different output
#
my $DEBUG = 0;
my $DEBUG_env = "${PROG}_DEBUG";
if ( defined($ENV{"$DEBUG_env"})) {
$DEBUG = $ENV{"$DEBUG_env"};
}
my ($showno, $showid, $missed);
my %showdata;
my @statustags = qw(
copied
parsed
format
metadata
pictures
assets
edited
converted
rendered
uploaded
database
reported
);
#
# Set up the tag hash
#
# {{{
my %taghash = (
'copied' => {
'description' => 'Copy notes',
'script' => 'copy_shownotes',
'level' => 1,
},
'parsed' => {
'description' => 'Parse raw',
'script' => 'do_parse',
'level' => 1,
},
'format' => {
'description' => 'Change format',
'script' => 'do_change_format',
'level' => 2,
},
'metadata' => {
'description' => 'Edit metadata',
'script' => 'do_edit_shownotes',
'level' => 2,
},
'pictures' => {
'description' => 'Process pictures',
'script' => 'do_pictures',
'level' => 2,
},
'assets' => {
'description' => 'Upload assets',
'script' => 'do_asset_upload',
'level' => 2,
},
'edited' => {
'description' => 'Edit notes',
'script' => 'do_vim',
'level' => 1,
},
'converted' => {
'description' => 'Run Pandoc',
'script' => 'do_pandoc',
'level' => 1,
},
'rendered' => {
'description' => 'Run browser',
'script' => 'do_browse',
'level' => 1,
},
'uploaded' => {
'description' => 'Upload HTML',
'script' => 'do_upload',
'level' => 1,
},
'database' => {
'description' => 'Change database status',
'script' => 'do_update_reservations',
'level' => 1,
},
'reported' => {
'description' => 'Report change to Matrix',
'script' => 'do_report',
'level' => 1,
},
);
# }}}
_debug($DEBUG == 3, '%taghash: ' . Dumper(\%taghash));
#
# Enable Unicode mode
#
binmode STDOUT, ":encoding(UTF-8)";
binmode STDERR, ":encoding(UTF-8)";
#-------------------------------------------------------------------------------
# Argument 1 must be the episode number in '1234' or 'hpr1234' form
#-------------------------------------------------------------------------------
$showno = shift;
die "Usage: $PROG epno\n" unless $showno;
if (($showno) =~ /^(?:hpr)?(\d+)$/) {
#
# Make an id in 'hpr1234' format
#
$showid = sprintf('hpr%04d',$showno);
}
else {
die "Invalid episode format: $showno\n";
}
#-------------------------------------------------------------------------------
# Set up logging keeping the default log layout except for the date. The format
# is "%T [%L] %m" where '%T' is the timestamp, '%L' is the log level and '%m is
# the message.
#-------------------------------------------------------------------------------
my $log = Log::Handler->new();
$log->add(
file => {
timeformat => "%Y/%m/%d %H:%M:%S",
filename => $logfile,
minlevel => 0,
maxlevel => 7,
}
);
#
# Check the existence of the requested show directory
#
unless (-d "$cache/$showid") {
die "Directory $cache/$showid not found\n";
}
$log->info("Processing show $showid");
#-------------------------------------------------------------------------------
# Collect show data
#-------------------------------------------------------------------------------
#
# Declared format
#
if ( -e "$cache/$showid/$formatfile" ) {
chomp($showdata{format} = read_text("$cache/$showid/$formatfile"));
}
else {
$showdata{format} = undef;
}
#
# Current status
#
if ( -e "$cache/$showid/$statusfile" ) {
$showdata{statuslist} = [];
push(@{$showdata{statuslist}}, read_lines("$cache/$showid/$statusfile"));
}
else {
$showdata{statuslist} = undef;
}
#
# Assets
#
if ( -e "$cache/$showid/$assetfile" ) {
$showdata{assets} = [];
push(@{$showdata{assets}}, read_lines("$cache/$showid/$assetfile"));
$taghash{assets}->{level} = 1;
}
else {
$showdata{assets} = undef;
}
#
# Pictures (a subset of assets)
#
if ( -e "$cache/$showid/$picturefile" ) {
$showdata{pictures} = [];
push(@{$showdata{pictures}}, read_lines("$cache/$showid/$picturefile"));
$taghash{pictures}->{level} = 1;
}
else {
$showdata{pictures} = undef;
}
_debug($DEBUG == 3, '%showdata: ' . Dumper(\%showdata));
#-------------------------------------------------------------------------------
# Determine the incomplete steps in the workflow
#-------------------------------------------------------------------------------
$missed = 0;
for my $tag (@statustags) {
my $th = $taghash{$tag};
if ( $th->{level} == 1
&& scalar( grep( /$tag/, @{ $showdata{statuslist} } ) ) == 0 )
{
printf "%-40s (./%s %d)\n",
"Missing '$th->{description}' step",
$th->{script}, $showno;
$missed++;
}
}
#
# Nothing missed, so say so
#
say "Nothing to do for show $showno" if ($missed == 0);
exit;
#=== FUNCTION ================================================================
# NAME: _debug
# PURPOSE: Prints debug reports
# PARAMETERS: $active Boolean: 1 for print, 0 for no print
# $messages... Arbitrary list of messages to print
# RETURNS: Nothing
# DESCRIPTION: Outputs messages if $active is true. It removes any trailing
# newline from each one and then adds one in the 'print' to the
# caller doesn't have to bother. Prepends each message with 'D>'
# to show it's a debug message.
# THROWS: No exceptions
# COMMENTS: Differs from other functions of the same name
# SEE ALSO: N/A
#===============================================================================
sub _debug {
my $active = shift;
my $message;
return unless $active;
while ($message = shift) {
chomp($message);
print STDERR "D> $message\n";
}
}
# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker

75
Show_Submission/sync_hpr Executable file
View File

@@ -0,0 +1,75 @@
#!/bin/bash -
#===============================================================================
#
# FILE: sync_hpr
#
# USAGE: ./sync_hpr
#
# DESCRIPTION: Runs rsync to pull the contents of the 'upload' directory from
# the HPR server. Uses a filter in the file '.rsync_hpr_upload'
# so that it only copies the files relevant to managing
# submitted shows. Files and directories deleted on the HPR
# server are also deleted here.
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.7
# CREATED: 2020-01-03 23:18:58
# REVISION: 2023-06-14 18:06:06
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
# DIR=${0%/*}
#
# Load library functions
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
# shellcheck source=/home/cendjm/bin/function_lib.sh
source "$LIB"
#
# Colour codes
#
define_colours
#
# Directories
#
BASEDIR="$HOME/HPR/Show_Submission"
HPRUPLOAD='hpr@hackerpublicradio.org:upload/'
UPLOAD="$BASEDIR/upload"
#
# Settings
#
# PORT=22074
PORT=22
#
# Change directory
#
cd "$BASEDIR" || { echo "$SCRIPT: Failed to change directory to $BASENAME"; exit 1; }
#
# Check the tunnel is open
#
if ! tunnel_is_open; then
echo "$SCRIPT: Open the tunnel before running"
exit 1
fi
#
# Use 'rsync' to collect relevant data from the HPR server's 'upload'
# directory.
#
rsync -vaP -e "ssh -p $PORT" --delete --filter=". .rsync_hpr_upload" "$HPRUPLOAD" "$UPLOAD"
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21

96
Show_Submission/validate_html Executable file
View File

@@ -0,0 +1,96 @@
#!/usr/bin/env perl
#===============================================================================
#
# FILE: validate_html
#
# USAGE: ./validate_html file
#
# DESCRIPTION: Validate HTML show notes using HTML::Valid
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: Much of this code is as seen in 'htmlok' because the
# documentation with HTML::Valid is poor.
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.3
# CREATED: 2018-06-01 16:25:27
# REVISION: 2019-11-10 21:25:06
#
#===============================================================================
use 5.010;
use strict;
use warnings;
use utf8;
# In the POD, without explanation
use FindBin '$Bin';
# Found in 'htmlok', not documented
use HTML::Valid 'sanitize_errors';
use Path::Tiny;
#
# Version number (manually incremented)
#
our $VERSION = '0.0.3';
#
# Script and directory names
#
( my $PROG = $0 ) =~ s|.*/||mx;
( my $DIR = $0 ) =~ s|/?[^/]*$||mx;
$DIR = '.' unless $DIR;
#-------------------------------------------------------------------------------
# Declarations
#-------------------------------------------------------------------------------
#
# Constants and other declarations
#
my $basedir = "$ENV{HOME}/HPR/Show_Submission";
#
# Enable Unicode mode
#
binmode STDOUT, ":encoding(UTF-8)";
binmode STDERR, ":encoding(UTF-8)";
#
# Create the object
#
my $htv = HTML::Valid->new(
quiet => 1, # omit summary, etc
doctype => 'omit',
show_body_only => 1, # print only the contents of the body tag as an HTML fragment
show_info => 1, # test; omit info-level messages
show_warnings => 1, # include warnings if set
);
#
# We expect a file as an argument
#
my $file = shift @ARGV;
die "Usage: $PROG filename\n" unless $file;
my $content;
if ( -f $file ) {
$content = path($file)->slurp_utf8();
$htv->set_filename($file);
}
else {
die "$PROG: Cannot find '$file'.\n";
}
#
# Largely undocumented but seen in 'htmlok'
#
my ( undef, $errors ) = $htv->run($content);
if ($errors) {
print sanitize_errors ($errors);
exit 1;
}
exit;
# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker