#!/bin/bash -
#===============================================================================
#
# FILE: do_pandoc
#
# USAGE: ./do_pandoc [option] <epno>
#
# DESCRIPTION: Performs an "intelligent" Pandoc run on various types of show
# notes. Converts various markup formats into HTML. Treate plain
# text as Markdown, though this depends on the prior editing
# step doing the right thing. Handles pictures and other assets
# in plain text shows that have them - this is done by the edit
# phase adding TT² macros and this script processing them with
# 'tpage'.
# Version 0:2:5 (released 2022-12-04) has not yet been fully
# tested. Seems reliable 2023-03-03.
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: 2021-04-03: removed the TOC option
# 2021-11-07: Added --strip-comments to the HTML snippet
# generation stage
# 2022-11-01: Big rewrite over the month. Refer to
# do_pandoc_0.1.6 for the previous version since there have been
# some big changes.
# 2022-12-17: Ending reliance on shownotes.txt, and using
# shownotes.json instead into the future. Some massive tidying
# and rationalisation are still required.
# 2023-03-03: If the title contained a quote then the previous
# algorithm made bad YAML which caused the 'full' html to fail.
# Fixed.
# 2023-11-15: The 'prefix' setting is wrong. It needs another
# 'hpr1234' directory level. This is needed because the
# top-level stuff for the show is already in such a directory,
# and while we have assets in a sub-directory we need to be
# careful about collisions.
#
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.2.10
# CREATED: 2016-08-16 15:34:30
# REVISION: 2024-02-18 13:27:40
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
#DIR=${0%/*}
VERSION='0.2.10'
STDOUT="/dev/fd/2"
#
# Load library functions
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
# shellcheck source=/home/cendjm/bin/function_lib.sh
source "$LIB"
#
# Colour codes
#
define_colours
# {{{ Functions: -- _usage -- _DEBUG --
#=== FUNCTION ================================================================
# NAME: _usage
# DESCRIPTION: Report usage
# PARAMETERS: None
# RETURNS: Nothing
#===============================================================================
_usage () {
cat >$STDOUT <<-endusage
Usage: ./${SCRIPT} [-h] [-d] [-D] shownumber
Version: $VERSION
Runs Pandoc against a particular show, choosing a format as
defined by the declared format (in the file '.format').
(In this version there is no method to force an explicit input format)
Options:
-h Print this help
-d Select dry run mode
-D Turn on debug mode with lots of extra output
The default behaviour is now to access the '.format' file in the show
directory.
Arguments:
shownumber
Examples
./${SCRIPT} -h
./${SCRIPT} -d 2240
./${SCRIPT} -D 2250
endusage
exit
}
#=== FUNCTION ================================================================
# NAME: _DEBUG
# DESCRIPTION: Writes one or more message lines if in DEBUG mode
# PARAMETERS: List of messages
# RETURNS: Nothing
#===============================================================================
_DEBUG () {
[ "$DEBUG" == 0 ] && return
for msg in "$@"; do
printf 'D> %s\n' "$msg"
done
}
# }}}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Formats offered by the web form on the HPR site:
#
# Keyword Explanation
# ------- -----------
# plain_text Plain text
# html5 HTML5 (preferred)
# markdown_standard Markdown (standard)
# Markdown_GitHub Markdown (GitHub flavoured)
# Markdown_Pandoc Markdown (Pandoc flavoured)
# restructured_text RestructuredText
# txt2tags txt2tags
#
#
# Hash to perform translation from declared format to Pandoc "-from" value
#
# {{{ -- 'lookup' hash --
declare -A lookup
lookup[plain_text]='markdown-implicit_figures'
lookup[html5]='html'
lookup[markdown_standard]='markdown_strict'
lookup[Markdown_GitHub]='gfm' # Extensions are limited
lookup[Markdown_Pandoc]='markdown-implicit_figures'
lookup[restructured_text]='rst'
lookup[txt2tags]='t2t'
# }}}
#
# Hash for options when generating standalone readable HTML
#
# (The 'smart' extension is only applicable to markdown, commonmark, latex,
# mediawiki, org, rst, twiki; we want to turn it off to remove smart quotes)
#
# {{{ -- 'options' hash --
declare -A options
options[plain_text]='-smart'
options[html5]=''
options[markdown_standard]='+yaml_metadata_block'
options[Markdown_GitHub]=''
options[Markdown_Pandoc]='-smart+yaml_metadata_block'
options[restructured_text]='-smart'
options[txt2tags]=''
# }}}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Sanity checks
#
JQ=$(command -v jq)
[ -n "$JQ" ] || { echo "Program 'jq' was not found"; exit 1; }
# YQ=$(command -v yq)
# [ -n "$YQ" ] || { echo "Program 'yq' was not found"; exit 1; }
#
# Process options first
#
while getopts :dDh opt
do
case "${opt}" in
d) DRYRUN=1;;
D) DEBUG=1;;
h) _usage;;
?) echo "$SCRIPT: Invalid option; aborting"; exit 1;;
esac
done
shift $((OPTIND - 1))
#
# Default options if not provided
#
DEBUG=${DEBUG:-0}
DRYRUN=${DRYRUN:-0}
#
# Check there's an argument after removing any options. Abort if not
#
if [[ $# -ne 1 ]]; then
_usage
fi
#
# Declare variables for later
# TODO: Is this necessary?
#
declare SHOWID FROM POPTIONS
#
# Make the explicit show id, catering for leading zeroes (belt & braces)
#
printf -v SHOWID 'hpr%04d' "$1"
#
# Make temporary files and set traps to delete them
#
TMP1=$(mktemp) || {
echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
exit 1
}
TMP2=$(mktemp) || {
echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
exit 1
}
TMP3=$(mktemp) || {
echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
exit 1
}
trap 'cleanup_temp $TMP1 $TMP2 $TMP3' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
#
# Paths to files
#
# ------------------------------------------------------------------------------
# Main directory
BASENAME="$HOME/HPR/Show_Submission"
# JSON to YAML Perl script
J2Y="$BASENAME/author_title.pl"
[ -e "$J2Y" ] || { echo "Program '$J2Y' was not found"; exit 1; }
# The notes for all shows are here
SHOWNOTES="$BASENAME/shownotes"
# Notes for this show are here
SHOWDIR="$SHOWNOTES/$SHOWID"
# Paths to all files already created or being created here
#RAWFILE="$SHOWDIR/shownotes.txt"
JSONFILE="$SHOWDIR/shownotes.json"
FMTFILE="$SHOWDIR/.format"
PICLIST="$SHOWDIR/.pictures"
STATUS="$SHOWDIR/.status"
EXTRACT="$SHOWDIR/${SHOWID}.out"
FULLHTML="$SHOWDIR/${SHOWID}_full.html"
HTML="$SHOWDIR/${SHOWID}.html"
# TT² macros and paths for adding pictures
PICTPL="$BASENAME/pic_definitions.tpl"
MANIFEST="$SHOWDIR/.pictures.mf" # From do_pictures
# ------------------------------------------------------------------------------
#
# The partial URL for pictures on the HPR site
#
BASEURL='https://hackerpublicradio.org/eps/'
#{{{ --- Obsolete 2022-12-17 ---
#
# Make a metadata file by parsing the raw data file fields with awk. Save it
# in a temporary file.
#
# See the original do_pandoc_0.1.6 for the extended comments that led to this
# design. We were trying to make acceptable YAML, but ended up extracting
# metadata from the result.
# ----
# TODO: 2022-12-01 Rationalise all of this; it's full of debris from previous
# attempts to solve the problem of passing metadata to Pandoc.
# ----
#
# awk -f - "$RAWFILE" > "$TMP1" <<'ENDAWK'
# BEGIN {print "---"}
# /^Title:/ && got_title == 0 {
# sub(/^Title:\s+/,"")
# printf "#title: %s\n",$0
# gsub(/'/,"''")
# printf "title: '%s'\n",$0
# got_title = 1
# }
# /^Host_Name:/ && got_author == 0 {
# sub(/^Host_Name:\s+/,"")
# printf "#author: %s\n",$0
# gsub(/'/,"''")
# printf "author: '%s'\n",$0
# got_author = 1
# }
# END {print "---"}
# ENDAWK
#}}}
#
# Use 'jq' to parse the JSON and make the metadata (in the form of YAML)
# needed for Pandoc
#
# Non-YAML alternative - not chosen
#jqprog="@text \"author: \(.host.Host_Name)\ntitle: \(.episode.Title)\""
#
# Testing another formatter (Journal 2023-03-03)
#jqprog="@sh \"---\nauthor: \(.host.Host_Name)\ntitle: \(.episode.Title)\n---\""
# Added quotes around the generated strings (2023-03-31)
# jqprog="@text \"---\nauthor: \(.host.Host_Name)\ntitle: \(.episode.Title)\n---\""
# Moved to 'yq' 2023-04-01
# jqprog="@text \"---\nauthor: '\(.host.Host_Name)'\ntitle: '\(.episode.Title)'\n---\""
# jq -r "$jqprog" "$JSONFILE" > "$TMP1"
# On 2023-10-01 wrote a Perl JSON to YAML generator just for these two
# elements. It's called 'author_title.pl'
# yqprog='{author:.host.Host_Name,title:.episode.Title}'
# ( echo "---"; $YQ -y "$yqprog" "$JSONFILE"; echo "---"; ) > "$TMP1"
$J2Y "$JSONFILE" "$TMP1"
_DEBUG "YAML:" "$(cat "$TMP1")"
#
# Check the main output file from do_parse exists
#
if [[ ! -e $EXTRACT ]]; then
echo "$SCRIPT: ${red}File not found: $EXTRACT${reset}"
exit 1
fi
#
# Get the format or fail with an error
#
if [[ -e $FMTFILE ]]; then
FORMAT=$(cat "$FMTFILE")
else
# TODO: Should we default to something rather than abort?
echo "$SCRIPT: ${red}Could not find declared format (.format file)${reset}"
echo "${yellow}Has do_parse been run? If so try and fix the .format file.${reset}"
exit
fi
#
# Need to match plain text and Markdown variants when deciding to use the
# manifest file and 'tpage'
#
FMTRE='^(plain_text|[Mm]arkdown_)'
#
# Determine if there are pictures
#
if [[ -e $PICLIST ]]; then
hasPictures=1
else
hasPictures=0
fi
#
# Here we use the declared format to determine what to do.
#
# This code now blocks HTML->HTML processing since it just confuses matters.
# The files hprNNNN.out and hprNNNN.html are linked to one another, so editing
# the former will edit the latter in preparation for uploading if there are
# changes that need to be made. See the journal discussion for 2018-05-24.
# ---
# TODO: 2022-12-01 If the notes are HTML but declared as something else then
# this check will not work. Trouble is, parse_JSON will have worked this out
# but not saved it so we can't avoid running Pandoc on HTML in this case.
# ---
# TODO: 2022-12-01 Look at resolving this in parse_JSON?
# ---
#
if [[ $FORMAT == 'html5' ]]; then
echo "${red}Running Pandoc on HTML is not allowed${reset}"
echo "${yellow}Run do_edit to edit as necessary and do_browser to view${reset}"
exit
elif [[ $FORMAT == 'plain_text' ]]; then
echo "${yellow}Format chosen is plain text${reset}"
if [[ $hasPictures -eq 0 ]]; then
echo "${yellow}This will be treated as Markdown${reset}"
else
echo "${yellow}This will be treated as Markdown and will need special action${reset}"
echo "${yellow}since there are pictures${reset}"
fi
fi
FROM=${lookup[$FORMAT]}
POPTIONS=${options[$FORMAT]}
echo "${green}Will process $FORMAT with 'pandoc -f ${FROM}'${reset}"
echo "${yellow}Options chosen for --standalone are '${POPTIONS}'${reset}"
#
# Only for plain text notes, process pictures for the HTML we'll be adding to
# the database.
#
# We need the following things:
# - The notes to be plain text format
# - Some pictures
# - Files written by do_parse and do_pictures:
# - .assets (not used here)
# - .pictures (were previously used here, but no longer)
# - .pictures.mf (needed by the TT² macros)
# - .pictures.tt2 (the TT² macro calls - already edited into the notes)
# - pic_definitions.tpl (macro definitions, common to all shows)
# - Not to be in dry run mode; if we are we just report intentions
#
if [[ $DRYRUN -eq 0 ]]; then
# if [[ $FORMAT == 'plain_text' && -e $MANIFEST ]]; then
if [[ $FORMAT =~ $FMTRE && -e $MANIFEST ]]; then
#
# Deal with pictures using the TT² macros
#
_DEBUG "Processing TT² inclusions"
# Make a picture manifest with a header
awk 'BEGIN{print "file : thumb"}{p1=$0; getline p2; printf "%s : %s\n",p1,p2}' \
"$MANIFEST" > "$TMP2"
_DEBUG "Picture list:" "$(cat "$TMP2")" "---" \
"BASEURL=${BASEURL}${SHOWID}/" \
"EXTRACT=$EXTRACT" \
"Extract file contents:" "$(cat "$EXTRACT")" "---"
# Run the macros on the notes to make $TMP3 for Pandoc
tpage --pre_process="$PICTPL" \
--define "piclist=$TMP2" \
--define "prefix=${BASEURL}${SHOWID}/${SHOWID}/" \
"$EXTRACT" > "$TMP3"
_DEBUG "Processed by tpage" "$(cat "$TMP3")" "---"
else
# Not plain text or a Markdown variant and no pictures, so put the
# notes in $TMP3 where Pandoc will look for them
cat "$EXTRACT" > "$TMP3"
fi
else
#
# We would not have used TT² unless the notes were plain text (or
# a Markdown variant) and there were pictures
#
# if [[ $FORMAT == 'plain_text' && -e $MANIFEST ]]; then
if [[ $FORMAT =~ $FMTRE && -e $MANIFEST ]]; then
echo "${yellow}Would have prepared TT² code for pandoc${reset}"
fi
fi
#
# Generate an HTML snippet for adding to the database.
# (Note 2021-11-24: Added --ascii option.)
#
if [[ $DRYRUN -eq 0 ]]; then
# shellcheck disable=SC2086
pandoc -f "$FROM"-smart -t html5 --ascii --no-highlight --strip-comments \
"$TMP3" -o "$HTML" # $EXTRAS
RES=$?
if [[ $RES -eq 0 ]]; then
echo "$SCRIPT: ${green}Created shownotes/$SHOWID/${HTML##*/}${reset}"
else
echo "$SCRIPT: ${red}Oops! Something went wrong! (line $LINENO)${reset}"
echo "${yellow}$SCRIPT: Aborting now${reset}"
exit 1
fi
else
echo "${yellow}Would have run pandoc to make HTML for upload${reset}"
fi
#
# Make HTML for proof reading. All pictures referenced are now on the HPR
# server (we ran 'do_asset_upload'), so we want to refer to them here.
#
# File $TMP2 contains the .pictures.mf contents with a header line; and it
# contains data for the macros. It was created when we prepared the main HTML
# for the database. We use $BASEURL again here because we want to reference
# the pictures on the server.
#
# We use the awk-formatted file (now yq-formatted) in $TMP1 from earlier to do
# this. At the end TMP3 contains Markdown for Pandoc.
#
if [[ $DRYRUN -eq 0 ]]; then
# if [[ $FORMAT == 'plain_text' && -e $MANIFEST ]]; then
if [[ $FORMAT =~ $FMTRE && -e $MANIFEST ]]; then
tpage --pre_process="$PICTPL" \
--define "piclist=$TMP2" \
--define "prefix=${BASEURL}${SHOWID}/${SHOWID}/" \
"$EXTRACT" > "$TMP3"
else
cat "$EXTRACT" > "$TMP3"
fi
#
# Generate complete HTML that we can proofread. We need metadata for this
# stand-alone HTML which is in the form of YAML in this version.
#
# ----------------------------------------------------------------------
# Original options below when using 'awk' to parse shownotes.txt:
# --metadata="$(sed -n '/^#author:/{s/#//;p}' "$TMP1")" \
# --metadata="$(sed -n '/^#title:/{s/#//;p}' "$TMP1")" \
#
# shellcheck disable=SC2086
pandoc -f ${FROM}${POPTIONS} -t html5 --ascii \
--standalone --template=hpr.html5 --no-highlight \
-c https://hackerpublicradio.org/css/hpr.css \
--metadata-file="$TMP1" -o "$FULLHTML" "$TMP3"
RES=$?
if [[ $RES -eq 0 ]]; then
echo "$SCRIPT: ${green}Created shownotes/$SHOWID/${FULLHTML##*/}${reset}"
else
echo "$SCRIPT: ${red}Oops! Something went wrong making the full HTML! (line $LINENO)${reset}"
fi
else
# Dry run
echo "${yellow}Would have run pandoc to make HTML for proof reading${reset}"
fi
#
# Set the status for this show
#
if [[ $DRYRUN -eq 0 ]]; then
echo "converted" >> "$STATUS"
fi
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker