525 lines
16 KiB
Bash
Executable File
525 lines
16 KiB
Bash
Executable File
#!/bin/bash -
|
|
#===============================================================================
|
|
#
|
|
# FILE: do_pandoc
|
|
#
|
|
# USAGE: ./do_pandoc [option] <epno>
|
|
#
|
|
# DESCRIPTION: Performs an "intelligent" Pandoc run on various types of show
|
|
# notes. Converts various markup formats into HTML. Treate plain
|
|
# text as Markdown, though this depends on the prior editing
|
|
# step doing the right thing. Handles pictures and other assets
|
|
# in plain text shows that have them - this is done by the edit
|
|
# phase adding TT² macros and this script processing them with
|
|
# 'tpage'.
|
|
# Version 0:2:5 (released 2022-12-04) has not yet been fully
|
|
# tested. Seems reliable 2023-03-03.
|
|
#
|
|
# OPTIONS: ---
|
|
# REQUIREMENTS: ---
|
|
# BUGS: ---
|
|
# NOTES: 2021-04-03: removed the TOC option
|
|
# 2021-11-07: Added --strip-comments to the HTML snippet
|
|
# generation stage
|
|
# 2022-11-01: Big rewrite over the month. Refer to
|
|
# do_pandoc_0.1.6 for the previous version since there have been
|
|
# some big changes.
|
|
# 2022-12-17: Ending reliance on shownotes.txt, and using
|
|
# shownotes.json instead into the future. Some massive tidying
|
|
# and rationalisation are still required.
|
|
# 2023-03-03: If the title contained a quote then the previous
|
|
# algorithm made bad YAML which caused the 'full' html to fail.
|
|
# Fixed.
|
|
# 2023-11-15: The 'prefix' setting is wrong. It needs another
|
|
# 'hpr1234' directory level. This is needed because the
|
|
# top-level stuff for the show is already in such a directory,
|
|
# and while we have assets in a sub-directory we need to be
|
|
# careful about collisions.
|
|
#
|
|
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
|
# VERSION: 0.2.10
|
|
# CREATED: 2016-08-16 15:34:30
|
|
# REVISION: 2024-02-18 13:27:40
|
|
#
|
|
#===============================================================================
|
|
|
|
set -o nounset # Treat unset variables as an error
|
|
|
|
SCRIPT=${0##*/}
|
|
#DIR=${0%/*}
|
|
|
|
VERSION='0.2.10'
|
|
|
|
STDOUT="/dev/fd/2"
|
|
|
|
#
|
|
# Load library functions
|
|
#
|
|
LIB="$HOME/bin/function_lib.sh"
|
|
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
|
|
# shellcheck source=/home/cendjm/bin/function_lib.sh
|
|
source "$LIB"
|
|
|
|
#
|
|
# Colour codes
|
|
#
|
|
define_colours
|
|
|
|
# {{{ Functions: -- _usage -- _DEBUG --
|
|
#=== FUNCTION ================================================================
|
|
# NAME: _usage
|
|
# DESCRIPTION: Report usage
|
|
# PARAMETERS: None
|
|
# RETURNS: Nothing
|
|
#===============================================================================
|
|
_usage () {
|
|
cat >$STDOUT <<-endusage
|
|
Usage: ./${SCRIPT} [-h] [-d] [-D] shownumber
|
|
|
|
Version: $VERSION
|
|
|
|
Runs Pandoc against a particular show, choosing a format as
|
|
defined by the declared format (in the file '.format').
|
|
|
|
(In this version there is no method to force an explicit input format)
|
|
|
|
Options:
|
|
-h Print this help
|
|
-d Select dry run mode
|
|
-D Turn on debug mode with lots of extra output
|
|
|
|
The default behaviour is now to access the '.format' file in the show
|
|
directory.
|
|
|
|
Arguments:
|
|
shownumber
|
|
|
|
Examples
|
|
./${SCRIPT} -h
|
|
./${SCRIPT} -d 2240
|
|
./${SCRIPT} -D 2250
|
|
|
|
endusage
|
|
exit
|
|
}
|
|
|
|
#=== FUNCTION ================================================================
|
|
# NAME: _DEBUG
|
|
# DESCRIPTION: Writes one or more message lines if in DEBUG mode
|
|
# PARAMETERS: List of messages
|
|
# RETURNS: Nothing
|
|
#===============================================================================
|
|
_DEBUG () {
|
|
[ "$DEBUG" == 0 ] && return
|
|
for msg in "$@"; do
|
|
printf 'D> %s\n' "$msg"
|
|
done
|
|
}
|
|
# }}}
|
|
|
|
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
#
|
|
# Formats offered by the web form on the HPR site:
|
|
#
|
|
# Keyword Explanation
|
|
# ------- -----------
|
|
# plain_text Plain text
|
|
# html5 HTML5 (preferred)
|
|
# markdown_standard Markdown (standard)
|
|
# Markdown_GitHub Markdown (GitHub flavoured)
|
|
# Markdown_Pandoc Markdown (Pandoc flavoured)
|
|
# restructured_text RestructuredText
|
|
# txt2tags txt2tags
|
|
#
|
|
|
|
#
|
|
# Hash to perform translation from declared format to Pandoc "-from" value
|
|
#
|
|
# {{{ -- 'lookup' hash --
|
|
declare -A lookup
|
|
lookup[plain_text]='markdown-implicit_figures'
|
|
lookup[html5]='html'
|
|
lookup[markdown_standard]='markdown_strict'
|
|
lookup[Markdown_GitHub]='gfm' # Extensions are limited
|
|
lookup[Markdown_Pandoc]='markdown-implicit_figures'
|
|
lookup[restructured_text]='rst'
|
|
lookup[txt2tags]='t2t'
|
|
# }}}
|
|
|
|
#
|
|
# Hash for options when generating standalone readable HTML
|
|
#
|
|
# (The 'smart' extension is only applicable to markdown, commonmark, latex,
|
|
# mediawiki, org, rst, twiki; we want to turn it off to remove smart quotes)
|
|
#
|
|
# {{{ -- 'options' hash --
|
|
declare -A options
|
|
options[plain_text]='-smart'
|
|
options[html5]=''
|
|
options[markdown_standard]='+yaml_metadata_block'
|
|
options[Markdown_GitHub]=''
|
|
options[Markdown_Pandoc]='-smart+yaml_metadata_block'
|
|
options[restructured_text]='-smart'
|
|
options[txt2tags]=''
|
|
# }}}
|
|
|
|
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
|
|
#
|
|
# Sanity checks
|
|
#
|
|
JQ=$(command -v jq)
|
|
[ -n "$JQ" ] || { echo "Program 'jq' was not found"; exit 1; }
|
|
# YQ=$(command -v yq)
|
|
# [ -n "$YQ" ] || { echo "Program 'yq' was not found"; exit 1; }
|
|
|
|
#
|
|
# Process options first
|
|
#
|
|
while getopts :dDh opt
|
|
do
|
|
case "${opt}" in
|
|
d) DRYRUN=1;;
|
|
D) DEBUG=1;;
|
|
h) _usage;;
|
|
?) echo "$SCRIPT: Invalid option; aborting"; exit 1;;
|
|
esac
|
|
done
|
|
shift $((OPTIND - 1))
|
|
|
|
#
|
|
# Default options if not provided
|
|
#
|
|
DEBUG=${DEBUG:-0}
|
|
DRYRUN=${DRYRUN:-0}
|
|
|
|
#
|
|
# Check there's an argument after removing any options. Abort if not
|
|
#
|
|
if [[ $# -ne 1 ]]; then
|
|
_usage
|
|
fi
|
|
|
|
#
|
|
# Declare variables for later
|
|
# TODO: Is this necessary?
|
|
#
|
|
declare SHOWID FROM POPTIONS
|
|
|
|
#
|
|
# Make the explicit show id, catering for leading zeroes (belt & braces)
|
|
#
|
|
printf -v SHOWID 'hpr%04d' "$1"
|
|
|
|
#
|
|
# Make temporary files and set traps to delete them
|
|
#
|
|
TMP1=$(mktemp) || {
|
|
echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
|
|
exit 1
|
|
}
|
|
TMP2=$(mktemp) || {
|
|
echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
|
|
exit 1
|
|
}
|
|
TMP3=$(mktemp) || {
|
|
echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
|
|
exit 1
|
|
}
|
|
trap 'cleanup_temp $TMP1 $TMP2 $TMP3' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
|
|
|
|
#
|
|
# Paths to files
|
|
#
|
|
# ------------------------------------------------------------------------------
|
|
# Main directory
|
|
BASENAME="$HOME/HPR/Show_Submission"
|
|
|
|
# JSON to YAML Perl script
|
|
J2Y="$BASENAME/author_title.pl"
|
|
[ -e "$J2Y" ] || { echo "Program '$J2Y' was not found"; exit 1; }
|
|
|
|
# The notes for all shows are here
|
|
SHOWNOTES="$BASENAME/shownotes"
|
|
|
|
# Notes for this show are here
|
|
SHOWDIR="$SHOWNOTES/$SHOWID"
|
|
|
|
# Paths to all files already created or being created here
|
|
#RAWFILE="$SHOWDIR/shownotes.txt"
|
|
JSONFILE="$SHOWDIR/shownotes.json"
|
|
FMTFILE="$SHOWDIR/.format"
|
|
PICLIST="$SHOWDIR/.pictures"
|
|
STATUS="$SHOWDIR/.status"
|
|
EXTRACT="$SHOWDIR/${SHOWID}.out"
|
|
FULLHTML="$SHOWDIR/${SHOWID}_full.html"
|
|
HTML="$SHOWDIR/${SHOWID}.html"
|
|
|
|
# TT² macros and paths for adding pictures
|
|
PICTPL="$BASENAME/pic_definitions.tpl"
|
|
MANIFEST="$SHOWDIR/.pictures.mf" # From do_pictures
|
|
# ------------------------------------------------------------------------------
|
|
|
|
#
|
|
# The partial URL for pictures on the HPR site
|
|
#
|
|
BASEURL='https://hackerpublicradio.org/eps/'
|
|
|
|
#{{{ --- Obsolete 2022-12-17 ---
|
|
#
|
|
# Make a metadata file by parsing the raw data file fields with awk. Save it
|
|
# in a temporary file.
|
|
#
|
|
# See the original do_pandoc_0.1.6 for the extended comments that led to this
|
|
# design. We were trying to make acceptable YAML, but ended up extracting
|
|
# metadata from the result.
|
|
# ----
|
|
# TODO: 2022-12-01 Rationalise all of this; it's full of debris from previous
|
|
# attempts to solve the problem of passing metadata to Pandoc.
|
|
# ----
|
|
#
|
|
# awk -f - "$RAWFILE" > "$TMP1" <<'ENDAWK'
|
|
# BEGIN {print "---"}
|
|
# /^Title:/ && got_title == 0 {
|
|
# sub(/^Title:\s+/,"")
|
|
# printf "#title: %s\n",$0
|
|
# gsub(/'/,"''")
|
|
# printf "title: '%s'\n",$0
|
|
# got_title = 1
|
|
# }
|
|
# /^Host_Name:/ && got_author == 0 {
|
|
# sub(/^Host_Name:\s+/,"")
|
|
# printf "#author: %s\n",$0
|
|
# gsub(/'/,"''")
|
|
# printf "author: '%s'\n",$0
|
|
# got_author = 1
|
|
# }
|
|
# END {print "---"}
|
|
# ENDAWK
|
|
#}}}
|
|
|
|
#
|
|
# Use 'jq' to parse the JSON and make the metadata (in the form of YAML)
|
|
# needed for Pandoc
|
|
#
|
|
# Non-YAML alternative - not chosen
|
|
#jqprog="@text \"author: \(.host.Host_Name)\ntitle: \(.episode.Title)\""
|
|
#
|
|
# Testing another formatter (Journal 2023-03-03)
|
|
#jqprog="@sh \"---\nauthor: \(.host.Host_Name)\ntitle: \(.episode.Title)\n---\""
|
|
# Added quotes around the generated strings (2023-03-31)
|
|
# jqprog="@text \"---\nauthor: \(.host.Host_Name)\ntitle: \(.episode.Title)\n---\""
|
|
# Moved to 'yq' 2023-04-01
|
|
# jqprog="@text \"---\nauthor: '\(.host.Host_Name)'\ntitle: '\(.episode.Title)'\n---\""
|
|
# jq -r "$jqprog" "$JSONFILE" > "$TMP1"
|
|
# On 2023-10-01 wrote a Perl JSON to YAML generator just for these two
|
|
# elements. It's called 'author_title.pl'
|
|
# yqprog='{author:.host.Host_Name,title:.episode.Title}'
|
|
# ( echo "---"; $YQ -y "$yqprog" "$JSONFILE"; echo "---"; ) > "$TMP1"
|
|
$J2Y "$JSONFILE" "$TMP1"
|
|
_DEBUG "YAML:" "$(cat "$TMP1")"
|
|
|
|
#
|
|
# Check the main output file from do_parse exists
|
|
#
|
|
if [[ ! -e $EXTRACT ]]; then
|
|
echo "$SCRIPT: ${red}File not found: $EXTRACT${reset}"
|
|
exit 1
|
|
fi
|
|
|
|
#
|
|
# Get the format or fail with an error
|
|
#
|
|
if [[ -e $FMTFILE ]]; then
|
|
FORMAT=$(cat "$FMTFILE")
|
|
else
|
|
# TODO: Should we default to something rather than abort?
|
|
echo "$SCRIPT: ${red}Could not find declared format (.format file)${reset}"
|
|
echo "${yellow}Has do_parse been run? If so try and fix the .format file.${reset}"
|
|
exit
|
|
fi
|
|
|
|
#
|
|
# Need to match plain text and Markdown variants when deciding to use the
|
|
# manifest file and 'tpage'
|
|
#
|
|
FMTRE='^(plain_text|[Mm]arkdown_)'
|
|
|
|
#
|
|
# Determine if there are pictures
|
|
#
|
|
if [[ -e $PICLIST ]]; then
|
|
hasPictures=1
|
|
else
|
|
hasPictures=0
|
|
fi
|
|
|
|
#
|
|
# Here we use the declared format to determine what to do.
|
|
#
|
|
# This code now blocks HTML->HTML processing since it just confuses matters.
|
|
# The files hprNNNN.out and hprNNNN.html are linked to one another, so editing
|
|
# the former will edit the latter in preparation for uploading if there are
|
|
# changes that need to be made. See the journal discussion for 2018-05-24.
|
|
# ---
|
|
# TODO: 2022-12-01 If the notes are HTML but declared as something else then
|
|
# this check will not work. Trouble is, parse_JSON will have worked this out
|
|
# but not saved it so we can't avoid running Pandoc on HTML in this case.
|
|
# ---
|
|
# TODO: 2022-12-01 Look at resolving this in parse_JSON?
|
|
# ---
|
|
#
|
|
if [[ $FORMAT == 'html5' ]]; then
|
|
echo "${red}Running Pandoc on HTML is not allowed${reset}"
|
|
echo "${yellow}Run do_edit to edit as necessary and do_browser to view${reset}"
|
|
exit
|
|
elif [[ $FORMAT == 'plain_text' ]]; then
|
|
echo "${yellow}Format chosen is plain text${reset}"
|
|
if [[ $hasPictures -eq 0 ]]; then
|
|
echo "${yellow}This will be treated as Markdown${reset}"
|
|
else
|
|
echo "${yellow}This will be treated as Markdown and will need special action${reset}"
|
|
echo "${yellow}since there are pictures${reset}"
|
|
fi
|
|
fi
|
|
FROM=${lookup[$FORMAT]}
|
|
POPTIONS=${options[$FORMAT]}
|
|
echo "${green}Will process $FORMAT with 'pandoc -f ${FROM}'${reset}"
|
|
echo "${yellow}Options chosen for --standalone are '${POPTIONS}'${reset}"
|
|
|
|
#
|
|
# Only for plain text notes, process pictures for the HTML we'll be adding to
|
|
# the database.
|
|
#
|
|
# We need the following things:
|
|
# - The notes to be plain text format
|
|
# - Some pictures
|
|
# - Files written by do_parse and do_pictures:
|
|
# - .assets (not used here)
|
|
# - .pictures (were previously used here, but no longer)
|
|
# - .pictures.mf (needed by the TT² macros)
|
|
# - .pictures.tt2 (the TT² macro calls - already edited into the notes)
|
|
# - pic_definitions.tpl (macro definitions, common to all shows)
|
|
# - Not to be in dry run mode; if we are we just report intentions
|
|
#
|
|
if [[ $DRYRUN -eq 0 ]]; then
|
|
# if [[ $FORMAT == 'plain_text' && -e $MANIFEST ]]; then
|
|
if [[ $FORMAT =~ $FMTRE && -e $MANIFEST ]]; then
|
|
#
|
|
# Deal with pictures using the TT² macros
|
|
#
|
|
_DEBUG "Processing TT² inclusions"
|
|
|
|
# Make a picture manifest with a header
|
|
awk 'BEGIN{print "file : thumb"}{p1=$0; getline p2; printf "%s : %s\n",p1,p2}' \
|
|
"$MANIFEST" > "$TMP2"
|
|
|
|
_DEBUG "Picture list:" "$(cat "$TMP2")" "---" \
|
|
"BASEURL=${BASEURL}${SHOWID}/" \
|
|
"EXTRACT=$EXTRACT" \
|
|
"Extract file contents:" "$(cat "$EXTRACT")" "---"
|
|
|
|
# Run the macros on the notes to make $TMP3 for Pandoc
|
|
tpage --pre_process="$PICTPL" \
|
|
--define "piclist=$TMP2" \
|
|
--define "prefix=${BASEURL}${SHOWID}/${SHOWID}/" \
|
|
"$EXTRACT" > "$TMP3"
|
|
|
|
_DEBUG "Processed by tpage" "$(cat "$TMP3")" "---"
|
|
else
|
|
# Not plain text or a Markdown variant and no pictures, so put the
|
|
# notes in $TMP3 where Pandoc will look for them
|
|
cat "$EXTRACT" > "$TMP3"
|
|
fi
|
|
else
|
|
#
|
|
# We would not have used TT² unless the notes were plain text (or
|
|
# a Markdown variant) and there were pictures
|
|
#
|
|
# if [[ $FORMAT == 'plain_text' && -e $MANIFEST ]]; then
|
|
if [[ $FORMAT =~ $FMTRE && -e $MANIFEST ]]; then
|
|
echo "${yellow}Would have prepared TT² code for pandoc${reset}"
|
|
fi
|
|
fi
|
|
|
|
#
|
|
# Generate an HTML snippet for adding to the database.
|
|
# (Note 2021-11-24: Added --ascii option.)
|
|
#
|
|
if [[ $DRYRUN -eq 0 ]]; then
|
|
# shellcheck disable=SC2086
|
|
pandoc -f "$FROM"-smart -t html5 --ascii --no-highlight --strip-comments \
|
|
"$TMP3" -o "$HTML" # $EXTRAS
|
|
RES=$?
|
|
|
|
if [[ $RES -eq 0 ]]; then
|
|
echo "$SCRIPT: ${green}Created shownotes/$SHOWID/${HTML##*/}${reset}"
|
|
else
|
|
echo "$SCRIPT: ${red}Oops! Something went wrong! (line $LINENO)${reset}"
|
|
echo "${yellow}$SCRIPT: Aborting now${reset}"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "${yellow}Would have run pandoc to make HTML for upload${reset}"
|
|
fi
|
|
|
|
#
|
|
# Make HTML for proof reading. All pictures referenced are now on the HPR
|
|
# server (we ran 'do_asset_upload'), so we want to refer to them here.
|
|
#
|
|
# File $TMP2 contains the .pictures.mf contents with a header line; and it
|
|
# contains data for the macros. It was created when we prepared the main HTML
|
|
# for the database. We use $BASEURL again here because we want to reference
|
|
# the pictures on the server.
|
|
#
|
|
# We use the awk-formatted file (now yq-formatted) in $TMP1 from earlier to do
|
|
# this. At the end TMP3 contains Markdown for Pandoc.
|
|
#
|
|
if [[ $DRYRUN -eq 0 ]]; then
|
|
# if [[ $FORMAT == 'plain_text' && -e $MANIFEST ]]; then
|
|
if [[ $FORMAT =~ $FMTRE && -e $MANIFEST ]]; then
|
|
tpage --pre_process="$PICTPL" \
|
|
--define "piclist=$TMP2" \
|
|
--define "prefix=${BASEURL}${SHOWID}/${SHOWID}/" \
|
|
"$EXTRACT" > "$TMP3"
|
|
else
|
|
cat "$EXTRACT" > "$TMP3"
|
|
fi
|
|
|
|
#
|
|
# Generate complete HTML that we can proofread. We need metadata for this
|
|
# stand-alone HTML which is in the form of YAML in this version.
|
|
#
|
|
# ----------------------------------------------------------------------
|
|
# Original options below when using 'awk' to parse shownotes.txt:
|
|
# --metadata="$(sed -n '/^#author:/{s/#//;p}' "$TMP1")" \
|
|
# --metadata="$(sed -n '/^#title:/{s/#//;p}' "$TMP1")" \
|
|
#
|
|
# shellcheck disable=SC2086
|
|
pandoc -f ${FROM}${POPTIONS} -t html5 --ascii \
|
|
--standalone --template=hpr.html5 --no-highlight \
|
|
-c https://hackerpublicradio.org/css/hpr.css \
|
|
--metadata-file="$TMP1" -o "$FULLHTML" "$TMP3"
|
|
RES=$?
|
|
|
|
if [[ $RES -eq 0 ]]; then
|
|
echo "$SCRIPT: ${green}Created shownotes/$SHOWID/${FULLHTML##*/}${reset}"
|
|
else
|
|
echo "$SCRIPT: ${red}Oops! Something went wrong making the full HTML! (line $LINENO)${reset}"
|
|
fi
|
|
else
|
|
# Dry run
|
|
echo "${yellow}Would have run pandoc to make HTML for proof reading${reset}"
|
|
fi
|
|
|
|
#
|
|
# Set the status for this show
|
|
#
|
|
if [[ $DRYRUN -eq 0 ]]; then
|
|
echo "converted" >> "$STATUS"
|
|
fi
|
|
|
|
exit
|
|
|
|
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker
|