hpr-tools/Show_Submission/do_pictures

#!/bin/bash -
#===============================================================================
#
#         FILE: do_pictures
#
#        USAGE: ./do_pictures [options] <epno>
#
#  DESCRIPTION: Process pictures associated with a show.
#
#               If there are pictures associated with an episode they will
#               have been found amongst the show media, including unpacking
#               any archives. Their names will have been written to a file
#               which is called '.pictures' in the show directory. The files
#               themselves will have been moved to a directory called
#               'uploads' in the show directory.
# {{{                                                           --- lengthy details ---
#               We no longer write files to the previous directory 'hpr<epno>'
#               because we will not make a tar file any more but will upload
#               directly to the server (elsewhere). We need the names because
#               we want to process them from the '.pictures' file. Processing
#               consists of renaming by replacing spaces with underscores and
#               using 'gm convert' to remove EXIF data. If the picture size is
#               too large a thumbnail is created, called
#               '<name>_thumbnail.<ext>'. If the notes are plain text (which
#               will be without links) then we will use `.pictures' as an
#               ordered list as we place them into the Markdown we create as
#               TT² macros. The list is reordered by asking the user to
#               resequence it with a list of reference numbers.
#
#               Also for plain text and Markdown variant notes we create
#               a file called '.pictures.tt2' which contains the TT² commands
#               to add the pictures.  When editing the notes this list is
#               shown in another window so the contents can be copied and
#               pasted. There are two TT² macros, 'pic1' for stand-alone
#               pictures and 'pic2' for pictures with thumbnails (clicking the
#               thumbnail causes the larger picture to be opened).
#
#               Running `do_pandoc` on plain text and Markdown variant notes
#               will assemble the TT² macros from 'pic_definitions.tpl', and
#               the notes containing the statements in '.pictures.tt2' placed
#               as required.
#
#               If the notes are not plain text then different paths are
#               taken:
#
#               Markdown variants: any links will require editing to refer to
#               any filename changes, and if the picture needs a thumbnail
#               then this will need to be catered for. There will be no need
#               for `.pictures', `.pictures.tt2' or 'pic_definitions.tpl'
#               unless ther are no links already. The notes will be processed
#               with 'do_pandoc' which will cater for these differences.
#
#               HTML: again, links might need adjusting, and as for Markdown,
#               no supplementary files will be required. The modified notes
#               will not be further processed before uploading.
# }}}
#
#      OPTIONS: ---
# REQUIREMENTS: ---
#         BUGS: ---
#        NOTES: ---
#       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
#      VERSION: 0.2.4
#      CREATED: 2020-05-25 13:20:18
#     REVISION: 2024-07-25 21:39:16
#
#===============================================================================

set -o nounset                              # Treat unset variables as an error

SCRIPT=${0##*/}
# DIR=${0%/*}

VERSION="0.2.4"

STDOUT="/dev/fd/2"

#
# Load library functions (make_file_list, range_parse, cleanup_temp)
#
LIB="$HOME/HPR/function_lib.sh"
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
# shellcheck source=/home/cendjm/HPR/function_lib.sh
source "$LIB"

#
# Colour codes
#
define_colours

# {{{ -- _usage - _silent -- _dryrun --
#===  FUNCTION  ================================================================
#         NAME: _usage
#  DESCRIPTION: Report usage
#   PARAMETERS: None
#      RETURNS: Nothing
#===============================================================================
_usage () {
    cat >$STDOUT <<-endusage
Usage: ./${SCRIPT} [-h] [-d] [-D] shownumber

Version: $VERSION

Processes pictures associated with an HPR show.

The script detects any pictures downloaded with an episode, looking for JPEG
and PNG. It copies them to a sub-directory called 'hpr<epno>' (since that's
the name of the directory where where they'll go on the server).

As it copies the files the script renames the copies if necessary, mainly by
replacing spaces with underscores. The copying process uses 'gm convert' and
removes EXIF data along the way. It also generates a thumbnail for each
picture if the size warrants it, called 'NAME_thumbnail.ext'.

The script also makes a list of the newly named pictures in a file called
'.pictures' which is used to refer to them in the notes (by position in the
list). The order in which the pictures are processed and the list built up can
be controlled by the user by the ordering of the presented enumerated list.

A file called '.pictures.tt2' contains the TT² commands to add the pictures.
When editing the notes this list is shown in another window so the contents
can be copied and pasted. There are two TT² macros, 'pic1' for stand-alone
pictures and 'pic2' for pictures with thumbnails (clicking the thumbnail
causes the larger picture to be opened).

Running 'do_pandoc' on the notes will assemble the TT² macros from
'pic_definitions.tpl', and the notes containing the statements in
'.pictures.tt2' which have been placed as required.

An 'index.html' file is created to allow the images to be visualised outside
the notes.

Options:
  -h                    Print this help
  -d                    Select dry run mode
  -D                    Select debug mode (works the same; more output)
  -s                    Silent mode; suppress as many output messages as
                        possible

Arguments:
    shownumber

Examples
  ./${SCRIPT} -h
  ./${SCRIPT} -d 3112
  ./${SCRIPT} -D -d 3112
  ./${SCRIPT} 3112
  ./${SCRIPT} -s 3112

endusage
    exit
}

#===  FUNCTION  ================================================================
#         NAME: _silent
#  DESCRIPTION: Output a message unless we're being silent
#   PARAMETERS: List of messages
#      RETURNS: Nothing
#===============================================================================
_silent () {
    [ "$SILENT" == 1 ] && return
    for msg in "$@"; do
        printf '%s\n' "$msg"
    done
}

#===  FUNCTION  ================================================================
#         NAME: _dryrun
#  DESCRIPTION: Output a dry run message
#   PARAMETERS: List of messages
#      RETURNS: Nothing
#===============================================================================
_dryrun () {
    local prefix="Dry run: "
    for msg in "$@"; do
        printf '%9s%s\n' "$prefix" "$msg"
        prefix=
    done
}
# }}}

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

#
# Debug mode, default off
#
DEBUG=0

#
# Option defaults
#
DRYRUN=0  # not dry run mode by default
SILENT=0  # not silent by default

#
# Process options
#
while getopts :hdDs opt
do
    case "${opt}" in
        h) _usage;;
        d) DRYRUN=1;;
        D) DEBUG=1;;
        s) SILENT=1;;
        ?) echo "$SCRIPT: Invalid option; aborting"; exit 1;;
    esac
done
shift $((OPTIND - 1))

#
# Check there's an argument after removing any options. Abort if not
#
if [[ $# -ne 1 ]]; then
    echo "${red}Missing shownumber argument${reset}"
    _usage
fi
epno="${1}"

#
# Ensure item spec is correctly formatted. Have to cater for leading zeroes
# being interpreted as octal.
#
if [[ $epno =~ ^(hpr)?([0-9]+)$ ]]; then
    epno="$((10#${BASH_REMATCH[-1]}))"
else
    coloured 'red' "Incorrect show specification: $epno"
    coloured 'yellow' "Use 'hpr9999' or '9999' formats"
    exit 1
fi

#
# Paths to files
#
BASEDIR="$HOME/HPR/Show_Submission"
SHOWDIR="$BASEDIR/shownotes/hpr${epno}"
# RAWFILE="$SHOWDIR/shownotes.txt"
JSONFILE="$SHOWDIR/shownotes.json"
PICDIR="$SHOWDIR/uploads"
# INDEXFILE="$PICDIR/index.html"
FORMAT="$SHOWDIR/.format"
ASSETS="$SHOWDIR/.assets"               # Created in parse_JSON
PICLIST="$SHOWDIR/.pictures"            # Created in parse_JSON
STATUS="$SHOWDIR/.status"

# ----
# Files to be used by the TT² macros that build Markdown from plain
# text notes.
# ----

# MANIFEST contains a list of pictures and thumbnails, with blank lines if no
# thumbnail.
MANIFEST="$SHOWDIR/.pictures.mf"

# PICTPL contains TT² macro calls which can be cut and pasted into the plain
# text notes
PICTPL="$SHOWDIR/.pictures.tt2"

# ----

#
# Constants
#
TNSIZE=800              # Thumbnail size
TNW=$TNSIZE             # Maximum width
TNH=$((TNSIZE*2))       # Maximum height

#
# Check the show directory exists
#
if [[ ! -d $SHOWDIR ]]; then
    echo "$SCRIPT: ${red}Directory $SHOWDIR not found${reset}"
    exit 1
fi

#
# Make temporary files and set traps to delete them.
#
# TMP1 - HTML header
# TMP2 - HTML footer
# TMP3 - call to TT² macro 'pic1' (make a plain picture link)
# TMP4 - call to TT² macro 'pic2' (make a thumbnail link)
#
TMP1=$(mktemp) || {
    echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
    exit 1
}
TMP2=$(mktemp) || {
    echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
    exit 1
}
TMP3=$(mktemp) || {
    echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
    exit 1
}
TMP4=$(mktemp) || {
    echo "$SCRIPT: ${red}creation of temporary file failed!${reset}"
    exit 1
}
trap 'cleanup_temp $TMP1 $TMP2 $TMP3 $TMP4' SIGHUP SIGINT SIGPIPE SIGTERM EXIT

#
# Parse these fields from the raw input file (shownotes.txt) into Bash
# variables so we can make an index.html later.
#
# ----
# NOTE: We're moving to the JSON file for show info. This could be parsed with
# 'jq'.  This filter seems reasonably close to our needs:
#
# command=$(jq -r '@sh "hostname=\(.host.Host_Name) title=\(.episode.Title) summary=\(.episode.Summary)"' $RAWFILE)
# eval ${command:1:-1}
#
# The '@sh' format thing ensures the output is OK for the shell.
# It seems 'jq' quotes the returned text (it's a string) which 'eval' doesn't
# like, but giving 'eval' the string without the quotes (${command:1:-1})
# makes this work.
# Update: adding the '-r' option removes the enclosing double quotes
# Use the '-f FILE' option to read the filter from a file.
# ----
#
# Title='' Summary='' Host_Name=''
# eval \
#     "$(sed -n '/^\(Title\|Summary\|Host_Name\):/{s/^\([^:]\+\):\t/\1="/;s/$/"/;p}' "$RAWFILE")"
#
Title='' Summary='' Host_Name=''
jqprog="@sh \"Host_Name=\(.host.Host_Name) Title=\(.episode.Title) Summary=\(.episode.Summary)\""
command=$(jq -r "$jqprog" "$JSONFILE")
eval "${command}"

#
# Make two HTML templates and two TT² templates. We do this in dry-run mode
# too then throw them away!
#
# {{{
cat > "$TMP1" <<ENDHTML1
<!DOCTYPE html>
<html>
<head>
  <meta charset="utf-8">
  <meta name="generator" content="pandoc">
  <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
  <meta name="author" content="$Host_Name">
  <title>$Title (HPR Show $1)</title>
  <style type="text/css">code{white-space: pre;}</style>
  <!--[if lt IE 9]>
    <script src="https://html5shim.googlecode.com/svn/trunk/html5.js"></script>
  <![endif]-->
  <link rel="stylesheet" href="https://hackerpublicradio.org/css/hpr.css">
</head>

<body id="home">
<div id="container" class="shadow">
<header>
<h1 class="title">$Title (HPR Show $1)</h1>
<h3 class="summary">$Summary</h3>
<h3 class="author">$Host_Name</h3>
<hr/>
</header>

<main id="maincontent">
<article>
<h3 id="index">Index</h3>
<ul>
ENDHTML1

cat > "$TMP2" <<ENDHTML2
</ul>
</article>
</main>
</div>
</body>
</html>
ENDHTML2

cat > "$TMP3" <<'ENDPIC1'
[% row = piclist.$ind -%]
[% pic1("$prefix",'*Title*',"$row.file") -%]

ENDPIC1

cat > "$TMP4" <<'ENDPIC2'
[% row = piclist.$ind -%]
[% pic2("$prefix",'*Title*',"$row.thumb","$row.file") -%]

ENDPIC2
# }}}

#
# Read the previously created list of pictures into array 'pix'
#
if [[ -e $PICLIST ]]; then
    declare -a pix
    mapfile -t pix < "$PICLIST"
else
    echo "$SCRIPT: ${red}No pictures found!${reset}"
    echo "$SCRIPT: ${red}Show has no $PICLIST file!${reset}"
    exit 1
fi

_DEBUG 'Pictures:' "${pix[@]}"

#
# The assets directory doesn't exist - something's wrong
#
if [[ ! -d $PICDIR ]]; then
    echo "$SCRIPT: ${red}No picture/asset directory found!${reset}"
    echo "$SCRIPT: ${red}Expected $PICDIR${reset}"
    exit 1
fi

#
# Look for the show note format file
#
# Formats offered by the web form on the HPR site:
#
# Keyword               Explanation
# -------               -----------
# plain_text            Plain text
# html5                 HTML5 (preferred)
# markdown_standard     Markdown (standard)
# Markdown_GitHub       Markdown (GitHub flavoured)
# Markdown_Pandoc       Markdown (Pandoc flavoured)
# restructured_text     RestructuredText
# txt2tags              txt2tags
#
NOTEFORMAT=
if [[ -e $FORMAT ]]; then
    NOTEFORMAT="$(cat "$FORMAT")"
else
    echo "$SCRIPT: ${red}No format file found!${reset}"
    echo "$SCRIPT: ${red}Expected $FORMAT${reset}"
    exit 1
fi

#
# Regex to match 'plain_text' or any Markdown variant
#
FMTRE='^(plain_text|[Mm]arkdown_)'

_DEBUG "Format: $NOTEFORMAT"

# ----
# Check whether we're trying to run this script a second time. It will
# potentially try and make thumbnails of thumbnails if we do!
# ----
if [[ -e $MANIFEST || -e $PICTPL || $(grep -c pictures "$STATUS") -gt 0 ]]; then
    echo "$SCRIPT: ${red}This doesn't seem to be the first run of this script!${reset}"
    echo "$SCRIPT: ${yellow}Delete $MANIFEST, $PICTPL and any thumbnails " \
        "and run it again - if you are certain about this!${reset}" | fmt -tu
    echo "$SCRIPT: ${red}Exiting...${reset}"
    exit 1
fi

#
# Start the TT² template and order the files (only for plain text and Markdown
# variant shows)
#
declare -a opix
if [[ $NOTEFORMAT =~ $FMTRE ]]; then
    if [[ $DRYRUN -eq 0 ]]; then
        echo "[%# Place at the top of the show notes -%]" > "$PICTPL"
        echo -e "[% ind = 0 -%]\n" >> "$PICTPL"
    else
        _dryrun "would have initialised $PICTPL"
    fi

    #
    # Get the order of the picture files, assuming they aren't in a usable sort
    # order, and store them in 'opix'. The user can just type 1-10 (or whatever
    # the count is) to use them in the presented order. The default prompt
    # offers such an order for speed. They can abort the script at this point
    # if they type CTRL-D.
    #
    _silent "${blue}Building ordered picture list${reset}" " "
    if ! make_file_list pix opix; then
        echo "Script aborted"
        exit 1
    fi
else
    #
    # Fill the 'opix' array, but we don't care about ordering
    #
    mapfile -t -d ' ' opix < <(echo -n "${pix[@]}")
fi

_DEBUG '(Possibly) re-ordered pictures:' "${opix[@]}"

#
# Convert the original pictures and make thumbnails if needed. Thumbnails get
# a name in the form 'NAME_thumbnail.EXT'. We record any thumbnails we create.
# In plain text mode we write details to a new file called '.pictures.mf'
# which is to be a manifest used by 'do_pandoc' to simplify the addition of
# Markdown picture and thumbnail links to the notes.
#
pcount=1
for pic in "${opix[@]}"; do
    #
    # The extension and name of the file we'll convert the original picture
    # into. No path here. We replace internal spaces with underscores.
    # NOTE: Space → underscore conversion has been done in parse_JSON
    #
    ext="${pic#*.}"
    newpic="${pic##*/}"
    # newpic="${newpic// /_}"

    #
    # What is this picture and does it need a thumbnail? Collect the type,
    # width and height.
    #
    if [[ -e "$PICDIR/$pic" ]]; then
        geo=$(gm identify -format "%m:%w:%h/" "$PICDIR/$pic")
        RES=$?
        [ $RES -ne 0 ] && { echo "Aborting!"; exit $RES; }
    fi

    #
    # Remove repetitions for GIFs and extract the type
    #
    geo="${geo%%/*}"
    type="${geo%%:*}"

    #
    # Special actions for GIFs, only needed if the notes are plain text or
    # a Markdown variant
    #
    if [[ $type == 'GIF' ]]; then
        echo "GIF detected"
        # Cater for plain text and Markdown variant format notes
        if [[ $NOTEFORMAT =~ $FMTRE ]]; then
            if [[ $DRYRUN -eq 0 ]]; then
                echo "$PICDIR/$newpic" >> "$MANIFEST"
                echo "" >> "$MANIFEST"

                if [[ $pcount -gt 1 ]]; then
                    echo "[% ind = ind + 1 -%]" >> "$PICTPL"
                fi

                # Add 'pic1' macro call
                cat "$TMP3" >> "$PICTPL"
            else
                _dryrun "Would have added ${yellow}$PICDIR/$newpic${reset} to" \
                    "${yellow}$MANIFEST${reset}"
                _dryrun "would have added a blank line to ${yellow}$MANIFEST${reset}"

                _dryrun "Would have added the ${yellow}pic1${reset} macro to" \
                    "${yellow}$PICTPL${reset}"
            fi
        fi

        #
        # Nothing else needs to be done, so skip the rest of the loop
        #
        _silent "No image processing performed for $type file $pic"
        echo "----"
        ((pcount++))
        continue
    else
        #
        # A simple image, get its size
        #
        geo="${geo#*:}"
        pw="${geo%:*}"
        ph="${geo#*:}"
        if [[ $pw -gt $TNW || $ph -gt $TNH ]]; then
            THUMB=1
            tn="${newpic%.*}_thumbnail.$ext"
        else
            THUMB=0
            tn=
        fi
    fi

    #
    # Include thumbnails or omit them
    #
    if [[ $THUMB -eq 1 ]]; then
        _DEBUG 'Thumbnail:' "${newpic}" "${ext}" "${tn}"

        #
        # Convert originals to new path/file (or document it in dry-run mode)
        #
        if [[ $DRYRUN -eq 0 ]]; then
            gm convert "$PICDIR/$pic" -strip -auto-orient "$PICDIR/$newpic"
            gm convert "$PICDIR/$pic" -strip -auto-orient -resize $TNSIZE "$PICDIR/$tn"

            #
            # Only make a (new) picture list if we have plain text or
            # a Markdown variant
            #
            if [[ $NOTEFORMAT =~ $FMTRE ]]; then
                echo "$PICDIR/$newpic" >> "$MANIFEST"
                echo "$PICDIR/$tn" >> "$MANIFEST"

                if [[ $pcount -gt 1 ]]; then
                    echo "[% ind = ind + 1 -%]" >> "$PICTPL"
                fi
                # Add 'pic2' macro call
                cat "$TMP4" >> "$PICTPL"
            fi

            #
            # All thumbnails are new so need to be recorded
            #
            echo "$tn" >> "$PICLIST"
            echo "$tn" >> "$ASSETS"

            _silent "Converted picture ${yellow}$pic${reset} with thumbnail"
        else
            _dryrun \
                "Would have converted ${yellow}$pic${reset} to " \
                "${yellow}$newpic${reset} and ${yellow}$tn${reset} in " \
                "${yellow}$(trimpath "$PICDIR" 3)${reset}"

            #
            # Only make a (new) picture list if we have plain text or
            # a Markdown variant
            #
            if [[ $NOTEFORMAT =~ $FMTRE ]]; then
                _dryrun "Would have added ${yellow}$PICDIR/$newpic${reset} to" \
                    "${yellow}$MANIFEST${reset}"
                _dryrun "would have added ${yellow}$PICDIR/$tn${reset} to" \
                    "${yellow}$MANIFEST${reset}"

                _dryrun "Would have added the ${yellow}pic2${reset} macro to" \
                    "${yellow}$PICTPL${reset}"
            fi

            echo "----"
        fi
    else
        _DEBUG 'No thumbnail:' "${newpic}" "${ext}"

        #
        # Convert originals to new path/file (or document it)
        #
        if [[ $DRYRUN -eq 0 ]]; then
            gm convert "$PICDIR/$pic" -strip -auto-orient "$PICDIR/$newpic"

            #
            # Only make a (new) picture list if we have plain text or
            # a Markdown variant
            #
            if [[ $NOTEFORMAT =~ $FMTRE ]]; then
                echo "$PICDIR/$newpic" >> "$MANIFEST"
                echo "" >> "$MANIFEST"

                if [[ $pcount -gt 1 ]]; then
                    echo "[% ind = ind + 1 -%]" >> "$PICTPL"
                fi
                # Add 'pic1' macro call
                cat "$TMP3" >> "$PICTPL"
            fi

            _silent "Converted picture ${yellow}$pic${reset} without thumbnail"
        else
            _dryrun "Would have converted ${yellow}$pic${reset} to " \
                "${yellow}$newpic${reset}"

            #
            # Only make a (new) picture list if we have plain text or
            # a Markdown variant
            #
            if [[ $NOTEFORMAT =~ $FMTRE ]]; then
                _dryrun "Would have added ${yellow}$PICDIR/$newpic${reset} to" \
                    "${yellow}$MANIFEST${reset}"
                _dryrun "would have added a blank line to ${yellow}$MANIFEST${reset}"

                _dryrun "Would have added the ${yellow}pic1${reset} macro to" \
                    "${yellow}$PICTPL${reset}"
            fi

            echo "----"
        fi
    fi

    ((pcount++))
done

#
# Final steps - not wanted in dry run mode
#
if [[ $DRYRUN -eq 0 ]]; then
    #
    # If we made thumbnails the .assets and .pictures files will have been
    # updated. Clean these up here.
    #
    perl -i.BAK -ne 'print if ! $x{$_}++' "$PICLIST"
    perl -i.BAK -ne 'print if ! $x{$_}++' "$ASSETS"

    #
    # Set the status for this show
    #
    echo "pictures" >> "$STATUS"
fi

exit

# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker