hpr-tools/InternetArchive/weekly_upload

#!/bin/bash -
#===============================================================================
#
#         FILE: weekly_upload
#
#        USAGE: ./weekly_upload [-h] [-r] [-v] [-d {0|1}] start [count]
#
#  DESCRIPTION: Run the commands necessary to upload a batch of HPR shows to
#               archive.org
#
#      OPTIONS: ---
# REQUIREMENTS: ---
#         BUGS: ---
#        NOTES: ---
#       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
#      VERSION: 0.0.9
#      CREATED: 2018-10-12 21:54:35
#     REVISION: 2020-10-13 13:54:32
#
#===============================================================================

set -o nounset                              # Treat unset variables as an error

SCRIPT=${0##*/}
#DIR=${0%/*}
VERSION="0.0.9"

STDOUT="/dev/fd/2"

#
# Select the appropriate working directory
#
case $(hostname) in
    i7-desktop)
        BASEDIR="$HOME/HPR/InternetArchive"
        UPLOAD="$BASEDIR/uploads"
        ;;
    hprvps|marvin|borg)
        BASEDIR="$HOME/IA"
        UPLOAD="/var/IA/uploads"
        ;;
    *)
        echo "Wrong host!"
        exit 1
        ;;
esac

cd "$BASEDIR" || exit 1

#
# Load library functions
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "Unable to source functions"; exit; }
# shellcheck disable=SC1090
source "$LIB"


#===  FUNCTION  ================================================================
#         NAME: check_uploads
#  DESCRIPTION: Determines if files exist for uploading
#   PARAMETERS: 1 - filename prefix e.g. 'hpr9999'
#      RETURNS: True/false
#===============================================================================
check_uploads () {
    local prefix=${1:?Usage: check_uploads prefix}
    local suff

    #
    # Look for files called hpr1234.flac and so on. Don't bother with the
    # hpr1234_source.flac one. As soon as a file is missing return with false.
    #
    for suff in flac mp3 ogg opus spx wav; do
        if [[ ! -e $UPLOAD/$prefix.$suff ]]; then
            return 1
        fi
    done

    return 0
}

#===  FUNCTION  ================================================================
#         NAME: _usage
#  DESCRIPTION: Reports usage; always exits the script after doing so
#   PARAMETERS: 1 - the integer to pass to the 'exit' command
#      RETURNS: Nothing
#===============================================================================
_usage () {
    local -i result=${1:-0}

    cat >$STDOUT <<-endusage
${SCRIPT} - version: ${VERSION}

Usage: ./${SCRIPT} [-h] [-r] [-v] [-d {0|1}] start [count]

Generates the necessary metadata and script and uses them to upload HPR audio
and other show-related files held on the VPS to the Internet Archive.

Options:
  -h                    Print this help
  -v                    Run in verbose mode where more information is reported
  -d 0|1                Dry run: -d 1 (the default) runs the script in dry-run
                        mode where nothing is changed but the actions that
                        will be taken are reported; -d 0 turns off dry-run
                        mode and the actions will be carried out.
  -r                    Run in 'remote' mode, using the live database over an
                        (already established) SSH tunnel. Default is to run
                        against the local database.
Arguments:
    start               the starting show number to be uploaded
    count               (optional, default 1) the number of shows to be
                        uploaded; cannot exceed 20

Notes:

1. When running on 'marvin' the method used is to run in faux 'local' mode.
   This means we have an open tunnel to the HPR server (mostly left open) and
   the default file .hpr_db.cfg points to the live database via this tunnel.
   So we do not use the -r option here. This is a bit of a hack! Sorry!

2. There are potential problems when a show has no tags which haven't been
   fully resolved. The make_metadata script fails in default mode when it
   finds such a show, but this (weekly_upload) script can continue on and run
   the generated script which uploads the source audio files. This can mean
   the IA items end up as books! In this mode the description is not stored
   and so there are no show notes.

endusage
    exit "$result"
}

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

#
# Process options
#
while getopts :d:hrv opt
do
    case "${opt}" in
        d) DRYRUN=$OPTARG;;
        h) _usage 1;;
        r) REMOTE=1;;
        v) VERBOSE=1;;
        *) _usage 1;;
    esac
done
shift $((OPTIND - 1))

DRYRUN=${DRYRUN:-1}
if [[ $DRYRUN -ne 0 && $DRYRUN -ne 1 ]]; then
    echo "** Use '-d 0' or '-d 1'"
    _usage 1
fi

VERBOSE=${VERBOSE:-0}

REMOTE=${REMOTE:-0}
if [[ $REMOTE -eq 0 ]]; then
    dbconfig="$BASEDIR/.hpr_db.cfg"
    [[ $VERBOSE -eq 1 ]] && echo "Local database mode"
else
    dbconfig="$BASEDIR/.hpr_livedb.cfg"
    [[ $VERBOSE -eq 1 ]] && echo "Remote database mode"
fi

#
# Check argument count
#
if [[ ! ( $# -eq 1 || $# -eq 2 ) ]]; then
    echo "Wrong number of arguments"
    _usage 1
fi

#
# Validate arguments
#
for arg; do
    if [[ ! $arg =~ ^[0-9]{1,4}$ ]]; then
        echo "Invalid number: $arg"
        echo "Use a plain number"
        exit 1
    fi
done

start=$1
count=${2:-1}
if [[ $count -gt 20 ]]; then
    echo "Can't process more than 20 shows at a time"
    exit 1
fi

[[ $VERBOSE -eq 1 && $DRYRUN -eq 1 ]] && echo "Dry run mode"

#
# Two paths; one for a single episode, the other for multiple episodes
#
if [[ $count -eq 1 ]]; then
    #
    # Single episode
    # --------------

    #
    # Check existence of files
    #
    if ! check_uploads "hpr$start"; then
        echo "Missing files for show $start. Aborted!"
        exit 1
    fi

    #
    # Define files
    #
    metadata="metadata_${start}.csv"
    script="script_${start}.sh"

    if [[ $DRYRUN -eq 1 ]]; then
        echo "Dry run: Would have uploaded $start"
        echo "Dry run: Would have created $metadata and $script"
        echo "Dry run: Would have uploaded $metadata and run $script"
        echo "Dry run: Would have used $dbconfig"
    else
        echo "Uploading $start"
        if yes_no "OK to continue? %s " "N"; then
            # shellcheck disable=SC2086
            {
                #
                # Make the metadata
                #
                $BASEDIR/make_metadata -dbconf=${dbconfig} -from=$start -verb -out -script
                RES=$?
                if [[ $RES -eq 0 ]]; then
                    ia upload --retries=3 --spreadsheet=${metadata} \
                        -H x-archive-keep-old-version:0 && ./${script}
                else
                    echo "Upload aborted due to errors"
                fi
            }
        else
            echo "Not uploaded"
        fi
    fi
else
    #
    # Multiple episodes
    # -----------------
    #
    # Compute end show number
    #
    ((end = start + count - 1))

    #
    # Check existence of files
    #
    for (( i = start; i < end; i++ )); do
        if ! check_uploads "hpr$i"; then
            echo "Missing files for show $i. Aborted!"
            exit 1
        fi
    done

    #
    # Define files
    #
    metadata="metadata_${start}-${end}.csv"
    script="script_${start}-${end}.sh"

    if [[ $DRYRUN -eq 1 ]]; then
        echo "Dry run: Would have uploaded $start to $end inclusive"
        echo "Dry run: Would have created $metadata and $script"
        echo "Dry run: Would have uploaded $metadata and run $script"
        echo "Dry run: Would have used $dbconfig"
    else
        echo "Uploading $start to $end inclusive"
        if yes_no "OK to continue? %s " "N"; then
            # shellcheck disable=2086
            {
                #
                # Make the metadata
                #
                $BASEDIR/make_metadata -dbconf=${dbconfig} -from=$start -count=$count -verb -out -script
                RES=$?
                if [[ $RES -eq 0 ]]; then
                    ia upload --retries=3 --spreadsheet=${metadata} \
                        -H x-archive-keep-old-version:0 && ./${script}
                else
                    echo "Upload aborted due to errors"
                fi
            }
        else
            echo "Not uploaded"
        fi
    fi
fi

exit

# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21