forked from HPR/hpr-tools
		
	
		
			
				
	
	
		
			607 lines
		
	
	
		
			18 KiB
		
	
	
	
		
			Bash
		
	
	
		
			Executable File
		
	
	
	
	
			
		
		
	
	
			607 lines
		
	
	
		
			18 KiB
		
	
	
	
		
			Bash
		
	
	
		
			Executable File
		
	
	
	
	
#!/bin/bash -
 | 
						|
#===============================================================================
 | 
						|
#
 | 
						|
#         FILE: future_upload
 | 
						|
#
 | 
						|
#        USAGE: ./future_upload
 | 
						|
#
 | 
						|
#  DESCRIPTION: Uploads future HPR shows based on what is in the upload area
 | 
						|
#
 | 
						|
#      OPTIONS: ---
 | 
						|
# REQUIREMENTS: ---
 | 
						|
#         BUGS: ---
 | 
						|
#        NOTES: Contains methods from 'delete_uploaded' and 'weekly_upload' as
 | 
						|
#               well as 'update_state'
 | 
						|
#       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
 | 
						|
#      VERSION: 0.0.16
 | 
						|
#      CREATED: 2021-01-07 12:11:02
 | 
						|
#     REVISION: 2025-01-01 11:48:40
 | 
						|
#
 | 
						|
#===============================================================================
 | 
						|
 | 
						|
set -o nounset                              # Treat unset variables as an error
 | 
						|
 | 
						|
SCRIPT=${0##*/}
 | 
						|
# DIR=${0%/*}
 | 
						|
 | 
						|
STDOUT="/dev/fd/2"
 | 
						|
 | 
						|
VERSION="0.0.16"
 | 
						|
 | 
						|
#
 | 
						|
# Load library functions
 | 
						|
#
 | 
						|
LIB="$HOME/bin/function_lib.sh"
 | 
						|
[ -e "$LIB" ] || { echo "Unable to source functions"; exit 1; }
 | 
						|
# shellcheck disable=SC1090
 | 
						|
source "$LIB"
 | 
						|
 | 
						|
# {{{ -- Functions -- check_uploads, _log, _usage
 | 
						|
 | 
						|
#===  FUNCTION  ================================================================
 | 
						|
#         NAME: check_uploads
 | 
						|
#  DESCRIPTION: Determines if files exist for uploading
 | 
						|
#   PARAMETERS: 1 - filename prefix e.g. 'hpr9999'
 | 
						|
#      RETURNS: True/false
 | 
						|
#===============================================================================
 | 
						|
check_uploads () {
 | 
						|
    local prefix=${1:?Usage: check_uploads prefix}
 | 
						|
    local suff
 | 
						|
 | 
						|
    #
 | 
						|
    # Look for files called hpr1234.flac and so on. Don't bother with the
 | 
						|
    # hpr1234_source.flac one. As soon as a file is missing return with false.
 | 
						|
    # 2025-01-01: Dropped 'spx' from the list
 | 
						|
    #
 | 
						|
    for suff in flac mp3 ogg opus wav; do
 | 
						|
        if [[ ! -e $UPLOADS/$prefix.$suff ]]; then
 | 
						|
            return 1
 | 
						|
        fi
 | 
						|
    done
 | 
						|
 | 
						|
    #
 | 
						|
    # Transcripts are (currently) in a sub-directory with the same name as the
 | 
						|
    # IA item. We only cater for two types as of 2025.
 | 
						|
    #
 | 
						|
    for suff in txt srt; do
 | 
						|
        if [[ ! -e $UPLOADS/$prefix/$prefix.$suff ]]; then
 | 
						|
            return 1
 | 
						|
        fi
 | 
						|
    done
 | 
						|
 | 
						|
    return 0
 | 
						|
}
 | 
						|
 | 
						|
#===  FUNCTION  ================================================================
 | 
						|
#         NAME: _log
 | 
						|
#  DESCRIPTION: Writes a log record to the predefined $LOGFILE in this script
 | 
						|
#               using the predefined $LOGREC, a template for 'printf'. If the
 | 
						|
#               latter is not defined the function will use a default.
 | 
						|
#               For some reason 'shellcheck' objects to this function. The
 | 
						|
#               first argument to 'printf' needs to be -1 to make the
 | 
						|
#               '%(fmt)T' use today's date and time.
 | 
						|
#   PARAMETERS: 1 - the message to write
 | 
						|
#      RETURNS: Nothing
 | 
						|
#===============================================================================
 | 
						|
# shellcheck disable=SC2317 disable=SC2059
 | 
						|
_log () {
 | 
						|
    local msg="$1"
 | 
						|
 | 
						|
#     echo "D> $LOGFILE $LOGREC"
 | 
						|
    [ -v LOGFILE ] || { echo "${FUNCNAME[0]}: \$LOGFILE is not defined"; exit 1; }
 | 
						|
    [ -v LOGREC ] || { local LOGREC='%(%F %T)T %s\n'; }
 | 
						|
 | 
						|
#     echo "D> $LOGFILE $LOGREC"
 | 
						|
    printf "$LOGREC" -1 "$msg" >> "$LOGFILE"
 | 
						|
 | 
						|
    return
 | 
						|
}
 | 
						|
 | 
						|
#===  FUNCTION  ================================================================
 | 
						|
#         NAME: _usage
 | 
						|
#  DESCRIPTION: Report usage
 | 
						|
#   PARAMETERS: 1       [optional] exit value
 | 
						|
#      RETURNS: Nothing
 | 
						|
#===============================================================================
 | 
						|
_usage () {
 | 
						|
    local -i res="${1:-0}"
 | 
						|
 | 
						|
    cat >$STDOUT <<-endusage
 | 
						|
${SCRIPT} - version: ${VERSION}
 | 
						|
 | 
						|
Usage: ./${SCRIPT} [-h] [-v] [-D] [-d {0|1}] [-F] [-r] [-l cp]
 | 
						|
 | 
						|
Uploads HPR shows to the Internet Archive that haven't yet been uploaded. This
 | 
						|
is as an alternative to uploading the next 5 shows each week for the coming
 | 
						|
week.
 | 
						|
 | 
						|
Options:
 | 
						|
  -h                    Print this help
 | 
						|
  -v                    Run in verbose mode where more information is reported
 | 
						|
  -D                    Run in debug mode where a lot more information is
 | 
						|
                        reported
 | 
						|
  -d 0|1                Dry run: -d 1 (the default) runs the script in dry-run
 | 
						|
                        mode where nothing is uploaded but the actions that
 | 
						|
                        will be taken are reported; -d 0 turns off dry-run
 | 
						|
                        mode and the actions will be carried out.
 | 
						|
  -F                    Force the upload of a show even if the checks for its
 | 
						|
                        state in the 'reservations' table gives the wrong
 | 
						|
                        state or none at all. This is a rare event which may
 | 
						|
                        come about if an "emergency" show is being deployed or
 | 
						|
                        in some other exceptional circumstance.
 | 
						|
  -r                    Run in 'remote' mode, using the live database over an
 | 
						|
                        (already established) SSH tunnel. Default is to run
 | 
						|
                        against the local database.
 | 
						|
  -l N                  Control the number of shows that can be uploaded at
 | 
						|
                        once. The range is 1 to $DEFLIMIT.
 | 
						|
 | 
						|
Notes:
 | 
						|
 | 
						|
1. When running on 'borg' the method used is to run in faux 'local' mode.
 | 
						|
   This means we have an open tunnel to the HPR server (mostly left open) and
 | 
						|
   the default file .hpr_db.cfg points to the live database via this tunnel.
 | 
						|
   So we do not use the -r option here. This is a bit of a hack! Sorry!
 | 
						|
 | 
						|
endusage
 | 
						|
    exit "$res"
 | 
						|
}
 | 
						|
 | 
						|
# }}}
 | 
						|
 | 
						|
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 | 
						|
 | 
						|
#
 | 
						|
# Configure depending whether local or on borg
 | 
						|
#
 | 
						|
case $HOSTNAME in
 | 
						|
    hprvps|marvin|borg)
 | 
						|
        UPLOADS="/data/IA/uploads"
 | 
						|
        BASEDIR="$HOME/IA" ;;
 | 
						|
    i7-desktop)
 | 
						|
        UPLOADS="$HOME/HPR/IA/uploads"
 | 
						|
        BASEDIR="$HOME/HPR/IA" ;;
 | 
						|
    *)
 | 
						|
        echo "Wrong host!"; exit 1 ;;
 | 
						|
esac
 | 
						|
 | 
						|
cd "$BASEDIR" || { echo "Can't cd to $BASEDIR"; exit 1; }
 | 
						|
 | 
						|
#
 | 
						|
# Prepare for logging
 | 
						|
#
 | 
						|
LOGS="$BASEDIR/logs"
 | 
						|
LOGFILE="$LOGS/$SCRIPT.log"
 | 
						|
LOGREC='%(%F %T)T %s\n'
 | 
						|
 | 
						|
#
 | 
						|
# Tools
 | 
						|
#
 | 
						|
BASECOM='curl -K ./.hpradmin_curlrc -s'
 | 
						|
URL="https://hub.hackerpublicradio.org/cms/status.php"
 | 
						|
# QUERY1="${BASECOM} ${URL}"
 | 
						|
QUERY2="${BASECOM} -o - ${URL}"
 | 
						|
UPSTATE="$BASEDIR/update_state"
 | 
						|
 | 
						|
#
 | 
						|
# Fallback URL
 | 
						|
#
 | 
						|
# URL_BAK="http://hub.hackerpublicradio.org/cms/status.php"
 | 
						|
# QUERY2_BAK="${BASECOM} -o - ${URL_BAK}"
 | 
						|
 | 
						|
#
 | 
						|
# Prerequisites
 | 
						|
#
 | 
						|
# jq=$(command -v jq)
 | 
						|
# [ -z "$jq" ] && { echo "Needs the 'jq' JSON filter"; exit 1; }
 | 
						|
ia=$(command -v ia)
 | 
						|
[ -z "$ia" ] && { echo "Needs the 'ia' Internet Archive script"; exit 1; }
 | 
						|
[ -e "$BASEDIR/make_metadata" ] || {
 | 
						|
    echo "Needs the 'make_metadata' script"
 | 
						|
    exit 1
 | 
						|
}
 | 
						|
[ -e "$UPSTATE" ] || {
 | 
						|
    echo "Needs the 'update_state' script"
 | 
						|
    exit 1
 | 
						|
}
 | 
						|
 | 
						|
#
 | 
						|
# File of processed shows
 | 
						|
#
 | 
						|
PROCFILE="$BASEDIR/.${SCRIPT}.dat"
 | 
						|
[ -e "$PROCFILE" ] || touch "$PROCFILE"
 | 
						|
 | 
						|
#
 | 
						|
# Constants
 | 
						|
#
 | 
						|
RETRIES=5
 | 
						|
DEFLIMIT=20
 | 
						|
 | 
						|
#
 | 
						|
# Process options
 | 
						|
#
 | 
						|
while getopts :d:FhvDr:l: opt
 | 
						|
do
 | 
						|
    case "${opt}" in
 | 
						|
        d) DRYRUN=$OPTARG;;
 | 
						|
        D) DEBUG=1;;
 | 
						|
        F) FORCE=1;;
 | 
						|
        h) _usage 0;;
 | 
						|
        v) VERBOSE=1;;
 | 
						|
        r) REMOTE=1;;
 | 
						|
        l) LIMIT=$OPTARG;;
 | 
						|
        *) _usage 1;;
 | 
						|
    esac
 | 
						|
done
 | 
						|
shift $((OPTIND - 1))
 | 
						|
 | 
						|
DRYRUN=${DRYRUN:-1}
 | 
						|
if [[ $DRYRUN -ne 0 && $DRYRUN -ne 1 ]]; then
 | 
						|
    echo "** Use '-d 0' or '-d 1'"
 | 
						|
    _usage 1
 | 
						|
fi
 | 
						|
[[ $DRYRUN -eq 1 ]] && echo "Dry run mode"
 | 
						|
 | 
						|
FORCE=${FORCE:-0}
 | 
						|
 | 
						|
VERBOSE=${VERBOSE:-0}
 | 
						|
 | 
						|
DEBUG=${DEBUG:-0}
 | 
						|
 | 
						|
REMOTE=${REMOTE:-0}
 | 
						|
if [[ $REMOTE -eq 0 ]]; then
 | 
						|
    dbconfig="$BASEDIR/.hpr_db.cfg"
 | 
						|
    [[ $VERBOSE -eq 1 ]] && echo "Local database mode"
 | 
						|
else
 | 
						|
    dbconfig="$BASEDIR/.hpr_livedb.cfg"
 | 
						|
    [[ $VERBOSE -eq 1 ]] && echo "Remote database mode"
 | 
						|
fi
 | 
						|
 | 
						|
LIMIT=${LIMIT:-$DEFLIMIT}
 | 
						|
if [[ $LIMIT -lt 1 || $LIMIT -gt $DEFLIMIT ]]; then
 | 
						|
    echo "** Use '-l 1' up to '-l $DEFLIMIT' or omit the option"
 | 
						|
    _usage 1
 | 
						|
fi
 | 
						|
 | 
						|
#
 | 
						|
# Should have no arguments
 | 
						|
#
 | 
						|
if [[ $# != 0 ]]; then
 | 
						|
    echo "** ${SCRIPT} takes no arguments"
 | 
						|
    _usage 1
 | 
						|
fi
 | 
						|
 | 
						|
#
 | 
						|
# Declarations
 | 
						|
#
 | 
						|
declare -A processed
 | 
						|
declare -A ready
 | 
						|
declare -A uploads
 | 
						|
minshow=
 | 
						|
maxshow=
 | 
						|
lastitem=
 | 
						|
 | 
						|
#
 | 
						|
# Load array of processed shows
 | 
						|
#
 | 
						|
while read -r item; do
 | 
						|
    processed+=([$item]=1)
 | 
						|
done < "$PROCFILE"
 | 
						|
[ "$VERBOSE" -eq 1 ] && echo "Number of shows in cache: ${#processed[@]}"
 | 
						|
 | 
						|
#
 | 
						|
# TODO: Create the associative array 'ready' containing the numbers of shows
 | 
						|
# ready for upload. This is a way to ensure that we don't try and upload shows
 | 
						|
# in transit to the upload area.
 | 
						|
#
 | 
						|
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 | 
						|
# Proposed code. Not sure what the actual URL will be nor what will be
 | 
						|
# returned if nothing is ready for upload yet
 | 
						|
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 | 
						|
#
 | 
						|
# json=$(curl http://hackerpublicradio.org/queue.php -s -o -)
 | 
						|
# while read -r showno; do
 | 
						|
#     ready+=([$showno]=1)
 | 
						|
# done < <(echo "${json}" | jq '.READY_FOR_IA_UPLOAD[] | tonumber')
 | 
						|
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 | 
						|
# Change of plan. Now we have a list of CSV values, so we need to do something
 | 
						|
# like this:
 | 
						|
#
 | 
						|
# reservations=$($BASECOM -o - $URL)
 | 
						|
# while read -r line; do
 | 
						|
#    if [[ $line =~ ^([^,]+),([^,]+),([^,]+),([^,]+),([^,]+),.*$ ]]; then
 | 
						|
#       state="${BASH_REMATCH[5]}"
 | 
						|
#       show="${BASH_REMATCH[2]}"
 | 
						|
#    fi
 | 
						|
#    if [[ $state = 'MEDIA_TRANSCODED' ]]; then
 | 
						|
#       ready+=([$show]=1)
 | 
						|
#    fi
 | 
						|
# done <<< $reservations
 | 
						|
#
 | 
						|
# At the end of this the associative array 'ready' will contain the keys of
 | 
						|
# shows that are ready for upload (presumably) so we can look in this array to
 | 
						|
# double check.
 | 
						|
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 | 
						|
 | 
						|
if [[ $FORCE -eq 0 ]]; then
 | 
						|
    #
 | 
						|
    # Collect the current table of shows requiring work. We expect something like:
 | 
						|
    # timestamp_epoc,ep_num,ep_date,key,status,email
 | 
						|
    # 1651286617,3617,2022-06-14,fda088e0e3bd5d0353ea6b7569e93b87626ca25976a0a,UPLOADED_TO_IA,lurkingprion@gmail.com
 | 
						|
    # 1651648589,3619,2022-06-16,e7d3810afa098863d81663418d8640276272284de68f1,UPLOADED_TO_IA,monochromec@gmail.com
 | 
						|
    # TODO: Check for a failure in the query?A
 | 
						|
    # NOTE: Problem encountered 2022-09-23 because the SSL certificate has expired
 | 
						|
    #
 | 
						|
    reservations=$($QUERY2) || {
 | 
						|
#         echo "Problem querying $URL"
 | 
						|
#         echo "Falling back to $URL_BAK"
 | 
						|
#         reservations=$($QUERY2_BAK) || {
 | 
						|
#             echo "Failed with fallback URL - aborting"
 | 
						|
#             exit 1
 | 
						|
#         }
 | 
						|
        echo "Failed to query $URL - aborting"
 | 
						|
        exit 1
 | 
						|
    }
 | 
						|
    _DEBUG "reservations = $reservations"
 | 
						|
 | 
						|
    #
 | 
						|
    # The query above might fail in a way that just returns an empty string,
 | 
						|
    # so check for that
 | 
						|
    #
 | 
						|
    if [[ -z $reservations ]]; then
 | 
						|
        echo "No reply from $URL - can't continue"
 | 
						|
        exit 1
 | 
						|
    fi
 | 
						|
 | 
						|
    #
 | 
						|
    # The query returns the bare number, but we're using 'hprxxxx' as the key in
 | 
						|
    # the 'ready' array.
 | 
						|
    #
 | 
						|
    while read -r line; do
 | 
						|
        if [[ $line =~ ^([^,]+),([^,]+),([^,]+),([^,]+),([^,]+),.*$ ]]; then
 | 
						|
            state="${BASH_REMATCH[5]}"
 | 
						|
            show="${BASH_REMATCH[2]}"
 | 
						|
        fi
 | 
						|
        if [[ $state = 'MEDIA_TRANSCODED' ]]; then
 | 
						|
            ready+=([hpr$show]=1)
 | 
						|
        fi
 | 
						|
    done <<< "$reservations"
 | 
						|
    _DEBUG "ready = ${!ready[*]}"
 | 
						|
else
 | 
						|
    [ "$VERBOSE" -eq 1 ] && {
 | 
						|
        echo "V: Not checking reservations table; force option used"
 | 
						|
    }
 | 
						|
fi
 | 
						|
 | 
						|
#
 | 
						|
# Process files. There will be several with the same prefix so look for
 | 
						|
# a change of prefix
 | 
						|
#
 | 
						|
while read -r path; do
 | 
						|
    #
 | 
						|
    # Extract the path relative to $UPLOADS and the IA item name from the
 | 
						|
    # returned path. Assume names are 'hpr9999' format (with leading zeroes if
 | 
						|
    # appropriate).
 | 
						|
    #
 | 
						|
    relpath="${path#"$UPLOADS"/}"
 | 
						|
    item="${relpath:0:7}"
 | 
						|
 | 
						|
    _DEBUG "Found path $path"
 | 
						|
    _DEBUG "Relative path $relpath"
 | 
						|
    _DEBUG "Item $item"
 | 
						|
 | 
						|
    #
 | 
						|
    # Detect that the item prefix has changed. If it has we're processing
 | 
						|
    # a new IA identifier, so work on this one
 | 
						|
    #
 | 
						|
    if [[ $item != "$lastitem" ]]; then
 | 
						|
        lastitem=$item
 | 
						|
        echo -n "$lastitem "
 | 
						|
 | 
						|
        #
 | 
						|
        # Have we already processed it? It might just not be in the cache even
 | 
						|
        # though processed by some other means.
 | 
						|
        #
 | 
						|
        if [[ ! -v "processed[$lastitem]" ]]; then
 | 
						|
 | 
						|
            [ "$VERBOSE" -eq 1 ] && {
 | 
						|
                echo "V: Not yet processed (or not cached) $lastitem"
 | 
						|
                echo "V: Checking IA for $lastitem"
 | 
						|
            }
 | 
						|
 | 
						|
            #
 | 
						|
            # Ask the IA
 | 
						|
            #
 | 
						|
            if ia list "$lastitem" > /dev/null 2>&1; then
 | 
						|
                #
 | 
						|
                # We can't tell with 'ia list' whether the show is
 | 
						|
                # a 'reserved' one. Need to look deeper.
 | 
						|
                #
 | 
						|
 | 
						|
                #
 | 
						|
                # It's on the IA already, save in the array
 | 
						|
                #
 | 
						|
                #[ $VERBOSE -eq 1 ] && echo "V: Already on IA - $lastitem"
 | 
						|
                echo "- cached"
 | 
						|
                processed+=([$lastitem]=1)
 | 
						|
            else
 | 
						|
                #
 | 
						|
                # Is the show ready for upload?
 | 
						|
                #
 | 
						|
                if [[ $FORCE -eq 0 ]]; then
 | 
						|
                    if [[ ! -v "ready[$lastitem]" ]]; then
 | 
						|
                        echo "- not ready"
 | 
						|
                        continue
 | 
						|
                    fi
 | 
						|
                fi
 | 
						|
 | 
						|
                #
 | 
						|
                # Need to upload this one
 | 
						|
                #
 | 
						|
                #[ $VERBOSE -eq 1 ] && echo "V: To be uploaded to IA - $lastitem"
 | 
						|
                echo "- to upload"
 | 
						|
 | 
						|
                uploads+=([${lastitem:3}]=1)
 | 
						|
 | 
						|
                #
 | 
						|
                # First show we find not on the IA should be the lowest number
 | 
						|
                #
 | 
						|
                [ -z "$minshow" ] && minshow="${lastitem:3}"
 | 
						|
 | 
						|
                #
 | 
						|
                # Last show number should be the highest
 | 
						|
                #
 | 
						|
                maxshow="${lastitem:3}"
 | 
						|
 | 
						|
                #
 | 
						|
                # Stop the loop if we have reached the limiting number
 | 
						|
                #
 | 
						|
                [[ ${#uploads[@]} -eq $LIMIT ]] && {
 | 
						|
                    echo "Upload limit ($LIMIT) reached"
 | 
						|
                    break
 | 
						|
                }
 | 
						|
            fi
 | 
						|
        else
 | 
						|
            #
 | 
						|
            # Recorded as having been seen
 | 
						|
            #
 | 
						|
            #[ $VERBOSE -eq 1 ] && echo "V: Already processed $lastitem"
 | 
						|
            echo "- processed"
 | 
						|
        fi
 | 
						|
 | 
						|
    fi
 | 
						|
 | 
						|
done < <(find "$UPLOADS" -regextype posix-extended -regex '.*hpr[0-9]{4}.*' | sort)
 | 
						|
 | 
						|
#
 | 
						|
# Write the processed array to the cache file unless in dry-run mode
 | 
						|
#
 | 
						|
# [ $DEBUG -eq 1 ] && { echo -n 'D> '; declare -p processed; }
 | 
						|
_DEBUG "processed = ${!processed[*]}"
 | 
						|
[ "$VERBOSE" -eq 1 ] && echo "Number of shows in cache: ${#processed[@]}"
 | 
						|
if [[ $DRYRUN -ne 1 ]]; then
 | 
						|
    while read -r item; do
 | 
						|
        echo "$item"
 | 
						|
    done < <(printf '%s\n' "${!processed[@]}" | sort -u ) > "$PROCFILE"
 | 
						|
fi
 | 
						|
 | 
						|
#
 | 
						|
# Generate the list of uploads for the 'make_metadata' option '-list=1,2,3'.
 | 
						|
# Order is unimportant because make_metadata sorts internally.
 | 
						|
#
 | 
						|
_DEBUG "uploads = ${!uploads[*]}"
 | 
						|
[ "$VERBOSE" -eq 1 ] && echo "Number of shows for upload: ${#uploads[@]}"
 | 
						|
printf -v list '%s,' "${!uploads[@]}"
 | 
						|
list="${list:0:-1}"
 | 
						|
 | 
						|
#
 | 
						|
# If there are no uploads to do we can stop
 | 
						|
#
 | 
						|
[[ ! -v uploads[@] ]] && { echo "Nothing to do!"; exit; }
 | 
						|
 | 
						|
#
 | 
						|
# Check that the shows being uploaded have all their files and log what is
 | 
						|
# happening.
 | 
						|
#
 | 
						|
while read -r show; do
 | 
						|
    echo "$(date +%Y%m%d%H%M%S) preparing to upload hpr$show" >> "$LOGFILE"
 | 
						|
 | 
						|
    if ! check_uploads "hpr$show"; then
 | 
						|
        echo "Missing files for show $show. Aborted!"
 | 
						|
        echo "$(date +%Y%m%d%H%M%S) aborting on hpr$show - missing files" >> "$LOGFILE"
 | 
						|
        exit 1
 | 
						|
    fi
 | 
						|
done < <(printf '%s\n' "${!uploads[@]}" | sort)
 | 
						|
 | 
						|
#
 | 
						|
# Define output files. If the list contains one element then it's a different
 | 
						|
# name from the multi-element case (make_metadata does this too).
 | 
						|
#
 | 
						|
if [[ ${#uploads[@]} -eq 1 ]]; then
 | 
						|
    metadata="metadata_${minshow}.csv"
 | 
						|
    script="script_${minshow}.sh"
 | 
						|
else
 | 
						|
    metadata="metadata_${minshow}-${maxshow}.csv"
 | 
						|
    script="script_${minshow}-${maxshow}.sh"
 | 
						|
fi
 | 
						|
 | 
						|
#
 | 
						|
# Perform the uploads or report what would be done
 | 
						|
#
 | 
						|
if [[ $DRYRUN -eq 1 ]]; then
 | 
						|
    echo "Dry run: Would have uploaded list '$list'"
 | 
						|
    echo "Dry run: Would have created $metadata and $script"
 | 
						|
    echo "Dry run: Would have uploaded $metadata and run $script"
 | 
						|
    echo "Dry run: Would have used $dbconfig"
 | 
						|
    echo "$BASEDIR/make_metadata -dbconf=${dbconfig} -list=\"$list\" -verb -out -script"
 | 
						|
 | 
						|
    echo "$(date +%Y%m%d%H%M%S) no uploads done - dry-run mode" >> "$LOGFILE"
 | 
						|
else
 | 
						|
    echo "Uploading the list '$list'"
 | 
						|
    if yes_no "OK to continue? %s " "N"; then
 | 
						|
        # shellcheck disable=2086
 | 
						|
        {
 | 
						|
            #
 | 
						|
            # Make the metadata
 | 
						|
            #
 | 
						|
            $BASEDIR/make_metadata -dbconf=${dbconfig} -list="$list" -verb -out -script
 | 
						|
            RES=$?
 | 
						|
            if [[ $RES -eq 0 ]]; then
 | 
						|
                #
 | 
						|
                # Upload in spreadsheet mode and run the generated script
 | 
						|
                #
 | 
						|
                ia upload --retries=$RETRIES --spreadsheet=${metadata} \
 | 
						|
                    -n -H x-archive-keep-old-version:0
 | 
						|
                RES=$?
 | 
						|
                if [[ $RES -eq 0 ]]; then
 | 
						|
                    #
 | 
						|
                    # Upload worked. Run the script if there is one
 | 
						|
                    #
 | 
						|
                    [[ -e $script ]] && ./${script}
 | 
						|
                else
 | 
						|
                    echo "Failed to upload to IA; aborting"
 | 
						|
                    echo "$(date +%Y%m%d%H%M%S) IA uploads aborted due to errors" >> "$LOGFILE"
 | 
						|
                    exit 1
 | 
						|
                fi
 | 
						|
 | 
						|
                #
 | 
						|
                # Append the sorted show details to the cache
 | 
						|
                #
 | 
						|
                echo "$list" |\
 | 
						|
                    sed -e 's/\([0-9]\{4\}\)/hpr\1/g; s/,/\n/g' | sort >> "$PROCFILE"
 | 
						|
 | 
						|
                echo "Uploaded ${#uploads[@]} shows"
 | 
						|
                echo "$(date +%Y%m%d%H%M%S) ${#uploads[@]} uploads completed" >> "$LOGFILE"
 | 
						|
 | 
						|
                #
 | 
						|
                # Update the state in the HPR database, unless we're using
 | 
						|
                # FORCE. Pass the limit used here to this script so it can
 | 
						|
                # stop looking for work unnecessarily
 | 
						|
                #
 | 
						|
                if [[ $FORCE -eq 0 ]]; then
 | 
						|
                    $UPSTATE -l$LIMIT
 | 
						|
                    RES=$?
 | 
						|
                    if [[ $RES -ne 0 ]]; then
 | 
						|
                        echo "Problem updating database state"
 | 
						|
                        exit 1
 | 
						|
                    fi
 | 
						|
                else
 | 
						|
                    echo "Not updating the database, FORCE mode is on"
 | 
						|
                fi
 | 
						|
 | 
						|
 | 
						|
            else
 | 
						|
                echo "Upload aborted due to errors"
 | 
						|
                echo "$(date +%Y%m%d%H%M%S) uploads aborted due to errors" >> "$LOGFILE"
 | 
						|
                exit 1
 | 
						|
            fi
 | 
						|
        }
 | 
						|
    else
 | 
						|
        echo "Not uploaded, as requested"
 | 
						|
        echo "$(date +%Y%m%d%H%M%S) uploads aborted by user" >> "$LOGFILE"
 | 
						|
    fi
 | 
						|
fi
 | 
						|
 | 
						|
exit
 | 
						|
 | 
						|
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker
 |