19030fee71
InternetArchive/future_upload: Added logging and debugging
InternetArchive/ia_db.sql: Added new tables
InternetArchive/recover_transcripts: New script to run on 'borg' and
copy missing files from the backup disk to the IA
InternetArchive/repair_assets: More comments, including one about a bug in the design.
InternetArchive/repair_item: Fix relating to octal numbers (if there are
leading zeroes in a number). '_DEBUG' is now in the function
library. Added comments to explain obscure stuff.
InternetArchive/snapshot_metadata: New Bash script (to run on my
desktop) which collects metadata for a show and stores in in the
'~/HPR/IA/assets' directory. Runs 'view_derivatives' on it to find
derivative files for deletion.
InternetArchive/tidy_uploaded: Moves files and directories containing
uploaded files into a holding area for later backup. Added
debugging, logging and a 'force' mode.
InternetArchive/upload_manager: Manages 'ia.db' (on my workstation).
Needs many updates which have just started to be added.
InternetArchive/weekly_upload: Old script, now obsolete.
305 lines
8.3 KiB
Bash
Executable File
305 lines
8.3 KiB
Bash
Executable File
#!/bin/bash -
|
|
#===============================================================================
|
|
#
|
|
# FILE: weekly_upload
|
|
#
|
|
# USAGE: ./weekly_upload [-h] [-r] [-v] [-d {0|1}] start [count]
|
|
#
|
|
# DESCRIPTION: Run the commands necessary to upload a batch of HPR shows to
|
|
# archive.org
|
|
#
|
|
# ** NOW OBSOLETE **
|
|
# We do uploads differently now.
|
|
#
|
|
# OPTIONS: ---
|
|
# REQUIREMENTS: ---
|
|
# BUGS: ---
|
|
# NOTES: ---
|
|
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
|
# VERSION: 0.0.9
|
|
# CREATED: 2018-10-12 21:54:35
|
|
# REVISION: 2020-10-13 13:54:32
|
|
#
|
|
#===============================================================================
|
|
|
|
echo "Obsolete script. Do not use!"
|
|
cmd='exit'; $cmd
|
|
|
|
set -o nounset # Treat unset variables as an error
|
|
|
|
SCRIPT=${0##*/}
|
|
#DIR=${0%/*}
|
|
VERSION="0.0.9"
|
|
|
|
STDOUT="/dev/fd/2"
|
|
|
|
#
|
|
# Select the appropriate working directory
|
|
#
|
|
case $(hostname) in
|
|
i7-desktop)
|
|
BASEDIR="$HOME/HPR/InternetArchive"
|
|
UPLOAD="$BASEDIR/uploads"
|
|
;;
|
|
hprvps|marvin|borg)
|
|
BASEDIR="$HOME/IA"
|
|
UPLOAD="/var/IA/uploads"
|
|
;;
|
|
*)
|
|
echo "Wrong host!"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
cd "$BASEDIR" || exit 1
|
|
|
|
#
|
|
# Load library functions
|
|
#
|
|
LIB="$HOME/bin/function_lib.sh"
|
|
[ -e "$LIB" ] || { echo "Unable to source functions"; exit; }
|
|
# shellcheck disable=SC1090
|
|
source "$LIB"
|
|
|
|
|
|
#=== FUNCTION ================================================================
|
|
# NAME: check_uploads
|
|
# DESCRIPTION: Determines if files exist for uploading
|
|
# PARAMETERS: 1 - filename prefix e.g. 'hpr9999'
|
|
# RETURNS: True/false
|
|
#===============================================================================
|
|
check_uploads () {
|
|
local prefix=${1:?Usage: check_uploads prefix}
|
|
local suff
|
|
|
|
#
|
|
# Look for files called hpr1234.flac and so on. Don't bother with the
|
|
# hpr1234_source.flac one. As soon as a file is missing return with false.
|
|
#
|
|
for suff in flac mp3 ogg opus spx wav; do
|
|
if [[ ! -e $UPLOAD/$prefix.$suff ]]; then
|
|
return 1
|
|
fi
|
|
done
|
|
|
|
return 0
|
|
}
|
|
|
|
#=== FUNCTION ================================================================
|
|
# NAME: _usage
|
|
# DESCRIPTION: Reports usage; always exits the script after doing so
|
|
# PARAMETERS: 1 - the integer to pass to the 'exit' command
|
|
# RETURNS: Nothing
|
|
#===============================================================================
|
|
_usage () {
|
|
local -i result=${1:-0}
|
|
|
|
cat >$STDOUT <<-endusage
|
|
${SCRIPT} - version: ${VERSION}
|
|
|
|
Usage: ./${SCRIPT} [-h] [-r] [-v] [-d {0|1}] start [count]
|
|
|
|
Generates the necessary metadata and script and uses them to upload HPR audio
|
|
and other show-related files held on the VPS to the Internet Archive.
|
|
|
|
Options:
|
|
-h Print this help
|
|
-v Run in verbose mode where more information is reported
|
|
-d 0|1 Dry run: -d 1 (the default) runs the script in dry-run
|
|
mode where nothing is changed but the actions that
|
|
will be taken are reported; -d 0 turns off dry-run
|
|
mode and the actions will be carried out.
|
|
-r Run in 'remote' mode, using the live database over an
|
|
(already established) SSH tunnel. Default is to run
|
|
against the local database.
|
|
Arguments:
|
|
start the starting show number to be uploaded
|
|
count (optional, default 1) the number of shows to be
|
|
uploaded; cannot exceed 20
|
|
|
|
Notes:
|
|
|
|
1. When running on 'marvin' the method used is to run in faux 'local' mode.
|
|
This means we have an open tunnel to the HPR server (mostly left open) and
|
|
the default file .hpr_db.cfg points to the live database via this tunnel.
|
|
So we do not use the -r option here. This is a bit of a hack! Sorry!
|
|
|
|
2. There are potential problems when a show has no tags which haven't been
|
|
fully resolved. The make_metadata script fails in default mode when it
|
|
finds such a show, but this (weekly_upload) script can continue on and run
|
|
the generated script which uploads the source audio files. This can mean
|
|
the IA items end up as books! In this mode the description is not stored
|
|
and so there are no show notes.
|
|
|
|
endusage
|
|
exit "$result"
|
|
}
|
|
|
|
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
|
|
#
|
|
# Process options
|
|
#
|
|
while getopts :d:hrv opt
|
|
do
|
|
case "${opt}" in
|
|
d) DRYRUN=$OPTARG;;
|
|
h) _usage 1;;
|
|
r) REMOTE=1;;
|
|
v) VERBOSE=1;;
|
|
*) _usage 1;;
|
|
esac
|
|
done
|
|
shift $((OPTIND - 1))
|
|
|
|
DRYRUN=${DRYRUN:-1}
|
|
if [[ $DRYRUN -ne 0 && $DRYRUN -ne 1 ]]; then
|
|
echo "** Use '-d 0' or '-d 1'"
|
|
_usage 1
|
|
fi
|
|
|
|
VERBOSE=${VERBOSE:-0}
|
|
|
|
REMOTE=${REMOTE:-0}
|
|
if [[ $REMOTE -eq 0 ]]; then
|
|
dbconfig="$BASEDIR/.hpr_db.cfg"
|
|
[[ $VERBOSE -eq 1 ]] && echo "Local database mode"
|
|
else
|
|
dbconfig="$BASEDIR/.hpr_livedb.cfg"
|
|
[[ $VERBOSE -eq 1 ]] && echo "Remote database mode"
|
|
fi
|
|
|
|
#
|
|
# Check argument count
|
|
#
|
|
if [[ ! ( $# -eq 1 || $# -eq 2 ) ]]; then
|
|
echo "Wrong number of arguments"
|
|
_usage 1
|
|
fi
|
|
|
|
#
|
|
# Validate arguments
|
|
#
|
|
for arg; do
|
|
if [[ ! $arg =~ ^[0-9]{1,4}$ ]]; then
|
|
echo "Invalid number: $arg"
|
|
echo "Use a plain number"
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
start=$1
|
|
count=${2:-1}
|
|
if [[ $count -gt 20 ]]; then
|
|
echo "Can't process more than 20 shows at a time"
|
|
exit 1
|
|
fi
|
|
|
|
[[ $VERBOSE -eq 1 && $DRYRUN -eq 1 ]] && echo "Dry run mode"
|
|
|
|
#
|
|
# Two paths; one for a single episode, the other for multiple episodes
|
|
#
|
|
if [[ $count -eq 1 ]]; then
|
|
#
|
|
# Single episode
|
|
# --------------
|
|
|
|
#
|
|
# Check existence of files
|
|
#
|
|
if ! check_uploads "hpr$start"; then
|
|
echo "Missing files for show $start. Aborted!"
|
|
exit 1
|
|
fi
|
|
|
|
#
|
|
# Define files
|
|
#
|
|
metadata="metadata_${start}.csv"
|
|
script="script_${start}.sh"
|
|
|
|
if [[ $DRYRUN -eq 1 ]]; then
|
|
echo "Dry run: Would have uploaded $start"
|
|
echo "Dry run: Would have created $metadata and $script"
|
|
echo "Dry run: Would have uploaded $metadata and run $script"
|
|
echo "Dry run: Would have used $dbconfig"
|
|
else
|
|
echo "Uploading $start"
|
|
if yes_no "OK to continue? %s " "N"; then
|
|
# shellcheck disable=SC2086
|
|
{
|
|
#
|
|
# Make the metadata
|
|
#
|
|
$BASEDIR/make_metadata -dbconf=${dbconfig} -from=$start -verb -out -script
|
|
RES=$?
|
|
if [[ $RES -eq 0 ]]; then
|
|
ia upload --retries=3 --spreadsheet=${metadata} \
|
|
-H x-archive-keep-old-version:0 && ./${script}
|
|
else
|
|
echo "Upload aborted due to errors"
|
|
fi
|
|
}
|
|
else
|
|
echo "Not uploaded"
|
|
fi
|
|
fi
|
|
else
|
|
#
|
|
# Multiple episodes
|
|
# -----------------
|
|
#
|
|
# Compute end show number
|
|
#
|
|
((end = start + count - 1))
|
|
|
|
#
|
|
# Check existence of files
|
|
#
|
|
for (( i = start; i < end; i++ )); do
|
|
if ! check_uploads "hpr$i"; then
|
|
echo "Missing files for show $i. Aborted!"
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
#
|
|
# Define files
|
|
#
|
|
metadata="metadata_${start}-${end}.csv"
|
|
script="script_${start}-${end}.sh"
|
|
|
|
if [[ $DRYRUN -eq 1 ]]; then
|
|
echo "Dry run: Would have uploaded $start to $end inclusive"
|
|
echo "Dry run: Would have created $metadata and $script"
|
|
echo "Dry run: Would have uploaded $metadata and run $script"
|
|
echo "Dry run: Would have used $dbconfig"
|
|
else
|
|
echo "Uploading $start to $end inclusive"
|
|
if yes_no "OK to continue? %s " "N"; then
|
|
# shellcheck disable=2086
|
|
{
|
|
#
|
|
# Make the metadata
|
|
#
|
|
$BASEDIR/make_metadata -dbconf=${dbconfig} -from=$start -count=$count -verb -out -script
|
|
RES=$?
|
|
if [[ $RES -eq 0 ]]; then
|
|
ia upload --retries=3 --spreadsheet=${metadata} \
|
|
-H x-archive-keep-old-version:0 && ./${script}
|
|
else
|
|
echo "Upload aborted due to errors"
|
|
fi
|
|
}
|
|
else
|
|
echo "Not uploaded"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
exit
|
|
|
|
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
|
|
|