diff --git a/InternetArchive/check_missing b/InternetArchive/check_missing deleted file mode 100755 index 1ded7b5..0000000 --- a/InternetArchive/check_missing +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -#=============================================================================== -# -# FILE: check_missing -# -# USAGE: ./check_missing [start] [end] -# -# DESCRIPTION: Looks for missing audio files in the upload area and in the -# upload journal on the VPS. The upload area is on the VPS at -# /var/IA/uploads and the journal is (currently) in -# ~dave/IA/ias3uploads.jnl -# -# OPTIONS: --- -# REQUIREMENTS: --- -# BUGS: --- -# NOTES: --- -# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com -# VERSION: 0.0.1 -# CREATED: 2015-06-14 13:07:23 -# REVISION: 2015-06-14 13:08:07 -# -#=============================================================================== - -set -o nounset # Treat unset variables as an error - -SCRIPT=${0##*/} -DIR=${0%/*} - -# -# Where the tools are -# -BASEDIR="$HOME/IA" - -# -# For the moment the files for uploading are in two places -# -UPLOADS="/var/IA/uploads" - -# -# Where the journal is -# -JNL="$BASEDIR/ias3upload.jnl" - -# -# Default episode range -# -START=${1:-1300} -END=${2:-1799} - -# -# Go looking for missing stuff -# -for (( i=${START}; i<=${END}; i++ )) -do - if [ ! -e "$UPLOADS/hpr$i.wav" ]; then - if ! grep -q "hpr$i.wav" $JNL; then - echo "Missing hpr$i.wav"; - fi; - fi; -done - -exit diff --git a/InternetArchive/delete_uploaded b/InternetArchive/delete_uploaded deleted file mode 100755 index abba341..0000000 --- a/InternetArchive/delete_uploaded +++ /dev/null @@ -1,224 +0,0 @@ -#!/bin/bash - -#=============================================================================== -# -# FILE: delete_uploaded -# -# USAGE: ./delete_uploaded [-h] [-v] [-d {0|1}] -# -# DESCRIPTION: Deletes HPR audio and other show-related files on the VPS -# after their shows have been uploaded to the Internet Archive -# -# OPTIONS: --- -# REQUIREMENTS: --- -# BUGS: --- -# NOTES: Was 'delete_uploaded_new' while in development. Now replaces -# the original 'delete_uploaded'. -# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com -# VERSION: 0.1.4 -# CREATED: 2017-08-12 12:32:13 -# REVISION: 2022-03-30 10:57:23 -# -#=============================================================================== - -set -o nounset # Treat unset variables as an error - -VERSION="0.1.4" - -SCRIPT=${0##*/} -#DIR=${0%/*} - -STDOUT="/dev/fd/2" - -# -# Make temporary files and set traps to delete them -# -TMP1=$(mktemp) || { echo "$SCRIPT: creation of temporary file failed!"; exit 1; } -trap 'cleanup_temp $TMP1' SIGHUP SIGINT SIGPIPE SIGTERM EXIT - -# -# Configure depending whether local or on the VPS -# -case $HOSTNAME in - hprvps|marvin|borg) UPLOADS="/var/IA/uploads" ;; - i7-desktop) UPLOADS="$HOME/HPR/IA/uploads" ;; - *) echo "Wrong host!"; exit 1 ;; -esac - -#=== FUNCTION ================================================================ -# NAME: cleanup_temp -# DESCRIPTION: Cleanup temporary files in case of a keyboard interrupt -# (SIGINT) or a termination signal (SIGTERM) and at script -# exit -# PARAMETERS: * - names of temporary files to delete -# RETURNS: Nothing -#=============================================================================== -function cleanup_temp { - for tmp in "$@"; do - [ -e "$tmp" ] && rm --force "$tmp" - done - exit 0 -} - -#=== FUNCTION ================================================================ -# NAME: is_empty -# DESCRIPTION: Check whether a directory is empty -# PARAMETERS: $1 Directory to test -# RETURNS: True if empty, otherwise false -#=============================================================================== -is_empty() { - test -z "$(find "$1" -mindepth 1 -printf X -quit)" -} - -#=== FUNCTION ================================================================ -# NAME: _usage -# DESCRIPTION: Report usage -# PARAMETERS: 1 [optional] exit value -# RETURNS: Nothing -#=============================================================================== -_usage () { - local -i res="${1:-0}" - - cat >$STDOUT <<-endusage -${SCRIPT} - version: ${VERSION} - -Usage: ./${SCRIPT} [-h] [-v] [-d {0|1}] - -Deletes HPR audio and other show-related files on the VPS after their shows -have been uploaded to the Internet Archive. - -Options: - -h Print this help - -v Run in verbose mode where more information is reported - -d 0|1 Dry run: -d 1 (the default) runs the script in dry-run - mode where nothing is deleted but the actions that - will be taken are reported; -d 0 turns off dry-run - mode and the actions will be carried out. - -endusage - exit "$res" -} - -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -# -# Process options -# -while getopts :d:hv opt -do - case "${opt}" in - d) DRYRUN=$OPTARG;; - h) _usage 0;; - v) VERBOSE=1;; - *) _usage 1;; - esac -done -shift $((OPTIND - 1)) - -DRYRUN=${DRYRUN:-1} -if [[ $DRYRUN -ne 0 && $DRYRUN -ne 1 ]]; then - echo "** Use '-d 0' or '-d 1'" - _usage 1 -fi -[[ $DRYRUN -eq 1 ]] && echo "Dry run mode" - -VERBOSE=${VERBOSE:-0} - -# -# Should have no arguments -# -if [[ $# != 0 ]]; then - echo "** ${SCRIPT} takes no arguments" - _usage 1 -fi - -# -# Declarations -# -#re="^hpr[0-9]{4}" -declare -a dirs -lastitem= - -while read -r path; do - # - # Extract the path relative to $UPLOADS and the IA item name from the - # returned path - # - relpath="${path#"$UPLOADS"/}" - item="${relpath:0:7}" - - [ $VERBOSE -eq 1 ] && echo "Found $path" - - # - # Record all directories from the 'find'. Note that this means the - # directory must begin with "^hpr[0-9]{4}", which may give a problem if - # there's a directory that doesn't conform - # - if [[ -d $path ]]; then - dirs+=("$path") - fi - - # - # Detect that the item prefix has changed. If it has we're processing - # a new IA identifier, so work on this one - # - if [[ $item != "$lastitem" ]]; then - lastitem=$item - - [ $VERBOSE -eq 1 ] && echo "Checking IA for $lastitem" - - if ia list "$lastitem" > "$TMP1"; then - # - # Scan the returned list to see if any files we have are online. - # Delete when there's a match. - # - while read -r file; do - if [[ -e "$UPLOADS/$file" ]]; then - # - # A file on the IA exists in the upload area. Delete the - # local one if we're not in dry-run mode, otherwise just - # report the deletion we would do. - # - if [[ $DRYRUN -eq 0 ]]; then - rm -f "$UPLOADS/$file" - echo "Deleted $UPLOADS/$file" - else - echo "Would delete $UPLOADS/$file" - fi - fi - done < "$TMP1" - else - # - # End the outer 'while' loop because we hit an item not on the IA. - # We rely on the list being sorted for this to be sensible - # - [ $VERBOSE -eq 1 ] && echo "Item not found on IA: $lastitem" - break - fi - else - # - # Ignore all but the first file belonging to an IA identifier - # - [ $VERBOSE -eq 1 ] && echo "Skipped $path" - continue - fi - -done < <(find "$UPLOADS" -regextype posix-extended -regex '.*hpr[0-9]{4}.*' | sort) - -# -# Clean up any empty directories -# -for dir in "${dirs[@]}"; do - if is_empty "$dir"; then - if [[ $DRYRUN -eq 0 ]]; then - rmdir "$dir" - echo "Deleted $dir" - else - echo "Would delete $dir" - fi - fi -done - -exit - -# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21 - diff --git a/InternetArchive/ia.db b/InternetArchive/ia.db deleted file mode 100644 index e50f2e0..0000000 Binary files a/InternetArchive/ia.db and /dev/null differ diff --git a/InternetArchive/weekly_upload b/InternetArchive/weekly_upload deleted file mode 100755 index 085126b..0000000 --- a/InternetArchive/weekly_upload +++ /dev/null @@ -1,304 +0,0 @@ -#!/bin/bash - -#=============================================================================== -# -# FILE: weekly_upload -# -# USAGE: ./weekly_upload [-h] [-r] [-v] [-d {0|1}] start [count] -# -# DESCRIPTION: Run the commands necessary to upload a batch of HPR shows to -# archive.org -# -# ** NOW OBSOLETE ** -# We do uploads differently now. -# -# OPTIONS: --- -# REQUIREMENTS: --- -# BUGS: --- -# NOTES: --- -# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com -# VERSION: 0.0.9 -# CREATED: 2018-10-12 21:54:35 -# REVISION: 2020-10-13 13:54:32 -# -#=============================================================================== - -echo "Obsolete script. Do not use!" -cmd='exit'; $cmd - -set -o nounset # Treat unset variables as an error - -SCRIPT=${0##*/} -#DIR=${0%/*} -VERSION="0.0.9" - -STDOUT="/dev/fd/2" - -# -# Select the appropriate working directory -# -case $(hostname) in - i7-desktop) - BASEDIR="$HOME/HPR/InternetArchive" - UPLOAD="$BASEDIR/uploads" - ;; - hprvps|marvin|borg) - BASEDIR="$HOME/IA" - UPLOAD="/var/IA/uploads" - ;; - *) - echo "Wrong host!" - exit 1 - ;; -esac - -cd "$BASEDIR" || exit 1 - -# -# Load library functions -# -LIB="$HOME/bin/function_lib.sh" -[ -e "$LIB" ] || { echo "Unable to source functions"; exit; } -# shellcheck disable=SC1090 -source "$LIB" - - -#=== FUNCTION ================================================================ -# NAME: check_uploads -# DESCRIPTION: Determines if files exist for uploading -# PARAMETERS: 1 - filename prefix e.g. 'hpr9999' -# RETURNS: True/false -#=============================================================================== -check_uploads () { - local prefix=${1:?Usage: check_uploads prefix} - local suff - - # - # Look for files called hpr1234.flac and so on. Don't bother with the - # hpr1234_source.flac one. As soon as a file is missing return with false. - # - for suff in flac mp3 ogg opus spx wav; do - if [[ ! -e $UPLOAD/$prefix.$suff ]]; then - return 1 - fi - done - - return 0 -} - -#=== FUNCTION ================================================================ -# NAME: _usage -# DESCRIPTION: Reports usage; always exits the script after doing so -# PARAMETERS: 1 - the integer to pass to the 'exit' command -# RETURNS: Nothing -#=============================================================================== -_usage () { - local -i result=${1:-0} - - cat >$STDOUT <<-endusage -${SCRIPT} - version: ${VERSION} - -Usage: ./${SCRIPT} [-h] [-r] [-v] [-d {0|1}] start [count] - -Generates the necessary metadata and script and uses them to upload HPR audio -and other show-related files held on the VPS to the Internet Archive. - -Options: - -h Print this help - -v Run in verbose mode where more information is reported - -d 0|1 Dry run: -d 1 (the default) runs the script in dry-run - mode where nothing is changed but the actions that - will be taken are reported; -d 0 turns off dry-run - mode and the actions will be carried out. - -r Run in 'remote' mode, using the live database over an - (already established) SSH tunnel. Default is to run - against the local database. -Arguments: - start the starting show number to be uploaded - count (optional, default 1) the number of shows to be - uploaded; cannot exceed 20 - -Notes: - -1. When running on 'marvin' the method used is to run in faux 'local' mode. - This means we have an open tunnel to the HPR server (mostly left open) and - the default file .hpr_db.cfg points to the live database via this tunnel. - So we do not use the -r option here. This is a bit of a hack! Sorry! - -2. There are potential problems when a show has no tags which haven't been - fully resolved. The make_metadata script fails in default mode when it - finds such a show, but this (weekly_upload) script can continue on and run - the generated script which uploads the source audio files. This can mean - the IA items end up as books! In this mode the description is not stored - and so there are no show notes. - -endusage - exit "$result" -} - -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -# -# Process options -# -while getopts :d:hrv opt -do - case "${opt}" in - d) DRYRUN=$OPTARG;; - h) _usage 1;; - r) REMOTE=1;; - v) VERBOSE=1;; - *) _usage 1;; - esac -done -shift $((OPTIND - 1)) - -DRYRUN=${DRYRUN:-1} -if [[ $DRYRUN -ne 0 && $DRYRUN -ne 1 ]]; then - echo "** Use '-d 0' or '-d 1'" - _usage 1 -fi - -VERBOSE=${VERBOSE:-0} - -REMOTE=${REMOTE:-0} -if [[ $REMOTE -eq 0 ]]; then - dbconfig="$BASEDIR/.hpr_db.cfg" - [[ $VERBOSE -eq 1 ]] && echo "Local database mode" -else - dbconfig="$BASEDIR/.hpr_livedb.cfg" - [[ $VERBOSE -eq 1 ]] && echo "Remote database mode" -fi - -# -# Check argument count -# -if [[ ! ( $# -eq 1 || $# -eq 2 ) ]]; then - echo "Wrong number of arguments" - _usage 1 -fi - -# -# Validate arguments -# -for arg; do - if [[ ! $arg =~ ^[0-9]{1,4}$ ]]; then - echo "Invalid number: $arg" - echo "Use a plain number" - exit 1 - fi -done - -start=$1 -count=${2:-1} -if [[ $count -gt 20 ]]; then - echo "Can't process more than 20 shows at a time" - exit 1 -fi - -[[ $VERBOSE -eq 1 && $DRYRUN -eq 1 ]] && echo "Dry run mode" - -# -# Two paths; one for a single episode, the other for multiple episodes -# -if [[ $count -eq 1 ]]; then - # - # Single episode - # -------------- - - # - # Check existence of files - # - if ! check_uploads "hpr$start"; then - echo "Missing files for show $start. Aborted!" - exit 1 - fi - - # - # Define files - # - metadata="metadata_${start}.csv" - script="script_${start}.sh" - - if [[ $DRYRUN -eq 1 ]]; then - echo "Dry run: Would have uploaded $start" - echo "Dry run: Would have created $metadata and $script" - echo "Dry run: Would have uploaded $metadata and run $script" - echo "Dry run: Would have used $dbconfig" - else - echo "Uploading $start" - if yes_no "OK to continue? %s " "N"; then - # shellcheck disable=SC2086 - { - # - # Make the metadata - # - $BASEDIR/make_metadata -dbconf=${dbconfig} -from=$start -verb -out -script - RES=$? - if [[ $RES -eq 0 ]]; then - ia upload --retries=3 --spreadsheet=${metadata} \ - -H x-archive-keep-old-version:0 && ./${script} - else - echo "Upload aborted due to errors" - fi - } - else - echo "Not uploaded" - fi - fi -else - # - # Multiple episodes - # ----------------- - # - # Compute end show number - # - ((end = start + count - 1)) - - # - # Check existence of files - # - for (( i = start; i < end; i++ )); do - if ! check_uploads "hpr$i"; then - echo "Missing files for show $i. Aborted!" - exit 1 - fi - done - - # - # Define files - # - metadata="metadata_${start}-${end}.csv" - script="script_${start}-${end}.sh" - - if [[ $DRYRUN -eq 1 ]]; then - echo "Dry run: Would have uploaded $start to $end inclusive" - echo "Dry run: Would have created $metadata and $script" - echo "Dry run: Would have uploaded $metadata and run $script" - echo "Dry run: Would have used $dbconfig" - else - echo "Uploading $start to $end inclusive" - if yes_no "OK to continue? %s " "N"; then - # shellcheck disable=2086 - { - # - # Make the metadata - # - $BASEDIR/make_metadata -dbconf=${dbconfig} -from=$start -count=$count -verb -out -script - RES=$? - if [[ $RES -eq 0 ]]; then - ia upload --retries=3 --spreadsheet=${metadata} \ - -H x-archive-keep-old-version:0 && ./${script} - else - echo "Upload aborted due to errors" - fi - } - else - echo "Not uploaded" - fi - fi -fi - -exit - -# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21 -