forked from HPR/hpr-tools
Removed obsolete scripts and the SQLite database
This commit is contained in:
Dave Morriss
@@ -1,62 +0,0 @@
|
||||
#!/bin/bash -
|
||||
#===============================================================================
|
||||
#
|
||||
# FILE: check_missing
|
||||
#
|
||||
# USAGE: ./check_missing [start] [end]
|
||||
#
|
||||
# DESCRIPTION: Looks for missing audio files in the upload area and in the
|
||||
# upload journal on the VPS. The upload area is on the VPS at
|
||||
# /var/IA/uploads and the journal is (currently) in
|
||||
# ~dave/IA/ias3uploads.jnl
|
||||
#
|
||||
# OPTIONS: ---
|
||||
# REQUIREMENTS: ---
|
||||
# BUGS: ---
|
||||
# NOTES: ---
|
||||
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
||||
# VERSION: 0.0.1
|
||||
# CREATED: 2015-06-14 13:07:23
|
||||
# REVISION: 2015-06-14 13:08:07
|
||||
#
|
||||
#===============================================================================
|
||||
|
||||
set -o nounset # Treat unset variables as an error
|
||||
|
||||
SCRIPT=${0##*/}
|
||||
DIR=${0%/*}
|
||||
|
||||
#
|
||||
# Where the tools are
|
||||
#
|
||||
BASEDIR="$HOME/IA"
|
||||
|
||||
#
|
||||
# For the moment the files for uploading are in two places
|
||||
#
|
||||
UPLOADS="/var/IA/uploads"
|
||||
|
||||
#
|
||||
# Where the journal is
|
||||
#
|
||||
JNL="$BASEDIR/ias3upload.jnl"
|
||||
|
||||
#
|
||||
# Default episode range
|
||||
#
|
||||
START=${1:-1300}
|
||||
END=${2:-1799}
|
||||
|
||||
#
|
||||
# Go looking for missing stuff
|
||||
#
|
||||
for (( i=${START}; i<=${END}; i++ ))
|
||||
do
|
||||
if [ ! -e "$UPLOADS/hpr$i.wav" ]; then
|
||||
if ! grep -q "hpr$i.wav" $JNL; then
|
||||
echo "Missing hpr$i.wav";
|
||||
fi;
|
||||
fi;
|
||||
done
|
||||
|
||||
exit
|
||||
@@ -1,224 +0,0 @@
|
||||
#!/bin/bash -
|
||||
#===============================================================================
|
||||
#
|
||||
# FILE: delete_uploaded
|
||||
#
|
||||
# USAGE: ./delete_uploaded [-h] [-v] [-d {0|1}]
|
||||
#
|
||||
# DESCRIPTION: Deletes HPR audio and other show-related files on the VPS
|
||||
# after their shows have been uploaded to the Internet Archive
|
||||
#
|
||||
# OPTIONS: ---
|
||||
# REQUIREMENTS: ---
|
||||
# BUGS: ---
|
||||
# NOTES: Was 'delete_uploaded_new' while in development. Now replaces
|
||||
# the original 'delete_uploaded'.
|
||||
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
||||
# VERSION: 0.1.4
|
||||
# CREATED: 2017-08-12 12:32:13
|
||||
# REVISION: 2022-03-30 10:57:23
|
||||
#
|
||||
#===============================================================================
|
||||
|
||||
set -o nounset # Treat unset variables as an error
|
||||
|
||||
VERSION="0.1.4"
|
||||
|
||||
SCRIPT=${0##*/}
|
||||
#DIR=${0%/*}
|
||||
|
||||
STDOUT="/dev/fd/2"
|
||||
|
||||
#
|
||||
# Make temporary files and set traps to delete them
|
||||
#
|
||||
TMP1=$(mktemp) || { echo "$SCRIPT: creation of temporary file failed!"; exit 1; }
|
||||
trap 'cleanup_temp $TMP1' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
|
||||
|
||||
#
|
||||
# Configure depending whether local or on the VPS
|
||||
#
|
||||
case $HOSTNAME in
|
||||
hprvps|marvin|borg) UPLOADS="/var/IA/uploads" ;;
|
||||
i7-desktop) UPLOADS="$HOME/HPR/IA/uploads" ;;
|
||||
*) echo "Wrong host!"; exit 1 ;;
|
||||
esac
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: cleanup_temp
|
||||
# DESCRIPTION: Cleanup temporary files in case of a keyboard interrupt
|
||||
# (SIGINT) or a termination signal (SIGTERM) and at script
|
||||
# exit
|
||||
# PARAMETERS: * - names of temporary files to delete
|
||||
# RETURNS: Nothing
|
||||
#===============================================================================
|
||||
function cleanup_temp {
|
||||
for tmp in "$@"; do
|
||||
[ -e "$tmp" ] && rm --force "$tmp"
|
||||
done
|
||||
exit 0
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: is_empty
|
||||
# DESCRIPTION: Check whether a directory is empty
|
||||
# PARAMETERS: $1 Directory to test
|
||||
# RETURNS: True if empty, otherwise false
|
||||
#===============================================================================
|
||||
is_empty() {
|
||||
test -z "$(find "$1" -mindepth 1 -printf X -quit)"
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: _usage
|
||||
# DESCRIPTION: Report usage
|
||||
# PARAMETERS: 1 [optional] exit value
|
||||
# RETURNS: Nothing
|
||||
#===============================================================================
|
||||
_usage () {
|
||||
local -i res="${1:-0}"
|
||||
|
||||
cat >$STDOUT <<-endusage
|
||||
${SCRIPT} - version: ${VERSION}
|
||||
|
||||
Usage: ./${SCRIPT} [-h] [-v] [-d {0|1}]
|
||||
|
||||
Deletes HPR audio and other show-related files on the VPS after their shows
|
||||
have been uploaded to the Internet Archive.
|
||||
|
||||
Options:
|
||||
-h Print this help
|
||||
-v Run in verbose mode where more information is reported
|
||||
-d 0|1 Dry run: -d 1 (the default) runs the script in dry-run
|
||||
mode where nothing is deleted but the actions that
|
||||
will be taken are reported; -d 0 turns off dry-run
|
||||
mode and the actions will be carried out.
|
||||
|
||||
endusage
|
||||
exit "$res"
|
||||
}
|
||||
|
||||
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
#
|
||||
# Process options
|
||||
#
|
||||
while getopts :d:hv opt
|
||||
do
|
||||
case "${opt}" in
|
||||
d) DRYRUN=$OPTARG;;
|
||||
h) _usage 0;;
|
||||
v) VERBOSE=1;;
|
||||
*) _usage 1;;
|
||||
esac
|
||||
done
|
||||
shift $((OPTIND - 1))
|
||||
|
||||
DRYRUN=${DRYRUN:-1}
|
||||
if [[ $DRYRUN -ne 0 && $DRYRUN -ne 1 ]]; then
|
||||
echo "** Use '-d 0' or '-d 1'"
|
||||
_usage 1
|
||||
fi
|
||||
[[ $DRYRUN -eq 1 ]] && echo "Dry run mode"
|
||||
|
||||
VERBOSE=${VERBOSE:-0}
|
||||
|
||||
#
|
||||
# Should have no arguments
|
||||
#
|
||||
if [[ $# != 0 ]]; then
|
||||
echo "** ${SCRIPT} takes no arguments"
|
||||
_usage 1
|
||||
fi
|
||||
|
||||
#
|
||||
# Declarations
|
||||
#
|
||||
#re="^hpr[0-9]{4}"
|
||||
declare -a dirs
|
||||
lastitem=
|
||||
|
||||
while read -r path; do
|
||||
#
|
||||
# Extract the path relative to $UPLOADS and the IA item name from the
|
||||
# returned path
|
||||
#
|
||||
relpath="${path#"$UPLOADS"/}"
|
||||
item="${relpath:0:7}"
|
||||
|
||||
[ $VERBOSE -eq 1 ] && echo "Found $path"
|
||||
|
||||
#
|
||||
# Record all directories from the 'find'. Note that this means the
|
||||
# directory must begin with "^hpr[0-9]{4}", which may give a problem if
|
||||
# there's a directory that doesn't conform
|
||||
#
|
||||
if [[ -d $path ]]; then
|
||||
dirs+=("$path")
|
||||
fi
|
||||
|
||||
#
|
||||
# Detect that the item prefix has changed. If it has we're processing
|
||||
# a new IA identifier, so work on this one
|
||||
#
|
||||
if [[ $item != "$lastitem" ]]; then
|
||||
lastitem=$item
|
||||
|
||||
[ $VERBOSE -eq 1 ] && echo "Checking IA for $lastitem"
|
||||
|
||||
if ia list "$lastitem" > "$TMP1"; then
|
||||
#
|
||||
# Scan the returned list to see if any files we have are online.
|
||||
# Delete when there's a match.
|
||||
#
|
||||
while read -r file; do
|
||||
if [[ -e "$UPLOADS/$file" ]]; then
|
||||
#
|
||||
# A file on the IA exists in the upload area. Delete the
|
||||
# local one if we're not in dry-run mode, otherwise just
|
||||
# report the deletion we would do.
|
||||
#
|
||||
if [[ $DRYRUN -eq 0 ]]; then
|
||||
rm -f "$UPLOADS/$file"
|
||||
echo "Deleted $UPLOADS/$file"
|
||||
else
|
||||
echo "Would delete $UPLOADS/$file"
|
||||
fi
|
||||
fi
|
||||
done < "$TMP1"
|
||||
else
|
||||
#
|
||||
# End the outer 'while' loop because we hit an item not on the IA.
|
||||
# We rely on the list being sorted for this to be sensible
|
||||
#
|
||||
[ $VERBOSE -eq 1 ] && echo "Item not found on IA: $lastitem"
|
||||
break
|
||||
fi
|
||||
else
|
||||
#
|
||||
# Ignore all but the first file belonging to an IA identifier
|
||||
#
|
||||
[ $VERBOSE -eq 1 ] && echo "Skipped $path"
|
||||
continue
|
||||
fi
|
||||
|
||||
done < <(find "$UPLOADS" -regextype posix-extended -regex '.*hpr[0-9]{4}.*' | sort)
|
||||
|
||||
#
|
||||
# Clean up any empty directories
|
||||
#
|
||||
for dir in "${dirs[@]}"; do
|
||||
if is_empty "$dir"; then
|
||||
if [[ $DRYRUN -eq 0 ]]; then
|
||||
rmdir "$dir"
|
||||
echo "Deleted $dir"
|
||||
else
|
||||
echo "Would delete $dir"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
exit
|
||||
|
||||
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
|
||||
|
||||
Binary file not shown.
@@ -1,304 +0,0 @@
|
||||
#!/bin/bash -
|
||||
#===============================================================================
|
||||
#
|
||||
# FILE: weekly_upload
|
||||
#
|
||||
# USAGE: ./weekly_upload [-h] [-r] [-v] [-d {0|1}] start [count]
|
||||
#
|
||||
# DESCRIPTION: Run the commands necessary to upload a batch of HPR shows to
|
||||
# archive.org
|
||||
#
|
||||
# ** NOW OBSOLETE **
|
||||
# We do uploads differently now.
|
||||
#
|
||||
# OPTIONS: ---
|
||||
# REQUIREMENTS: ---
|
||||
# BUGS: ---
|
||||
# NOTES: ---
|
||||
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
||||
# VERSION: 0.0.9
|
||||
# CREATED: 2018-10-12 21:54:35
|
||||
# REVISION: 2020-10-13 13:54:32
|
||||
#
|
||||
#===============================================================================
|
||||
|
||||
echo "Obsolete script. Do not use!"
|
||||
cmd='exit'; $cmd
|
||||
|
||||
set -o nounset # Treat unset variables as an error
|
||||
|
||||
SCRIPT=${0##*/}
|
||||
#DIR=${0%/*}
|
||||
VERSION="0.0.9"
|
||||
|
||||
STDOUT="/dev/fd/2"
|
||||
|
||||
#
|
||||
# Select the appropriate working directory
|
||||
#
|
||||
case $(hostname) in
|
||||
i7-desktop)
|
||||
BASEDIR="$HOME/HPR/InternetArchive"
|
||||
UPLOAD="$BASEDIR/uploads"
|
||||
;;
|
||||
hprvps|marvin|borg)
|
||||
BASEDIR="$HOME/IA"
|
||||
UPLOAD="/var/IA/uploads"
|
||||
;;
|
||||
*)
|
||||
echo "Wrong host!"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
cd "$BASEDIR" || exit 1
|
||||
|
||||
#
|
||||
# Load library functions
|
||||
#
|
||||
LIB="$HOME/bin/function_lib.sh"
|
||||
[ -e "$LIB" ] || { echo "Unable to source functions"; exit; }
|
||||
# shellcheck disable=SC1090
|
||||
source "$LIB"
|
||||
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: check_uploads
|
||||
# DESCRIPTION: Determines if files exist for uploading
|
||||
# PARAMETERS: 1 - filename prefix e.g. 'hpr9999'
|
||||
# RETURNS: True/false
|
||||
#===============================================================================
|
||||
check_uploads () {
|
||||
local prefix=${1:?Usage: check_uploads prefix}
|
||||
local suff
|
||||
|
||||
#
|
||||
# Look for files called hpr1234.flac and so on. Don't bother with the
|
||||
# hpr1234_source.flac one. As soon as a file is missing return with false.
|
||||
#
|
||||
for suff in flac mp3 ogg opus spx wav; do
|
||||
if [[ ! -e $UPLOAD/$prefix.$suff ]]; then
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: _usage
|
||||
# DESCRIPTION: Reports usage; always exits the script after doing so
|
||||
# PARAMETERS: 1 - the integer to pass to the 'exit' command
|
||||
# RETURNS: Nothing
|
||||
#===============================================================================
|
||||
_usage () {
|
||||
local -i result=${1:-0}
|
||||
|
||||
cat >$STDOUT <<-endusage
|
||||
${SCRIPT} - version: ${VERSION}
|
||||
|
||||
Usage: ./${SCRIPT} [-h] [-r] [-v] [-d {0|1}] start [count]
|
||||
|
||||
Generates the necessary metadata and script and uses them to upload HPR audio
|
||||
and other show-related files held on the VPS to the Internet Archive.
|
||||
|
||||
Options:
|
||||
-h Print this help
|
||||
-v Run in verbose mode where more information is reported
|
||||
-d 0|1 Dry run: -d 1 (the default) runs the script in dry-run
|
||||
mode where nothing is changed but the actions that
|
||||
will be taken are reported; -d 0 turns off dry-run
|
||||
mode and the actions will be carried out.
|
||||
-r Run in 'remote' mode, using the live database over an
|
||||
(already established) SSH tunnel. Default is to run
|
||||
against the local database.
|
||||
Arguments:
|
||||
start the starting show number to be uploaded
|
||||
count (optional, default 1) the number of shows to be
|
||||
uploaded; cannot exceed 20
|
||||
|
||||
Notes:
|
||||
|
||||
1. When running on 'marvin' the method used is to run in faux 'local' mode.
|
||||
This means we have an open tunnel to the HPR server (mostly left open) and
|
||||
the default file .hpr_db.cfg points to the live database via this tunnel.
|
||||
So we do not use the -r option here. This is a bit of a hack! Sorry!
|
||||
|
||||
2. There are potential problems when a show has no tags which haven't been
|
||||
fully resolved. The make_metadata script fails in default mode when it
|
||||
finds such a show, but this (weekly_upload) script can continue on and run
|
||||
the generated script which uploads the source audio files. This can mean
|
||||
the IA items end up as books! In this mode the description is not stored
|
||||
and so there are no show notes.
|
||||
|
||||
endusage
|
||||
exit "$result"
|
||||
}
|
||||
|
||||
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
#
|
||||
# Process options
|
||||
#
|
||||
while getopts :d:hrv opt
|
||||
do
|
||||
case "${opt}" in
|
||||
d) DRYRUN=$OPTARG;;
|
||||
h) _usage 1;;
|
||||
r) REMOTE=1;;
|
||||
v) VERBOSE=1;;
|
||||
*) _usage 1;;
|
||||
esac
|
||||
done
|
||||
shift $((OPTIND - 1))
|
||||
|
||||
DRYRUN=${DRYRUN:-1}
|
||||
if [[ $DRYRUN -ne 0 && $DRYRUN -ne 1 ]]; then
|
||||
echo "** Use '-d 0' or '-d 1'"
|
||||
_usage 1
|
||||
fi
|
||||
|
||||
VERBOSE=${VERBOSE:-0}
|
||||
|
||||
REMOTE=${REMOTE:-0}
|
||||
if [[ $REMOTE -eq 0 ]]; then
|
||||
dbconfig="$BASEDIR/.hpr_db.cfg"
|
||||
[[ $VERBOSE -eq 1 ]] && echo "Local database mode"
|
||||
else
|
||||
dbconfig="$BASEDIR/.hpr_livedb.cfg"
|
||||
[[ $VERBOSE -eq 1 ]] && echo "Remote database mode"
|
||||
fi
|
||||
|
||||
#
|
||||
# Check argument count
|
||||
#
|
||||
if [[ ! ( $# -eq 1 || $# -eq 2 ) ]]; then
|
||||
echo "Wrong number of arguments"
|
||||
_usage 1
|
||||
fi
|
||||
|
||||
#
|
||||
# Validate arguments
|
||||
#
|
||||
for arg; do
|
||||
if [[ ! $arg =~ ^[0-9]{1,4}$ ]]; then
|
||||
echo "Invalid number: $arg"
|
||||
echo "Use a plain number"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
start=$1
|
||||
count=${2:-1}
|
||||
if [[ $count -gt 20 ]]; then
|
||||
echo "Can't process more than 20 shows at a time"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
[[ $VERBOSE -eq 1 && $DRYRUN -eq 1 ]] && echo "Dry run mode"
|
||||
|
||||
#
|
||||
# Two paths; one for a single episode, the other for multiple episodes
|
||||
#
|
||||
if [[ $count -eq 1 ]]; then
|
||||
#
|
||||
# Single episode
|
||||
# --------------
|
||||
|
||||
#
|
||||
# Check existence of files
|
||||
#
|
||||
if ! check_uploads "hpr$start"; then
|
||||
echo "Missing files for show $start. Aborted!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
#
|
||||
# Define files
|
||||
#
|
||||
metadata="metadata_${start}.csv"
|
||||
script="script_${start}.sh"
|
||||
|
||||
if [[ $DRYRUN -eq 1 ]]; then
|
||||
echo "Dry run: Would have uploaded $start"
|
||||
echo "Dry run: Would have created $metadata and $script"
|
||||
echo "Dry run: Would have uploaded $metadata and run $script"
|
||||
echo "Dry run: Would have used $dbconfig"
|
||||
else
|
||||
echo "Uploading $start"
|
||||
if yes_no "OK to continue? %s " "N"; then
|
||||
# shellcheck disable=SC2086
|
||||
{
|
||||
#
|
||||
# Make the metadata
|
||||
#
|
||||
$BASEDIR/make_metadata -dbconf=${dbconfig} -from=$start -verb -out -script
|
||||
RES=$?
|
||||
if [[ $RES -eq 0 ]]; then
|
||||
ia upload --retries=3 --spreadsheet=${metadata} \
|
||||
-H x-archive-keep-old-version:0 && ./${script}
|
||||
else
|
||||
echo "Upload aborted due to errors"
|
||||
fi
|
||||
}
|
||||
else
|
||||
echo "Not uploaded"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
#
|
||||
# Multiple episodes
|
||||
# -----------------
|
||||
#
|
||||
# Compute end show number
|
||||
#
|
||||
((end = start + count - 1))
|
||||
|
||||
#
|
||||
# Check existence of files
|
||||
#
|
||||
for (( i = start; i < end; i++ )); do
|
||||
if ! check_uploads "hpr$i"; then
|
||||
echo "Missing files for show $i. Aborted!"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
#
|
||||
# Define files
|
||||
#
|
||||
metadata="metadata_${start}-${end}.csv"
|
||||
script="script_${start}-${end}.sh"
|
||||
|
||||
if [[ $DRYRUN -eq 1 ]]; then
|
||||
echo "Dry run: Would have uploaded $start to $end inclusive"
|
||||
echo "Dry run: Would have created $metadata and $script"
|
||||
echo "Dry run: Would have uploaded $metadata and run $script"
|
||||
echo "Dry run: Would have used $dbconfig"
|
||||
else
|
||||
echo "Uploading $start to $end inclusive"
|
||||
if yes_no "OK to continue? %s " "N"; then
|
||||
# shellcheck disable=2086
|
||||
{
|
||||
#
|
||||
# Make the metadata
|
||||
#
|
||||
$BASEDIR/make_metadata -dbconf=${dbconfig} -from=$start -count=$count -verb -out -script
|
||||
RES=$?
|
||||
if [[ $RES -eq 0 ]]; then
|
||||
ia upload --retries=3 --spreadsheet=${metadata} \
|
||||
-H x-archive-keep-old-version:0 && ./${script}
|
||||
else
|
||||
echo "Upload aborted due to errors"
|
||||
fi
|
||||
}
|
||||
else
|
||||
echo "Not uploaded"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
exit
|
||||
|
||||
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
|
||||
|
||||
Reference in New Issue
Block a user