Removed obsolete scripts and the SQLite database

This commit is contained in:
Dave Morriss 2024-08-22 19:39:46 +01:00
parent 0ccbb6419a
commit d3c4f3907f
4 changed files with 0 additions and 590 deletions

View File

@ -1,62 +0,0 @@
#!/bin/bash -
#===============================================================================
#
# FILE: check_missing
#
# USAGE: ./check_missing [start] [end]
#
# DESCRIPTION: Looks for missing audio files in the upload area and in the
# upload journal on the VPS. The upload area is on the VPS at
# /var/IA/uploads and the journal is (currently) in
# ~dave/IA/ias3uploads.jnl
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.1
# CREATED: 2015-06-14 13:07:23
# REVISION: 2015-06-14 13:08:07
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
DIR=${0%/*}
#
# Where the tools are
#
BASEDIR="$HOME/IA"
#
# For the moment the files for uploading are in two places
#
UPLOADS="/var/IA/uploads"
#
# Where the journal is
#
JNL="$BASEDIR/ias3upload.jnl"
#
# Default episode range
#
START=${1:-1300}
END=${2:-1799}
#
# Go looking for missing stuff
#
for (( i=${START}; i<=${END}; i++ ))
do
if [ ! -e "$UPLOADS/hpr$i.wav" ]; then
if ! grep -q "hpr$i.wav" $JNL; then
echo "Missing hpr$i.wav";
fi;
fi;
done
exit

View File

@ -1,224 +0,0 @@
#!/bin/bash -
#===============================================================================
#
# FILE: delete_uploaded
#
# USAGE: ./delete_uploaded [-h] [-v] [-d {0|1}]
#
# DESCRIPTION: Deletes HPR audio and other show-related files on the VPS
# after their shows have been uploaded to the Internet Archive
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: Was 'delete_uploaded_new' while in development. Now replaces
# the original 'delete_uploaded'.
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.1.4
# CREATED: 2017-08-12 12:32:13
# REVISION: 2022-03-30 10:57:23
#
#===============================================================================
set -o nounset # Treat unset variables as an error
VERSION="0.1.4"
SCRIPT=${0##*/}
#DIR=${0%/*}
STDOUT="/dev/fd/2"
#
# Make temporary files and set traps to delete them
#
TMP1=$(mktemp) || { echo "$SCRIPT: creation of temporary file failed!"; exit 1; }
trap 'cleanup_temp $TMP1' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
#
# Configure depending whether local or on the VPS
#
case $HOSTNAME in
hprvps|marvin|borg) UPLOADS="/var/IA/uploads" ;;
i7-desktop) UPLOADS="$HOME/HPR/IA/uploads" ;;
*) echo "Wrong host!"; exit 1 ;;
esac
#=== FUNCTION ================================================================
# NAME: cleanup_temp
# DESCRIPTION: Cleanup temporary files in case of a keyboard interrupt
# (SIGINT) or a termination signal (SIGTERM) and at script
# exit
# PARAMETERS: * - names of temporary files to delete
# RETURNS: Nothing
#===============================================================================
function cleanup_temp {
for tmp in "$@"; do
[ -e "$tmp" ] && rm --force "$tmp"
done
exit 0
}
#=== FUNCTION ================================================================
# NAME: is_empty
# DESCRIPTION: Check whether a directory is empty
# PARAMETERS: $1 Directory to test
# RETURNS: True if empty, otherwise false
#===============================================================================
is_empty() {
test -z "$(find "$1" -mindepth 1 -printf X -quit)"
}
#=== FUNCTION ================================================================
# NAME: _usage
# DESCRIPTION: Report usage
# PARAMETERS: 1 [optional] exit value
# RETURNS: Nothing
#===============================================================================
_usage () {
local -i res="${1:-0}"
cat >$STDOUT <<-endusage
${SCRIPT} - version: ${VERSION}
Usage: ./${SCRIPT} [-h] [-v] [-d {0|1}]
Deletes HPR audio and other show-related files on the VPS after their shows
have been uploaded to the Internet Archive.
Options:
-h Print this help
-v Run in verbose mode where more information is reported
-d 0|1 Dry run: -d 1 (the default) runs the script in dry-run
mode where nothing is deleted but the actions that
will be taken are reported; -d 0 turns off dry-run
mode and the actions will be carried out.
endusage
exit "$res"
}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Process options
#
while getopts :d:hv opt
do
case "${opt}" in
d) DRYRUN=$OPTARG;;
h) _usage 0;;
v) VERBOSE=1;;
*) _usage 1;;
esac
done
shift $((OPTIND - 1))
DRYRUN=${DRYRUN:-1}
if [[ $DRYRUN -ne 0 && $DRYRUN -ne 1 ]]; then
echo "** Use '-d 0' or '-d 1'"
_usage 1
fi
[[ $DRYRUN -eq 1 ]] && echo "Dry run mode"
VERBOSE=${VERBOSE:-0}
#
# Should have no arguments
#
if [[ $# != 0 ]]; then
echo "** ${SCRIPT} takes no arguments"
_usage 1
fi
#
# Declarations
#
#re="^hpr[0-9]{4}"
declare -a dirs
lastitem=
while read -r path; do
#
# Extract the path relative to $UPLOADS and the IA item name from the
# returned path
#
relpath="${path#"$UPLOADS"/}"
item="${relpath:0:7}"
[ $VERBOSE -eq 1 ] && echo "Found $path"
#
# Record all directories from the 'find'. Note that this means the
# directory must begin with "^hpr[0-9]{4}", which may give a problem if
# there's a directory that doesn't conform
#
if [[ -d $path ]]; then
dirs+=("$path")
fi
#
# Detect that the item prefix has changed. If it has we're processing
# a new IA identifier, so work on this one
#
if [[ $item != "$lastitem" ]]; then
lastitem=$item
[ $VERBOSE -eq 1 ] && echo "Checking IA for $lastitem"
if ia list "$lastitem" > "$TMP1"; then
#
# Scan the returned list to see if any files we have are online.
# Delete when there's a match.
#
while read -r file; do
if [[ -e "$UPLOADS/$file" ]]; then
#
# A file on the IA exists in the upload area. Delete the
# local one if we're not in dry-run mode, otherwise just
# report the deletion we would do.
#
if [[ $DRYRUN -eq 0 ]]; then
rm -f "$UPLOADS/$file"
echo "Deleted $UPLOADS/$file"
else
echo "Would delete $UPLOADS/$file"
fi
fi
done < "$TMP1"
else
#
# End the outer 'while' loop because we hit an item not on the IA.
# We rely on the list being sorted for this to be sensible
#
[ $VERBOSE -eq 1 ] && echo "Item not found on IA: $lastitem"
break
fi
else
#
# Ignore all but the first file belonging to an IA identifier
#
[ $VERBOSE -eq 1 ] && echo "Skipped $path"
continue
fi
done < <(find "$UPLOADS" -regextype posix-extended -regex '.*hpr[0-9]{4}.*' | sort)
#
# Clean up any empty directories
#
for dir in "${dirs[@]}"; do
if is_empty "$dir"; then
if [[ $DRYRUN -eq 0 ]]; then
rmdir "$dir"
echo "Deleted $dir"
else
echo "Would delete $dir"
fi
fi
done
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21

Binary file not shown.

View File

@ -1,304 +0,0 @@
#!/bin/bash -
#===============================================================================
#
# FILE: weekly_upload
#
# USAGE: ./weekly_upload [-h] [-r] [-v] [-d {0|1}] start [count]
#
# DESCRIPTION: Run the commands necessary to upload a batch of HPR shows to
# archive.org
#
# ** NOW OBSOLETE **
# We do uploads differently now.
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.9
# CREATED: 2018-10-12 21:54:35
# REVISION: 2020-10-13 13:54:32
#
#===============================================================================
echo "Obsolete script. Do not use!"
cmd='exit'; $cmd
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
#DIR=${0%/*}
VERSION="0.0.9"
STDOUT="/dev/fd/2"
#
# Select the appropriate working directory
#
case $(hostname) in
i7-desktop)
BASEDIR="$HOME/HPR/InternetArchive"
UPLOAD="$BASEDIR/uploads"
;;
hprvps|marvin|borg)
BASEDIR="$HOME/IA"
UPLOAD="/var/IA/uploads"
;;
*)
echo "Wrong host!"
exit 1
;;
esac
cd "$BASEDIR" || exit 1
#
# Load library functions
#
LIB="$HOME/bin/function_lib.sh"
[ -e "$LIB" ] || { echo "Unable to source functions"; exit; }
# shellcheck disable=SC1090
source "$LIB"
#=== FUNCTION ================================================================
# NAME: check_uploads
# DESCRIPTION: Determines if files exist for uploading
# PARAMETERS: 1 - filename prefix e.g. 'hpr9999'
# RETURNS: True/false
#===============================================================================
check_uploads () {
local prefix=${1:?Usage: check_uploads prefix}
local suff
#
# Look for files called hpr1234.flac and so on. Don't bother with the
# hpr1234_source.flac one. As soon as a file is missing return with false.
#
for suff in flac mp3 ogg opus spx wav; do
if [[ ! -e $UPLOAD/$prefix.$suff ]]; then
return 1
fi
done
return 0
}
#=== FUNCTION ================================================================
# NAME: _usage
# DESCRIPTION: Reports usage; always exits the script after doing so
# PARAMETERS: 1 - the integer to pass to the 'exit' command
# RETURNS: Nothing
#===============================================================================
_usage () {
local -i result=${1:-0}
cat >$STDOUT <<-endusage
${SCRIPT} - version: ${VERSION}
Usage: ./${SCRIPT} [-h] [-r] [-v] [-d {0|1}] start [count]
Generates the necessary metadata and script and uses them to upload HPR audio
and other show-related files held on the VPS to the Internet Archive.
Options:
-h Print this help
-v Run in verbose mode where more information is reported
-d 0|1 Dry run: -d 1 (the default) runs the script in dry-run
mode where nothing is changed but the actions that
will be taken are reported; -d 0 turns off dry-run
mode and the actions will be carried out.
-r Run in 'remote' mode, using the live database over an
(already established) SSH tunnel. Default is to run
against the local database.
Arguments:
start the starting show number to be uploaded
count (optional, default 1) the number of shows to be
uploaded; cannot exceed 20
Notes:
1. When running on 'marvin' the method used is to run in faux 'local' mode.
This means we have an open tunnel to the HPR server (mostly left open) and
the default file .hpr_db.cfg points to the live database via this tunnel.
So we do not use the -r option here. This is a bit of a hack! Sorry!
2. There are potential problems when a show has no tags which haven't been
fully resolved. The make_metadata script fails in default mode when it
finds such a show, but this (weekly_upload) script can continue on and run
the generated script which uploads the source audio files. This can mean
the IA items end up as books! In this mode the description is not stored
and so there are no show notes.
endusage
exit "$result"
}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Process options
#
while getopts :d:hrv opt
do
case "${opt}" in
d) DRYRUN=$OPTARG;;
h) _usage 1;;
r) REMOTE=1;;
v) VERBOSE=1;;
*) _usage 1;;
esac
done
shift $((OPTIND - 1))
DRYRUN=${DRYRUN:-1}
if [[ $DRYRUN -ne 0 && $DRYRUN -ne 1 ]]; then
echo "** Use '-d 0' or '-d 1'"
_usage 1
fi
VERBOSE=${VERBOSE:-0}
REMOTE=${REMOTE:-0}
if [[ $REMOTE -eq 0 ]]; then
dbconfig="$BASEDIR/.hpr_db.cfg"
[[ $VERBOSE -eq 1 ]] && echo "Local database mode"
else
dbconfig="$BASEDIR/.hpr_livedb.cfg"
[[ $VERBOSE -eq 1 ]] && echo "Remote database mode"
fi
#
# Check argument count
#
if [[ ! ( $# -eq 1 || $# -eq 2 ) ]]; then
echo "Wrong number of arguments"
_usage 1
fi
#
# Validate arguments
#
for arg; do
if [[ ! $arg =~ ^[0-9]{1,4}$ ]]; then
echo "Invalid number: $arg"
echo "Use a plain number"
exit 1
fi
done
start=$1
count=${2:-1}
if [[ $count -gt 20 ]]; then
echo "Can't process more than 20 shows at a time"
exit 1
fi
[[ $VERBOSE -eq 1 && $DRYRUN -eq 1 ]] && echo "Dry run mode"
#
# Two paths; one for a single episode, the other for multiple episodes
#
if [[ $count -eq 1 ]]; then
#
# Single episode
# --------------
#
# Check existence of files
#
if ! check_uploads "hpr$start"; then
echo "Missing files for show $start. Aborted!"
exit 1
fi
#
# Define files
#
metadata="metadata_${start}.csv"
script="script_${start}.sh"
if [[ $DRYRUN -eq 1 ]]; then
echo "Dry run: Would have uploaded $start"
echo "Dry run: Would have created $metadata and $script"
echo "Dry run: Would have uploaded $metadata and run $script"
echo "Dry run: Would have used $dbconfig"
else
echo "Uploading $start"
if yes_no "OK to continue? %s " "N"; then
# shellcheck disable=SC2086
{
#
# Make the metadata
#
$BASEDIR/make_metadata -dbconf=${dbconfig} -from=$start -verb -out -script
RES=$?
if [[ $RES -eq 0 ]]; then
ia upload --retries=3 --spreadsheet=${metadata} \
-H x-archive-keep-old-version:0 && ./${script}
else
echo "Upload aborted due to errors"
fi
}
else
echo "Not uploaded"
fi
fi
else
#
# Multiple episodes
# -----------------
#
# Compute end show number
#
((end = start + count - 1))
#
# Check existence of files
#
for (( i = start; i < end; i++ )); do
if ! check_uploads "hpr$i"; then
echo "Missing files for show $i. Aborted!"
exit 1
fi
done
#
# Define files
#
metadata="metadata_${start}-${end}.csv"
script="script_${start}-${end}.sh"
if [[ $DRYRUN -eq 1 ]]; then
echo "Dry run: Would have uploaded $start to $end inclusive"
echo "Dry run: Would have created $metadata and $script"
echo "Dry run: Would have uploaded $metadata and run $script"
echo "Dry run: Would have used $dbconfig"
else
echo "Uploading $start to $end inclusive"
if yes_no "OK to continue? %s " "N"; then
# shellcheck disable=2086
{
#
# Make the metadata
#
$BASEDIR/make_metadata -dbconf=${dbconfig} -from=$start -count=$count -verb -out -script
RES=$?
if [[ $RES -eq 0 ]]; then
ia upload --retries=3 --spreadsheet=${metadata} \
-H x-archive-keep-old-version:0 && ./${script}
else
echo "Upload aborted due to errors"
fi
}
else
echo "Not uploaded"
fi
fi
fi
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21