Updates for show "repair" processing
InternetArchive/future_upload: Added logging and debugging
InternetArchive/ia_db.sql: Added new tables
InternetArchive/recover_transcripts: New script to run on 'borg' and
copy missing files from the backup disk to the IA
InternetArchive/repair_assets: More comments, including one about a bug in the design.
InternetArchive/repair_item: Fix relating to octal numbers (if there are
leading zeroes in a number). '_DEBUG' is now in the function
library. Added comments to explain obscure stuff.
InternetArchive/snapshot_metadata: New Bash script (to run on my
desktop) which collects metadata for a show and stores in in the
'~/HPR/IA/assets' directory. Runs 'view_derivatives' on it to find
derivative files for deletion.
InternetArchive/tidy_uploaded: Moves files and directories containing
uploaded files into a holding area for later backup. Added
debugging, logging and a 'force' mode.
InternetArchive/upload_manager: Manages 'ia.db' (on my workstation).
Needs many updates which have just started to be added.
InternetArchive/weekly_upload: Old script, now obsolete.
This commit is contained in:
Dave Morriss
197
InternetArchive/snapshot_metadata
Executable file
197
InternetArchive/snapshot_metadata
Executable file
@@ -0,0 +1,197 @@
|
||||
#!/bin/bash -
|
||||
#===============================================================================
|
||||
#
|
||||
# FILE: snapshot_metadata
|
||||
#
|
||||
# USAGE: ./snapshot_metadata episode_number
|
||||
#
|
||||
# DESCRIPTION: Collects metadata from the IA for a given show and stores it
|
||||
# in the cache.
|
||||
#
|
||||
# OPTIONS: ---
|
||||
# REQUIREMENTS: ---
|
||||
# BUGS: ---
|
||||
# NOTES: ---
|
||||
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
||||
# VERSION: 0.0.2
|
||||
# CREATED: 2024-08-16 20:36:51
|
||||
# REVISION: 2024-08-17 10:31:15
|
||||
#
|
||||
#===============================================================================
|
||||
|
||||
set -o nounset # Treat unset variables as an error
|
||||
|
||||
VERSION="0.0.2"
|
||||
|
||||
SCRIPT=${0##*/}
|
||||
# DIR=${0%/*}
|
||||
|
||||
STDOUT="/dev/fd/2"
|
||||
|
||||
#
|
||||
# Select the appropriate working directory for the host
|
||||
#
|
||||
case $(hostname) in
|
||||
i7-desktop)
|
||||
BASEDIR="$HOME/HPR/InternetArchive"
|
||||
;;
|
||||
borg)
|
||||
BASEDIR="$HOME/IA"
|
||||
;;
|
||||
*)
|
||||
echo "Wrong host!"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
cd "$BASEDIR" || { echo "Failed to cd to $BASEDIR"; exit 1; }
|
||||
|
||||
#
|
||||
# Load library functions
|
||||
#
|
||||
LIB="$HOME/HPR/function_lib.sh"
|
||||
[ -e "$LIB" ] || { echo "Unable to source functions"; exit; }
|
||||
# shellcheck disable=SC1090
|
||||
source "$LIB"
|
||||
|
||||
#
|
||||
# Enable coloured messages
|
||||
#
|
||||
define_colours
|
||||
|
||||
#
|
||||
# Sanity checks
|
||||
#
|
||||
IA=$(command -v ia)
|
||||
[ -n "$IA" ] || { echo "Program 'ia' was not found"; exit 1; }
|
||||
VIEWD="$BASEDIR/view_derivatives"
|
||||
[ -e "$VIEWD" ] || { echo "Program '$VIEWD' was not found"; exit 1; }
|
||||
|
||||
# {{{ -- Functions -- _usage
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: make_dir
|
||||
# DESCRIPTION: Make a directory if it doesn't exist, failing gracefully on
|
||||
# errors.
|
||||
# PARAMETERS: $1 directory path
|
||||
# RETURNS: True if success, otherwise exits the caller script
|
||||
#===============================================================================
|
||||
make_dir () {
|
||||
local dir="${1}"
|
||||
|
||||
if [[ ! -d $dir ]]; then
|
||||
mkdir -p "$dir" || {
|
||||
coloured 'red' "Failed to create $dir"
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: _usage
|
||||
# DESCRIPTION: Reports usage; always exits the script after doing so
|
||||
# PARAMETERS: 1 - the integer to pass to the 'exit' command
|
||||
# RETURNS: Nothing
|
||||
#===============================================================================
|
||||
_usage () {
|
||||
local -i result=${1:-0}
|
||||
|
||||
cat >$STDOUT <<-endusage
|
||||
${SCRIPT} - version: ${VERSION}
|
||||
|
||||
Usage: ./${SCRIPT} showid
|
||||
|
||||
Collects notes for a show and adds them to the cache directory
|
||||
|
||||
Arguments:
|
||||
showid The show id in the form 'hpr1234'
|
||||
|
||||
endusage
|
||||
exit "$result"
|
||||
}
|
||||
|
||||
# }}}
|
||||
|
||||
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Argument check
|
||||
#-------------------------------------------------------------------------------
|
||||
# Should have one argument
|
||||
#
|
||||
if [[ $# != 1 ]]; then
|
||||
coloured 'red' "Missing argument"
|
||||
_usage 1
|
||||
fi
|
||||
show="${1,,}"
|
||||
|
||||
#
|
||||
# Ensure show id is correctly formatted. We want it to be 'hpr1234'
|
||||
#
|
||||
if [[ $show =~ (hpr)?([0-9]+) ]]; then
|
||||
printf -v show 'hpr%04d' "${BASH_REMATCH[2]}"
|
||||
else
|
||||
coloured 'red' "Incorrect show specification: $show"
|
||||
coloured 'yellow' "Use 'hpr9999' or '9999' format"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Setting up paths
|
||||
#-------------------------------------------------------------------------------
|
||||
#
|
||||
# CACHEDIR is where we store asset details and files
|
||||
#
|
||||
CACHEDIR="$BASEDIR/assets"
|
||||
[ ! -d "$CACHEDIR" ] && {
|
||||
coloured 'red' "Creating cache directory"
|
||||
make_dir "$CACHEDIR"
|
||||
}
|
||||
|
||||
#
|
||||
# Pointers into the cache:
|
||||
# LOCAL_ASSETDIR - where the cache for this show lives
|
||||
#
|
||||
LOCAL_ASSETDIR="$CACHEDIR/${show}"
|
||||
[ ! -d "$LOCAL_ASSETDIR" ] && {
|
||||
coloured 'green' "Creating cache directory for $show"
|
||||
make_dir "$LOCAL_ASSETDIR"
|
||||
}
|
||||
|
||||
METADATA="$CACHEDIR/$show/metadata.json"
|
||||
DERIVED="$CACHEDIR/$show/derived.lis"
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Save the IA metadata unless we already have the file
|
||||
#-------------------------------------------------------------------------------
|
||||
if [[ ! -e $METADATA ]]; then
|
||||
if ia metadata "$show" > "$METADATA"; then
|
||||
coloured 'green' "Created metadata file"
|
||||
if [[ ! -s $METADATA ]]; then
|
||||
coloured 'red' "Metadata file is empty"
|
||||
fi
|
||||
else
|
||||
coloured 'red' "Creation of metadata file failed"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
coloured 'yellow' "Metadata already exists, not replacing it"
|
||||
fi
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Use the collected metadata to view the state of the IA, and collect the derived file names
|
||||
#-------------------------------------------------------------------------------
|
||||
coloured 'blue' "Viewing IA files"
|
||||
"$VIEWD" -verb "$METADATA"
|
||||
|
||||
if "$VIEWD" -list "$METADATA" > "$DERIVED"; then
|
||||
nfiles="$(wc -l < "$DERIVED")"
|
||||
coloured 'green' "Saved 'derived' files for show $show ($nfiles)"
|
||||
else
|
||||
coloured 'red' "Creation of $DERIVED file failed"
|
||||
fi
|
||||
|
||||
exit
|
||||
|
||||
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker
|
||||
Reference in New Issue
Block a user