Updates for show "repair" processing
InternetArchive/future_upload: Added logging and debugging
InternetArchive/ia_db.sql: Added new tables
InternetArchive/recover_transcripts: New script to run on 'borg' and
copy missing files from the backup disk to the IA
InternetArchive/repair_assets: More comments, including one about a bug in the design.
InternetArchive/repair_item: Fix relating to octal numbers (if there are
leading zeroes in a number). '_DEBUG' is now in the function
library. Added comments to explain obscure stuff.
InternetArchive/snapshot_metadata: New Bash script (to run on my
desktop) which collects metadata for a show and stores in in the
'~/HPR/IA/assets' directory. Runs 'view_derivatives' on it to find
derivative files for deletion.
InternetArchive/tidy_uploaded: Moves files and directories containing
uploaded files into a holding area for later backup. Added
debugging, logging and a 'force' mode.
InternetArchive/upload_manager: Manages 'ia.db' (on my workstation).
Needs many updates which have just started to be added.
InternetArchive/weekly_upload: Old script, now obsolete.
This commit is contained in:
Dave Morriss
@@ -13,15 +13,15 @@
|
||||
# BUGS: ---
|
||||
# NOTES: ---
|
||||
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
||||
# VERSION: 0.0.10
|
||||
# VERSION: 0.0.11
|
||||
# CREATED: 2022-03-30 17:38:01
|
||||
# REVISION: 2022-07-30 14:30:43
|
||||
# REVISION: 2024-07-29 18:24:26
|
||||
#
|
||||
#===============================================================================
|
||||
|
||||
set -o nounset # Treat unset variables as an error
|
||||
|
||||
VERSION="0.0.10"
|
||||
VERSION="0.0.11"
|
||||
|
||||
SCRIPT=${0##*/}
|
||||
# DIR=${0%/*}
|
||||
@@ -55,6 +55,8 @@ case $HOSTNAME in
|
||||
*) echo "Wrong host!"; exit 1 ;;
|
||||
esac
|
||||
|
||||
# {{{ -- Functions -- exists_in, queued_tasks, movefile, is_empty, _log, _usage
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: exists_in
|
||||
# DESCRIPTION: Checks the existence of a key in an associative array
|
||||
@@ -99,9 +101,12 @@ queued_tasks () {
|
||||
# RETURNS: True if a move was done, otherwise False
|
||||
#===============================================================================
|
||||
movefile () {
|
||||
local fromdir="${1:?Usage: movefile fromdir todir path}"
|
||||
local todir="${2:?Usage: movefile fromdir todir path}"
|
||||
local path="${3:?Usage: movefile fromdir todir path}"
|
||||
local fromdir="${1:?Usage: movefile fromdir todir path [FORCE]}"
|
||||
local todir="${2:?Usage: movefile fromdir todir path [FORCE]}"
|
||||
local path="${3:?Usage: movefile fromdir todir path [FORCE]}"
|
||||
local FORCE="${4:-0}"
|
||||
|
||||
[[ ! -v FORCE ]] && FORCE=0
|
||||
|
||||
#
|
||||
# Chop up the path. If it's just a file name then $dir and $file are the
|
||||
@@ -126,8 +131,16 @@ movefile () {
|
||||
# TODO: Compare the two files?
|
||||
#
|
||||
if [[ -e $todir/$path ]]; then
|
||||
echo "File already exists: $todir/$path"
|
||||
return 1
|
||||
if [[ $FORCE -eq 1 ]]; then
|
||||
echo "File exists: $todir/$path"
|
||||
echo "FORCE mode is ON so overwriting"
|
||||
mv --force "$fromdir/$path" "$todir/$path"
|
||||
echo "Moved $fromdir/$path"
|
||||
return 0
|
||||
else
|
||||
echo "File already exists: $todir/$path"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
mv "$fromdir/$path" "$todir/$path"
|
||||
echo "Moved $fromdir/$path"
|
||||
@@ -147,16 +160,28 @@ is_empty() {
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: _DEBUG
|
||||
# DESCRIPTION: Writes a message if in DEBUG mode
|
||||
# PARAMETERS: List of messages
|
||||
# NAME: _log
|
||||
# DESCRIPTION: Writes a log record to the predefined $LOGFILE in this script
|
||||
# using the predefined $LOGREC, a template for 'printf'. If the
|
||||
# latter is not defined the function will use a default.
|
||||
# For some reason 'shellcheck' objects to this function. The
|
||||
# first argument to 'printf' needs to be -1 to make the
|
||||
# '%(fmt)T' use today's date and time.
|
||||
# PARAMETERS: 1 - the message to write
|
||||
# RETURNS: Nothing
|
||||
#===============================================================================
|
||||
_DEBUG () {
|
||||
[ "$DEBUG" == 0 ] && return
|
||||
for msg in "$@"; do
|
||||
printf 'D> %s\n' "$msg"
|
||||
done
|
||||
# shellcheck disable=SC2317 disable=SC2059
|
||||
_log () {
|
||||
local msg="$1"
|
||||
|
||||
# echo "D> $LOGFILE $LOGREC"
|
||||
[ -v LOGFILE ] || { echo "${FUNCNAME[0]}: \$LOGFILE is not defined"; exit 1; }
|
||||
[ -v LOGREC ] || { local LOGREC='%(%F %T)T %s\n'; }
|
||||
|
||||
# echo "D> $LOGFILE $LOGREC"
|
||||
printf "$LOGREC" -1 "$msg" >> "$LOGFILE"
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
@@ -189,6 +214,11 @@ Options:
|
||||
to stop at.
|
||||
-D Run in debug mode where a lot more information is
|
||||
reported
|
||||
-F Turn on FORCE mode (normally off). In this mode when
|
||||
the files being tidied (moved) already exist, they are
|
||||
overwritten. This is for the very rare case when
|
||||
a show's audio has to be re-uploaded because of bad
|
||||
audio or the wrong file being sent.
|
||||
|
||||
Examples
|
||||
./tidy_uploaded # Run in (default) dry-run mode
|
||||
@@ -196,11 +226,14 @@ Examples
|
||||
./tidy_uploaded -d0 # Live mode (without verbose messages)
|
||||
./tidy_uploaded -c1 # Process 1 show in dry-run mode
|
||||
./tidy_uploaded -D # Run with debugging enabled
|
||||
./tidy_uploaded -F # Run with FORCE mode on
|
||||
|
||||
endusage
|
||||
exit "$res"
|
||||
}
|
||||
|
||||
# }}}
|
||||
|
||||
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
#
|
||||
@@ -208,16 +241,18 @@ endusage
|
||||
#
|
||||
LOGS="$BASEDIR/logs"
|
||||
LOGFILE="$LOGS/$SCRIPT.log"
|
||||
LOGREC='%(%F %T)T %s\n'
|
||||
|
||||
#
|
||||
# Process options
|
||||
#
|
||||
while getopts :c:d:Dhv opt
|
||||
while getopts :c:d:DFhv opt
|
||||
do
|
||||
case "${opt}" in
|
||||
c) COUNT=$OPTARG;;
|
||||
D) DEBUG=1;;
|
||||
d) DRYRUN=$OPTARG;;
|
||||
F) FORCE=1;;
|
||||
h) _usage 0;;
|
||||
v) VERBOSE=1;;
|
||||
*) echo "** Unknown option"
|
||||
@@ -239,6 +274,9 @@ if [[ $DRYRUN -ne 0 && $DRYRUN -ne 1 ]]; then
|
||||
fi
|
||||
[[ $DRYRUN -eq 1 ]] && echo "Dry run mode"
|
||||
|
||||
FORCE=${FORCE:-0}
|
||||
[[ $FORCE -eq 1 ]] && echo "Force mode - overwriting existing files"
|
||||
|
||||
VERBOSE=${VERBOSE:-0}
|
||||
|
||||
DEBUG=${DEBUG:-0}
|
||||
@@ -361,12 +399,17 @@ while read -r path; do
|
||||
#
|
||||
# A file on the IA exists in the upload area. Move the
|
||||
# local one if we're not in dry-run mode, otherwise just
|
||||
# report the move we would do.
|
||||
# report the move we would do. If FORCE mode is on
|
||||
# overwrite the file.
|
||||
#
|
||||
if [[ $DRYRUN -eq 0 ]]; then
|
||||
movefile "$UPLOADS" "$ARCHIVE" "$file" && ((moves++))
|
||||
movefile "$UPLOADS" "$ARCHIVE" "$file" "$FORCE" && ((moves++))
|
||||
else
|
||||
printf 'Would move %s\n\tto %s\n' "$frompath" "$topath"
|
||||
if [[ $FORCE -eq 0 ]]; then
|
||||
printf 'Would move %s\n\tto %s\n' "$frompath" "$topath"
|
||||
else
|
||||
printf 'Would move %s\n\toverwriting %s\n' "$frompath" "$topath"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done < "$TMP1"
|
||||
|
||||
Reference in New Issue
Block a user