50edeccc88
FAQ/FAQ.mkd, FAQ/Makefile: this version of the FAQ is now out of date
and probably should be deleted.
InternetArchive/repair_item: script to upload missing shows after tie
out errors during the normal upload; still under development.
InternetArchive/update_state: script to update show state in the
'reservations' table in the database. Uses the CMS interface.
Link_Checker/scan_links: under development. Not currently usable.
Miscellaneous/fix_tags: audio metadata manipulation script. Recently
added to this repo for convenience. Updates for 'experimental::try',
the official Perl try/catch.
PostgreSQL_Database/add_hosts_to_show, PostgreSQL_Database/hpr_schema_2.pgsql,
PostgreSQL_Database/nuke_n_pave.sh: an old experimental Pg database
to take over from the previous MySQL version (from before 2023).
Kept for reference; never implemented.
343 lines
10 KiB
Bash
Executable File
343 lines
10 KiB
Bash
Executable File
#!/bin/bash -
|
|
#===============================================================================
|
|
#
|
|
# FILE: update_state
|
|
#
|
|
# USAGE: ./update_state
|
|
#
|
|
# DESCRIPTION: A script to update the state of shows which have been sent to
|
|
# the IA. It looks at the current state of the 'reservations'
|
|
# table on the HPR database and selects all shows which are in
|
|
# the state 'MEDIA_TRANSCODED'. It checks each one to see if it
|
|
# known to the IA and if so changes state to 'UPLOADED_TO_IA'.
|
|
#
|
|
# The IA check can be overridden using the '-F' option, but care
|
|
# should be taken not to do this unless it is known all eligible
|
|
# shows are uploaded.
|
|
#
|
|
# Note that the algorithm described here does not work for
|
|
# reserved shows like the Community News episodes since they are
|
|
# not submitted as such and have no entry in the 'reservations'
|
|
# table.
|
|
#
|
|
# OPTIONS: ---
|
|
# REQUIREMENTS: ---
|
|
# BUGS: ---
|
|
# NOTES: ---
|
|
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
|
# VERSION: 0.0.10
|
|
# CREATED: 2022-04-19 12:50:52
|
|
# REVISION: 2024-06-01 14:19:20
|
|
#
|
|
#===============================================================================
|
|
|
|
set -o nounset # Treat unset variables as an error
|
|
|
|
SCRIPT=${0##*/}
|
|
# DIR=${0%/*}
|
|
|
|
# shellcheck disable=SC2034
|
|
VERSION="0.0.10"
|
|
|
|
STDOUT="/dev/fd/2"
|
|
|
|
#
|
|
# Load library functions
|
|
#
|
|
LIB="$HOME/bin/function_lib.sh"
|
|
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
|
|
# shellcheck source=/home/cendjm/bin/function_lib.sh
|
|
source "$LIB"
|
|
|
|
#
|
|
# Colour codes
|
|
#
|
|
define_colours
|
|
|
|
# {{{ ---- Functions: ---- _usage _DEBUG
|
|
|
|
#=== FUNCTION ================================================================
|
|
# NAME: _usage
|
|
# DESCRIPTION: Report usage
|
|
# PARAMETERS: None
|
|
# RETURNS: Nothing
|
|
#===============================================================================
|
|
_usage () {
|
|
local -i res="${1:-0}"
|
|
|
|
cat >$STDOUT <<-endusage
|
|
Usage: ./${SCRIPT} [-h] [-D] [-d] [-F] [-l N] [-m]
|
|
|
|
Version: $VERSION
|
|
|
|
Script to update the status in the 'reservations' table after a show has been
|
|
processed.
|
|
|
|
Options:
|
|
-h Print this help
|
|
-D Enable DEBUG mode where a lot of information about the working
|
|
of the script is displayed
|
|
-d Dry-run mode. Reports what it would do but doesn't do it
|
|
-F Force the update(s) without checking the state of the show on
|
|
the IA
|
|
-l N Limit the number of shows processed to N
|
|
-m Monochrome mode - no colours
|
|
-R Normally, if a show is not in the IA, the script retries
|
|
waiting for it to be uploaded (assuming it's being worked on
|
|
by the IA servers). Including -R limits the retries to one
|
|
which is useful when uploading multiple shows one at a time.
|
|
|
|
Examples
|
|
./${SCRIPT} -h
|
|
./${SCRIPT} -m
|
|
./${SCRIPT} -d
|
|
./${SCRIPT} -dm
|
|
./${SCRIPT} -Dd
|
|
./${SCRIPT} -F
|
|
./${SCRIPT} -l1
|
|
./${SCRIPT} -m
|
|
./${SCRIPT} -R
|
|
./${SCRIPT}
|
|
|
|
endusage
|
|
exit "$res"
|
|
}
|
|
|
|
#=== FUNCTION ================================================================
|
|
# NAME: _DEBUG
|
|
# DESCRIPTION: Writes a message if in DEBUG mode
|
|
# PARAMETERS: List of messages
|
|
# RETURNS: Nothing
|
|
#===============================================================================
|
|
_DEBUG () {
|
|
[ "$DEBUG" == 0 ] && return
|
|
for msg in "$@"; do
|
|
printf 'D> %s\n' "$msg"
|
|
done
|
|
}
|
|
|
|
# }}}
|
|
|
|
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
|
|
#
|
|
# Configure depending whether local or on borg
|
|
#
|
|
case $HOSTNAME in
|
|
hprvps|marvin|borg)
|
|
# UPLOADS="/data/IA/uploads"
|
|
BASEDIR="$HOME/IA" ;;
|
|
i7-desktop)
|
|
# UPLOADS="$HOME/HPR/IA/uploads"
|
|
BASEDIR="$HOME/HPR/IA" ;;
|
|
*)
|
|
echo "Wrong host!"; exit 1 ;;
|
|
esac
|
|
|
|
cd "$BASEDIR" || { echo "Can't cd to $BASEDIR"; exit 1; }
|
|
|
|
|
|
#
|
|
# Tools
|
|
#
|
|
BASECOM='curl -K ./.hpradmin_curlrc -s'
|
|
URL="https://hub.hackerpublicradio.org/cms/status.php"
|
|
QUERY1="${BASECOM} ${URL}"
|
|
QUERY2="${BASECOM} -o - ${URL}"
|
|
|
|
#
|
|
# Fallback URLs and commands
|
|
#
|
|
URL_BAK="http://hub.hackerpublicradio.org/cms/status.php"
|
|
QUERY1_BAK="${BASECOM} ${URL_BAK}"
|
|
QUERY2_BAK="${BASECOM} -o - ${URL_BAK}"
|
|
|
|
#
|
|
# Number of retries per show
|
|
#
|
|
RETRIES=3
|
|
|
|
#
|
|
# Option defaults
|
|
#
|
|
COLOUR=1 # use colours by default
|
|
DRYRUN=0 # live mode by default
|
|
DEBUG=0
|
|
FORCE=0
|
|
RETRYING=1 # retry if a show's not on the IA
|
|
DEFLIMIT=20
|
|
|
|
#
|
|
# Process options
|
|
#
|
|
while getopts :hdDFl:mR opt
|
|
do
|
|
case "${opt}" in
|
|
h) _usage;;
|
|
d) DRYRUN=1;;
|
|
D) DEBUG=1;;
|
|
F) FORCE=1;;
|
|
l) LIMIT=$OPTARG;;
|
|
m) COLOUR=0;;
|
|
R) RETRYING=0;;
|
|
?) echo "$SCRIPT: Invalid option; aborting"; exit 1;;
|
|
esac
|
|
done
|
|
shift $((OPTIND - 1))
|
|
|
|
#
|
|
# Cancel colours if requested
|
|
#
|
|
if [[ $COLOUR -eq 0 ]]; then
|
|
undefine_colours
|
|
fi
|
|
|
|
LIMIT=${LIMIT:-$DEFLIMIT}
|
|
if [[ $LIMIT -lt 1 || $LIMIT -gt $DEFLIMIT ]]; then
|
|
echo "** Use '-l 1' up to '-l $DEFLIMIT' or omit the option"
|
|
_usage 1
|
|
fi
|
|
|
|
if [[ $FORCE -eq 1 ]]; then
|
|
coloured 'yellow' "Forcing updates without checking the IA state"
|
|
fi
|
|
|
|
if [[ $RETRYING -eq 0 ]]; then
|
|
coloured 'yellow' "Not retrying updates if the show is missing"
|
|
fi
|
|
|
|
#
|
|
# Check the argument count after any options
|
|
#
|
|
if [[ $# -ne 0 ]]; then
|
|
coloured 'red' "** ${SCRIPT} takes no arguments"
|
|
_usage 1
|
|
fi
|
|
|
|
#
|
|
# Collect the current table of shows requiring work. We expect something like:
|
|
# timestamp_epoc,ep_num,ep_date,key,status,email
|
|
# 1651286617,3617,2022-06-14,fda088e0e3bd5d0353ea6b7569e93b87626ca25976a0a,UPLOADED_TO_IA,lurkingprion@gmail.com
|
|
# 1651648589,3619,2022-06-16,e7d3810afa098863d81663418d8640276272284de68f1,UPLOADED_TO_IA,monochromec@gmail.com
|
|
# TODO: Check for a failure in the query?
|
|
# NOTE: Problem encountered 2022-09-23 because the SSL certificate has expired
|
|
#
|
|
reservations=$($QUERY2) || {
|
|
coloured 'red' "Problem querying $URL"
|
|
coloured 'yellow' "Falling back to $URL_BAK"
|
|
reservations=$($QUERY2_BAK) || {
|
|
coloured 'red' "Failed with fallback URL - aborting"
|
|
exit 1
|
|
}
|
|
}
|
|
_DEBUG "reservations = $reservations"
|
|
|
|
#
|
|
# Check which shows are on the IA and can be flagged as such. We get the work
|
|
# "queue" from the variable 'reservations' which contains lines returned from
|
|
# querying the CMS status interface.
|
|
#
|
|
showcount=0
|
|
while read -r line; do
|
|
if [[ $line =~ ^([^,]+),([^,]+),([^,]+),([^,]+),([^,]+),.*$ ]]; then
|
|
state="${BASH_REMATCH[5]}"
|
|
show="${BASH_REMATCH[2]}"
|
|
|
|
#
|
|
# Process shows in just one of the states
|
|
#
|
|
if [[ $state = 'MEDIA_TRANSCODED' ]]; then
|
|
_DEBUG "show = $show, state = $state"
|
|
|
|
#
|
|
# If we're retrying (waiting for a show to be uploaded) then loop
|
|
# $RETRIES times, otherwise don't retry at all
|
|
#
|
|
if [[ $RETRYING -eq 1 ]]; then
|
|
retry_count=$RETRIES
|
|
else
|
|
retry_count=1
|
|
fi
|
|
|
|
while [ $retry_count -gt 0 ]; do
|
|
#
|
|
# Look for the show on the IA. If not found we sleep 30
|
|
# seconds and look again. This happens a limited number of
|
|
# times, controlled by $RETRIES, then we give up this show. If
|
|
# there are more shows then we keep going.
|
|
#
|
|
if [ $FORCE -eq 1 ] || ia metadata "hpr$show" --exists > /dev/null 2>&1; then
|
|
# if [ $FORCE -eq 1 ] || ia list "hpr$show" > /dev/null 2>&1; then
|
|
command="${QUERY1}?ep_num=${show}&status=UPLOADED_TO_IA"
|
|
command_bak="${QUERY1_BAK}?ep_num=${show}&status=UPLOADED_TO_IA"
|
|
|
|
#
|
|
# In dry-run mode we count this iteration as success. In
|
|
# live mode we exit if the command fails. Otherwise we
|
|
# assume the command succeeds and exit the retry loop.
|
|
#
|
|
if [[ $DRYRUN -eq 1 ]]; then
|
|
echo -e "Dry-run: would have run\n${yellow}$command${reset}"
|
|
else
|
|
coloured 'yellow' "$command"
|
|
$command || {
|
|
coloured 'red' "Problem querying $URL"
|
|
coloured 'yellow' "Falling back to $URL_BAK"
|
|
$command_bak || {
|
|
coloured 'red' "Failed with fallback URL - aborting"
|
|
exit 1
|
|
}
|
|
}
|
|
RES=$?
|
|
if [[ $RES -ne 0 ]]; then
|
|
coloured 'red' "** Problem running $command; aborting"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
#
|
|
# Success. Stop the loop
|
|
#
|
|
break
|
|
else
|
|
#
|
|
# Failed to find the show, have another go after a wait
|
|
#
|
|
coloured 'red' "Show $show is not yet uploaded"
|
|
sleep 30
|
|
fi
|
|
((retry_count--))
|
|
done
|
|
|
|
#
|
|
# Are all retries done, and are we retrying anyway?
|
|
#
|
|
if [[ $retry_count -eq 0 && $RETRYING -eq 1 ]]; then
|
|
coloured 'red' "Failed to update show $show; retry count reached"
|
|
coloured 'yellow' "The command 'ia list hpr$show' repeatedly returned \"failure\""
|
|
coloured 'yellow' "Database updates not done"
|
|
coloured 'yellow' "Try again later with './${SCRIPT}'"
|
|
fi
|
|
|
|
#
|
|
# Stop the loop if we have reached the limiting number
|
|
#
|
|
((showcount++))
|
|
[[ $showcount -eq $LIMIT ]] && {
|
|
echo "Upload limit ($LIMIT) reached"
|
|
((--showcount))
|
|
break
|
|
}
|
|
|
|
fi
|
|
fi
|
|
done <<< "$reservations"
|
|
|
|
if [[ $DRYRUN -eq 0 ]]; then
|
|
echo "Number of shows processed successfully: $showcount"
|
|
fi
|
|
|
|
exit
|
|
|
|
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker
|