forked from HPR/hpr-tools
Updates from previous repo
FAQ/FAQ.mkd, FAQ/Makefile: this version of the FAQ is now out of date
and probably should be deleted.
InternetArchive/repair_item: script to upload missing shows after tie
out errors during the normal upload; still under development.
InternetArchive/update_state: script to update show state in the
'reservations' table in the database. Uses the CMS interface.
Link_Checker/scan_links: under development. Not currently usable.
Miscellaneous/fix_tags: audio metadata manipulation script. Recently
added to this repo for convenience. Updates for 'experimental::try',
the official Perl try/catch.
PostgreSQL_Database/add_hosts_to_show, PostgreSQL_Database/hpr_schema_2.pgsql,
PostgreSQL_Database/nuke_n_pave.sh: an old experimental Pg database
to take over from the previous MySQL version (from before 2023).
Kept for reference; never implemented.
This commit is contained in:
Dave Morriss
@@ -25,9 +25,9 @@
|
||||
# BUGS: ---
|
||||
# NOTES: ---
|
||||
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
||||
# VERSION: 0.0.9
|
||||
# VERSION: 0.0.10
|
||||
# CREATED: 2022-04-19 12:50:52
|
||||
# REVISION: 2023-10-21 22:51:42
|
||||
# REVISION: 2024-06-01 14:19:20
|
||||
#
|
||||
#===============================================================================
|
||||
|
||||
@@ -37,7 +37,7 @@ SCRIPT=${0##*/}
|
||||
# DIR=${0%/*}
|
||||
|
||||
# shellcheck disable=SC2034
|
||||
VERSION="0.0.9"
|
||||
VERSION="0.0.10"
|
||||
|
||||
STDOUT="/dev/fd/2"
|
||||
|
||||
@@ -54,6 +54,8 @@ source "$LIB"
|
||||
#
|
||||
define_colours
|
||||
|
||||
# {{{ ---- Functions: ---- _usage _DEBUG
|
||||
|
||||
#=== FUNCTION ================================================================
|
||||
# NAME: _usage
|
||||
# DESCRIPTION: Report usage
|
||||
@@ -73,13 +75,17 @@ processed.
|
||||
|
||||
Options:
|
||||
-h Print this help
|
||||
-D Enable DEBUG mode where a lot of information about the workins
|
||||
-D Enable DEBUG mode where a lot of information about the working
|
||||
of the script is displayed
|
||||
-d Dry-run mode. Reports what it will do but doesn't do it
|
||||
-d Dry-run mode. Reports what it would do but doesn't do it
|
||||
-F Force the update(s) without checking the state of the show on
|
||||
the IA
|
||||
-l N Limit the number of shows processed to N
|
||||
-m Monochrome mode - no colours
|
||||
-R Normally, if a show is not in the IA, the script retries
|
||||
waiting for it to be uploaded (assuming it's being worked on
|
||||
by the IA servers). Including -R limits the retries to one
|
||||
which is useful when uploading multiple shows one at a time.
|
||||
|
||||
Examples
|
||||
./${SCRIPT} -h
|
||||
@@ -87,7 +93,10 @@ Examples
|
||||
./${SCRIPT} -d
|
||||
./${SCRIPT} -dm
|
||||
./${SCRIPT} -Dd
|
||||
./${SCRIPT} -F
|
||||
./${SCRIPT} -l1
|
||||
./${SCRIPT} -m
|
||||
./${SCRIPT} -R
|
||||
./${SCRIPT}
|
||||
|
||||
endusage
|
||||
@@ -107,6 +116,8 @@ _DEBUG () {
|
||||
done
|
||||
}
|
||||
|
||||
# }}}
|
||||
|
||||
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
#
|
||||
@@ -114,10 +125,10 @@ _DEBUG () {
|
||||
#
|
||||
case $HOSTNAME in
|
||||
hprvps|marvin|borg)
|
||||
# UPLOADS="/data/IA/uploads"
|
||||
# UPLOADS="/data/IA/uploads"
|
||||
BASEDIR="$HOME/IA" ;;
|
||||
i7-desktop)
|
||||
# UPLOADS="$HOME/HPR/IA/uploads"
|
||||
# UPLOADS="$HOME/HPR/IA/uploads"
|
||||
BASEDIR="$HOME/HPR/IA" ;;
|
||||
*)
|
||||
echo "Wrong host!"; exit 1 ;;
|
||||
@@ -149,16 +160,17 @@ RETRIES=3
|
||||
#
|
||||
# Option defaults
|
||||
#
|
||||
COLOUR=1 # use colours by default
|
||||
DRYRUN=0 # live mode by default
|
||||
COLOUR=1 # use colours by default
|
||||
DRYRUN=0 # live mode by default
|
||||
DEBUG=0
|
||||
FORCE=0
|
||||
RETRYING=1 # retry if a show's not on the IA
|
||||
DEFLIMIT=20
|
||||
|
||||
#
|
||||
# Process options
|
||||
#
|
||||
while getopts :hdDFl:m opt
|
||||
while getopts :hdDFl:mR opt
|
||||
do
|
||||
case "${opt}" in
|
||||
h) _usage;;
|
||||
@@ -167,6 +179,7 @@ do
|
||||
F) FORCE=1;;
|
||||
l) LIMIT=$OPTARG;;
|
||||
m) COLOUR=0;;
|
||||
R) RETRYING=0;;
|
||||
?) echo "$SCRIPT: Invalid option; aborting"; exit 1;;
|
||||
esac
|
||||
done
|
||||
@@ -189,6 +202,10 @@ if [[ $FORCE -eq 1 ]]; then
|
||||
coloured 'yellow' "Forcing updates without checking the IA state"
|
||||
fi
|
||||
|
||||
if [[ $RETRYING -eq 0 ]]; then
|
||||
coloured 'yellow' "Not retrying updates if the show is missing"
|
||||
fi
|
||||
|
||||
#
|
||||
# Check the argument count after any options
|
||||
#
|
||||
@@ -220,7 +237,7 @@ _DEBUG "reservations = $reservations"
|
||||
# "queue" from the variable 'reservations' which contains lines returned from
|
||||
# querying the CMS status interface.
|
||||
#
|
||||
count=0
|
||||
showcount=0
|
||||
while read -r line; do
|
||||
if [[ $line =~ ^([^,]+),([^,]+),([^,]+),([^,]+),([^,]+),.*$ ]]; then
|
||||
state="${BASH_REMATCH[5]}"
|
||||
@@ -232,7 +249,16 @@ while read -r line; do
|
||||
if [[ $state = 'MEDIA_TRANSCODED' ]]; then
|
||||
_DEBUG "show = $show, state = $state"
|
||||
|
||||
retry_count=$RETRIES
|
||||
#
|
||||
# If we're retrying (waiting for a show to be uploaded) then loop
|
||||
# $RETRIES times, otherwise don't retry at all
|
||||
#
|
||||
if [[ $RETRYING -eq 1 ]]; then
|
||||
retry_count=$RETRIES
|
||||
else
|
||||
retry_count=1
|
||||
fi
|
||||
|
||||
while [ $retry_count -gt 0 ]; do
|
||||
#
|
||||
# Look for the show on the IA. If not found we sleep 30
|
||||
@@ -240,7 +266,8 @@ while read -r line; do
|
||||
# times, controlled by $RETRIES, then we give up this show. If
|
||||
# there are more shows then we keep going.
|
||||
#
|
||||
if [ $FORCE -eq 1 ] || ia list "hpr$show" > /dev/null 2>&1; then
|
||||
if [ $FORCE -eq 1 ] || ia metadata "hpr$show" --exists > /dev/null 2>&1; then
|
||||
# if [ $FORCE -eq 1 ] || ia list "hpr$show" > /dev/null 2>&1; then
|
||||
command="${QUERY1}?ep_num=${show}&status=UPLOADED_TO_IA"
|
||||
command_bak="${QUERY1_BAK}?ep_num=${show}&status=UPLOADED_TO_IA"
|
||||
|
||||
@@ -282,19 +309,23 @@ while read -r line; do
|
||||
((retry_count--))
|
||||
done
|
||||
|
||||
if [[ $retry_count -eq 0 ]]; then
|
||||
#
|
||||
# Are all retries done, and are we retrying anyway?
|
||||
#
|
||||
if [[ $retry_count -eq 0 && $RETRYING -eq 1 ]]; then
|
||||
coloured 'red' "Failed to update show $show; retry count reached"
|
||||
coloured 'yellow' "The command 'ia list hpr$show' repeatedly returned \"failure\""
|
||||
coloured 'yellow' "Database updates not done"
|
||||
coloured 'yellow' "Try again later with './update_state'"
|
||||
coloured 'yellow' "Try again later with './${SCRIPT}'"
|
||||
fi
|
||||
|
||||
#
|
||||
# Stop the loop if we have reached the limiting number
|
||||
#
|
||||
((count++))
|
||||
[[ $count -eq $LIMIT ]] && {
|
||||
((showcount++))
|
||||
[[ $showcount -eq $LIMIT ]] && {
|
||||
echo "Upload limit ($LIMIT) reached"
|
||||
((--showcount))
|
||||
break
|
||||
}
|
||||
|
||||
@@ -303,9 +334,9 @@ while read -r line; do
|
||||
done <<< "$reservations"
|
||||
|
||||
if [[ $DRYRUN -eq 0 ]]; then
|
||||
echo "Number of shows processed successfully: $count"
|
||||
echo "Number of shows processed successfully: $showcount"
|
||||
fi
|
||||
|
||||
exit
|
||||
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
|
||||
|
||||
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker
|
||||
|
||||
Reference in New Issue
Block a user