HPR/hpr-tools
5
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Updated 'repair_item'

Browse Source 
InternetArchive/repair_item: originally planned in 2020 as a Bash script
    to find missing files in shows and then add them, it was not turned
    into the current form until May 2024. Now, with the heavy loading of
    the IA servers, normal uploads are timing out and being aborted.
    This script is more "determined" to upload files and usually
    successfully "repairs" shows that need it.
This commit is contained in:
Dave Morriss 2024-06-15 17:14:22 +01:00
parent 7a4290fcdd
commit 6805cd662b
1 changed files with 19 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
InternetArchive/repair_item
View File

@ -23,15 +23,15 @@
# BUGS: --- # BUGS: ---
# NOTES: --- # NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com # AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.7 # VERSION: 0.0.9
# CREATED: 2020-01-05 22:42:46 # CREATED: 2020-01-05 22:42:46
# REVISION: 2024-05-17 20:49:47 # REVISION: 2024-06-14 18:03:58
# #
#=============================================================================== #===============================================================================
#set -o nounset # Treat unset variables as an error #set -o nounset # Treat unset variables as an error
VERSION="0.0.7" VERSION="0.0.9"
SCRIPT=${0##*/} SCRIPT=${0##*/}
# DIR=${0%/*} # DIR=${0%/*}
@ -90,13 +90,15 @@ trap 'cleanup_temp $TMP1' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
#=== FUNCTION ================================================================ #=== FUNCTION ================================================================
# NAME: Upload # NAME: Upload
# DESCRIPTION: Upload a file to the Internet Archive with various options # DESCRIPTION: Uploads a file to the Internet Archive with various options.
# Any output from the 'ia' command invocation is saved in
# a temporary file and the name reported to the caller
# PARAMETERS: 1 - the item id (e.g. 'hpr1234' # PARAMETERS: 1 - the item id (e.g. 'hpr1234'
# 2 - the path to the file for upload # 2 - the path to the file for upload
# 3 - (optional) the path to the file on the IA # 3 - (optional) the path to the file on the IA
# 4 - (optional) list of options for 'ia upload' enclosed as # 4 - (optional) list of options for 'ia upload' enclosed as
# a string # a string
# RETURNS: Nothing # RETURNS: Exit code of last command
#=============================================================================== #===============================================================================
Upload () { Upload () {
local id=${1} local id=${1}
@ -104,16 +106,23 @@ Upload () {
local remote=${3:-} local remote=${3:-}
local options=${4:-} local options=${4:-}
local RES
if [[ -e $file ]]; then if [[ -e $file ]]; then
if [[ -z $remote ]]; then if [[ -z $remote ]]; then
# shellcheck disable=SC2086 # shellcheck disable=SC2086
ia upload ${id} ${file} ${options} ia upload ${id} ${file} ${options} > /dev/null 2>&1
RES=$?
return $RES
else else
# shellcheck disable=SC2086 # shellcheck disable=SC2086
ia upload ${id} ${file} --remote-name=${remote} ${options} ia upload ${id} ${file} --remote-name=${remote} ${options} > /dev/null 2>&1
RES=$?
return $RES
fi fi
else else
echo "File missing: $file" echo "File missing: $file"
return 1
fi fi
} }
@ -418,7 +427,7 @@ else
# upload when the system is overloaded, but these are non-fatal. # upload when the system is overloaded, but these are non-fatal.
# #
until eval "$cmd"; do until eval "$cmd"; do
coloured 'red' "Failure when invoking the Upload command!" coloured 'red' "Failure when uploading $file"
((retries++)) ((retries++))
printf '%s Failed to upload %s to the IA [%d]\n' \ printf '%s Failed to upload %s to the IA [%d]\n' \
@ -435,6 +444,7 @@ else
sleep $sleeptime sleep $sleeptime
done # until eval ... done # until eval ...
coloured 'green' "Uploaded $file to the IA"
echo "$(date +%Y%m%d%H%M%S) Uploaded $file to the IA" >> "$LOGFILE" echo "$(date +%Y%m%d%H%M%S) Uploaded $file to the IA" >> "$LOGFILE"
fi fi
@ -462,7 +472,7 @@ fi
if [[ $failures -gt 0 ]]; then if [[ $failures -gt 0 ]]; then
coloured 'red' \ coloured 'red' \
"There $(ngettext "was $failures upload failure" "were $failures upload failures" $failures)" "There $(ngettext "was $failures upload failure" "were $failures upload failures" $failures)"
coloured 'yellow' 'Run this script again to repeat the repair attemmpt' coloured 'yellow' 'Run this script again to repeat the repair attempt'
fi fi
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker # vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker