HPR/hpr-tools
5
0
Fork 1
Code Issues 12 Pull Requests Packages Projects Releases Wiki Activity

Fix variable override

Browse Source
This commit is contained in:
Ken Fallon 2025-05-12 15:31:12 +02:00
parent 946fb47508
commit 6150943cb3
1 changed files with 117 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

203
workflow/process_episode.bash
View File

@ -75,7 +75,7 @@ function program_checks() {
done done
} }
is_installed audio2image.bash awk base64 cat csvtojson curl date detox eval extract_images ffprobe file find grep head jq kate magick mediainfo mv realpath remove-image.pl rsync seamonkey sed sed sort sponge ssh touch touch wget is_installed audio2image.bash awk base64 cat csvtojson curl date detox eval extract_images ffprobe file find grep head jq kate magick mediainfo mv realpath remove-image.pl rsync seamonkey sed sed sort sponge ssh touch touch wget hpr-check-ccdn-links
for arg in $* for arg in $*
do do
@ -114,8 +114,29 @@ function check_variable_is_correct() {
echo_error "The \"ep_num\" variable is not a valid number between 1 and 9999." echo_error "The \"ep_num\" variable is not a valid number between 1 and 9999."
fi fi
;; ;;
piper_bin)
if [ -z "${piper_bin}" ]
then
echo_error "The \"ep_num\" variable is missing."
fi
;;
piper_voice)
if [ -z "${ep_num}" ]
then
echo_error "The \"ep_num\" variable is missing."
fi
;;
shownotes_html)
if [[ ! -s "${shownotes_html}" || -z "${shownotes_html}" ]]
then
echo_error "The \"shownotes_html\" variable/file is missing."
fi
if [ "$( file --brief --mime-type "${shownotes_html}" | grep --count 'text/html' )" -ne "1" ]
then
echo_error "The \"shownotes_html\" variable has not a valid \"text/html\" mime type."
fi
;;
shownotes_edited) shownotes_edited)
if [[ ! -s "${shownotes_edited}" || -z "${shownotes_edited}" ]] if [[ ! -s "${shownotes_edited}" || -z "${shownotes_edited}" ]]
then then
@ -507,27 +528,33 @@ function get_episode_metadata() {
</html>' </html>'
) | sponge "${shownotes_html}" ) | sponge "${shownotes_html}"
variables=( shownotes_json shownotes_html hostid host_name email title summary series_id series_name explicit episode_license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised ) episode_variables=( shownotes_json shownotes_html hostid host_name email title summary series_id series_name explicit episode_license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised )
for variable in "${variables[@]}" for episode_variable in "${episode_variables[@]}"
do do
if [[ -z ${!variable} && "${variable}" != "remote_media" ]] if [[ -z ${!episode_variable} && "${episode_variable}" != "remote_media" ]]
then # indirect expansion here then # indirect expansion here
echo_error "The variable \"${variable}\" is missing."; echo_error "The episode_variable \"${episode_variable}\" is missing.";
else else
echo_debug "The variable \"${variable}\" is set to \"${!variable}\""; echo_debug "The episode_variable \"${episode_variable}\" is set to \"${!episode_variable}\"";
fi fi
done done
# Argument Override # Argument Override
if [ $# -gt 0 ] if [ $# -gt 0 ]
then then
declare -A hash
for argument for argument
do do
if [[ $argument =~ ^[^=]+=.*$ ]] if [[ $argument =~ ^[^=]+=.*$ ]]
then then
this_key="${argument%=*}" this_key="${argument%=*}"
echo "${this_key}"
echo "${episode_variables[*]}"
if [ "$( echo "${episode_variables[*]}" | grep --count "${this_key}" )" -eq "0" ]
then
echo_debug "Skipping replacing \"${this_key}\".";
continue
fi
this_value="${argument#*=}" this_value="${argument#*=}"
this_value="$( echo "${this_value}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" this_value="$( echo "${this_value}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
eval "${this_key}=${this_value}" eval "${this_key}=${this_value}"
@ -1032,6 +1059,7 @@ function post_show_to_hpr_db() {
echo_error "The Episode hpr${ep_num} has not been posted" echo_error "The Episode hpr${ep_num} has not been posted"
fi fi
xdg-open https://archive.org/details/hpr${ep_num}
} }
################################################# #################################################
@ -1050,7 +1078,7 @@ function get_variables_from_episode_summary_json() {
if [ ! -s "${episode_summary_json}" ] if [ ! -s "${episode_summary_json}" ]
then then
echo_debug "The \"episode_summary_json\" variable/file is missing." echo_debug "The \"episode_summary_json\" variable/file is missing, getting a new version."
if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output "${episode_summary_json}" )" != 200 ] if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output "${episode_summary_json}" )" != 200 ]
then then
echo_error "The Episode hpr${ep_num} has not been posted" echo_error "The Episode hpr${ep_num} has not been posted"
@ -1529,15 +1557,15 @@ function register_assets() {
if [ -s "${assets_csv}" ] if [ -s "${assets_csv}" ]
then then
cat "${assets_csv}" | csvtojson | jq '{"assets":[.[]]}' | tee "assets_json" cat "${assets_csv}" | csvtojson | jq '{"assets":[.[]]}' | tee "${assets_json}"
fi fi
if [ ! -s "assets_json" ] if [ ! -s "${assets_json}" ]
then then
echo_error "The asset json file \"assets_json\" is missing."; echo_error "The asset json file \"${assets_json}\" is missing.";
fi fi
response="$( curl --silent --netrc-file $HOME/.netrc --write-out '%{http_code}' --output /dev/null --request POST https://hub.hackerpublicradio.org/cms/assets.php --data-ascii @"assets_json" --header "Content-Type: application/json" )" response="$( curl --silent --netrc-file $HOME/.netrc --write-out '%{http_code}' --output /dev/null --request POST https://hub.hackerpublicradio.org/cms/assets.php --data-ascii @"${assets_json}" --header "Content-Type: application/json" )"
if [[ -z "${response}" || "${response}" != "200" ]] if [[ -z "${response}" || "${response}" != "200" ]]
then then
echo_error "The assets for episode hpr${ep_num} has not been registered. The response was \"${response}\"" echo_error "The assets for episode hpr${ep_num} has not been registered. The response was \"${response}\""
@ -1735,13 +1763,10 @@ function upload_files_to_the_internet_archive() {
files_xml="${working_dir}/hpr${ep_num}_files.xml" files_xml="${working_dir}/hpr${ep_num}_files.xml"
fi fi
if [ ! -s "${files_xml}" ] echo_debug "Getting the \"files_xml\" file from the IA."
if [ "$( curl --silent --netrc --location --write-out '%{http_code}' https://archive.org/download/hpr${ep_num}/hpr${ep_num}_files.xml --output "${files_xml}" )" != 200 ]
then then
echo_debug "Getting the \"files_xml\" variable/file from the HPR site." echo_error "Could not find file list \"https://archive.org/download/hpr${ep_num}/hpr${ep_num}_files.xml\""
if [ "$( curl --silent --netrc --location --write-out '%{http_code}' https://archive.org/download/hpr${ep_num}/hpr${ep_num}_files.xml --output "${files_xml}" )" != 200 ]
then
echo_error "Could not find file list \"https://archive.org/download/hpr${ep_num}/hpr${ep_num}_files.xml\""
fi
fi fi
check_variable_is_correct files_xml check_variable_is_correct files_xml
@ -1753,7 +1778,7 @@ function upload_files_to_the_internet_archive() {
if [ ! -s "${files_json}" ] if [ ! -s "${files_json}" ]
then then
echo_debug "Getting the \"files_json\" variable/file from the HPR site." echo_debug "Getting the \"files_json\" file from the HPR site."
if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/assets.php?id=${ep_num} --output "${files_json}" )" != 200 ] if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/assets.php?id=${ep_num} --output "${files_json}" )" != 200 ]
then then
echo_error "The Episode hpr${ep_num} has not been posted" echo_error "The Episode hpr${ep_num} has not been posted"
@ -1802,48 +1827,13 @@ function upload_files_to_the_internet_archive() {
done done
wait_for_the_internet_archive_to_process
} }
function hide_me() { function hide_me() {
echo "hidden" echo "hidden"
#
# Compare assets_json to files_xml
# locate the different files and upload them
# if [ "$( find ${working_dir} -mindepth 1 -maxdepth 1 -type f \( -iname "hpr${ep_num}.flac" -or -iname "hpr${ep_num}.mp3" -or -iname "hpr${ep_num}.ogg" -or -iname "hpr${ep_num}.opus" -or -iname "hpr${ep_num}_source.*" -or -iname "hpr${ep_num}.srt" -or -iname "hpr${ep_num}.txt" -or -iname "hpr${ep_num}.wav" \) | wc --lines )" -ne "8" ]
# then
# echo_error "There are not 8 derived files for the Internet Archive."
# fi
#
# while read this_ia_file
# do
# if [[ ! -s "${this_ia_file}" ]]
# then
# ls -al "${this_ia_file}"
# echo_error "The derived files to the Internet Archive are missing \"${this_ia_file}\"."
# fi
#
# echo_debug "Sending \"${this_ia_file}\" to the Internet Archive"
#
#
# # origin_sha1sum="$( echo $( ssh rsync.net "sha1 hpr/eps/hpr${ep_num}/${this_asset}" 2> /dev/null ) | awk '{print $NF}' )"
# # this_asset_sha1sum="$( sha1sum "${this_file}" | awk '{print $1}' )"
# #
# # if [[ -z "${origin_sha1sum}" || -z "${this_asset_sha1sum}" ]]
# # then
# # echo_error "Could not determine the local/origin sha1sum for file \"${this_file}\"."
# # fi
# #
# # ia download hpr_761c0f557b87090db3f8d4d9bce7fc70 hpr_761c0f557b87090db3f8d4d9bce7fc70_files.xml
#
# echo ia upload hpr${ep_num} "${this_ia_file}" --header "x-archive-keep-old-version:0" --retries=5 --no-derive --no-backup
#
#
#
# done < <( find ${working_dir} -mindepth 1 -maxdepth 1 -type f \( -iname "hpr${ep_num}.flac" -or -iname "hpr${ep_num}.mp3" -or -iname "hpr${ep_num}.ogg" -or -iname "hpr${ep_num}.opus" -or -iname "hpr${ep_num}_source.*" -or -iname "hpr${ep_num}.srt" -or -iname "hpr${ep_num}.txt" -or -iname "hpr${ep_num}.wav" \) )
} }
@ -1880,58 +1870,67 @@ then
done done
fi fi
########################################################################################
# Posting show
#TODO add logic to check where we are and what we need to do.
# program_checks # We know that all the programs and variables are set # All functions should skip if their work is done. Except when mode=refresh
# #
# get_working_dir $@ # We have a working directory and a valid json file
# #
# get_episode_metadata $@ # We have all the metadata we need to process the show.
# #
# extract_images_brute_force # We have extracted the images by brute force # If there is a working dir then
# # - is the show posted etc, shownotes_edited.html
# media_checks # # - do the assets match hpr${ep_num}_assets.json
# # -
# generate_initial_report # Generate Initial Report for review by the Janitors # #
# # # program_checks # We know that all the programs and variables are set
# manual_shownotes_review # Janitors review audio and shownote. Skips if done. # #
# # # get_working_dir $@ # We have a working directory and a valid json file
# post_show_to_hpr_db # Posts the episode to HPR. Skips if it is already posted. # #
# # # get_episode_metadata $@ # We have all the metadata we need to process the show.
# #
######################################################################################### # # extract_images_brute_force # We have extracted the images by brute force
# Using DB info from here # #
# # media_checks #
get_variables_from_episode_summary_json # Get the episode from HPR. # #
# # generate_initial_report # Generate Initial Report for review by the Janitors
# # #
# create_tts_summary # Generate text to speech summary # # manual_shownotes_review # Janitors review audio and shownote. Skips if done.
# # #
# generate_intro # Generate Intro from the intro theme with overlay of a lead in silence then the tts summary # # post_show_to_hpr_db # Posts the episode to HPR. Skips if it is already posted.
# # #
# generate_parent_audio # Combines the intro, the episode, and the outro to a final cut. # # ########################################################################################
# # # # Using DB info from here
# generate_derived_media # Generate the flac wav mp3 ogg opus files # #
# # # get_variables_from_episode_summary_json # Get the episode from HPR.
# generate_show_transcript # #
# # # create_tts_summary # Generate text to speech summary
# generate_final_report # #
# # # generate_intro # Generate Intro from the intro theme with overlay of a lead in silence then the tts summary
# manual_final_review # #
# # # generate_parent_audio # Combines the intro, the episode, and the outro to a final cut.
# register_assets # #
# # generate_derived_media # Generate the flac wav mp3 ogg opus files
# copy_files_to_origin_server # #
# # generate_show_transcript
#check_variable_is_correct working_dir ep_num shownotes_edited # #
# # generate_final_report
# copy_derived_files_to_borg # #
# # manual_final_review
# create_item_on_the_internet_archive # #
# # register_assets
# #
# # copy_files_to_origin_server
# #
# # check_variable_is_correct working_dir ep_num shownotes_edited
# #
# # copy_derived_files_to_borg
# #
# # create_item_on_the_internet_archive
upload_files_to_the_internet_archive upload_files_to_the_internet_archive
#for i in {4301..4305};do echo ${i};/home/ken/sourcecode/personal/bin/hpr-check-ccdn-links.bash ${i};done hpr-check-ccdn-links
echo_debug "The End" echo_debug "The End"
exit 0 exit 0