HPR/hpr-tools
8
1
Fork 2
Code Issues 14 Pull Requests Packages Projects Releases Wiki Activity

refactoring to get show varibles from posted episode data

Browse Source
This commit is contained in:
Ken Fallon
2025-05-09 19:14:05 +02:00
parent 62071280a5
commit 240ece066b
1 changed files with 466 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

313
workflow/process_episode.bash
View File

@@ -93,7 +93,7 @@ function program_checks() {
function check_variable_is_correct() { function check_variable_is_correct() {
echo_debug "Checking variables ${*}. check_variable_is_correct()" #echo_debug "Checking variables ${*}. check_variable_is_correct()"
for argument in "$@" for argument in "$@"
do do
@@ -115,16 +115,138 @@ function check_variable_is_correct() {
fi fi
;; ;;
shownotes_edited) shownotes_edited)
if [ ! -s "${shownotes_edited}" ] if [[ ! -s "${shownotes_edited}" || -z "${shownotes_edited}" ]]
then then
echo_debug "The \"shownotes_edited\" variable is missing." echo_error "The \"shownotes_edited\" variable/file is missing."
return
fi fi
if [ "$( file --brief --mime-type "${shownotes_edited}" | grep --count 'text/html' )" -ne "1" ] if [ "$( file --brief --mime-type "${shownotes_edited}" | grep --count 'text/html' )" -ne "1" ]
then then
echo_error "The \"shownotes_edited\" variable has not a valid \"text/html\" mime type." echo_error "The \"shownotes_edited\" variable has not a valid \"text/html\" mime type."
fi fi
;; ;;
episode_summary_json)
if [[ ! -s "${episode_summary_json}" || -z "${episode_summary_json}" ]]
then
echo_error "The \"episode_summary_json\" variable/file is missing."
fi
if [ "$( file --brief --mime-type "${episode_summary_json}" | grep --count 'application/json' )" -ne "1" ]
then
echo_error "The \"episode_summary_json\" variable has not a valid \"application/json\" mime type."
fi
;;
album)
if [[ -z "${album}" || "${album}" == "null" ]]
then
echo_error "The \"album\" variable is missing."
fi
if [ "$( echo "${album}" | grep --perl-regexp '^Hacker Public Radio$' | wc --lines )" -eq "0" ]
then
echo_error "The \"album\" variable is not \"Hacker Public Radio\"."
fi
;;
artist)
if [[ -z "${artist}" || "${artist}" == "null" ]]
then
echo_error "The \"artist\" variable is missing."
fi
;;
comment)
if [[ -z "${comment}" || "${comment}" == "null" ]]
then
echo_error "The \"comment\" variable is missing."
fi
;;
date)
if [[ -z "${date}" || "${date}" == "null" ]]
then
echo_error "The \"date\" variable is missing."
fi
;;
duration)
if [[ -z "${duration}" || "${duration}" == "null" ]]
then
echo_error "The \"duration\" variable is missing."
fi
if [[ -z "${duration}" || "${duration}" -lt "30" || "${duration}" -gt "30000" ]]
then
echo_error "Invalid duration missing or outside range 30 to 30000." >&2
fi
;;
duration_iso8601)
if [[ -z "${duration_iso8601}" || "${duration_iso8601}" == "null" ]]
then
echo_error "The \"duration_iso8601\" variable is missing."
fi
;;
explicit)
if [[ -z "${explicit}" || "${explicit}" == "null" ]]
then
echo_error "The \"explicit\" variable is missing."
fi
;;
genre)
if [[ -z "${genre}" || "${genre}" == "null" ]]
then
echo_error "The \"genre\" variable is missing."
fi
;;
hostid)
if [[ -z "${hostid}" || "${hostid}" == "null" ]]
then
echo_error "The \"hostid\" variable is missing."
fi
;;
license)
if [[ -z "${license}" || "${license}" == "null" ]]
then
echo_error "The \"license\" variable is missing."
fi
;;
license_url)
if [[ -z "${license_url}" || "${license_url}" == "null" ]]
then
echo_error "The \"license_url\" variable is missing."
fi
;;
summary)
if [[ -z "${summary}" || "${summary}" == "null" ]]
then
echo_error "The \"summary\" variable is missing."
fi
;;
synopsis)
if [[ -z "${synopsis}" || "${synopsis}" == "null" ]]
then
echo_error "The \"synopsis\" variable is missing."
fi
;;
tags)
if [[ -z "${tags}" || "${tags}" == "null" ]]
then
echo_error "The \"tags\" variable is missing."
fi
;;
title)
if [[ -z "${title}" || "${title}" == "null" ]]
then
echo_error "The \"title\" variable is missing."
fi
;;
track)
if [[ -z "${track}" || "${track}" == "null" ]]
then
echo_error "The \"track\" variable is missing."
fi
;;
year)
if [[ -z "${year}" || "${year}" == "null" ]]
then
echo_error "The \"year\" variable is missing."
fi
;;
*)
echo_error "An unknown variable \"${argument}\" was provided."
;;
esac esac
done done
@@ -133,9 +255,9 @@ function check_variable_is_correct() {
################################################# #################################################
# Get the next show in the queue # Get the next show in the queue
function get_working_dir_from_hpr_hub() { function get_ep_num_from_hpr_hub() {
echo_debug "Processing the next HPR Show in the queue. get_working_dir_from_hpr_hub()" echo_debug "Processing the next HPR Show in the queue. get_ep_num_from_hpr_hub()"
if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${processing_dir}/status.csv" )" != 200 ] if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${processing_dir}/status.csv" )" != 200 ]
then then
@@ -189,14 +311,11 @@ function get_working_dir_from_hpr_hub() {
################################################# #################################################
# Get the show information from a local directory # Get the show information from a local directory
function get_working_dir_from_local_dir() { function get_ep_num_from_local_dir() {
echo_debug "Processing a local directory. get_working_dir_from_local_dir()" echo_debug "Processing a local directory. get_ep_num_from_local_dir()"
if [[ ! -d "${working_dir}" || -z "${working_dir}" ]] check_variable_is_correct working_dir
then
echo_error "The working dir is missing. Please supply a working directory."
fi
if [ ! -s "${working_dir}/shownotes.json" ] if [ ! -s "${working_dir}/shownotes.json" ]
then then
@@ -245,9 +364,15 @@ function get_working_dir() {
if [ $# -eq 0 ] if [ $# -eq 0 ]
then then
get_working_dir_from_hpr_hub get_ep_num_from_hpr_hub
else fi
get_working_dir_from_local_dir $@
check_variable_is_correct working_dir
if [ -z "${ep_num}" ]
then
get_ep_num_from_local_dir $@
check_variable_is_correct ep_num
fi fi
if [ ! -s "${working_dir}/shownotes.json" ] if [ ! -s "${working_dir}/shownotes.json" ]
@@ -845,6 +970,43 @@ function post_show_to_hpr_db() {
} }
#################################################
# Get the
function get_variables_from_episode_summary_json() {
echo_debug "Creating Text to Speech summary. get_variables_from_episode_summary_json()"
check_variable_is_correct ep_num working_dir
if [ -z "${episode_summary_json}" ]
then
episode_summary_json="${working_dir}/episode_summary.json"
fi
if [ ! -s "${episode_summary_json}" ]
then
echo_debug "The \"episode_summary_json\" variable/file is missing."
if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output "${episode_summary_json}" )" != 200 ]
then
echo_error "The Episode hpr${ep_num} has not been posted"
fi
fi
check_variable_is_correct episode_summary_json
for episode_summary_key in $( jq --raw-output '. | keys | @tsv' "${episode_summary_json}" )
do
episode_summary_value="$( jq --raw-output ".${episode_summary_key}" "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
declare "${episode_summary_key}=${episode_summary_value}"
echo_debug "Setting \"${episode_summary_key}\" to \"${episode_summary_value}\" from \"$( basename ${episode_summary_json} )\""
check_variable_is_correct ${episode_summary_key}
done
duration_iso8601="$( \date -d@${duration} -u +%H:%M:%S )"
check_variable_is_correct duration_iso8601
}
################################################# #################################################
# Generate text to speech summary # Generate text to speech summary
@@ -852,28 +1014,7 @@ function create_tts_summary() {
echo_debug "Creating Text to Speech summary. create_tts_summary()" echo_debug "Creating Text to Speech summary. create_tts_summary()"
if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output "${working_dir}/episode_summary.json" )" != 200 ] check_variable_is_correct working_dir duration synopsis piper_bin piper_voice
then
echo_error "The Episode hpr${ep_num} has not been posted"
fi
if [ ! -s "${working_dir}/episode_summary.json" ]
then
echo_error "Failed to find the extracted shownote html file \"episode_summary.json\""
fi
duration="$( jq --raw-output '.duration' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
if [[ -z "${duration}" || "${duration}" -lt "30" || "${duration}" -gt "30000" ]]
then
echo_error "Invalid duration found in \"episode_summary.json\"" >&2
fi
synopsis="$( jq --raw-output '.synopsis' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
if [[ -z "${synopsis}" || "${synopsis}" == "null" ]]
then
echo_error "Could not retrieve the synopsis for the text to speech."
fi
echo_debug "Converting text synopsis \"${synopsis}\" to speech." echo_debug "Converting text synopsis \"${synopsis}\" to speech."
@@ -953,7 +1094,7 @@ function generate_derived_media() {
echo_error "The final cut is not available." echo_error "The final cut is not available."
fi fi
episode_comment="$( jq --raw-output '.comment' "${working_dir}/episode_summary.json" )" episode_comment="$( jq --raw-output '.comment' "${episode_summary_json}" )"
episode_year="$( echo "${ep_date}" | cut -c -4 )" episode_year="$( echo "${ep_date}" | cut -c -4 )"
# https://wiki.multimedia.cx/index.php?title=FFmpeg_Metadata # https://wiki.multimedia.cx/index.php?title=FFmpeg_Metadata
@@ -1011,19 +1152,19 @@ function generate_show_transcript() {
# Copy in the intro subtitle template and replace each line with the text with the summary # Copy in the intro subtitle template and replace each line with the text with the summary
date="$( jq --raw-output '.date' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )" date="$( jq --raw-output '.date' "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
title="$( jq --raw-output '.title' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )" title="$( jq --raw-output '.title' "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
duration="$( jq --raw-output '.duration' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )" duration="$( jq --raw-output '.duration' "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
duration_iso8601="$( \date -d@${duration} -u +%H:%M:%S )" duration_iso8601="$( \date -d@${duration} -u +%H:%M:%S )"
artist="$( jq --raw-output '.artist' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )" artist="$( jq --raw-output '.artist' "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
explicit="$( jq --raw-output '.explicit' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )" explicit="$( jq --raw-output '.explicit' "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
license="$( jq --raw-output '.license' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )" license="$( jq --raw-output '.license' "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
summary="$( jq --raw-output '.summary' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )" summary="$( jq --raw-output '.summary' "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
if [[ -z "${date}" || "${date}" == "null" || -z "${title}" || "${title}" == "null" || -z "${duration_iso8601}" || "${duration_iso8601}" == "null" || -z "${artist}" || "${artist}" == "null" || -z "${explicit}" || "${explicit}" == "null" || -z "${license}" || "${license}" == "null" || -z "${summary}" || "${summary}" == "null" ]] if [[ -z "${date}" || "${date}" == "null" || -z "${title}" || "${title}" == "null" || -z "${duration_iso8601}" || "${duration_iso8601}" == "null" || -z "${artist}" || "${artist}" == "null" || -z "${explicit}" || "${explicit}" == "null" || -z "${license}" || "${license}" == "null" || -z "${summary}" || "${summary}" == "null" ]]
then then
echo_error "Could not retrieve the synopsis for the text to speech." echo_error "Could not retrieve the synopsis for the text to speech."
ls -al "${working_dir}/episode_summary.json" ls -al "${episode_summary_json}"
fi fi
REPLACE_LINE_1="This is Hacker Public Radio Episode ${ep_num}, for ${date}" REPLACE_LINE_1="This is Hacker Public Radio Episode ${ep_num}, for ${date}"
@@ -1471,17 +1612,46 @@ function wait_for_the_internet_archive_to_process() {
echo_debug "Waiting for the Internet Archive to finish processing. wait_for_the_internet_archive_to_process()" echo_debug "Waiting for the Internet Archive to finish processing. wait_for_the_internet_archive_to_process()"
if [ -z "${ep_num}x" ] check_variable_is_correct ep_num
then
echo_error "Could not find the episode number while uploading to the Internet Archive" while [ "$( ia tasks "hpr${ep_num}" | jq --slurp '[.[] | if .category == "catalog" then .status else empty end] | length' )" -ne "0" ]
fi do
echo_debug "Waiting for the Internet Archive to finish processing \"hpr${ep_num}\"."
date
sleep 1m
done
} }
################################################# #################################################
# Send the derived files to the server borg to be sent to the Internet Archive # Send the derived files to the server borg to be sent to the Internet Archive
function upload_to_the_internet_archive() { function create_item_on_the_internet_archive() {
echo_debug "Sending the derived files to Internet Archive. upload_to_the_internet_archive()" echo_debug "Sending the derived files to Internet Archive. create_item_on_the_internet_archive()"
check_variable_is_correct ep_num
if [ ! -s "${working_dir}/hpr${ep_num}.txt" ]
then
echo_error "The Internet Archive \"${working_dir}/hpr${ep_num}.txt\" is missing."
fi
echo ia upload hpr${ep_num} "${working_dir}/hpr${ep_num}.txt" --metadata=mediatype:audio --metadata="contributor:HackerPublicRadio" --metadata="creator:HPR Volunteers" --metadata="date:2025-05-05" --metadata="description:This show is a placeholder and will be updated soon." --metadata="language:eng" --metadata="licenseurl:http://creativecommons.org/licenses/by-sa/4.0" --metadata="title:A placeholder for hpr${ep_num}." --metadata=reviews-allowed:none --header x-archive-keep-old-version:0 --retries=5 --no-derive --no-backup
# curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output "${episode_summary_json}"
# curl --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/shownotes.php?id=${ep_num} --output "${working_dir}/shownotes.json"
wait_for_the_internet_archive_to_process
}
#################################################
# Send the derived files to the server borg to be sent to the Internet Archive
function upload_files_to_the_internet_archive() {
echo_debug "Sending the derived files to Internet Archive. upload_files_to_the_internet_archive()"
check_variable_is_correct working_dir ep_num shownotes_html check_variable_is_correct working_dir ep_num shownotes_html
@@ -1495,17 +1665,21 @@ function upload_to_the_internet_archive() {
# hpr4371.wav # hpr4371.wav
# #
find ${working_dir} -mindepth 1 -maxdepth 1 -type f \( -iname "hpr${ep_num}.flac" -or -iname "hpr${ep_num}.mp3" -or -iname "hpr${ep_num}.ogg" -or -iname "hpr${ep_num}.opus" -or -iname "hpr${ep_num}_source.*" -or -iname "hpr${ep_num}.srt" -or -iname "hpr${ep_num}.txt" -or -iname "hpr${ep_num}.wav" \) if [ "$( find ${working_dir} -mindepth 1 -maxdepth 1 -type f \( -iname "hpr${ep_num}.flac" -or -iname "hpr${ep_num}.mp3" -or -iname "hpr${ep_num}.ogg" -or -iname "hpr${ep_num}.opus" -or -iname "hpr${ep_num}_source.*" -or -iname "hpr${ep_num}.srt" -or -iname "hpr${ep_num}.txt" -or -iname "hpr${ep_num}.wav" \) | wc --lines )" -ne "8" ]
for extension in flac mp3 ogg opus _source.* srt txt wav
do
if [[ ! -s "${working_dir}/hpr${ep_num}.${extension}" ]]
then then
echo_error "The derived files to the Internet Archive are missing \"hpr${ep_num}.${extension}\"." echo_error "There are not 8 derived files for the Internet Archive."
ls -al "${working_dir}/hpr${ep_num}.${extension}"
fi fi
done
while read this_ia_file
do
if [[ ! -s "${this_ia_file}" ]]
then
ls -al "${this_ia_file}"
echo_error "The derived files to the Internet Archive are missing \"${this_ia_file}\"."
fi
done < <( find ${working_dir} -mindepth 1 -maxdepth 1 -type f \( -iname "hpr${ep_num}.flac" -or -iname "hpr${ep_num}.mp3" -or -iname "hpr${ep_num}.ogg" -or -iname "hpr${ep_num}.opus" -or -iname "hpr${ep_num}_source.*" -or -iname "hpr${ep_num}.srt" -or -iname "hpr${ep_num}.txt" -or -iname "hpr${ep_num}.wav" \) )
# #
# "${working_dir}/hpr${ep_num}.${extension}" # "${working_dir}/hpr${ep_num}.${extension}"
# #
@@ -1618,7 +1792,7 @@ function upload_to_the_internet_archive() {
# Get supplied working dir and ep_num if provided # Get supplied working dir and ep_num if provided
if [ $# -gt 0 ] if [ $# -gt 0 ]
then then
declare -A hash # declare -A hash
for argument for argument
do do
if [[ $argument =~ ^[^=]+=.*$ ]] if [[ $argument =~ ^[^=]+=.*$ ]]
@@ -1649,6 +1823,17 @@ fi
# manual_shownotes_review # Janitors review audio and shownote. Skips if done. # manual_shownotes_review # Janitors review audio and shownote. Skips if done.
# #
# post_show_to_hpr_db # Posts the episode to HPR. Skips if it is already posted. # post_show_to_hpr_db # Posts the episode to HPR. Skips if it is already posted.
#
#########################################################################################
# Using DB info from here
get_variables_from_episode_summary_json # Get the episode from HPR.
# #
# create_tts_summary # Generate text to speech summary # create_tts_summary # Generate text to speech summary
# #
@@ -1672,7 +1857,9 @@ fi
# copy_derived_files_to_borg # copy_derived_files_to_borg
upload_to_the_internet_archive #create_item_on_the_internet_archive
#upload_files_to_the_internet_archive
#for i in {4301..4305};do echo ${i};/home/ken/sourcecode/personal/bin/hpr-check-ccdn-links.bash ${i};done #for i in {4301..4305};do echo ${i};/home/ken/sourcecode/personal/bin/hpr-check-ccdn-links.bash ${i};done