HPR/hpr-tools
5
0
Fork 1
Code Issues 13 Pull Requests Packages Projects Releases Wiki Activity

2025-05-16_13-14-56Z_Friday

Browse Source
This commit is contained in:
Ken Fallon 2025-05-16 15:14:56 +02:00
parent 6150943cb3
commit e8c203debf
1 changed files with 257 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

429
workflow/process_episode.bash
View File

@ -89,77 +89,31 @@ function program_checks() {
}
#################################################
# Program Checks
# Variable Checks
#album artist assets_csv assets_json comment date duration duration_iso8601 email email_unpadded ep_date episode_summary_json ep_num explicit files_json files_xml genre HOME hostid intro_srt key license license_url outro_flac outro_srt piper_bin piper_voice processing_dir shownotes_edited shownotes_html silence summary synopsis tags theme timestamp_epoc title track working_dir year
function check_variable_is_correct() {
#echo_debug "Checking variables ${*}. check_variable_is_correct()"
for argument in "$@"
do
case "${argument}" in
working_dir)
if [[ ! -d "${working_dir}" || -z "${working_dir}" ]]
then
echo_error "The \"working dir\" variable is missing."
fi
;;
ep_num)
if [ -z "${ep_num}" ]
then
echo_error "The \"ep_num\" variable is missing."
fi
if [ "$( echo "${ep_num}" | grep --perl-regexp '^(0{0,3}[1-9]\d{0,2}|[1-9]\d{0,3})$' | wc --lines )" -eq "0" ]
then
echo_error "The \"ep_num\" variable is not a valid number between 1 and 9999."
fi
;;
piper_bin)
if [ -z "${piper_bin}" ]
then
echo_error "The \"ep_num\" variable is missing."
fi
;;
piper_voice)
if [ -z "${ep_num}" ]
then
echo_error "The \"ep_num\" variable is missing."
fi
;;
shownotes_html)
if [[ ! -s "${shownotes_html}" || -z "${shownotes_html}" ]]
album)
if [[ -z "${album}" || "${album}" == "null" ]]
then
echo_error "The \"shownotes_html\" variable/file is missing."
echo_error "The \"album\" variable is missing."
fi
if [ "$( file --brief --mime-type "${shownotes_html}" | grep --count 'text/html' )" -ne "1" ]
if [ "$( echo "${album}" | grep --perl-regexp '^Hacker Public Radio$' | wc --lines )" -eq "0" ]
then
echo_error "The \"shownotes_html\" variable has not a valid \"text/html\" mime type."
echo_error "The \"album\" variable is not \"Hacker Public Radio\"."
fi
;;
shownotes_edited)
if [[ ! -s "${shownotes_edited}" || -z "${shownotes_edited}" ]]
artist)
if [[ -z "${artist}" || "${artist}" == "null" ]]
then
echo_error "The \"shownotes_edited\" variable/file is missing."
fi
if [ "$( file --brief --mime-type "${shownotes_edited}" | grep --count 'text/html' )" -ne "1" ]
then
echo_error "The \"shownotes_edited\" variable has not a valid \"text/html\" mime type."
fi
;;
episode_summary_json)
if [[ ! -s "${episode_summary_json}" || -z "${episode_summary_json}" ]]
then
echo_error "The \"episode_summary_json\" variable/file is missing."
fi
if [ "$( file --brief --mime-type "${episode_summary_json}" | grep --count 'application/json' )" -ne "1" ]
then
echo_error "The \"episode_summary_json\" variable has not a valid \"application/json\" mime type."
fi
jq '.' "${episode_summary_json}" >/dev/null 2>&1
if [ $? -ne 0 ]
then
echo_error "The file \"${episode_summary_json}\" is not valid json."
echo_error "The \"artist\" variable is missing."
fi
;;
assets_csv)
@ -187,6 +141,83 @@ function check_variable_is_correct() {
echo_error "The file \"${assets_json}\" is not valid json."
fi
;;
comment)
if [[ -z "${comment}" || "${comment}" == "null" ]]
then
echo_error "The \"comment\" variable is missing."
fi
;;
date)
if [[ -z "${date}" || "${date}" == "null" ]]
then
echo_error "The \"date\" variable is missing."
fi
;;
duration)
if [[ -z "${duration}" || "${duration}" == "null" ]]
then
echo_error "The \"duration\" variable is missing."
fi
if [[ -z "${duration}" || "${duration}" -lt "30" || "${duration}" -gt "30000" ]]
then
echo_error "Invalid duration missing or outside range 30 to 30000." >&2
fi
;;
duration_iso8601)
if [[ -z "${duration_iso8601}" || "${duration_iso8601}" == "null" ]]
then
echo_error "The \"duration_iso8601\" variable is missing."
fi
;;
email)
if [ -z "${email}" ]
then
echo_error "The \"email\" variable is missing."
fi
;;
email_unpadded)
if [ -z "${email_unpadded}" ]
then
echo_error "The \"email_unpadded\" variable is missing."
fi
;;
episode_summary_json)
if [[ ! -s "${episode_summary_json}" || -z "${episode_summary_json}" ]]
then
echo_error "The \"episode_summary_json\" variable/file is missing."
fi
if [ "$( file --brief --mime-type "${episode_summary_json}" | grep --count 'application/json' )" -ne "1" ]
then
echo_error "The \"episode_summary_json\" variable has not a valid \"application/json\" mime type."
fi
jq '.' "${episode_summary_json}" >/dev/null 2>&1
if [ $? -ne 0 ]
then
echo_error "The file \"${episode_summary_json}\" is not valid json."
fi
;;
ep_date)
if [ -z "${ep_date}" ]
then
echo_error "The \"ep_date\" variable is missing."
fi
;;
ep_num)
if [ -z "${ep_num}" ]
then
echo_error "The \"ep_num\" variable is missing."
fi
if [ "$( echo "${ep_num}" | grep --perl-regexp '^(0{0,3}[1-9]\d{0,2}|[1-9]\d{0,3})$' | wc --lines )" -eq "0" ]
then
echo_error "The \"ep_num\" variable is not a valid number between 1 and 9999."
fi
;;
explicit)
if [[ -z "${explicit}" || "${explicit}" == "null" ]]
then
echo_error "The \"explicit\" variable is missing."
fi
;;
files_json)
if [[ ! -s "${files_json}" || -z "${files_json}" ]]
then
@ -217,70 +248,36 @@ function check_variable_is_correct() {
echo_error "The file \"${files_xml}\" is not valid xml."
fi
;;
album)
if [[ -z "${album}" || "${album}" == "null" ]]
then
echo_error "The \"album\" variable is missing."
fi
if [ "$( echo "${album}" | grep --perl-regexp '^Hacker Public Radio$' | wc --lines )" -eq "0" ]
then
echo_error "The \"album\" variable is not \"Hacker Public Radio\"."
fi
;;
artist)
if [[ -z "${artist}" || "${artist}" == "null" ]]
then
echo_error "The \"artist\" variable is missing."
fi
;;
comment)
if [[ -z "${comment}" || "${comment}" == "null" ]]
then
echo_error "The \"comment\" variable is missing."
fi
;;
date)
if [[ -z "${date}" || "${date}" == "null" ]]
then
echo_error "The \"date\" variable is missing."
fi
;;
duration)
if [[ -z "${duration}" || "${duration}" == "null" ]]
then
echo_error "The \"duration\" variable is missing."
fi
if [[ -z "${duration}" || "${duration}" -lt "30" || "${duration}" -gt "30000" ]]
then
echo_error "Invalid duration missing or outside range 30 to 30000." >&2
fi
;;
duration_iso8601)
if [[ -z "${duration_iso8601}" || "${duration_iso8601}" == "null" ]]
then
echo_error "The \"duration_iso8601\" variable is missing."
fi
;;
explicit)
if [[ -z "${explicit}" || "${explicit}" == "null" ]]
then
echo_error "The \"explicit\" variable is missing."
fi
;;
genre)
if [[ -z "${genre}" || "${genre}" == "null" ]]
then
echo_error "The \"genre\" variable is missing."
fi
;;
HOME)
if [[ ! -d "${HOME}" || -z "${HOME}" ]]
then
echo_error "The \"HOME\" variable is missing or file \"${HOME}\" does not exist."
fi
;;
hostid)
if [[ -z "${hostid}" || "${hostid}" == "null" ]]
then
echo_error "The \"hostid\" variable is missing."
fi
;;
intro_srt)
if [[ ! -d "${intro_srt}" || -z "${intro_srt}" ]]
then
echo_error "The \"intro_srt\" variable is missing or file \"${intro_srt}\" does not exist."
fi
;;
key)
if [ -z "${key}" ]
then
echo_error "The \"key\" variable is missing."
fi
;;
license)
if [[ -z "${license}" || "${license}" == "null" ]]
then
@ -293,6 +290,62 @@ function check_variable_is_correct() {
echo_error "The \"license_url\" variable is missing."
fi
;;
outro_flac)
if [[ ! -d "${outro_flac}" || -z "${outro_flac}" ]]
then
echo_error "The \"outro_flac\" variable is missing or file \"${outro_flac}\" does not exist."
fi
;;
outro_srt)
if [[ ! -d "${outro_srt}" || -z "${outro_srt}" ]]
then
echo_error "The \"outro_srt\" variable is missing or file \"${outro_srt}\" does not exist."
fi
;;
piper_bin)
if [ -z "${piper_bin}" ]
then
echo_error "The \"ep_num\" variable is missing."
fi
;;
piper_voice)
if [ -z "${ep_num}" ]
then
echo_error "The \"ep_num\" variable is missing."
fi
;;
processing_dir)
if [[ ! -d "${processing_dir}" || -z "${processing_dir}" ]]
then
echo_error "The \"processing_dir\" variable is missing or is not a directory."
fi
;;
shownotes_edited)
if [[ ! -s "${shownotes_edited}" || -z "${shownotes_edited}" ]]
then
echo_error "The \"shownotes_edited\" variable/file is missing."
fi
# if [ "$( file --brief --mime-type "${shownotes_edited}" | grep --count 'text/html' )" -ne "1" ]
# then
# echo_error "The \"shownotes_edited\" variable has not a valid \"text/html\" mime type \"${shownotes_edited}\"."
# fi
;;
shownotes_html)
if [[ ! -s "${shownotes_html}" || -z "${shownotes_html}" ]]
then
echo_error "The \"shownotes_html\" variable/file is missing."
fi
# if [ "$( file --brief --mime-type "${shownotes_html}" | grep --count 'text/html' )" -ne "1" ]
# then
# echo_error "The \"shownotes_html\" variable has not a valid \"text/html\" mime type. \"${shownotes_html}\""
# fi
;;
silence)
if [[ ! -d "${silence}" || -z "${silence}" ]]
then
echo_error "The \"silence\" variable is missing or file \"${silence}\" does not exist."
fi
;;
summary)
if [[ -z "${summary}" || "${summary}" == "null" ]]
then
@ -311,6 +364,18 @@ function check_variable_is_correct() {
echo_error "The \"tags\" variable is missing."
fi
;;
theme)
if [[ ! -d "${theme}" || -z "${theme}" ]]
then
echo_error "The \"theme\" variable is missing or file \"${theme}\" does not exist."
fi
;;
timestamp_epoc)
if [ -z "${timestamp_epoc}" ]
then
echo_error "The \"timestamp_epoc\" variable is missing."
fi
;;
title)
if [[ -z "${title}" || "${title}" == "null" ]]
then
@ -323,12 +388,19 @@ function check_variable_is_correct() {
echo_error "The \"track\" variable is missing."
fi
;;
working_dir)
if [[ ! -d "${working_dir}" || -z "${working_dir}" ]]
then
echo_error "The \"working_dir\" variable is missing or is not a directory."
fi
;;
year)
if [[ -z "${year}" || "${year}" == "null" ]]
then
echo_error "The \"year\" variable is missing."
fi
;;
*)
echo_error "An unknown variable \"${argument}\" was provided."
;;
@ -340,9 +412,11 @@ function check_variable_is_correct() {
#################################################
# Get the next show in the queue
function get_ep_num_from_hpr_hub() {
function get_next_show_from_hpr_hub() {
echo_debug "Processing the next HPR Show in the queue. get_ep_num_from_hpr_hub()"
echo_debug "Processing the next HPR Show in the queue. get_next_show_from_hpr_hub()"
check_variable_is_correct processing_dir HOME
if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${processing_dir}/status.csv" )" != 200 ]
then
@ -354,7 +428,7 @@ function get_ep_num_from_hpr_hub() {
echo_error "Failed to retrieve \"${processing_dir}/status.csv\" from server."
fi
response=$( cat "${processing_dir}/status.csv" | grep ',SHOW_SUBMITTED,' | head -1 | sed 's/,/ /g' )
response=$( cat "${processing_dir}/status.csv" | grep ',SHOW_SUBMITTED,' | sort -t ',' -k 2 | head -1 | sed 's/,/ /g' )
if [ -z "${response}" ]
then
echo_debug "Getting a list of all the reservations."
@ -373,6 +447,8 @@ function get_ep_num_from_hpr_hub() {
source_dir="hpr:${hpr_upload_dir}"
dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
working_dir="${processing_dir}/${dest_dir}"
check_variable_is_correct timestamp_epoc ep_num ep_date key email email_unpadded
echo_debug "Downloading hpr${ep_num} from ${email_unpadded}"
echo_debug ""
@ -449,7 +525,7 @@ function get_working_dir() {
if [ $# -eq 0 ]
then
get_ep_num_from_hpr_hub
get_next_show_from_hpr_hub
fi
check_variable_is_correct working_dir
@ -548,8 +624,6 @@ function get_episode_metadata() {
if [[ $argument =~ ^[^=]+=.*$ ]]
then
this_key="${argument%=*}"
echo "${this_key}"
echo "${episode_variables[*]}"
if [ "$( echo "${episode_variables[*]}" | grep --count "${this_key}" )" -eq "0" ]
then
echo_debug "Skipping replacing \"${this_key}\".";
@ -1059,7 +1133,6 @@ function post_show_to_hpr_db() {
echo_error "The Episode hpr${ep_num} has not been posted"
fi
xdg-open https://archive.org/details/hpr${ep_num}
}
#################################################
@ -1677,7 +1750,7 @@ function copy_derived_files_to_borg() {
# Try and change the online db status to MEDIA_TRANSCODED
if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' "https://hub.hackerpublicradio.org/cms/status.php?ep_num=${ep_num}&status=MEDIA_TRANSCODED" )" != 200 ]
then
echo_error "Could not change the status of \"${ep_num}\" to \"MEDIA_TRANSCODED\""
echo_debug "Could not change the status of \"${ep_num}\" to \"MEDIA_TRANSCODED\""
fi
# Get the current status
@ -1691,7 +1764,7 @@ function copy_derived_files_to_borg() {
if [ -z "${response}" ]
then
grep --perl-regexp ",${ep_num},.*MEDIA_TRANSCODED," "${processing_dir}/status.csv"
echo_error "The show \"${ep_num}\" hasn't the correct status of \"MEDIA_TRANSCODED\" in the database."
echo_debug "The show \"${ep_num}\" hasn't the correct status of \"MEDIA_TRANSCODED\" in the database."
fi
echo_debug "The show \"${ep_num}\" has the correct status of \"MEDIA_TRANSCODED\" in the database."
@ -1743,7 +1816,6 @@ function create_item_on_the_internet_archive() {
echo_debug "The episode has been posted to the Internet Archive"
fi
xdg-open https://archive.org/details/hpr${ep_num}
wait_for_the_internet_archive_to_process
}
@ -1762,7 +1834,12 @@ function upload_files_to_the_internet_archive() {
then
files_xml="${working_dir}/hpr${ep_num}_files.xml"
fi
if [ -s "${files_xml}" ]
then
echo_debug "Removing stale \"files_xml\" file \"${files_xml}\"."
rm -v "${files_xml}"
fi
echo_debug "Getting the \"files_xml\" file from the IA."
if [ "$( curl --silent --netrc --location --write-out '%{http_code}' https://archive.org/download/hpr${ep_num}/hpr${ep_num}_files.xml --output "${files_xml}" )" != 200 ]
then
@ -1776,13 +1853,16 @@ function upload_files_to_the_internet_archive() {
files_json="${working_dir}/hpr${ep_num}_files.json"
fi
if [ ! -s "${files_json}" ]
if [ -s "${files_json}" ]
then
echo_debug "Getting the \"files_json\" file from the HPR site."
if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/assets.php?id=${ep_num} --output "${files_json}" )" != 200 ]
then
echo_error "The Episode hpr${ep_num} has not been posted"
fi
echo_debug "Removing stale \"files_json\" file \"${files_json}\"."
rm -v "${files_json}"
fi
echo_debug "Getting the \"files_json\" file from the HPR site."
if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/assets.php?id=${ep_num} --output "${files_json}" )" != 200 ]
then
echo_error "The Episode hpr${ep_num} has not been posted"
fi
check_variable_is_correct files_json
@ -1882,57 +1962,62 @@ fi
# - is the show posted etc, shownotes_edited.html
# - do the assets match hpr${ep_num}_assets.json
# -
# #
# # program_checks # We know that all the programs and variables are set
# #
# # get_working_dir $@ # We have a working directory and a valid json file
# #
# # get_episode_metadata $@ # We have all the metadata we need to process the show.
# #
# # extract_images_brute_force # We have extracted the images by brute force
# #
# # media_checks #
# #
# # generate_initial_report # Generate Initial Report for review by the Janitors
# #
# # manual_shownotes_review # Janitors review audio and shownote. Skips if done.
# #
# # post_show_to_hpr_db # Posts the episode to HPR. Skips if it is already posted.
# #
# # ########################################################################################
# # # Using DB info from here
# #
# # get_variables_from_episode_summary_json # Get the episode from HPR.
# #
# # create_tts_summary # Generate text to speech summary
# #
# # generate_intro # Generate Intro from the intro theme with overlay of a lead in silence then the tts summary
# #
# # generate_parent_audio # Combines the intro, the episode, and the outro to a final cut.
# #
# # generate_derived_media # Generate the flac wav mp3 ogg opus files
# #
# # generate_show_transcript
# #
# # generate_final_report
# #
# # manual_final_review
# #
# # register_assets
# #
# # copy_files_to_origin_server
# #
# # check_variable_is_correct working_dir ep_num shownotes_edited
# #
# # copy_derived_files_to_borg
# #
# # create_item_on_the_internet_archive
program_checks # We know that all the programs and variables are set
get_working_dir $@ # We have a working directory and a valid json file
get_episode_metadata $@ # We have all the metadata we need to process the show.
extract_images_brute_force # We have extracted the images by brute force
media_checks #
generate_initial_report # Generate Initial Report for review by the Janitors
manual_shownotes_review # Janitors review audio and shownote. Skips if done.
post_show_to_hpr_db # Posts the episode to HPR. Skips if it is already posted.
# #######################################################################################
# Using DB info from here
get_variables_from_episode_summary_json # Get the episode from HPR.
create_tts_summary # Generate text to speech summary
generate_intro # Generate Intro from the intro theme with overlay of a lead in silence then the tts summary
generate_parent_audio # Combines the intro, the episode, and the outro to a final cut.
generate_derived_media # Generate the flac wav mp3 ogg opus files
generate_show_transcript
generate_final_report
manual_final_review
register_assets
copy_files_to_origin_server
check_variable_is_correct working_dir ep_num shownotes_edited
copy_derived_files_to_borg
create_item_on_the_internet_archive
upload_files_to_the_internet_archive
hpr-check-ccdn-links
hpr-check-ccdn-links ${ep_num}
echo_debug "The End"
xdg-open https://archive.org/details/hpr${ep_num}
xdg-open https://hackerpublicradio.org/eps/hpr${ep_num}/index.html
exit 0
# if ${force_overwrite}