diff --git a/workflow/process_episode.bash b/workflow/process_episode.bash index 2b4323f..64c562f 100755 --- a/workflow/process_episode.bash +++ b/workflow/process_episode.bash @@ -89,77 +89,31 @@ function program_checks() { } ################################################# -# Program Checks +# Variable Checks + +#album artist assets_csv assets_json comment date duration duration_iso8601 email email_unpadded ep_date episode_summary_json ep_num explicit files_json files_xml genre HOME hostid intro_srt key license license_url outro_flac outro_srt piper_bin piper_voice processing_dir shownotes_edited shownotes_html silence summary synopsis tags theme timestamp_epoc title track working_dir year function check_variable_is_correct() { - #echo_debug "Checking variables ${*}. check_variable_is_correct()" for argument in "$@" do case "${argument}" in - working_dir) - if [[ ! -d "${working_dir}" || -z "${working_dir}" ]] - then - echo_error "The \"working dir\" variable is missing." - fi - ;; - ep_num) - if [ -z "${ep_num}" ] - then - echo_error "The \"ep_num\" variable is missing." - fi - if [ "$( echo "${ep_num}" | grep --perl-regexp '^(0{0,3}[1-9]\d{0,2}|[1-9]\d{0,3})$' | wc --lines )" -eq "0" ] - then - echo_error "The \"ep_num\" variable is not a valid number between 1 and 9999." - fi - ;; - piper_bin) - if [ -z "${piper_bin}" ] - then - echo_error "The \"ep_num\" variable is missing." - fi - ;; - piper_voice) - if [ -z "${ep_num}" ] - then - echo_error "The \"ep_num\" variable is missing." - fi - ;; - shownotes_html) - if [[ ! -s "${shownotes_html}" || -z "${shownotes_html}" ]] + album) + if [[ -z "${album}" || "${album}" == "null" ]] then - echo_error "The \"shownotes_html\" variable/file is missing." + echo_error "The \"album\" variable is missing." fi - if [ "$( file --brief --mime-type "${shownotes_html}" | grep --count 'text/html' )" -ne "1" ] + if [ "$( echo "${album}" | grep --perl-regexp '^Hacker Public Radio$' | wc --lines )" -eq "0" ] then - echo_error "The \"shownotes_html\" variable has not a valid \"text/html\" mime type." + echo_error "The \"album\" variable is not \"Hacker Public Radio\"." fi ;; - shownotes_edited) - if [[ ! -s "${shownotes_edited}" || -z "${shownotes_edited}" ]] + artist) + if [[ -z "${artist}" || "${artist}" == "null" ]] then - echo_error "The \"shownotes_edited\" variable/file is missing." - fi - if [ "$( file --brief --mime-type "${shownotes_edited}" | grep --count 'text/html' )" -ne "1" ] - then - echo_error "The \"shownotes_edited\" variable has not a valid \"text/html\" mime type." - fi - ;; - episode_summary_json) - if [[ ! -s "${episode_summary_json}" || -z "${episode_summary_json}" ]] - then - echo_error "The \"episode_summary_json\" variable/file is missing." - fi - if [ "$( file --brief --mime-type "${episode_summary_json}" | grep --count 'application/json' )" -ne "1" ] - then - echo_error "The \"episode_summary_json\" variable has not a valid \"application/json\" mime type." - fi - jq '.' "${episode_summary_json}" >/dev/null 2>&1 - if [ $? -ne 0 ] - then - echo_error "The file \"${episode_summary_json}\" is not valid json." + echo_error "The \"artist\" variable is missing." fi ;; assets_csv) @@ -187,6 +141,83 @@ function check_variable_is_correct() { echo_error "The file \"${assets_json}\" is not valid json." fi ;; + comment) + if [[ -z "${comment}" || "${comment}" == "null" ]] + then + echo_error "The \"comment\" variable is missing." + fi + ;; + date) + if [[ -z "${date}" || "${date}" == "null" ]] + then + echo_error "The \"date\" variable is missing." + fi + ;; + duration) + if [[ -z "${duration}" || "${duration}" == "null" ]] + then + echo_error "The \"duration\" variable is missing." + fi + if [[ -z "${duration}" || "${duration}" -lt "30" || "${duration}" -gt "30000" ]] + then + echo_error "Invalid duration missing or outside range 30 to 30000." >&2 + fi + ;; + duration_iso8601) + if [[ -z "${duration_iso8601}" || "${duration_iso8601}" == "null" ]] + then + echo_error "The \"duration_iso8601\" variable is missing." + fi + ;; + email) + if [ -z "${email}" ] + then + echo_error "The \"email\" variable is missing." + fi + ;; + email_unpadded) + if [ -z "${email_unpadded}" ] + then + echo_error "The \"email_unpadded\" variable is missing." + fi + ;; + episode_summary_json) + if [[ ! -s "${episode_summary_json}" || -z "${episode_summary_json}" ]] + then + echo_error "The \"episode_summary_json\" variable/file is missing." + fi + if [ "$( file --brief --mime-type "${episode_summary_json}" | grep --count 'application/json' )" -ne "1" ] + then + echo_error "The \"episode_summary_json\" variable has not a valid \"application/json\" mime type." + fi + jq '.' "${episode_summary_json}" >/dev/null 2>&1 + if [ $? -ne 0 ] + then + echo_error "The file \"${episode_summary_json}\" is not valid json." + fi + ;; + ep_date) + if [ -z "${ep_date}" ] + then + echo_error "The \"ep_date\" variable is missing." + fi + ;; + ep_num) + if [ -z "${ep_num}" ] + then + echo_error "The \"ep_num\" variable is missing." + fi + if [ "$( echo "${ep_num}" | grep --perl-regexp '^(0{0,3}[1-9]\d{0,2}|[1-9]\d{0,3})$' | wc --lines )" -eq "0" ] + then + echo_error "The \"ep_num\" variable is not a valid number between 1 and 9999." + fi + ;; + explicit) + if [[ -z "${explicit}" || "${explicit}" == "null" ]] + then + echo_error "The \"explicit\" variable is missing." + fi + ;; files_json) if [[ ! -s "${files_json}" || -z "${files_json}" ]] then @@ -217,70 +248,36 @@ function check_variable_is_correct() { echo_error "The file \"${files_xml}\" is not valid xml." fi ;; - - - album) - if [[ -z "${album}" || "${album}" == "null" ]] - then - echo_error "The \"album\" variable is missing." - fi - if [ "$( echo "${album}" | grep --perl-regexp '^Hacker Public Radio$' | wc --lines )" -eq "0" ] - then - echo_error "The \"album\" variable is not \"Hacker Public Radio\"." - fi - ;; - artist) - if [[ -z "${artist}" || "${artist}" == "null" ]] - then - echo_error "The \"artist\" variable is missing." - fi - ;; - comment) - if [[ -z "${comment}" || "${comment}" == "null" ]] - then - echo_error "The \"comment\" variable is missing." - fi - ;; - date) - if [[ -z "${date}" || "${date}" == "null" ]] - then - echo_error "The \"date\" variable is missing." - fi - ;; - duration) - if [[ -z "${duration}" || "${duration}" == "null" ]] - then - echo_error "The \"duration\" variable is missing." - fi - if [[ -z "${duration}" || "${duration}" -lt "30" || "${duration}" -gt "30000" ]] - then - echo_error "Invalid duration missing or outside range 30 to 30000." >&2 - fi - ;; - duration_iso8601) - if [[ -z "${duration_iso8601}" || "${duration_iso8601}" == "null" ]] - then - echo_error "The \"duration_iso8601\" variable is missing." - fi - ;; - explicit) - if [[ -z "${explicit}" || "${explicit}" == "null" ]] - then - echo_error "The \"explicit\" variable is missing." - fi - ;; genre) if [[ -z "${genre}" || "${genre}" == "null" ]] then echo_error "The \"genre\" variable is missing." fi ;; + HOME) + if [[ ! -d "${HOME}" || -z "${HOME}" ]] + then + echo_error "The \"HOME\" variable is missing or file \"${HOME}\" does not exist." + fi + ;; hostid) if [[ -z "${hostid}" || "${hostid}" == "null" ]] then echo_error "The \"hostid\" variable is missing." fi ;; + intro_srt) + if [[ ! -d "${intro_srt}" || -z "${intro_srt}" ]] + then + echo_error "The \"intro_srt\" variable is missing or file \"${intro_srt}\" does not exist." + fi + ;; + key) + if [ -z "${key}" ] + then + echo_error "The \"key\" variable is missing." + fi + ;; license) if [[ -z "${license}" || "${license}" == "null" ]] then @@ -293,6 +290,62 @@ function check_variable_is_correct() { echo_error "The \"license_url\" variable is missing." fi ;; + outro_flac) + if [[ ! -d "${outro_flac}" || -z "${outro_flac}" ]] + then + echo_error "The \"outro_flac\" variable is missing or file \"${outro_flac}\" does not exist." + fi + ;; + outro_srt) + if [[ ! -d "${outro_srt}" || -z "${outro_srt}" ]] + then + echo_error "The \"outro_srt\" variable is missing or file \"${outro_srt}\" does not exist." + fi + ;; + piper_bin) + if [ -z "${piper_bin}" ] + then + echo_error "The \"ep_num\" variable is missing." + fi + ;; + piper_voice) + if [ -z "${ep_num}" ] + then + echo_error "The \"ep_num\" variable is missing." + fi + ;; + processing_dir) + if [[ ! -d "${processing_dir}" || -z "${processing_dir}" ]] + then + echo_error "The \"processing_dir\" variable is missing or is not a directory." + fi + ;; + shownotes_edited) + if [[ ! -s "${shownotes_edited}" || -z "${shownotes_edited}" ]] + then + echo_error "The \"shownotes_edited\" variable/file is missing." + fi +# if [ "$( file --brief --mime-type "${shownotes_edited}" | grep --count 'text/html' )" -ne "1" ] +# then +# echo_error "The \"shownotes_edited\" variable has not a valid \"text/html\" mime type \"${shownotes_edited}\"." +# fi + ;; + shownotes_html) + if [[ ! -s "${shownotes_html}" || -z "${shownotes_html}" ]] + then + echo_error "The \"shownotes_html\" variable/file is missing." + fi +# if [ "$( file --brief --mime-type "${shownotes_html}" | grep --count 'text/html' )" -ne "1" ] +# then +# echo_error "The \"shownotes_html\" variable has not a valid \"text/html\" mime type. \"${shownotes_html}\"" +# fi + ;; + silence) + if [[ ! -d "${silence}" || -z "${silence}" ]] + then + echo_error "The \"silence\" variable is missing or file \"${silence}\" does not exist." + fi + ;; summary) if [[ -z "${summary}" || "${summary}" == "null" ]] then @@ -311,6 +364,18 @@ function check_variable_is_correct() { echo_error "The \"tags\" variable is missing." fi ;; + theme) + if [[ ! -d "${theme}" || -z "${theme}" ]] + then + echo_error "The \"theme\" variable is missing or file \"${theme}\" does not exist." + fi + ;; + timestamp_epoc) + if [ -z "${timestamp_epoc}" ] + then + echo_error "The \"timestamp_epoc\" variable is missing." + fi + ;; title) if [[ -z "${title}" || "${title}" == "null" ]] then @@ -323,12 +388,19 @@ function check_variable_is_correct() { echo_error "The \"track\" variable is missing." fi ;; + working_dir) + if [[ ! -d "${working_dir}" || -z "${working_dir}" ]] + then + echo_error "The \"working_dir\" variable is missing or is not a directory." + fi + ;; year) if [[ -z "${year}" || "${year}" == "null" ]] then echo_error "The \"year\" variable is missing." fi ;; + *) echo_error "An unknown variable \"${argument}\" was provided." ;; @@ -340,9 +412,11 @@ function check_variable_is_correct() { ################################################# # Get the next show in the queue -function get_ep_num_from_hpr_hub() { +function get_next_show_from_hpr_hub() { - echo_debug "Processing the next HPR Show in the queue. get_ep_num_from_hpr_hub()" + echo_debug "Processing the next HPR Show in the queue. get_next_show_from_hpr_hub()" + + check_variable_is_correct processing_dir HOME if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${processing_dir}/status.csv" )" != 200 ] then @@ -354,7 +428,7 @@ function get_ep_num_from_hpr_hub() { echo_error "Failed to retrieve \"${processing_dir}/status.csv\" from server." fi - response=$( cat "${processing_dir}/status.csv" | grep ',SHOW_SUBMITTED,' | head -1 | sed 's/,/ /g' ) + response=$( cat "${processing_dir}/status.csv" | grep ',SHOW_SUBMITTED,' | sort -t ',' -k 2 | head -1 | sed 's/,/ /g' ) if [ -z "${response}" ] then echo_debug "Getting a list of all the reservations." @@ -373,6 +447,8 @@ function get_ep_num_from_hpr_hub() { source_dir="hpr:${hpr_upload_dir}" dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}" working_dir="${processing_dir}/${dest_dir}" + + check_variable_is_correct timestamp_epoc ep_num ep_date key email email_unpadded echo_debug "Downloading hpr${ep_num} from ${email_unpadded}" echo_debug "" @@ -449,7 +525,7 @@ function get_working_dir() { if [ $# -eq 0 ] then - get_ep_num_from_hpr_hub + get_next_show_from_hpr_hub fi check_variable_is_correct working_dir @@ -548,8 +624,6 @@ function get_episode_metadata() { if [[ $argument =~ ^[^=]+=.*$ ]] then this_key="${argument%=*}" - echo "${this_key}" - echo "${episode_variables[*]}" if [ "$( echo "${episode_variables[*]}" | grep --count "${this_key}" )" -eq "0" ] then echo_debug "Skipping replacing \"${this_key}\"."; @@ -1059,7 +1133,6 @@ function post_show_to_hpr_db() { echo_error "The Episode hpr${ep_num} has not been posted" fi - xdg-open https://archive.org/details/hpr${ep_num} } ################################################# @@ -1677,7 +1750,7 @@ function copy_derived_files_to_borg() { # Try and change the online db status to MEDIA_TRANSCODED if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' "https://hub.hackerpublicradio.org/cms/status.php?ep_num=${ep_num}&status=MEDIA_TRANSCODED" )" != 200 ] then - echo_error "Could not change the status of \"${ep_num}\" to \"MEDIA_TRANSCODED\"" + echo_debug "Could not change the status of \"${ep_num}\" to \"MEDIA_TRANSCODED\"" fi # Get the current status @@ -1691,7 +1764,7 @@ function copy_derived_files_to_borg() { if [ -z "${response}" ] then grep --perl-regexp ",${ep_num},.*MEDIA_TRANSCODED," "${processing_dir}/status.csv" - echo_error "The show \"${ep_num}\" hasn't the correct status of \"MEDIA_TRANSCODED\" in the database." + echo_debug "The show \"${ep_num}\" hasn't the correct status of \"MEDIA_TRANSCODED\" in the database." fi echo_debug "The show \"${ep_num}\" has the correct status of \"MEDIA_TRANSCODED\" in the database." @@ -1743,7 +1816,6 @@ function create_item_on_the_internet_archive() { echo_debug "The episode has been posted to the Internet Archive" fi - xdg-open https://archive.org/details/hpr${ep_num} wait_for_the_internet_archive_to_process } @@ -1762,7 +1834,12 @@ function upload_files_to_the_internet_archive() { then files_xml="${working_dir}/hpr${ep_num}_files.xml" fi - + if [ -s "${files_xml}" ] + then + echo_debug "Removing stale \"files_xml\" file \"${files_xml}\"." + rm -v "${files_xml}" + fi + echo_debug "Getting the \"files_xml\" file from the IA." if [ "$( curl --silent --netrc --location --write-out '%{http_code}' https://archive.org/download/hpr${ep_num}/hpr${ep_num}_files.xml --output "${files_xml}" )" != 200 ] then @@ -1776,13 +1853,16 @@ function upload_files_to_the_internet_archive() { files_json="${working_dir}/hpr${ep_num}_files.json" fi - if [ ! -s "${files_json}" ] + if [ -s "${files_json}" ] then - echo_debug "Getting the \"files_json\" file from the HPR site." - if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/assets.php?id=${ep_num} --output "${files_json}" )" != 200 ] - then - echo_error "The Episode hpr${ep_num} has not been posted" - fi + echo_debug "Removing stale \"files_json\" file \"${files_json}\"." + rm -v "${files_json}" + fi + + echo_debug "Getting the \"files_json\" file from the HPR site." + if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/assets.php?id=${ep_num} --output "${files_json}" )" != 200 ] + then + echo_error "The Episode hpr${ep_num} has not been posted" fi check_variable_is_correct files_json @@ -1882,57 +1962,62 @@ fi # - is the show posted etc, shownotes_edited.html # - do the assets match hpr${ep_num}_assets.json # - -# # -# # program_checks # We know that all the programs and variables are set -# # -# # get_working_dir $@ # We have a working directory and a valid json file -# # -# # get_episode_metadata $@ # We have all the metadata we need to process the show. -# # -# # extract_images_brute_force # We have extracted the images by brute force -# # -# # media_checks # -# # -# # generate_initial_report # Generate Initial Report for review by the Janitors -# # -# # manual_shownotes_review # Janitors review audio and shownote. Skips if done. -# # -# # post_show_to_hpr_db # Posts the episode to HPR. Skips if it is already posted. -# # -# # ######################################################################################## -# # # Using DB info from here -# # -# # get_variables_from_episode_summary_json # Get the episode from HPR. -# # -# # create_tts_summary # Generate text to speech summary -# # -# # generate_intro # Generate Intro from the intro theme with overlay of a lead in silence then the tts summary -# # -# # generate_parent_audio # Combines the intro, the episode, and the outro to a final cut. -# # -# # generate_derived_media # Generate the flac wav mp3 ogg opus files -# # -# # generate_show_transcript -# # -# # generate_final_report -# # -# # manual_final_review -# # -# # register_assets -# # -# # copy_files_to_origin_server -# # -# # check_variable_is_correct working_dir ep_num shownotes_edited -# # -# # copy_derived_files_to_borg -# # -# # create_item_on_the_internet_archive + +program_checks # We know that all the programs and variables are set + +get_working_dir $@ # We have a working directory and a valid json file + +get_episode_metadata $@ # We have all the metadata we need to process the show. + +extract_images_brute_force # We have extracted the images by brute force + +media_checks # + +generate_initial_report # Generate Initial Report for review by the Janitors + +manual_shownotes_review # Janitors review audio and shownote. Skips if done. + +post_show_to_hpr_db # Posts the episode to HPR. Skips if it is already posted. + +# ####################################################################################### +# Using DB info from here + +get_variables_from_episode_summary_json # Get the episode from HPR. + +create_tts_summary # Generate text to speech summary + +generate_intro # Generate Intro from the intro theme with overlay of a lead in silence then the tts summary + +generate_parent_audio # Combines the intro, the episode, and the outro to a final cut. + +generate_derived_media # Generate the flac wav mp3 ogg opus files + +generate_show_transcript + +generate_final_report + +manual_final_review + +register_assets + +copy_files_to_origin_server + +check_variable_is_correct working_dir ep_num shownotes_edited + +copy_derived_files_to_borg + +create_item_on_the_internet_archive upload_files_to_the_internet_archive -hpr-check-ccdn-links +hpr-check-ccdn-links ${ep_num} echo_debug "The End" + +xdg-open https://archive.org/details/hpr${ep_num} + +xdg-open https://hackerpublicradio.org/eps/hpr${ep_num}/index.html + exit 0 # if ${force_overwrite}