diff --git a/workflow/duration.bash b/workflow/duration.bash index 91c9580..d66d2fc 100755 --- a/workflow/duration.bash +++ b/workflow/duration.bash @@ -2,7 +2,14 @@ # Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ #============================================================ -find ${HOME}/processing/ -type f | egrep -v '/sponsor-anhonesthost.com-hpr15.flac|/outro.flac|/intro.flac|/sponsor-archive.org.flac' | while read mediafile +search_dir="${HOME}/processing/" + +if [ -d "${1}" ] +then + search_dir="${1}" +fi + +find ${search_dir} -type f | grep -vP '/sponsor-anhonesthost.com-hpr15.flac|/outro.flac|/intro.flac|/sponsor-archive.org.flac' | while read mediafile do duration=$( mediainfo --full --Output=XML "${mediafile}" | xmlstarlet sel -T -t -m "_:MediaInfo/_:media/_:track[@type='Audio']/_:Duration[1]" -v "." -n - | awk -F '.' '{print $1}' ) if [ "${duration}" != "" ] diff --git a/workflow/process_episode.bash b/workflow/process_episode.bash index b7471e6..73b0929 100755 --- a/workflow/process_episode.bash +++ b/workflow/process_episode.bash @@ -20,6 +20,12 @@ intro_srt="${processing_dir}/intro.srt" # The intro subtitle template file piper_bin="/opt/bin/piper/piper/piper" # The location of the text to speech engine piper_voice="/opt/bin/piper/piper/piper-voices/en/en_US/lessac/medium/en_US-lessac-medium.onnx" +working_dir_bypass="false" +skip_post_show="false" +acceptable_duration_difference="2" # Seconds +unsafe_episode_variables=( shownotes_json shownotes_html hostid artist email title summary series_id series_name explicit license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised ) +audio_formats=( flac wav mp3 ogg opus spx ) + ################################################# # Display Error message, display help and exit @@ -39,6 +45,13 @@ function echo_debug() { fi } +################################################# +# Display Information message + +function abs_diff() { + echo $(($1 >= $2 ? $1 - $2 : $2 - $1)) +} + ################################################# # Display Help @@ -88,14 +101,50 @@ function program_checks() { } +################################################# +# Allows override of arguments from the commandline + +function argument_override() { + + echo_debug "Overriding values with terminal arguments. argument_override()" + + + + # Argument Override + if [ $# -gt 0 ] + then + for argument + do + if [[ $argument =~ ^[^=]+=.*$ ]] + then + this_key="${argument%=*}" + this_value="${argument#*=}" + for this_episode_variable in "${unsafe_episode_variables[@]}" + do + if [ "$( echo "${this_episode_variable}" | grep --perl-regexp --count "^${this_key}$" )" -ne "0" ] + then + this_value="$( echo "${this_value}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" + echo_debug "URL encoding \"${this_key}\" as \"${this_value}\"." + fi + done + eval "${this_key}=${this_value}" + echo_debug "Replacing \"${this_key}\" with \"${this_value}\"." + check_variable_is_correct ${this_key} + if [ "${this_key}" == "working_dir" ] + then + working_dir_bypass="true" + set_working_processing_dir + fi + fi + done + fi +} + ################################################# # Variable Checks -#album artist assets_csv assets_json comment date duration duration_iso8601 email email_unpadded ep_date episode_summary_json ep_num explicit files_json files_xml genre HOME hostid intro_srt key license license_url outro_flac outro_srt piper_bin piper_voice processing_dir shownotes_edited shownotes_html silence summary synopsis tags theme timestamp_epoc title track working_dir year - function check_variable_is_correct() { - for argument in "$@" do case "${argument}" in @@ -296,14 +345,20 @@ function check_variable_is_correct() { echo_error "The \"media\" variable is missing." fi ;; + media_basename) + if [[ ! -d "${media_basename}" || -z "${media_basename}" ]] + then + echo_error "The \"media_basename\" variable is missing or is not a directory." + fi + ;; outro_flac) - if [[ ! -d "${outro_flac}" || -z "${outro_flac}" ]] + if [[ ! -s "${outro_flac}" || -z "${outro_flac}" ]] then echo_error "The \"outro_flac\" variable is missing or file \"${outro_flac}\" does not exist." fi ;; outro_srt) - if [[ ! -d "${outro_srt}" || -z "${outro_srt}" ]] + if [[ ! -s "${outro_srt}" || -z "${outro_srt}" ]] then echo_error "The \"outro_srt\" variable is missing or file \"${outro_srt}\" does not exist." fi @@ -346,6 +401,21 @@ function check_variable_is_correct() { # echo_error "The \"shownotes_html\" variable has not a valid \"text/html\" mime type. \"${shownotes_html}\"" # fi ;; + shownotes_json) + if [[ ! -s "${shownotes_json}" || -z "${shownotes_json}" ]] + then + echo_error "The \"shownotes_json\" variable/file is missing." + fi + if [ "$( file --brief --mime-type "${shownotes_json}" | grep --count 'application/json' )" -ne "1" ] + then + echo_error "The \"shownotes_json\" variable has not a valid \"application/json\" mime type." + fi + jq '.' "${shownotes_json}" >/dev/null 2>&1 + if [ $? -ne 0 ] + then + echo_error "The file \"${shownotes_json}\" is not valid json." + fi + ;; silence) if [[ ! -d "${silence}" || -z "${silence}" ]] then @@ -371,7 +441,7 @@ function check_variable_is_correct() { fi ;; theme) - if [[ ! -d "${theme}" || -z "${theme}" ]] + if [[ ! -s "${theme}" || -z "${theme}" ]] then echo_error "The \"theme\" variable is missing or file \"${theme}\" does not exist." fi @@ -400,6 +470,12 @@ function check_variable_is_correct() { echo_error "The \"working_dir\" variable is missing or is not a directory." fi ;; + working_processing_dir) + if [[ ! -d "${working_processing_dir}" || -z "${working_processing_dir}" ]] + then + echo_error "The \"working_processing_dir\" variable is missing or is not a directory." + fi + ;; year) if [[ -z "${year}" || "${year}" == "null" ]] then @@ -549,15 +625,25 @@ function get_working_dir() { echo_debug "Found working directory as \"${working_dir}\"" - if [ ! -d "${working_dir}/processing/" ] - then - mkdir -v "${working_dir}/processing/" - if [ ! -d "${working_dir}/processing/" ] - then - echo_error "Could not create the processing directory \"${working_dir}/processing/\"." - fi - fi +} +################################################# +# Create a sub dir to store technical data. + +function set_working_processing_dir() { + + echo_debug "Checking the processing directory in the current working directory. set_working_processing_dir()" + + check_variable_is_correct working_dir + + working_processing_dir="${working_dir}/processing" + + if [ ! -d "${working_processing_dir}/" ] + then + mkdir -v "${working_processing_dir}/" + fi + + check_variable_is_correct working_processing_dir } ################################################# @@ -566,6 +652,14 @@ function get_working_dir() { function get_episode_metadata() { echo_debug "Extracting the episode metadata. get_episode_metadata()" + + if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]] + then + echo_debug "The Episode hpr${ep_num} has already been posted. Skipping get_episode_metadata()" + return + fi + + check_variable_is_correct working_dir if [[ -s "${working_dir}/shownotes.json" && "$( file --brief --mime-type "${working_dir}/shownotes.json" | grep --count "application/json" )" -eq 0 ]] then @@ -575,6 +669,8 @@ function get_episode_metadata() { shownotes_json="${working_dir}/shownotes.json" shownotes_html="${working_dir}/shownotes.html" shownotes_edited="${working_dir}/shownotes_edited.html" + + check_variable_is_correct shownotes_json hostid="$( jq --raw-output '.host.Host_ID' ${shownotes_json} )" artist="$( jq --raw-output '.host.Host_Name' ${shownotes_json} )" @@ -598,11 +694,8 @@ function get_episode_metadata() { echo_debug "Extracting shownotes html from json file." jq --raw-output '.episode.Show_Notes' "${shownotes_json}" > "${shownotes_html}" - - if [ ! -s "${shownotes_html}" ] - then - echo_error "Failed to extract the shownote html file \"${shownotes_html}\"" - fi + + check_variable_is_correct shownotes_html ( echo '' cat "${shownotes_html}" @@ -610,9 +703,9 @@ function get_episode_metadata() { ' ) | sponge "${shownotes_html}" - episode_variables=( shownotes_json shownotes_html hostid artist email title summary series_id series_name explicit license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised ) + unsafe_episode_variables=( shownotes_json shownotes_html hostid artist email title summary series_id series_name explicit license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised ) - for episode_variable in "${episode_variables[@]}" + for episode_variable in "${unsafe_episode_variables[@]}" do if [[ -z ${!episode_variable} && "${episode_variable}" != "remote_media" ]] then # indirect expansion here @@ -622,26 +715,7 @@ function get_episode_metadata() { fi done - # Argument Override - if [ $# -gt 0 ] - then - for argument - do - if [[ $argument =~ ^[^=]+=.*$ ]] - then - this_key="${argument%=*}" - if [ "$( echo "${episode_variables[*]}" | grep --count "${this_key}" )" -eq "0" ] - then - echo_debug "Skipping replacing \"${this_key}\"."; - continue - fi - this_value="${argument#*=}" - this_value="$( echo "${this_value}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" - eval "${this_key}=${this_value}" - echo_debug "Replacing \"${this_key}\" with \"${this_value}\"." - fi - done - fi + argument_override "$@" # Hosts need to exist in the database if [ "${hostid}" == '0' ] @@ -657,6 +731,12 @@ function get_episode_metadata() { function extract_images_brute_force() { echo_debug "Extracting images with grep. extract_images_brute_force()" + + if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]] + then + echo_debug "The Episode hpr${ep_num} has already been posted. Skipping extract_images_brute_force()" + return + fi if [ -s "${shownotes_edited}" ] then @@ -803,6 +883,8 @@ function extract_images_brute_force() { function media_checks() { echo_debug "Running media checks. media_checks()" + + check_variable_is_correct working_dir working_processing_dir if [[ -n "${remote_media}" && "${remote_media}" != "null" ]] then @@ -849,6 +931,8 @@ function media_checks() { then echo_error "Could not find the subtitles for media \"${media}\"" fi + + #TODO fix close duration # Find duration duration=$( mediainfo --full --Output=JSON "${media}" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' ) @@ -857,6 +941,10 @@ function media_checks() { echo_error "Invalid duration found in \"${media}\"" >&2 fi echo_debug "The Duration is \"${duration}\" seconds from \"${media}\"" + + + + #TODO fix close duration # Find number of channels ( 1=mono or 2=stereo) supplied_channels=$( mediainfo --full --Output=JSON "${media}" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Channels' ) @@ -867,20 +955,20 @@ function media_checks() { echo_debug "The number of audio channels is \"${supplied_channels}\" from \"${media}\" ." # Gernerate the Spectrum and Waveform image - ffmpeg -hide_banner -loglevel error -y -i "${media}" -lavfi "showspectrumpic=s=960x540" "${working_dir}/processing/${media_basename%.*}_spectrum.png" - audio2image.bash "${media}" && mv -v "${media%.*}.png" "${working_dir}/processing/${media_basename%.*}_waveform.png" + ffmpeg -hide_banner -loglevel error -y -i "${media}" -lavfi "showspectrumpic=s=960x540" "${working_processing_dir}/${media_basename%.*}_spectrum.png" + audio2image.bash "${media}" && mv -v "${media%.*}.png" "${working_processing_dir}/${media_basename%.*}_waveform.png" # Getting metadata - mediainfo "${media}" > "${working_dir}/processing/${media_basename%.*}_mediainfo.txt" + mediainfo "${media}" > "${working_processing_dir}/${media_basename%.*}_mediainfo.txt" - exiftool "${media}" > "${working_dir}/processing/${media_basename%.*}_exiftool.txt" + exiftool "${media}" > "${working_processing_dir}/${media_basename%.*}_exiftool.txt" for check_file in spectrum.png waveform.png mediainfo.txt exiftool.txt do - if [ ! -s "${working_dir}/processing/${media_basename%.*}_${check_file}" ] + if [ ! -s "${working_processing_dir}/${media_basename%.*}_${check_file}" ] then - echo_error "The ${check_file} file was not generated for the \"${working_dir}/processing/${media_basename%.*}_${check_file}\"" >&2 + echo_error "The ${check_file} file was not generated for the \"${working_processing_dir}/${media_basename%.*}_${check_file}\"" >&2 fi done @@ -902,6 +990,13 @@ function media_checks() { function generate_initial_report() { echo_debug "Generating the initial report. generate_initial_report()" + + if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]] + then + echo_debug "The Episode hpr${ep_num} has already been posted. Skipping generate_initial_report()" + return + fi + # TODO list the images. @@ -964,25 +1059,25 @@ ${shownotes_json_sanatised}

mediainfo report

-$( cat "${working_dir}/processing/${media_basename%.*}_mediainfo.txt" )
+$( cat "${working_processing_dir}/${media_basename%.*}_mediainfo.txt" )

exiftool report

-$( cat "${working_dir}/processing/${media_basename%.*}_exiftool.txt" )
+$( cat "${working_processing_dir}/${media_basename%.*}_exiftool.txt" )

Audio Spectrum

-\"Spectrum\" +\"Spectrum\"

Audio Waveform

-\"Waveform\" +\"Waveform\" 

@@ -1006,7 +1101,7 @@ $(cat "${shownotes_srt}" )
-" > "${working_dir}/processing/${media_basename%.*}_media_report.html" +" > "${working_processing_dir}/${media_basename%.*}_media_report.html" } @@ -1016,11 +1111,17 @@ $(cat "${shownotes_srt}" ) function manual_shownotes_review() { echo_debug "Validating the initial report. manual_shownotes_review()" + + if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]] + then + echo_debug "The Episode hpr${ep_num} has already been posted. Skipping manual_shownotes_review()" + return + fi - if [[ -z "${shownotes_html}" || ! -s "${shownotes_html}" || ! -s "${working_dir}/processing/${media_basename%.*}_media_report.html" ]] + if [[ -z "${shownotes_html}" || ! -s "${shownotes_html}" || ! -s "${working_processing_dir}/${media_basename%.*}_media_report.html" ]] then echo "shownotes_html: ${shownotes_html}" - ls -al "${shownotes_html}" "${working_dir}/processing/${media_basename%.*}_media_report.html" + ls -al "${shownotes_html}" "${working_processing_dir}/${media_basename%.*}_media_report.html" echo_error "The files needed for to generate the inital report information are not available." fi @@ -1038,7 +1139,7 @@ function manual_shownotes_review() { fi kate "${shownotes_edited}" >/dev/null 2>&1 & - librewolf "${working_dir}/processing/${media_basename%.*}_media_report.html" >/dev/null 2>&1 & + librewolf "${working_processing_dir}/${media_basename%.*}_media_report.html" >/dev/null 2>&1 & seamonkey "${shownotes_edited}" >/dev/null 2>&1 & # # # # bluefish "${shownotes_edited}" >/dev/null 2>&1 & # https://markdowntohtml.com/ @@ -1070,9 +1171,9 @@ function post_show_to_hpr_db() { echo_debug "Posting the show to the HPR DB. post_show_to_hpr_db()" - if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output /dev/null )" == 200 ] + if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]] then - echo "WARN: The Episode hpr${ep_num} has already been posted" + echo_debug "The Episode hpr${ep_num} has already been posted. Skipping post_show_to_hpr_db()" return fi @@ -1176,6 +1277,8 @@ function get_variables_from_episode_summary_json() { echo_debug "Setting \"${episode_summary_key}\" to \"${episode_summary_value}\" from \"$( basename ${episode_summary_json} )\"" check_variable_is_correct ${episode_summary_key} done + + argument_override "$@" duration_iso8601="$( \date -d@${duration} -u +%H:%M:%S )" check_variable_is_correct duration_iso8601 @@ -1189,15 +1292,15 @@ function create_tts_summary() { echo_debug "Creating Text to Speech summary. create_tts_summary()" - check_variable_is_correct working_dir duration synopsis piper_bin piper_voice + check_variable_is_correct working_dir duration synopsis piper_bin piper_voice working_processing_dir echo_debug "Converting text synopsis \"${synopsis}\" to speech." - echo "${synopsis}" | "${piper_bin}" --model "${piper_voice}" --output_file "${working_dir}/processing/episode_tts.wav" + echo "${synopsis}" | "${piper_bin}" --model "${piper_voice}" --output_file "${working_processing_dir}/episode_tts.wav" - if [ ! -s "${working_dir}/processing/episode_tts.wav" ] + if [ ! -s "${working_processing_dir}/episode_tts.wav" ] then - echo_error "The text to speech episode summary was not created \"${working_dir}/processing/episode_tts.wav\"." + echo_error "The text to speech episode summary was not created \"${working_processing_dir}/episode_tts.wav\"." fi } @@ -1208,26 +1311,22 @@ function create_tts_summary() { function generate_intro() { echo_debug "Generating the intro. generate_intro()" - - if [[ ! -s "${working_dir}/processing/episode_tts.wav" || ! -s "${theme}" || ! -s "${media}" || ! -s "${outro_flac}" || ! -d "${working_dir}/processing/" ]] - then - echo_error "The files for the intro are not available." - ls -al "${working_dir}/processing/episode_tts.wav" "${theme}" "${media}" "${outro_flac}" "${working_dir}/processing/" - fi + + check_variable_is_correct working_dir working_processing_dir theme media outro_flac # Everything needs to be in the same format for the intro, 1 channel (mono) Sampling rate 44.1 kHz - ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/processing/episode_tts.wav" -ar 44100 -ac 1 "${working_dir}/processing//episode_tts.flac" + ffmpeg -hide_banner -loglevel error -y -i "${working_processing_dir}/episode_tts.wav" -ar 44100 -ac 1 "${working_processing_dir}//episode_tts.flac" # A level of silence is added at the beginning of the text to speech - sox -V2 "${silence}" "${working_dir}/processing//episode_tts.flac" "${working_dir}/processing/episode_summary.flac" + sox -V2 "${silence}" "${working_processing_dir}//episode_tts.flac" "${working_processing_dir}/episode_summary.flac" # The tracks are merged together resulting in the theme playing first, then after a period of silence the text to speech enters - sox -V2 -m "${working_dir}/processing/episode_summary.flac" "${theme}" "${working_dir}/processing/episode_intro.flac" + sox -V2 -m "${working_processing_dir}/episode_summary.flac" "${theme}" "${working_processing_dir}/episode_intro.flac" - if [[ ! -s "${working_dir}/processing//episode_tts.flac" || ! -s "${working_dir}/processing/episode_summary.flac" || ! -s "${working_dir}/processing/episode_intro.flac" ]] + if [[ ! -s "${working_processing_dir}//episode_tts.flac" || ! -s "${working_processing_dir}/episode_summary.flac" || ! -s "${working_processing_dir}/episode_intro.flac" ]] then echo_error "The files for the theme audio sandwich are not available." - ls -al "${working_dir}/processing//episode_tts.flac" "${working_dir}/processing/episode_summary.flac" "${theme}" "${working_dir}/processing/episode_intro.flac" + ls -al "${working_processing_dir}//episode_tts.flac" "${working_processing_dir}/episode_summary.flac" "${theme}" "${working_processing_dir}/episode_intro.flac" fi } @@ -1239,20 +1338,20 @@ function generate_parent_audio() { echo_debug "Generating the parent audio - the sandwitch. generate_parent_audio()" - if [[ ! -s "${working_dir}/processing/episode_intro.flac" || ! -s "${media}" || ! -s "${outro_flac}" ]] + if [[ ! -s "${working_processing_dir}/episode_intro.flac" || ! -s "${media}" || ! -s "${outro_flac}" ]] then echo_error "The files for the sandwich are not available." ls -al fi # Everything needs to be in the same format so the text to speech needs to be converted to 2 channel Sampling rate 44.1 kHz - ffmpeg -hide_banner -loglevel error -y -i "${media}" -ar 44100 -ac 1 "${working_dir}/processing/episode_body.flac" + ffmpeg -hide_banner -loglevel error -y -i "${media}" -ar 44100 -ac 1 "${working_processing_dir}/episode_body.flac" # Combine the components together - sox -V2 "${working_dir}/processing/episode_intro.flac" "${working_dir}/processing/episode_body.flac" "${outro_flac}" "${working_dir}/processing/episode_sandwitch.flac" + sox -V2 "${working_processing_dir}/episode_intro.flac" "${working_processing_dir}/episode_body.flac" "${outro_flac}" "${working_processing_dir}/episode_sandwitch.flac" # Normalise the audio - ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/processing/episode_sandwitch.flac" -af loudnorm=I=-16:LRA=11:TP=-1.5 "${working_dir}/processing/episode_final.flac" + ffmpeg -hide_banner -loglevel error -y -i "${working_processing_dir}/episode_sandwitch.flac" -af loudnorm=I=-16:LRA=11:TP=-1.5 "${working_processing_dir}/episode_final.flac" } @@ -1265,23 +1364,20 @@ function generate_derived_media() { check_variable_is_correct media working_dir ep_num title artist license - if [[ ! -s "${working_dir}/processing/episode_final.flac" ]] + if [[ ! -s "${working_processing_dir}/episode_final.flac" ]] then ls -al echo_error "The final cut is not available." fi - - comment="$( jq --raw-output '.comment' "${episode_summary_json}" )" - year="$( echo "${ep_date}" | cut -c -4 )" check_variable_is_correct comment year # https://wiki.multimedia.cx/index.php?title=FFmpeg_Metadata - for extension in flac wav mp3 ogg opus + for extension in flac wav mp3 ogg opus spx do echo_debug "Generating \"hpr${ep_num}.${extension}\"." - ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/processing/episode_final.flac" \ + ffmpeg -hide_banner -loglevel error -y -i "${working_processing_dir}/episode_final.flac" \ -metadata title="${title}" \ -metadata artist="${artist}" \ -metadata author="${artist}" \ @@ -1304,6 +1400,7 @@ function generate_derived_media() { done + #TODO fix close duration lengths=$( for extension in flac wav mp3 ogg opus do mediainfo --full --Output=XML "${working_dir}/hpr${ep_num}.${extension}" | xmlstarlet sel -T -t -m "_:MediaInfo/_:media/_:track[@type='Audio']/_:Duration[1]" -v "." -n - | awk -F '.' '{print $1}' @@ -1312,7 +1409,7 @@ function generate_derived_media() { then echo_error "The duration of the derived media is not correct." fi - +#TODO fix close duration cp -v "${media}" "${working_dir}/hpr${ep_num}_source.${media##*.}" if [[ ! -s "${working_dir}/hpr${ep_num}_source.${media##*.}" ]] @@ -1332,9 +1429,9 @@ function generate_show_transcript() { # TODO Currently processed elsewhere by hpr-get-and-transcode.bash and uploaded to hpr:upload/ to be synced with media above - if [[ ! -s "${media}" || ! -s "${media%.*}.srt" || ! -s "${intro_srt}" || ! -s "${outro_srt}" || ! -s "${working_dir}/processing/episode_intro.flac" || ! -s "${working_dir}/processing/episode_body.flac" ]] + if [[ ! -s "${media}" || ! -s "${media%.*}.srt" || ! -s "${intro_srt}" || ! -s "${outro_srt}" || ! -s "${working_processing_dir}/episode_intro.flac" || ! -s "${working_processing_dir}/episode_body.flac" ]] then - ls -al "${media}" "${media%.*}.srt" "${intro_srt}" "${outro_srt}" "${working_dir}/processing/episode_intro.flac" "${working_dir}/processing/episode_body.flac" + ls -al "${media}" "${media%.*}.srt" "${intro_srt}" "${outro_srt}" "${working_processing_dir}/episode_intro.flac" "${working_processing_dir}/episode_body.flac" echo_error "The transcriptions files are not available." fi @@ -1361,32 +1458,32 @@ function generate_show_transcript() { REPLACE_LINE_4="The flag is ${explicit}, and the license is ${license}" REPLACE_LINE_5="The summary is \"${summary}\"" - cp -v ${intro_srt} "${working_dir}/processing/episode_intro.srt" - cp -v ${outro_srt} "${working_dir}/processing/episode_outro.srt" + cp -v ${intro_srt} "${working_processing_dir}/episode_intro.srt" + cp -v ${outro_srt} "${working_processing_dir}/episode_outro.srt" - sed -e "s~REPLACE_LINE_1~${REPLACE_LINE_1}~g" -e "s~REPLACE_LINE_2~${REPLACE_LINE_2}~g" -e "s~REPLACE_LINE_3~${REPLACE_LINE_3}~g" -e "s~REPLACE_LINE_4~${REPLACE_LINE_4}~g" -e "s~REPLACE_LINE_5~${REPLACE_LINE_5}~g" -i "${working_dir}/processing/episode_intro.srt" + sed -e "s~REPLACE_LINE_1~${REPLACE_LINE_1}~g" -e "s~REPLACE_LINE_2~${REPLACE_LINE_2}~g" -e "s~REPLACE_LINE_3~${REPLACE_LINE_3}~g" -e "s~REPLACE_LINE_4~${REPLACE_LINE_4}~g" -e "s~REPLACE_LINE_5~${REPLACE_LINE_5}~g" -i "${working_processing_dir}/episode_intro.srt" - if [ "$( grep --count REPLACE_LINE "${working_dir}/processing/episode_intro.srt" )" -ne "0" ] + if [ "$( grep --count REPLACE_LINE "${working_processing_dir}/episode_intro.srt" )" -ne "0" ] then - echo_error "The intro subtitles were not correctly generated \"${working_dir}/processing/episode_intro.srt\"." + echo_error "The intro subtitles were not correctly generated \"${working_processing_dir}/episode_intro.srt\"." fi # Time shift the media subtitles on by the duration of the intro wav file # https://trac.ffmpeg.org/wiki/UnderstandingItsoffset - itsoffset_intro="$( mediainfo --full --Output=JSON "${working_dir}/processing/episode_intro.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )" + itsoffset_intro="$( mediainfo --full --Output=JSON "${working_processing_dir}/episode_intro.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )" if [[ -z "${itsoffset_intro}" || "${itsoffset_intro}" == "null" ]] then echo_error "Could not retrieve the itsoffset_intro to correct the timing of the subtitles." fi - ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_intro}" -i "${media%.*}.srt" -c copy "${working_dir}/processing/episode_body.srt" + ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_intro}" -i "${media%.*}.srt" -c copy "${working_processing_dir}/episode_body.srt" # Timeshift the outro by the duration of the intro and the supplied media - itsoffset_body="$( mediainfo --full --Output=JSON "${working_dir}/processing/episode_body.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )" + itsoffset_body="$( mediainfo --full --Output=JSON "${working_processing_dir}/episode_body.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )" if [[ -z "${itsoffset_body}" || "${itsoffset_body}" == "null" ]] then @@ -1395,18 +1492,18 @@ function generate_show_transcript() { itsoffset_body=$((itsoffset_intro + $itsoffset_body)) - ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_body}" -i "${working_dir}/processing/episode_outro.srt" -c copy "${working_dir}/processing/episode_outro_shifted.srt" + ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_body}" -i "${working_processing_dir}/episode_outro.srt" -c copy "${working_processing_dir}/episode_outro_shifted.srt" # Combine the intro, timeshifted media subtitles, and the timeshifted outro subtitles. - cat "${working_dir}/processing/episode_intro.srt" "${working_dir}/processing/episode_body.srt" "${working_dir}/processing/episode_outro_shifted.srt" > "${working_dir}/processing/episode.srt" + cat "${working_processing_dir}/episode_intro.srt" "${working_processing_dir}/episode_body.srt" "${working_processing_dir}/episode_outro_shifted.srt" > "${working_processing_dir}/episode.srt" # Parse the resulting subtitle file fixing the numberic counter # https://en.wikipedia.org/wiki/SubRip count=1 - cat "${working_dir}/processing/episode.srt" | while read this_line + cat "${working_processing_dir}/episode.srt" | while read this_line do if [ "$( echo "${this_line}" | grep -c --perl-regexp '^[0-9]+$' )" -eq "1" ] then @@ -1435,8 +1532,16 @@ function generate_show_transcript() { function generate_final_report() { echo_debug "Generating the final report. generate_final_report()" + + if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]] + then + echo_debug "The Episode hpr${ep_num} has already been posted. Skipping generate_final_report()" + return + fi + + check_variable_is_correct working_dir working_processing_dir ep_num media_basename shownotes_edited synopsis - final_report="${working_dir}/processing/hpr${ep_num}_report.html" + final_report="${working_processing_dir}/hpr${ep_num}_report.html" for this_file_extension_to_check in flac mp3 ogg opus srt txt wav do @@ -1447,13 +1552,13 @@ function generate_final_report() { fi done - if [[ ! -s "${working_dir}/processing/${media_basename%.*}_media_report.html" ]] + if [[ ! -s "${working_processing_dir}/${media_basename%.*}_media_report.html" ]] then - ls -al "${working_dir}/processing/${media_basename%.*}_media_report.html" - echo_error "The initial report is not available.\"${working_dir}/processing/${media_basename%.*}_media_report.html\"" + ls -al "${working_processing_dir}/${media_basename%.*}_media_report.html" + echo_error "The initial report is not available.\"${working_processing_dir}/${media_basename%.*}_media_report.html\"" fi - grep -Pv '|' "${working_dir}/processing/${media_basename%.*}_media_report.html" > "${final_report}" + grep -Pv '|' "${working_processing_dir}/${media_basename%.*}_media_report.html" > "${final_report}" echo "

Text To Speech

@@ -1461,33 +1566,33 @@ $( echo "${synopsis}" )


-${working_dir}/processing//episode_tts.flac +${working_processing_dir}//episode_tts.flac

" >> "${final_report}" for this_file_extension_to_check in flac mp3 ogg opus wav do - ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" -lavfi "showspectrumpic=s=960x540" "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png" + ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" -lavfi "showspectrumpic=s=960x540" "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png" - audio2image.bash "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" && mv -v "${working_dir}/hpr${ep_num}.png" "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_waveform.png" + audio2image.bash "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" && mv -v "${working_dir}/hpr${ep_num}.png" "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_waveform.png" - mediainfo "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" + mediainfo "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" - exiftool "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" + exiftool "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" - ffprobe "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" 2>&1 | grep Audio: | sed 's/^.\s//g' > "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" - file --brief --mime "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" >> "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" + ffprobe "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" 2>&1 | grep Audio: | sed 's/^.\s//g' > "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" + file --brief --mime "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" >> "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" this_file_mime_type="$( file --brief --mime-type "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" )" for this_file_to_check in spectrum.png waveform.png mediainfo.txt exiftool.txt ffprobe.txt do - if [[ ! -s "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" ]] + if [[ ! -s "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" ]] then - ls -al "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" + ls -al "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" echo_error "The inital report information is missing \"${this_file_to_check}\"." fi done @@ -1497,29 +1602,29 @@ $( echo "${synopsis}" )

mediainfo report

-$( cat "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" )
+$( cat "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" )

exiftool report

-$( cat "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" )
+$( cat "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" )

Audio Spectrum

-\"Spectrum\" +\"Spectrum\"

Audio Waveform

-\"Waveform\" +\"Waveform\" 

-$( cat "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" )
+$( cat "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" )

@@ -1571,6 +1676,12 @@ $(cat "${working_dir}/hpr${ep_num}.txt" ) function manual_final_review() { echo_debug "Validating the final report. manual_final_review()" + + if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]] + then + echo_debug "The Episode hpr${ep_num} has already been posted. Skipping generate_final_report()" + return + fi if [[ -z "${final_report}" || ! -s "${final_report}" ]] then @@ -1719,6 +1830,7 @@ function copy_files_to_origin_server() { ################################################# # Send the derived files to the server borg to be sent to borg + function copy_derived_files_to_borg() { echo_debug "Sending the derived files to the admin server borg. copy_derived_files_to_borg()" @@ -1938,12 +2050,6 @@ function hide_me() { echo "hidden" -} - -################################################# -# Main exceution starts here -# -# This tool will process the HPR shows allowing the janitor to review the media and fix shownotes. # # TODO Add support for reserve queue - process validate and move to reserve dir # TODO Add support to reprocess processed shows - when given onlay new media reprocess it, update duration on hub, generate and download shownotes.{html,json} from db @@ -1955,27 +2061,6 @@ echo "hidden" # TODO Add chapter support # TODO incorporate direct upload to the IA # TODO copy the files to the backup disk - -# Get supplied working dir and ep_num if provided -if [ $# -gt 0 ] -then -# declare -A hash - for argument - do - if [[ $argument =~ ^[^=]+=.*$ ]] - then - key="${argument%=*}" - value="${argument#*=}" - eval "${key}=${value}" - echo_debug "Using supplied \"${key}\" of \"${value}\"" - check_variable_is_correct ${key} - fi - done -fi - -######################################################################################## -# Posting show - #TODO add logic to check where we are and what we need to do. # All functions should skip if their work is done. Except when mode=refresh # @@ -1986,9 +2071,53 @@ fi # - do the assets match hpr${ep_num}_assets.json # - +# # # # Get supplied working dir and ep_num if provided +# # # if [ $# -gt 0 ] +# # # then +# # # # declare -A hash +# # # for argument +# # # do +# # # if [[ $argument =~ ^[^=]+=.*$ ]] +# # # then +# # # key="${argument%=*}" +# # # value="${argument#*=}" +# # # eval "${key}=${value}" +# # # echo_debug "Using supplied \"${key}\" of \"${value}\"" +# # # check_variable_is_correct ${key} +# # # fi +# # # done +# # # fi + +} + +################################################# +# Main exceution starts here +# +# This tool will process the HPR shows allowing the janitor to review the media and fix shownotes. + + +argument_override "$@" + +######################################################################################## +# Posting show + + program_checks # We know that all the programs and variables are set -get_working_dir $@ # We have a working directory and a valid json file +if [[ -z "${working_dir_bypass}" || "${working_dir_bypass}" != "true" ]] +then + get_working_dir $@ # We have a working directory and a valid json file +else + echo_debug "Skipping get_working_dir()" +fi + +set_working_processing_dir + +if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output /dev/null )" == 200 ] +then + skip_post_show="true" + echo_debug "The Episode hpr${ep_num} has already been posted." +fi get_episode_metadata $@ # We have all the metadata we need to process the show. @@ -2005,7 +2134,7 @@ post_show_to_hpr_db # Posts the episode to HPR. Skips if it is already p # ####################################################################################### # Using DB info from here -get_variables_from_episode_summary_json # Get the episode from HPR. +get_variables_from_episode_summary_json "$@" # Get the episode from HPR. create_tts_summary # Generate text to speech summary