diff --git a/workflow/process_episode.bash b/workflow/process_episode.bash index 73b0929..e77aa45 100755 --- a/workflow/process_episode.bash +++ b/workflow/process_episode.bash @@ -22,9 +22,12 @@ piper_voice="/opt/bin/piper/piper/piper-voices/en/en_US/lessac/medium/en_US-less working_dir_bypass="false" skip_post_show="false" +skip_media_encoding="false" + acceptable_duration_difference="2" # Seconds unsafe_episode_variables=( shownotes_json shownotes_html hostid artist email title summary series_id series_name explicit license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised ) audio_formats=( flac wav mp3 ogg opus spx ) +episode_formats=( flac wav mp3 ogg opus spx srt txt ) ################################################# # Display Error message, display help and exit @@ -133,7 +136,6 @@ function argument_override() { if [ "${this_key}" == "working_dir" ] then working_dir_bypass="true" - set_working_processing_dir fi fi done @@ -174,6 +176,10 @@ function check_variable_is_correct() { then echo_error "The \"assets_csv\" variable has not a valid \"text/csv\" mime type." fi + if [ "$( wc --lines ${assets_csv} | awk '{print $1}' )" -le "1" ] + then + echo_error "The \"${assets_csv}\" file is empty." + fi ;; assets_json) if [[ ! -s "${assets_json}" || -z "${assets_json}" ]] @@ -230,6 +236,72 @@ function check_variable_is_correct() { echo_error "The \"email_unpadded\" variable is missing." fi ;; + episode_body_flac) + if [ -z "${episode_body_flac}" ] + then + echo_error "The \"episode_body_flac\" variable is missing." + fi + ;; + episode_body_srt) + if [ -z "${episode_body_srt}" ] + then + echo_error "The \"episode_body_srt\" variable is missing." + fi + ;; + episode_final_flac) + if [ -z "${episode_final_flac}" ] + then + echo_error "The \"episode_final_flac\" variable is missing." + fi + ;; + episode_intro_flac) + if [ -z "${episode_intro_flac}" ] + then + echo_error "The \"episode_intro_flac\" variable is missing." + fi + ;; + episode_intro_srt) + if [ -z "${episode_intro_srt}" ] + then + echo_error "The \"episode_intro_srt\" variable is missing." + fi + ;; + episode_outro_srt) + if [ -z "${episode_outro_srt}" ] + then + echo_error "The \"episode_outro_srt\" variable is missing." + fi + ;; + episode_sandwitch_flac) + if [ -z "${episode_sandwitch_flac}" ] + then + echo_error "The \"episode_sandwitch_flac\" variable is missing." + fi + ;; + episode_srt) + if [ -z "${episode_srt}" ] + then + echo_error "The \"episode_srt\" variable is missing." + fi + ;; + episode_summary_flac) + if [ -z "${episode_summary_flac}" ] + then + echo_error "The \"episode_summary_flac\" variable is missing." + fi + ;; + episode_tts_flac) + if [ -z "${episode_tts_flac}" ] + then + echo_error "The \"episode_tts_flac\" variable is missing." + fi + ;; + episode_tts_wav) + if [ -z "${episode_tts_wav}" ] + then + echo_error "The \"episode_tts_wav\" variable is missing." + fi + ;; episode_summary_json) if [[ ! -s "${episode_summary_json}" || -z "${episode_summary_json}" ]] then @@ -261,6 +333,12 @@ function check_variable_is_correct() { echo_error "The \"ep_num\" variable is not a valid number between 1 and 9999." fi ;; + expected_duration) + if [ -z "${expected_duration}" ] + then + echo_error "The \"expected_duration\" variable is missing." + fi + ;; explicit) if [[ -z "${explicit}" || "${explicit}" == "null" ]] then @@ -315,6 +393,12 @@ function check_variable_is_correct() { echo_error "The \"hostid\" variable is missing." fi ;; + intro_duration) + if [ -z "${intro_duration}" ] + then + echo_error "The \"intro_duration\" variable is missing." + fi + ;; intro_srt) if [[ ! -d "${intro_srt}" || -z "${intro_srt}" ]] then @@ -346,11 +430,35 @@ function check_variable_is_correct() { fi ;; media_basename) - if [[ ! -d "${media_basename}" || -z "${media_basename}" ]] + if [[ ! -s "${media_basename}" || -z "${media_basename}" ]] then echo_error "The \"media_basename\" variable is missing or is not a directory." fi ;; + media_ffprobe) + if [ -z "${media_ffprobe}" ] + then + echo_error "The \"media_ffprobe\" variable is missing." + fi + ;; + media_file_mime) + if [ -z "${media_file_mime}" ] + then + echo_error "The \"media_file_mime\" variable is missing." + fi + ;; + media_file_mime_type) + if [ -z "${media_file_mime_type}" ] + then + echo_error "The \"media_file_mime_type\" variable is missing." + fi + ;; + outro_duration) + if [ -z "${outro_duration}" ] + then + echo_error "The \"outro_duration\" variable is missing." + fi + ;; outro_flac) if [[ ! -s "${outro_flac}" || -z "${outro_flac}" ]] then @@ -422,6 +530,12 @@ function check_variable_is_correct() { echo_error "The \"silence\" variable is missing or file \"${silence}\" does not exist." fi ;; + source_duration) + if [ -z "${source_duration}" ] + then + echo_error "The \"source_duration\" variable is missing." + fi + ;; summary) if [[ -z "${summary}" || "${summary}" == "null" ]] then @@ -470,12 +584,6 @@ function check_variable_is_correct() { echo_error "The \"working_dir\" variable is missing or is not a directory." fi ;; - working_processing_dir) - if [[ ! -d "${working_processing_dir}" || -z "${working_processing_dir}" ]] - then - echo_error "The \"working_processing_dir\" variable is missing or is not a directory." - fi - ;; year) if [[ -z "${year}" || "${year}" == "null" ]] then @@ -538,16 +646,7 @@ function get_next_show_from_hpr_hub() { ssh hpr -t "detox -v ${hpr_upload_dir}/" rsync -ave ssh --partial --progress ${source_dir}/ ${working_dir}/ - - if [ ! -s "${working_dir}/shownotes.json" ] - then - echo_error "The working dir is missing the shownotes file \"${working_dir}/shownotes.json\"" - fi - - if [ "$( file --brief --mime-type "${working_dir}/shownotes.json" | grep --count "application/json" )" -eq 0 ] - then - echo_error "The \"${working_dir}/shownotes.json\" is not a \"application/json\" file" - fi + check_variable_is_correct shownotes_json } @@ -560,7 +659,7 @@ function get_ep_num_from_local_dir() { check_variable_is_correct working_dir - if [ ! -s "${working_dir}/shownotes.json" ] + if [ ! -s "${shownotes_json}" ] then echo_debug "Could not find a \"shownotes.json\" in the working directory \"${working_dir}/\"" @@ -578,22 +677,19 @@ function get_ep_num_from_local_dir() { echo_debug "Attempting to download information for episode \"${ep_num}\"" - if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/shownotes.php?id=${ep_num} --output "${working_dir}/shownotes.json" )" != 200 ] + if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/shownotes.php?id=${ep_num} --output "${shownotes_json}" )" != 200 ] then echo_error "The Episode hpr${ep_num} has not been posted." fi - if [ ! -s "${working_dir}/shownotes.json" ] + if [ ! -s "${shownotes_json}" ] then echo_error "The Episode information for hpr${ep_num} failed to download." fi fi - if [[ -s "${working_dir}/shownotes.json" && "$( file --brief --mime-type "${working_dir}/shownotes.json" | grep --count "application/json" )" -eq 0 ]] - then - echo_error "\"${working_dir}/shownotes.json\" is not a \"application/json\" file" - fi + check_variable_is_correct shownotes_json } @@ -618,32 +714,34 @@ function get_working_dir() { check_variable_is_correct ep_num fi - if [ ! -s "${working_dir}/shownotes.json" ] - then - echo_error "The working dir \"${working_dir}\" could not be found." - fi - echo_debug "Found working directory as \"${working_dir}\"" } + ################################################# -# Create a sub dir to store technical data. +# Once the working_dir is known, set the other variables -function set_working_processing_dir() { +function set_working_dir_variables() { - echo_debug "Checking the processing directory in the current working directory. set_working_processing_dir()" + echo_debug "Setting the other working directory and variables. set_working_dir_variables()" check_variable_is_correct working_dir - - working_processing_dir="${working_dir}/processing" - if [ ! -d "${working_processing_dir}/" ] - then - mkdir -v "${working_processing_dir}/" - fi - - check_variable_is_correct working_processing_dir + shownotes_json="${working_dir}/shownotes.json" + shownotes_html="${working_dir}/shownotes.html" + shownotes_edited="${working_dir}/shownotes_edited.html" + episode_tts_wav="${working_dir}/episode_tts.wav" + episode_tts_flac="${working_dir}/episode_tts.flac" + episode_summary_flac="${working_dir}/episode_summary.flac" + episode_intro_flac="${working_dir}/episode_intro.flac" + episode_body_flac="${working_dir}/episode_body.flac" + episode_sandwitch_flac="${working_dir}/episode_sandwitch.flac" + episode_final_flac="${working_dir}/episode_final.flac" + episode_intro_srt="${working_dir}/episode_intro.srt" + episode_body_srt="${working_dir}/episode_body.srt" + episode_outro_srt="${working_dir}/episode_outro.srt" + episode_srt="${working_dir}/episode.srt" } ################################################# @@ -659,17 +757,10 @@ function get_episode_metadata() { return fi - check_variable_is_correct working_dir + check_variable_is_correct working_dir shownotes_json - if [[ -s "${working_dir}/shownotes.json" && "$( file --brief --mime-type "${working_dir}/shownotes.json" | grep --count "application/json" )" -eq 0 ]] - then - echo_error "\"${working_dir}/shownotes.json\" is not a \"application/json\" file" - fi + set_working_dir_variables - shownotes_json="${working_dir}/shownotes.json" - shownotes_html="${working_dir}/shownotes.html" - shownotes_edited="${working_dir}/shownotes_edited.html" - check_variable_is_correct shownotes_json hostid="$( jq --raw-output '.host.Host_ID' ${shownotes_json} )" @@ -884,7 +975,7 @@ function media_checks() { echo_debug "Running media checks. media_checks()" - check_variable_is_correct working_dir working_processing_dir + check_variable_is_correct working_dir if [[ -n "${remote_media}" && "${remote_media}" != "null" ]] then @@ -912,15 +1003,23 @@ function media_checks() { if [ "$( echo "${media}" | wc --lines )" -ne 1 ] then echo "Multiple files found. Which one do you want to use ?" - select this_media in $( echo "${media}" ) + select this_media in "Skip asset creation" $( echo "${media}" ) do + if [ "${this_media}" == "Skip asset creation" ] + then + echo_debug "Skip asset creation" + check_derived_assets + skip_media_encoding="true" + return + fi ls -al "${this_media}" media="${this_media}" break done fi - echo_debug "You selected \"${media}\"." + echo_debug "You selected \"${media}\"." + if [[ -z "${media}" || ! -s "${media}" ]] then echo_error "Could not find the media \"${media}/\"" @@ -955,32 +1054,28 @@ function media_checks() { echo_debug "The number of audio channels is \"${supplied_channels}\" from \"${media}\" ." # Gernerate the Spectrum and Waveform image - ffmpeg -hide_banner -loglevel error -y -i "${media}" -lavfi "showspectrumpic=s=960x540" "${working_processing_dir}/${media_basename%.*}_spectrum.png" - audio2image.bash "${media}" && mv -v "${media%.*}.png" "${working_processing_dir}/${media_basename%.*}_waveform.png" + ffmpeg -hide_banner -loglevel error -y -i "${media}" -lavfi "showspectrumpic=s=960x540" "${working_dir}/${media_basename%.*}_spectrum.png" + audio2image.bash "${media}" && mv -v "${media%.*}.png" "${working_dir}/${media_basename%.*}_waveform.png" # Getting metadata - mediainfo "${media}" > "${working_processing_dir}/${media_basename%.*}_mediainfo.txt" + mediainfo "${media}" > "${working_dir}/${media_basename%.*}_mediainfo.txt" - exiftool "${media}" > "${working_processing_dir}/${media_basename%.*}_exiftool.txt" + exiftool "${media}" > "${working_dir}/${media_basename%.*}_exiftool.txt" for check_file in spectrum.png waveform.png mediainfo.txt exiftool.txt do - if [ ! -s "${working_processing_dir}/${media_basename%.*}_${check_file}" ] + if [ ! -s "${working_dir}/${media_basename%.*}_${check_file}" ] then - echo_error "The ${check_file} file was not generated for the \"${working_processing_dir}/${media_basename%.*}_${check_file}\"" >&2 + echo_error "The ${check_file} file was not generated for the \"${working_dir}/${media_basename%.*}_${check_file}\"" >&2 fi done - ffprobe="$( ffprobe ${media} 2>&1 | grep Audio: | sed 's/^.\s//g' )" - file_mime="$( file --brief --mime ${media} )" - file_mime_type="$( file --brief --mime-type ${media} )" - - if [[ -z "${ffprobe}" || -z ${file_mime} || -s ${file_mime_type} ]] - then - echo "ffprobe: ${ffprobe}, file_mime: ${file_mime},file_mime_type: ${file_mime_type}" - echo_error "Wasn't able to find mime metadata from \"${media}/\"" - fi + media_ffprobe="$( ffprobe ${media} 2>&1 | grep Audio: | sed 's/^.\s//g' )" + media_file_mime="$( file --brief --mime ${media} )" + media_file_mime_type="$( file --brief --mime-type ${media} )" + + check_variable_is_correct media_ffprobe media_file_mime media_file_mime_type } @@ -991,11 +1086,11 @@ function generate_initial_report() { echo_debug "Generating the initial report. generate_initial_report()" - if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]] - then - echo_debug "The Episode hpr${ep_num} has already been posted. Skipping generate_initial_report()" - return - fi +# if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]] +# then +# echo_debug "The Episode hpr${ep_num} has already been posted. Skipping generate_initial_report()" +# return +# fi # TODO list the images. @@ -1059,35 +1154,35 @@ ${shownotes_json_sanatised}

mediainfo report

-$( cat "${working_processing_dir}/${media_basename%.*}_mediainfo.txt" )
+$( cat "${working_dir}/${media_basename%.*}_mediainfo.txt" )

exiftool report

-$( cat "${working_processing_dir}/${media_basename%.*}_exiftool.txt" )
+$( cat "${working_dir}/${media_basename%.*}_exiftool.txt" )

Audio Spectrum

-\"Spectrum\" +\"Spectrum\"

Audio Waveform

-\"Waveform\" +\"Waveform\" 

-${ffprobe}
-${file_mime}
+${media_ffprobe}
+${media_file_mime}


${media} @@ -1101,7 +1196,7 @@ $(cat "${shownotes_srt}" )

-" > "${working_processing_dir}/${media_basename%.*}_media_report.html" +" > "${working_dir}/${media_basename%.*}_media_report.html" } @@ -1118,10 +1213,10 @@ function manual_shownotes_review() { return fi - if [[ -z "${shownotes_html}" || ! -s "${shownotes_html}" || ! -s "${working_processing_dir}/${media_basename%.*}_media_report.html" ]] + if [[ -z "${shownotes_html}" || ! -s "${shownotes_html}" || ! -s "${working_dir}/${media_basename%.*}_media_report.html" ]] then echo "shownotes_html: ${shownotes_html}" - ls -al "${shownotes_html}" "${working_processing_dir}/${media_basename%.*}_media_report.html" + ls -al "${shownotes_html}" "${working_dir}/${media_basename%.*}_media_report.html" echo_error "The files needed for to generate the inital report information are not available." fi @@ -1139,7 +1234,7 @@ function manual_shownotes_review() { fi kate "${shownotes_edited}" >/dev/null 2>&1 & - librewolf "${working_processing_dir}/${media_basename%.*}_media_report.html" >/dev/null 2>&1 & + librewolf "${working_dir}/${media_basename%.*}_media_report.html" >/dev/null 2>&1 & seamonkey "${shownotes_edited}" >/dev/null 2>&1 & # # # # bluefish "${shownotes_edited}" >/dev/null 2>&1 & # https://markdowntohtml.com/ @@ -1251,9 +1346,7 @@ function get_variables_from_episode_summary_json() { check_variable_is_correct ep_num working_dir - shownotes_json="${working_dir}/shownotes.json" - shownotes_html="${working_dir}/shownotes.html" - shownotes_edited="${working_dir}/shownotes_edited.html" + set_working_dir_variables if [ -z "${episode_summary_json}" ] then @@ -1291,17 +1384,20 @@ function get_variables_from_episode_summary_json() { function create_tts_summary() { echo_debug "Creating Text to Speech summary. create_tts_summary()" + + if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]] + then + echo_debug "Skipping media creation from create_tts_summary()" + return + fi - check_variable_is_correct working_dir duration synopsis piper_bin piper_voice working_processing_dir + check_variable_is_correct working_dir duration synopsis piper_bin piper_voice echo_debug "Converting text synopsis \"${synopsis}\" to speech." - echo "${synopsis}" | "${piper_bin}" --model "${piper_voice}" --output_file "${working_processing_dir}/episode_tts.wav" - - if [ ! -s "${working_processing_dir}/episode_tts.wav" ] - then - echo_error "The text to speech episode summary was not created \"${working_processing_dir}/episode_tts.wav\"." - fi + echo "${synopsis}" | "${piper_bin}" --model "${piper_voice}" --output_file "${episode_tts_wav}" + + check_variable_is_correct episode_tts_wav } @@ -1312,23 +1408,26 @@ function generate_intro() { echo_debug "Generating the intro. generate_intro()" - check_variable_is_correct working_dir working_processing_dir theme media outro_flac + if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]] + then + echo_debug "Skipping media creation from generate_intro()" + return + fi + + check_variable_is_correct working_dir theme media outro_flac # Everything needs to be in the same format for the intro, 1 channel (mono) Sampling rate 44.1 kHz - ffmpeg -hide_banner -loglevel error -y -i "${working_processing_dir}/episode_tts.wav" -ar 44100 -ac 1 "${working_processing_dir}//episode_tts.flac" + ffmpeg -hide_banner -loglevel error -y -i "${episode_tts_wav}" -ar 44100 -ac 1 "${episode_tts_flac}" + check_variable_is_correct episode_tts_flac # A level of silence is added at the beginning of the text to speech - sox -V2 "${silence}" "${working_processing_dir}//episode_tts.flac" "${working_processing_dir}/episode_summary.flac" + sox -V2 "${silence}" "${episode_tts_flac}" "${episode_summary_flac}" + check_variable_is_correct episode_summary_flac # The tracks are merged together resulting in the theme playing first, then after a period of silence the text to speech enters - sox -V2 -m "${working_processing_dir}/episode_summary.flac" "${theme}" "${working_processing_dir}/episode_intro.flac" - - if [[ ! -s "${working_processing_dir}//episode_tts.flac" || ! -s "${working_processing_dir}/episode_summary.flac" || ! -s "${working_processing_dir}/episode_intro.flac" ]] - then - echo_error "The files for the theme audio sandwich are not available." - ls -al "${working_processing_dir}//episode_tts.flac" "${working_processing_dir}/episode_summary.flac" "${theme}" "${working_processing_dir}/episode_intro.flac" - fi - + sox -V2 -m "${episode_summary_flac}" "${theme}" ${episode_intro_flac} + check_variable_is_correct episode_intro_flac + } ################################################# @@ -1337,21 +1436,25 @@ function generate_intro() { function generate_parent_audio() { echo_debug "Generating the parent audio - the sandwitch. generate_parent_audio()" - - if [[ ! -s "${working_processing_dir}/episode_intro.flac" || ! -s "${media}" || ! -s "${outro_flac}" ]] + + if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]] then - echo_error "The files for the sandwich are not available." - ls -al + echo_debug "Skipping media creation from generate_parent_audio()" + return fi + + check_variable_is_correct episode_intro_flac media outro_flac # Everything needs to be in the same format so the text to speech needs to be converted to 2 channel Sampling rate 44.1 kHz - ffmpeg -hide_banner -loglevel error -y -i "${media}" -ar 44100 -ac 1 "${working_processing_dir}/episode_body.flac" + ffmpeg -hide_banner -loglevel error -y -i "${media}" -ar 44100 -ac 1 "${episode_body_flac}" + check_variable_is_correct episode_body_flac # Combine the components together - sox -V2 "${working_processing_dir}/episode_intro.flac" "${working_processing_dir}/episode_body.flac" "${outro_flac}" "${working_processing_dir}/episode_sandwitch.flac" - + sox -V2 ${episode_intro_flac} "${episode_body_flac}" "${outro_flac}" "${episode_sandwitch_flac}" + check_variable_is_correct episode_sandwitch_flac # Normalise the audio - ffmpeg -hide_banner -loglevel error -y -i "${working_processing_dir}/episode_sandwitch.flac" -af loudnorm=I=-16:LRA=11:TP=-1.5 "${working_processing_dir}/episode_final.flac" + ffmpeg -hide_banner -loglevel error -y -i "${episode_sandwitch_flac}" -af loudnorm=I=-16:LRA=11:TP=-1.5 "${episode_final_flac}" + check_variable_is_correct episode_final_flac } @@ -1362,9 +1465,15 @@ function generate_derived_media() { echo_debug "Generating derived audio. generate_derived_media()" + if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]] + then + echo_debug "Skipping media creation from generate_derived_media()" + return + fi + check_variable_is_correct media working_dir ep_num title artist license - if [[ ! -s "${working_processing_dir}/episode_final.flac" ]] + if [[ ! -s "${episode_final_flac}" ]] then ls -al echo_error "The final cut is not available." @@ -1374,10 +1483,10 @@ function generate_derived_media() { # https://wiki.multimedia.cx/index.php?title=FFmpeg_Metadata - for extension in flac wav mp3 ogg opus spx + for extension in "${audio_formats[@]}" do echo_debug "Generating \"hpr${ep_num}.${extension}\"." - ffmpeg -hide_banner -loglevel error -y -i "${working_processing_dir}/episode_final.flac" \ + ffmpeg -hide_banner -loglevel error -y -i "${episode_final_flac}" \ -metadata title="${title}" \ -metadata artist="${artist}" \ -metadata author="${artist}" \ @@ -1401,15 +1510,44 @@ function generate_derived_media() { done #TODO fix close duration - lengths=$( for extension in flac wav mp3 ogg opus + +# shownotes_json="${working_dir}/shownotes.json" +# shownotes_html="${working_dir}/shownotes.html" +# shownotes_edited="${working_dir}/shownotes_edited.html" +# episode_tts_wav="${working_dir}/episode_tts.wav" +# episode_tts_flac="${working_dir}/episode_tts.flac" +# episode_summary_flac="${working_dir}/episode_summary.flac" +# episode_intro_flac="${working_dir}/episode_intro.flac" +# episode_body_flac="${working_dir}/episode_body.flac" +# episode_sandwitch_flac="${working_dir}/episode_sandwitch.flac" +# episode_final_flac="${working_dir}/episode_final.flac" +# episode_intro_srt="${working_dir}/episode_intro.srt" +# episode_body_srt="${working_dir}/episode_body.srt" +# episode_outro_srt="${working_dir}/episode_outro.srt" +# episode_srt="${working_dir}/episode.srt" + + check_variable_is_correct episode_intro_flac outro_flac media duration + + + intro_duration="$( mediainfo --Output=XML --Full "${episode_intro_flac}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" + outro_duration="$( mediainfo --Output=XML --Full "${outro_flac}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" + source_duration="$( mediainfo --Output=XML --Full "${media}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" + expected_duration=$(( ${intro_duration} + ${duration} + ${outro_duration} )) + check_variable_is_correct intro_duration outro_duration source_duration expected_duration + + echo_debug "Intro(${intro_duration}) + Show(${duration}) + Outro(${outro_duration}) = ${expected_duration}" + + for extension in "${audio_formats[@]}" do - mediainfo --full --Output=XML "${working_dir}/hpr${ep_num}.${extension}" | xmlstarlet sel -T -t -m "_:MediaInfo/_:media/_:track[@type='Audio']/_:Duration[1]" -v "." -n - | awk -F '.' '{print $1}' - done | sort | uniq | wc -l ) - if [ "${lengths}" -ne "1" ] - then - echo_error "The duration of the derived media is not correct." - fi -#TODO fix close duration + this_duration=$( mediainfo --full --Output=XML "${working_dir}/hpr${ep_num}.${extension}" | xmlstarlet sel -T -t -m "_:MediaInfo/_:media/_:track[@type='Audio']/_:Duration[1]" -v "." -n - | awk -F '.' '{print $1}' ) + if [ $( abs_diff "${this_duration}" "${expected_duration}" ) -le "${acceptable_duration_difference}" ] + then + echo_debug "The file \"hpr${ep_num}.${extension}\" duration of ${this_duration} is close enough to ${expected_duration}" + else + echo_error "The file \"hpr${ep_num}.${extension}\" actual duration of ${this_duration} is not close enough to posted duration of ${expected_duration}. Fix or update the posted duration to ${source_duration}." + fi + done + cp -v "${media}" "${working_dir}/hpr${ep_num}_source.${media##*.}" if [[ ! -s "${working_dir}/hpr${ep_num}_source.${media##*.}" ]] @@ -1426,12 +1564,18 @@ function generate_derived_media() { function generate_show_transcript() { echo_debug "Generate show transcript and subtitles. generate_show_transcript()" + + if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]] + then + echo_debug "Skipping media creation from generate_show_transcript()" + return + fi # TODO Currently processed elsewhere by hpr-get-and-transcode.bash and uploaded to hpr:upload/ to be synced with media above - if [[ ! -s "${media}" || ! -s "${media%.*}.srt" || ! -s "${intro_srt}" || ! -s "${outro_srt}" || ! -s "${working_processing_dir}/episode_intro.flac" || ! -s "${working_processing_dir}/episode_body.flac" ]] + if [[ ! -s "${media}" || ! -s "${media%.*}.srt" || ! -s "${intro_srt}" || ! -s "${outro_srt}" || ! -s ${episode_intro_flac} || ! -s "${episode_body_flac}" ]] then - ls -al "${media}" "${media%.*}.srt" "${intro_srt}" "${outro_srt}" "${working_processing_dir}/episode_intro.flac" "${working_processing_dir}/episode_body.flac" + ls -al "${media}" "${media%.*}.srt" "${intro_srt}" "${outro_srt}" ${episode_intro_flac} "${episode_body_flac}" echo_error "The transcriptions files are not available." fi @@ -1458,32 +1602,32 @@ function generate_show_transcript() { REPLACE_LINE_4="The flag is ${explicit}, and the license is ${license}" REPLACE_LINE_5="The summary is \"${summary}\"" - cp -v ${intro_srt} "${working_processing_dir}/episode_intro.srt" - cp -v ${outro_srt} "${working_processing_dir}/episode_outro.srt" + cp -v ${intro_srt} "${episode_intro_srt}" + cp -v ${outro_srt} "${episode_outro_srt}" - sed -e "s~REPLACE_LINE_1~${REPLACE_LINE_1}~g" -e "s~REPLACE_LINE_2~${REPLACE_LINE_2}~g" -e "s~REPLACE_LINE_3~${REPLACE_LINE_3}~g" -e "s~REPLACE_LINE_4~${REPLACE_LINE_4}~g" -e "s~REPLACE_LINE_5~${REPLACE_LINE_5}~g" -i "${working_processing_dir}/episode_intro.srt" + sed -e "s~REPLACE_LINE_1~${REPLACE_LINE_1}~g" -e "s~REPLACE_LINE_2~${REPLACE_LINE_2}~g" -e "s~REPLACE_LINE_3~${REPLACE_LINE_3}~g" -e "s~REPLACE_LINE_4~${REPLACE_LINE_4}~g" -e "s~REPLACE_LINE_5~${REPLACE_LINE_5}~g" -i "${episode_intro_srt}" - if [ "$( grep --count REPLACE_LINE "${working_processing_dir}/episode_intro.srt" )" -ne "0" ] + if [ "$( grep --count REPLACE_LINE "${episode_intro_srt}" )" -ne "0" ] then - echo_error "The intro subtitles were not correctly generated \"${working_processing_dir}/episode_intro.srt\"." + echo_error "The intro subtitles were not correctly generated \"${episode_intro_srt}\"." fi # Time shift the media subtitles on by the duration of the intro wav file # https://trac.ffmpeg.org/wiki/UnderstandingItsoffset - itsoffset_intro="$( mediainfo --full --Output=JSON "${working_processing_dir}/episode_intro.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )" + itsoffset_intro="$( mediainfo --full --Output=JSON ${episode_intro_flac} | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )" if [[ -z "${itsoffset_intro}" || "${itsoffset_intro}" == "null" ]] then echo_error "Could not retrieve the itsoffset_intro to correct the timing of the subtitles." fi - ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_intro}" -i "${media%.*}.srt" -c copy "${working_processing_dir}/episode_body.srt" + ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_intro}" -i "${media%.*}.srt" -c copy "${episode_body_srt}" # Timeshift the outro by the duration of the intro and the supplied media - itsoffset_body="$( mediainfo --full --Output=JSON "${working_processing_dir}/episode_body.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )" + itsoffset_body="$( mediainfo --full --Output=JSON "${episode_body_flac}" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )" if [[ -z "${itsoffset_body}" || "${itsoffset_body}" == "null" ]] then @@ -1492,18 +1636,18 @@ function generate_show_transcript() { itsoffset_body=$((itsoffset_intro + $itsoffset_body)) - ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_body}" -i "${working_processing_dir}/episode_outro.srt" -c copy "${working_processing_dir}/episode_outro_shifted.srt" + ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_body}" -i "${episode_outro_srt}" -c copy "${working_dir}/episode_outro_shifted.srt" # Combine the intro, timeshifted media subtitles, and the timeshifted outro subtitles. - cat "${working_processing_dir}/episode_intro.srt" "${working_processing_dir}/episode_body.srt" "${working_processing_dir}/episode_outro_shifted.srt" > "${working_processing_dir}/episode.srt" + cat "${episode_intro_srt}" "${episode_body_srt}" "${working_dir}/episode_outro_shifted.srt" > "${episode_srt}" # Parse the resulting subtitle file fixing the numberic counter # https://en.wikipedia.org/wiki/SubRip count=1 - cat "${working_processing_dir}/episode.srt" | while read this_line + cat "${episode_srt}" | while read this_line do if [ "$( echo "${this_line}" | grep -c --perl-regexp '^[0-9]+$' )" -eq "1" ] then @@ -1533,17 +1677,17 @@ function generate_final_report() { echo_debug "Generating the final report. generate_final_report()" - if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]] + if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]] then - echo_debug "The Episode hpr${ep_num} has already been posted. Skipping generate_final_report()" + echo_debug "Skipping media creation from ()" return fi - check_variable_is_correct working_dir working_processing_dir ep_num media_basename shownotes_edited synopsis + check_variable_is_correct working_dir ep_num media_basename shownotes_edited synopsis - final_report="${working_processing_dir}/hpr${ep_num}_report.html" + final_report="${working_dir}/hpr${ep_num}_report.html" - for this_file_extension_to_check in flac mp3 ogg opus srt txt wav + for this_file_extension_to_check in "${episode_formats[@]}" do if [[ ! -s "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" ]] then @@ -1552,13 +1696,13 @@ function generate_final_report() { fi done - if [[ ! -s "${working_processing_dir}/${media_basename%.*}_media_report.html" ]] + if [[ ! -s "${working_dir}/${media_basename%.*}_media_report.html" ]] then - ls -al "${working_processing_dir}/${media_basename%.*}_media_report.html" - echo_error "The initial report is not available.\"${working_processing_dir}/${media_basename%.*}_media_report.html\"" + ls -al "${working_dir}/${media_basename%.*}_media_report.html" + echo_error "The initial report is not available.\"${working_dir}/${media_basename%.*}_media_report.html\"" fi - grep -Pv '|' "${working_processing_dir}/${media_basename%.*}_media_report.html" > "${final_report}" + grep -Pv '|' "${working_dir}/${media_basename%.*}_media_report.html" > "${final_report}" echo "

Text To Speech

@@ -1566,33 +1710,33 @@ $( echo "${synopsis}" )


-${working_processing_dir}//episode_tts.flac +${episode_tts_flac}

" >> "${final_report}" - for this_file_extension_to_check in flac mp3 ogg opus wav + for this_file_extension_to_check in "${audio_formats[@]}" do - ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" -lavfi "showspectrumpic=s=960x540" "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png" + ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" -lavfi "showspectrumpic=s=960x540" "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png" - audio2image.bash "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" && mv -v "${working_dir}/hpr${ep_num}.png" "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_waveform.png" + audio2image.bash "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" && mv -v "${working_dir}/hpr${ep_num}.png" "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_waveform.png" - mediainfo "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" + mediainfo "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" - exiftool "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" + exiftool "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" - ffprobe "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" 2>&1 | grep Audio: | sed 's/^.\s//g' > "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" - file --brief --mime "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" >> "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" + ffprobe "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" 2>&1 | grep Audio: | sed 's/^.\s//g' > "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" + file --brief --mime "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" >> "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" this_file_mime_type="$( file --brief --mime-type "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" )" for this_file_to_check in spectrum.png waveform.png mediainfo.txt exiftool.txt ffprobe.txt do - if [[ ! -s "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" ]] + if [[ ! -s "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" ]] then - ls -al "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" + ls -al "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" echo_error "The inital report information is missing \"${this_file_to_check}\"." fi done @@ -1602,29 +1746,29 @@ $( echo "${synopsis}" )

mediainfo report

-$( cat "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" )
+$( cat "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" )

exiftool report

-$( cat "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" )
+$( cat "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" )

Audio Spectrum

-\"Spectrum\" +\"Spectrum\"

Audio Waveform

-\"Waveform\" +\"Waveform\" 

-$( cat "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" )
+$( cat "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" )

@@ -1677,9 +1821,9 @@ function manual_final_review() { echo_debug "Validating the final report. manual_final_review()" - if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]] + if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]] then - echo_debug "The Episode hpr${ep_num} has already been posted. Skipping generate_final_report()" + echo_debug "Skipping media creation from ()" return fi @@ -1703,14 +1847,38 @@ function manual_final_review() { } +################################################# +# Checks derived assets are correct + +function check_derived_assets () { + + check_variable_is_correct working_dir ep_num + + #TODO Extend checks to media internals + + for extension in "${audio_formats[@]}" + do + echo_debug "Checking \"hpr${ep_num}.${extension}\"." + + if [[ ! -s "${working_dir}/hpr${ep_num}.${extension}" ]] + then + ls -al "${working_dir}/hpr${ep_num}.${extension}" + echo_error "Failed to locate \"${working_dir}/hpr${ep_num}.${extension}\"." + fi + done + +} + ################################################# # Register the assets with the hpr database function register_assets() { echo_debug "Registering the assets with the hpr database. register_assets()" - + check_variable_is_correct working_dir ep_num + + check_derived_assets assets_csv="${working_dir}/hpr${ep_num}_assets.csv" assets_json="${working_dir}/hpr${ep_num}_assets.json" @@ -1723,7 +1891,8 @@ function register_assets() { echo '"episode_id","filename","extension","size", "sha1sum", "mime_type", "file_type"' | tee "${assets_csv}" - for this_asset_filename in hpr${ep_num}.flac hpr${ep_num}.wav hpr${ep_num}.mp3 hpr${ep_num}.ogg hpr${ep_num}.opus hpr${ep_num}.srt hpr${ep_num}.txt $( find "${working_dir}/" -maxdepth 1 -type f -iname "hpr${ep_num}_image_*.*" ) + find "${working_dir}/" -maxdepth 1 -type f \( -iname "hpr${ep_num}.*" -or -iname "hpr${ep_num}_image_*.*" \) | \ + while read for this_asset_filename do this_asset_filename="$( basename "${this_asset_filename}" )" echo_debug "Registering \"${this_asset_filename}\"." @@ -1760,9 +1929,11 @@ function register_assets() { echo "${ep_num},\"${this_asset_basename}\",\"${this_asset_extension}\",\"${this_asset_size}\",\"${this_asset_sha1sum}\",\"${this_asset_mime_type}\",\"${this_asset_file_type}\"" | tee --append "${assets_csv}" - done + done + + check_variable_is_correct assets_csv - if [ -s "${assets_csv}" ] + if [[ -s "${assets_csv}" && "$( wc -l "${assets_csv}" )" -gt "1" ]] then cat "${assets_csv}" | csvtojson | jq '{"assets":[.[]]}' | tee "${assets_json}" fi @@ -2095,13 +2266,8 @@ echo "hidden" # # This tool will process the HPR shows allowing the janitor to review the media and fix shownotes. - argument_override "$@" -######################################################################################## -# Posting show - - program_checks # We know that all the programs and variables are set if [[ -z "${working_dir_bypass}" || "${working_dir_bypass}" != "true" ]] @@ -2111,8 +2277,6 @@ else echo_debug "Skipping get_working_dir()" fi -set_working_processing_dir - if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output /dev/null )" == 200 ] then skip_post_show="true"