HPR/hpr-tools
5
0
Fork 2
Code Issues 13 Pull Requests 1 Packages Projects Releases Wiki Activity

fixing encoding timing errors and restoring spx - why not

Browse Source
This commit is contained in:
Ken Fallon 2025-06-04 21:18:21 +02:00
parent 6f3c6c2596
commit dd97a672aa
1 changed files with 339 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

514
workflow/process_episode.bash
View File

@ -22,9 +22,12 @@ piper_voice="/opt/bin/piper/piper/piper-voices/en/en_US/lessac/medium/en_US-less
working_dir_bypass="false"
skip_post_show="false"
skip_media_encoding="false"
acceptable_duration_difference="2" # Seconds
unsafe_episode_variables=( shownotes_json shownotes_html hostid artist email title summary series_id series_name explicit license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised )
audio_formats=( flac wav mp3 ogg opus spx )
episode_formats=( flac wav mp3 ogg opus spx srt txt )
#################################################
# Display Error message, display help and exit
@ -133,7 +136,6 @@ function argument_override() {
if [ "${this_key}" == "working_dir" ]
then
working_dir_bypass="true"
set_working_processing_dir
fi
fi
done
@ -174,6 +176,10 @@ function check_variable_is_correct() {
then
echo_error "The \"assets_csv\" variable has not a valid \"text/csv\" mime type."
fi
if [ "$( wc --lines ${assets_csv} | awk '{print $1}' )" -le "1" ]
then
echo_error "The \"${assets_csv}\" file is empty."
fi
;;
assets_json)
if [[ ! -s "${assets_json}" || -z "${assets_json}" ]]
@ -230,6 +236,72 @@ function check_variable_is_correct() {
echo_error "The \"email_unpadded\" variable is missing."
fi
;;
episode_body_flac)
if [ -z "${episode_body_flac}" ]
then
echo_error "The \"episode_body_flac\" variable is missing."
fi
;;
episode_body_srt)
if [ -z "${episode_body_srt}" ]
then
echo_error "The \"episode_body_srt\" variable is missing."
fi
;;
episode_final_flac)
if [ -z "${episode_final_flac}" ]
then
echo_error "The \"episode_final_flac\" variable is missing."
fi
;;
episode_intro_flac)
if [ -z "${episode_intro_flac}" ]
then
echo_error "The \"episode_intro_flac\" variable is missing."
fi
;;
episode_intro_srt)
if [ -z "${episode_intro_srt}" ]
then
echo_error "The \"episode_intro_srt\" variable is missing."
fi
;;
episode_outro_srt)
if [ -z "${episode_outro_srt}" ]
then
echo_error "The \"episode_outro_srt\" variable is missing."
fi
;;
episode_sandwitch_flac)
if [ -z "${episode_sandwitch_flac}" ]
then
echo_error "The \"episode_sandwitch_flac\" variable is missing."
fi
;;
episode_srt)
if [ -z "${episode_srt}" ]
then
echo_error "The \"episode_srt\" variable is missing."
fi
;;
episode_summary_flac)
if [ -z "${episode_summary_flac}" ]
then
echo_error "The \"episode_summary_flac\" variable is missing."
fi
;;
episode_tts_flac)
if [ -z "${episode_tts_flac}" ]
then
echo_error "The \"episode_tts_flac\" variable is missing."
fi
;;
episode_tts_wav)
if [ -z "${episode_tts_wav}" ]
then
echo_error "The \"episode_tts_wav\" variable is missing."
fi
;;
episode_summary_json)
if [[ ! -s "${episode_summary_json}" || -z "${episode_summary_json}" ]]
then
@ -261,6 +333,12 @@ function check_variable_is_correct() {
echo_error "The \"ep_num\" variable is not a valid number between 1 and 9999."
fi
;;
expected_duration)
if [ -z "${expected_duration}" ]
then
echo_error "The \"expected_duration\" variable is missing."
fi
;;
explicit)
if [[ -z "${explicit}" || "${explicit}" == "null" ]]
then
@ -315,6 +393,12 @@ function check_variable_is_correct() {
echo_error "The \"hostid\" variable is missing."
fi
;;
intro_duration)
if [ -z "${intro_duration}" ]
then
echo_error "The \"intro_duration\" variable is missing."
fi
;;
intro_srt)
if [[ ! -d "${intro_srt}" || -z "${intro_srt}" ]]
then
@ -346,11 +430,35 @@ function check_variable_is_correct() {
fi
;;
media_basename)
if [[ ! -d "${media_basename}" || -z "${media_basename}" ]]
if [[ ! -s "${media_basename}" || -z "${media_basename}" ]]
then
echo_error "The \"media_basename\" variable is missing or is not a directory."
fi
;;
media_ffprobe)
if [ -z "${media_ffprobe}" ]
then
echo_error "The \"media_ffprobe\" variable is missing."
fi
;;
media_file_mime)
if [ -z "${media_file_mime}" ]
then
echo_error "The \"media_file_mime\" variable is missing."
fi
;;
media_file_mime_type)
if [ -z "${media_file_mime_type}" ]
then
echo_error "The \"media_file_mime_type\" variable is missing."
fi
;;
outro_duration)
if [ -z "${outro_duration}" ]
then
echo_error "The \"outro_duration\" variable is missing."
fi
;;
outro_flac)
if [[ ! -s "${outro_flac}" || -z "${outro_flac}" ]]
then
@ -422,6 +530,12 @@ function check_variable_is_correct() {
echo_error "The \"silence\" variable is missing or file \"${silence}\" does not exist."
fi
;;
source_duration)
if [ -z "${source_duration}" ]
then
echo_error "The \"source_duration\" variable is missing."
fi
;;
summary)
if [[ -z "${summary}" || "${summary}" == "null" ]]
then
@ -470,12 +584,6 @@ function check_variable_is_correct() {
echo_error "The \"working_dir\" variable is missing or is not a directory."
fi
;;
working_processing_dir)
if [[ ! -d "${working_processing_dir}" || -z "${working_processing_dir}" ]]
then
echo_error "The \"working_processing_dir\" variable is missing or is not a directory."
fi
;;
year)
if [[ -z "${year}" || "${year}" == "null" ]]
then
@ -538,16 +646,7 @@ function get_next_show_from_hpr_hub() {
ssh hpr -t "detox -v ${hpr_upload_dir}/"
rsync -ave ssh --partial --progress ${source_dir}/ ${working_dir}/
if [ ! -s "${working_dir}/shownotes.json" ]
then
echo_error "The working dir is missing the shownotes file \"${working_dir}/shownotes.json\""
fi
if [ "$( file --brief --mime-type "${working_dir}/shownotes.json" | grep --count "application/json" )" -eq 0 ]
then
echo_error "The \"${working_dir}/shownotes.json\" is not a \"application/json\" file"
fi
check_variable_is_correct shownotes_json
}
@ -560,7 +659,7 @@ function get_ep_num_from_local_dir() {
check_variable_is_correct working_dir
if [ ! -s "${working_dir}/shownotes.json" ]
if [ ! -s "${shownotes_json}" ]
then
echo_debug "Could not find a \"shownotes.json\" in the working directory \"${working_dir}/\""
@ -578,22 +677,19 @@ function get_ep_num_from_local_dir() {
echo_debug "Attempting to download information for episode \"${ep_num}\""
if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/shownotes.php?id=${ep_num} --output "${working_dir}/shownotes.json" )" != 200 ]
if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/shownotes.php?id=${ep_num} --output "${shownotes_json}" )" != 200 ]
then
echo_error "The Episode hpr${ep_num} has not been posted."
fi
if [ ! -s "${working_dir}/shownotes.json" ]
if [ ! -s "${shownotes_json}" ]
then
echo_error "The Episode information for hpr${ep_num} failed to download."
fi
fi
if [[ -s "${working_dir}/shownotes.json" && "$( file --brief --mime-type "${working_dir}/shownotes.json" | grep --count "application/json" )" -eq 0 ]]
then
echo_error "\"${working_dir}/shownotes.json\" is not a \"application/json\" file"
fi
check_variable_is_correct shownotes_json
}
@ -618,32 +714,34 @@ function get_working_dir() {
check_variable_is_correct ep_num
fi
if [ ! -s "${working_dir}/shownotes.json" ]
then
echo_error "The working dir \"${working_dir}\" could not be found."
fi
echo_debug "Found working directory as \"${working_dir}\""
}
#################################################
# Create a sub dir to store technical data.
# Once the working_dir is known, set the other variables
function set_working_processing_dir() {
function set_working_dir_variables() {
echo_debug "Checking the processing directory in the current working directory. set_working_processing_dir()"
echo_debug "Setting the other working directory and variables. set_working_dir_variables()"
check_variable_is_correct working_dir
working_processing_dir="${working_dir}/processing"
if [ ! -d "${working_processing_dir}/" ]
then
mkdir -v "${working_processing_dir}/"
fi
check_variable_is_correct working_processing_dir
shownotes_json="${working_dir}/shownotes.json"
shownotes_html="${working_dir}/shownotes.html"
shownotes_edited="${working_dir}/shownotes_edited.html"
episode_tts_wav="${working_dir}/episode_tts.wav"
episode_tts_flac="${working_dir}/episode_tts.flac"
episode_summary_flac="${working_dir}/episode_summary.flac"
episode_intro_flac="${working_dir}/episode_intro.flac"
episode_body_flac="${working_dir}/episode_body.flac"
episode_sandwitch_flac="${working_dir}/episode_sandwitch.flac"
episode_final_flac="${working_dir}/episode_final.flac"
episode_intro_srt="${working_dir}/episode_intro.srt"
episode_body_srt="${working_dir}/episode_body.srt"
episode_outro_srt="${working_dir}/episode_outro.srt"
episode_srt="${working_dir}/episode.srt"
}
#################################################
@ -659,17 +757,10 @@ function get_episode_metadata() {
return
fi
check_variable_is_correct working_dir
check_variable_is_correct working_dir shownotes_json
if [[ -s "${working_dir}/shownotes.json" && "$( file --brief --mime-type "${working_dir}/shownotes.json" | grep --count "application/json" )" -eq 0 ]]
then
echo_error "\"${working_dir}/shownotes.json\" is not a \"application/json\" file"
fi
set_working_dir_variables
shownotes_json="${working_dir}/shownotes.json"
shownotes_html="${working_dir}/shownotes.html"
shownotes_edited="${working_dir}/shownotes_edited.html"
check_variable_is_correct shownotes_json
hostid="$( jq --raw-output '.host.Host_ID' ${shownotes_json} )"
@ -884,7 +975,7 @@ function media_checks() {
echo_debug "Running media checks. media_checks()"
check_variable_is_correct working_dir working_processing_dir
check_variable_is_correct working_dir
if [[ -n "${remote_media}" && "${remote_media}" != "null" ]]
then
@ -912,15 +1003,23 @@ function media_checks() {
if [ "$( echo "${media}" | wc --lines )" -ne 1 ]
then
echo "Multiple files found. Which one do you want to use ?"
select this_media in $( echo "${media}" )
select this_media in "Skip asset creation" $( echo "${media}" )
do
if [ "${this_media}" == "Skip asset creation" ]
then
echo_debug "Skip asset creation"
check_derived_assets
skip_media_encoding="true"
return
fi
ls -al "${this_media}"
media="${this_media}"
break
done
fi
echo_debug "You selected \"${media}\"."
echo_debug "You selected \"${media}\"."
if [[ -z "${media}" || ! -s "${media}" ]]
then
echo_error "Could not find the media \"${media}/\""
@ -955,32 +1054,28 @@ function media_checks() {
echo_debug "The number of audio channels is \"${supplied_channels}\" from \"${media}\" ."
# Gernerate the Spectrum and Waveform image
ffmpeg -hide_banner -loglevel error -y -i "${media}" -lavfi "showspectrumpic=s=960x540" "${working_processing_dir}/${media_basename%.*}_spectrum.png"
audio2image.bash "${media}" && mv -v "${media%.*}.png" "${working_processing_dir}/${media_basename%.*}_waveform.png"
ffmpeg -hide_banner -loglevel error -y -i "${media}" -lavfi "showspectrumpic=s=960x540" "${working_dir}/${media_basename%.*}_spectrum.png"
audio2image.bash "${media}" && mv -v "${media%.*}.png" "${working_dir}/${media_basename%.*}_waveform.png"
# Getting metadata
mediainfo "${media}" > "${working_processing_dir}/${media_basename%.*}_mediainfo.txt"
mediainfo "${media}" > "${working_dir}/${media_basename%.*}_mediainfo.txt"
exiftool "${media}" > "${working_processing_dir}/${media_basename%.*}_exiftool.txt"
exiftool "${media}" > "${working_dir}/${media_basename%.*}_exiftool.txt"
for check_file in spectrum.png waveform.png mediainfo.txt exiftool.txt
do
if [ ! -s "${working_processing_dir}/${media_basename%.*}_${check_file}" ]
if [ ! -s "${working_dir}/${media_basename%.*}_${check_file}" ]
then
echo_error "The ${check_file} file was not generated for the \"${working_processing_dir}/${media_basename%.*}_${check_file}\"" >&2
echo_error "The ${check_file} file was not generated for the \"${working_dir}/${media_basename%.*}_${check_file}\"" >&2
fi
done
ffprobe="$( ffprobe ${media} 2>&1 | grep Audio: | sed 's/^.\s//g' )"
file_mime="$( file --brief --mime ${media} )"
file_mime_type="$( file --brief --mime-type ${media} )"
if [[ -z "${ffprobe}" || -z ${file_mime} || -s ${file_mime_type} ]]
then
echo "ffprobe: ${ffprobe}, file_mime: ${file_mime},file_mime_type: ${file_mime_type}"
echo_error "Wasn't able to find mime metadata from \"${media}/\""
fi
media_ffprobe="$( ffprobe ${media} 2>&1 | grep Audio: | sed 's/^.\s//g' )"
media_file_mime="$( file --brief --mime ${media} )"
media_file_mime_type="$( file --brief --mime-type ${media} )"
check_variable_is_correct media_ffprobe media_file_mime media_file_mime_type
}
@ -991,11 +1086,11 @@ function generate_initial_report() {
echo_debug "Generating the initial report. generate_initial_report()"
if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
then
echo_debug "The Episode hpr${ep_num} has already been posted. Skipping generate_initial_report()"
return
fi
# if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
# then
# echo_debug "The Episode hpr${ep_num} has already been posted. Skipping generate_initial_report()"
# return
# fi
# TODO list the images.
@ -1059,35 +1154,35 @@ ${shownotes_json_sanatised}
<h3>mediainfo report</h3>
<code>
<pre>
$( cat "${working_processing_dir}/${media_basename%.*}_mediainfo.txt" )
$( cat "${working_dir}/${media_basename%.*}_mediainfo.txt" )
</pre>
</code>
<h3>exiftool report</h3>
<code>
<pre>
$( cat "${working_processing_dir}/${media_basename%.*}_exiftool.txt" )
$( cat "${working_dir}/${media_basename%.*}_exiftool.txt" )
</pre>
</code>
<h3>Audio Spectrum</h3>
<p>
<img src=\"${working_processing_dir}/${media_basename%.*}_spectrum.png\" alt=\"Spectrum\" />
<img src=\"${working_dir}/${media_basename%.*}_spectrum.png\" alt=\"Spectrum\" />
</p>
<h3>Audio Waveform</h3>
<p>
<img src=\"${working_processing_dir}/${media_basename%.*}_waveform.png\" alt=\"Waveform\" />
<img src=\"${working_dir}/${media_basename%.*}_waveform.png\" alt=\"Waveform\" />
</p>
<pre>
${ffprobe}
${file_mime}
${media_ffprobe}
${media_file_mime}
</pre>
<p>
<audio controls=\"\" preload=\"none\" style=\"width:600px;\" >
<source src=\"${media}\" type=\"${file_mime_type}\">
<source src=\"${media}\" type=\"${media_file_mime_type}\">
</audio>
<br />
<a href=\"${media}\">${media}</a>
@ -1101,7 +1196,7 @@ $(cat "${shownotes_srt}" )
</pre>
<hr />
</body>
</html>" > "${working_processing_dir}/${media_basename%.*}_media_report.html"
</html>" > "${working_dir}/${media_basename%.*}_media_report.html"
}
@ -1118,10 +1213,10 @@ function manual_shownotes_review() {
return
fi
if [[ -z "${shownotes_html}" || ! -s "${shownotes_html}" || ! -s "${working_processing_dir}/${media_basename%.*}_media_report.html" ]]
if [[ -z "${shownotes_html}" || ! -s "${shownotes_html}" || ! -s "${working_dir}/${media_basename%.*}_media_report.html" ]]
then
echo "shownotes_html: ${shownotes_html}"
ls -al "${shownotes_html}" "${working_processing_dir}/${media_basename%.*}_media_report.html"
ls -al "${shownotes_html}" "${working_dir}/${media_basename%.*}_media_report.html"
echo_error "The files needed for to generate the inital report information are not available."
fi
@ -1139,7 +1234,7 @@ function manual_shownotes_review() {
fi
kate "${shownotes_edited}" >/dev/null 2>&1 &
librewolf "${working_processing_dir}/${media_basename%.*}_media_report.html" >/dev/null 2>&1 &
librewolf "${working_dir}/${media_basename%.*}_media_report.html" >/dev/null 2>&1 &
seamonkey "${shownotes_edited}" >/dev/null 2>&1 &
# # # # bluefish "${shownotes_edited}" >/dev/null 2>&1 &
# https://markdowntohtml.com/
@ -1251,9 +1346,7 @@ function get_variables_from_episode_summary_json() {
check_variable_is_correct ep_num working_dir
shownotes_json="${working_dir}/shownotes.json"
shownotes_html="${working_dir}/shownotes.html"
shownotes_edited="${working_dir}/shownotes_edited.html"
set_working_dir_variables
if [ -z "${episode_summary_json}" ]
then
@ -1291,17 +1384,20 @@ function get_variables_from_episode_summary_json() {
function create_tts_summary() {
echo_debug "Creating Text to Speech summary. create_tts_summary()"
if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]]
then
echo_debug "Skipping media creation from create_tts_summary()"
return
fi
check_variable_is_correct working_dir duration synopsis piper_bin piper_voice working_processing_dir
check_variable_is_correct working_dir duration synopsis piper_bin piper_voice
echo_debug "Converting text synopsis \"${synopsis}\" to speech."
echo "${synopsis}" | "${piper_bin}" --model "${piper_voice}" --output_file "${working_processing_dir}/episode_tts.wav"
if [ ! -s "${working_processing_dir}/episode_tts.wav" ]
then
echo_error "The text to speech episode summary was not created \"${working_processing_dir}/episode_tts.wav\"."
fi
echo "${synopsis}" | "${piper_bin}" --model "${piper_voice}" --output_file "${episode_tts_wav}"
check_variable_is_correct episode_tts_wav
}
@ -1312,23 +1408,26 @@ function generate_intro() {
echo_debug "Generating the intro. generate_intro()"
check_variable_is_correct working_dir working_processing_dir theme media outro_flac
if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]]
then
echo_debug "Skipping media creation from generate_intro()"
return
fi
check_variable_is_correct working_dir theme media outro_flac
# Everything needs to be in the same format for the intro, 1 channel (mono) Sampling rate 44.1 kHz
ffmpeg -hide_banner -loglevel error -y -i "${working_processing_dir}/episode_tts.wav" -ar 44100 -ac 1 "${working_processing_dir}//episode_tts.flac"
ffmpeg -hide_banner -loglevel error -y -i "${episode_tts_wav}" -ar 44100 -ac 1 "${episode_tts_flac}"
check_variable_is_correct episode_tts_flac
# A level of silence is added at the beginning of the text to speech
sox -V2 "${silence}" "${working_processing_dir}//episode_tts.flac" "${working_processing_dir}/episode_summary.flac"
sox -V2 "${silence}" "${episode_tts_flac}" "${episode_summary_flac}"
check_variable_is_correct episode_summary_flac
# The tracks are merged together resulting in the theme playing first, then after a period of silence the text to speech enters
sox -V2 -m "${working_processing_dir}/episode_summary.flac" "${theme}" "${working_processing_dir}/episode_intro.flac"
if [[ ! -s "${working_processing_dir}//episode_tts.flac" || ! -s "${working_processing_dir}/episode_summary.flac" || ! -s "${working_processing_dir}/episode_intro.flac" ]]
then
echo_error "The files for the theme audio sandwich are not available."
ls -al "${working_processing_dir}//episode_tts.flac" "${working_processing_dir}/episode_summary.flac" "${theme}" "${working_processing_dir}/episode_intro.flac"
fi
sox -V2 -m "${episode_summary_flac}" "${theme}" ${episode_intro_flac}
check_variable_is_correct episode_intro_flac
}
#################################################
@ -1337,21 +1436,25 @@ function generate_intro() {
function generate_parent_audio() {
echo_debug "Generating the parent audio - the sandwitch. generate_parent_audio()"
if [[ ! -s "${working_processing_dir}/episode_intro.flac" || ! -s "${media}" || ! -s "${outro_flac}" ]]
if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]]
then
echo_error "The files for the sandwich are not available."
ls -al
echo_debug "Skipping media creation from generate_parent_audio()"
return
fi
check_variable_is_correct episode_intro_flac media outro_flac
# Everything needs to be in the same format so the text to speech needs to be converted to 2 channel Sampling rate 44.1 kHz
ffmpeg -hide_banner -loglevel error -y -i "${media}" -ar 44100 -ac 1 "${working_processing_dir}/episode_body.flac"
ffmpeg -hide_banner -loglevel error -y -i "${media}" -ar 44100 -ac 1 "${episode_body_flac}"
check_variable_is_correct episode_body_flac
# Combine the components together
sox -V2 "${working_processing_dir}/episode_intro.flac" "${working_processing_dir}/episode_body.flac" "${outro_flac}" "${working_processing_dir}/episode_sandwitch.flac"
sox -V2 ${episode_intro_flac} "${episode_body_flac}" "${outro_flac}" "${episode_sandwitch_flac}"
check_variable_is_correct episode_sandwitch_flac
# Normalise the audio
ffmpeg -hide_banner -loglevel error -y -i "${working_processing_dir}/episode_sandwitch.flac" -af loudnorm=I=-16:LRA=11:TP=-1.5 "${working_processing_dir}/episode_final.flac"
ffmpeg -hide_banner -loglevel error -y -i "${episode_sandwitch_flac}" -af loudnorm=I=-16:LRA=11:TP=-1.5 "${episode_final_flac}"
check_variable_is_correct episode_final_flac
}
@ -1362,9 +1465,15 @@ function generate_derived_media() {
echo_debug "Generating derived audio. generate_derived_media()"
if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]]
then
echo_debug "Skipping media creation from generate_derived_media()"
return
fi
check_variable_is_correct media working_dir ep_num title artist license
if [[ ! -s "${working_processing_dir}/episode_final.flac" ]]
if [[ ! -s "${episode_final_flac}" ]]
then
ls -al
echo_error "The final cut is not available."
@ -1374,10 +1483,10 @@ function generate_derived_media() {
# https://wiki.multimedia.cx/index.php?title=FFmpeg_Metadata
for extension in flac wav mp3 ogg opus spx
for extension in "${audio_formats[@]}"
do
echo_debug "Generating \"hpr${ep_num}.${extension}\"."
ffmpeg -hide_banner -loglevel error -y -i "${working_processing_dir}/episode_final.flac" \
ffmpeg -hide_banner -loglevel error -y -i "${episode_final_flac}" \
-metadata title="${title}" \
-metadata artist="${artist}" \
-metadata author="${artist}" \
@ -1401,15 +1510,44 @@ function generate_derived_media() {
done
#TODO fix close duration
lengths=$( for extension in flac wav mp3 ogg opus
# shownotes_json="${working_dir}/shownotes.json"
# shownotes_html="${working_dir}/shownotes.html"
# shownotes_edited="${working_dir}/shownotes_edited.html"
# episode_tts_wav="${working_dir}/episode_tts.wav"
# episode_tts_flac="${working_dir}/episode_tts.flac"
# episode_summary_flac="${working_dir}/episode_summary.flac"
# episode_intro_flac="${working_dir}/episode_intro.flac"
# episode_body_flac="${working_dir}/episode_body.flac"
# episode_sandwitch_flac="${working_dir}/episode_sandwitch.flac"
# episode_final_flac="${working_dir}/episode_final.flac"
# episode_intro_srt="${working_dir}/episode_intro.srt"
# episode_body_srt="${working_dir}/episode_body.srt"
# episode_outro_srt="${working_dir}/episode_outro.srt"
# episode_srt="${working_dir}/episode.srt"
check_variable_is_correct episode_intro_flac outro_flac media duration
intro_duration="$( mediainfo --Output=XML --Full "${episode_intro_flac}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
outro_duration="$( mediainfo --Output=XML --Full "${outro_flac}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
source_duration="$( mediainfo --Output=XML --Full "${media}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
expected_duration=$(( ${intro_duration} + ${duration} + ${outro_duration} ))
check_variable_is_correct intro_duration outro_duration source_duration expected_duration
echo_debug "Intro(${intro_duration}) + Show(${duration}) + Outro(${outro_duration}) = ${expected_duration}"
for extension in "${audio_formats[@]}"
do
mediainfo --full --Output=XML "${working_dir}/hpr${ep_num}.${extension}" | xmlstarlet sel -T -t -m "_:MediaInfo/_:media/_:track[@type='Audio']/_:Duration[1]" -v "." -n - | awk -F '.' '{print $1}'
done | sort | uniq | wc -l )
if [ "${lengths}" -ne "1" ]
then
echo_error "The duration of the derived media is not correct."
fi
#TODO fix close duration
this_duration=$( mediainfo --full --Output=XML "${working_dir}/hpr${ep_num}.${extension}" | xmlstarlet sel -T -t -m "_:MediaInfo/_:media/_:track[@type='Audio']/_:Duration[1]" -v "." -n - | awk -F '.' '{print $1}' )
if [ $( abs_diff "${this_duration}" "${expected_duration}" ) -le "${acceptable_duration_difference}" ]
then
echo_debug "The file \"hpr${ep_num}.${extension}\" duration of ${this_duration} is close enough to ${expected_duration}"
else
echo_error "The file \"hpr${ep_num}.${extension}\" actual duration of ${this_duration} is not close enough to posted duration of ${expected_duration}. Fix or update the posted duration to ${source_duration}."
fi
done
cp -v "${media}" "${working_dir}/hpr${ep_num}_source.${media##*.}"
if [[ ! -s "${working_dir}/hpr${ep_num}_source.${media##*.}" ]]
@ -1426,12 +1564,18 @@ function generate_derived_media() {
function generate_show_transcript() {
echo_debug "Generate show transcript and subtitles. generate_show_transcript()"
if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]]
then
echo_debug "Skipping media creation from generate_show_transcript()"
return
fi
# TODO Currently processed elsewhere by hpr-get-and-transcode.bash and uploaded to hpr:upload/ to be synced with media above
if [[ ! -s "${media}" || ! -s "${media%.*}.srt" || ! -s "${intro_srt}" || ! -s "${outro_srt}" || ! -s "${working_processing_dir}/episode_intro.flac" || ! -s "${working_processing_dir}/episode_body.flac" ]]
if [[ ! -s "${media}" || ! -s "${media%.*}.srt" || ! -s "${intro_srt}" || ! -s "${outro_srt}" || ! -s ${episode_intro_flac} || ! -s "${episode_body_flac}" ]]
then
ls -al "${media}" "${media%.*}.srt" "${intro_srt}" "${outro_srt}" "${working_processing_dir}/episode_intro.flac" "${working_processing_dir}/episode_body.flac"
ls -al "${media}" "${media%.*}.srt" "${intro_srt}" "${outro_srt}" ${episode_intro_flac} "${episode_body_flac}"
echo_error "The transcriptions files are not available."
fi
@ -1458,32 +1602,32 @@ function generate_show_transcript() {
REPLACE_LINE_4="The flag is ${explicit}, and the license is ${license}"
REPLACE_LINE_5="The summary is \"${summary}\""
cp -v ${intro_srt} "${working_processing_dir}/episode_intro.srt"
cp -v ${outro_srt} "${working_processing_dir}/episode_outro.srt"
cp -v ${intro_srt} "${episode_intro_srt}"
cp -v ${outro_srt} "${episode_outro_srt}"
sed -e "s~REPLACE_LINE_1~${REPLACE_LINE_1}~g" -e "s~REPLACE_LINE_2~${REPLACE_LINE_2}~g" -e "s~REPLACE_LINE_3~${REPLACE_LINE_3}~g" -e "s~REPLACE_LINE_4~${REPLACE_LINE_4}~g" -e "s~REPLACE_LINE_5~${REPLACE_LINE_5}~g" -i "${working_processing_dir}/episode_intro.srt"
sed -e "s~REPLACE_LINE_1~${REPLACE_LINE_1}~g" -e "s~REPLACE_LINE_2~${REPLACE_LINE_2}~g" -e "s~REPLACE_LINE_3~${REPLACE_LINE_3}~g" -e "s~REPLACE_LINE_4~${REPLACE_LINE_4}~g" -e "s~REPLACE_LINE_5~${REPLACE_LINE_5}~g" -i "${episode_intro_srt}"
if [ "$( grep --count REPLACE_LINE "${working_processing_dir}/episode_intro.srt" )" -ne "0" ]
if [ "$( grep --count REPLACE_LINE "${episode_intro_srt}" )" -ne "0" ]
then
echo_error "The intro subtitles were not correctly generated \"${working_processing_dir}/episode_intro.srt\"."
echo_error "The intro subtitles were not correctly generated \"${episode_intro_srt}\"."
fi
# Time shift the media subtitles on by the duration of the intro wav file
# https://trac.ffmpeg.org/wiki/UnderstandingItsoffset
itsoffset_intro="$( mediainfo --full --Output=JSON "${working_processing_dir}/episode_intro.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )"
itsoffset_intro="$( mediainfo --full --Output=JSON ${episode_intro_flac} | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )"
if [[ -z "${itsoffset_intro}" || "${itsoffset_intro}" == "null" ]]
then
echo_error "Could not retrieve the itsoffset_intro to correct the timing of the subtitles."
fi
ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_intro}" -i "${media%.*}.srt" -c copy "${working_processing_dir}/episode_body.srt"
ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_intro}" -i "${media%.*}.srt" -c copy "${episode_body_srt}"
# Timeshift the outro by the duration of the intro and the supplied media
itsoffset_body="$( mediainfo --full --Output=JSON "${working_processing_dir}/episode_body.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )"
itsoffset_body="$( mediainfo --full --Output=JSON "${episode_body_flac}" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )"
if [[ -z "${itsoffset_body}" || "${itsoffset_body}" == "null" ]]
then
@ -1492,18 +1636,18 @@ function generate_show_transcript() {
itsoffset_body=$((itsoffset_intro + $itsoffset_body))
ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_body}" -i "${working_processing_dir}/episode_outro.srt" -c copy "${working_processing_dir}/episode_outro_shifted.srt"
ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_body}" -i "${episode_outro_srt}" -c copy "${working_dir}/episode_outro_shifted.srt"
# Combine the intro, timeshifted media subtitles, and the timeshifted outro subtitles.
cat "${working_processing_dir}/episode_intro.srt" "${working_processing_dir}/episode_body.srt" "${working_processing_dir}/episode_outro_shifted.srt" > "${working_processing_dir}/episode.srt"
cat "${episode_intro_srt}" "${episode_body_srt}" "${working_dir}/episode_outro_shifted.srt" > "${episode_srt}"
# Parse the resulting subtitle file fixing the numberic counter
# https://en.wikipedia.org/wiki/SubRip
count=1
cat "${working_processing_dir}/episode.srt" | while read this_line
cat "${episode_srt}" | while read this_line
do
if [ "$( echo "${this_line}" | grep -c --perl-regexp '^[0-9]+$' )" -eq "1" ]
then
@ -1533,17 +1677,17 @@ function generate_final_report() {
echo_debug "Generating the final report. generate_final_report()"
if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]]
then
echo_debug "The Episode hpr${ep_num} has already been posted. Skipping generate_final_report()"
echo_debug "Skipping media creation from ()"
return
fi
check_variable_is_correct working_dir working_processing_dir ep_num media_basename shownotes_edited synopsis
check_variable_is_correct working_dir ep_num media_basename shownotes_edited synopsis
final_report="${working_processing_dir}/hpr${ep_num}_report.html"
final_report="${working_dir}/hpr${ep_num}_report.html"
for this_file_extension_to_check in flac mp3 ogg opus srt txt wav
for this_file_extension_to_check in "${episode_formats[@]}"
do
if [[ ! -s "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" ]]
then
@ -1552,13 +1696,13 @@ function generate_final_report() {
fi
done
if [[ ! -s "${working_processing_dir}/${media_basename%.*}_media_report.html" ]]
if [[ ! -s "${working_dir}/${media_basename%.*}_media_report.html" ]]
then
ls -al "${working_processing_dir}/${media_basename%.*}_media_report.html"
echo_error "The initial report is not available.\"${working_processing_dir}/${media_basename%.*}_media_report.html\""
ls -al "${working_dir}/${media_basename%.*}_media_report.html"
echo_error "The initial report is not available.\"${working_dir}/${media_basename%.*}_media_report.html\""
fi
grep -Pv '</body>|</html>' "${working_processing_dir}/${media_basename%.*}_media_report.html" > "${final_report}"
grep -Pv '</body>|</html>' "${working_dir}/${media_basename%.*}_media_report.html" > "${final_report}"
echo "<h3 id=\"derived_media\" >Text To Speech</h3>
<p>
@ -1566,33 +1710,33 @@ $( echo "${synopsis}" )
</p>
<p>
<audio controls=\"\" preload=\"none\" style=\"width:600px;\" >
<source src=\"${working_processing_dir}//episode_tts.flac\" type=\"audio/flac\">
<source src=\"${episode_tts_flac}\" type=\"audio/flac\">
</audio>
<br />
<a href=\"${working_processing_dir}//episode_tts.flac\">${working_processing_dir}//episode_tts.flac</a>
<a href=\"${episode_tts_flac}\">${episode_tts_flac}</a>
</p>
<hr />" >> "${final_report}"
for this_file_extension_to_check in flac mp3 ogg opus wav
for this_file_extension_to_check in "${audio_formats[@]}"
do
ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" -lavfi "showspectrumpic=s=960x540" "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png"
ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" -lavfi "showspectrumpic=s=960x540" "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png"
audio2image.bash "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" && mv -v "${working_dir}/hpr${ep_num}.png" "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_waveform.png"
audio2image.bash "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" && mv -v "${working_dir}/hpr${ep_num}.png" "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_waveform.png"
mediainfo "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt"
mediainfo "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt"
exiftool "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt"
exiftool "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt"
ffprobe "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" 2>&1 | grep Audio: | sed 's/^.\s//g' > "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt"
file --brief --mime "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" >> "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt"
ffprobe "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" 2>&1 | grep Audio: | sed 's/^.\s//g' > "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt"
file --brief --mime "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" >> "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt"
this_file_mime_type="$( file --brief --mime-type "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" )"
for this_file_to_check in spectrum.png waveform.png mediainfo.txt exiftool.txt ffprobe.txt
do
if [[ ! -s "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" ]]
if [[ ! -s "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" ]]
then
ls -al "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}"
ls -al "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}"
echo_error "The inital report information is missing \"${this_file_to_check}\"."
fi
done
@ -1602,29 +1746,29 @@ $( echo "${synopsis}" )
<h3>mediainfo report</h3>
<code>
<pre>
$( cat "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" )
$( cat "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" )
</pre>
</code>
<h3>exiftool report</h3>
<code>
<pre>
$( cat "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" )
$( cat "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" )
</pre>
</code>
<h3>Audio Spectrum</h3>
<p>
<img src=\"${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png\" alt=\"Spectrum\" />
<img src=\"${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png\" alt=\"Spectrum\" />
</p>
<h3>Audio Waveform</h3>
<p>
<img src=\"${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_waveform.png\" alt=\"Waveform\" />
<img src=\"${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_waveform.png\" alt=\"Waveform\" />
</p>
<pre>
$( cat "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" )
$( cat "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" )
</pre>
<p>
@ -1677,9 +1821,9 @@ function manual_final_review() {
echo_debug "Validating the final report. manual_final_review()"
if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]]
then
echo_debug "The Episode hpr${ep_num} has already been posted. Skipping generate_final_report()"
echo_debug "Skipping media creation from ()"
return
fi
@ -1703,14 +1847,38 @@ function manual_final_review() {
}
#################################################
# Checks derived assets are correct
function check_derived_assets () {
check_variable_is_correct working_dir ep_num
#TODO Extend checks to media internals
for extension in "${audio_formats[@]}"
do
echo_debug "Checking \"hpr${ep_num}.${extension}\"."
if [[ ! -s "${working_dir}/hpr${ep_num}.${extension}" ]]
then
ls -al "${working_dir}/hpr${ep_num}.${extension}"
echo_error "Failed to locate \"${working_dir}/hpr${ep_num}.${extension}\"."
fi
done
}
#################################################
# Register the assets with the hpr database
function register_assets() {
echo_debug "Registering the assets with the hpr database. register_assets()"
check_variable_is_correct working_dir ep_num
check_derived_assets
assets_csv="${working_dir}/hpr${ep_num}_assets.csv"
assets_json="${working_dir}/hpr${ep_num}_assets.json"
@ -1723,7 +1891,8 @@ function register_assets() {
echo '"episode_id","filename","extension","size", "sha1sum", "mime_type", "file_type"' | tee "${assets_csv}"
for this_asset_filename in hpr${ep_num}.flac hpr${ep_num}.wav hpr${ep_num}.mp3 hpr${ep_num}.ogg hpr${ep_num}.opus hpr${ep_num}.srt hpr${ep_num}.txt $( find "${working_dir}/" -maxdepth 1 -type f -iname "hpr${ep_num}_image_*.*" )
find "${working_dir}/" -maxdepth 1 -type f \( -iname "hpr${ep_num}.*" -or -iname "hpr${ep_num}_image_*.*" \) | \
while read for this_asset_filename
do
this_asset_filename="$( basename "${this_asset_filename}" )"
echo_debug "Registering \"${this_asset_filename}\"."
@ -1760,9 +1929,11 @@ function register_assets() {
echo "${ep_num},\"${this_asset_basename}\",\"${this_asset_extension}\",\"${this_asset_size}\",\"${this_asset_sha1sum}\",\"${this_asset_mime_type}\",\"${this_asset_file_type}\"" | tee --append "${assets_csv}"
done
done
check_variable_is_correct assets_csv
if [ -s "${assets_csv}" ]
if [[ -s "${assets_csv}" && "$( wc -l "${assets_csv}" )" -gt "1" ]]
then
cat "${assets_csv}" | csvtojson | jq '{"assets":[.[]]}' | tee "${assets_json}"
fi
@ -2095,13 +2266,8 @@ echo "hidden"
#
# This tool will process the HPR shows allowing the janitor to review the media and fix shownotes.
argument_override "$@"
########################################################################################
# Posting show
program_checks # We know that all the programs and variables are set
if [[ -z "${working_dir_bypass}" || "${working_dir_bypass}" != "true" ]]
@ -2111,8 +2277,6 @@ else
echo_debug "Skipping get_working_dir()"
fi
set_working_processing_dir
if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output /dev/null )" == 200 ]
then
skip_post_show="true"