2025-06-01_18-03-47Z_Sunday

This commit is contained in:
Ken Fallon 2025-06-01 20:03:47 +02:00
parent 1dd1c8c100
commit 6f3c6c2596
2 changed files with 282 additions and 146 deletions

View File

@ -2,7 +2,14 @@
# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/
#============================================================
find ${HOME}/processing/ -type f | egrep -v '/sponsor-anhonesthost.com-hpr15.flac|/outro.flac|/intro.flac|/sponsor-archive.org.flac' | while read mediafile
search_dir="${HOME}/processing/"
if [ -d "${1}" ]
then
search_dir="${1}"
fi
find ${search_dir} -type f | grep -vP '/sponsor-anhonesthost.com-hpr15.flac|/outro.flac|/intro.flac|/sponsor-archive.org.flac' | while read mediafile
do
duration=$( mediainfo --full --Output=XML "${mediafile}" | xmlstarlet sel -T -t -m "_:MediaInfo/_:media/_:track[@type='Audio']/_:Duration[1]" -v "." -n - | awk -F '.' '{print $1}' )
if [ "${duration}" != "" ]

View File

@ -20,6 +20,12 @@ intro_srt="${processing_dir}/intro.srt" # The intro subtitle template file
piper_bin="/opt/bin/piper/piper/piper" # The location of the text to speech engine
piper_voice="/opt/bin/piper/piper/piper-voices/en/en_US/lessac/medium/en_US-lessac-medium.onnx"
working_dir_bypass="false"
skip_post_show="false"
acceptable_duration_difference="2" # Seconds
unsafe_episode_variables=( shownotes_json shownotes_html hostid artist email title summary series_id series_name explicit license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised )
audio_formats=( flac wav mp3 ogg opus spx )
#################################################
# Display Error message, display help and exit
@ -39,6 +45,13 @@ function echo_debug() {
fi
}
#################################################
# Display Information message
function abs_diff() {
echo $(($1 >= $2 ? $1 - $2 : $2 - $1))
}
#################################################
# Display Help
@ -88,14 +101,50 @@ function program_checks() {
}
#################################################
# Allows override of arguments from the commandline
function argument_override() {
echo_debug "Overriding values with terminal arguments. argument_override()"
# Argument Override
if [ $# -gt 0 ]
then
for argument
do
if [[ $argument =~ ^[^=]+=.*$ ]]
then
this_key="${argument%=*}"
this_value="${argument#*=}"
for this_episode_variable in "${unsafe_episode_variables[@]}"
do
if [ "$( echo "${this_episode_variable}" | grep --perl-regexp --count "^${this_key}$" )" -ne "0" ]
then
this_value="$( echo "${this_value}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
echo_debug "URL encoding \"${this_key}\" as \"${this_value}\"."
fi
done
eval "${this_key}=${this_value}"
echo_debug "Replacing \"${this_key}\" with \"${this_value}\"."
check_variable_is_correct ${this_key}
if [ "${this_key}" == "working_dir" ]
then
working_dir_bypass="true"
set_working_processing_dir
fi
fi
done
fi
}
#################################################
# Variable Checks
#album artist assets_csv assets_json comment date duration duration_iso8601 email email_unpadded ep_date episode_summary_json ep_num explicit files_json files_xml genre HOME hostid intro_srt key license license_url outro_flac outro_srt piper_bin piper_voice processing_dir shownotes_edited shownotes_html silence summary synopsis tags theme timestamp_epoc title track working_dir year
function check_variable_is_correct() {
for argument in "$@"
do
case "${argument}" in
@ -296,14 +345,20 @@ function check_variable_is_correct() {
echo_error "The \"media\" variable is missing."
fi
;;
media_basename)
if [[ ! -d "${media_basename}" || -z "${media_basename}" ]]
then
echo_error "The \"media_basename\" variable is missing or is not a directory."
fi
;;
outro_flac)
if [[ ! -d "${outro_flac}" || -z "${outro_flac}" ]]
if [[ ! -s "${outro_flac}" || -z "${outro_flac}" ]]
then
echo_error "The \"outro_flac\" variable is missing or file \"${outro_flac}\" does not exist."
fi
;;
outro_srt)
if [[ ! -d "${outro_srt}" || -z "${outro_srt}" ]]
if [[ ! -s "${outro_srt}" || -z "${outro_srt}" ]]
then
echo_error "The \"outro_srt\" variable is missing or file \"${outro_srt}\" does not exist."
fi
@ -346,6 +401,21 @@ function check_variable_is_correct() {
# echo_error "The \"shownotes_html\" variable has not a valid \"text/html\" mime type. \"${shownotes_html}\""
# fi
;;
shownotes_json)
if [[ ! -s "${shownotes_json}" || -z "${shownotes_json}" ]]
then
echo_error "The \"shownotes_json\" variable/file is missing."
fi
if [ "$( file --brief --mime-type "${shownotes_json}" | grep --count 'application/json' )" -ne "1" ]
then
echo_error "The \"shownotes_json\" variable has not a valid \"application/json\" mime type."
fi
jq '.' "${shownotes_json}" >/dev/null 2>&1
if [ $? -ne 0 ]
then
echo_error "The file \"${shownotes_json}\" is not valid json."
fi
;;
silence)
if [[ ! -d "${silence}" || -z "${silence}" ]]
then
@ -371,7 +441,7 @@ function check_variable_is_correct() {
fi
;;
theme)
if [[ ! -d "${theme}" || -z "${theme}" ]]
if [[ ! -s "${theme}" || -z "${theme}" ]]
then
echo_error "The \"theme\" variable is missing or file \"${theme}\" does not exist."
fi
@ -400,6 +470,12 @@ function check_variable_is_correct() {
echo_error "The \"working_dir\" variable is missing or is not a directory."
fi
;;
working_processing_dir)
if [[ ! -d "${working_processing_dir}" || -z "${working_processing_dir}" ]]
then
echo_error "The \"working_processing_dir\" variable is missing or is not a directory."
fi
;;
year)
if [[ -z "${year}" || "${year}" == "null" ]]
then
@ -549,15 +625,25 @@ function get_working_dir() {
echo_debug "Found working directory as \"${working_dir}\""
if [ ! -d "${working_dir}/processing/" ]
then
mkdir -v "${working_dir}/processing/"
if [ ! -d "${working_dir}/processing/" ]
then
echo_error "Could not create the processing directory \"${working_dir}/processing/\"."
fi
fi
}
#################################################
# Create a sub dir to store technical data.
function set_working_processing_dir() {
echo_debug "Checking the processing directory in the current working directory. set_working_processing_dir()"
check_variable_is_correct working_dir
working_processing_dir="${working_dir}/processing"
if [ ! -d "${working_processing_dir}/" ]
then
mkdir -v "${working_processing_dir}/"
fi
check_variable_is_correct working_processing_dir
}
#################################################
@ -566,6 +652,14 @@ function get_working_dir() {
function get_episode_metadata() {
echo_debug "Extracting the episode metadata. get_episode_metadata()"
if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
then
echo_debug "The Episode hpr${ep_num} has already been posted. Skipping get_episode_metadata()"
return
fi
check_variable_is_correct working_dir
if [[ -s "${working_dir}/shownotes.json" && "$( file --brief --mime-type "${working_dir}/shownotes.json" | grep --count "application/json" )" -eq 0 ]]
then
@ -575,6 +669,8 @@ function get_episode_metadata() {
shownotes_json="${working_dir}/shownotes.json"
shownotes_html="${working_dir}/shownotes.html"
shownotes_edited="${working_dir}/shownotes_edited.html"
check_variable_is_correct shownotes_json
hostid="$( jq --raw-output '.host.Host_ID' ${shownotes_json} )"
artist="$( jq --raw-output '.host.Host_Name' ${shownotes_json} )"
@ -598,11 +694,8 @@ function get_episode_metadata() {
echo_debug "Extracting shownotes html from json file."
jq --raw-output '.episode.Show_Notes' "${shownotes_json}" > "${shownotes_html}"
if [ ! -s "${shownotes_html}" ]
then
echo_error "Failed to extract the shownote html file \"${shownotes_html}\""
fi
check_variable_is_correct shownotes_html
( echo '<!DOCTYPE HTML><html lang="en"><head><meta charset="utf-8" /></head><body>'
cat "${shownotes_html}"
@ -610,9 +703,9 @@ function get_episode_metadata() {
</html>'
) | sponge "${shownotes_html}"
episode_variables=( shownotes_json shownotes_html hostid artist email title summary series_id series_name explicit license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised )
unsafe_episode_variables=( shownotes_json shownotes_html hostid artist email title summary series_id series_name explicit license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised )
for episode_variable in "${episode_variables[@]}"
for episode_variable in "${unsafe_episode_variables[@]}"
do
if [[ -z ${!episode_variable} && "${episode_variable}" != "remote_media" ]]
then # indirect expansion here
@ -622,26 +715,7 @@ function get_episode_metadata() {
fi
done
# Argument Override
if [ $# -gt 0 ]
then
for argument
do
if [[ $argument =~ ^[^=]+=.*$ ]]
then
this_key="${argument%=*}"
if [ "$( echo "${episode_variables[*]}" | grep --count "${this_key}" )" -eq "0" ]
then
echo_debug "Skipping replacing \"${this_key}\".";
continue
fi
this_value="${argument#*=}"
this_value="$( echo "${this_value}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
eval "${this_key}=${this_value}"
echo_debug "Replacing \"${this_key}\" with \"${this_value}\"."
fi
done
fi
argument_override "$@"
# Hosts need to exist in the database
if [ "${hostid}" == '0' ]
@ -657,6 +731,12 @@ function get_episode_metadata() {
function extract_images_brute_force() {
echo_debug "Extracting images with grep. extract_images_brute_force()"
if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
then
echo_debug "The Episode hpr${ep_num} has already been posted. Skipping extract_images_brute_force()"
return
fi
if [ -s "${shownotes_edited}" ]
then
@ -803,6 +883,8 @@ function extract_images_brute_force() {
function media_checks() {
echo_debug "Running media checks. media_checks()"
check_variable_is_correct working_dir working_processing_dir
if [[ -n "${remote_media}" && "${remote_media}" != "null" ]]
then
@ -849,6 +931,8 @@ function media_checks() {
then
echo_error "Could not find the subtitles for media \"${media}\""
fi
#TODO fix close duration
# Find duration
duration=$( mediainfo --full --Output=JSON "${media}" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )
@ -857,6 +941,10 @@ function media_checks() {
echo_error "Invalid duration found in \"${media}\"" >&2
fi
echo_debug "The Duration is \"${duration}\" seconds from \"${media}\""
#TODO fix close duration
# Find number of channels ( 1=mono or 2=stereo)
supplied_channels=$( mediainfo --full --Output=JSON "${media}" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Channels' )
@ -867,20 +955,20 @@ function media_checks() {
echo_debug "The number of audio channels is \"${supplied_channels}\" from \"${media}\" ."
# Gernerate the Spectrum and Waveform image
ffmpeg -hide_banner -loglevel error -y -i "${media}" -lavfi "showspectrumpic=s=960x540" "${working_dir}/processing/${media_basename%.*}_spectrum.png"
audio2image.bash "${media}" && mv -v "${media%.*}.png" "${working_dir}/processing/${media_basename%.*}_waveform.png"
ffmpeg -hide_banner -loglevel error -y -i "${media}" -lavfi "showspectrumpic=s=960x540" "${working_processing_dir}/${media_basename%.*}_spectrum.png"
audio2image.bash "${media}" && mv -v "${media%.*}.png" "${working_processing_dir}/${media_basename%.*}_waveform.png"
# Getting metadata
mediainfo "${media}" > "${working_dir}/processing/${media_basename%.*}_mediainfo.txt"
mediainfo "${media}" > "${working_processing_dir}/${media_basename%.*}_mediainfo.txt"
exiftool "${media}" > "${working_dir}/processing/${media_basename%.*}_exiftool.txt"
exiftool "${media}" > "${working_processing_dir}/${media_basename%.*}_exiftool.txt"
for check_file in spectrum.png waveform.png mediainfo.txt exiftool.txt
do
if [ ! -s "${working_dir}/processing/${media_basename%.*}_${check_file}" ]
if [ ! -s "${working_processing_dir}/${media_basename%.*}_${check_file}" ]
then
echo_error "The ${check_file} file was not generated for the \"${working_dir}/processing/${media_basename%.*}_${check_file}\"" >&2
echo_error "The ${check_file} file was not generated for the \"${working_processing_dir}/${media_basename%.*}_${check_file}\"" >&2
fi
done
@ -902,6 +990,13 @@ function media_checks() {
function generate_initial_report() {
echo_debug "Generating the initial report. generate_initial_report()"
if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
then
echo_debug "The Episode hpr${ep_num} has already been posted. Skipping generate_initial_report()"
return
fi
# TODO list the images.
@ -964,25 +1059,25 @@ ${shownotes_json_sanatised}
<h3>mediainfo report</h3>
<code>
<pre>
$( cat "${working_dir}/processing/${media_basename%.*}_mediainfo.txt" )
$( cat "${working_processing_dir}/${media_basename%.*}_mediainfo.txt" )
</pre>
</code>
<h3>exiftool report</h3>
<code>
<pre>
$( cat "${working_dir}/processing/${media_basename%.*}_exiftool.txt" )
$( cat "${working_processing_dir}/${media_basename%.*}_exiftool.txt" )
</pre>
</code>
<h3>Audio Spectrum</h3>
<p>
<img src=\"${working_dir}/processing/${media_basename%.*}_spectrum.png\" alt=\"Spectrum\" />
<img src=\"${working_processing_dir}/${media_basename%.*}_spectrum.png\" alt=\"Spectrum\" />
</p>
<h3>Audio Waveform</h3>
<p>
<img src=\"${working_dir}/processing/${media_basename%.*}_waveform.png\" alt=\"Waveform\" />
<img src=\"${working_processing_dir}/${media_basename%.*}_waveform.png\" alt=\"Waveform\" />
</p>
<pre>
@ -1006,7 +1101,7 @@ $(cat "${shownotes_srt}" )
</pre>
<hr />
</body>
</html>" > "${working_dir}/processing/${media_basename%.*}_media_report.html"
</html>" > "${working_processing_dir}/${media_basename%.*}_media_report.html"
}
@ -1016,11 +1111,17 @@ $(cat "${shownotes_srt}" )
function manual_shownotes_review() {
echo_debug "Validating the initial report. manual_shownotes_review()"
if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
then
echo_debug "The Episode hpr${ep_num} has already been posted. Skipping manual_shownotes_review()"
return
fi
if [[ -z "${shownotes_html}" || ! -s "${shownotes_html}" || ! -s "${working_dir}/processing/${media_basename%.*}_media_report.html" ]]
if [[ -z "${shownotes_html}" || ! -s "${shownotes_html}" || ! -s "${working_processing_dir}/${media_basename%.*}_media_report.html" ]]
then
echo "shownotes_html: ${shownotes_html}"
ls -al "${shownotes_html}" "${working_dir}/processing/${media_basename%.*}_media_report.html"
ls -al "${shownotes_html}" "${working_processing_dir}/${media_basename%.*}_media_report.html"
echo_error "The files needed for to generate the inital report information are not available."
fi
@ -1038,7 +1139,7 @@ function manual_shownotes_review() {
fi
kate "${shownotes_edited}" >/dev/null 2>&1 &
librewolf "${working_dir}/processing/${media_basename%.*}_media_report.html" >/dev/null 2>&1 &
librewolf "${working_processing_dir}/${media_basename%.*}_media_report.html" >/dev/null 2>&1 &
seamonkey "${shownotes_edited}" >/dev/null 2>&1 &
# # # # bluefish "${shownotes_edited}" >/dev/null 2>&1 &
# https://markdowntohtml.com/
@ -1070,9 +1171,9 @@ function post_show_to_hpr_db() {
echo_debug "Posting the show to the HPR DB. post_show_to_hpr_db()"
if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output /dev/null )" == 200 ]
if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
then
echo "WARN: The Episode hpr${ep_num} has already been posted"
echo_debug "The Episode hpr${ep_num} has already been posted. Skipping post_show_to_hpr_db()"
return
fi
@ -1176,6 +1277,8 @@ function get_variables_from_episode_summary_json() {
echo_debug "Setting \"${episode_summary_key}\" to \"${episode_summary_value}\" from \"$( basename ${episode_summary_json} )\""
check_variable_is_correct ${episode_summary_key}
done
argument_override "$@"
duration_iso8601="$( \date -d@${duration} -u +%H:%M:%S )"
check_variable_is_correct duration_iso8601
@ -1189,15 +1292,15 @@ function create_tts_summary() {
echo_debug "Creating Text to Speech summary. create_tts_summary()"
check_variable_is_correct working_dir duration synopsis piper_bin piper_voice
check_variable_is_correct working_dir duration synopsis piper_bin piper_voice working_processing_dir
echo_debug "Converting text synopsis \"${synopsis}\" to speech."
echo "${synopsis}" | "${piper_bin}" --model "${piper_voice}" --output_file "${working_dir}/processing/episode_tts.wav"
echo "${synopsis}" | "${piper_bin}" --model "${piper_voice}" --output_file "${working_processing_dir}/episode_tts.wav"
if [ ! -s "${working_dir}/processing/episode_tts.wav" ]
if [ ! -s "${working_processing_dir}/episode_tts.wav" ]
then
echo_error "The text to speech episode summary was not created \"${working_dir}/processing/episode_tts.wav\"."
echo_error "The text to speech episode summary was not created \"${working_processing_dir}/episode_tts.wav\"."
fi
}
@ -1208,26 +1311,22 @@ function create_tts_summary() {
function generate_intro() {
echo_debug "Generating the intro. generate_intro()"
if [[ ! -s "${working_dir}/processing/episode_tts.wav" || ! -s "${theme}" || ! -s "${media}" || ! -s "${outro_flac}" || ! -d "${working_dir}/processing/" ]]
then
echo_error "The files for the intro are not available."
ls -al "${working_dir}/processing/episode_tts.wav" "${theme}" "${media}" "${outro_flac}" "${working_dir}/processing/"
fi
check_variable_is_correct working_dir working_processing_dir theme media outro_flac
# Everything needs to be in the same format for the intro, 1 channel (mono) Sampling rate 44.1 kHz
ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/processing/episode_tts.wav" -ar 44100 -ac 1 "${working_dir}/processing//episode_tts.flac"
ffmpeg -hide_banner -loglevel error -y -i "${working_processing_dir}/episode_tts.wav" -ar 44100 -ac 1 "${working_processing_dir}//episode_tts.flac"
# A level of silence is added at the beginning of the text to speech
sox -V2 "${silence}" "${working_dir}/processing//episode_tts.flac" "${working_dir}/processing/episode_summary.flac"
sox -V2 "${silence}" "${working_processing_dir}//episode_tts.flac" "${working_processing_dir}/episode_summary.flac"
# The tracks are merged together resulting in the theme playing first, then after a period of silence the text to speech enters
sox -V2 -m "${working_dir}/processing/episode_summary.flac" "${theme}" "${working_dir}/processing/episode_intro.flac"
sox -V2 -m "${working_processing_dir}/episode_summary.flac" "${theme}" "${working_processing_dir}/episode_intro.flac"
if [[ ! -s "${working_dir}/processing//episode_tts.flac" || ! -s "${working_dir}/processing/episode_summary.flac" || ! -s "${working_dir}/processing/episode_intro.flac" ]]
if [[ ! -s "${working_processing_dir}//episode_tts.flac" || ! -s "${working_processing_dir}/episode_summary.flac" || ! -s "${working_processing_dir}/episode_intro.flac" ]]
then
echo_error "The files for the theme audio sandwich are not available."
ls -al "${working_dir}/processing//episode_tts.flac" "${working_dir}/processing/episode_summary.flac" "${theme}" "${working_dir}/processing/episode_intro.flac"
ls -al "${working_processing_dir}//episode_tts.flac" "${working_processing_dir}/episode_summary.flac" "${theme}" "${working_processing_dir}/episode_intro.flac"
fi
}
@ -1239,20 +1338,20 @@ function generate_parent_audio() {
echo_debug "Generating the parent audio - the sandwitch. generate_parent_audio()"
if [[ ! -s "${working_dir}/processing/episode_intro.flac" || ! -s "${media}" || ! -s "${outro_flac}" ]]
if [[ ! -s "${working_processing_dir}/episode_intro.flac" || ! -s "${media}" || ! -s "${outro_flac}" ]]
then
echo_error "The files for the sandwich are not available."
ls -al
fi
# Everything needs to be in the same format so the text to speech needs to be converted to 2 channel Sampling rate 44.1 kHz
ffmpeg -hide_banner -loglevel error -y -i "${media}" -ar 44100 -ac 1 "${working_dir}/processing/episode_body.flac"
ffmpeg -hide_banner -loglevel error -y -i "${media}" -ar 44100 -ac 1 "${working_processing_dir}/episode_body.flac"
# Combine the components together
sox -V2 "${working_dir}/processing/episode_intro.flac" "${working_dir}/processing/episode_body.flac" "${outro_flac}" "${working_dir}/processing/episode_sandwitch.flac"
sox -V2 "${working_processing_dir}/episode_intro.flac" "${working_processing_dir}/episode_body.flac" "${outro_flac}" "${working_processing_dir}/episode_sandwitch.flac"
# Normalise the audio
ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/processing/episode_sandwitch.flac" -af loudnorm=I=-16:LRA=11:TP=-1.5 "${working_dir}/processing/episode_final.flac"
ffmpeg -hide_banner -loglevel error -y -i "${working_processing_dir}/episode_sandwitch.flac" -af loudnorm=I=-16:LRA=11:TP=-1.5 "${working_processing_dir}/episode_final.flac"
}
@ -1265,23 +1364,20 @@ function generate_derived_media() {
check_variable_is_correct media working_dir ep_num title artist license
if [[ ! -s "${working_dir}/processing/episode_final.flac" ]]
if [[ ! -s "${working_processing_dir}/episode_final.flac" ]]
then
ls -al
echo_error "The final cut is not available."
fi
comment="$( jq --raw-output '.comment' "${episode_summary_json}" )"
year="$( echo "${ep_date}" | cut -c -4 )"
check_variable_is_correct comment year
# https://wiki.multimedia.cx/index.php?title=FFmpeg_Metadata
for extension in flac wav mp3 ogg opus
for extension in flac wav mp3 ogg opus spx
do
echo_debug "Generating \"hpr${ep_num}.${extension}\"."
ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/processing/episode_final.flac" \
ffmpeg -hide_banner -loglevel error -y -i "${working_processing_dir}/episode_final.flac" \
-metadata title="${title}" \
-metadata artist="${artist}" \
-metadata author="${artist}" \
@ -1304,6 +1400,7 @@ function generate_derived_media() {
done
#TODO fix close duration
lengths=$( for extension in flac wav mp3 ogg opus
do
mediainfo --full --Output=XML "${working_dir}/hpr${ep_num}.${extension}" | xmlstarlet sel -T -t -m "_:MediaInfo/_:media/_:track[@type='Audio']/_:Duration[1]" -v "." -n - | awk -F '.' '{print $1}'
@ -1312,7 +1409,7 @@ function generate_derived_media() {
then
echo_error "The duration of the derived media is not correct."
fi
#TODO fix close duration
cp -v "${media}" "${working_dir}/hpr${ep_num}_source.${media##*.}"
if [[ ! -s "${working_dir}/hpr${ep_num}_source.${media##*.}" ]]
@ -1332,9 +1429,9 @@ function generate_show_transcript() {
# TODO Currently processed elsewhere by hpr-get-and-transcode.bash and uploaded to hpr:upload/ to be synced with media above
if [[ ! -s "${media}" || ! -s "${media%.*}.srt" || ! -s "${intro_srt}" || ! -s "${outro_srt}" || ! -s "${working_dir}/processing/episode_intro.flac" || ! -s "${working_dir}/processing/episode_body.flac" ]]
if [[ ! -s "${media}" || ! -s "${media%.*}.srt" || ! -s "${intro_srt}" || ! -s "${outro_srt}" || ! -s "${working_processing_dir}/episode_intro.flac" || ! -s "${working_processing_dir}/episode_body.flac" ]]
then
ls -al "${media}" "${media%.*}.srt" "${intro_srt}" "${outro_srt}" "${working_dir}/processing/episode_intro.flac" "${working_dir}/processing/episode_body.flac"
ls -al "${media}" "${media%.*}.srt" "${intro_srt}" "${outro_srt}" "${working_processing_dir}/episode_intro.flac" "${working_processing_dir}/episode_body.flac"
echo_error "The transcriptions files are not available."
fi
@ -1361,32 +1458,32 @@ function generate_show_transcript() {
REPLACE_LINE_4="The flag is ${explicit}, and the license is ${license}"
REPLACE_LINE_5="The summary is \"${summary}\""
cp -v ${intro_srt} "${working_dir}/processing/episode_intro.srt"
cp -v ${outro_srt} "${working_dir}/processing/episode_outro.srt"
cp -v ${intro_srt} "${working_processing_dir}/episode_intro.srt"
cp -v ${outro_srt} "${working_processing_dir}/episode_outro.srt"
sed -e "s~REPLACE_LINE_1~${REPLACE_LINE_1}~g" -e "s~REPLACE_LINE_2~${REPLACE_LINE_2}~g" -e "s~REPLACE_LINE_3~${REPLACE_LINE_3}~g" -e "s~REPLACE_LINE_4~${REPLACE_LINE_4}~g" -e "s~REPLACE_LINE_5~${REPLACE_LINE_5}~g" -i "${working_dir}/processing/episode_intro.srt"
sed -e "s~REPLACE_LINE_1~${REPLACE_LINE_1}~g" -e "s~REPLACE_LINE_2~${REPLACE_LINE_2}~g" -e "s~REPLACE_LINE_3~${REPLACE_LINE_3}~g" -e "s~REPLACE_LINE_4~${REPLACE_LINE_4}~g" -e "s~REPLACE_LINE_5~${REPLACE_LINE_5}~g" -i "${working_processing_dir}/episode_intro.srt"
if [ "$( grep --count REPLACE_LINE "${working_dir}/processing/episode_intro.srt" )" -ne "0" ]
if [ "$( grep --count REPLACE_LINE "${working_processing_dir}/episode_intro.srt" )" -ne "0" ]
then
echo_error "The intro subtitles were not correctly generated \"${working_dir}/processing/episode_intro.srt\"."
echo_error "The intro subtitles were not correctly generated \"${working_processing_dir}/episode_intro.srt\"."
fi
# Time shift the media subtitles on by the duration of the intro wav file
# https://trac.ffmpeg.org/wiki/UnderstandingItsoffset
itsoffset_intro="$( mediainfo --full --Output=JSON "${working_dir}/processing/episode_intro.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )"
itsoffset_intro="$( mediainfo --full --Output=JSON "${working_processing_dir}/episode_intro.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )"
if [[ -z "${itsoffset_intro}" || "${itsoffset_intro}" == "null" ]]
then
echo_error "Could not retrieve the itsoffset_intro to correct the timing of the subtitles."
fi
ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_intro}" -i "${media%.*}.srt" -c copy "${working_dir}/processing/episode_body.srt"
ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_intro}" -i "${media%.*}.srt" -c copy "${working_processing_dir}/episode_body.srt"
# Timeshift the outro by the duration of the intro and the supplied media
itsoffset_body="$( mediainfo --full --Output=JSON "${working_dir}/processing/episode_body.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )"
itsoffset_body="$( mediainfo --full --Output=JSON "${working_processing_dir}/episode_body.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )"
if [[ -z "${itsoffset_body}" || "${itsoffset_body}" == "null" ]]
then
@ -1395,18 +1492,18 @@ function generate_show_transcript() {
itsoffset_body=$((itsoffset_intro + $itsoffset_body))
ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_body}" -i "${working_dir}/processing/episode_outro.srt" -c copy "${working_dir}/processing/episode_outro_shifted.srt"
ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_body}" -i "${working_processing_dir}/episode_outro.srt" -c copy "${working_processing_dir}/episode_outro_shifted.srt"
# Combine the intro, timeshifted media subtitles, and the timeshifted outro subtitles.
cat "${working_dir}/processing/episode_intro.srt" "${working_dir}/processing/episode_body.srt" "${working_dir}/processing/episode_outro_shifted.srt" > "${working_dir}/processing/episode.srt"
cat "${working_processing_dir}/episode_intro.srt" "${working_processing_dir}/episode_body.srt" "${working_processing_dir}/episode_outro_shifted.srt" > "${working_processing_dir}/episode.srt"
# Parse the resulting subtitle file fixing the numberic counter
# https://en.wikipedia.org/wiki/SubRip
count=1
cat "${working_dir}/processing/episode.srt" | while read this_line
cat "${working_processing_dir}/episode.srt" | while read this_line
do
if [ "$( echo "${this_line}" | grep -c --perl-regexp '^[0-9]+$' )" -eq "1" ]
then
@ -1435,8 +1532,16 @@ function generate_show_transcript() {
function generate_final_report() {
echo_debug "Generating the final report. generate_final_report()"
if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
then
echo_debug "The Episode hpr${ep_num} has already been posted. Skipping generate_final_report()"
return
fi
check_variable_is_correct working_dir working_processing_dir ep_num media_basename shownotes_edited synopsis
final_report="${working_dir}/processing/hpr${ep_num}_report.html"
final_report="${working_processing_dir}/hpr${ep_num}_report.html"
for this_file_extension_to_check in flac mp3 ogg opus srt txt wav
do
@ -1447,13 +1552,13 @@ function generate_final_report() {
fi
done
if [[ ! -s "${working_dir}/processing/${media_basename%.*}_media_report.html" ]]
if [[ ! -s "${working_processing_dir}/${media_basename%.*}_media_report.html" ]]
then
ls -al "${working_dir}/processing/${media_basename%.*}_media_report.html"
echo_error "The initial report is not available.\"${working_dir}/processing/${media_basename%.*}_media_report.html\""
ls -al "${working_processing_dir}/${media_basename%.*}_media_report.html"
echo_error "The initial report is not available.\"${working_processing_dir}/${media_basename%.*}_media_report.html\""
fi
grep -Pv '</body>|</html>' "${working_dir}/processing/${media_basename%.*}_media_report.html" > "${final_report}"
grep -Pv '</body>|</html>' "${working_processing_dir}/${media_basename%.*}_media_report.html" > "${final_report}"
echo "<h3 id=\"derived_media\" >Text To Speech</h3>
<p>
@ -1461,33 +1566,33 @@ $( echo "${synopsis}" )
</p>
<p>
<audio controls=\"\" preload=\"none\" style=\"width:600px;\" >
<source src=\"${working_dir}/processing//episode_tts.flac\" type=\"audio/flac\">
<source src=\"${working_processing_dir}//episode_tts.flac\" type=\"audio/flac\">
</audio>
<br />
<a href=\"${working_dir}/processing//episode_tts.flac\">${working_dir}/processing//episode_tts.flac</a>
<a href=\"${working_processing_dir}//episode_tts.flac\">${working_processing_dir}//episode_tts.flac</a>
</p>
<hr />" >> "${final_report}"
for this_file_extension_to_check in flac mp3 ogg opus wav
do
ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" -lavfi "showspectrumpic=s=960x540" "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png"
ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" -lavfi "showspectrumpic=s=960x540" "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png"
audio2image.bash "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" && mv -v "${working_dir}/hpr${ep_num}.png" "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_waveform.png"
audio2image.bash "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" && mv -v "${working_dir}/hpr${ep_num}.png" "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_waveform.png"
mediainfo "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt"
mediainfo "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt"
exiftool "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt"
exiftool "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt"
ffprobe "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" 2>&1 | grep Audio: | sed 's/^.\s//g' > "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt"
file --brief --mime "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" >> "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt"
ffprobe "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" 2>&1 | grep Audio: | sed 's/^.\s//g' > "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt"
file --brief --mime "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" >> "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt"
this_file_mime_type="$( file --brief --mime-type "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" )"
for this_file_to_check in spectrum.png waveform.png mediainfo.txt exiftool.txt ffprobe.txt
do
if [[ ! -s "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" ]]
if [[ ! -s "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" ]]
then
ls -al "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}"
ls -al "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}"
echo_error "The inital report information is missing \"${this_file_to_check}\"."
fi
done
@ -1497,29 +1602,29 @@ $( echo "${synopsis}" )
<h3>mediainfo report</h3>
<code>
<pre>
$( cat "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" )
$( cat "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" )
</pre>
</code>
<h3>exiftool report</h3>
<code>
<pre>
$( cat "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" )
$( cat "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" )
</pre>
</code>
<h3>Audio Spectrum</h3>
<p>
<img src=\"${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png\" alt=\"Spectrum\" />
<img src=\"${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png\" alt=\"Spectrum\" />
</p>
<h3>Audio Waveform</h3>
<p>
<img src=\"${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_waveform.png\" alt=\"Waveform\" />
<img src=\"${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_waveform.png\" alt=\"Waveform\" />
</p>
<pre>
$( cat "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" )
$( cat "${working_processing_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" )
</pre>
<p>
@ -1571,6 +1676,12 @@ $(cat "${working_dir}/hpr${ep_num}.txt" )
function manual_final_review() {
echo_debug "Validating the final report. manual_final_review()"
if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
then
echo_debug "The Episode hpr${ep_num} has already been posted. Skipping generate_final_report()"
return
fi
if [[ -z "${final_report}" || ! -s "${final_report}" ]]
then
@ -1719,6 +1830,7 @@ function copy_files_to_origin_server() {
#################################################
# Send the derived files to the server borg to be sent to borg
function copy_derived_files_to_borg() {
echo_debug "Sending the derived files to the admin server borg. copy_derived_files_to_borg()"
@ -1938,12 +2050,6 @@ function hide_me() {
echo "hidden"
}
#################################################
# Main exceution starts here
#
# This tool will process the HPR shows allowing the janitor to review the media and fix shownotes.
#
# TODO Add support for reserve queue - process validate and move to reserve dir
# TODO Add support to reprocess processed shows - when given onlay new media reprocess it, update duration on hub, generate and download shownotes.{html,json} from db
@ -1955,27 +2061,6 @@ echo "hidden"
# TODO Add chapter support
# TODO incorporate direct upload to the IA
# TODO copy the files to the backup disk
# Get supplied working dir and ep_num if provided
if [ $# -gt 0 ]
then
# declare -A hash
for argument
do
if [[ $argument =~ ^[^=]+=.*$ ]]
then
key="${argument%=*}"
value="${argument#*=}"
eval "${key}=${value}"
echo_debug "Using supplied \"${key}\" of \"${value}\""
check_variable_is_correct ${key}
fi
done
fi
########################################################################################
# Posting show
#TODO add logic to check where we are and what we need to do.
# All functions should skip if their work is done. Except when mode=refresh
#
@ -1986,9 +2071,53 @@ fi
# - do the assets match hpr${ep_num}_assets.json
# -
# # # # Get supplied working dir and ep_num if provided
# # # if [ $# -gt 0 ]
# # # then
# # # # declare -A hash
# # # for argument
# # # do
# # # if [[ $argument =~ ^[^=]+=.*$ ]]
# # # then
# # # key="${argument%=*}"
# # # value="${argument#*=}"
# # # eval "${key}=${value}"
# # # echo_debug "Using supplied \"${key}\" of \"${value}\""
# # # check_variable_is_correct ${key}
# # # fi
# # # done
# # # fi
}
#################################################
# Main exceution starts here
#
# This tool will process the HPR shows allowing the janitor to review the media and fix shownotes.
argument_override "$@"
########################################################################################
# Posting show
program_checks # We know that all the programs and variables are set
get_working_dir $@ # We have a working directory and a valid json file
if [[ -z "${working_dir_bypass}" || "${working_dir_bypass}" != "true" ]]
then
get_working_dir $@ # We have a working directory and a valid json file
else
echo_debug "Skipping get_working_dir()"
fi
set_working_processing_dir
if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output /dev/null )" == 200 ]
then
skip_post_show="true"
echo_debug "The Episode hpr${ep_num} has already been posted."
fi
get_episode_metadata $@ # We have all the metadata we need to process the show.
@ -2005,7 +2134,7 @@ post_show_to_hpr_db # Posts the episode to HPR. Skips if it is already p
# #######################################################################################
# Using DB info from here
get_variables_from_episode_summary_json # Get the episode from HPR.
get_variables_from_episode_summary_json "$@" # Get the episode from HPR.
create_tts_summary # Generate text to speech summary