diff --git a/workflow/hpr-publish.bash b/workflow/hpr-publish.bash deleted file mode 100755 index 3a20bdd..0000000 --- a/workflow/hpr-publish.bash +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ -#============================================================ - -git_dir="$HOME/tmp/hpr/hpr_generator/sourcecode" -if [ ! -d "${git_dir}/.git" ] -then - git clone gitea@repo.anhonesthost.net:HPR/hpr_generator.git "${git_dir}" -fi - -cd "${git_dir}" - -git pull - -ssh hpr -t "ls -al /home/hpr/www/hpr.sql;md5sum /home/hpr/www/hpr.sql" -ssh hpr -t "/home/hpr/bin/hpr_db_backup.bash" -ssh hpr -t "ls -al /home/hpr/www/hpr.sql;md5sum /home/hpr/www/hpr.sql" - -./utils/update-hpr-db.sh -if [ $? -ne 0 ] -then - echo 'Terminating...' >&2 - exit 1 -fi - -./site-generator --all --verbose -if [ $? -ne 0 ] -then - echo 'Terminating...' >&2 - exit 1 -fi - -rsync -av --partial --progress "${git_dir}/public_html/" hpr:/home/hpr/public_html/ -rsync -av --partial --progress "${git_dir}/public_html/" hobbypublicradio.org:hobbypublicradio.org/ - -cd $HOME/sourcecode/hpr/hpr_hub/ -git pull -cd $HOME/sourcecode/hpr/hpr_hub/sql -split --hex-suffixes --lines=1000 --additional-suffix=.sql hpr.sql hpr-db-part- -cd $HOME/sourcecode/hpr/hpr_hub/ -git add $HOME/sourcecode/hpr/hpr_hub/sql/hpr*sql -git commit -m "$(\date -u +%Y-%m-%d_%H-%M-%SZ_%A ) database changed" -git push -#xdg-open https://hackerpublicradio.org/ diff --git a/workflow/hprtranscode.bash b/workflow/hprtranscode.bash deleted file mode 100755 index 9455020..0000000 --- a/workflow/hprtranscode.bash +++ /dev/null @@ -1,685 +0,0 @@ -#!/usr/bin/env bash -# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ -#============================================================ - -PATH=$PATH:$HOME/sourcecode/hpr/hpr_hub/bin/ -source $HOME/tmp/pip3.9/bin/activate -# https://hub.tcno.co/ai/whisper/install/ - -TEMP_DIR="/var/tmp/" -CHANNELS="2" -FIXAUDIO="1" -ARTIST="EMPTY" -TITLE="EMPTY" -YEAR="EMPTY" -SLOT="EMPTY" -basedir="/var/IA" -upload_dir="${basedir}/uploads" -theme="${basedir}/theme.wav" -outro="${basedir}/2022-03-07-outro.wav" -outro_c2="${basedir}/2022-03-07-outro_c2.wav" -ttsserver="http://localhost:5500" -processing_dir="$HOME/tmp/hpr/processing" -git_image_dir="$HOME/sourcecode/hpr/HPR_Public_Code/www/images/hosts" -acceptable_duration_difference="2" # Seconds -thedate=$(/usr/bin/date -u +%Y-%m-%d_%H-%M-%SZ_%A) -echo "Processing the next HPR Show in the queue" - - -if [ $( curl -s -o /dev/null -w "%{http_code}" -X 'GET' "${ttsserver}/api/voices" -H 'accept: */*' ) != "200" ] -then - echo "Please start the tts-server \"podman run -it -p 5500:5500 synesthesiam/opentts:en\"" - exit -fi - -function abs_diff { - echo $(($1 >= $2 ? $1 - $2 : $2 - $1)) -} - -function create_tts_summary { - HPR_summary="$( cat "hpr${ep_num}_summary.txt" )" - echo "INFO: Converting text \"${HPR_summary}\" to speech." - curl -X 'GET' -G --data-urlencode "voice=coqui-tts:en_ljspeech" --data-urlencode "text=${HPR_summary}" --data-urlencode "vocoder=high" --data-urlencode "denoiserStrength=0.03" --data-urlencode "cache=false" ${ttsserver}/api/tts -H 'accept: */*' --output ~hpr${ep_num}_summary.wav -} - -################### -# Locate media directory -# -# -show_type="" -if [[ -f "${1}" && -n "${2}" ]] -then - echo "DEBUG: Show type is local" 1>&2 - show_type="local" - mediafile="${1}" - media_dir="$( dirname "${mediafile}" )" - ep_num="${2}" - echo "The duration is \"$( \date -ud "1970-01-01 $( ffprobe -i "${mediafile}" 2>&1| awk -F ': |, ' '/Duration:/ { print $2 }' )" +%s )\"." -else - echo "DEBUG: Show type is remote" 1>&2 - show_type="remote" - response=$( curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | \ - grep 'SHOW_POSTED' | \ - head -1 | \ - sed 's/,/ /g' ) - - echo "DEBUG: Getting server response" 1>&2 - if [ -z "${response}" ] - then - echo "INFO: There appear to be no more shows with the status \"SHOW_POSTED\"." - echo "Getting a list of all the reservations." - curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" - exit 3 - fi - - timestamp_epoc="$( echo ${response} | awk '{print $1}' )" - ep_num="$( echo ${response} | awk '{print $2}' )" - ep_date="$( echo ${response} | awk '{print $3}' )" - key="$( echo ${response} | awk '{print $4}' )" - status="$( echo ${response} | awk '{print $5}' )" - email="$( echo ${response} | awk '{print $6}' )" - - #source_dir="hpr:/home/hpr/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}" - dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}" - media_dir="${processing_dir}/${timestamp_epoc}_${ep_num}_${ep_date}_${key}" - -fi - -upload_dir_ep_num="${upload_dir}/hpr${ep_num}" -mkdir -p "${upload_dir_ep_num}" 2>/dev/null -if [ ! -d "${upload_dir_ep_num}" ] -then - echo "DEBUG: Missing directory \"upload_dir_ep_num\" \"${upload_dir}\", /hpr and \"${ep_num}\"" 1>&2 - exit -fi - -echo "DEBUG: Checking variables" 1>&2 -if [ -z "${show_type}" ] -then - echo "sorry, variable \"show_type\" is not set." - exit -fi - -if [ -z "${media_dir}" ] -then - echo "sorry, variable \"media_dir\" is not set." - exit -fi - -if [ ! -d "${media_dir}" ] -then - echo "sorry, directory \"media_dir: ${media_dir}\" does not exist" - exit 1 -fi - -echo detox -v "${media_dir}/" -detox -vr "${media_dir}/" - -echo "DEBUG: Using media directory as \"${media_dir}\"" - -if [[ "$( find "${media_dir}" \( -iname "hpr${ep_num}_sandwitch.wav" -o -iname "hpr${ep_num}.mp3" -o -iname "hpr${ep_num}.ogg" -o -iname "hpr${ep_num}.spx" -o -iname "hpr${ep_num}_summary.txt" -o -iname "hpr${ep_num}_summary.wav" -o -iname "hpr${ep_num}_sandwitch.wav" -o -iname "*_mez.wav" -o -iname "*_sox_norm.wav" -o -iname "*_mezzanine.wav" -o -iname "*_tmp.pcm" -o -iname "hpr${ep_num}_intro*.wav" -o -iname "~hpr${ep_num}_summary.wav" -o -iname "~~hpr${ep_num}_summary.wav" -o -iname "*_tmp.log" -o -iname "silence.wav" -o -iname "hpr${ep_num}_norm.wav" \) | wc -l )" -ne 0 ]] -then - echo "Files for this episode already exist." - find "${media_dir}" \( -iname "hpr${ep_num}_sandwitch.wav" -o -iname "hpr${ep_num}.mp3" -o -iname "hpr${ep_num}.ogg" -o -iname "hpr${ep_num}.spx" -o -iname "hpr${ep_num}_summary.txt" -o -iname "hpr${ep_num}_summary.wav" -o -iname "hpr${ep_num}_sandwitch.wav" -o -iname "*_mez.wav" -o -iname "*_sox_norm.wav" -o -iname "*_mezzanine.wav" -o -iname "*_tmp.pcm" -o -iname "hpr${ep_num}_intro*.wav" -o -iname "~hpr${ep_num}_summary.wav" -o -iname "~~hpr${ep_num}_summary.wav" -o -iname "*_tmp.log" -o -iname "silence.wav" -o -iname "hpr${ep_num}_norm.wav" \) - read -p "Shall I remove them ? (N|y) ? " -n 1 -r - echo # (optional) move to a new line - if [[ ! $REPLY =~ ^[yY]$ ]] - then - echo "skipping...." - exit - else - echo "Removing old files ...." - find "${media_dir}" \( -iname "hpr${ep_num}_sandwitch.wav" -o -iname "hpr${ep_num}.mp3" -o -iname "hpr${ep_num}.ogg" -o -iname "hpr${ep_num}.spx" -o -iname "hpr${ep_num}_summary.txt" -o -iname "hpr${ep_num}_summary.wav" -o -iname "hpr${ep_num}_sandwitch.wav" -o -iname "*_mez.wav" -o -iname "*_sox_norm.wav" -o -iname "*_mezzanine.wav" -o -iname "*_tmp.pcm" -o -iname "hpr${ep_num}_intro*.wav" -o -iname "~hpr${ep_num}_summary.wav" -o -iname "~~hpr${ep_num}_summary.wav" -o -iname "*_tmp.log" -o -iname "silence.wav" -o -iname "hpr${ep_num}_norm.wav" \) -delete -print - fi -fi - - -##################################################### -# Process media - -media=$( find "${media_dir}" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' ) -if [ -z "${media}" ] -then - echo "ERROR: Can't find any media in \"${media_dir}/\"" - find "${media_dir}/" -type f - exit 1 -fi - -mediafile="" - -echo "Found more than one media file. Please select which one to use ?" -if [ "$( echo "${media}" | wc -l )" -ne 1 ] -then - select this_media in $( echo "${media}" ) - do - echo "INFO: You selected \"${this_media}\"." - ls -al "${this_media}" - mediafile="${this_media}" - break - done -else - echo "INFO: Selecting media as \"${media}\"." - mediafile="${media}" -fi - -# extract file name and extension -fname="$( basename "${mediafile%.*}" )" -ext="${mediafile/*./}" - -cd "${media_dir}/" -pwd - -echo "INFO: Processing hpr${ep_num} from ${email}" -echo "INFO: Working directory is \"${media_dir}/\"" -echo "" - - -if [ -z "${mediafile}" ] -then - echo "sorry, variable \"mediafile\" is not set." - exit -fi - -if [ -z "${fname}" ] -then - echo "sorry, variable \"fname\" is not set." - exit -fi - -if [ -z "${ext}" ] -then - echo "sorry, variable \"ext\" is not set." - exit -fi - -if [ ! -f "${mediafile}" ] -then - echo "sorry, media file \"${mediafile}\" does not exist" - exit -fi - -echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log -echo "File information for \"${mediafile}\"" | tee -a ${fname}_tmp.log -ffprobe ${mediafile} 2>&1 | grep Audio: -mediainfo ${mediafile} -audio2image.bash ${mediafile} -xdg-open ${mediafile%.*}.png >/dev/null 2>&1 & -unset REPLY -until [[ $REPLY =~ ^[yYnN]$ ]] -do - read -p "Source Waveform look ok ? (N|y) ? " -n 1 -r - echo "" -done - -if [[ ! $REPLY =~ ^[yY]$ ]] -then - echo "skipping.... $REPLY" - exit -fi - - -re='^[0-9]+$' -if ! [[ $ep_num =~ $re ]] -then - echo "error: episode \"${ep_num}\" is not a number" - exit 1 -fi - -if [ ! -f "${outro}" ] -then - echo "sorry, file \"${outro}\" does not exist" - exit 1 -fi - -if [ ! -f "${outro_c2}" ] -then - echo "sorry, file \"${outro_c2}\" does not exist" - exit 1 -fi - - -if [ ! -f "${theme}" ] -then - echo "sorry, file \"${theme}\" does not exist" - exit 1 -fi - -if [ ! -r "${mediafile}" ] -then - echo "sorry, media file \"${mediafile}\" is not readable" - exit -fi - -if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ] -then - echo "sorry, media file \"${mediafile}\" has no audio track" - exit -fi - -xdg-open "https://hackerpublicradio.org/eps/hpr${ep_num}/index.html" >/dev/null 2>&1 & - -mpv -vo=null "${mediafile}" - -unset REPLY -until [[ $REPLY =~ ^[yYnN]$ ]] -do - read -p "Is the audio ok (n|Y) ? " -n 1 -r - echo "" -done - -if [[ ! $REPLY =~ ^[yY]$ ]] -then - echo "skipping.... $REPLY" - exit -fi - -echo "--------------------------------------------------------------------------------" -echo "Geting metadata for hpr${ep_num}" - -while read -r line -do - field=$(echo $line | awk -F ':' '{print $1}') - case $field in - "HPR_summary") - HPR_summary=$(echo $line | grep "HPR_summary: " | cut -c 14- ) - continue - ;; - "HPR_album") - HPR_album=$(echo $line | grep "HPR_album: " | cut -c 12- ) - continue - ;; - "HPR_artist") - HPR_artist=$(echo $line | grep "HPR_artist: " | cut -c 13- ) - continue - ;; - "HPR_comment") - HPR_comment=$(echo $line | grep "HPR_comment: " | cut -c 14- ) - continue - ;; - "HPR_genre") - HPR_genre=$(echo $line | grep "HPR_genre: " | cut -c 12- ) - continue - ;; - "HPR_title") - HPR_title=$(echo $line | grep "HPR_title: " | cut -c 12- ) - continue - ;; - "HPR_track") - HPR_track=$(echo $line | grep "HPR_track: " | cut -c 12- ) - continue - ;; - "HPR_year") - HPR_year=$(echo $line | grep "HPR_year: " | cut -c 11- ) - continue - ;; - "HPR_duration") - HPR_duration=$(echo $line | grep "HPR_duration: " | cut -c 15- ) - continue - ;; - "HPR_explicit") - HPR_explicit=$(echo $line | grep "HPR_explicit: " | cut -c 15- ) - continue - ;; - "HPR_license") - HPR_license=$(echo $line | grep "HPR_license: " | cut -c 14- ) - continue - ;; - esac -done < <( curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num}" ) - -if [[ -z "$HPR_album" || -z "$HPR_artist" || -z "$HPR_comment" || -z "$HPR_genre" || -z "$HPR_title" || -z "$HPR_track" || -z "$HPR_year" || -z "$HPR_summary" || -z "$HPR_duration" || -z "$HPR_explicit" || -z "$HPR_license" ]] -then - echo "Could not find information on ${ep_num}. Has the show been posted ?" - exit; -fi -echo "--------------------------------------------------------------------------------" -echo "album : $HPR_album" -echo "artist : $HPR_artist" -echo "comment : $HPR_comment" -echo "genre : $HPR_genre" -echo "title : $HPR_title" -echo "track : $HPR_track" -echo "year : $HPR_year" -echo "summary : $HPR_summary" -echo "duration : $HPR_duration" -echo "explicit : $HPR_explicit" -echo "license : $HPR_license" -echo "media_dir : ${media_dir}" -echo "mediafile : ${mediafile}" -echo "fname : ${fname}" -echo "ext : ${ext}" - -if [[ $HPR_duration == "0" ]] -then - echo "The duration is set to 0. Please update the show with the correct time." - exit; -fi - -#============================================================ -# Preproc`s the source file - -echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log -echo "Prepare mezzanine file" | tee -a ${fname}_tmp.log -ffmpeg -y -i ${mediafile} -ar 44100 -ac $CHANNELS ${fname}_mezzanine.wav >> ${fname}_tmp.log 2>&1 - -echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log -echo "Add HPR Branding" | tee -a ${fname}_tmp.log - -echo "Creating the summary" | tee -a ${fname}_tmp.log -#echo "$HPR_summary" - | text2wave -F 44100 - -o hpr${ep_num}_summary.wav #festival --tts -#echo "$HPR_summary" - | espeak -w ~hpr${ep_num}_summary.wav - -echo "$HPR_summary" > "hpr${ep_num}_summary.txt" - -create_tts_summary_ok="not-ok" - -while [ "${create_tts_summary_ok}" != "OK" ] -do - create_tts_summary - - xdg-open "hpr${ep_num}_summary.txt" 2>&1 & - mpv --speed=1.8 ~hpr${ep_num}_summary.wav - - read -p "Is the text to speech correct (y|N) ? " -n 1 -r - echo # (optional) move to a new line - if [[ $REPLY =~ ^[Yy]$ ]] - then - create_tts_summary_ok="OK" - else - echo "WARN: Please correct the text and try again." - inotifywait --event modify "hpr${ep_num}_summary.txt" - fi -done -echo "INFO: TTS complete." - -ffmpeg -y -i ~hpr${ep_num}_summary.wav -ar 44100 ~~hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1 - -sox -V2 -n -r 44100 -c 1 silence.wav trim 0.0 6.0 >> ${fname}_tmp.log 2>&1 - -sox -V2 silence.wav ~~hpr${ep_num}_summary.wav hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1 - -echo "Adding the theme" | tee -a ${fname}_tmp.log -sox -V2 -m "hpr${ep_num}_summary.wav" "${theme}" "hpr${ep_num}_intro.wav" >> ${fname}_tmp.log 2>&1 - - -if [ ${CHANNELS} -eq "2" ] -then - echo "Converting mono to Stero" | tee -a ${fname}_tmp.log - ffmpeg -y -i "hpr${ep_num}_intro.wav" -ac 2 "hpr${ep_num}_intro_c2.wav" - echo "Creating the sandwitch: \"hpr${ep_num}_intro_c2.wav\" \"${fname}_mezzanine.wav\" \"${outro}\" \"hpr${ep_num}_sandwitch.wav\" " | tee -a ${fname}_tmp.log - sox -V2 "hpr${ep_num}_intro_c2.wav" "${fname}_mezzanine.wav" "${outro_c2}" "hpr${ep_num}_sandwitch.wav" >> ${fname}_tmp.log 2>&1 -else - echo "Creating the sandwitch: \"hpr${ep_num}_intro.wav\" \"${fname}_mezzanine.wav\" \"${outro}\" \"hpr${ep_num}_sandwitch.wav\" " | tee -a ${fname}_tmp.log - sox -V2 "hpr${ep_num}_intro.wav" "${fname}_mezzanine.wav" "${outro}" "hpr${ep_num}_sandwitch.wav" >> ${fname}_tmp.log 2>&1 -fi - - - -echo "Normalizing the wav files" | tee -a ${fname}_tmp.log -#sox --temp "${TEMP_DIR}" --norm hpr${ep_num}_sandwitch.wav hpr${ep_num}_norm.wav -#ffmpeg -y -i hpr${ep_num}_sandwitch.wav -filter:a "dynaudnorm=p=0.9:s=5" hpr${ep_num}_norm.wav >> ${fname}_tmp.log 2>&1 -ffmpeg -y -i hpr${ep_num}.wav -af loudnorm=I=-16:LRA=11:TP=-1.5 hpr${ep_num}_norm.wav >> ${fname}_tmp.log 2>&1 - -if [ ! -d "${upload_dir_ep_num}" ] -then - echo "DEBUG: Missing directory \"upload_dir_ep_num\" \"${upload_dir}\", /hpr and \"${ep_num}\"" 1>&2 - exit -fi - -if [ ! -e "hpr${ep_num}_norm.wav" ] -then - echo "DEBUG: Missing file \"hpr${ep_num}_norm.wav\"" 1>&2 - exit -fi - -cp -v hpr${ep_num}_norm.wav ${upload_dir_ep_num}/hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1 - -if [ ! -e "${upload_dir_ep_num}/hpr${ep_num}.wav" ] -then - echo "DEBUG: Missing file \"${upload_dir_ep_num}/hpr${ep_num}.wav\"" 1>&2 - exit -fi - - -echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log -echo "File information" | tee -a ${fname}_tmp.log -ffprobe ${upload_dir_ep_num}/hpr${ep_num}.wav 2>&1 | grep Audio: -mediainfo ${upload_dir_ep_num}/hpr${ep_num}.wav -audio2image.bash ${upload_dir_ep_num}/hpr${ep_num}.wav -xdg-open ${upload_dir_ep_num}/hpr${ep_num}.png >/dev/null 2>&1 & -read -p "Processed Waveform look ok ? (N|y) ? " -n 1 -r -echo # (optional) move to a new line -if [[ ! $REPLY =~ ^[yY]$ ]] -then - echo "skipping...." - exit -fi -#rm -v ${upload_dir_ep_num}/hpr${ep_num}.png - - -echo "--------------------------------------------------------------------------------" -echo "Geting transcript of hpr${ep_num}" - -echo whisper --model tiny --language en --output_dir "${media_dir}" "${upload_dir_ep_num}/hpr${ep_num}.wav" | tee -a ${fname}_tmp.log -whisper --model tiny --language en --output_dir "${media_dir}" "${upload_dir_ep_num}/hpr${ep_num}.wav" | tee -a ${fname}_tmp.log - -ls -al "${media_dir}/hpr${ep_num}".* -xdg-open "${media_dir}/hpr${ep_num}".txt 2>&1 & -echo mpv --no-audio-display --audio-channels=stereo --speed="3.5" "${media_dir}/hpr${ep_num}".txt -unset REPLY -until [[ $REPLY =~ ^[yYnN]$ ]] -do - read -p "Processed transcript look ok ? (N|y) ? " -n 1 -r - echo "" -done - -if [[ ! $REPLY =~ ^[yY]$ ]] -then - echo "skipping.... $REPLY" - exit -fi - -mkdir "${upload_dir_ep_num}/hpr${ep_num}" >/dev/null 2>&1 - -cp -v "${media_dir}/hpr${ep_num}".vtt "${upload_dir_ep_num}/hpr${ep_num}.vtt" | tee -a ${fname}_tmp.log -cp -v "${media_dir}/hpr${ep_num}".srt "${upload_dir_ep_num}/hpr${ep_num}.srt" | tee -a ${fname}_tmp.log -cp -v "${media_dir}/hpr${ep_num}".txt "${upload_dir_ep_num}/hpr${ep_num}.txt" | tee -a ${fname}_tmp.log -cp -v "${media_dir}/hpr${ep_num}".tsv "${upload_dir_ep_num}/hpr${ep_num}.tsv" | tee -a ${fname}_tmp.log -cp -v "${media_dir}/hpr${ep_num}".json "${upload_dir_ep_num}/hpr${ep_num}.json" | tee -a ${fname}_tmp.log - -echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log -echo "Convert to opus" | tee -a ${fname}_tmp.log -ffmpeg -y -i ${upload_dir_ep_num}/hpr${ep_num}.wav ${upload_dir_ep_num}/hpr${ep_num}.opus 2>> ${fname}_tmp.log 1>&2 - -echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log -echo "Convert to flac" | tee -a ${fname}_tmp.log -ffmpeg -y -i ${upload_dir_ep_num}/hpr${ep_num}.wav ${upload_dir_ep_num}/hpr${ep_num}.flac 2>> ${fname}_tmp.log 1>&2 - -echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log -echo "Convert to mp3" | tee -a ${fname}_tmp.log -ffmpeg -y -i ${upload_dir_ep_num}/hpr${ep_num}.wav ${upload_dir_ep_num}/hpr${ep_num}.mp3 2>> ${fname}_tmp.log 1>&2 - -echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log -echo "Convert to ogg" | tee -a ${fname}_tmp.log -#ffmpeg -y -i ${upload_dir_ep_num}/hpr${ep_num}.wav ${upload_dir_ep_num}/hpr${ep_num}.ogg 2>> ${fname}_tmp.log 1>&2 -ffmpeg -y -i ${upload_dir_ep_num}/hpr${ep_num}.wav -acodec libopus -f ogg ${upload_dir_ep_num}/hpr${ep_num}.ogg 2>> ${fname}_tmp.log 1>&2 - -echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log -echo "Convert to spx" | tee -a ${fname}_tmp.log -ffmpeg -y -i ${upload_dir_ep_num}/hpr${ep_num}.wav ${upload_dir_ep_num}/hpr${ep_num}.spx 2>> ${fname}_tmp.log 1>&2 - -### End conversion - -intro_duration="$( mediainfo --Output=XML --Full "hpr${ep_num}_intro.wav" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" -outro_duration="$( mediainfo --Output=XML --Full "${outro}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" -source_duration="$( mediainfo --Output=XML --Full "${mediafile}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" -expected_duration=$(( ${intro_duration} + ${HPR_duration} + ${outro_duration} )) - -echo "Intro(${intro_duration}) + Show(${HPR_duration}) + Outro(${outro_duration}) = ${expected_duration}" - -media_error="0" - -for check_file in \ -${upload_dir_ep_num}/hpr${ep_num}.wav \ -${upload_dir_ep_num}/hpr${ep_num}.opus \ -${upload_dir_ep_num}/hpr${ep_num}.flac \ -${upload_dir_ep_num}/hpr${ep_num}.mp3 \ -${upload_dir_ep_num}/hpr${ep_num}.spx \ -${upload_dir_ep_num}/hpr${ep_num}.ogg -do - # ${upload_dir_ep_num}/hpr${ep_num}.spx - echo "INFO: Processing the file \"${check_file}\"" - if [[ ! -s "${check_file}" ]] - then - echo "ERROR: Something went wrong encoding of the file \"${check_file}\"" - exit - else - mediainfo --Output=XML --Full "${check_file}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' - this_duration=$( mediainfo --Output=XML --Full "${check_file}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' ) - if [[ $(abs_diff "${this_duration}" "${expected_duration}" ) -le "${acceptable_duration_difference}" ]] - then - echo "INFO: The file \"${check_file}\" duration of ${this_duration} () is close enough to ${expected_duration}" - else - echo "ERROR: The file \"${check_file}\" actual duration of ${this_duration} is not close enough to posted duration of ${expected_duration}." - echo " Fix or update the posted duration to ${source_duration}." - media_error="1" - fi - #${expected_duration} - fi -done -echo "Source: ${source_duration}" - -if [[ "${media_error}" -eq "1" ]] -then - echo "ERROR: Media is not encoded correctly" - exit -else - echo "INFO: Media duration is correct" -fi - - -if [[ ! -s ${upload_dir_ep_num}/hpr${ep_num}.wav ]] || [[ ! -s ${upload_dir_ep_num}/hpr${ep_num}.opus ]] || [[ ! -s ${upload_dir_ep_num}/hpr${ep_num}.flac ]] || [[ ! -s ${upload_dir_ep_num}/hpr${ep_num}.mp3 ]] || [[ ! -s ${upload_dir_ep_num}/hpr${ep_num}.ogg ]] || [[ ! -s ${upload_dir_ep_num}/hpr${ep_num}.spx ]] -then - echo "ERROR: Something went wrong encoding the files" - exit 1 -fi - - -echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log -echo "Fixing Tags" | tee -a ${fname}_tmp.log - -fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="${HPR_comment} The license is ${HPR_license}" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" ${upload_dir_ep_num}/hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2 - -fix_tags ${upload_dir_ep_num}/hpr${ep_num}* - -#echo "Changing the file dates to the time of upload" -touch -r ${mediafile} hpr${ep_num}* -touch -r ${mediafile} ${upload_dir_ep_num}/hpr${ep_num}* - -ls -al hpr${ep_num}* ${upload_dir_ep_num}/hpr${ep_num}* | tee -a ${fname}_tmp.log - -# # echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log -# # echo "Getting info for the asset table" | tee -a ${fname}_tmp.log -# # -# # echo "INSERT INTO assets (episode_id,filename,extension,size,sha1sum,mime_type,file_type) VALUES" -# # -# # for asset_file in \ -# # ${upload_dir_ep_num}/hpr${ep_num}.wav \ -# # ${upload_dir_ep_num}/hpr${ep_num}.opus \ -# # ${upload_dir_ep_num}/hpr${ep_num}.flac \ -# # ${upload_dir_ep_num}/hpr${ep_num}.mp3 \ -# # ${upload_dir_ep_num}/hpr${ep_num}.spx \ -# # ${upload_dir_ep_num}/hpr${ep_num}.ogg -# # do -# # size="$( ls -al "${asset_file}" | awk '{print $5}' )" -# # sha1sum="$( sha1sum "${asset_file}" | awk '{print $1}' )" -# # mime_type=$( file --dereference --brief --mime "${asset_file}" ) -# # file_type=$( file --dereference --brief "${asset_file}" ) -# # echo "(${ep_num},'${filename}','${extension}','${size}','${sha1sum}','${mime_type}','${file_type}')," -# # done - -echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log -echo "Transferring files to Servers" | tee -a ${fname}_tmp.log - - -#rsync -ave ssh --partial --progress --ignore-existing ${upload_dir_ep_num}/hpr${ep_num}.mp3 ${upload_dir_ep_num}/hpr${ep_num}.ogg ${upload_dir_ep_num}/hpr${ep_num}.spx hpr:www/eps/ - -#firefox "https://hackerpublicradio.org/local/hpr${ep_num}.mp3" >/dev/null 2>&1 & -#firefox "https://hackerpublicradio.org/local/hpr${ep_num}.ogg" >/dev/null 2>&1 & -firefox "file://${upload_dir_ep_num}/hpr${ep_num}.mp3" >/dev/null 2>&1 & -firefox "file://${upload_dir_ep_num}/hpr${ep_num}.ogg" >/dev/null 2>&1 & -#firefox "https://hackerpublicradio.org/eps.php?id=${ep_num}" >/dev/null 2>&1 & - -#mpv "http://hackerpublicradio.org/local/hpr${ep_num}.spx" "http://hackerpublicradio.org/local/hpr${ep_num}.ogg" "http://hackerpublicradio.org/local/hpr${ep_num}.mp3" "file://${upload_dir_ep_num}/hpr${ep_num}.spx" "file://${upload_dir_ep_num}/hpr${ep_num}.opus" "file://${upload_dir_ep_num}/hpr${ep_num}.ogg" "file://${upload_dir_ep_num}/hpr${ep_num}.mp3" "file://${upload_dir_ep_num}/hpr${ep_num}.flac" -mpv "file://${upload_dir_ep_num}/hpr${ep_num}.spx" "file://${upload_dir_ep_num}/hpr${ep_num}.opus" "file://${upload_dir_ep_num}/hpr${ep_num}.ogg" "file://${upload_dir_ep_num}/hpr${ep_num}.mp3" "file://${upload_dir_ep_num}/hpr${ep_num}.flac" - -read -p "Remove files for \"${fname}\" (y|N) ? " -n 1 -r -echo # (optional) move to a new line -if [[ $REPLY =~ ^[Yy]$ ]] -then - mediafilename=$(basename "${mediafile}") - mediaextension="${mediafilename##*.}" # " -# ssh hpr -t "mkdir /home/hpr/www/eps/hpr${ep_num}" >/dev/null 2>&1 -# rsync -ave ssh --partial --progress --ignore-existing "${mediafile}" hpr:www/eps/hpr${ep_num}/hpr${ep_num}_source.${mediaextension} | tee -a ${fname}_tmp.log -# -# rsync -ave ssh --partial --progress --ignore-existing "${media_dir}/hpr${ep_num}.wav".vtt hpr:www/eps/hpr${ep_num}/hpr${ep_num}.vtt | tee -a ${fname}_tmp.log -# rsync -ave ssh --partial --progress --ignore-existing "${media_dir}/hpr${ep_num}.wav".srt hpr:www/eps/hpr${ep_num}/hpr${ep_num}.srt | tee -a ${fname}_tmp.log -# rsync -ave ssh --partial --progress --ignore-existing "${media_dir}/hpr${ep_num}.wav".txt hpr:www/eps/hpr${ep_num}/hpr${ep_num}.txt | tee -a ${fname}_tmp.log -# f -# ssh hpr -t "ls -al /home/hpr/www/eps/hpr${ep_num}*" - cp -v "${mediafile}" "${upload_dir_ep_num}/hpr${ep_num}_source.${mediaextension}" - - #echo "Remove temp files" - # rm -v ~hpr${ep_num}_summary.wav ~~hpr${ep_num}_summary.wav silence.wav - # rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav - #mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/ - -# wget --timeout=0 -q "http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1" -O - | xmlstarlet val --err - -# wget --timeout=0 -q "http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1" -O - | xmlstarlet val --err - -# wget --timeout=0 -q "http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1" -O - | xmlstarlet val --err - -# wget --timeout=0 -q "http://hackerpublicradio.org/rss-future.php" -O - | xmlstarlet val --err - - - echo "INFO: rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* borg:/data/IA/uploads/" - rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* borg:/data/IA/uploads/ - echo "INFO: rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* rsync.net:processing/" - rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* rsync.net:processing/ - echo "$( ssh borg -t "ls -al /data/IA/uploads/hpr${ep_num}*" ; ssh rsync.net -t "ls -al rsync.net:processing/hpr${ep_num}*" ; ls -al ${upload_dir}/hpr${ep_num}* )" | grep "hpr${ep_num}" | awk '{print $5, $NF}' | sort -# ken@kalani: -# rsync -av --partial --progress /var/IA/uploads/ vger:processing/ -# $HOME/sourcecode/hpr/hpr_hub.wip/bin/hpr-assets.bash - -# root@vger:~# -# rsync -av --progress --partial $HOME/processing/ /mnt/data/hpr/eps/ -# find /mnt/data/hpr/eps/ -mindepth 1 -maxdepth 1 -type f | grep -vE 'assets.csv|assets.json' | while read i;do mv -fv ${i} $(dirname ${i})/$(basename ${i%.*} | cut -c -7 )/;done -# find /mnt/data/hpr/eps/ -type f \( -iname "*source*" -o -iname "*flac*" -o -iname "*wav*" \) -delete -print -# chcon -R --type=httpd_sys_rw_content_t /mnt/data/hpr/ -# chown apache:apache /mnt/data/hpr/ - -# ken@kalani: -# sshfs vger:/ /mnt/sshfs/vger/ -# sshfs rsync.net: /mnt/sshfs/rsync.net/ -# rsync -av --partial --progress /mnt/sshfs/vger/mnt/data/hpr/eps/ /mnt/sshfs/rsync.net/hpr/eps/ - -# https://hpr.nyc3.cdn.digitaloceanspaces.com/eps/hpr1404/hpr1404.mp3 -# https://alpha.nl.eu.mirror.hackerpublicradio.org/eps/hpr4271/hpr4271.mp3 -# https://archive.org/download/hpr4268/hpr4268.ogg - -# set eps.valid=1 - - if [ ${show_type} == "remote" ] - then - echo "INFO: Setting the status" - # SHOW_SUBMITTED → METADATA_PROCESSED → SHOW_POSTED → MEDIA_TRANSCODED → UPLOADED_TO_IA → UPLOADED_TO_RSYNC_NET - curl --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php?ep_num=${ep_num}&status=MEDIA_TRANSCODED" - curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" - fi -else - echo "skipping...." - echo "cp -v \"${mediafile}\" \"${upload_dir_ep_num}/hpr${ep_num}_source.${mediaextension}\"" -# echo "rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav" - #echo "mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/" - echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1\" -O - | xmlstarlet val --err -" - echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1\" -O - | xmlstarlet val --err -" - echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1\" -O - | xmlstarlet val --err -" - echo "rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* borg:/data/IA/uploads/" -fi diff --git a/workflow/intro.srt b/workflow/intro.srt new file mode 100644 index 0000000..90fab7c --- /dev/null +++ b/workflow/intro.srt @@ -0,0 +1,20 @@ +1 +00:00:00,000 --> 00:00:13,200 +REPLACE_LINE_1 + +2 +00:00:13,200 --> 00:00:16,400 +REPLACE_LINE_2 + +3 +00:00:16,400 --> 00:00:20,040 +REPLACE_LINE_3 + +4 +00:00:20,040 --> 00:00:22,080 +REPLACE_LINE_4 + +5 +00:00:22,080 --> 00:00:27,320 +REPLACE_LINE_5 + diff --git a/workflow/outro.flac b/workflow/outro.flac new file mode 100644 index 0000000..3b2dcf9 Binary files /dev/null and b/workflow/outro.flac differ diff --git a/workflow/outro.srt b/workflow/outro.srt new file mode 100644 index 0000000..bacf385 --- /dev/null +++ b/workflow/outro.srt @@ -0,0 +1,31 @@ +1 +00:00:00,000 --> 00:00:07,720 +You have been listening to Hacker Public Radio at Hacker Public Radio.org. + +2 +00:00:07,720 --> 00:00:11,520 +Today's show was contributed by a HPR listener like yourself. + +3 +00:00:11,520 --> 00:00:18,000 +If you ever thought of recording podcast, click on our upload link + +4 +00:00:18,000 --> 00:00:19,000 + to find out how easy it is. + +5 +00:00:19,000 --> 00:00:26,480 +Hosting for HPR has been kindly provided by an AnHonestHost.com, the Internet Archive, + +6 +00:00:26,480 --> 00:00:27,480 +rsync.net, and our mirror network. + +7 +00:00:27,480 --> 00:00:33,480 +Unless otherwise stated, today's show is released under a Creative Commons + +8 +00:00:33,480 --> 00:00:36,480 +Attribution-ShareAlike 4.0 International (CC BY-SA 4.0) license. diff --git a/workflow/postshow.bash b/workflow/postshow.bash deleted file mode 100755 index d057d4a..0000000 --- a/workflow/postshow.bash +++ /dev/null @@ -1,339 +0,0 @@ -#!/usr/bin/env bash -# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ -#============================================================ - -processing_dir="$HOME/tmp/hpr/processing" # The directory where the files will be copied to for processing - -if [ ! -d "${processing_dir}" ] -then - echo "ERROR: The application \"${this_program}\" is required but is not installed." - exit 1 -fi - -################### -# Check that all the programs are installed - -function is_installed () { - for this_program in "$@" - do - if ! command -v ${this_program} 2>&1 >/dev/null - then - echo "ERROR: The application \"${this_program}\" is required but is not installed." - exit 2 - fi - done -} - -is_installed awk base64 cat curl curl date detox eval ffprobe file find grep grep head jq jq kate magick mediainfo mv mv rsync rsync seamonkey sed sed sort sponge ssh touch touch wget - -echo "Processing the next HPR Show in the queue" - -################### -# Get the show -# -# Replaced METADATA_PROCESSED with SHOW_SUBMITTED -response=$( curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | \ -grep ',SHOW_SUBMITTED,' | \ -head -1 | \ -sed 's/,/ /g' ) - -if [ -z "${response}" ] -then - echo "INFO: There appear to be no more shows with the status \"SHOW_SUBMITTED\"." - echo "Getting a list of all the reservations." - curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | sort -n - exit 3 -fi - -timestamp_epoc="$( echo ${response} | awk '{print $1}' )" -ep_num="$( echo ${response} | awk '{print $2}' )" -ep_date="$( echo ${response} | awk '{print $3}' )" -key="$( echo ${response} | awk '{print $4}' )" -status="$( echo ${response} | awk '{print $5}' )" -email="$( echo ${response} | awk '{print $6}' )" -email_unpadded="$( echo $email | sed 's/.nospam@nospam./@/g' )" - -upload_dir="/home/hpr/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}" -source_dir="hpr:${upload_dir}" -publish_dir="hpr:www/eps/hpr${ep_num}" -dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}" - -ssh hpr -t "detox -v ${upload_dir}/" - -echo "INFO: Downloading hpr${ep_num} from ${email_unpadded}" -echo "" -echo rsync -ave ssh --partial --progress ${source_dir}/ ${processing_dir}/${dest_dir}/ -rsync -ave ssh --partial --progress ${source_dir}/ ${processing_dir}/${dest_dir}/ -echo "" -echo "INFO: Working directory is \"${processing_dir}/${dest_dir}/\"" -echo "" - -shownotes_json="${processing_dir}/${dest_dir}/shownotes.json" - -if [ ! -s "${shownotes_json}" ] -then - echo "ERROR: \"${shownotes_json}\" is missing" - exit 4 -fi - -if [ "$( file "${shownotes_json}" | grep -ic "text" )" -eq 0 ] -then - echo "ERROR: \"${shownotes_json}\" is not a text file" - exit 5 -fi - -mv -v "${shownotes_json}" "${shownotes_json%.*}_origional.json" - -jq '.' "${shownotes_json%.*}_origional.json" > "${shownotes_json}" - -################### -# Get the media -# - -remote_media="$( jq --raw-output '.metadata.url' "${shownotes_json}" )" - -if [ -n "${remote_media}" ] -then - echo "INFO: Fetching remote media from \"${remote_media}\"" - wget --timestamping --directory-prefix="${processing_dir}/${dest_dir}/" "${remote_media}" - if [ $? -ne 0 ] - then - echo "ERROR: Could not get the remote media" - exit 6 - fi -fi - -shownotes_html="${processing_dir}/${dest_dir}/shownotes.html" - -jq --raw-output '.episode.Show_Notes' "${shownotes_json}" > "${shownotes_html}" - -if [ ! -s "${shownotes_html}" ] -then - echo "ERROR: \"${shownotes_html}\" is missing" - exit 7 -fi - -# Process Shownotes -sed "s#>#>\n#g" "${shownotes_html}" | sponge "${shownotes_html}" - -# Extract Images -## TODO Temp fix until https://repo.anhonesthost.net/HPR/hpr-tools/issues/3 - -image_count="1" - -for image in $( grep --color=never --perl-regexp --only-matching 'data:image/[^;]*;base64,\K[a-zA-Z0-9+/=]*' "${shownotes_html}" ) -do - this_image="${processing_dir}/${dest_dir}/hpr${ep_num}_${image_count}" - echo -n "$image" | base64 -di > ${this_image} - this_ext="$( file --mime-type ${this_image} | awk -F '/' '{print $NF}' )" - mv -v "${this_image}" "${this_image}.${this_ext}" - this_width="$( mediainfo "${this_image}.${this_ext}" | grep Width | awk -F ': | pixels' '{print $2}' | sed 's/ //g' )" - if [ "${this_width}" -gt "400" ] - then - magick "${this_image}.${this_ext}" -resize 400x "${this_image}_tn.${this_ext}" - fi - ((image_count=image_count+1)) -done - -for image in $( grep --color=never --perl-regexp --only-matching '' "${shownotes_html}" | awk -F 'src=' '{print $2}' | awk -F '"' '{print $2}' ) -do - this_image="${processing_dir}/${dest_dir}/hpr${ep_num}_${image_count}" - wget "${image}" --output-document=${this_image} - this_ext="$( file --mime-type ${this_image} | awk -F '/' '{print $NF}' )" - if [ ! -e "${this_image}.${this_ext}" ] - then - mv -v "${this_image%.*}" "${this_image}.${this_ext}" - fi - this_width="$( mediainfo "${this_image}.${this_ext}" | grep Width | awk -F ': | pixels' '{print $2}' | sed 's/ //g' )" - if [ "${this_width}" -gt "400" ] - then - magick "${this_image}.${this_ext}" -resize 400x "${this_image}_tn.${this_ext}" - fi - ((image_count=image_count+1)) -done - -## TODO End Temp fix until https://repo.anhonesthost.net/HPR/hpr-tools/issues/3 - -ls -al "${processing_dir}/${dest_dir}/" - -## Manually edit the shownotes to fix issues - -kate "${shownotes_html}" >/dev/null 2>&1 & -# librewolf "${shownotes_html}" >/dev/null 2>&1 & -seamonkey "${shownotes_html}" >/dev/null 2>&1 & -# bluefish "${shownotes_html}" >/dev/null 2>&1 & - -read -p "Does the metadata 'look ok ? (N|y) ? " -n 1 -r -echo # (optional) move to a new line -if [[ ! $REPLY =~ ^[yY]$ ]] -then - echo "skipping...." - exit 8 -fi - -media=$( find "${processing_dir}/${dest_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' ) -if [ -z "${media}" ] -then - echo "ERROR: Can't find any media in \"${processing_dir}/${dest_dir}/\"" - find "${processing_dir}/${dest_dir}/" -type f - exit 9 -fi - -the_show_media="" - -if [ "$( echo "${media}" | wc -l )" -ne 1 ] -then - echo "Multiple files found. Which one do you want to use ?" - select this_media in $( echo "${media}" ) - do - echo "INFO: You selected \"${this_media}\"." - ls -al "${this_media}" - the_show_media="${this_media}" - break - done -else - echo "INFO: Selecting media as \"${media}\"." - the_show_media="${media}" -fi - -duration=$( \date -ud "1970-01-01 $( ffprobe -i "${the_show_media}" 2>&1| awk -F ': |, ' '/Duration:/ { print $2 }' )" +%s ) -if [ $? -ne 0 ] -then - echo 'ERROR: Invalid duration found in '\"${media}\" >&2 - exit 10 -fi - -################### -# Gather episode information -# - -if [ "$( curl --silent --write-out '%{http_code}' http://hackerpublicradio.org/say.php?id=${ep_num} --output /dev/null )" == 200 ] -then - echo "ERROR: The Episode hpr${ep_num} has already been posted" - exit 11 -fi - -if [ "$( jq --raw-output '.metadata.Episode_Number' ${shownotes_json} )" != "${ep_num}" ] -then - echo "ERROR: The Episode_Number: \"${ep_num}\" was not found in \"${shownotes_json}\"" - exit 12 -fi - -if [ "$( jq --raw-output '.metadata.Episode_Date' ${shownotes_json} )" != "${ep_date}" ] -then - echo "ERROR: The Episode_Date: \"${ep_date}\" was not found in \"${shownotes_json}\"" - exit 13 -fi - -if [ "$( jq --raw-output '.host.Host_Email' ${shownotes_json} )" != "${email_unpadded}" ] -then - echo "ERROR: The Host_Email: \"${email_unpadded}\" was not found in \"${shownotes_json}\"" - exit 14 -fi - -################### -# Assemble the components -# -# https://newbedev.com/how-to-urlencode-data-for-curl-command/ -hostid="$( jq --raw-output '.host.Host_ID' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" -host_name="$( jq --raw-output '.host.Host_Name' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" -title=$( jq --raw-output '.episode.Title' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' ) -summary="$( jq --raw-output '.episode.Summary' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" -series_id="$( jq --raw-output '.episode.Series' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" -series_name="$( jq --raw-output '.episode.Series_Name' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" -explicit="$( jq --raw-output '.episode.Explicit' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" -episode_license="$( jq --raw-output '.episode.Show_License' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" -tags="$( jq --raw-output '.episode.Tags' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" -host_license=$( jq --raw-output '.host.Host_License' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' ) -host_profile=$( jq --raw-output '.host.Host_Profile' ${shownotes_json} | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' ) - -notes="$( cat "${shownotes_html}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" - -if [ $# -gt 0 ] -then - declare -A hash - for argument - do - if [[ $argument =~ ^[^=]+=.*$ ]] - then - this_key="${argument%=*}" # "${} Kate format hack - this_value="${argument#*=}" # "${} Kate format hack - this_value="$( echo "${this_value}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" - eval "${this_key}=${this_value}" - echo "INFO: Replacing \"${this_key}\" with \"${this_value}\"." - fi - done -fi - -if [ "${hostid}" == '0' ] -then - echo "ERROR: The hostid is 0. Create the host and use \"hostid=???\" to override" - exit 15 -fi - -################### -# Post show to HPR -# - -post_show_json="${processing_dir}/${dest_dir}/post_show.json" -post_show_response="${processing_dir}/${dest_dir}/post_show_response.txt" - -echo "Sending:" -cat "${post_show}" -echo "key=${key} -ep_num=${ep_num} -ep_date=${ep_date} -email=${email} -title=${title} -duration=${duration} -summary=${summary} -series_id=${series_id} -series_name=${series_name} -explicit=${explicit} -episode_license=${episode_license} -tags=${tags} -hostid=${hostid} -host_name=${host_name} -host_license=${host_license} -host_profile=${host_profile} -notes=${notes}" - -echo "{ - \"key\": \"${key}\", - \"ep_num\": \"${ep_num}\", - \"ep_date\": \"${ep_date}\", - \"email\": \"${email}\", - \"title\": \"${title}\", - \"duration\": \"${duration}\", - \"summary\": \"${summary}\", - \"series_id\": \"${series_id}\", - \"series_name\": \"${series_name}\", - \"explicit\": \"${explicit}\", - \"episode_license\": \"${episode_license}\", - \"tags\": \"${tags}\", - \"hostid\": \"${hostid}\", - \"host_name\": \"${host_name}\", - \"host_license\": \"${host_license}\", - \"host_profile\": \"${host_profile}\", - \"notes\": \"${notes}\" -}" | tee "${post_show_json}" - -echo "INFO: Uploading processed files to \"${processing_dir}/${dest_dir}/\"" -echo "" -echo rsync -ave ssh --partial --progress ${processing_dir}/${dest_dir}/ ${source_dir}/ -rsync -ave ssh --partial --progress ${processing_dir}/${dest_dir}/ ${source_dir}/ -echo "" - -curl --netrc --include --request POST "https://hub.hackerpublicradio.org/cms/add_show_json.php" --header "Content-Type: application/json" --data-binary "@${post_show_json}" - -if [ "$( curl --silent --write-out '%{http_code}' http://hackerpublicradio.org/say.php?id=${ep_num} --output /dev/null )" != 200 ] -then - echo "ERROR: The Episode hpr${ep_num} has not been posted" - exit 16 -fi - -echo "INFO: Uploading completed files to \"${publish_dir}/\"" -echo "" -echo "rsync -ave ssh --partial --progress \"${processing_dir}/${dest_dir}/hpr${ep_num}\"* \"${publish_dir}/\"" -rsync -ave ssh --partial --progress "${processing_dir}/${dest_dir}/hpr${ep_num}"* "${publish_dir}/" -echo "" diff --git a/workflow/process_episode.bash b/workflow/process_episode.bash new file mode 100755 index 0000000..83f6ff7 --- /dev/null +++ b/workflow/process_episode.bash @@ -0,0 +1,1277 @@ +#!/usr/bin/env bash +# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ +#============================================================ + +# git clone https://github.com/davmo/fix_tags.git +# git clone https://repo.anhonesthost.net/HPR/hpr-tools.git + +################################################# +# Variables + +debug="1" +force_overwrite=true # Sets the behavour when files exist or not +processing_dir="$HOME/tmp/hpr/processing" # The directory where the files will be copied to for processing +theme="${processing_dir}/theme.flac" # The hpr theme +silence="${processing_dir}/silence.flac" # A segment of silence to offset the tts in the intro +outro_flac="${processing_dir}/outro.flac" # The outro edited media +outro_srt="${processing_dir}/outro.srt" # The outro subtitle file +intro_srt="${processing_dir}/intro.srt" # The intro subtitle template file + +piper_bin="/opt/bin/piper/piper/piper" # The location of the text to speech engine +piper_voice="/opt/bin/piper/piper/piper-voices/en/en_US/lessac/medium/en_US-lessac-medium.onnx" + +################################################# +# Display Error message, display help and exit + +function echo_error() { + echo -e "ERROR: $@" #1>&2 + display_help_and_exit + exit 1 +} + +################################################# +# Display Information message + +function echo_debug() { + if [ "${debug}" != "0" ] + then + echo -e "INFO: $@" #1>&2 + fi +} + +################################################# +# Display Help + +function display_help_and_exit() { + echo_debug "For more information see https://repo.anhonesthost.net/HPR/hpr_documentation" + exit 1 +} + +################################################# +# Program Checks + +function program_checks() { + + if [ ! -d "${processing_dir}" ] + then + echo_error "The \"${processing_dir}\" is required but is not defined." + fi + + if [[ ! -s "${theme}" || ! -s "${silence}" || ! -s "${outro_flac}" || ! -s "${outro_srt}" || ! -s "${intro_srt}" ]] + then + echo_error "The files for the theme are not available." + ls -al "${theme}" "${silence}" "${outro_flac}" "${outro_srt}" "${intro_srt}" + fi + + function is_installed() { + for this_program in "$@" + do + if ! command -v ${this_program} 2>&1 >/dev/null + then + echo_error "The application \"${this_program}\" is required but is not installed." + fi + done + } + + is_installed audio2image.bash awk base64 cat curl date detox eval extract_images ffprobe file find grep grep head jq kate magick mediainfo mv realpath rsync seamonkey sed sed sort sponge ssh touch touch wget + + for arg in $* + do + if [ "$( echo "${arg}" | grep --count --ignore-case --perl-regexp -- '-h|--help' )" -ne "0" ] + then + echo_debug "Process the next SHOW_SUBMITTED show from the queue." + echo_debug "If a directory is provided then the shownotes.json will be used." + fi + done + + if ${force_overwrite} + then + echo_debug "The setting \"force_overwrite\" is set to true, so files will be overwritten." + else + echo_debug "The setting \"force_overwrite\" is set to false, so when files exist the program will skip files if they exist." + fi + +} + +################################################# +# Get the next show in the queue + +function get_working_dir_from_hpr_hub() { + + echo_debug "Processing the next HPR Show in the queue" + + if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${processing_dir}/status.csv" )" != 200 ] + then + echo_error "Could not get a list of the queue status from \"https://hub.hackerpublicradio.org/cms/status.php\"" + fi + + if [ ! -s "${processing_dir}/status.csv" ] + then + echo_error "Failed to retrieve \"${processing_dir}/status.csv\" from server." + fi + + response=$( cat "${processing_dir}/status.csv" | grep ',SHOW_SUBMITTED,' | head -1 | sed 's/,/ /g' ) + if [ -z "${response}" ] + then + echo_debug "Getting a list of all the reservations." + curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | sort -n + echo_error "There appear to be no more shows with the status \"SHOW_SUBMITTED\"." + fi + + timestamp_epoc="$( echo ${response} | awk '{print $1}' )" + ep_num="$( echo ${response} | awk '{print $2}' )" + ep_date="$( echo ${response} | awk '{print $3}' )" + key="$( echo ${response} | awk '{print $4}' )" + email="$( echo ${response} | awk '{print $6}' )" + email_unpadded="$( echo $email | sed 's/.nospam@nospam./@/g' )" + + hpr_upload_dir="/home/hpr/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}" + source_dir="hpr:${hpr_upload_dir}" + dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}" + working_dir="${processing_dir}/${dest_dir}" + + echo_debug "Downloading hpr${ep_num} from ${email_unpadded}" + echo_debug "" + echo_debug "rsync -ave ssh --partial --progress ${source_dir}/ ${working_dir}/" + ssh hpr -t "detox -v ${hpr_upload_dir}/" + rsync -ave ssh --partial --progress ${source_dir}/ ${working_dir}/ + + + if [ ! -s "${working_dir}/shownotes.json" ] + then + echo_error "The working dir is missing the shownotes file \"${working_dir}/shownotes.json\"" + fi + + if [ "$( file --brief --mime-type "${working_dir}/shownotes.json" | grep --count "application/json" )" -eq 0 ] + then + echo_error "The \"${working_dir}/shownotes.json\" is not a \"application/json\" file" + fi + +} + +################################################# +# Get the show information from a local directory + +function get_working_dir_from_local_dir() { + + echo_debug "Processing a local directory" + + # Get supplied working dir and ep_num if provided + for argument in "$@" + do + + + if [ -d "${argument}" ] + then + working_dir="$( realpath ${argument} )" + fi + + if [ "$( echo "${argument}" | grep --count "ep_num=" )" -eq "1" ] + then + ep_num="$( echo "${argument}" | sed 's/^.*ep_num=//g' | awk '{print $1}' )" + fi + + done + + if [[ ! -d "${working_dir}" || -z "${working_dir}" ]] + then + echo_error "The working dir is missing. Please supply a working directory." + fi + + if [ ! -s "${working_dir}/shownotes.json" ] + then + + echo_debug "Could not find a \"shownotes.json\" in the working directory \"${working_dir}/\"" + + if [ -z "${ep_num}" ] + then + echo_debug "Attempting to get episode number from the \"${working_dir}\"" + ep_num="$( echo "${working_dir}" | grep --color=never --perl-regexp --only-matching '_[0-9]{4}_' | sed 's/_//g' )" + fi + + if [ -z "${ep_num}" ] + then + echo_error "Could not find the episode number - please rerun with \"$0 ep_num=9876\"" + fi + + echo_debug "Attempting to download information for episode \"${ep_num}\"" + + if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/shownotes.php?id=${ep_num} --output "${working_dir}/shownotes.json" )" != 200 ] + then + echo_error "The Episode hpr${ep_num} has not been posted." + fi + + if [ ! -s "${working_dir}/shownotes.json" ] + then + echo_error "The Episode information for hpr${ep_num} failed to download." + fi + + fi + + if [[ -s "${working_dir}/shownotes.json" && "$( file --brief --mime-type "${working_dir}/shownotes.json" | grep --count "application/json" )" -eq 0 ]] + then + echo_error "\"${working_dir}/shownotes.json\" is not a \"application/json\" file" + fi + +} + +################################################# +# Get the show either the next in the queue +# or from a local queue directory + +function get_working_dir() { + + echo_debug "Getting working directory and populating show information. $#" + + if [ $# -eq 0 ] + then + get_working_dir_from_hpr_hub + else + get_working_dir_from_local_dir $@ + fi + + if [ ! -s "${working_dir}/shownotes.json" ] + then + echo_error "The working dir \"${working_dir}\" could not be found." + fi + + echo_debug "Found working directory as \"${working_dir}\"" + + if [ ! -d "${working_dir}/processing/" ] + then + mkdir -v "${working_dir}/processing/" + if [ ! -d "${working_dir}/processing/" ] + then + echo_error "Could not create the processing directory \"${working_dir}/processing/\"." + fi + fi + +} + +################################################# +# Provides all the metadata we need to process the show. + +function get_episode_metadata() { + + echo_debug "Extracting the episode metadata" + + if [[ -s "${working_dir}/shownotes.json" && "$( file --brief --mime-type "${working_dir}/shownotes.json" | grep --count "application/json" )" -eq 0 ]] + then + echo_error "\"${working_dir}/shownotes.json\" is not a \"application/json\" file" + fi + + shownotes_json="${working_dir}/shownotes.json" + shownotes_html="${working_dir}/shownotes.html" + + hostid="$( jq --raw-output '.host.Host_ID' ${shownotes_json} )" + host_name="$( jq --raw-output '.host.Host_Name' ${shownotes_json} )" + email="$( jq --raw-output '.host.Host_Email' ${shownotes_json} )" + email_padded="$( echo $email | sed 's/@/.nospam@nospam./g' )" + title="$( jq --raw-output '.episode.Title' ${shownotes_json} )" + summary="$( jq --raw-output '.episode.Summary' ${shownotes_json} )" + series_id="$( jq --raw-output '.episode.Series' ${shownotes_json} )" + series_name="$( jq --raw-output '.episode.Series_Name' ${shownotes_json} )" + explicit="$( jq --raw-output '.episode.Explicit' ${shownotes_json} )" + episode_license="$( jq --raw-output '.episode.Show_License' ${shownotes_json} )" + ep_date="$( jq --raw-output '.metadata.Episode_Date' ${shownotes_json} )" + ep_num="$( jq --raw-output '.metadata.Episode_Number' ${shownotes_json} )" + key="$( jq --raw-output '.metadata.Key' ${shownotes_json} )" + tags="$( jq --raw-output '.episode.Tags' ${shownotes_json} )" + host_license="$( jq --raw-output '.host.Host_License' ${shownotes_json} )" + host_profile="$( jq --raw-output '.host.Host_Profile' ${shownotes_json} )" + remote_media="$( jq --raw-output '.metadata.url' ${shownotes_json} )" + shownotes_json_sanatised=$( jq 'del(.episode.Show_Notes, .metadata.Host_IP)' "${shownotes_json}" ) + + echo_debug "Extracting shownotes html from json file." + + jq --raw-output '.episode.Show_Notes' "${shownotes_json}" > "${shownotes_html}" + + if [ ! -s "${shownotes_html}" ] + then + echo_error "Failed to extract the shownote html file \"${shownotes_html}\"" + fi + + variables=( shownotes_json shownotes_html hostid host_name email title summary series_id series_name explicit episode_license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised ) + + for variable in "${variables[@]}" + do + if [[ -z ${!variable} && "${variable}" != "remote_media" ]] + then # indirect expansion here + echo_error "The variable \"${variable}\" is missing."; + else + echo_debug "The variable \"${variable}\" is set to \"${!variable}\""; + fi + done + + # Argument Override + if [ $# -gt 0 ] + then + declare -A hash + for argument + do + if [[ $argument =~ ^[^=]+=.*$ ]] + then + this_key="${argument%=*}" + this_value="${argument#*=}" + this_value="$( echo "${this_value}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" + eval "${this_key}=${this_value}" + echo_debug "Replacing \"${this_key}\" with \"${this_value}\"." + fi + done + fi + + # Hosts need to exist in the database + if [ "${hostid}" == '0' ] + then + echo_error "The hostid is 0. Create the host and use \"hostid=???\" to override" + fi + +} + +################################################# +# Extract_images by brute force + +function extract_images_brute_force() { + + echo_debug "Extracting images with grep." + + if [[ -z "${shownotes_html}" || ! -s "${shownotes_html}" ]] + then + echo_error "The shownotes_html file \"${shownotes_html}\" could not be found." + fi + + ## TODO Temp fix until https://repo.anhonesthost.net/HPR/hpr-tools/issues/3 is available + + sed "s#>#>\n#g" "${shownotes_html}" | sponge "${shownotes_html}" + + image_count="1" + + # Extract embedded images + + for image in $( grep --color=never --perl-regexp --only-matching 'data:image/[^;]*;base64,\K[a-zA-Z0-9+/=]*' "${shownotes_html}" ) + do + this_image="${working_dir}/hpr${ep_num}_image_${image_count}" + echo -n "$image" | base64 -di > ${this_image} + this_ext="$( file --mime-type ${this_image} | awk -F '/' '{print $NF}' )" + mv -v "${this_image}" "${this_image}.${this_ext}" + this_width="$( mediainfo "${this_image}.${this_ext}" | grep Width | awk -F ': | pixels' '{print $2}' | sed 's/ //g' )" + if [ "${this_width}" -gt "400" ] + then + magick "${this_image}.${this_ext}" -resize 400x "${this_image}_tn.${this_ext}" + fi + ((image_count=image_count+1)) + done + + # Download referenced images + + for image in $( grep --color=never --perl-regexp --only-matching '' "${shownotes_html}" | awk -F 'src=' '{print $2}' | awk -F '"' '{print $2}' ) + do + this_image="${working_dir}/hpr${ep_num}_image_${image_count}" + wget "${image}" --output-document=${this_image} + this_ext="$( file --mime-type ${this_image} | awk -F '/' '{print $NF}' )" + if [ ! -e "${this_image}.${this_ext}" ] + then + mv -v "${this_image%.*}" "${this_image}.${this_ext}" + fi + this_width="$( mediainfo "${this_image}.${this_ext}" | grep Width | awk -F ': | pixels' '{print $2}' | sed 's/ //g' )" + if [ "${this_width}" -gt "400" ] + then + magick "${this_image}.${this_ext}" -resize 400x "${this_image}_tn.${this_ext}" + fi + ((image_count=image_count+1)) + done + + # TODO Link up image_count > TODO Link up image_count by looping ` + +## TODO End Temp fix + +} + +################################################# +## Media Checks + +function media_checks() { + + if [ -n "${remote_media}" ] + then + echo_debug "Fetching remote media from \"${remote_media}\"" + wget --timestamping --directory-prefix="${working_dir}/" "${remote_media}" + if [ $? -ne 0 ] + then + echo_error "Could not get the remote media" + fi + fi + + media=$( find "${working_dir}/" -maxdepth 1 -type f -exec file --mime-type {} \; | grep -Ei ' audio/| video/' | awk -F ': ' '{print $1}' ) + if [ -z "${media}" ] + then + find "${working_dir}/" -type f + echo_error "Can't find any media in \"${working_dir}/\"" + fi + + media_basename="$( basename "${media}" )" + if [ -z "${media_basename}" ] + then + echo_error "Could not create the media_basename \"${media_basename}/\"" + fi + + if [ "$( echo "${media}" | wc -l )" -ne 1 ] + then + echo "Multiple files found. Which one do you want to use ?" + select this_media in $( echo "${media}" ) + do + ls -al "${this_media}" + media="${this_media}" + break + done + fi + echo_debug "You selected \"${media}\"." + + if [[ -z "${media}" || ! -s "${media}" ]] + then + echo_error "Could not find the media \"${media}/\"" + fi + + shownotes_srt="${media%.*}.srt" + if [[ -z "${shownotes_srt}" || ! -s "${shownotes_srt}" ]] + then + echo_error "Could not find the subtitles for media \"${media}/\"" + fi + + # Find duration + duration=$( mediainfo --full --Output=JSON "${media}" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' ) + if [[ -z "${duration}" || "${duration}" -lt "30" || "${duration}" -gt "30000" ]] + then + echo_error "Invalid duration found in \"${media}\"" >&2 + fi + echo_debug "The Duration is \"${duration}\" seconds from \"${media}\"" + + # Find number of channels ( 1=mono or 2=stereo) + supplied_channels=$( mediainfo --full --Output=JSON "${media}" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Channels' ) + if [[ -z "${supplied_channels}" || "${supplied_channels}" -lt "1" || "${supplied_channels}" -gt "2" ]] + then + echo_error "Invalid number of audio channels \"${supplied_channels}\" found in \"${media}\"" >&2 + fi + echo_debug "The number of audio channels is \"${supplied_channels}\" from \"${media}\" ." + + # Gernerate the Spectrum and Waveform image + ffmpeg -hide_banner -loglevel error -y -i "${media}" -lavfi "showspectrumpic=s=960x540" "${working_dir}/processing/${media_basename%.*}_spectrum.png" + audio2image.bash "${media}" && mv -v "${media%.*}.png" "${working_dir}/processing/${media_basename%.*}_waveform.png" + + # Getting metadata + + mediainfo "${media}" > "${working_dir}/processing/${media_basename%.*}_mediainfo.txt" + + exiftool "${media}" > "${working_dir}/processing/${media_basename%.*}_exiftool.txt" + + for check_file in spectrum.png waveform.png mediainfo.txt exiftool.txt + do + if [ ! -s "${working_dir}/processing/${media_basename%.*}_${check_file}" ] + then + echo_error "The ${check_file} file was not generated for the \"${working_dir}/processing/${media_basename%.*}_${check_file}\"" >&2 + fi + done + + ffprobe="$( ffprobe ${media} 2>&1 | grep Audio: | sed 's/^.\s//g' )" + file_mime="$( file --brief --mime ${media} )" + file_mime_type="$( file --brief --mime-type ${media} )" + + if [[ -z "${ffprobe}" || -z ${file_mime} || -s ${file_mime_type} ]] + then + echo "ffprobe: ${ffprobe}, file_mime: ${file_mime},file_mime_type: ${file_mime_type}" + echo_error "Wasn't able to find mime metadata from \"${media}/\"" + fi + +} + +################################################# +## Generate Initial Report for review by the Janitors + +function generate_initial_report() { + + echo_debug "Generating the initial report." + + # TODO list the images. + + echo " + + + Hacker Public Radio ~ The Technology Community Podcast + + + + + + +

Field Mapping

+ + + + + + + + + + + + + + + + + + + +
FieldValue
hostid${hostid}
host_name${host_name}
title${title}
summary${summary}
series_id${series_id}
series_name${series_name}
explicit${explicit}
episode_license${episode_license}
tags${tags}
host_license${host_license}
host_profile${host_profile}
+ +

Raw shownotes.json

+ +
+${shownotes_json_sanatised}
+
+ + + +

Audio

+ +

mediainfo report

+ +
+$( cat "${working_dir}/processing/${media_basename%.*}_mediainfo.txt" )
+
+ + +

exiftool report

+ +
+$( cat "${working_dir}/processing/${media_basename%.*}_exiftool.txt" )
+
+ + +

Audio Spectrum

+

+\"Spectrum\" +

+ +

Audio Waveform

+

+\"Waveform\" +

+ +
+${ffprobe}
+${file_mime}
+
+ +

+ +
+${media} +

+ +

Transcript

+ +
+
+$(cat "${shownotes_srt}" )
+
+
+ +" > "${working_dir}/processing/${media_basename%.*}_media_report.html" + +} + +################################################# +## Manually edit the shownotes to fix issues + +function manual_shownotes_review() { + + echo_debug "Validating the initial report." + + if [[ -z "${shownotes_html}" || ! -s "${shownotes_html}" || ! -s "${working_dir}/processing/${media_basename%.*}_media_report.html" ]] + then + echo "shownotes_html: ${shownotes_html}" + ls -al "${shownotes_html}" "${working_dir}/processing/${media_basename%.*}_media_report.html" + echo_error "The files needed for to generate the inital report information are not available." + fi + + if [ -s "${shownotes_html%.*}_edited.html" ] + then + echo_debug "There is already an edited version of the shownotes at \"${shownotes_html%.*}_edited.html\"." + return + fi + + cp -v "${shownotes_html}" "${shownotes_html%.*}_edited.html" + + if [ ! -s "${shownotes_html%.*}_edited.html" ] + then + echo_error "The edited shownotes are missing \"${shownotes_html%.*}_edited.html\"." + fi + + kate "${shownotes_html%.*}_edited.html" >/dev/null 2>&1 & + librewolf "${working_dir}/processing/${media_basename%.*}_media_report.html" >/dev/null 2>&1 & + seamonkey "${shownotes_html%.*}_edited.html" >/dev/null 2>&1 & + # # # # bluefish "${shownotes_html%.*}_edited.html" >/dev/null 2>&1 & + # https://markdowntohtml.com/ + + read -p "Does the metadata 'look ok ? (N|y) ? " -n 1 -r + echo # (optional) move to a new line + if [[ ! $REPLY =~ ^[yY]$ ]] + then + echo_error "The final review was not approved." + fi + + # remove extra wrappers that seamonkey adds + grep --invert-match --perl-regexp '|head>|' "${shownotes_html%.*}_edited.html" | sponge "${shownotes_html%.*}_edited.html" + + # Check to see if images have been linked TODO make a loop for found images + if [ "$( find "${working_dir}" -type f -iname "*_image_*" | wc -l )" -ne "0" ] + then + if [ "$( grep --count "_image_" "${shownotes_html%.*}_edited.html" )" -eq "0" ] + then + echo_error "The extracted images were not linked in the shownotes \"${shownotes_html%.*}_edited.html\"." + fi + fi +} + +################################################# +# Post show to HPR + +function post_show_to_hpr_db() { + + if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output /dev/null )" == 200 ] + then + echo "WARN: The Episode hpr${ep_num} has already been posted" + return + fi + + if [ ! -s "${shownotes_html%.*}_edited.html" ] + then + echo_error "Failed to find the extracted shownote html file \"${shownotes_html%.*}_edited.html\"" + fi + notes="$( cat "${shownotes_html%.*}_edited.html" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" + host_profile_encoded="$( echo "${host_profile}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )" + + + post_show_json="${working_dir}/post_show.json" + + echo "Sending:" + echo " key=${key} + ep_num=${ep_num} + ep_date=${ep_date} + email=${email_padded} + title=${title} + duration=${duration} + summary=${summary} + series_id=${series_id} + series_name=${series_name} + explicit=${explicit} + episode_license=${episode_license} + tags=${tags} + hostid=${hostid} + host_name=${host_name} + host_license=${host_license} + host_profile=${host_profile_encoded} + notes=REMOVED" + + echo "{ + \"key\": \"${key}\", + \"ep_num\": \"${ep_num}\", + \"ep_date\": \"${ep_date}\", + \"email\": \"${email_padded}\", + \"title\": \"${title}\", + \"duration\": \"${duration}\", + \"summary\": \"${summary}\", + \"series_id\": \"${series_id}\", + \"series_name\": \"${series_name}\", + \"explicit\": \"${explicit}\", + \"episode_license\": \"${episode_license}\", + \"tags\": \"${tags}\", + \"hostid\": \"${hostid}\", + \"host_name\": \"${host_name}\", + \"host_license\": \"${host_license}\", + \"host_profile\": \"${host_profile_encoded}\", + \"notes\": \"${notes}\" + }" > "${post_show_json}" + + jq '.' "${post_show_json}" + + if [ $? -ne 0 ] + then + echo_error "The file \"${post_show_json}\" is not valid json." + fi + + curl --netrc --include --request POST "https://hub.hackerpublicradio.org/cms/add_show_json.php" --header "Content-Type: application/json" --data-binary "@${post_show_json}" + + if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output /dev/null )" != 200 ] + then + echo_error "The Episode hpr${ep_num} has not been posted" + fi + +} + +################################################# +# Generate text to speech summary + +function create_tts_summary { + + if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output "${working_dir}/episode_summary.json" )" != 200 ] + then + echo_error "The Episode hpr${ep_num} has not been posted" + fi + + synopsis="$( jq --raw-output '.synopsis' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )" + + if [[ -z "${synopsis}" || "${synopsis}" == "null" ]] + then + echo_error "Could not retrieve the synopsis for the text to speech." + fi + + echo_debug "Converting text synopsis \"${synopsis}\" to speech." + + echo "${synopsis}" | "${piper_bin}" --model "${piper_voice}" --output_file "${working_dir}/processing/episode_tts.wav" + + if [ ! -s "${working_dir}/processing/episode_tts.wav" ] + then + echo_error "The text to speech episode summary was not created \"${working_dir}/processing/episode_tts.wav\"." + fi + +} + +################################################# +# Generate Intro + +function generate_intro { + + echo_debug "Generating the intro." + + if [[ ! -s "${working_dir}/processing/episode_tts.wav" || ! -s "${theme}" || ! -s "${media}" || ! -s "${outro_flac}" || ! -d "${working_dir}/processing/" ]] + then + echo_error "The files for the intro are not available." + ls -al "${working_dir}/processing/episode_tts.wav" "${theme}" "${media}" "${outro_flac}" "${working_dir}/processing/" + fi + + # Everything needs to be in the same format for the intro, 1 channel (mono) Sampling rate 44.1 kHz + ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/processing/episode_tts.wav" -ar 44100 -ac 1 "${working_dir}/processing//episode_tts.flac" + + # A level of silence is added at the beginning of the text to speech + sox -V2 "${silence}" "${working_dir}/processing//episode_tts.flac" "${working_dir}/processing/episode_summary.flac" + + # The tracks are merged together resulting in the theme playing first, then after a period of silence the text to speech enters + sox -V2 -m "${working_dir}/processing/episode_summary.flac" "${theme}" "${working_dir}/processing/episode_intro.flac" + + if [[ ! -s "${working_dir}/processing//episode_tts.flac" || ! -s "${working_dir}/processing/episode_summary.flac" || ! -s "${working_dir}/processing/episode_intro.flac" ]] + then + echo_error "The files for the theme audio sandwich are not available." + ls -al "${working_dir}/processing//episode_tts.flac" "${working_dir}/processing/episode_summary.flac" "${theme}" "${working_dir}/processing/episode_intro.flac" + fi + +} + +################################################# +# Generate parent audio - the sandwitch + +function generate_parent_audio { + + echo_debug "Generating the parent audio - the sandwitch." + + if [[ ! -s "${working_dir}/processing/episode_intro.flac" || ! -s "${media}" || ! -s "${outro_flac}" ]] + then + echo_error "The files for the sandwich are not available." + ls -al + fi + + # Everything needs to be in the same format so the text to speech needs to be converted to 2 channel Sampling rate 44.1 kHz + ffmpeg -hide_banner -loglevel error -y -i "${media}" -ar 44100 -ac 1 "${working_dir}/processing/episode_body.flac" + + # Combine the components together + sox -V2 "${working_dir}/processing/episode_intro.flac" "${working_dir}/processing/episode_body.flac" "${outro_flac}" "${working_dir}/processing/episode_sandwitch.flac" + + # Normalise the audio + ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/processing/episode_sandwitch.flac" -af loudnorm=I=-16:LRA=11:TP=-1.5 "${working_dir}/processing/episode_final.flac" + +} + +################################################# +# Generate derived media + +function generate_derived_media { + + echo_debug "Generating derived audio." + + if [[ ! -s "${working_dir}/processing/episode_final.flac" ]] + then + ls -al + echo_error "The final cut is not available." + fi + + episode_comment="$( jq --raw-output '.comment' "${working_dir}/episode_summary.json" )" + episode_year="$( echo "${ep_date}" | cut -c -4 )" + + # https://wiki.multimedia.cx/index.php?title=FFmpeg_Metadata + + for ext in flac wav mp3 ogg opus + do + echo_debug "Generating \"hpr${ep_num}.${ext}\"." + ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/processing/episode_final.flac" \ + -metadata title="${title}" \ + -metadata artist="${host_name}" \ + -metadata author="${host_name}" \ + -metadata album="Hacker Public Radio" \ + -metadata comment="${episode_comment} The license is ${episode_license}" \ + -metadata year="${episode_year}" \ + -metadata track="${ep_num}" \ + -metadata genre="Podcast" \ + -metadata language="English" \ + -metadata copyright="${episode_license}" \ + "${working_dir}/hpr${ep_num}.${ext}" + + fix_tags -album="Hacker Public Radio" -artist="${host_name}" -comment="${episode_comment} The license is ${episode_license}" -genre="Podcast" -title="${title}" -track="${ep_num}" -year="${episode_year}" "${working_dir}/hpr${ep_num}.${ext}" + + if [[ ! -s "${working_dir}/hpr${ep_num}.${ext}" ]] + then + echo_error "Failed to generate \"${working_dir}/hpr${ep_num}.${ext}\"." + ls -al "${working_dir}/hpr${ep_num}.${ext}" + fi + + done + + cp -v "${media}" "${working_dir}/hpr${ep_num}_source.${media##*.}" + + if [[ ! -s "${working_dir}/hpr${ep_num}_source.${media##*.}" ]] + then + echo_error "Failed to copy \"${working_dir}/hpr${ep_num}_source.${media##*.}\"." + ls -al "${working_dir}/hpr${ep_num}_source.${media##*.}" + fi + +} + +################################################# +# Generate Subtitles + +function generate_show_transcript() { + + echo_debug "Generate show transcript and subtitles." + + # TODO Currently processed elsewhere by hpr-get-and-transcode.bash and uploaded to hpr:upload/ to be synced with media above + + if [[ ! -s "${media}" || ! -s "${media%.*}.srt" || ! -s "${intro_srt}" || ! -s "${outro_srt}" || ! -s "${working_dir}/processing/episode_intro.flac" || ! -s "${working_dir}/processing/episode_body.flac" ]] + then + ls -al "${media}" "${media%.*}.srt" "${intro_srt}" "${outro_srt}" "${working_dir}/processing/episode_intro.flac" "${working_dir}/processing/episode_body.flac" + echo_error "The transcriptions files are not available." + fi + + # Copy in the intro subtitle template and replace each line with the text with the summary + + date="$( jq --raw-output '.date' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )" + title="$( jq --raw-output '.title' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )" + duration="$( jq --raw-output '.duration' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )" + duration="$( \date -d@${duration} -u +%H:%M:%S )" + artist="$( jq --raw-output '.artist' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )" + explicit="$( jq --raw-output '.explicit' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )" + license="$( jq --raw-output '.license' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )" + summary="$( jq --raw-output '.summary' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )" + + if [[ -z "${date}" || "${date}" == "null" || -z "${title}" || "${title}" == "null" || -z "${duration}" || "${duration}" == "null" || -z "${duration}" || "${duration}" == "null" || -z "${artist}" || "${artist}" == "null" || -z "${explicit}" || "${explicit}" == "null" || -z "${license}" || "${license}" == "null" || -z "${summary}" || "${summary}" == "null" ]] + then + echo_error "Could not retrieve the synopsis for the text to speech." + ls -al "${working_dir}/episode_summary.json" + fi + + REPLACE_LINE_1="This is Hacker Public Radio Episode ${ep_num}, for ${date}" + REPLACE_LINE_2="Today's show is entitled, \"${title}\"" + REPLACE_LINE_3="The host is ${artist} and the duration is ${duration}" + REPLACE_LINE_4="The flag is ${explicit}, and the license is ${license}" + REPLACE_LINE_5="The summary is \"${summary}\"" + + cp -v ${intro_srt} "${working_dir}/processing/episode_intro.srt" + cp -v ${outro_srt} "${working_dir}/processing/episode_outro.srt" + + sed -e "s~REPLACE_LINE_1~${REPLACE_LINE_1}~g" -e "s~REPLACE_LINE_2~${REPLACE_LINE_2}~g" -e "s~REPLACE_LINE_3~${REPLACE_LINE_3}~g" -e "s~REPLACE_LINE_4~${REPLACE_LINE_4}~g" -e "s~REPLACE_LINE_5~${REPLACE_LINE_5}~g" -i "${working_dir}/processing/episode_intro.srt" + + if [ "$( grep --count REPLACE_LINE "${working_dir}/processing/episode_intro.srt" )" -ne "0" ] + then + echo_error "The intro subtitles were not correctly generated \"${working_dir}/processing/episode_intro.srt\"." + fi + + + # Time shift the media subtitles on by the duration of the intro wav file + # https://trac.ffmpeg.org/wiki/UnderstandingItsoffset + + itsoffset_intro="$( mediainfo --full --Output=JSON "${working_dir}/processing/episode_intro.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )" + + if [[ -z "${itsoffset_intro}" || "${itsoffset_intro}" == "null" ]] + then + echo_error "Could not retrieve the itsoffset_intro to correct the timing of the subtitles." + fi + + ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_intro}" -i "${media%.*}.srt" -c copy "${working_dir}/processing/episode_body.srt" + + # Timeshift the outro by the duration of the intro and the supplied media + + itsoffset_body="$( mediainfo --full --Output=JSON "${working_dir}/processing/episode_body.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )" + + if [[ -z "${itsoffset_body}" || "${itsoffset_body}" == "null" ]] + then + echo_error "Could not retrieve the itsoffset_body to correct the timing of the subtitles." + fi + + itsoffset_body=$((itsoffset_intro + $itsoffset_body)) + + ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_body}" -i "${working_dir}/processing/episode_outro.srt" -c copy "${working_dir}/processing/episode_outro_shifted.srt" + + # Combine the intro, timeshifted media subtitles, and the timeshifted outro subtitles. + + cat "${working_dir}/processing/episode_intro.srt" "${working_dir}/processing/episode_body.srt" "${working_dir}/processing/episode_outro_shifted.srt" > "${working_dir}/processing/episode.srt" + + # Parse the resulting subtitle file fixing the numberic counter + # https://en.wikipedia.org/wiki/SubRip + + count=1 + + cat "${working_dir}/processing/episode.srt" | while read this_line + do + if [ "$( echo "${this_line}" | grep -c -P '^[0-9]+$' )" -eq "1" ] + then + echo "${count}" + count=$((count+1)) + else + echo "${this_line}" + fi + done > "${working_dir}/hpr${ep_num}.srt" + + # extract the txt version + + grep -Pv -- '-->|^$|^[0-9]+$' "${working_dir}/hpr${ep_num}.srt" > "${working_dir}/hpr${ep_num}.txt" + + if [[ ! -s "${working_dir}/hpr${ep_num}.srt" || ! -s "${working_dir}/hpr${ep_num}.txt" ]] + then + echo_error "The transcriptions files were not generated." + ls -al "${working_dir}/hpr${ep_num}.srt" "${working_dir}/hpr${ep_num}.txt" + fi + +} + +################################################# +## Generate Final Report + +function generate_final_report() { + + echo_debug "Generating the final report." + + final_report="${working_dir}/processing/hpr${ep_num}_report.html" + + for this_file_extension_to_check in flac mp3 ogg opus srt txt wav + do + if [[ ! -s "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" ]] + then + ls -al "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" + echo_error "The generated media is missing \"${this_file_extension_to_check}\"." + fi + done + + if [[ ! -s "${working_dir}/processing/${media_basename%.*}_media_report.html" ]] + then + ls -al "${working_dir}/processing/${media_basename%.*}_media_report.html" + echo_error "The initial report is not available.\"${working_dir}/processing/${media_basename%.*}_media_report.html\"" + fi + + grep -Pv '|' "${working_dir}/processing/${media_basename%.*}_media_report.html" > "${final_report}" + + echo "

Text To Speech

+

+$( echo "${synopsis}" ) +

+

+ +
+${working_dir}/processing//episode_tts.flac +

+
" >> "${final_report}" + + + for this_file_extension_to_check in flac mp3 ogg opus wav + do + ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" -lavfi "showspectrumpic=s=960x540" "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png" + + audio2image.bash "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" && mv -v "${working_dir}/hpr${ep_num}.png" "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_waveform.png" + + mediainfo "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" + + exiftool "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" + + ffprobe "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" 2>&1 | grep Audio: | sed 's/^.\s//g' > "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" + file --brief --mime "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" >> "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" + this_file_mime_type="$( file --brief --mime-type "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" )" + + for this_file_to_check in spectrum.png waveform.png mediainfo.txt exiftool.txt ffprobe.txt + do + if [[ ! -s "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" ]] + then + ls -al "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" + echo_error "The inital report information is missing \"${this_file_to_check}\"." + fi + done + + echo "

Report for derived media file \"hpr${ep_num}.${this_file_extension_to_check}\"

+ +

mediainfo report

+ +
+$( cat "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" )
+
+ + +

exiftool report

+ +
+$( cat "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" )
+
+ + +

Audio Spectrum

+

+\"Spectrum\" +

+ +

Audio Waveform

+

+\"Waveform\" +

+ +
+$( cat "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" )
+
+ +

+ +
+${working_dir}/hpr${ep_num}.${this_file_extension_to_check} +

" >> "${final_report}" + + done + + echo " + + +

Rendered shownotes.html

+ +
+ +$(cat "${shownotes_html%.*}_edited.html" ) + +
+ +

Subtitle File

+ +
+
+$(cat "${working_dir}/hpr${ep_num}.srt" )
+
+
+ +

Raw shownotes.html

+ +
+
+$(cat "${working_dir}/hpr${ep_num}.txt" )
+
+
+ + + +" >> "${final_report}" + +} + +################################################# +## Manually edit the shownotes to fix issues + +function manual_final_review() { + + echo_debug "Validating the final report." + + if [[ -z "${final_report}" || ! -s "${final_report}" ]] + then + ls -al "${final_report}" + echo_error "The files needed for to generate the final report information are not available." + fi + + librewolf "${final_report}" >/dev/null 2>&1 & + # # # # bluefish "${shownotes_html%.*}_edited.html" >/dev/null 2>&1 & + # https://markdowntohtml.com/ + + read -p "Does the metadata 'look ok ? (N|y) ? " -n 1 -r + echo # (optional) move to a new line + if [[ ! $REPLY =~ ^[yY]$ ]] + then + echo "skipping...." + echo_error "The final review was not approved." + fi + +} + +################################################# +# Send the derived files to the server borg to be sent to the Internet Archive + +function copy_derived_files_to_borg_for_the_internet_archive() { + + echo_debug "Sending the derived files to the server borg to be sent to the Internet Archive" + + for ext in flac mp3 ogg opus wav + do + if [[ ! -s "${working_dir}/hpr${ep_num}.${ext}" ]] + then + echo_error "The inital report information is missing \"${ext}\"." + ls -al "${working_dir}/hpr${ep_num}.${ext}" + fi + done + + echo_debug "rsync -ave ssh --partial --progress \"${working_dir}/hpr${ep_num}.{flac,mp3,ogg,opus,wav}\" borg:/data/IA/uploads/" + rsync -ave ssh --partial --progress "${working_dir}/hpr${ep_num}".{flac,mp3,ogg,opus,wav} borg:/data/IA/uploads/ + rsync_error="${?}" + if [ "${rsync_error}" -ne "0" ] + then + echo_error "rsync to \"borg:/data/IA/uploads/\" failed with error ${rsync_error}" + fi + + rsync -ave ssh --partial --progress "${working_dir}/hpr${ep_num}".{txt,srt} borg:/data/IA/uploads/hpr${ep_num}/ + rsync_error="${?}" + if [ "${rsync_error}" -ne "0" ] + then + echo_error "rsync to \"borg:/data/IA/uploads/hpr${ep_num}/\" failed with error ${rsync_error}" + fi + + rsync -ave ssh --partial --progress "${shownotes_html%.*}_edited.html" borg:/data/IA/uploads/hpr${ep_num}/shownotes.html + rsync_error="${?}" + if [ "${rsync_error}" -ne "0" ] + then + echo_error "rsync to \"borg:/data/IA/uploads/hpr${ep_num}/shownotes.html\" failed with error ${rsync_error}" + fi + + # Get the current status + if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${processing_dir}/status.csv" )" != 200 ] + then + echo_error "Could not get a list of the queue status from \"https://hub.hackerpublicradio.org/cms/status.php\"" + fi + + # Check the current status is correct SHOW_POSTED + response=$( grep -P ",${ep_num},.*SHOW_POSTED," "${processing_dir}/status.csv" | head -1 | sed 's/,/ /g' ) + if [ -z "${response}" ] + then + grep -P ",${ep_num},.*SHOW_POSTED," "${processing_dir}/status.csv" + echo_debug "The show \"${ep_num}\" hasn't the correct status of \"SHOW_POSTED\" in the database." + fi + + # Try and change the online db status to MEDIA_TRANSCODED + if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' "https://hub.hackerpublicradio.org/cms/status.php?ep_num=${ep_num}&status=MEDIA_TRANSCODED" )" != 200 ] + then + echo_error "Could not change the status of \"${ep_num}\" to \"MEDIA_TRANSCODED\"" + fi + + # Get the current status + if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${processing_dir}/status.csv" )" != 200 ] + then + echo_error "Could not get a list of the queue status from \"https://hub.hackerpublicradio.org/cms/status.php\"" + fi + + # Check the current status is correct MEDIA_TRANSCODED + response=$( grep -P ",${ep_num},.*MEDIA_TRANSCODED," "${processing_dir}/status.csv" | head -1 | sed 's/,/ /g' ) + if [ -z "${response}" ] + then + grep -P ",${ep_num},.*MEDIA_TRANSCODED," "${processing_dir}/status.csv" + echo_error "The show \"${ep_num}\" hasn't the correct status of \"MEDIA_TRANSCODED\" in the database." + fi + + echo_debug "The show \"${ep_num}\" has the correct status of \"MEDIA_TRANSCODED\" in the database." + + #TODO images + + # Picture 1 shows the broken dog walking accessory. + +} + +################################################# +# Main exceution starts here +# +# This tool will process the HPR shows allowing the janitor to review the media and fix shownotes. +# +# TODO Add support for reserve queue - process validate and move to reserve dir +# TODO Add support to reprocess processed shows - when given onlay new media reprocess it, update duration on hub, generate and download shownotes.{html,json} from db +# TODO Add support for community news - reusing ^^^ +# TODO Add support for stereo for some episodes that request it +# TODO Include links in extract_images_brute_force +# TODO run hpr_generator to genrate only the new episode +# TODO take screenshots of the rendered episode on the hpr website +# TODO audio_channels default to mono - stereo as an option +# TODO check the channels on the source audio and add a warning in the report to check it's ok to reduce to mono +# TODO Add chapter support +# TODO incorporate direct upload to the IA +# TODO change MEDIA_TRANSCODED +# TODO incorporate assets +# TODO copy the files to the backup disk +# TODO copy the derived files to the ccdn origin server +# TODO fix permissions on vger(two) + +# TODO + +program_checks # We know that all the programs and variables are set + +get_working_dir $@ # We have a working directory and a valid json file + +get_episode_metadata $@ # We have all the metadata we need to process the show. + +extract_images_brute_force # Extract_images by brute force + +media_checks # + +generate_initial_report # Generate Initial Report for review by the Janitors + +manual_shownotes_review # Janitors review audio and shownote. Skips if done. + +post_show_to_hpr_db # Posts the episode to HPR. Skips if it is already posted. + +create_tts_summary # Generate text to speech summary + +generate_intro # Generate Intro from the intro theme with overlay of a lead in silence then the tts summary + +generate_parent_audio # Combines the intro, the episode, and the outro to a final cut. + +generate_derived_media # Generate the flac wav mp3 ogg opus files + +generate_show_transcript + +generate_final_report + +manual_final_review + +#TODO update the assets table + +copy_derived_files_to_borg_for_the_internet_archive + +echo_debug "here" ; exit # TODO diff --git a/workflow/silence.flac b/workflow/silence.flac new file mode 100644 index 0000000..4a292ce Binary files /dev/null and b/workflow/silence.flac differ diff --git a/workflow/theme.flac b/workflow/theme.flac new file mode 100644 index 0000000..cd88175 Binary files /dev/null and b/workflow/theme.flac differ