From c22336b59c1cac39b7c3ebf6ad3059b4b8aa4aa1 Mon Sep 17 00:00:00 2001 From: Ken Fallon Date: Fri, 17 Nov 2023 11:56:08 +0100 Subject: [PATCH] 2023-11-17_10-56-08Z_Friday --- bin/hprtranscode.bash | 732 ++++++++++++++++++++++++++---------------- 1 file changed, 455 insertions(+), 277 deletions(-) diff --git a/bin/hprtranscode.bash b/bin/hprtranscode.bash index 5829048..191f1ae 100755 --- a/bin/hprtranscode.bash +++ b/bin/hprtranscode.bash @@ -1,173 +1,250 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ #============================================================ -# Check input -usage="usage: $(basename $0 ) [ -i ] [ -o ] , -i to add intro and -o outro, fname is a file with audio for HPR" - -# Argument = -t test -r server -p password -v - -echoerr() -{ -echo "$@" 1>&2; -} - -usage() -{ -cat << EOF -usage: $0 [options] {media file to encode} {episode number} - -OPTIONS: - -h Show this message - -p add a promo - -i add the intro - -s Automatically generate the summary - -o add the outro - -b add the intro and outro ( and summary default) - -2 encode to 2 channels - -n no audio normalization - -x eXclude sponsor mention - -EOF -} - TEMP_DIR="/var/tmp/" CHANNELS="1" FIXAUDIO="1" -ADDSUMMARY="n" -ADDINTRO="n" -ADDPROMO="n" -ADDOUTRO="n" -ADDSPONSOR="y" ARTIST="EMPTY" TITLE="EMPTY" YEAR="EMPTY" SLOT="EMPTY" basedir="/var/IA" -intro="${basedir}/intro.flac" -promo="${basedir}/promo.flac" -outro="${basedir}/outro.flac" -anhonesthost="${basedir}/sponsor-anhonesthost.com-hpr15.flac" -internetarchive="${basedir}/sponsor-archive.org.flac" +upload_dir="${basedir}/uploads" +theme="${basedir}/theme.wav" +outro="${basedir}/2022-03-07-outro.wav" +ttsserver="http://localhost:5500" +processing_dir="/home/ken/tmp/hpr/processing" +git_image_dir="/home/ken/sourcecode/hpr/HPR_Public_Code/www/images/hosts" +acceptable_duration_difference="2" # Seconds +thedate=$(/usr/bin/date -u +%Y-%m-%d_%H-%M-%SZ_%A) +echo "Processing the next HPR Show in the queue" -while getopts "hipsob2nx" OPTION -do - case $OPTION in - h) - usage - exit 1 - ;; - s) - ADDSUMMARY="y" - ;; - i) - ADDINTRO="y" - ;; - p) - ADDPROMO="y" - ;; - o) - ADDOUTRO="y" - ;; - b) - ADDINTRO="y" - ADDOUTRO="y" - ;; - 2) - CHANNELS="2" - ;; - n) - FIXAUDIO="0" - ;; - x) - ADDSPONSOR="n" - ;; - ?) - usage - exit - ;; - esac -done -shift $(($OPTIND - 1)) +if [ $( curl -s -o /dev/null -w "%{http_code}" -X 'GET' "${ttsserver}/api/voices" -H 'accept: */*' ) != "200" ] +then + echo "Please start the tts-server \"podman run -it -p 5500:5500 synesthesiam/opentts:en\"" + exit +fi -if [ "$#" -ne 2 ]; then - echoerr "Please enter the source file and episode number" +function abs_diff { + echo $(($1 >= $2 ? $1 - $2 : $2 - $1)) +} + +function create_tts_summary { + HPR_summary="$( cat "hpr${ep_num}_summary.txt" )" + echo "INFO: Converting text \"${HPR_summary}\" to speech." + curl -X 'GET' -G --data-urlencode "voice=coqui-tts:en_ljspeech" --data-urlencode "text=${HPR_summary}" --data-urlencode "vocoder=high" --data-urlencode "denoiserStrength=0.03" --data-urlencode "cache=false" ${ttsserver}/api/tts -H 'accept: */*' --output ~hpr${ep_num}_summary.wav +} + +################### +# Locate media directory +# +# +show_type="" +if [[ -f "${1}" && -n "${2}" ]] +then + show_type="local" + mediafile="${1}" + media_dir="$( dirname "${mediafile}" )" + ep_num="${2}" + echo "The duration is \"$( \date -ud "1970-01-01 $( ffprobe -i "${mediafile}" 2>&1| awk -F ': |, ' '/Duration:/ { print $2 }' )" +%s )\"." +else + show_type="remote" + response=$( curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | \ + grep 'SHOW_POSTED' | \ + head -1 | \ + sed 's/,/ /g' ) + + if [ -z "${response}" ] + then + echo "INFO: There appear to be no more shows with the status \"SHOW_POSTED\"." + echo "Getting a list of all the reservations." + curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" + exit 3 + fi + + timestamp_epoc="$( echo ${response} | awk '{print $1}' )" + ep_num="$( echo ${response} | awk '{print $2}' )" + ep_date="$( echo ${response} | awk '{print $3}' )" + key="$( echo ${response} | awk '{print $4}' )" + status="$( echo ${response} | awk '{print $5}' )" + email="$( echo ${response} | awk '{print $6}' )" + + #source_dir="hpr:/home/hpr/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}" + dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}" + media_dir="${processing_dir}/${timestamp_epoc}_${ep_num}_${ep_date}_${key}" +fi + +if [ -z "${show_type}" ] +then + echo "sorry, variable \"show_type\" is not set." + exit +fi + +if [ -z "${media_dir}" ] +then + echo "sorry, variable \"media_dir\" is not set." + exit +fi + +if [ ! -d "${media_dir}" ] +then + echo "sorry, directory \"media_dir: ${media_dir}\" does not exist" + exit 1 +fi + +echo detox -v "${media_dir}/" +detox -vr "${media_dir}/" + +if [[ "$( find "${media_dir}" \( -iname "hpr${ep_num}.wav" -o -iname "hpr${ep_num}.mp3" -o -iname "hpr${ep_num}.ogg" -o -iname "hpr${ep_num}.spx" -o -iname "hpr${ep_num}_summary.txt" -o -iname "hpr${ep_num}_summary.wav" -o -iname "hpr${ep_num}.wav" -o -iname "*_mez.wav" -o -iname "*_sox_norm.wav" -o -iname "*_mezzanine.wav" -o -iname "*_tmp.pcm" -o -iname "hpr${ep_num}_intro.wav" -o -iname "~hpr${ep_num}_summary.wav" -o -iname "~~hpr${ep_num}_summary.wav" -o -iname "*_tmp.log" -o -iname "silence.wav" -o -iname "hpr${ep_num}_norm.wav" \) | wc -l )" -ne 0 ]] +then + echo "Files for this episode already exist." + find "${media_dir}" \( -iname "hpr${ep_num}.wav" -o -iname "hpr${ep_num}.mp3" -o -iname "hpr${ep_num}.ogg" -o -iname "hpr${ep_num}.spx" -o -iname "hpr${ep_num}_summary.txt" -o -iname "hpr${ep_num}_summary.wav" -o -iname "hpr${ep_num}.wav" -o -iname "*_mez.wav" -o -iname "*_sox_norm.wav" -o -iname "*_mezzanine.wav" -o -iname "*_tmp.pcm" -o -iname "hpr${ep_num}_intro.wav" -o -iname "~hpr${ep_num}_summary.wav" -o -iname "~~hpr${ep_num}_summary.wav" -o -iname "*_tmp.log" -o -iname "silence.wav" -o -iname "hpr${ep_num}_norm.wav" \) + read -p "Shall I remove them ? (N|y) ? " -n 1 -r + echo # (optional) move to a new line + if [[ ! $REPLY =~ ^[yY]$ ]] + then + echo "skipping...." exit -fi - -mediafile=${1} -ep_num=${2} - -ep_num=$(echo $ep_num | sed 's/hpr//g') -re='^[0-9]+$' -if ! [[ $ep_num =~ $re ]] ; then - echoerr "error: episode \"${ep_num}\" is not a number" - exit 1 -fi - -if [ ! -f "${anhonesthost}" ]; then - echoerr "sorry, file \"${anhonesthost}\" does not exist" - exit 1 -fi - -if [ ! -f "${internetarchive}" ]; then - echoerr "sorry, file \"${internetarchive}\" does not exist" - exit 1 -fi - -if [ ! -f "${intro}" ]; then - echoerr "sorry, file \"intro.flac\" does not exist" - exit 1 -fi - -if [ "$ADDPROMO" = 'y' ]; then - if [ ! -f "${promo}" ]; then - echoerr "sorry, file \"promo.flac\" does not exist" - exit 1 + else + echo "Removing old files ...." + find "${media_dir}" \( -iname "hpr${ep_num}.wav" -o -iname "hpr${ep_num}.mp3" -o -iname "hpr${ep_num}.ogg" -o -iname "hpr${ep_num}.spx" -o -iname "hpr${ep_num}_summary.txt" -o -iname "hpr${ep_num}_summary.wav" -o -iname "hpr${ep_num}.wav" -o -iname "*_mez.wav" -o -iname "*_sox_norm.wav" -o -iname "*_mezzanine.wav" -o -iname "*_tmp.pcm" -o -iname "hpr${ep_num}_intro.wav" -o -iname "~hpr${ep_num}_summary.wav" -o -iname "~~hpr${ep_num}_summary.wav" -o -iname "*_tmp.log" -o -iname "silence.wav" -o -iname "hpr${ep_num}_norm.wav" \) -delete -print fi fi -if [ ! -f "${outro}" ]; then - echoerr "sorry, file \"outro.flac\" does not exist" + +##################################################### +# Process media + +media=$( find "${media_dir}" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' ) +if [ -z "${media}" ] +then + echo "ERROR: Can't find any media in \"${media_dir}/\"" + find "${media_dir}/" -type f exit 1 fi -if [ ! -f "${mediafile}" ]; then - echoerr "sorry, media file \"${mediafile}\" does not exist" - exit -fi +mediafile="" -if [ ! -r "${mediafile}" ]; then - echoerr "sorry, media file \"${mediafile}\" is not readable" - exit -fi - -if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ]; then - echoerr "sorry, media file \"${mediafile}\" has no audio track" - exit +echo "Found more than one media file. Please select which one to use ?" +if [ "$( echo "${media}" | wc -l )" -ne 1 ] +then + select this_media in $( echo "${media}" ) + do + echo "INFO: You selected \"${this_media}\"." + ls -al "${this_media}" + mediafile="${this_media}" + break + done +else + echo "INFO: Selecting media as \"${media}\"." + mediafile="${media}" fi # extract file name and extension -media_dir=$(dirname ${mediafile}) -fname=${mediafile%.*} -ext=${mediafile/*./} +fname="$( basename "${mediafile%.*}" )" +ext="${mediafile/*./}" -if [ "$ADDSUMMARY" = 'n' ]; then - if [ ! -e "${media_dir}/summary.wav" ] - then - echoerr "ERROR: Can not find the summary file \"${media_dir}/summary.wav\"" - wget -O- --timeout=10 --tries=1 --quiet http://hackerpublicradio.org/say.php?id=${ep_num} | grep HPR_summary - exit 1 - fi +cd "${media_dir}/" +pwd + +echo "INFO: Processing hpr${ep_num} from ${email}" +echo "INFO: Working directory is \"${media_dir}/\"" +echo "" + + +if [ -z "${mediafile}" ] +then + echo "sorry, variable \"mediafile\" is not set." + exit fi -if [[ -e hpr${ep_num}.wav ]] || [[ -e hpr${ep_num}.mp3 ]] || [[ -e hpr${ep_num}.ogg ]] || [[ -e hpr${ep_num}.spx ]] || [[ -e hpr${ep_num}_summary.wav ]] || [[ -e ${fname}_mez_norm.wav ]] || [[ -e ${fname}_mez.wav ]] || [[ -e ${fname}_sox_norm.wav ]] || [[ -e ${fname}_sox.wav ]] || [[ -e ${fname}_tmp_hh.pcm ]] || [[ -e ${fname}_tmp_ia.pcm ]] || [[ -e ${fname}_tmp.log ]] +if [ -z "${fname}" ] then - echoerr "Files for this episode already exist." - ls -1 hpr${ep_num}* ${fname}_* 2>/dev/null + echo "sorry, variable \"fname\" is not set." + exit +fi + +if [ -z "${ext}" ] +then + echo "sorry, variable \"ext\" is not set." + exit +fi + +if [ ! -f "${mediafile}" ] +then + echo "sorry, media file \"${mediafile}\" does not exist" + exit +fi + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "File information for \"${mediafile}\"" | tee -a ${fname}_tmp.log +ffprobe ${mediafile} 2>&1 | grep Audio: +mediainfo ${mediafile} +audio2image.bash ${mediafile} +xdg-open ${mediafile%.*}.png >/dev/null 2>&1 & +unset REPLY +until [[ $REPLY =~ ^[yYnN]$ ]] +do + read -p "Source Waveform look ok ? (N|y) ? " -n 1 -r + echo "" +done + +if [[ ! $REPLY =~ ^[yY]$ ]] +then + echo "skipping.... $REPLY" + exit +fi + + +re='^[0-9]+$' +if ! [[ $ep_num =~ $re ]] +then + echo "error: episode \"${ep_num}\" is not a number" + exit 1 +fi + +if [ ! -f "${outro}" ] +then + echo "sorry, file \"${outro}\" does not exist" exit 1 fi +if [ ! -f "${theme}" ] +then + echo "sorry, file \"${theme}\" does not exist" + exit 1 +fi + +if [ ! -r "${mediafile}" ] +then + echo "sorry, media file \"${mediafile}\" is not readable" + exit +fi + +if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ] +then + echo "sorry, media file \"${mediafile}\" has no audio track" + exit +fi + +xdg-open "https://hackerpublicradio.org/eps/hpr${ep_num}/index.html" >/dev/null 2>&1 & + +mpv -vo=null "${mediafile}" + +unset REPLY +until [[ $REPLY =~ ^[yYnN]$ ]] +do + read -p "Is the audio ok (n|Y) ? " -n 1 -r + echo "" +done + +if [[ ! $REPLY =~ ^[yY]$ ]] +then + echo "skipping.... $REPLY" + exit +fi + echo "--------------------------------------------------------------------------------" echo "Geting metadata for hpr${ep_num}" @@ -220,18 +297,14 @@ do continue ;; esac -done < <( wget --timeout=10 --tries=1 --quiet http://hackerpublicradio.org/say.php?id=${ep_num} -O - ) +done < <( curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num}" ) -if [[ -z "$ADDINTRO" || -z "$ADDPROMO" || -z "$ADDSUMMARY" || -z "$ADDOUTRO" || -z "$HPR_album" || -z "$HPR_artist" || -z "$HPR_comment" || -z "$HPR_genre" || -z "$HPR_title" || -z "$HPR_track" || -z "$HPR_year" || -z "$HPR_summary" || -z "$HPR_duration" || -z "$HPR_explicit" || -z "$HPR_license" ]] +if [[ -z "$HPR_album" || -z "$HPR_artist" || -z "$HPR_comment" || -z "$HPR_genre" || -z "$HPR_title" || -z "$HPR_track" || -z "$HPR_year" || -z "$HPR_summary" || -z "$HPR_duration" || -z "$HPR_explicit" || -z "$HPR_license" ]] then - echoerr "Could not find information on ${ep_num}. Has the show been posted ?" + echo "Could not find information on ${ep_num}. Has the show been posted ?" exit; fi echo "--------------------------------------------------------------------------------" -echo "Add intro : $ADDINTRO" -echo "Add promo : $ADDPROMO" -echo "Add Summary : $ADDSUMMARY" -echo "Add outro : $ADDOUTRO" echo "album : $HPR_album" echo "artist : $HPR_artist" echo "comment : $HPR_comment" @@ -243,183 +316,288 @@ echo "summary : $HPR_summary" echo "duration : $HPR_duration" echo "explicit : $HPR_explicit" echo "license : $HPR_license" +echo "media_dir : ${media_dir}" +echo "mediafile : ${mediafile}" +echo "fname : ${fname}" +echo "ext : ${ext}" if [[ $HPR_duration == "0" ]] then - echoerr "The duration is set to 0. Please update the show with the correct time." + echo "The duration is set to 0. Please update the show with the correct time." exit; fi #============================================================ # Preproc`s the source file -echo "--------------------------------------------------------------------------------" -echo "Prepare mezzanine file" -ffmpeg -i ${mediafile} -ar 44100 -ac $CHANNELS ${fname}_sox.wav > ${fname}_tmp.log 2>&1 +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Prepare mezzanine file" | tee -a ${fname}_tmp.log +ffmpeg -y -i ${mediafile} -ar 44100 -ac $CHANNELS ${fname}_mezzanine.wav > ${fname}_tmp.log 2>&1 -if [ "$FIXAUDIO" = "1" ];then - echo "normalizing audio" - sox --temp "${TEMP_DIR}" --norm ${fname}_sox.wav ${fname}_sox_norm.wav - mv -v ${fname}_sox_norm.wav ${fname}_sox.wav >> ${fname}_tmp.log 2>&1 -# normalize -a 0.5 ${fname}_sox.wav >> ${fname}_tmp.log 2>&1 -fi +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Add HPR Branding" | tee -a ${fname}_tmp.log -echo "--------------------------------------------------------------------------------" -echo "Add HPR Branding" +echo "Creating the summary" | tee -a ${fname}_tmp.log +#echo "$HPR_summary" - | text2wave -F 44100 - -o hpr${ep_num}_summary.wav #festival --tts +#echo "$HPR_summary" - | espeak -w ~hpr${ep_num}_summary.wav -if [ "$ADDSUMMARY" = 'y' ]; then - echo "Creating the summary" - #echo "$HPR_summary" - | espeak -w hpr${ep_num}_summary.wav - echo "$HPR_summary" - | text2wave - -o hpr${ep_num}_summary.wav #festival --tts - #echo "${HPR_summary}" | gtts-cli - --output hpr${ep_num}_summary.mp3 - #ffmpeg -i hpr${ep_num}_summary.mp3 hpr${ep_num}_summary.wav - #rm -v hpr${ep_num}_summary.mp3 -else - echo "Copying the supplied summary" - if [ ! -e "${media_dir}/summary.wav" ] +echo "$HPR_summary" > "hpr${ep_num}_summary.txt" + +create_tts_summary_ok="not-ok" + +while [ "${create_tts_summary_ok}" != "OK" ] +do + create_tts_summary + + xdg-open "hpr${ep_num}_summary.txt" 2>&1 & + mpv --speed=1.8 ~hpr${ep_num}_summary.wav + + read -p "Is the text to speech correct (y|N) ? " -n 1 -r + echo # (optional) move to a new line + if [[ $REPLY =~ ^[Yy]$ ]] then - echoerr "ERROR: Can not find the summary file \"${media_dir}/summary.wav\"" - exit 1 + create_tts_summary_ok="OK" + else + echo "WARN: Please correct the text and try again." + inotifywait --event modify "hpr${ep_num}_summary.txt" fi - cp -v "${media_dir}/summary.wav" hpr${ep_num}_summary.wav -fi +done +echo "INFO: TTS complete." -ffmpeg -i hpr${ep_num}_summary.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log -ffmpeg -i hpr${ep_num}_summary.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log -rm hpr${ep_num}_summary.wav +ffmpeg -y -i ~hpr${ep_num}_summary.wav -ar 44100 ~~hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1 -if [ "$ADDSPONSOR" = 'y' ]; then - echo "Adding the sponsor" - ffmpeg -i "$anhonesthost" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log - ffmpeg -i "$internetarchive" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log -else - echo "NOT Adding the sponsor" -fi +sox -V2 -n -r 44100 -c 1 silence.wav trim 0.0 6.0 >> ${fname}_tmp.log 2>&1 -if [ "$ADDPROMO" = 'y' ]; then - echo "Adding the promo" - ffmpeg -i "$promo" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log - ffmpeg -i "$promo" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log -fi +sox -V2 silence.wav ~~hpr${ep_num}_summary.wav hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1 -if [ "$ADDINTRO" = 'y' ]; then - echo "Adding the intro" - ffmpeg -i "$intro" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log - ffmpeg -i "$intro" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log -fi +echo "Adding the theme" | tee -a ${fname}_tmp.log +sox -V2 -m "hpr${ep_num}_summary.wav" "${theme}" "hpr${ep_num}_intro.wav" >> ${fname}_tmp.log 2>&1 -echo "convert the uploaded episode and add it to the temp pcm file" -ffmpeg -i ${fname}_sox.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log -ffmpeg -i ${fname}_sox.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log +echo "Creating the sandwitch: \"hpr${ep_num}_intro.wav\" \"${fname}_mezzanine.wav\" \"${outro}\" \"hpr${ep_num}.wav\" " | tee -a ${fname}_tmp.log -if [ "$ADDOUTRO" = 'y' ]; then - echo "Adding the outro" - ffmpeg -i "$outro" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log - ffmpeg -i "$outro" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log -fi +sox -V2 "hpr${ep_num}_intro.wav" "${fname}_mezzanine.wav" "${outro}" "hpr${ep_num}.wav" >> ${fname}_tmp.log 2>&1 -echo "Convert the pcm file to a know wav format" -ffmpeg -f s16le -ar 44100 -ac $CHANNELS -acodec pcm_s16le -i ${fname}_tmp_hh.pcm ${fname}_mez.wav 2>> ${fname}_tmp.log -ffmpeg -f s16le -ar 44100 -ac $CHANNELS -acodec pcm_s16le -i ${fname}_tmp_ia.pcm hpr${ep_num}.wav 2>> ${fname}_tmp.log +echo "Normalizing the wav files" | tee -a ${fname}_tmp.log +#sox --temp "${TEMP_DIR}" --norm hpr${ep_num}.wav hpr${ep_num}_norm.wav +ffmpeg -y -i hpr${ep_num}.wav -af loudnorm=I=-16:LRA=11:TP=-1.5 hpr${ep_num}_norm.wav >> ${fname}_tmp.log 2>&1 -echo "Normalizing the wav files" -sox --temp "${TEMP_DIR}" --norm ${fname}_mez.wav ${fname}_mez_norm.wav -mv -v ${fname}_mez_norm.wav ${fname}_mez.wav >> ${fname}_tmp.log 2>&1 -# normalize -a 0.5 ${fname}_mez.wav >> ${fname}_tmp.log 2>&1 -sox --temp "${TEMP_DIR}" --norm hpr${ep_num}.wav hpr${ep_num}_norm.wav -mv -v hpr${ep_num}_norm.wav /var/IA/uploads/hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1 -# normalize -a 0.5 hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1 +mv -v hpr${ep_num}_norm.wav ${upload_dir}/hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1 -echo "--------------------------------------------------------------------------------" -echo "File information" -ffprobe ${fname}_mez.wav 2>&1 | grep Audio: -mediainfo ${fname}_mez.wav - -echo "--------------------------------------------------------------------------------" -echo "Convert to opus for IA" -opusenc hpr${ep_num}.wav /var/IA/uploads/hpr${ep_num}.opus - -echo "--------------------------------------------------------------------------------" -echo "Convert to flac for IA" -sox --temp "${TEMP_DIR}" -S hpr${ep_num}.wav /var/IA/uploads/hpr${ep_num}.flac - -echo "--------------------------------------------------------------------------------" -echo "Convert to mp3 for HPR" -sox --temp "${TEMP_DIR}" -S ${fname}_mez.wav hpr${ep_num}.mp3 -echo "Convert to mp3 for IA" -sox --temp "${TEMP_DIR}" -S hpr${ep_num}.wav /var/IA/uploads/hpr${ep_num}.mp3 - -echo "--------------------------------------------------------------------------------" -echo "Convert to ogg for HPR" -sox --temp "${TEMP_DIR}" -S ${fname}_mez.wav hpr${ep_num}.ogg -echo "Convert to ogg for IA" -sox --temp "${TEMP_DIR}" -S hpr${ep_num}.wav /var/IA/uploads/hpr${ep_num}.ogg - -echo "--------------------------------------------------------------------------------" -echo "Convert to spx for HPR" -sox --temp "${TEMP_DIR}" -S ${fname}_mez.wav -c 1 -r 16000 -t wav - | speexenc - hpr${ep_num}.spx -echo "Convert to spx for IA" -sox --temp "${TEMP_DIR}" -S hpr${ep_num}.wav -c 1 -r 16000 -t wav - | speexenc - /var/IA/uploads/hpr${ep_num}.spx - - -if [[ ! -s /var/IA/uploads/hpr${ep_num}.wav ]] || [[ ! -s /var/IA/uploads/hpr${ep_num}.mp3 ]] || [[ ! -s /var/IA/uploads/hpr${ep_num}.ogg ]] || [[ ! -s /var/IA/uploads/hpr${ep_num}.spx ]] || [[ ! -s hpr${ep_num}.mp3 ]] || [[ ! -s hpr${ep_num}.ogg ]] || [[ ! -s hpr${ep_num}.spx ]] +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "File information" | tee -a ${fname}_tmp.log +ffprobe ${upload_dir}/hpr${ep_num}.wav 2>&1 | grep Audio: +mediainfo ${upload_dir}/hpr${ep_num}.wav +audio2image.bash ${upload_dir}/hpr${ep_num}.wav +xdg-open ${upload_dir}/hpr${ep_num}.png >/dev/null 2>&1 & +read -p "Processed Waveform look ok ? (N|y) ? " -n 1 -r +echo # (optional) move to a new line +if [[ ! $REPLY =~ ^[yY]$ ]] then - echoerr "ERROR: Something went wrong encoding the files" + echo "skipping...." + exit +fi +rm -v ${upload_dir}/hpr${ep_num}.png + + +echo "--------------------------------------------------------------------------------" +echo "Geting transcript of hpr${ep_num}" + +whisper --model tiny --language en --output_dir "${media_dir}" "${upload_dir}/hpr${ep_num}.wav" | tee -a ${fname}_tmp.log + +ls -al "${media_dir}/hpr${ep_num}".* +xdg-open "${media_dir}/hpr${ep_num}".txt 2>&1 & +echo mpv --no-audio-display --audio-channels=stereo --speed="3.5" "${media_dir}/hpr${ep_num}".txt +unset REPLY +until [[ $REPLY =~ ^[yYnN]$ ]] +do + read -p "Processed transcript look ok ? (N|y) ? " -n 1 -r + echo "" +done + +if [[ ! $REPLY =~ ^[yY]$ ]] +then + echo "skipping.... $REPLY" + exit +fi + +mkdir "${upload_dir}/hpr${ep_num}" >/dev/null 2>&1 + +cp -v "${media_dir}/hpr${ep_num}".vtt "${upload_dir}/hpr${ep_num}/hpr${ep_num}.vtt" | tee -a ${fname}_tmp.log +cp -v "${media_dir}/hpr${ep_num}".srt "${upload_dir}/hpr${ep_num}/hpr${ep_num}.srt" | tee -a ${fname}_tmp.log +cp -v "${media_dir}/hpr${ep_num}".txt "${upload_dir}/hpr${ep_num}/hpr${ep_num}.txt" | tee -a ${fname}_tmp.log +cp -v "${media_dir}/hpr${ep_num}".tsv "${upload_dir}/hpr${ep_num}/hpr${ep_num}.tsv" | tee -a ${fname}_tmp.log +cp -v "${media_dir}/hpr${ep_num}".json "${upload_dir}/hpr${ep_num}/hpr${ep_num}.json" | tee -a ${fname}_tmp.log + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Convert to opus" | tee -a ${fname}_tmp.log +ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.opus 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Convert to flac" | tee -a ${fname}_tmp.log +ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.flac 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Convert to mp3" | tee -a ${fname}_tmp.log +ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.mp3 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Convert to ogg" | tee -a ${fname}_tmp.log +#ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.ogg 2>> ${fname}_tmp.log 1>&2 +ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav -acodec libopus -f ogg ${upload_dir}/hpr${ep_num}.ogg 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Convert to spx" | tee -a ${fname}_tmp.log +ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.spx 2>> ${fname}_tmp.log 1>&2 + +### End conversion + +intro_duration="$( mediainfo --Output=XML --Full "hpr${ep_num}_intro.wav" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" +outro_duration="$( mediainfo --Output=XML --Full "${outro}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" +source_duration="$( mediainfo --Output=XML --Full "${mediafile}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" +expected_duration=$(( ${intro_duration} + ${HPR_duration} + ${outro_duration} )) + +echo "Intro(${intro_duration}) + Show(${HPR_duration}) + Outro(${outro_duration}) = ${expected_duration}" + +media_error="0" + +for check_file in \ +${upload_dir}/hpr${ep_num}.wav \ +${upload_dir}/hpr${ep_num}.opus \ +${upload_dir}/hpr${ep_num}.flac \ +${upload_dir}/hpr${ep_num}.mp3 \ +${upload_dir}/hpr${ep_num}.spx \ +${upload_dir}/hpr${ep_num}.ogg +do + # ${upload_dir}/hpr${ep_num}.spx + echo "INFO: Processing the file \"${check_file}\"" + if [[ ! -s "${check_file}" ]] + then + echo "ERROR: Something went wrong encoding of the file \"${check_file}\"" + exit + else + mediainfo --Output=XML --Full "${check_file}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' + this_duration=$( mediainfo --Output=XML --Full "${check_file}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' ) + if [[ $(abs_diff "${this_duration}" "${expected_duration}" ) -le "${acceptable_duration_difference}" ]] + then + echo "INFO: The file \"${check_file}\" duration of ${this_duration} () is close enough to ${expected_duration}" + else + echo "ERROR: The file \"${check_file}\" actual duration of ${this_duration} is not close enough to posted duration of ${expected_duration}." + echo " Fix or update the posted duration to ${source_duration}." + media_error="1" + fi + #${expected_duration} + fi +done +echo "Source: ${source_duration}" + +if [[ "${media_error}" -eq "1" ]] +then + echo "ERROR: Media is not encoded correctly" + exit +else + echo "INFO: Media duration is correct" +fi + + +if [[ ! -s ${upload_dir}/hpr${ep_num}.wav ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.opus ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.flac ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.mp3 ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.ogg ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.spx ]] +then + echo "ERROR: Something went wrong encoding the files" exit 1 fi -fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="${HPR_comment} The license is ${HPR_license}" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2 -fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="$HPR_comment" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" /var/IA/uploads/hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2 -fix_tags hpr${ep_num}* -fix_tags /var/IA/uploads/hpr${ep_num}* + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Fixing Tags" | tee -a ${fname}_tmp.log + +fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="${HPR_comment} The license is ${HPR_license}" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" ${upload_dir}/hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2 + +fix_tags ${upload_dir}/hpr${ep_num}* #echo "Changing the file dates to the time of upload" touch -r ${mediafile} hpr${ep_num}* -touch -r ${mediafile} /var/IA/uploads/hpr${ep_num}* +touch -r ${mediafile} ${upload_dir}/hpr${ep_num}* + +ls -al hpr${ep_num}* ${upload_dir}/hpr${ep_num}* | tee -a ${fname}_tmp.log + +# # echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +# # echo "Getting info for the asset table" | tee -a ${fname}_tmp.log +# # +# # echo "INSERT INTO assets (episode_id,filename,extension,size,sha1sum,mime_type,file_type) VALUES" +# # +# # for asset_file in \ +# # ${upload_dir}/hpr${ep_num}.wav \ +# # ${upload_dir}/hpr${ep_num}.opus \ +# # ${upload_dir}/hpr${ep_num}.flac \ +# # ${upload_dir}/hpr${ep_num}.mp3 \ +# # ${upload_dir}/hpr${ep_num}.spx \ +# # ${upload_dir}/hpr${ep_num}.ogg +# # do +# # size="$( ls -al "${asset_file}" | awk '{print $5}' )" +# # sha1sum="$( sha1sum "${asset_file}" | awk '{print $1}' )" +# # mime_type=$( file --dereference --brief --mime "${asset_file}" ) +# # file_type=$( file --dereference --brief "${asset_file}" ) +# # echo "(${ep_num},'${filename}','${extension}','${size}','${sha1sum}','${mime_type}','${file_type}')," +# # done + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Transferring files to Servers" | tee -a ${fname}_tmp.log -rsync -ave ssh --partial --progress --ignore-existing hpr${ep_num}.mp3 hpr${ep_num}.ogg hpr${ep_num}.spx hpr:www/eps/ +#rsync -ave ssh --partial --progress --ignore-existing ${upload_dir}/hpr${ep_num}.mp3 ${upload_dir}/hpr${ep_num}.ogg ${upload_dir}/hpr${ep_num}.spx hpr:www/eps/ -firefox http://hackerpublicradio.org/local/hpr${ep_num}.mp3 -firefox http://hackerpublicradio.org/local/hpr${ep_num}.ogg -firefox file:///var/IA/uploads/hpr${ep_num}.mp3 -firefox file:///var/IA/uploads/hpr${ep_num}.ogg -mpv "http://hackerpublicradio.org/local/hpr${ep_num}.spx" "http://hackerpublicradio.org/local/hpr${ep_num}.ogg" "http://hackerpublicradio.org/local/hpr${ep_num}.mp3" "file:///var/IA/uploads/hpr${ep_num}.spx" "file:///var/IA/uploads/hpr${ep_num}.opus" "file:///var/IA/uploads/hpr${ep_num}.ogg" "file:///var/IA/uploads/hpr${ep_num}.mp3" "file:///var/IA/uploads/hpr${ep_num}.flac" +#firefox "https://hackerpublicradio.org/local/hpr${ep_num}.mp3" >/dev/null 2>&1 & +#firefox "https://hackerpublicradio.org/local/hpr${ep_num}.ogg" >/dev/null 2>&1 & +firefox "file://${upload_dir}/hpr${ep_num}.mp3" >/dev/null 2>&1 & +firefox "file://${upload_dir}/hpr${ep_num}.ogg" >/dev/null 2>&1 & +#firefox "https://hackerpublicradio.org/eps.php?id=${ep_num}" >/dev/null 2>&1 & -echo "Source: $( mediainfo --Output=XML --Full "${mediafile}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" -mediainfo --Output=XML --Full hpr${ep_num}* /var/IA/uploads/hpr${ep_num}* | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' | sort | uniq -c +#mpv "http://hackerpublicradio.org/local/hpr${ep_num}.spx" "http://hackerpublicradio.org/local/hpr${ep_num}.ogg" "http://hackerpublicradio.org/local/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.spx" "file://${upload_dir}/hpr${ep_num}.opus" "file://${upload_dir}/hpr${ep_num}.ogg" "file://${upload_dir}/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.flac" +mpv "file://${upload_dir}/hpr${ep_num}.spx" "file://${upload_dir}/hpr${ep_num}.opus" "file://${upload_dir}/hpr${ep_num}.ogg" "file://${upload_dir}/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.flac" read -p "Remove files for \"${fname}\" (y|N) ? " -n 1 -r echo # (optional) move to a new line if [[ $REPLY =~ ^[Yy]$ ]] then mediafilename=$(basename "${mediafile}") - mediaextension="${mediafilename##*.}" - - ssh hpr -t "mkdir /home/hpr/www/eps/hpr${ep_num}" >/dev/null 2>&1 - rsync -ave ssh --partial --progress --ignore-existing "${mediafile}" hpr:www/eps/hpr${ep_num}/hpr${ep_num}_source.${mediaextension} - ssh hpr -t "ls -al /home/hpr/www/eps/hpr${ep_num}*" - cp -v "${mediafile}" "/var/IA/uploads/hpr${ep_num}_source.${mediaextension}" + mediaextension="${mediafilename##*.}" # " +# ssh hpr -t "mkdir /home/hpr/www/eps/hpr${ep_num}" >/dev/null 2>&1 +# rsync -ave ssh --partial --progress --ignore-existing "${mediafile}" hpr:www/eps/hpr${ep_num}/hpr${ep_num}_source.${mediaextension} | tee -a ${fname}_tmp.log +# +# rsync -ave ssh --partial --progress --ignore-existing "${media_dir}/hpr${ep_num}.wav".vtt hpr:www/eps/hpr${ep_num}/hpr${ep_num}.vtt | tee -a ${fname}_tmp.log +# rsync -ave ssh --partial --progress --ignore-existing "${media_dir}/hpr${ep_num}.wav".srt hpr:www/eps/hpr${ep_num}/hpr${ep_num}.srt | tee -a ${fname}_tmp.log +# rsync -ave ssh --partial --progress --ignore-existing "${media_dir}/hpr${ep_num}.wav".txt hpr:www/eps/hpr${ep_num}/hpr${ep_num}.txt | tee -a ${fname}_tmp.log +# +# ssh hpr -t "ls -al /home/hpr/www/eps/hpr${ep_num}*" +# cp -v "${mediafile}" "${upload_dir}/hpr${ep_num}_source.${mediaextension}" #echo "Remove temp files" - rm -v ${fname}_sox.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav - mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/ - wget --timeout=0 -q "http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1" -O - | xmlstarlet val --err - - wget --timeout=0 -q "http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1" -O - | xmlstarlet val --err - - wget --timeout=0 -q "http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1" -O - | xmlstarlet val --err - - wget --timeout=0 -q "http://hackerpublicradio.org/rss-future.php" -O - | xmlstarlet val --err - + rm -v ~hpr${ep_num}_summary.wav ~~hpr${ep_num}_summary.wav silence.wav + rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav + #mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/ + +# wget --timeout=0 -q "http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1" -O - | xmlstarlet val --err - +# wget --timeout=0 -q "http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1" -O - | xmlstarlet val --err - +# wget --timeout=0 -q "http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1" -O - | xmlstarlet val --err - +# wget --timeout=0 -q "http://hackerpublicradio.org/rss-future.php" -O - | xmlstarlet val --err - - #rsync -ave ssh --partial --progress /var/IA/uploads/ hpr:/home/hpr/upload/processed/ - rsync -ave ssh --partial --progress /var/IA/uploads/ borg:/data/IA/uploads/ - echo "$( ssh borg -t "ls -al /data/IA/uploads/hpr${ep_num}*" ; ls -al /var/IA/uploads/hpr${ep_num}* )" | grep "hpr${ep_num}" | awk '{print $5, $NF}' | sort - find /var/IA/done/ -empty -delete + echo "INFO: rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* borg:/data/IA/uploads/" + rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* borg:/data/IA/uploads/ + echo "$( ssh borg -t "ls -al /data/IA/uploads/hpr${ep_num}*" ; ls -al ${upload_dir}/hpr${ep_num}* )" | grep "hpr${ep_num}" | awk '{print $5, $NF}' | sort + + if [ ${show_type} == "remote" ] + then + echo "INFO: Setting the status" + # SHOW_SUBMITTED → METADATA_PROCESSED → SHOW_POSTED → MEDIA_TRANSCODED → UPLOADED_TO_IA → UPLOADED_TO_RSYNC_NET + curl --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php?ep_num=${ep_num}&status=MEDIA_TRANSCODED" + curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" + fi else echo "skipping...." - echo "cp -v \"${mediafile}\" \"/var/IA/uploads/hpr${ep_num}_source.${mediaextension}\"" - echo "rm -v ${fname}_sox.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav" - echo "mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/" + echo "cp -v \"${mediafile}\" \"${upload_dir}/hpr${ep_num}_source.${mediaextension}\"" + echo "rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav" + #echo "mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/" echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1\" -O - | xmlstarlet val --err -" echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1\" -O - | xmlstarlet val --err -" echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1\" -O - | xmlstarlet val --err -" - echo "rsync -ave ssh --partial --progress /var/IA/uploads/ borg:/data/IA/uploads/" + echo "rsync -ave ssh --partial --progress ${upload_dir}/ borg:/data/IA/uploads/" fi