hpr-tools/workflow/hprtranscode-simple.bash.2022-07-31

#!/usr/bin/env bash
# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
#============================================================
TEMP_DIR="/var/tmp/"
CHANNELS="1"
FIXAUDIO="1"
ARTIST="EMPTY"
TITLE="EMPTY"
YEAR="EMPTY"
SLOT="EMPTY"
basedir="/var/IA"
upload_dir="${basedir}/uploads"
theme="${basedir}/theme.wav"
outro="${basedir}/2022-03-07-outro.wav"
ttsserver="http://localhost:5500"
processing_dir="/home/ken/tmp/hpr/processing"
git_image_dir="/home/ken/sourcecode/hpr/HPR_Public_Code/www/images/hosts"
thedate=$(/usr/bin/date -u +%Y-%m-%d_%H-%M-%SZ_%A)
echo "Processing the next HPR Show in the queue"


if [ $( curl -s -o /dev/null -w "%{http_code}" -X 'GET' "${ttsserver}/api/voices" -H 'accept: */*' ) != "200" ]
then
  echo "Please start the tts-server \"podman run -it -p 5500:5500 synesthesiam/opentts:en\""
  exit
fi

###################
# Get the show
#
# 
if [[ -f "${1}" && -n "${2}" ]]
then
  mediafile="${1}"
  ep_num="${2}"
  echo "The duration is \"$( \date -ud "1970-01-01 $( ffprobe -i "${mediafile}" 2>&1| awk -F ': |, ' '/Duration:/ { print $2 }' )" +%s )\"."
else
  response=$( curl --silent --netrc-file ${HOME}/.netrc "https://hackerpublicradio.org/cms/status.php" | \
  grep 'SHOW_POSTED' | \
  head -1 | \
  sed 's/,/ /g' )

  if [ -z "${response}" ]
  then
    echo "INFO: There appear to be no more shows with the status \"SHOW_POSTED\"."
    echo "Getting a list of all the reservations."
    curl --silent --netrc-file ${HOME}/.netrc "https://hackerpublicradio.org/cms/status.php"
    exit 3
  fi

  timestamp_epoc="$( echo ${response} | awk '{print $1}' )"
  ep_num="$( echo ${response} | awk '{print $2}' )"
  ep_date="$( echo ${response} | awk '{print $3}' )"
  key="$( echo ${response} | awk '{print $4}' )"
  status="$( echo ${response} | awk '{print $5}' )"
  email="$( echo ${response} | awk '{print $6}' )"

  #source_dir="hpr:/home/hpr/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
  dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
  echo detox -v "${processing_dir}/${dest_dir}/"
  detox -vr "${processing_dir}/${dest_dir}/"
  cd "${processing_dir}/${dest_dir}/"
  pwd

  echo "INFO: Processing hpr${ep_num} from ${email}"
  echo "INFO: Working directory is \"${processing_dir}/${dest_dir}/\""
  echo ""

  media=$( find "${processing_dir}/${timestamp_epoc}_${ep_num}_${ep_date}_${key}" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' )
  if [ -z "${media}" ]
  then
    echo "ERROR: Can't find any media in \"${processing_dir}/${dest_dir}/\""
    find "${processing_dir}/${dest_dir}/" -type f
    exit 1
  fi

  mediafile=""

  if [ "$( echo "${media}" | wc -l )" -ne 1 ]
  then
    select this_media in $( echo "${media}" )
    do 
      echo "INFO: You selected \"${this_media}\"."
      ls -al "${this_media}"
      mediafile="${this_media}"
      break
    done
  else
    echo "INFO: Selecting media as \"${media}\"."
    mediafile="${media}"
  fi
  echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
  echo "File information for \"${mediafile}\"" | tee -a ${fname}_tmp.log
  ffprobe ${mediafile} 2>&1 | grep Audio:
  mediainfo ${mediafile}
  audio2image.bash ${mediafile}
  xdg-open ${mediafile%.*}.png >/dev/null 2>&1 &
  read -p "Source Waveform look ok ? (N|y) ? " -n 1 -r
  echo    # (optional) move to a new line
  if [[ ! $REPLY =~ ^[yY]$ ]]
  then
    echo "skipping...."
    exit
  fi

fi

re='^[0-9]+$'
if ! [[ $ep_num =~ $re ]] 
then
   echo "error: episode \"${ep_num}\" is not a number"
   exit 1
fi

if [ ! -f "${outro}" ]
then
  echo "sorry, file \"${outro}\" does not exist"
  exit 1
fi

if [ ! -f "${theme}" ]
then
  echo "sorry, file \"${theme}\" does not exist"
  exit 1
fi

if [ ! -f "${mediafile}" ]
then
  echo "sorry, media file \"${mediafile}\" does not exist"
  exit
fi

if [ ! -r "${mediafile}" ]
then
  echo "sorry, media file \"${mediafile}\" is not readable"
  exit
fi

if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ]
then
  echo "sorry, media file \"${mediafile}\" has no audio track"
  exit
fi

mpv -vo=null "${mediafile}"
read -p "Is there any problems with the audio (N|y) ? " -n 1 -r
echo    # (optional) move to a new line
if [[ $REPLY =~ ^[yY]$ ]]
then
  echo "skipping...."
  exit 1
fi

# extract file name and extension
media_dir=$(dirname ${mediafile})
fname=${mediafile%.*}
ext=${mediafile/*./}

if [[ -e hpr${ep_num}.wav ]] || [[ -e hpr${ep_num}.mp3 ]] || [[ -e hpr${ep_num}.ogg ]] || [[ -e hpr${ep_num}.spx ]] || [[ -e hpr${ep_num}_summary.wav ]] || [[ -e hpr${ep_num}.wav ]] || [[ -e ${fname}_mez.wav ]] || [[ -e ${fname}_sox_norm.wav ]] || [[ -e ${fname}_mezzanine.wav ]] || [[ -e ${fname}_tmp.pcm ]] || [[ -e ${fname}_tmp.log ]] || [[ -e hpr${ep_num}_intro.wav ]] || [[ -e ~hpr${ep_num}_summary.wav ]] || [[ -e ~~hpr${ep_num}_summary.wav ]] 
then
  echo "Files for this episode already exist."
  ls -1 *hpr${ep_num}* ${fname}_* 2>/dev/null
  exit 1
fi

echo "--------------------------------------------------------------------------------"
echo "Geting metadata for hpr${ep_num}"

while read -r line
do
  field=$(echo $line | awk -F ':' '{print $1}')
  case $field in
  "HPR_summary")
      HPR_summary=$(echo $line | grep "HPR_summary: " | cut -c 14- )
      continue
      ;;
  "HPR_album")
     HPR_album=$(echo $line | grep "HPR_album: " | cut -c 12- )
      continue
      ;;
  "HPR_artist")
    HPR_artist=$(echo $line | grep "HPR_artist: "  | cut -c 13- )
      continue
      ;;
  "HPR_comment")
    HPR_comment=$(echo $line | grep "HPR_comment: " | cut -c 14- )
      continue
      ;;
  "HPR_genre")
    HPR_genre=$(echo $line | grep "HPR_genre: " | cut -c 12- )
      continue
      ;;
  "HPR_title")
    HPR_title=$(echo $line | grep "HPR_title: " | cut -c 12- )
      continue
      ;;
  "HPR_track")
    HPR_track=$(echo $line | grep "HPR_track: " | cut -c 12- )
      continue
      ;;
  "HPR_year")
    HPR_year=$(echo $line | grep "HPR_year: " | cut -c 11- )
      continue
      ;;
  "HPR_duration")
    HPR_duration=$(echo $line | grep "HPR_duration: " | cut -c 15- )
      continue
      ;;
  "HPR_explicit")
    HPR_explicit=$(echo $line | grep "HPR_explicit: " | cut -c 15- )
      continue
      ;;
  "HPR_license")
    HPR_license=$(echo $line | grep "HPR_license: " | cut -c 14- )
      continue
      ;;
  esac
done < <( wget --timeout=10 --tries=1 --quiet http://hackerpublicradio.org/say.php?id=${ep_num} -O - )

if [[ -z "$HPR_album" || -z "$HPR_artist" || -z "$HPR_comment" || -z "$HPR_genre" || -z "$HPR_title" || -z "$HPR_track" || -z "$HPR_year" || -z "$HPR_summary" || -z "$HPR_duration" || -z "$HPR_explicit" || -z "$HPR_license" ]]
then
  echo "Could not find information on ${ep_num}. Has the show been posted ?"
  exit;
fi
echo "--------------------------------------------------------------------------------"
echo "album       : $HPR_album"
echo "artist      : $HPR_artist"
echo "comment     : $HPR_comment"
echo "genre       : $HPR_genre"
echo "title       : $HPR_title"
echo "track       : $HPR_track"
echo "year        : $HPR_year"
echo "summary     : $HPR_summary"
echo "duration    : $HPR_duration"
echo "explicit    : $HPR_explicit"
echo "license     : $HPR_license"

if [[ $HPR_duration == "0" ]]
then
  echo "The duration is set to 0. Please update the show with the correct time."
  exit;
fi

#============================================================
# Preproc`s the source file

echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
echo "Prepare mezzanine file" | tee -a ${fname}_tmp.log
ffmpeg -i ${mediafile} -ar 44100 -ac $CHANNELS ${fname}_mezzanine.wav > ${fname}_tmp.log 2>&1

echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
echo "Add HPR Branding" | tee -a ${fname}_tmp.log

echo "Creating the summary" | tee -a ${fname}_tmp.log
#echo "$HPR_summary" - | text2wave -F 44100 - -o hpr${ep_num}_summary.wav #festival --tts
#echo "$HPR_summary" - | espeak -w ~hpr${ep_num}_summary.wav
echo "$HPR_summary" - | curl -X 'GET' -G --data-urlencode "voice=coqui-tts:en_ljspeech" --data-urlencode "text=${HPR_summary}" --data-urlencode "vocoder=high" --data-urlencode "denoiserStrength=0.03" --data-urlencode "cache=false" ${ttsserver}/api/tts -H 'accept: */*' --output ~hpr${ep_num}_summary.wav

ffmpeg -i ~hpr${ep_num}_summary.wav -ar 44100 ~~hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1

sox -n -r 44100 -c 1 silence.wav trim 0.0 6.0 >> ${fname}_tmp.log 2>&1

sox silence.wav ~~hpr${ep_num}_summary.wav hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1

echo "Adding the theme" | tee -a ${fname}_tmp.log
sox -m "hpr${ep_num}_summary.wav" "${theme}" "hpr${ep_num}_intro.wav" >> ${fname}_tmp.log 2>&1

echo "Creating the sandwitch" | tee -a ${fname}_tmp.log

sox "hpr${ep_num}_intro.wav" "${fname}_mezzanine.wav" "${outro}" "hpr${ep_num}.wav" >> ${fname}_tmp.log 2>&1

echo "Normalizing the wav files" | tee -a ${fname}_tmp.log
#sox --temp "${TEMP_DIR}" --norm hpr${ep_num}.wav hpr${ep_num}_norm.wav
ffmpeg -i hpr${ep_num}.wav -af loudnorm=I=-16:LRA=11:TP=-1.5  hpr${ep_num}_norm.wav  >> ${fname}_tmp.log 2>&1

mv -v hpr${ep_num}_norm.wav ${upload_dir}/hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1

echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
echo "File information" | tee -a ${fname}_tmp.log
ffprobe ${upload_dir}/hpr${ep_num}.wav 2>&1 | grep Audio:
mediainfo ${upload_dir}/hpr${ep_num}.wav
audio2image.bash ${upload_dir}/hpr${ep_num}.wav
xdg-open ${upload_dir}/hpr${ep_num}.png >/dev/null 2>&1 &
read -p "Processed Waveform look ok ? (N|y) ? " -n 1 -r
echo    # (optional) move to a new line
if [[ ! $REPLY =~ ^[yY]$ ]]
then
  echo "skipping...."
  exit
fi
rm -v ${upload_dir}/hpr${ep_num}.png

echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
echo "Convert to opus" | tee -a ${fname}_tmp.log
ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.opus 2>> ${fname}_tmp.log 1>&2

echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
echo "Convert to flac" | tee -a ${fname}_tmp.log
sox --temp "${TEMP_DIR}" -S ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.flac 2>> ${fname}_tmp.log 1>&2

echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
echo "Convert to mp3" | tee -a ${fname}_tmp.log
sox --temp "${TEMP_DIR}" -S ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.mp3 2>> ${fname}_tmp.log 1>&2

echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
echo "Convert to ogg" | tee -a ${fname}_tmp.log
ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.ogg 2>> ${fname}_tmp.log 1>&2

echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
echo "Convert to spx" | tee -a ${fname}_tmp.log
ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.spx 2>> ${fname}_tmp.log 1>&2

for check_file in \
${upload_dir}/hpr${ep_num}.wav \
${upload_dir}/hpr${ep_num}.opus \
${upload_dir}/hpr${ep_num}.flac \
${upload_dir}/hpr${ep_num}.mp3 \
${upload_dir}/hpr${ep_num}.ogg \
${upload_dir}/hpr${ep_num}.spx
do
  if [[ ! -s "${check_file}" ]]
  then
    echo "ERROR: Something went wrong encoding of the file \"${check_file}\""
  fi
done

if [[ ! -s ${upload_dir}/hpr${ep_num}.wav ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.opus ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.flac ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.mp3 ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.ogg ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.spx ]] 
then
  echo "ERROR: Something went wrong encoding the files"
  exit 1
fi

echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
echo "Fixing Tags" | tee -a ${fname}_tmp.log

fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="${HPR_comment} The license is ${HPR_license}" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" ${upload_dir}/hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2

fix_tags ${upload_dir}/hpr${ep_num}*

#echo "Changing the file dates to the time of upload"
touch -r ${mediafile} hpr${ep_num}*
touch -r ${mediafile} ${upload_dir}/hpr${ep_num}*

ls -al hpr${ep_num}* ${upload_dir}/hpr${ep_num}* | tee -a ${fname}_tmp.log


echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log
echo "Transferring files to Servers" | tee -a ${fname}_tmp.log


rsync -ave ssh --partial --progress --ignore-existing ${upload_dir}/hpr${ep_num}.mp3 ${upload_dir}/hpr${ep_num}.ogg ${upload_dir}/hpr${ep_num}.spx hpr:www/eps/

firefox "https://hackerpublicradio.org/local/hpr${ep_num}.mp3" >/dev/null 2>&1 &
firefox "https://hackerpublicradio.org/local/hpr${ep_num}.ogg" >/dev/null 2>&1 &
firefox "file://${upload_dir}/hpr${ep_num}.mp3" >/dev/null 2>&1 &
firefox "file://${upload_dir}/hpr${ep_num}.ogg" >/dev/null 2>&1 &
firefox "https://hackerpublicradio.org/eps.php?id=${ep_num}" >/dev/null 2>&1 &

mpv "http://hackerpublicradio.org/local/hpr${ep_num}.spx" "http://hackerpublicradio.org/local/hpr${ep_num}.ogg" "http://hackerpublicradio.org/local/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.spx" "file://${upload_dir}/hpr${ep_num}.opus" "file://${upload_dir}/hpr${ep_num}.ogg" "file://${upload_dir}/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.flac"

echo "Source: $( mediainfo --Output=XML --Full "${mediafile}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
mediainfo --Output=XML --Full hpr${ep_num}* ${upload_dir}/hpr${ep_num}* | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' | sort | uniq -c

read -p "Remove files for \"${fname}\" (y|N) ? " -n 1 -r
echo    # (optional) move to a new line
if [[ $REPLY =~ ^[Yy]$ ]]
then
  mediafilename=$(basename "${mediafile}")
  mediaextension="${mediafilename##*.}"                                             # "
  ssh hpr -t "mkdir /home/hpr/www/eps/hpr${ep_num}" >/dev/null 2>&1
  rsync -ave ssh --partial --progress --ignore-existing "${mediafile}" hpr:www/eps/hpr${ep_num}/hpr${ep_num}_source.${mediaextension}
  ssh hpr -t "ls -al /home/hpr/www/eps/hpr${ep_num}*" 
  cp -v "${mediafile}" "${upload_dir}/hpr${ep_num}_source.${mediaextension}"

  #echo "Remove temp files"
  rm -v ~hpr${ep_num}_summary.wav ~~hpr${ep_num}_summary.wav silence.wav 
  rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav
  #mv -v ${fname}* hpr${ep_num}*  *_${ep_num}_* /var/IA/done/
  wget --timeout=0 -q "http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1" -O - | xmlstarlet val --err -
  wget --timeout=0 -q "http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1" -O - | xmlstarlet val --err -
  wget --timeout=0 -q "http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1" -O - | xmlstarlet val --err -
  wget --timeout=0 -q "http://hackerpublicradio.org/rss-future.php" -O - | xmlstarlet val --err -

  rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* borg:/data/IA/uploads/
  echo "$( ssh borg -t "ls -al /data/IA/uploads/hpr${ep_num}*" ; ls -al ${upload_dir}/hpr${ep_num}* )" | grep "hpr${ep_num}" | awk '{print $5, $NF}' | sort
  # SHOW_SUBMITTED → METADATA_PROCESSED → SHOW_POSTED → MEDIA_TRANSCODED → UPLOADED_TO_IA → UPLOADED_TO_RSYNC_NET
  curl --netrc-file ${HOME}/.netrc "https://hackerpublicradio.org/cms/status.php?ep_num=${ep_num}&status=MEDIA_TRANSCODED"
else
  echo "skipping...."
  echo "cp -v \"${mediafile}\" \"${upload_dir}/hpr${ep_num}_source.${mediaextension}\""
  echo "rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav"
  #echo "mv -v ${fname}* hpr${ep_num}*  *_${ep_num}_* /var/IA/done/"
  echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
  echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
  echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1\" -O - | xmlstarlet val --err -"
  echo "rsync -ave ssh --partial --progress ${upload_dir}/ borg:/data/IA/uploads/"
fi