hpr-tools/workflow/process_episode.bash

#!/usr/bin/env bash
# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/
#============================================================

# git clone https://github.com/davmo/fix_tags.git
# git clone https://repo.anhonesthost.net/HPR/hpr-tools.git

#################################################
# Variables

debug="1"
force_overwrite=true                      # Sets the behavour when files exist or not
processing_dir="$HOME/tmp/hpr/processing" # The directory where the files will be copied to for processing
theme="${processing_dir}/theme.flac"      # The hpr theme
silence="${processing_dir}/silence.flac"  # A segment of silence to offset the tts in the intro
outro_flac="${processing_dir}/outro.flac" # The outro edited media
outro_srt="${processing_dir}/outro.srt"   # The outro subtitle file
intro_srt="${processing_dir}/intro.srt"   # The intro subtitle template file

piper_bin="/opt/bin/piper/piper/piper"    # The location of the text to speech engine
piper_voice="/opt/bin/piper/piper/piper-voices/en/en_US/lessac/medium/en_US-lessac-medium.onnx"

working_dir_bypass="false"
skip_post_show="false"
skip_media_encoding="false"

acceptable_duration_difference="2" # Seconds
unsafe_episode_variables=( shownotes_json shownotes_html hostid artist email title summary series_id series_name explicit license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised )
audio_formats=( flac wav mp3 ogg opus spx )
episode_formats=( flac wav mp3 ogg opus spx srt txt )

#################################################
# Display Error message, display help and exit

function echo_error() {
  echo -e "ERROR: $@" #1>&2
  display_help_and_exit
  exit 1
}

#################################################
# Display Information message

function echo_debug() {
  if [ "${debug}" != "0"  ]
  then
    echo -e "INFO: $@" #1>&2
  fi
}

#################################################
# Display Help

function display_help_and_exit() {
  echo_debug "For more information see https://repo.anhonesthost.net/HPR/hpr_documentation"
  exit 1
}

#################################################
# Calculate difference 

function abs_diff() {
  echo $(($1 >= $2 ? $1 - $2 : $2 - $1))
}

#################################################
# Program Checks

function program_checks() {

  echo_debug "Completing program checks. program_checks()"

  if [ ! -d "${processing_dir}" ]
  then
    echo_error "The \"${processing_dir}\" is required but is not defined."
  fi

  if [[ ! -s "${theme}" || ! -s "${silence}" || ! -s "${outro_flac}" || ! -s "${outro_srt}" || ! -s "${intro_srt}" ]]
  then
    echo_error "The files for the theme are not available."
    ls -al "${theme}" "${silence}" "${outro_flac}" "${outro_srt}" "${intro_srt}"
  fi

  function is_installed() {
    for this_program in "$@"
    do
      if ! command -v ${this_program} 2>&1 >/dev/null
      then
        echo_error "The application \"${this_program}\" is required but is not installed."
      fi
    done
  }

  is_installed audio2image.bash awk base64 cat csvtojson curl date eval extract_images ffprobe ffmpeg file find grep head jq kate magick mediainfo mv realpath remove-image.pl rsync seamonkey sed sed sort sponge ssh touch touch wget hpr-check-ccdn-links

  for arg in $*
  do
    if [ "$( echo "${arg}" | grep --count --ignore-case --perl-regexp -- '-h|--help' )" -ne "0" ]
    then
      echo_debug "Process the next SHOW_SUBMITTED show from the queue."
      echo_debug "If a directory is provided then the shownotes.json will be used."
    fi
  done

}

#################################################
# Allows override of arguments from the commandline

function argument_override() {

  echo_debug "Overriding values with terminal arguments. argument_override()"
  
  

  # Argument Override
  if [ $# -gt 0 ]
  then
    for argument
    do
      if [[ $argument =~ ^[^=]+=.*$ ]]
      then
        this_key="${argument%=*}"
        this_value="${argument#*=}"
        for this_episode_variable in "${unsafe_episode_variables[@]}"
        do
          if [ "$( echo "${this_episode_variable}" | grep --perl-regexp --count "^${this_key}$" )" -ne "0" ]
          then
            this_value="$( echo "${this_value}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g'  )"
            echo_debug "URL encoding \"${this_key}\" as \"${this_value}\"."
          fi
        done
        eval "${this_key}=${this_value}"
        echo_debug "Replacing \"${this_key}\" with \"${this_value}\"."
        check_variable_is_correct ${this_key}
        if [ "${this_key}" == "working_dir" ]
        then
          working_dir_bypass="true"
        fi       
      fi
    done
  fi
}

#################################################
# Variable Checks

function check_variable_is_correct() {

  for argument in "$@"
  do
    case "${argument}" in

      album)
        if [[ -z "${album}" || "${album}" == "null" ]]
        then
          echo_error "The \"album\" variable is missing."
        fi
        if [ "$( echo "${album}" | grep --perl-regexp '^Hacker Public Radio$' | wc --lines )" -eq "0" ]
        then
          echo_error "The \"album\" variable is not \"Hacker Public Radio\"."
        fi
      ;;
      artist)
        if [[ -z "${artist}" || "${artist}" == "null" ]]
        then
          echo_error "The \"artist\" variable is missing."
        fi
      ;;
      assets_csv)
        if [[ ! -s "${assets_csv}" || -z "${assets_csv}" ]]
        then
          echo_error "The \"assets_csv\" variable/file is missing."
        fi
        if [ "$( file --brief --mime-type "${assets_csv}" | grep --count 'text/csv' )" -ne "1" ]
        then
          echo_error "The \"assets_csv\" file has not a valid \"text/csv\" mime type."
        fi
        if [ "$( wc --lines ${assets_csv} | awk '{print $1}' )" -le "1" ]
        then
          echo_error "The \"${assets_csv}\" file is empty."
        fi
      ;;
      assets_json)
        if [[ ! -s "${assets_json}" || -z "${assets_json}" ]]
        then
          echo_error "The \"assets_json\" variable/file is missing."
        fi
        if [ "$( file --brief --mime-type "${assets_json}" | grep --count 'application/json' )" -ne "1" ]
        then
          echo_error "The \"assets_json\" variable has not a valid \"application/json\" mime type."
        fi
        jq '.' "${assets_json}"  >/dev/null 2>&1
        if [ $? -ne 0 ]
        then
          echo_error "The file \"${assets_json}\" is not valid json."
        fi
      ;;
      comment)
        if [[ -z "${comment}" || "${comment}" == "null" ]]
        then
          echo_error "The \"comment\" variable is missing."
        fi
      ;;
      date)
        if [[ -z "${date}" || "${date}" == "null" ]]
        then
          echo_error "The \"date\" variable is missing."
        fi
      ;;
      duration)
        if [[ -z "${duration}" || "${duration}" == "null" ]]
        then
          echo_error "The \"duration\" variable is missing."
        fi
        if [[ -z "${duration}" || "${duration}" -lt "30" || "${duration}" -gt "30000" ]]
        then
          echo_error "Invalid duration missing or outside range 30 to 30000." >&2
        fi
      ;;
      duration_iso8601)
        if [[ -z "${duration_iso8601}" || "${duration_iso8601}" == "null" ]]
        then
          echo_error "The \"duration_iso8601\" variable is missing."
        fi
      ;;
      email)
        if [ -z "${email}" ]
        then
          echo_error "The \"email\" variable is missing."
        fi
      ;;
      email_unpadded)
        if [ -z "${email_unpadded}" ]
        then
          echo_error "The \"email_unpadded\" variable is missing."
        fi
      ;;
      episode_body_flac)
        if [ -z "${episode_body_flac}" ]
        then
          echo_error "The \"episode_body_flac\" variable is missing." 
       fi
      ;;
      episode_body_srt)
        if [ -z "${episode_body_srt}" ]
        then
          echo_error "The \"episode_body_srt\" variable is missing." 
       fi
      ;;
      episode_final_flac)
        if [ -z "${episode_final_flac}" ]
        then
          echo_error "The \"episode_final_flac\" variable is missing." 
       fi
      ;;
      episode_intro_flac)
        if [ -z "${episode_intro_flac}" ]
        then
          echo_error "The \"episode_intro_flac\" variable is missing." 
       fi
      ;;
      episode_intro_srt)
        if [ -z "${episode_intro_srt}" ]
        then
          echo_error "The \"episode_intro_srt\" variable is missing." 
       fi
      ;;
      episode_outro_srt)
        if [ -z "${episode_outro_srt}" ]
        then
          echo_error "The \"episode_outro_srt\" variable is missing." 
       fi
      ;;
      episode_sandwitch_flac)
        if [ -z "${episode_sandwitch_flac}" ]
        then
          echo_error "The \"episode_sandwitch_flac\" variable is missing." 
       fi
      ;;
      episode_srt)
        if [ -z "${episode_srt}" ]
        then
          echo_error "The \"episode_srt\" variable is missing." 
       fi
      ;;
      episode_summary_flac)
        if [ -z "${episode_summary_flac}" ]
        then
          echo_error "The \"episode_summary_flac\" variable is missing." 
       fi
      ;;
      episode_tts_flac)
        if [ -z "${episode_tts_flac}" ]
        then
          echo_error "The \"episode_tts_flac\" variable is missing." 
       fi
      ;;
      episode_tts_wav)
        if [ -z "${episode_tts_wav}" ]
        then
          echo_error "The \"episode_tts_wav\" variable is missing." 
       fi
      ;;
      episode_summary_json)
        if [[ ! -s "${episode_summary_json}" || -z "${episode_summary_json}" ]]
        then
          echo_error "The \"episode_summary_json\" variable/file is missing."
        fi
        if [ "$( file --brief --mime-type "${episode_summary_json}" | grep --count 'application/json' )" -ne "1" ]
        then
          echo_error "The \"episode_summary_json\" variable has not a valid \"application/json\" mime type."
        fi
        jq '.' "${episode_summary_json}"  >/dev/null 2>&1
        if [ $? -ne 0 ]
        then
          echo_error "The file \"${episode_summary_json}\" is not valid json."
        fi
      ;;
      ep_date)
        if [ -z "${ep_date}" ]
        then
          echo_error "The \"ep_date\" variable is missing."
        fi
      ;;
      ep_num)
        if [ -z "${ep_num}" ]
        then
          echo_error "The \"ep_num\" variable is missing."
        fi
        if [ "$( echo "${ep_num}" | grep --perl-regexp '^(0{0,3}[1-9]\d{0,2}|[1-9]\d{0,3})$' | wc --lines )" -eq "0" ]
        then
          echo_error "The \"ep_num\" variable is not a valid number between 1 and 9999."
        fi
      ;;
      expected_duration)
        if [ -z "${expected_duration}" ]
        then
          echo_error "The \"expected_duration\" variable is missing." 
       fi
      ;;
      explicit)
        if [[ -z "${explicit}" || "${explicit}" == "null" ]]
        then
          echo_error "The \"explicit\" variable is missing."
        fi
      ;;
      files_json)
        if [[ ! -s "${files_json}" || -z "${files_json}" ]]
        then
          echo_error "The \"files_json\" variable/file is missing."
        fi
        if [ "$( file --brief --mime-type "${files_json}" | grep --count 'application/json' )" -ne "1" ]
        then
          echo_error "The \"files_json\" variable has not a valid \"application/json\" mime type."
        fi
        jq '.' "${files_json}"  >/dev/null 2>&1
        if [ $? -ne 0 ]
        then
          echo_error "The file \"${files_json}\" is not valid json."
        fi
      ;;
      files_xml)
        if [[ ! -s "${files_xml}" || -z "${files_xml}" ]]
        then
          echo_error "The \"files_xml\" variable/file is missing."
        fi
        if [ "$( file --brief --mime-type "${files_xml}" | grep --count 'text/xml' )" -ne "1" ]
        then
          echo_error "The \"files_xml\" variable has not a valid \"text/xml\" mime type."
        fi
        xmllint --format "${files_xml}" >/dev/null 2>&1
        if [ $? -ne 0 ]
        then
          echo_error "The file \"${files_xml}\" is not valid xml."
        fi
      ;;
      genre)
        if [[ -z "${genre}" || "${genre}" == "null" ]]
        then
          echo_error "The \"genre\" variable is missing."
        fi
      ;;
      HOME)
        if [[ ! -d "${HOME}" || -z "${HOME}" ]]
        then
          echo_error "The \"HOME\" variable is missing or file \"${HOME}\" does not exist."
        fi
      ;;
      hostid)
        if [[ -z "${hostid}" || "${hostid}" == "null" ]]
        then
          echo_error "The \"hostid\" variable is missing."
        fi
      ;;
      intro_duration)
        if [ -z "${intro_duration}" ]
        then
          echo_error "The \"intro_duration\" variable is missing." 
       fi
      ;;
      intro_srt)
        if [[ ! -d "${intro_srt}" || -z "${intro_srt}" ]]
        then
          echo_error "The \"intro_srt\" variable is missing or file \"${intro_srt}\" does not exist."
        fi
      ;;
      key)
        if [ -z "${key}" ]
        then
          echo_error "The \"key\" variable is missing."
        fi
      ;;
      license)
        if [[ -z "${license}" || "${license}" == "null" ]]
        then
          echo_error "The \"license\" variable is missing."
        fi
      ;;
      license_url)
        if [[ -z "${license_url}" || "${license_url}" == "null" ]]
        then
          echo_error "The \"license_url\" variable is missing."
        fi
      ;;
      media)
        if [ -z "${media}" ]
        then
          echo_error "The \"media\" variable is missing."
        fi
      ;;
      media_basename)
        if [ -z "${media_basename}" ]
        then
          echo_error "The \"media_basename\" variable is missing ."
        fi
      ;;
      media_ffprobe)
        if [ -z "${media_ffprobe}" ]
        then
          echo_error "The \"media_ffprobe\" variable is missing." 
       fi
      ;;
      media_file_mime)
        if [ -z "${media_file_mime}" ]
        then
          echo_error "The \"media_file_mime\" variable is missing." 
       fi
      ;;
      media_file_mime_type)
        if [ -z "${media_file_mime_type}" ]
        then
          echo_error "The \"media_file_mime_type\" variable is missing." 
       fi
      ;;
      outro_duration)
        if [ -z "${outro_duration}" ]
        then
          echo_error "The \"outro_duration\" variable is missing." 
       fi
      ;;
      outro_flac)
        if [[ ! -s "${outro_flac}" || -z "${outro_flac}" ]]
        then
          echo_error "The \"outro_flac\" variable is missing or file \"${outro_flac}\" does not exist."
        fi
      ;;
      outro_srt)
        if [[ ! -s "${outro_srt}" || -z "${outro_srt}" ]]
        then
          echo_error "The \"outro_srt\" variable is missing or file \"${outro_srt}\" does not exist."
        fi
      ;;
      piper_bin)
        if [ -z "${piper_bin}" ]
        then
          echo_error "The \"ep_num\" variable is missing."
        fi
      ;;
      piper_voice)
        if [ -z "${ep_num}" ]
        then
          echo_error "The \"ep_num\" variable is missing."
        fi
      ;;
      processing_dir)
        if [[ ! -d "${processing_dir}" || -z "${processing_dir}" ]]
        then
          echo_error "The \"processing_dir\" variable is missing or is not a directory."
        fi
      ;;
      shownotes_edited)
        if [[ ! -s "${shownotes_edited}" || -z "${shownotes_edited}" ]]
        then
          echo_error "The \"shownotes_edited\" variable/file is missing."
        fi
#         if [ "$( file --brief --mime-type "${shownotes_edited}" | grep --count 'text/html' )" -ne "1" ]
#         then
#           echo_error "The \"shownotes_edited\" variable has not a valid \"text/html\" mime type \"${shownotes_edited}\"."
#         fi
      ;;
      shownotes_html)
        if [[ ! -s "${shownotes_html}" || -z "${shownotes_html}" ]]
        then
          echo_error "The \"shownotes_html\" variable/file is missing."
        fi
#         if [ "$( file --brief --mime-type "${shownotes_html}" | grep --count 'text/html' )" -ne "1" ]
#         then
#           echo_error "The \"shownotes_html\" variable has not a valid \"text/html\" mime type. \"${shownotes_html}\""
#         fi
      ;;
      shownotes_json)
        if [ ! -s "${shownotes_json}" ]
        then
          echo_error "The \"shownotes_json\" variable is missing."
        fi
        if [ -z "${shownotes_json}" ]
        then
          echo_error "The \"shownotes_json\" file is missing."
        fi
#         if [ "$( file --brief --mime-type "${shownotes_json}" | grep --count 'application/json' )" -ne "1" ]
#         then
#           echo_error "The \"shownotes_json\" variable has not a valid \"application/json\" mime type."
#         fi
        jq '.' "${shownotes_json}"  >/dev/null 2>&1
        if [ $? -ne 0 ]
        then
          echo_error "The file \"${shownotes_json}\" is not valid json."
        fi
      ;;
      silence)
        if [[ ! -d "${silence}" || -z "${silence}" ]]
        then
          echo_error "The \"silence\" variable is missing or file \"${silence}\" does not exist."
        fi
      ;;
      source_duration)
        if [ -z "${source_duration}" ]
        then
          echo_error "The \"source_duration\" variable is missing." 
       fi
      ;;
      summary)
        if [[ -z "${summary}" || "${summary}" == "null" ]]
        then
          echo_error "The \"summary\" variable is missing."
        fi
      ;;
      synopsis)
        if [[ -z "${synopsis}" || "${synopsis}" == "null" ]]
        then
          echo_error "The \"synopsis\" variable is missing."
        fi
      ;;
      tags)
        if [[ -z "${tags}" || "${tags}" == "null" ]]
        then
          echo_error "The \"tags\" variable is missing."
        fi
      ;;
      theme)
        if [[ ! -s "${theme}" || -z "${theme}" ]]
        then
          echo_error "The \"theme\" variable is missing or file \"${theme}\" does not exist."
        fi
      ;;
      timestamp_epoc)
        if [ -z "${timestamp_epoc}" ]
        then
          echo_error "The \"timestamp_epoc\" variable is missing."
        fi
      ;;
      title)
        if [[ -z "${title}" || "${title}" == "null" ]]
        then
          echo_error "The \"title\" variable is missing."
        fi
      ;;
      track)
        if [[ -z "${track}" || "${track}" == "null" ]]
        then
          echo_error "The \"track\" variable is missing."
        fi
      ;;
      working_dir)
        if [[ ! -d "${working_dir}" || -z "${working_dir}" ]]
        then
          echo_error "The \"working_dir\" variable is missing or is not a directory."
        fi
      ;;
      year)
        if [[ -z "${year}" || "${year}" == "null" ]]
        then
          echo_error "The \"year\" variable is missing."
        fi
      ;;

      *)
        echo_error "An unknown variable \"${argument}\" was provided."
      ;;
    esac
  done

}

#################################################
# Get the next show in the queue

function get_next_show_from_hpr_hub() {

  echo_debug "Processing the next HPR Show in the queue. get_next_show_from_hpr_hub()"
  
  check_variable_is_correct processing_dir HOME

  if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${processing_dir}/status.csv" )" != 200 ]
  then
    echo_error "Could not get a list of the queue status from \"https://hub.hackerpublicradio.org/cms/status.php\""
  fi

  if [ ! -s "${processing_dir}/status.csv" ]
  then
    echo_error "Failed to retrieve \"${processing_dir}/status.csv\" from server."
  fi

  response=$( cat "${processing_dir}/status.csv" | grep ',SHOW_SUBMITTED,' | sort -t ',' -k 2 | head -1 | sed 's/,/ /g' )
  if [ -z "${response}" ]
  then
    echo_debug "Getting a list of all the reservations."
    curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | sort -n
    echo_error "There appear to be no more shows with the status \"SHOW_SUBMITTED\"."
  fi

  timestamp_epoc="$( echo ${response} | awk '{print $1}' )"
  ep_num="$( echo ${response} | awk '{print $2}' )"
  ep_date="$( echo ${response} | awk '{print $3}' )"
  key="$( echo ${response} | awk '{print $4}' )"
  email="$( echo ${response} | awk '{print $6}' )"
  email_unpadded="$( echo $email | sed 's/.nospam@nospam./@/g' )"

  hpr_upload_dir="hub.hackerpublicradio.org/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
  source_dir="hpr:${hpr_upload_dir}"
  dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
  working_dir="${processing_dir}/${dest_dir}"
  
  check_variable_is_correct timestamp_epoc ep_num ep_date key email email_unpadded

  echo_debug "Downloading hpr${ep_num} from ${email_unpadded}"
  echo_debug ""
  echo_debug "rsync -ave ssh --partial --progress ${source_dir}/ ${working_dir}/"
  rsync -ave ssh --partial --progress ${source_dir}/ ${working_dir}/

}

#################################################
# Get the show information from a local directory

function get_ep_num_from_local_dir() {

  echo_debug "Processing a local directory. get_ep_num_from_local_dir()"

  check_variable_is_correct working_dir

  if [ ! -s "${shownotes_json}" ]
  then

    echo_debug "Could not find a \"shownotes.json\" in the working directory \"${working_dir}/\""

    if [ -z "${ep_num}" ]
    then
      echo_debug "Attempting to get episode number from the \"${working_dir}\""
      ep_num="$( echo "${working_dir}" | grep --color=never --perl-regexp --only-matching '_[0-9]{4}_' | sed 's/_//g' )"
    fi

    if [ -z "${ep_num}" ]
    then
      echo_error "Could not find the episode number - please rerun with \"$0 ep_num=9876\""
    fi

    echo_debug "Attempting to download information for episode \"${ep_num}\""

    if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}'  https://hub.hackerpublicradio.org/cms/shownotes.php?id=${ep_num} --output  "${shownotes_json}" )" != 200 ]
    then
      echo_error "The Episode hpr${ep_num} has not been posted."
    fi

    if [ ! -s "${shownotes_json}" ]
    then
      echo_error "The Episode information for hpr${ep_num} failed to download."
    fi

  fi

  check_variable_is_correct shownotes_json

}

#################################################
# Get the show either the next in the queue
# or from a local queue directory

function get_working_dir() {

  echo_debug "Getting working directory and populating show information. get_working_dir()"

  if [ $# -eq 0 ]
  then
    get_next_show_from_hpr_hub
  fi

  check_variable_is_correct working_dir

  if [ -z "${ep_num}" ]
  then
    get_ep_num_from_local_dir $@
    check_variable_is_correct ep_num
  fi

  echo_debug "Found working directory as \"${working_dir}\""

}

#################################################
# Once the working_dir is known, set the other variables

function set_working_dir_variables() {

  echo_debug "Setting the other working directory and variables. set_working_dir_variables()"

  check_variable_is_correct working_dir

  shownotes_json="${working_dir}/shownotes.json"
  shownotes_html="${working_dir}/shownotes.html"
  shownotes_edited="${working_dir}/shownotes_edited.html"
  episode_tts_wav="${working_dir}/episode_tts.wav"
  episode_tts_flac="${working_dir}/episode_tts.flac"
  episode_summary_flac="${working_dir}/episode_summary.flac"
  episode_intro_flac="${working_dir}/episode_intro.flac"
  episode_body_flac="${working_dir}/episode_body.flac"
  episode_sandwitch_flac="${working_dir}/episode_sandwitch.flac"
  episode_final_flac="${working_dir}/episode_final.flac"
  episode_intro_srt="${working_dir}/episode_intro.srt"
  episode_body_srt="${working_dir}/episode_body.srt"
  episode_outro_srt="${working_dir}/episode_outro.srt"
  episode_srt="${working_dir}/episode.srt"
}

#################################################
# Provides all the metadata we need to process the show.

function get_episode_metadata() {

  echo_debug "Extracting the episode metadata. get_episode_metadata()"
  
  check_variable_is_correct working_dir

  set_working_dir_variables

#   check_variable_is_correct shownotes_html shownotes_edited
  
  if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
  then
    echo_debug "The Episode hpr${ep_num} has already been posted. Skipping get_episode_metadata()"
    return
  fi
  
  check_variable_is_correct shownotes_json

  hostid="$( jq --raw-output '.host.Host_ID' ${shownotes_json} )"
  artist="$( jq --raw-output '.host.Host_Name' ${shownotes_json} )"
  email="$( jq --raw-output '.host.Host_Email' ${shownotes_json} )"
  email_padded="$( echo $email | sed 's/@/.nospam@nospam./g' )"
  title="$( jq --raw-output '.episode.Title' ${shownotes_json} )"
  summary="$( jq --raw-output '.episode.Summary' ${shownotes_json} )"
  series_id="$( jq --raw-output '.episode.Series' ${shownotes_json} )"
  series_name="$( jq --raw-output '.episode.Series_Name' ${shownotes_json} )"
  explicit="$( jq --raw-output '.episode.Explicit' ${shownotes_json} )"
  license="$( jq --raw-output '.episode.Show_License' ${shownotes_json} )"
  ep_date="$( jq --raw-output '.metadata.Episode_Date' ${shownotes_json} )"
  ep_num="$( jq --raw-output '.metadata.Episode_Number' ${shownotes_json} )"
  key="$( jq --raw-output '.metadata.Key' ${shownotes_json} )"
  tags="$( jq --raw-output '.episode.Tags' ${shownotes_json} )"
  host_license="$( jq --raw-output '.host.Host_License' ${shownotes_json} )"
  host_profile="$( jq --raw-output '.host.Host_Profile' ${shownotes_json} )"
  remote_media="$( jq --raw-output '.metadata.url' ${shownotes_json} )"
  shownotes_json_sanatised=$( jq 'del(.episode.Show_Notes, .metadata.Host_IP)' "${shownotes_json}" )

  echo_debug "Extracting shownotes html from json file."

  jq --raw-output '.episode.Show_Notes' "${shownotes_json}" > "${shownotes_html}"
  
  check_variable_is_correct shownotes_html

  ( echo '<!DOCTYPE HTML><html lang="en"><head><meta charset="utf-8" /></head><body>'
  cat "${shownotes_html}"
  echo '</body>
  </html>'
  ) | sponge "${shownotes_html}"

  unsafe_episode_variables=( shownotes_json shownotes_html hostid artist email title summary series_id series_name explicit license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised )

  for episode_variable in "${unsafe_episode_variables[@]}"
  do
    if [[ -z ${!episode_variable} && "${episode_variable}" != "remote_media" ]]
    then   # indirect expansion here
      echo_error "The episode_variable \"${episode_variable}\" is missing.";
    else
      echo_debug "The episode_variable \"${episode_variable}\" is set to \"${!episode_variable}\"";
    fi
  done

  argument_override "$@"

  # Hosts need to exist in the database
  if [ "${hostid}" == '0' ]
  then
    echo_error "The hostid is 0. Create the host and use \"hostid=???\" to override"
  fi

}

#################################################
# Extract_images by brute force

function extract_images_brute_force() {

  echo_debug "Extracting images with grep. extract_images_brute_force()"
  
  if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
  then
    echo_debug "The Episode hpr${ep_num} has already been posted. Skipping extract_images_brute_force()"
    return
  fi

  if [ -s "${shownotes_edited}" ]
  then
    echo_debug "There is already an edited version of the shownotes at \"${shownotes_edited}\", slipping image extraction."
    return
  fi

  if [[ -z "${shownotes_html}" || ! -s "${shownotes_html}" ]]
  then
    echo_error "The shownotes_html file \"${shownotes_html}\" could not be found."
  fi

  ## TODO Temp fix until https://repo.anhonesthost.net/HPR/hpr-tools/issues/3 is available

  sed "s#>#>\n#g" "${shownotes_html}" | sponge "${shownotes_html}"


  # Extract embedded images
  image_count_embedded="1"

  for image in $( grep --color=never --perl-regexp --only-matching 'data:image/[^;]*;base64,\K[a-zA-Z0-9+/=]*' "${shownotes_html}" )
  do
    this_image="${working_dir}/hpr${ep_num}_image_${image_count_embedded}"
    echo -n "$image" | base64 -di > ${this_image}
    this_ext="$( file --mime-type ${this_image} | awk -F '/' '{print $NF}' )"
    mv -v "${this_image}" "${this_image}.${this_ext}"
    this_width="$( mediainfo "${this_image}.${this_ext}" | grep Width | awk -F ': | pixels' '{print $2}' | sed 's/ //g' )"
    if [ "${this_width}" -gt "400" ]
    then
      echo_debug "Generating thumbnail for embedded image \"${this_image}.${this_ext}\"."
      magick "${this_image}.${this_ext}" -resize 400x "${this_image}_tn.${this_ext}"
    fi
    ((image_count_embedded=image_count_embedded+1))
  done

  # Download referenced images
  image_count_external="1"

  for image in $( grep --color=never --perl-regexp --only-matching '<img.*src.*http.*://.*>' "${shownotes_html}" | awk -F 'src=' '{print $2}' | awk -F '"' '{print $2}' )
  do
    this_image="${working_dir}/hpr${ep_num}_image_ext_${image_count_external}"
    wget "${image}" --output-document=${this_image}
    if [ -s "${this_image}" ]
    then
      this_ext="$( file --mime-type ${this_image} | awk -F '/' '{print $NF}' )"
      mv -v "${this_image%.*}" "${this_image}.${this_ext}"
      this_width="$( mediainfo "${this_image}.${this_ext}" | grep Width | awk -F ': | pixels' '{print $2}' | sed 's/ //g' )"
      if [ "${this_width}" -gt "400" ]
      then
        echo_debug "Generating thumbnail for external image \"${this_image}.${this_ext}\"."
        magick "${this_image}.${this_ext}" -resize 400x "${this_image}_tn.${this_ext}"
      fi
      ((image_count_external=image_count_external+1))
    else
      echo_debug "Could not download external image \"${image}\"."
    fi
  done

  cat "${shownotes_html}" | remove-image.pl | sponge "${shownotes_html}"

  if [ "${image_count_embedded}" -gt "1" ]
  then
    image_count_embedded="1"

    touch "${shownotes_html}.embedded_images"

    cat "${shownotes_html}" | while read this_line
    do
      if [ "$( echo "${this_line}" | grep --count "LOCAL_IMAGE_REMOVED" )" -eq "0" ]
      then
        echo "${this_line}" >> "${shownotes_html}.embedded_images"
      else
        this_image="$( find "${working_dir}/" -type f -iname "hpr${ep_num}_image_${image_count_embedded}.*" )"
        if [[ -z "${this_image}" || ! -s "${this_image}" ]]
        then
          echo_error "Unable to find an image for \"${image_count_embedded}\", \"${this_image}\"."
        fi
        this_image="$( basename "${this_image}" )"
        this_image_tn="$( find "${working_dir}/" -type f -iname "${this_image%.*}_tn.*" )"
        if [[ -z "${this_image_tn}" || ! -s "${this_image_tn}" ]]
        then
          echo "${this_line}" | sed "s@LOCAL_IMAGE_REMOVED@${this_image}@g" >> "${shownotes_html}.embedded_images"
        else
          this_image_tn="$( basename "${this_image_tn}" )"
          echo "<a href=\"${this_image}\">" >> "${shownotes_html}.embedded_images"
          echo "${this_line}" | sed "s@LOCAL_IMAGE_REMOVED@${this_image_tn}@g" >> "${shownotes_html}.embedded_images"
          echo "</a>" >> "${shownotes_html}.embedded_images"
        fi
        ((image_count_embedded=image_count_embedded+1))
      fi
    done

    mv -v "${shownotes_html}.embedded_images" "${shownotes_html}"

  else
    echo_debug "No embedded images found. ${image_count_embedded}"
  fi

  if [ "${image_count_external}" -gt "1" ]
  then
    image_count_external="1"

    touch "${shownotes_html}.external_images"

    cat "${shownotes_html}" | remove-image.pl | while read this_line
    do
      if [ "$( echo "${this_line}" | grep --count "REMOTE_IMAGE_REMOVED" )" -eq "0" ]
      then
        echo "${this_line}" >> "${shownotes_html}.external_images"
      else
        this_image="$( find "${working_dir}/" -type f -iname "hpr${ep_num}_image_ext_${image_count_external}.*" )"
        if [[ -z "${this_image}" || ! -s "${this_image}" ]]
        then
          echo_error "Unable to find an image for \"${image_count_external}\", \"${this_image}\"."
        fi
        this_image="$( basename "${this_image}" )"
        this_image_tn="$( find "${working_dir}/" -type f -iname "${this_image%.*}_tn.*" )"
        if [[ -z "${this_image_tn}" || ! -s "${this_image_tn}" ]]
        then
          echo "${this_line}" | sed "s@REMOTE_IMAGE_REMOVED@${this_image}@g" >> "${shownotes_html}.external_images"
        else
          this_image_tn="$( basename "${this_image_tn}" )"
          echo "<a href=\"${this_image}\">" >> "${shownotes_html}.external_images"
          echo "${this_line}" | sed "s@REMOTE_IMAGE_REMOVED@${this_image_tn}@g" >> "${shownotes_html}.external_images"
          echo "</a>" >> "${shownotes_html}.external_images"
        fi
        ((image_count_external=image_count_external+1))
      fi
    done

    mv -v "${shownotes_html}.external_images" "${shownotes_html}"

  else
    echo_debug "No external images found."
  fi

## TODO End Temp fix

}

#################################################
## Media Checks

function media_checks() {

  echo_debug "Running media checks. media_checks()"
  
  check_variable_is_correct working_dir

  if [[ -n "${remote_media}" && "${remote_media}" != "null" ]]
  then
    echo_debug "Fetching remote media from \"${remote_media}\""
    wget --timestamping  --directory-prefix="${working_dir}/" "${remote_media}"
    if [ $? -ne 0 ]
    then
      echo_error "Could not get the remote media"
    fi
  fi

  media=$( find "${working_dir}/" -maxdepth 1 -type f -exec file --mime-type {} \; | grep -Ei ' audio/| video/' | awk -F ': ' '{print $1}' )
  if [ -z "${media}" ]
  then
    find "${working_dir}/" -type f
    echo_error "Can't find any media in \"${working_dir}/\""
  fi

  media_basename="$( basename "${media}" )"
  if [ -z "${media_basename}" ]
  then
    echo_error "Could not create the media_basename \"${media_basename}/\""
  fi

  if [ "$( echo "${media}" | wc --lines )" -ne 1 ]
  then
    echo "Multiple files found. Which one do you want to use ?"
    select this_media in "Skip asset creation" $( echo "${media}" )
    do
      if [ "${this_media}" == "Skip asset creation" ]
      then
        echo_debug "Skip asset creation"
        check_derived_assets
        skip_media_encoding="true"
        return
      fi
      ls -al "${this_media}"
      media="${this_media}"
      break
    done
  fi

  echo_debug "You selected \"${media}\"."
  
  if [[ -z "${media}" || ! -s "${media}" ]]
  then
    echo_error "Could not find the media \"${media}/\""
  fi

  shownotes_srt="${media%.*}.srt"
  if [[ -z "${shownotes_srt}" || ! -s "${shownotes_srt}" ]]
  then
    echo_error "Could not find the subtitles for media \"${media}\" in \"${shownotes_srt}\""
  fi
  
  #TODO fix close duration

  # Find duration
  duration=$( mediainfo --full --Output=JSON "${media}" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )
  if [[ -z "${duration}" || "${duration}" -lt "30" || "${duration}" -gt "30000" ]]
  then
    echo_error "Invalid duration found in \"${media}\"" >&2
  fi
  echo_debug "The Duration is \"${duration}\" seconds from \"${media}\""
  
  
  
  #TODO fix close duration

  # Find number of channels ( 1=mono or 2=stereo)
  supplied_channels=$( mediainfo --full --Output=JSON "${media}" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Channels' )
  if [[ -z "${supplied_channels}" || "${supplied_channels}" -lt "1" || "${supplied_channels}" -gt "2" ]]
  then
    echo_error "Invalid number of audio channels \"${supplied_channels}\" found in \"${media}\"" >&2
  fi
  echo_debug "The number of audio channels is \"${supplied_channels}\" from \"${media}\" ."

  # Gernerate the Spectrum and Waveform image
  ffmpeg -hide_banner -loglevel error -y -i "${media}" -lavfi "showspectrumpic=s=960x540" "${working_dir}/${media_basename%.*}_spectrum.png"
  audio2image.bash "${media}" && mv -v "${media%.*}.png" "${working_dir}/${media_basename%.*}_waveform.png"

  # Getting metadata

  mediainfo "${media}" > "${working_dir}/${media_basename%.*}_mediainfo.txt"

  exiftool "${media}" > "${working_dir}/${media_basename%.*}_exiftool.txt"

  for check_file in spectrum.png waveform.png mediainfo.txt exiftool.txt
  do
    if [ ! -s "${working_dir}/${media_basename%.*}_${check_file}" ]
    then
      echo_error "The ${check_file} file was not generated for the \"${working_dir}/${media_basename%.*}_${check_file}\"" >&2
    fi
  done

  media_ffprobe="$( ffprobe ${media} 2>&1 | grep Audio: | sed 's/^.\s//g' )"
  media_file_mime="$( file --brief --mime ${media} )"
  media_file_mime_type="$( file --brief --mime-type ${media} )"
  
  check_variable_is_correct media_ffprobe media_file_mime media_file_mime_type

}

#################################################
## Generate Initial Report for review by the Janitors

function generate_initial_report() {

  echo_debug "Generating the initial report. generate_initial_report()"
  
#   if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
#   then
#     echo_debug "The Episode hpr${ep_num} has already been posted. Skipping generate_initial_report()"
#     return
#   fi
  

  # TODO list the images.

  echo "<!DOCTYPE HTML>
<html lang=\"en\">
<head>
  <title>Hacker Public Radio ~ The Technology Community Podcast</title>
  <meta charset=\"utf-8\" />
  <style>
  body {
    font-family: sans-serif;
  }
  table {
    border-collapse: collapse;
    text-align: left;
    vertical-align: middle;
  }

  th, td {
    border: 1px solid black;
  }
  
  #waveform {
    position: relative;
    left: 50px;
  }
  </style>
</head>

<body>
<p>Skip to <a href=\"#derived_media\">Derived Media</a></p>

<h3>Field Mapping</h3>
<table>
<thead>
<tr>
  <th>Field</th><th>Value</th>
</tr>
</thead>
<tbody>
<tr><th>hostid</th><td>${hostid}</td></tr>
<tr><th>host_name</th><td>${artist}</td></tr>
<tr><th>title</th><td>${title}</td></tr>
<tr><th>summary</th><td>${summary}</td></tr>
<tr><th>series_id</th><td>${series_id}</td></tr>
<tr><th>series_name</th><td>${series_name}</td></tr>
<tr><th>explicit</th><td>${explicit}</td></tr>
<tr><th>episode_license</th><td>${license}</td></tr>
<tr><th>tags</th><td>${tags}</td></tr>
<tr><th>host_license</th><td>${host_license}</td></tr>
<tr><th>host_profile</th><td>${host_profile}</td></tr>
<tbody>
</table>

<h3>Raw shownotes.json</h3>
<code>
<pre>
${shownotes_json_sanatised}
</pre>
</code>


<h2>Audio</h2>

<h3>mediainfo report</h3>
<code>
<pre>
$( cat "${working_dir}/${media_basename%.*}_mediainfo.txt" )
</pre>
</code>

<h3>exiftool report</h3>
<code>
<pre>
$( cat "${working_dir}/${media_basename%.*}_exiftool.txt" )
</pre>
</code>

<h3>Audio Spectrum</h3>
<p>
<img src=\"${working_dir}/${media_basename%.*}_spectrum.png\" alt=\"Spectrum\" />
</p>

<h3>Audio Waveform</h3>
<p>
<img id=\"waveform\" src=\"${working_dir}/${media_basename%.*}_waveform.png\" alt=\"Waveform\" />
</p>

<pre>
${media_ffprobe}
${media_file_mime}
</pre>

<p>
<audio controls=\"\" preload=\"none\" style=\"width:1040px;\" >
  <source src=\"${media}\" type=\"${media_file_mime_type}\">
</audio>
<br />
<a href=\"${media}\">${media}</a>
</p>

<h3>Transcript</h3>

<hr />
<pre>
$(cat "${shownotes_srt}" )
</pre>
<hr />
</body>
</html>" > "${working_dir}/${media_basename%.*}_media_report.html"

}

#################################################
## Manually edit the shownotes to fix issues

function manual_shownotes_review() {

  echo_debug "Validating the initial report. manual_shownotes_review()"
  
  if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
  then
    echo_debug "The Episode hpr${ep_num} has already been posted. Skipping manual_shownotes_review()"
    return
  fi

  if [[ -z "${shownotes_html}" || ! -s "${shownotes_html}" || ! -s "${working_dir}/${media_basename%.*}_media_report.html" ]]
  then
    echo "shownotes_html: ${shownotes_html}"
    ls -al "${shownotes_html}" "${working_dir}/${media_basename%.*}_media_report.html"
    echo_error "The files needed for to generate the inital report information are not available."
  fi

  if [ -s "${shownotes_edited}" ]
  then
    echo_debug "There is already an edited version of the shownotes at \"${shownotes_edited}\"."
    return
  fi

  cp -v "${shownotes_html}" "${shownotes_edited}"

  if [ ! -s "${shownotes_edited}" ]
  then
    echo_error "The edited shownotes are missing \"${shownotes_edited}\"."
  fi

  kate "${shownotes_edited}" >/dev/null 2>&1 &
  librewolf "${working_dir}/${media_basename%.*}_media_report.html" >/dev/null 2>&1 &
  seamonkey "${shownotes_edited}" >/dev/null 2>&1 &
  # # # # bluefish "${shownotes_edited}" >/dev/null 2>&1 &
  # https://markdowntohtml.com/

  read -p "Does the metadata 'look ok ? (N|y) ? " -n 1 -r
  echo    # (optional) move to a new line
  if [[ ! $REPLY =~ ^[yY]$ ]]
  then
    echo_error "The final review was not approved."
  fi

  # remove extra wrappers that seamonkey adds
  grep --invert-match --perl-regexp  '<!DOCTYPE|<html|html>|head>|<meta|body>' "${shownotes_edited}" | sponge "${shownotes_edited}"

  # Check to see if images have been linked TODO make a loop for found images
  if [ "$( find "${working_dir}" -type f -iname "*_image_*" | wc --lines )" -ne "0" ]
  then
    if [ "$( grep --count "_image_" "${shownotes_edited}" )" -eq "0" ]
    then
      echo_error "The extracted images were not linked in the shownotes \"${shownotes_edited}\"."
    fi
  fi
}

#################################################
# Post show to HPR

function post_show_to_hpr_db() {

  echo_debug "Posting the show to the HPR DB. post_show_to_hpr_db()"

  if [[ -n "${skip_post_show}" && "${skip_post_show}" == "true" ]]
  then
    echo_debug "The Episode hpr${ep_num} has already been posted. Skipping post_show_to_hpr_db()"
    return
  fi

  if [ ! -s "${shownotes_edited}" ]
  then
    echo_error "Failed to find the extracted shownote html file \"${shownotes_edited}\""
  fi
  notes="$( cat "${shownotes_edited}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g'  )"
  host_profile_encoded="$( echo "${host_profile}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g'  )"
  title_encoded="$( echo "${title}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g'  )"
  summary_encoded="$( echo "${summary}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g'  )"

  post_show_json="${working_dir}/post_show.json"

  echo "Sending:"
  echo "  key=${key}
  ep_num=${ep_num}
  ep_date=${ep_date}
  email=${email_padded}
  title=${title_encoded}
  duration=${duration}
  summary=${summary_encoded}
  series_id=${series_id}
  series_name=${series_name}
  explicit=${explicit}
  episode_license=${license}
  tags=${tags}
  hostid=${hostid}
  host_name=${artist}
  host_license=${host_license}
  host_profile=${host_profile_encoded}
  notes=REMOVED"

  echo "{
    \"key\": \"${key}\",
    \"ep_num\": \"${ep_num}\",
    \"ep_date\": \"${ep_date}\",
    \"email\": \"${email_padded}\",
    \"title\": \"${title_encoded}\",
    \"duration\": \"${duration}\",
    \"summary\": \"${summary_encoded}\",
    \"series_id\": \"${series_id}\",
    \"series_name\": \"${series_name}\",
    \"explicit\": \"${explicit}\",
    \"episode_license\": \"${license}\",
    \"tags\": \"${tags}\",
    \"hostid\": \"${hostid}\",
    \"host_name\": \"${artist}\",
    \"host_license\": \"${host_license}\",
    \"host_profile\": \"${host_profile_encoded}\",
    \"notes\": \"${notes}\"
  }" > "${post_show_json}"

  jq '.' "${post_show_json}"

  if [ $? -ne 0 ]
  then
    echo_error "The file \"${post_show_json}\" is not valid json."
  fi

  curl --netrc --show-headers --request POST "https://hub.hackerpublicradio.org/cms/add_show_json.php" --header "Content-Type: application/json" --data-binary "@${post_show_json}"

  if [ "$( curl --silent --netrc --write-out '%{http_code}'  https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output /dev/null )" != 200 ]
  then
    echo_error "The Episode hpr${ep_num} has not been posted"
  fi

}

#################################################
# Get the

function get_variables_from_episode_summary_json() {

  echo_debug "Creating Text to Speech summary. get_variables_from_episode_summary_json()"

  check_variable_is_correct ep_num working_dir
  
  set_working_dir_variables

  if [ -z "${episode_summary_json}" ]
  then
    episode_summary_json="${working_dir}/episode_summary.json"
  fi

  if [ ! -s "${episode_summary_json}" ]
  then
    echo_debug "The \"episode_summary_json\" variable/file is missing, getting a new version."
    if [ "$( curl --silent --netrc --write-out '%{http_code}'  https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output "${episode_summary_json}" )" != 200 ]
    then
      echo_error "The Episode hpr${ep_num} has not been posted"
    fi
  fi
  check_variable_is_correct episode_summary_json

  for episode_summary_key in $( jq --raw-output '. | keys | @tsv' "${episode_summary_json}" )
  do
    episode_summary_value="$( jq --raw-output ".${episode_summary_key}" "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/  / /g' )"
    export "${episode_summary_key}=${episode_summary_value}"
    echo_debug "Setting \"${episode_summary_key}\" to \"${episode_summary_value}\" from \"$( basename ${episode_summary_json} )\""
    check_variable_is_correct ${episode_summary_key}
  done
  
  argument_override "$@"

  duration_iso8601="$( \date -d@${duration} -u +%H:%M:%S )"
  check_variable_is_correct duration_iso8601

}

#################################################
# Generate text to speech summary

function create_tts_summary() {

  echo_debug "Creating Text to Speech summary. create_tts_summary()"
  
  if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]]
  then
    echo_debug "Skipping media creation from create_tts_summary()"
    return
  fi

  check_variable_is_correct working_dir duration synopsis piper_bin piper_voice

  echo_debug "Converting text synopsis \"${synopsis}\" to speech."

  echo "${synopsis}" | "${piper_bin}" --model "${piper_voice}" --output_file "${episode_tts_wav}"
  
  check_variable_is_correct episode_tts_wav

}

#################################################
# Generate Intro

function generate_intro() {

  echo_debug "Generating the intro. generate_intro()"
  
  if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]]
  then
    echo_debug "Skipping media creation from generate_intro()"
    return
  fi
  
  check_variable_is_correct working_dir theme media outro_flac

  # Everything needs to be in the same format for the intro, 1 channel (mono) Sampling rate 44.1 kHz
  ffmpeg -hide_banner -loglevel error -y -i "${episode_tts_wav}" -ar 44100 -ac 1 "${episode_tts_flac}"
  check_variable_is_correct episode_tts_flac

  # A level of silence is added at the beginning of the text to speech
  sox -V2 "${silence}" "${episode_tts_flac}" "${episode_summary_flac}"
  check_variable_is_correct episode_summary_flac

  # The tracks are merged together resulting in the theme playing first, then after a period of silence the text to speech enters
  sox -V2 -m "${episode_summary_flac}" "${theme}" ${episode_intro_flac}
  check_variable_is_correct episode_intro_flac
  
}

#################################################
# Generate parent audio - the sandwitch

function generate_parent_audio() {

  echo_debug "Generating the parent audio - the sandwitch. generate_parent_audio()"
  
  if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]]
  then
    echo_debug "Skipping media creation from generate_parent_audio()"
    return
  fi
  
  check_variable_is_correct episode_intro_flac media outro_flac

  # Everything needs to be in the same format so the text to speech needs to be converted to 2 channel Sampling rate 44.1 kHz
  ffmpeg -hide_banner -loglevel error -y -i "${media}" -ar 44100 -ac 1 "${episode_body_flac}"
  check_variable_is_correct episode_body_flac

  # Combine the components together
  sox -V2 ${episode_intro_flac} "${episode_body_flac}" "${outro_flac}" "${episode_sandwitch_flac}"
  check_variable_is_correct episode_sandwitch_flac
  # Normalise the audio
  ffmpeg -hide_banner -loglevel error -y -i "${episode_sandwitch_flac}" -af loudnorm=I=-16:LRA=11:TP=-1.5 "${episode_final_flac}"
  check_variable_is_correct episode_final_flac

}

#################################################
# Generate derived media

function generate_derived_media() {

  echo_debug "Generating derived audio. generate_derived_media()"
  
  if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]]
  then
    echo_debug "Skipping media creation from generate_derived_media()"
    return
  fi
  
  check_variable_is_correct media working_dir ep_num title artist license 

  if [[ ! -s "${episode_final_flac}" ]]
  then
    ls -al
    echo_error "The final cut is not available."
  fi
  
  check_variable_is_correct comment year

  # https://wiki.multimedia.cx/index.php?title=FFmpeg_Metadata

  for extension in "${audio_formats[@]}"
  do
    echo_debug "Generating \"hpr${ep_num}.${extension}\"."
    ffmpeg -hide_banner -loglevel error -y -i "${episode_final_flac}" \
      -metadata title="${title}" \
      -metadata artist="${artist}" \
      -metadata author="${artist}" \
      -metadata album="Hacker Public Radio" \
      -metadata comment="${comment} The license is ${license}" \
      -metadata year="${year}" \
      -metadata track="${ep_num}" \
      -metadata genre="Podcast" \
      -metadata language="English" \
      -metadata copyright="${license}" \
      "${working_dir}/hpr${ep_num}.${extension}"

    fix_tags -album="Hacker Public Radio" -artist="${artist}" -comment="${comment} The license is ${license}" -genre="Podcast" -title="${title}" -track="${ep_num}" -year="${year}" "${working_dir}/hpr${ep_num}.${extension}"

    if [[ ! -s "${working_dir}/hpr${ep_num}.${extension}" ]]
    then
      echo_error "Failed to generate \"${working_dir}/hpr${ep_num}.${extension}\"."
      ls -al "${working_dir}/hpr${ep_num}.${extension}"
    fi

  done

  #TODO fix close duration
  
#   shownotes_json="${working_dir}/shownotes.json"
#   shownotes_html="${working_dir}/shownotes.html"
#   shownotes_edited="${working_dir}/shownotes_edited.html"
#   episode_tts_wav="${working_dir}/episode_tts.wav"
#   episode_tts_flac="${working_dir}/episode_tts.flac"
#   episode_summary_flac="${working_dir}/episode_summary.flac"
#   episode_intro_flac="${working_dir}/episode_intro.flac"
#   episode_body_flac="${working_dir}/episode_body.flac"
#   episode_sandwitch_flac="${working_dir}/episode_sandwitch.flac"
#   episode_final_flac="${working_dir}/episode_final.flac"
#   episode_intro_srt="${working_dir}/episode_intro.srt"
#   episode_body_srt="${working_dir}/episode_body.srt"
#   episode_outro_srt="${working_dir}/episode_outro.srt"
#   episode_srt="${working_dir}/episode.srt"

  check_variable_is_correct episode_intro_flac outro_flac media duration

  
  intro_duration="$( mediainfo --Output=XML --Full "${episode_intro_flac}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
  outro_duration="$( mediainfo --Output=XML --Full "${outro_flac}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
  source_duration="$( mediainfo --Output=XML --Full "${media}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )"
  expected_duration=$(( ${intro_duration} + ${duration} + ${outro_duration} ))
  check_variable_is_correct intro_duration outro_duration source_duration expected_duration

  echo_debug "Intro(${intro_duration}) + Show(${duration}) + Outro(${outro_duration}) = ${expected_duration}"
  
  for extension in "${audio_formats[@]}"
  do
    this_duration=$( mediainfo --full --Output=XML "${working_dir}/hpr${ep_num}.${extension}" | xmlstarlet sel -T -t -m "_:MediaInfo/_:media/_:track[@type='Audio']/_:Duration[1]" -v "." -n - | awk -F '.' '{print $1}' )
    if [ $( abs_diff "${this_duration}" "${expected_duration}" ) -le "${acceptable_duration_difference}" ]
    then
      echo_debug "The file \"hpr${ep_num}.${extension}\" duration of ${this_duration} is close enough to ${expected_duration}"
    else
      echo_error "The file \"hpr${ep_num}.${extension}\" actual duration of ${this_duration} is not close enough to posted duration of ${expected_duration}. Fix or update the posted duration to ${source_duration}."
    fi
  done

  cp -v "${media}" "${working_dir}/hpr${ep_num}_source.${media##*.}"

  if [[ ! -s "${working_dir}/hpr${ep_num}_source.${media##*.}" ]]
  then
    ls -al "${working_dir}/hpr${ep_num}_source.${media##*.}"
    echo_error "Failed to copy \"${working_dir}/hpr${ep_num}_source.${media##*.}\"."
  fi

}

#################################################
# Generate Subtitles

function generate_show_transcript() {

  echo_debug "Generate show transcript and subtitles. generate_show_transcript()"
  
  if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]]
  then
    echo_debug "Skipping media creation from generate_show_transcript()"
    return
  fi

  # TODO Currently processed elsewhere by hpr-get-and-transcode.bash and uploaded to hpr:upload/ to be synced with media above

  if [[ ! -s "${media}" || ! -s "${media%.*}.srt" || ! -s "${intro_srt}" || ! -s "${outro_srt}" || ! -s ${episode_intro_flac} || ! -s "${episode_body_flac}" ]]
  then
    ls -al "${media}" "${media%.*}.srt" "${intro_srt}" "${outro_srt}" ${episode_intro_flac} "${episode_body_flac}"
    echo_error "The transcriptions files are not available."
  fi

  # Copy in the intro subtitle template and replace each line with the text with the summary

  date="$( jq --raw-output '.date' "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/  / /g' )"
  title="$( jq --raw-output '.title' "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/  / /g' )"
  duration="$( jq --raw-output '.duration' "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/  / /g' )"
  duration_iso8601="$( \date -d@${duration} -u +%H:%M:%S )"
  artist="$( jq --raw-output '.artist' "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/  / /g' )"
  explicit="$( jq --raw-output '.explicit' "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/  / /g' )"
  license="$( jq --raw-output '.license' "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/  / /g' )"
  summary="$( jq --raw-output '.summary' "${episode_summary_json}" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/  / /g' )"

  if [[ -z "${date}" || "${date}" == "null" || -z "${title}" || "${title}" == "null" || -z "${duration_iso8601}" || "${duration_iso8601}" == "null" || -z "${artist}" || "${artist}" == "null" || -z "${explicit}" || "${explicit}" == "null" || -z "${license}" || "${license}" == "null" || -z "${summary}" || "${summary}" == "null"  ]]
  then
    echo_error "Could not retrieve the synopsis for the text to speech."
    ls -al "${episode_summary_json}"
  fi

  REPLACE_LINE_1="This is Hacker Public Radio Episode ${ep_num}, for ${date}"
  REPLACE_LINE_2="Today's show is entitled, \"${title}\""
  REPLACE_LINE_3="The host is ${artist} and the duration is ${duration_iso8601}"
  REPLACE_LINE_4="The flag is ${explicit}, and the license is ${license}"
  REPLACE_LINE_5="The summary is \"${summary}\""

  cp -v ${intro_srt} "${episode_intro_srt}"
  cp -v ${outro_srt} "${episode_outro_srt}"

  sed -e "s~REPLACE_LINE_1~${REPLACE_LINE_1}~g" -e "s~REPLACE_LINE_2~${REPLACE_LINE_2}~g" -e "s~REPLACE_LINE_3~${REPLACE_LINE_3}~g" -e "s~REPLACE_LINE_4~${REPLACE_LINE_4}~g" -e "s~REPLACE_LINE_5~${REPLACE_LINE_5}~g" -i "${episode_intro_srt}"

  if [ "$( grep --count REPLACE_LINE "${episode_intro_srt}" )" -ne "0" ]
  then
    echo_error "The intro subtitles were not correctly generated \"${episode_intro_srt}\"."
  fi


  # Time shift the media subtitles on by the duration of the intro wav file
  # https://trac.ffmpeg.org/wiki/UnderstandingItsoffset

  itsoffset_intro="$( mediainfo --full --Output=JSON ${episode_intro_flac} | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )"

  if [[ -z "${itsoffset_intro}" || "${itsoffset_intro}" == "null" ]]
  then
    echo_error "Could not retrieve the itsoffset_intro to correct the timing of the subtitles."
  fi

  ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_intro}" -i "${media%.*}.srt" -c copy "${episode_body_srt}"

  # Timeshift the outro by the duration of the intro and the supplied media

  itsoffset_body="$( mediainfo --full --Output=JSON "${episode_body_flac}" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )"

  if [[ -z "${itsoffset_body}" || "${itsoffset_body}" == "null" ]]
  then
    echo_error "Could not retrieve the itsoffset_body to correct the timing of the subtitles."
  fi

  itsoffset_body=$((itsoffset_intro + $itsoffset_body))

  ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_body}" -i "${episode_outro_srt}" -c copy "${working_dir}/episode_outro_shifted.srt"

  # Combine the intro, timeshifted media subtitles, and the timeshifted outro subtitles.

  cat "${episode_intro_srt}" "${episode_body_srt}" "${working_dir}/episode_outro_shifted.srt" > "${episode_srt}"

  # Parse the resulting subtitle file fixing the numberic counter
  # https://en.wikipedia.org/wiki/SubRip

  count=1

  cat "${episode_srt}" | while read this_line
  do
    if [ "$( echo "${this_line}" | grep --count --perl-regexp '^[0-9]+$' )" -eq "1" ]
    then
      echo "${count}"
      count=$((count+1))
    else
      echo "${this_line}"
    fi
  done >  "${working_dir}/hpr${ep_num}.srt"

  # extract the txt version

  grep -Pv -- '-->|^$|^[0-9]+$' "${working_dir}/hpr${ep_num}.srt" > "${working_dir}/hpr${ep_num}.txt"

  if [[ ! -s "${working_dir}/hpr${ep_num}.srt" || ! -s "${working_dir}/hpr${ep_num}.txt"  ]]
  then
    echo_error "The transcriptions files were not generated."
    ls -al "${working_dir}/hpr${ep_num}.srt" "${working_dir}/hpr${ep_num}.txt"
  fi

}

#################################################
## Generate Final Report

function generate_final_report() {

  echo_debug "Generating the final report. generate_final_report()"
  
  if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]]
  then
    echo_debug "Skipping media creation from ()"
    return
  fi
  
  check_variable_is_correct working_dir ep_num media_basename shownotes_edited synopsis

  final_report="${working_dir}/hpr${ep_num}_report.html"

  for this_file_extension_to_check in "${episode_formats[@]}"
  do
    if [[ ! -s "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" ]]
    then
      ls -al "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}"
      echo_error "The generated media is missing \"${this_file_extension_to_check}\"."
    fi
  done

  if [[ ! -s "${working_dir}/${media_basename%.*}_media_report.html" ]]
  then
    ls -al "${working_dir}/${media_basename%.*}_media_report.html"
    echo_error "The initial report is not available.\"${working_dir}/${media_basename%.*}_media_report.html\""
  fi

  grep -Pv '</body>|</html>' "${working_dir}/${media_basename%.*}_media_report.html" > "${final_report}"

  echo "<h3 id=\"derived_media\" >Text To Speech</h3>
<p>
$( echo "${synopsis}" )
</p>
<p>
<audio controls=\"\" preload=\"none\" style=\"width:1040px;\" >
  <source src=\"${episode_tts_flac}\" type=\"audio/flac\">
</audio>
<br />
<a href=\"${episode_tts_flac}\">${episode_tts_flac}</a>
</p>
<hr />" >> "${final_report}"


  for this_file_extension_to_check in "${audio_formats[@]}"
  do
    ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" -lavfi "showspectrumpic=s=960x540" "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png"

    audio2image.bash "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" && mv -v "${working_dir}/hpr${ep_num}.png" "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_waveform.png"

    mediainfo "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt"

    exiftool "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt"

    ffprobe "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" 2>&1 | grep Audio: | sed 's/^.\s//g' > "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt"
    file --brief --mime "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" >> "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt"
    this_file_mime_type="$( file --brief --mime-type "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" )"

    for this_file_to_check in spectrum.png waveform.png mediainfo.txt exiftool.txt ffprobe.txt
    do
      if [[ ! -s "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" ]]
      then
        ls -al "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}"
        echo_error "The inital report information is missing \"${this_file_to_check}\"."
      fi
    done

    echo "<h2>Report for derived media file \"hpr${ep_num}.${this_file_extension_to_check}\"</h2>

<h3>mediainfo report</h3>
<code>
<pre>
$( cat "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" )
</pre>
</code>

<h3>exiftool report</h3>
<code>
<pre>
$( cat "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" )
</pre>
</code>

<h3>Audio Spectrum</h3>
<p>
<img src=\"${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png\" alt=\"Spectrum\" />
</p>

<h3>Audio Waveform</h3>
<p>
<img id=\"waveform\" src=\"${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_waveform.png\" alt=\"Waveform\" />
</p>

<pre>
$( cat "${working_dir}/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" )
</pre>

<p>
<audio controls=\"\" preload=\"none\" style=\"width:1040px;\" >
  <source src=\"${working_dir}/hpr${ep_num}.${this_file_extension_to_check}\" type=\"${this_file_mime_type}\">
</audio>
<br />
<a href=\"${working_dir}/hpr${ep_num}.${this_file_extension_to_check}\">${working_dir}/hpr${ep_num}.${this_file_extension_to_check}</a>
</p>" >> "${final_report}"

  done

  echo "


<h3>Rendered shownotes.html</h3>

<hr />

$(cat "${shownotes_edited}" )

<hr />

<h3>Subtitle File</h3>

<hr />
<pre>
$(cat "${working_dir}/hpr${ep_num}.srt" )
</pre>
<hr />

<h3>Transcript File</h3>

<hr />
<pre>
$(cat "${working_dir}/hpr${ep_num}.txt" )
</pre>
<hr />


</body>
</html>" >> "${final_report}"

}

#################################################
## Manually edit the shownotes to fix issues

function manual_final_review() {

  echo_debug "Validating the final report. manual_final_review()"
  
  if [[ -n "${skip_media_encoding}" && "${skip_media_encoding}" == "true" ]]
  then
    echo_debug "Skipping media creation from ()"
    return
  fi

  if [[ -z "${final_report}" || ! -s "${final_report}" ]]
  then
    ls -al "${final_report}"
    echo_error "The files needed for to generate the final report information are not available."
  fi

  librewolf "${final_report}" >/dev/null 2>&1 &
  # # # # bluefish "${shownotes_edited}" >/dev/null 2>&1 &
  # https://markdowntohtml.com/

  read -p "Does the metadata 'look ok ? (N|y) ? " -n 1 -r
  echo    # (optional) move to a new line
  if [[ ! $REPLY =~ ^[yY]$ ]]
  then
    echo "skipping...."
    echo_error "The final review was not approved."
  fi

}

#################################################
# Checks derived assets are correct

function check_derived_assets () {

  check_variable_is_correct working_dir ep_num
  
  #TODO Extend checks to media internals

  for extension in "${audio_formats[@]}"
  do
    echo_debug "Checking \"hpr${ep_num}.${extension}\"."

    if [[ ! -s "${working_dir}/hpr${ep_num}.${extension}" ]]
    then
      ls -al "${working_dir}/hpr${ep_num}.${extension}"
      echo_error "Failed to locate \"${working_dir}/hpr${ep_num}.${extension}\"."
    fi
  done

}

#################################################
# Register the assets with the hpr database

function register_assets() {

  echo_debug "Registering the assets with the hpr database. register_assets()"
  
  check_variable_is_correct working_dir ep_num
  
  check_derived_assets 

  assets_csv="${working_dir}/hpr${ep_num}_assets.csv"
  assets_json="${working_dir}/hpr${ep_num}_assets.json"

  if [[ -s "${assets_csv}" ]]
  then
    echo_debug "Removing \"${assets_csv}\"."
    rm -v "${assets_csv}"
  fi

  echo '"episode_id","filename","extension","size", "sha1sum", "mime_type", "file_type"' | tee "${assets_csv}"

  find "${working_dir}/" -maxdepth 1 -type f \( -iname "hpr${ep_num}.*" -or -iname "hpr${ep_num}_image_*.*" -or -iname "hpr${ep_num}_extra_*.*" \) | \
  while read this_asset_filename
  do
    this_asset_filename="$( basename "${this_asset_filename}" )"
    echo_debug "Registering \"${this_asset_filename}\"."
    this_asset="${working_dir}/${this_asset_filename}"

    if [[ ! -s "${this_asset}" ]]
    then
      echo_error "Failed to register missing file \"${this_asset}\"."
      ls -al "${this_asset}"
    fi
    this_asset_basename=$( basename "${this_asset}" )
    this_asset_extension="${this_asset_basename##*.}"
    this_asset_size="$( ls -al "${this_asset}" | awk '{print $5}' )"
    this_asset_sha1sum="$( sha1sum "${this_asset}" | awk '{print $1}' )"
    this_asset_mime_type=$( file --dereference --brief --mime "${this_asset}" )
    this_asset_file_type=$( file --dereference --brief "${this_asset}" )

    if [ "$( echo ${this_asset_file_type} | wc --chars )" -gt "130" ]
    then
      this_asset_file_type="${this_asset_mime_type}"
    fi

    variables=( ep_num this_asset_basename this_asset_extension this_asset_size this_asset_sha1sum this_asset_mime_type this_asset_file_type working_dir ep_num )

    for variable in "${variables[@]}"
    do
      if [ -z "${!variable}" ]
      then   # indirect expansion here
        echo_error "The variable \"${variable}\" is missing.";
      else
        echo_debug "The variable \"${variable}\" is set to \"${!variable}\"";
      fi
    done

    echo "${ep_num},\"${this_asset_basename}\",\"${this_asset_extension}\",\"${this_asset_size}\",\"${this_asset_sha1sum}\",\"${this_asset_mime_type}\",\"${this_asset_file_type}\"" | tee --append "${assets_csv}"

  done 
  
  check_variable_is_correct assets_csv
  
  echo_debug "${assets_csv}, ${assets_json}"

  if [[ -s "${assets_csv}" && "$( wc -l "${assets_csv}" | awk '{print $1}' )" -gt "1" ]]
  then
    cat "${assets_csv}" | csvtojson | jq '{"assets":[.[]]}' | tee "${assets_json}"
  fi
  
  check_variable_is_correct assets_json

  if [[ ! -s "${assets_json}" && "$( wc -l "${assets_json}" | awk '{print $1}' )" -gt "4" ]]
  then
    echo_error "The asset json file \"${assets_json}\" is missing.";
  fi

  response="$( curl --silent --netrc-file $HOME/.netrc --write-out '%{http_code}' --output /dev/null --request POST https://hub.hackerpublicradio.org/cms/assets.php --data-ascii @"${assets_json}" --header "Content-Type: application/json" )"
  if [[ -z "${response}" || "${response}" != "200" ]]
  then
    echo_error "The assets for episode hpr${ep_num} has not been registered. The response was \"${response}\""
  else
    echo_debug "The assets for episode hpr${ep_num} have been registered. The response was \"${response}\""
  fi
  
}

#################################################
# Copy the files to the origin server

function copy_files_to_origin_server() {

  echo_debug "Copying the files to the origin server. copy_files_to_origin_server()"

  check_variable_is_correct working_dir ep_num

  # TODO get a origin server capable of storing all the files
  for this_asset in hpr${ep_num}.mp3 hpr${ep_num}.ogg hpr${ep_num}.opus hpr${ep_num}.srt hpr${ep_num}.txt $( find "${working_dir}/" -type f -iname "hpr${ep_num}_image_*.*" )
  do
    this_asset="$( basename ${this_asset} )"
    this_file="${working_dir}/${this_asset}"

    echo_debug "Copying \"${this_file}\" to the origin server."

    if [[ ! -s "${this_file}" ]]
    then
      echo_error "Failed to transfer missing file \"${this_file}\"."
      ls -al "${this_file}"
    fi

    if [ "$( ssh rsync.net ls hpr/eps/hpr${ep_num}/ 2>/dev/null | wc --lines )" -eq "0" ]
    then
      echo_debug "Creating \"hpr/eps/hpr${ep_num}/\" on the origin server."
      ssh rsync.net mkdir hpr/eps/hpr${ep_num}/ 2>/dev/null
    fi

    rsync --archive --quiet --partial --progress "${this_file}" rsync.net:hpr/eps/hpr${ep_num}/${this_asset}

    origin_sha1sum="$( echo $( ssh rsync.net "sha1 hpr/eps/hpr${ep_num}/${this_asset}" 2> /dev/null ) | awk '{print $NF}' )"
    this_asset_sha1sum="$( sha1sum "${this_file}" | awk '{print $1}' )"

    if [[ -z "${origin_sha1sum}" || -z "${this_asset_sha1sum}" ]]
    then
      echo_error "Could not determine the local/origin sha1sum for file \"${this_file}\"."
    fi

    if [ "${origin_sha1sum}" != "${this_asset_sha1sum}" ]
    then
      echo_error "The local sha1sum \"${origin_sha1sum}\" and origin \"${this_asset_sha1sum}\" are mismatched for file \"${this_file}\"."
    fi

  done
}

#################################################
# Copy the images to the hpr server

function copy_files_to_hpr_server() {

  echo_debug "Copying the files to the hpr server. copy_files_to_hpr_server()"

  check_variable_is_correct working_dir ep_num

  # Get the file list from the HPR db
  if [ -z "${files_json}" ]
  then
    files_json="${working_dir}/hpr${ep_num}_files.json"
  fi

  if [ -s "${files_json}" ]
  then
    echo_debug "Removing stale \"files_json\" file \"${files_json}\"."
    rm -v "${files_json}"
  fi
  
  echo_debug "Getting the \"files_json\" file from the HPR site."
  if [ "$( curl --silent --netrc --write-out '%{http_code}'  https://hub.hackerpublicradio.org/cms/assets.php?id=${ep_num} --output "${files_json}" )" != 200 ]
  then
    echo_error "The Episode hpr${ep_num} has not been posted"
  fi
  check_variable_is_correct files_json

  ssh hpr -t "mkdir -v hackerpublicradio.org/public_html/eps/hpr${ep_num}"
  
  for this_file in $( jq --raw-output ".hpr${ep_num} | keys | @tsv" "${files_json}" )
  do
    skip_file="$( echo "$this_file" | grep --count --perl-regexp "hpr${ep_num}.flac|hpr${ep_num}.mp3|hpr${ep_num}.ogg|hpr${ep_num}.opus|hpr${ep_num}.spx|hpr${ep_num}.srt|hpr${ep_num}.txt|hpr${ep_num}.wav" )"
    if [ "${skip_file}" -eq "1" ]
    then
      continue
    fi
    scp "${working_dir}/$this_file" hpr:hackerpublicradio.org/public_html/eps/hpr${ep_num}/
  done

}

#################################################
# Send the derived files to the server borg to be sent to borg

function copy_derived_files_to_borg() {

  echo_debug "Sending the derived files to the admin server borg. copy_derived_files_to_borg()"

  check_variable_is_correct working_dir ep_num shownotes_html

  for extension in flac mp3 ogg opus wav
  do
    if [[ ! -s "${working_dir}/hpr${ep_num}.${extension}" ]]
    then
      echo_error "The derived files to the admin server borg is missing \"hpr${ep_num}.${extension}\"."
      ls -al "${working_dir}/hpr${ep_num}.${extension}"
    fi
  done

  echo_debug "rsync -ave ssh --partial --progress \"${working_dir}/hpr${ep_num}.{flac,mp3,ogg,opus,wav}\" hpradmin@borg:/data/IA/uploads/"
  rsync -ave ssh --partial --progress "${working_dir}/hpr${ep_num}".{flac,mp3,ogg,opus,wav} hpradmin@borg:/data/IA/uploads/
  rsync_error="${?}"
  if [ "${rsync_error}" -ne "0" ]
  then
    echo_error "rsync to \"hpradmin@borg:/data/IA/uploads/\" failed with error ${rsync_error}"
  fi

  rsync -ave ssh --partial --progress "${working_dir}/hpr${ep_num}".{txt,srt} hpradmin@borg:/data/IA/uploads/hpr${ep_num}/
  rsync_error="${?}"
  if [ "${rsync_error}" -ne "0" ]
  then
    echo_error "rsync to \"hpradmin@borg:/data/IA/uploads/hpr${ep_num}/\" failed with error ${rsync_error}"
  fi

  rsync -ave ssh --partial --progress "${shownotes_edited}" hpradmin@borg:/data/IA/uploads/hpr${ep_num}/shownotes.html
  rsync_error="${?}"
  if [ "${rsync_error}" -ne "0" ]
  then
    echo_error "rsync to \"hpradmin@borg:/data/IA/uploads/hpr${ep_num}/shownotes.html\" failed with error ${rsync_error}"
  fi

  # Get the current status
  if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${processing_dir}/status.csv" )" != 200 ]
  then
    echo_error "Could not get a list of the queue status from \"https://hub.hackerpublicradio.org/cms/status.php\""
  fi

  # Check the current status is correct SHOW_POSTED
  response=$( grep --perl-regexp ",${ep_num},.*SHOW_POSTED," "${processing_dir}/status.csv" | head -1 | sed 's/,/ /g' )
  if [ -z "${response}" ]
  then
    grep --perl-regexp ",${ep_num},.*SHOW_POSTED," "${processing_dir}/status.csv"
    echo_debug "The show \"${ep_num}\" hasn't the correct status of \"SHOW_POSTED\" in the database."
  fi

  # Try and change the online db status to MEDIA_TRANSCODED
  if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' "https://hub.hackerpublicradio.org/cms/status.php?ep_num=${ep_num}&status=MEDIA_TRANSCODED" )" != 200 ]
  then
    echo_debug "Could not change the status of \"${ep_num}\" to \"MEDIA_TRANSCODED\""
  fi

  # Get the current status
  if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${processing_dir}/status.csv" )" != 200 ]
  then
    echo_error "Could not get a list of the queue status from \"https://hub.hackerpublicradio.org/cms/status.php\""
  fi

  # Check the current status is correct MEDIA_TRANSCODED
  response=$( grep --perl-regexp ",${ep_num},.*MEDIA_TRANSCODED," "${processing_dir}/status.csv" | head -1 | sed 's/,/ /g' )
  if [ -z "${response}" ]
  then
    grep --perl-regexp ",${ep_num},.*MEDIA_TRANSCODED," "${processing_dir}/status.csv"
    echo_debug "The show \"${ep_num}\" hasn't the correct status of \"MEDIA_TRANSCODED\" in the database."
  fi

  echo_debug "The show \"${ep_num}\" has the correct status of \"MEDIA_TRANSCODED\" in the database."

  #TODO images

  # <img alt="Picture 1 shows the broken dog    walking accessory." border="0" height="300" src="https://archive.org/download/hpr4283/hpr4283/hpr4283_1_tn.jpeg" width="400" />

}

#################################################
# Wait for the Internet Archive to finish processing

function wait_for_the_internet_archive_to_process() {

  echo_debug "Waiting for the Internet Archive to finish processing. wait_for_the_internet_archive_to_process()"

  check_variable_is_correct ep_num

  while [ "$( ia tasks "hpr${ep_num}" | jq --slurp '[.[] | if .category == "catalog" then .status else empty end] | length' )" -ne "0" ]
  do
    echo_debug "Waiting for the Internet Archive to finish processing \"hpr${ep_num}\"."
    date
    sleep 1m
  done

}

#################################################
# Send the derived files to the server borg to be sent to the Internet Archive

function create_item_on_the_internet_archive() {

  echo_debug "Sending the derived files to Internet Archive. create_item_on_the_internet_archive()"

  check_variable_is_correct working_dir ep_num artist date synopsis title license_url

  if [ ! -s "${working_dir}/hpr${ep_num}.txt" ]
  then
    echo_error "The Internet Archive \"${working_dir}/hpr${ep_num}.txt\" is missing."
  fi

  ia list "hpr${ep_num}" > /dev/null
  if [ $? -ne 0 ]
  then
    echo_debug "The episode has not been posted to the Internet Archive"
    ia upload hpr${ep_num} "${working_dir}/hpr${ep_num}.txt" --metadata=mediatype:audio --metadata="contributor:HackerPublicRadio" --metadata="creator:${artist}" --metadata="date:${date}" --metadata="description:${synopsis}." --metadata="language:eng" --metadata="licenseurl:${license_url}" --metadata="title:${title}." --metadata=reviews-allowed:none --header x-archive-keep-old-version:0 --retries=5 --no-derive --no-backup
  else
    echo_debug "The episode has been posted to the Internet Archive"
  fi

  wait_for_the_internet_archive_to_process

}

#################################################
# Send the derived files to the server borg to be sent to the Internet Archive

function upload_files_to_the_internet_archive() {

  echo_debug "Sending the derived files to Internet Archive. upload_files_to_the_internet_archive()"

  check_variable_is_correct working_dir ep_num

  # Get the file list from the IA
  if [ -z "${files_xml}" ]
  then
    files_xml="${working_dir}/hpr${ep_num}_files.xml"
  fi
  if [ -s "${files_xml}" ]
  then
    echo_debug "Removing stale \"files_xml\" file \"${files_xml}\"."
    rm -v "${files_xml}"
  fi
  
  echo_debug "Getting the \"files_xml\" file from the IA."
  if [ "$( curl --silent --netrc --location --write-out '%{http_code}' https://archive.org/download/hpr${ep_num}/hpr${ep_num}_files.xml --output "${files_xml}" )" != 200 ]
  then
    echo_error "Could not find file list \"https://archive.org/download/hpr${ep_num}/hpr${ep_num}_files.xml\""
  fi
  check_variable_is_correct files_xml

  # Get the file list from the HPR db
  if [ -z "${files_json}" ]
  then
    files_json="${working_dir}/hpr${ep_num}_files.json"
  fi

  if [ -s "${files_json}" ]
  then
    echo_debug "Removing stale \"files_json\" file \"${files_json}\"."
    rm -v "${files_json}"
  fi
  
  echo_debug "Getting the \"files_json\" file from the HPR site."
  if [ "$( curl --silent --netrc --write-out '%{http_code}'  https://hub.hackerpublicradio.org/cms/assets.php?id=${ep_num} --output "${files_json}" )" != 200 ]
  then
    echo_error "The Episode hpr${ep_num} has not been posted"
  fi
  check_variable_is_correct files_json

  for this_file in $( jq --raw-output ".hpr${ep_num} | keys | @tsv" "${files_json}" )
  do
    db_file_info="$( jq --raw-output ".hpr${ep_num}.[] | select(.filename==\"${this_file}\") | [ .filename, .size, .sha1sum ] | @tsv" "${files_json}" )"
    ia_file_info="$( xmlstarlet sel -T -t -m "/files/file[@name=\"${this_file}\"]" -v "concat(@name, '→', size, '→', sha1)" -n "${files_xml}" | sed 's/→/\t/g' )"
    if [ -z "${db_file_info}" ]
    then
      echo_error "The information for \"${this_file}\" could not be found in \"${files_json}\""
    fi
    if [ ! -z "${ia_file_info}" ]
    then
      if [ "${db_file_info}" == "${ia_file_info}" ]
      then
        echo_debug "The file \"${this_file}\" on HPR and on the IA matches."
        continue
      else
        echo_debug "The file \"${this_file}\" on HPR and on the IA do not match."
      fi
    fi

    echo_debug "Transferring \"${this_file}\" to the IA"

    this_file_path="${working_dir}/${this_file}"

    if [ ! -s "${this_file_path}" ]
    then
      echo_error "The file \"${this_file}\" could not be found in the working directory \"${working_dir}\"."
    fi

    this_file_path_sha1="$( sha1sum $this_file_path | awk '{print $1}' )"
    db_file_info_sha1="$( echo ${db_file_info} | awk '{print $NF}' )"

    if [ "${this_file_path_sha1}" != "${db_file_info_sha1}" ]
    then
      echo_error "The sha1 sum of the local file \"${this_file}\" does not match the HPR database."
      continue
    else
      ia upload hpr${ep_num} "${this_file_path}" --header "x-archive-keep-old-version:0" --retries=5 --no-derive --no-backup
    fi

  done
  
  wait_for_the_internet_archive_to_process

}

function hide_me() {

echo "hidden"

#
# TODO Add support for reserve queue - process validate and move to reserve dir
# TODO Add support to reprocess processed shows - when given onlay new media reprocess it, update duration on hub, generate and download shownotes.{html,json} from db
# TODO Add support for community news - reusing ^^^
# TODO Add support for stereo for some episodes that request it
# TODO Include links in extract_images_brute_force
# TODO take screenshots of the rendered episode on the hpr website
# TODO audio_channels default to mono - stereo as an option
# TODO Add chapter support
# TODO incorporate direct upload to the IA
# TODO copy the files to the backup disk
#TODO add logic to check where we are and what we need to do.
# All functions should skip if their work is done. Except when mode=refresh
# 
# 
# 
# If there is a working dir then 
# - is the show posted etc, shownotes_edited.html
# - do the assets match hpr${ep_num}_assets.json
# - 

# # # # Get supplied working dir and ep_num if provided
# # # if [ $# -gt 0 ]
# # # then
# # # #   declare -A hash
# # #   for argument
# # #   do
# # #     if [[ $argument =~ ^[^=]+=.*$ ]]
# # #     then
# # #       key="${argument%=*}"
# # #       value="${argument#*=}"
# # #       eval "${key}=${value}"
# # #       echo_debug "Using supplied \"${key}\" of \"${value}\""
# # #       check_variable_is_correct ${key}
# # #     fi
# # #   done
# # # fi

}

#################################################
# Main exceution starts here
#
# This tool will process the HPR shows allowing the janitor to review the media and fix shownotes.

argument_override "$@"

program_checks              # We know that all the programs and variables are set

if [[ -z "${working_dir_bypass}" || "${working_dir_bypass}" != "true" ]]
then
  get_working_dir $@          # We have a working directory and a valid json file
else
  echo_debug "Skipping get_working_dir()"
fi

if [ "$( curl --silent --netrc --write-out '%{http_code}'  https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output /dev/null )" == 200 ]
then
  skip_post_show="true"
  echo_debug "The Episode hpr${ep_num} has already been posted."
fi

get_episode_metadata $@     # We have all the metadata we need to process the show.

extract_images_brute_force  # We have extracted the images by brute force

media_checks                #

generate_initial_report     # Generate Initial Report for review by the Janitors

manual_shownotes_review     # Janitors review audio and shownote. Skips if done.

post_show_to_hpr_db         # Posts the episode to HPR. Skips if it is already posted.

# #######################################################################################
# Using DB info from here

get_variables_from_episode_summary_json  "$@"         # Get the episode from HPR.

create_tts_summary          # Generate text to speech summary

generate_intro              # Generate Intro from the intro theme with overlay of a lead in silence then the tts summary

generate_parent_audio       # Combines the intro, the episode, and the outro to a final cut.

generate_derived_media      # Generate the flac wav mp3 ogg opus files

generate_show_transcript

generate_final_report

manual_final_review

register_assets

copy_files_to_origin_server

check_variable_is_correct working_dir ep_num shownotes_edited

copy_files_to_hpr_server

copy_derived_files_to_borg

create_item_on_the_internet_archive

upload_files_to_the_internet_archive

hpr-check-ccdn-links ${ep_num}

echo_debug "The End"

xdg-open https://archive.org/details/hpr${ep_num}

xdg-open https://hackerpublicradio.org/eps/hpr${ep_num}/index.html

exit 0

#   if ${force_overwrite}
#   then
#     echo_debug "The setting \"force_overwrite\" is set to true, so files will be overwritten."
#   else
#     echo_debug "The setting \"force_overwrite\" is set to false, so when files exist the program will skip files if they exist."
#   fi

# curl --silent --netrc --write-out '%{http_code}'  https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output  "${episode_summary_json}"
# curl --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/shownotes.php?id=${ep_num} --output "${working_dir}/shownotes.json"