1278 lines
46 KiB
Bash
Executable File
1278 lines
46 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/
|
|
#============================================================
|
|
|
|
# git clone https://github.com/davmo/fix_tags.git
|
|
# git clone https://repo.anhonesthost.net/HPR/hpr-tools.git
|
|
|
|
#################################################
|
|
# Variables
|
|
|
|
debug="1"
|
|
force_overwrite=true # Sets the behavour when files exist or not
|
|
processing_dir="$HOME/tmp/hpr/processing" # The directory where the files will be copied to for processing
|
|
theme="${processing_dir}/theme.flac" # The hpr theme
|
|
silence="${processing_dir}/silence.flac" # A segment of silence to offset the tts in the intro
|
|
outro_flac="${processing_dir}/outro.flac" # The outro edited media
|
|
outro_srt="${processing_dir}/outro.srt" # The outro subtitle file
|
|
intro_srt="${processing_dir}/intro.srt" # The intro subtitle template file
|
|
|
|
piper_bin="/opt/bin/piper/piper/piper" # The location of the text to speech engine
|
|
piper_voice="/opt/bin/piper/piper/piper-voices/en/en_US/lessac/medium/en_US-lessac-medium.onnx"
|
|
|
|
#################################################
|
|
# Display Error message, display help and exit
|
|
|
|
function echo_error() {
|
|
echo -e "ERROR: $@" #1>&2
|
|
display_help_and_exit
|
|
exit 1
|
|
}
|
|
|
|
#################################################
|
|
# Display Information message
|
|
|
|
function echo_debug() {
|
|
if [ "${debug}" != "0" ]
|
|
then
|
|
echo -e "INFO: $@" #1>&2
|
|
fi
|
|
}
|
|
|
|
#################################################
|
|
# Display Help
|
|
|
|
function display_help_and_exit() {
|
|
echo_debug "For more information see https://repo.anhonesthost.net/HPR/hpr_documentation"
|
|
exit 1
|
|
}
|
|
|
|
#################################################
|
|
# Program Checks
|
|
|
|
function program_checks() {
|
|
|
|
if [ ! -d "${processing_dir}" ]
|
|
then
|
|
echo_error "The \"${processing_dir}\" is required but is not defined."
|
|
fi
|
|
|
|
if [[ ! -s "${theme}" || ! -s "${silence}" || ! -s "${outro_flac}" || ! -s "${outro_srt}" || ! -s "${intro_srt}" ]]
|
|
then
|
|
echo_error "The files for the theme are not available."
|
|
ls -al "${theme}" "${silence}" "${outro_flac}" "${outro_srt}" "${intro_srt}"
|
|
fi
|
|
|
|
function is_installed() {
|
|
for this_program in "$@"
|
|
do
|
|
if ! command -v ${this_program} 2>&1 >/dev/null
|
|
then
|
|
echo_error "The application \"${this_program}\" is required but is not installed."
|
|
fi
|
|
done
|
|
}
|
|
|
|
is_installed audio2image.bash awk base64 cat curl date detox eval extract_images ffprobe file find grep grep head jq kate magick mediainfo mv realpath rsync seamonkey sed sed sort sponge ssh touch touch wget
|
|
|
|
for arg in $*
|
|
do
|
|
if [ "$( echo "${arg}" | grep --count --ignore-case --perl-regexp -- '-h|--help' )" -ne "0" ]
|
|
then
|
|
echo_debug "Process the next SHOW_SUBMITTED show from the queue."
|
|
echo_debug "If a directory is provided then the shownotes.json will be used."
|
|
fi
|
|
done
|
|
|
|
if ${force_overwrite}
|
|
then
|
|
echo_debug "The setting \"force_overwrite\" is set to true, so files will be overwritten."
|
|
else
|
|
echo_debug "The setting \"force_overwrite\" is set to false, so when files exist the program will skip files if they exist."
|
|
fi
|
|
|
|
}
|
|
|
|
#################################################
|
|
# Get the next show in the queue
|
|
|
|
function get_working_dir_from_hpr_hub() {
|
|
|
|
echo_debug "Processing the next HPR Show in the queue"
|
|
|
|
if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${processing_dir}/status.csv" )" != 200 ]
|
|
then
|
|
echo_error "Could not get a list of the queue status from \"https://hub.hackerpublicradio.org/cms/status.php\""
|
|
fi
|
|
|
|
if [ ! -s "${processing_dir}/status.csv" ]
|
|
then
|
|
echo_error "Failed to retrieve \"${processing_dir}/status.csv\" from server."
|
|
fi
|
|
|
|
response=$( cat "${processing_dir}/status.csv" | grep ',SHOW_SUBMITTED,' | head -1 | sed 's/,/ /g' )
|
|
if [ -z "${response}" ]
|
|
then
|
|
echo_debug "Getting a list of all the reservations."
|
|
curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | sort -n
|
|
echo_error "There appear to be no more shows with the status \"SHOW_SUBMITTED\"."
|
|
fi
|
|
|
|
timestamp_epoc="$( echo ${response} | awk '{print $1}' )"
|
|
ep_num="$( echo ${response} | awk '{print $2}' )"
|
|
ep_date="$( echo ${response} | awk '{print $3}' )"
|
|
key="$( echo ${response} | awk '{print $4}' )"
|
|
email="$( echo ${response} | awk '{print $6}' )"
|
|
email_unpadded="$( echo $email | sed 's/.nospam@nospam./@/g' )"
|
|
|
|
hpr_upload_dir="/home/hpr/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
|
|
source_dir="hpr:${hpr_upload_dir}"
|
|
dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
|
|
working_dir="${processing_dir}/${dest_dir}"
|
|
|
|
echo_debug "Downloading hpr${ep_num} from ${email_unpadded}"
|
|
echo_debug ""
|
|
echo_debug "rsync -ave ssh --partial --progress ${source_dir}/ ${working_dir}/"
|
|
ssh hpr -t "detox -v ${hpr_upload_dir}/"
|
|
rsync -ave ssh --partial --progress ${source_dir}/ ${working_dir}/
|
|
|
|
|
|
if [ ! -s "${working_dir}/shownotes.json" ]
|
|
then
|
|
echo_error "The working dir is missing the shownotes file \"${working_dir}/shownotes.json\""
|
|
fi
|
|
|
|
if [ "$( file --brief --mime-type "${working_dir}/shownotes.json" | grep --count "application/json" )" -eq 0 ]
|
|
then
|
|
echo_error "The \"${working_dir}/shownotes.json\" is not a \"application/json\" file"
|
|
fi
|
|
|
|
}
|
|
|
|
#################################################
|
|
# Get the show information from a local directory
|
|
|
|
function get_working_dir_from_local_dir() {
|
|
|
|
echo_debug "Processing a local directory"
|
|
|
|
# Get supplied working dir and ep_num if provided
|
|
for argument in "$@"
|
|
do
|
|
|
|
|
|
if [ -d "${argument}" ]
|
|
then
|
|
working_dir="$( realpath ${argument} )"
|
|
fi
|
|
|
|
if [ "$( echo "${argument}" | grep --count "ep_num=" )" -eq "1" ]
|
|
then
|
|
ep_num="$( echo "${argument}" | sed 's/^.*ep_num=//g' | awk '{print $1}' )"
|
|
fi
|
|
|
|
done
|
|
|
|
if [[ ! -d "${working_dir}" || -z "${working_dir}" ]]
|
|
then
|
|
echo_error "The working dir is missing. Please supply a working directory."
|
|
fi
|
|
|
|
if [ ! -s "${working_dir}/shownotes.json" ]
|
|
then
|
|
|
|
echo_debug "Could not find a \"shownotes.json\" in the working directory \"${working_dir}/\""
|
|
|
|
if [ -z "${ep_num}" ]
|
|
then
|
|
echo_debug "Attempting to get episode number from the \"${working_dir}\""
|
|
ep_num="$( echo "${working_dir}" | grep --color=never --perl-regexp --only-matching '_[0-9]{4}_' | sed 's/_//g' )"
|
|
fi
|
|
|
|
if [ -z "${ep_num}" ]
|
|
then
|
|
echo_error "Could not find the episode number - please rerun with \"$0 ep_num=9876\""
|
|
fi
|
|
|
|
echo_debug "Attempting to download information for episode \"${ep_num}\""
|
|
|
|
if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/shownotes.php?id=${ep_num} --output "${working_dir}/shownotes.json" )" != 200 ]
|
|
then
|
|
echo_error "The Episode hpr${ep_num} has not been posted."
|
|
fi
|
|
|
|
if [ ! -s "${working_dir}/shownotes.json" ]
|
|
then
|
|
echo_error "The Episode information for hpr${ep_num} failed to download."
|
|
fi
|
|
|
|
fi
|
|
|
|
if [[ -s "${working_dir}/shownotes.json" && "$( file --brief --mime-type "${working_dir}/shownotes.json" | grep --count "application/json" )" -eq 0 ]]
|
|
then
|
|
echo_error "\"${working_dir}/shownotes.json\" is not a \"application/json\" file"
|
|
fi
|
|
|
|
}
|
|
|
|
#################################################
|
|
# Get the show either the next in the queue
|
|
# or from a local queue directory
|
|
|
|
function get_working_dir() {
|
|
|
|
echo_debug "Getting working directory and populating show information. $#"
|
|
|
|
if [ $# -eq 0 ]
|
|
then
|
|
get_working_dir_from_hpr_hub
|
|
else
|
|
get_working_dir_from_local_dir $@
|
|
fi
|
|
|
|
if [ ! -s "${working_dir}/shownotes.json" ]
|
|
then
|
|
echo_error "The working dir \"${working_dir}\" could not be found."
|
|
fi
|
|
|
|
echo_debug "Found working directory as \"${working_dir}\""
|
|
|
|
if [ ! -d "${working_dir}/processing/" ]
|
|
then
|
|
mkdir -v "${working_dir}/processing/"
|
|
if [ ! -d "${working_dir}/processing/" ]
|
|
then
|
|
echo_error "Could not create the processing directory \"${working_dir}/processing/\"."
|
|
fi
|
|
fi
|
|
|
|
}
|
|
|
|
#################################################
|
|
# Provides all the metadata we need to process the show.
|
|
|
|
function get_episode_metadata() {
|
|
|
|
echo_debug "Extracting the episode metadata"
|
|
|
|
if [[ -s "${working_dir}/shownotes.json" && "$( file --brief --mime-type "${working_dir}/shownotes.json" | grep --count "application/json" )" -eq 0 ]]
|
|
then
|
|
echo_error "\"${working_dir}/shownotes.json\" is not a \"application/json\" file"
|
|
fi
|
|
|
|
shownotes_json="${working_dir}/shownotes.json"
|
|
shownotes_html="${working_dir}/shownotes.html"
|
|
|
|
hostid="$( jq --raw-output '.host.Host_ID' ${shownotes_json} )"
|
|
host_name="$( jq --raw-output '.host.Host_Name' ${shownotes_json} )"
|
|
email="$( jq --raw-output '.host.Host_Email' ${shownotes_json} )"
|
|
email_padded="$( echo $email | sed 's/@/.nospam@nospam./g' )"
|
|
title="$( jq --raw-output '.episode.Title' ${shownotes_json} )"
|
|
summary="$( jq --raw-output '.episode.Summary' ${shownotes_json} )"
|
|
series_id="$( jq --raw-output '.episode.Series' ${shownotes_json} )"
|
|
series_name="$( jq --raw-output '.episode.Series_Name' ${shownotes_json} )"
|
|
explicit="$( jq --raw-output '.episode.Explicit' ${shownotes_json} )"
|
|
episode_license="$( jq --raw-output '.episode.Show_License' ${shownotes_json} )"
|
|
ep_date="$( jq --raw-output '.metadata.Episode_Date' ${shownotes_json} )"
|
|
ep_num="$( jq --raw-output '.metadata.Episode_Number' ${shownotes_json} )"
|
|
key="$( jq --raw-output '.metadata.Key' ${shownotes_json} )"
|
|
tags="$( jq --raw-output '.episode.Tags' ${shownotes_json} )"
|
|
host_license="$( jq --raw-output '.host.Host_License' ${shownotes_json} )"
|
|
host_profile="$( jq --raw-output '.host.Host_Profile' ${shownotes_json} )"
|
|
remote_media="$( jq --raw-output '.metadata.url' ${shownotes_json} )"
|
|
shownotes_json_sanatised=$( jq 'del(.episode.Show_Notes, .metadata.Host_IP)' "${shownotes_json}" )
|
|
|
|
echo_debug "Extracting shownotes html from json file."
|
|
|
|
jq --raw-output '.episode.Show_Notes' "${shownotes_json}" > "${shownotes_html}"
|
|
|
|
if [ ! -s "${shownotes_html}" ]
|
|
then
|
|
echo_error "Failed to extract the shownote html file \"${shownotes_html}\""
|
|
fi
|
|
|
|
variables=( shownotes_json shownotes_html hostid host_name email title summary series_id series_name explicit episode_license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised )
|
|
|
|
for variable in "${variables[@]}"
|
|
do
|
|
if [[ -z ${!variable} && "${variable}" != "remote_media" ]]
|
|
then # indirect expansion here
|
|
echo_error "The variable \"${variable}\" is missing.";
|
|
else
|
|
echo_debug "The variable \"${variable}\" is set to \"${!variable}\"";
|
|
fi
|
|
done
|
|
|
|
# Argument Override
|
|
if [ $# -gt 0 ]
|
|
then
|
|
declare -A hash
|
|
for argument
|
|
do
|
|
if [[ $argument =~ ^[^=]+=.*$ ]]
|
|
then
|
|
this_key="${argument%=*}"
|
|
this_value="${argument#*=}"
|
|
this_value="$( echo "${this_value}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
|
|
eval "${this_key}=${this_value}"
|
|
echo_debug "Replacing \"${this_key}\" with \"${this_value}\"."
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# Hosts need to exist in the database
|
|
if [ "${hostid}" == '0' ]
|
|
then
|
|
echo_error "The hostid is 0. Create the host and use \"hostid=???\" to override"
|
|
fi
|
|
|
|
}
|
|
|
|
#################################################
|
|
# Extract_images by brute force
|
|
|
|
function extract_images_brute_force() {
|
|
|
|
echo_debug "Extracting images with grep."
|
|
|
|
if [[ -z "${shownotes_html}" || ! -s "${shownotes_html}" ]]
|
|
then
|
|
echo_error "The shownotes_html file \"${shownotes_html}\" could not be found."
|
|
fi
|
|
|
|
## TODO Temp fix until https://repo.anhonesthost.net/HPR/hpr-tools/issues/3 is available
|
|
|
|
sed "s#>#>\n#g" "${shownotes_html}" | sponge "${shownotes_html}"
|
|
|
|
image_count="1"
|
|
|
|
# Extract embedded images
|
|
|
|
for image in $( grep --color=never --perl-regexp --only-matching 'data:image/[^;]*;base64,\K[a-zA-Z0-9+/=]*' "${shownotes_html}" )
|
|
do
|
|
this_image="${working_dir}/hpr${ep_num}_image_${image_count}"
|
|
echo -n "$image" | base64 -di > ${this_image}
|
|
this_ext="$( file --mime-type ${this_image} | awk -F '/' '{print $NF}' )"
|
|
mv -v "${this_image}" "${this_image}.${this_ext}"
|
|
this_width="$( mediainfo "${this_image}.${this_ext}" | grep Width | awk -F ': | pixels' '{print $2}' | sed 's/ //g' )"
|
|
if [ "${this_width}" -gt "400" ]
|
|
then
|
|
magick "${this_image}.${this_ext}" -resize 400x "${this_image}_tn.${this_ext}"
|
|
fi
|
|
((image_count=image_count+1))
|
|
done
|
|
|
|
# Download referenced images
|
|
|
|
for image in $( grep --color=never --perl-regexp --only-matching '<img.*src.*http.*>' "${shownotes_html}" | awk -F 'src=' '{print $2}' | awk -F '"' '{print $2}' )
|
|
do
|
|
this_image="${working_dir}/hpr${ep_num}_image_${image_count}"
|
|
wget "${image}" --output-document=${this_image}
|
|
this_ext="$( file --mime-type ${this_image} | awk -F '/' '{print $NF}' )"
|
|
if [ ! -e "${this_image}.${this_ext}" ]
|
|
then
|
|
mv -v "${this_image%.*}" "${this_image}.${this_ext}"
|
|
fi
|
|
this_width="$( mediainfo "${this_image}.${this_ext}" | grep Width | awk -F ': | pixels' '{print $2}' | sed 's/ //g' )"
|
|
if [ "${this_width}" -gt "400" ]
|
|
then
|
|
magick "${this_image}.${this_ext}" -resize 400x "${this_image}_tn.${this_ext}"
|
|
fi
|
|
((image_count=image_count+1))
|
|
done
|
|
|
|
# TODO Link up image_count > TODO Link up image_count by looping `<img` tags with images found on the disk.
|
|
|
|
#grep --color=never --perl-regexp --invert-match 'data:image/[^;]*;base64,\K[a-zA-Z0-9+/=]*' "${shownotes_html}"
|
|
|
|
|
|
# <img alt="Picture 1 shows the broken dog walking accessory." border="0" height="300" src="hpr4283_image_1_tn.jpeg" width="400" />
|
|
|
|
## TODO End Temp fix
|
|
|
|
}
|
|
|
|
#################################################
|
|
## Media Checks
|
|
|
|
function media_checks() {
|
|
|
|
if [ -n "${remote_media}" ]
|
|
then
|
|
echo_debug "Fetching remote media from \"${remote_media}\""
|
|
wget --timestamping --directory-prefix="${working_dir}/" "${remote_media}"
|
|
if [ $? -ne 0 ]
|
|
then
|
|
echo_error "Could not get the remote media"
|
|
fi
|
|
fi
|
|
|
|
media=$( find "${working_dir}/" -maxdepth 1 -type f -exec file --mime-type {} \; | grep -Ei ' audio/| video/' | awk -F ': ' '{print $1}' )
|
|
if [ -z "${media}" ]
|
|
then
|
|
find "${working_dir}/" -type f
|
|
echo_error "Can't find any media in \"${working_dir}/\""
|
|
fi
|
|
|
|
media_basename="$( basename "${media}" )"
|
|
if [ -z "${media_basename}" ]
|
|
then
|
|
echo_error "Could not create the media_basename \"${media_basename}/\""
|
|
fi
|
|
|
|
if [ "$( echo "${media}" | wc -l )" -ne 1 ]
|
|
then
|
|
echo "Multiple files found. Which one do you want to use ?"
|
|
select this_media in $( echo "${media}" )
|
|
do
|
|
ls -al "${this_media}"
|
|
media="${this_media}"
|
|
break
|
|
done
|
|
fi
|
|
echo_debug "You selected \"${media}\"."
|
|
|
|
if [[ -z "${media}" || ! -s "${media}" ]]
|
|
then
|
|
echo_error "Could not find the media \"${media}/\""
|
|
fi
|
|
|
|
shownotes_srt="${media%.*}.srt"
|
|
if [[ -z "${shownotes_srt}" || ! -s "${shownotes_srt}" ]]
|
|
then
|
|
echo_error "Could not find the subtitles for media \"${media}/\""
|
|
fi
|
|
|
|
# Find duration
|
|
duration=$( mediainfo --full --Output=JSON "${media}" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )
|
|
if [[ -z "${duration}" || "${duration}" -lt "30" || "${duration}" -gt "30000" ]]
|
|
then
|
|
echo_error "Invalid duration found in \"${media}\"" >&2
|
|
fi
|
|
echo_debug "The Duration is \"${duration}\" seconds from \"${media}\""
|
|
|
|
# Find number of channels ( 1=mono or 2=stereo)
|
|
supplied_channels=$( mediainfo --full --Output=JSON "${media}" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Channels' )
|
|
if [[ -z "${supplied_channels}" || "${supplied_channels}" -lt "1" || "${supplied_channels}" -gt "2" ]]
|
|
then
|
|
echo_error "Invalid number of audio channels \"${supplied_channels}\" found in \"${media}\"" >&2
|
|
fi
|
|
echo_debug "The number of audio channels is \"${supplied_channels}\" from \"${media}\" ."
|
|
|
|
# Gernerate the Spectrum and Waveform image
|
|
ffmpeg -hide_banner -loglevel error -y -i "${media}" -lavfi "showspectrumpic=s=960x540" "${working_dir}/processing/${media_basename%.*}_spectrum.png"
|
|
audio2image.bash "${media}" && mv -v "${media%.*}.png" "${working_dir}/processing/${media_basename%.*}_waveform.png"
|
|
|
|
# Getting metadata
|
|
|
|
mediainfo "${media}" > "${working_dir}/processing/${media_basename%.*}_mediainfo.txt"
|
|
|
|
exiftool "${media}" > "${working_dir}/processing/${media_basename%.*}_exiftool.txt"
|
|
|
|
for check_file in spectrum.png waveform.png mediainfo.txt exiftool.txt
|
|
do
|
|
if [ ! -s "${working_dir}/processing/${media_basename%.*}_${check_file}" ]
|
|
then
|
|
echo_error "The ${check_file} file was not generated for the \"${working_dir}/processing/${media_basename%.*}_${check_file}\"" >&2
|
|
fi
|
|
done
|
|
|
|
ffprobe="$( ffprobe ${media} 2>&1 | grep Audio: | sed 's/^.\s//g' )"
|
|
file_mime="$( file --brief --mime ${media} )"
|
|
file_mime_type="$( file --brief --mime-type ${media} )"
|
|
|
|
if [[ -z "${ffprobe}" || -z ${file_mime} || -s ${file_mime_type} ]]
|
|
then
|
|
echo "ffprobe: ${ffprobe}, file_mime: ${file_mime},file_mime_type: ${file_mime_type}"
|
|
echo_error "Wasn't able to find mime metadata from \"${media}/\""
|
|
fi
|
|
|
|
}
|
|
|
|
#################################################
|
|
## Generate Initial Report for review by the Janitors
|
|
|
|
function generate_initial_report() {
|
|
|
|
echo_debug "Generating the initial report."
|
|
|
|
# TODO list the images.
|
|
|
|
echo "<!DOCTYPE HTML>
|
|
<html lang=\"en\">
|
|
<head>
|
|
<title>Hacker Public Radio ~ The Technology Community Podcast</title>
|
|
<meta charset=\"utf-8\" />
|
|
<style>
|
|
body {
|
|
font-family: sans-serif;
|
|
}
|
|
table {
|
|
border-collapse: collapse;
|
|
text-align: left;
|
|
vertical-align: middle;
|
|
}
|
|
|
|
th, td {
|
|
border: 1px solid black;
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<body>
|
|
|
|
<h3>Field Mapping</h3>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Field</th><th>Value</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr><th>hostid</th><td>${hostid}</td></tr>
|
|
<tr><th>host_name</th><td>${host_name}</td></tr>
|
|
<tr><th>title</th><td>${title}</td></tr>
|
|
<tr><th>summary</th><td>${summary}</td></tr>
|
|
<tr><th>series_id</th><td>${series_id}</td></tr>
|
|
<tr><th>series_name</th><td>${series_name}</td></tr>
|
|
<tr><th>explicit</th><td>${explicit}</td></tr>
|
|
<tr><th>episode_license</th><td>${episode_license}</td></tr>
|
|
<tr><th>tags</th><td>${tags}</td></tr>
|
|
<tr><th>host_license</th><td>${host_license}</td></tr>
|
|
<tr><th>host_profile</th><td>${host_profile}</td></tr>
|
|
<tbody>
|
|
</table>
|
|
|
|
<h3>Raw shownotes.json</h3>
|
|
<code>
|
|
<pre>
|
|
${shownotes_json_sanatised}
|
|
</pre>
|
|
</code>
|
|
|
|
|
|
<h2>Audio</h2>
|
|
|
|
<h3>mediainfo report</h3>
|
|
<code>
|
|
<pre>
|
|
$( cat "${working_dir}/processing/${media_basename%.*}_mediainfo.txt" )
|
|
</pre>
|
|
</code>
|
|
|
|
<h3>exiftool report</h3>
|
|
<code>
|
|
<pre>
|
|
$( cat "${working_dir}/processing/${media_basename%.*}_exiftool.txt" )
|
|
</pre>
|
|
</code>
|
|
|
|
<h3>Audio Spectrum</h3>
|
|
<p>
|
|
<img src=\"${working_dir}/processing/${media_basename%.*}_spectrum.png\" alt=\"Spectrum\" />
|
|
</p>
|
|
|
|
<h3>Audio Waveform</h3>
|
|
<p>
|
|
<img src=\"${working_dir}/processing/${media_basename%.*}_waveform.png\" alt=\"Waveform\" />
|
|
</p>
|
|
|
|
<pre>
|
|
${ffprobe}
|
|
${file_mime}
|
|
</pre>
|
|
|
|
<p>
|
|
<audio controls=\"\" preload=\"none\" style=\"width:600px;\" >
|
|
<source src=\"${media}\" type=\"${file_mime_type}\">
|
|
</audio>
|
|
<br />
|
|
<a href=\"${media}\">${media}</a>
|
|
</p>
|
|
|
|
<h3>Transcript</h3>
|
|
|
|
<hr />
|
|
<pre>
|
|
$(cat "${shownotes_srt}" )
|
|
</pre>
|
|
<hr />
|
|
</body>
|
|
</html>" > "${working_dir}/processing/${media_basename%.*}_media_report.html"
|
|
|
|
}
|
|
|
|
#################################################
|
|
## Manually edit the shownotes to fix issues
|
|
|
|
function manual_shownotes_review() {
|
|
|
|
echo_debug "Validating the initial report."
|
|
|
|
if [[ -z "${shownotes_html}" || ! -s "${shownotes_html}" || ! -s "${working_dir}/processing/${media_basename%.*}_media_report.html" ]]
|
|
then
|
|
echo "shownotes_html: ${shownotes_html}"
|
|
ls -al "${shownotes_html}" "${working_dir}/processing/${media_basename%.*}_media_report.html"
|
|
echo_error "The files needed for to generate the inital report information are not available."
|
|
fi
|
|
|
|
if [ -s "${shownotes_html%.*}_edited.html" ]
|
|
then
|
|
echo_debug "There is already an edited version of the shownotes at \"${shownotes_html%.*}_edited.html\"."
|
|
return
|
|
fi
|
|
|
|
cp -v "${shownotes_html}" "${shownotes_html%.*}_edited.html"
|
|
|
|
if [ ! -s "${shownotes_html%.*}_edited.html" ]
|
|
then
|
|
echo_error "The edited shownotes are missing \"${shownotes_html%.*}_edited.html\"."
|
|
fi
|
|
|
|
kate "${shownotes_html%.*}_edited.html" >/dev/null 2>&1 &
|
|
librewolf "${working_dir}/processing/${media_basename%.*}_media_report.html" >/dev/null 2>&1 &
|
|
seamonkey "${shownotes_html%.*}_edited.html" >/dev/null 2>&1 &
|
|
# # # # bluefish "${shownotes_html%.*}_edited.html" >/dev/null 2>&1 &
|
|
# https://markdowntohtml.com/
|
|
|
|
read -p "Does the metadata 'look ok ? (N|y) ? " -n 1 -r
|
|
echo # (optional) move to a new line
|
|
if [[ ! $REPLY =~ ^[yY]$ ]]
|
|
then
|
|
echo_error "The final review was not approved."
|
|
fi
|
|
|
|
# remove extra wrappers that seamonkey adds
|
|
grep --invert-match --perl-regexp '<!DOCTYPE|html>|head>|<meta|body>' "${shownotes_html%.*}_edited.html" | sponge "${shownotes_html%.*}_edited.html"
|
|
|
|
# Check to see if images have been linked TODO make a loop for found images
|
|
if [ "$( find "${working_dir}" -type f -iname "*_image_*" | wc -l )" -ne "0" ]
|
|
then
|
|
if [ "$( grep --count "_image_" "${shownotes_html%.*}_edited.html" )" -eq "0" ]
|
|
then
|
|
echo_error "The extracted images were not linked in the shownotes \"${shownotes_html%.*}_edited.html\"."
|
|
fi
|
|
fi
|
|
}
|
|
|
|
#################################################
|
|
# Post show to HPR
|
|
|
|
function post_show_to_hpr_db() {
|
|
|
|
if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output /dev/null )" == 200 ]
|
|
then
|
|
echo "WARN: The Episode hpr${ep_num} has already been posted"
|
|
return
|
|
fi
|
|
|
|
if [ ! -s "${shownotes_html%.*}_edited.html" ]
|
|
then
|
|
echo_error "Failed to find the extracted shownote html file \"${shownotes_html%.*}_edited.html\""
|
|
fi
|
|
notes="$( cat "${shownotes_html%.*}_edited.html" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
|
|
host_profile_encoded="$( echo "${host_profile}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g' )"
|
|
|
|
|
|
post_show_json="${working_dir}/post_show.json"
|
|
|
|
echo "Sending:"
|
|
echo " key=${key}
|
|
ep_num=${ep_num}
|
|
ep_date=${ep_date}
|
|
email=${email_padded}
|
|
title=${title}
|
|
duration=${duration}
|
|
summary=${summary}
|
|
series_id=${series_id}
|
|
series_name=${series_name}
|
|
explicit=${explicit}
|
|
episode_license=${episode_license}
|
|
tags=${tags}
|
|
hostid=${hostid}
|
|
host_name=${host_name}
|
|
host_license=${host_license}
|
|
host_profile=${host_profile_encoded}
|
|
notes=REMOVED"
|
|
|
|
echo "{
|
|
\"key\": \"${key}\",
|
|
\"ep_num\": \"${ep_num}\",
|
|
\"ep_date\": \"${ep_date}\",
|
|
\"email\": \"${email_padded}\",
|
|
\"title\": \"${title}\",
|
|
\"duration\": \"${duration}\",
|
|
\"summary\": \"${summary}\",
|
|
\"series_id\": \"${series_id}\",
|
|
\"series_name\": \"${series_name}\",
|
|
\"explicit\": \"${explicit}\",
|
|
\"episode_license\": \"${episode_license}\",
|
|
\"tags\": \"${tags}\",
|
|
\"hostid\": \"${hostid}\",
|
|
\"host_name\": \"${host_name}\",
|
|
\"host_license\": \"${host_license}\",
|
|
\"host_profile\": \"${host_profile_encoded}\",
|
|
\"notes\": \"${notes}\"
|
|
}" > "${post_show_json}"
|
|
|
|
jq '.' "${post_show_json}"
|
|
|
|
if [ $? -ne 0 ]
|
|
then
|
|
echo_error "The file \"${post_show_json}\" is not valid json."
|
|
fi
|
|
|
|
curl --netrc --include --request POST "https://hub.hackerpublicradio.org/cms/add_show_json.php" --header "Content-Type: application/json" --data-binary "@${post_show_json}"
|
|
|
|
if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output /dev/null )" != 200 ]
|
|
then
|
|
echo_error "The Episode hpr${ep_num} has not been posted"
|
|
fi
|
|
|
|
}
|
|
|
|
#################################################
|
|
# Generate text to speech summary
|
|
|
|
function create_tts_summary {
|
|
|
|
if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output "${working_dir}/episode_summary.json" )" != 200 ]
|
|
then
|
|
echo_error "The Episode hpr${ep_num} has not been posted"
|
|
fi
|
|
|
|
synopsis="$( jq --raw-output '.synopsis' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
|
|
|
|
if [[ -z "${synopsis}" || "${synopsis}" == "null" ]]
|
|
then
|
|
echo_error "Could not retrieve the synopsis for the text to speech."
|
|
fi
|
|
|
|
echo_debug "Converting text synopsis \"${synopsis}\" to speech."
|
|
|
|
echo "${synopsis}" | "${piper_bin}" --model "${piper_voice}" --output_file "${working_dir}/processing/episode_tts.wav"
|
|
|
|
if [ ! -s "${working_dir}/processing/episode_tts.wav" ]
|
|
then
|
|
echo_error "The text to speech episode summary was not created \"${working_dir}/processing/episode_tts.wav\"."
|
|
fi
|
|
|
|
}
|
|
|
|
#################################################
|
|
# Generate Intro
|
|
|
|
function generate_intro {
|
|
|
|
echo_debug "Generating the intro."
|
|
|
|
if [[ ! -s "${working_dir}/processing/episode_tts.wav" || ! -s "${theme}" || ! -s "${media}" || ! -s "${outro_flac}" || ! -d "${working_dir}/processing/" ]]
|
|
then
|
|
echo_error "The files for the intro are not available."
|
|
ls -al "${working_dir}/processing/episode_tts.wav" "${theme}" "${media}" "${outro_flac}" "${working_dir}/processing/"
|
|
fi
|
|
|
|
# Everything needs to be in the same format for the intro, 1 channel (mono) Sampling rate 44.1 kHz
|
|
ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/processing/episode_tts.wav" -ar 44100 -ac 1 "${working_dir}/processing//episode_tts.flac"
|
|
|
|
# A level of silence is added at the beginning of the text to speech
|
|
sox -V2 "${silence}" "${working_dir}/processing//episode_tts.flac" "${working_dir}/processing/episode_summary.flac"
|
|
|
|
# The tracks are merged together resulting in the theme playing first, then after a period of silence the text to speech enters
|
|
sox -V2 -m "${working_dir}/processing/episode_summary.flac" "${theme}" "${working_dir}/processing/episode_intro.flac"
|
|
|
|
if [[ ! -s "${working_dir}/processing//episode_tts.flac" || ! -s "${working_dir}/processing/episode_summary.flac" || ! -s "${working_dir}/processing/episode_intro.flac" ]]
|
|
then
|
|
echo_error "The files for the theme audio sandwich are not available."
|
|
ls -al "${working_dir}/processing//episode_tts.flac" "${working_dir}/processing/episode_summary.flac" "${theme}" "${working_dir}/processing/episode_intro.flac"
|
|
fi
|
|
|
|
}
|
|
|
|
#################################################
|
|
# Generate parent audio - the sandwitch
|
|
|
|
function generate_parent_audio {
|
|
|
|
echo_debug "Generating the parent audio - the sandwitch."
|
|
|
|
if [[ ! -s "${working_dir}/processing/episode_intro.flac" || ! -s "${media}" || ! -s "${outro_flac}" ]]
|
|
then
|
|
echo_error "The files for the sandwich are not available."
|
|
ls -al
|
|
fi
|
|
|
|
# Everything needs to be in the same format so the text to speech needs to be converted to 2 channel Sampling rate 44.1 kHz
|
|
ffmpeg -hide_banner -loglevel error -y -i "${media}" -ar 44100 -ac 1 "${working_dir}/processing/episode_body.flac"
|
|
|
|
# Combine the components together
|
|
sox -V2 "${working_dir}/processing/episode_intro.flac" "${working_dir}/processing/episode_body.flac" "${outro_flac}" "${working_dir}/processing/episode_sandwitch.flac"
|
|
|
|
# Normalise the audio
|
|
ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/processing/episode_sandwitch.flac" -af loudnorm=I=-16:LRA=11:TP=-1.5 "${working_dir}/processing/episode_final.flac"
|
|
|
|
}
|
|
|
|
#################################################
|
|
# Generate derived media
|
|
|
|
function generate_derived_media {
|
|
|
|
echo_debug "Generating derived audio."
|
|
|
|
if [[ ! -s "${working_dir}/processing/episode_final.flac" ]]
|
|
then
|
|
ls -al
|
|
echo_error "The final cut is not available."
|
|
fi
|
|
|
|
episode_comment="$( jq --raw-output '.comment' "${working_dir}/episode_summary.json" )"
|
|
episode_year="$( echo "${ep_date}" | cut -c -4 )"
|
|
|
|
# https://wiki.multimedia.cx/index.php?title=FFmpeg_Metadata
|
|
|
|
for ext in flac wav mp3 ogg opus
|
|
do
|
|
echo_debug "Generating \"hpr${ep_num}.${ext}\"."
|
|
ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/processing/episode_final.flac" \
|
|
-metadata title="${title}" \
|
|
-metadata artist="${host_name}" \
|
|
-metadata author="${host_name}" \
|
|
-metadata album="Hacker Public Radio" \
|
|
-metadata comment="${episode_comment} The license is ${episode_license}" \
|
|
-metadata year="${episode_year}" \
|
|
-metadata track="${ep_num}" \
|
|
-metadata genre="Podcast" \
|
|
-metadata language="English" \
|
|
-metadata copyright="${episode_license}" \
|
|
"${working_dir}/hpr${ep_num}.${ext}"
|
|
|
|
fix_tags -album="Hacker Public Radio" -artist="${host_name}" -comment="${episode_comment} The license is ${episode_license}" -genre="Podcast" -title="${title}" -track="${ep_num}" -year="${episode_year}" "${working_dir}/hpr${ep_num}.${ext}"
|
|
|
|
if [[ ! -s "${working_dir}/hpr${ep_num}.${ext}" ]]
|
|
then
|
|
echo_error "Failed to generate \"${working_dir}/hpr${ep_num}.${ext}\"."
|
|
ls -al "${working_dir}/hpr${ep_num}.${ext}"
|
|
fi
|
|
|
|
done
|
|
|
|
cp -v "${media}" "${working_dir}/hpr${ep_num}_source.${media##*.}"
|
|
|
|
if [[ ! -s "${working_dir}/hpr${ep_num}_source.${media##*.}" ]]
|
|
then
|
|
echo_error "Failed to copy \"${working_dir}/hpr${ep_num}_source.${media##*.}\"."
|
|
ls -al "${working_dir}/hpr${ep_num}_source.${media##*.}"
|
|
fi
|
|
|
|
}
|
|
|
|
#################################################
|
|
# Generate Subtitles
|
|
|
|
function generate_show_transcript() {
|
|
|
|
echo_debug "Generate show transcript and subtitles."
|
|
|
|
# TODO Currently processed elsewhere by hpr-get-and-transcode.bash and uploaded to hpr:upload/ to be synced with media above
|
|
|
|
if [[ ! -s "${media}" || ! -s "${media%.*}.srt" || ! -s "${intro_srt}" || ! -s "${outro_srt}" || ! -s "${working_dir}/processing/episode_intro.flac" || ! -s "${working_dir}/processing/episode_body.flac" ]]
|
|
then
|
|
ls -al "${media}" "${media%.*}.srt" "${intro_srt}" "${outro_srt}" "${working_dir}/processing/episode_intro.flac" "${working_dir}/processing/episode_body.flac"
|
|
echo_error "The transcriptions files are not available."
|
|
fi
|
|
|
|
# Copy in the intro subtitle template and replace each line with the text with the summary
|
|
|
|
date="$( jq --raw-output '.date' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
|
|
title="$( jq --raw-output '.title' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
|
|
duration="$( jq --raw-output '.duration' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
|
|
duration="$( \date -d@${duration} -u +%H:%M:%S )"
|
|
artist="$( jq --raw-output '.artist' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
|
|
explicit="$( jq --raw-output '.explicit' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
|
|
license="$( jq --raw-output '.license' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
|
|
summary="$( jq --raw-output '.summary' "${working_dir}/episode_summary.json" | sed -e 's/ \././g' -e 's/\.\./\./g' -e 's/ / /g' )"
|
|
|
|
if [[ -z "${date}" || "${date}" == "null" || -z "${title}" || "${title}" == "null" || -z "${duration}" || "${duration}" == "null" || -z "${duration}" || "${duration}" == "null" || -z "${artist}" || "${artist}" == "null" || -z "${explicit}" || "${explicit}" == "null" || -z "${license}" || "${license}" == "null" || -z "${summary}" || "${summary}" == "null" ]]
|
|
then
|
|
echo_error "Could not retrieve the synopsis for the text to speech."
|
|
ls -al "${working_dir}/episode_summary.json"
|
|
fi
|
|
|
|
REPLACE_LINE_1="This is Hacker Public Radio Episode ${ep_num}, for ${date}"
|
|
REPLACE_LINE_2="Today's show is entitled, \"${title}\""
|
|
REPLACE_LINE_3="The host is ${artist} and the duration is ${duration}"
|
|
REPLACE_LINE_4="The flag is ${explicit}, and the license is ${license}"
|
|
REPLACE_LINE_5="The summary is \"${summary}\""
|
|
|
|
cp -v ${intro_srt} "${working_dir}/processing/episode_intro.srt"
|
|
cp -v ${outro_srt} "${working_dir}/processing/episode_outro.srt"
|
|
|
|
sed -e "s~REPLACE_LINE_1~${REPLACE_LINE_1}~g" -e "s~REPLACE_LINE_2~${REPLACE_LINE_2}~g" -e "s~REPLACE_LINE_3~${REPLACE_LINE_3}~g" -e "s~REPLACE_LINE_4~${REPLACE_LINE_4}~g" -e "s~REPLACE_LINE_5~${REPLACE_LINE_5}~g" -i "${working_dir}/processing/episode_intro.srt"
|
|
|
|
if [ "$( grep --count REPLACE_LINE "${working_dir}/processing/episode_intro.srt" )" -ne "0" ]
|
|
then
|
|
echo_error "The intro subtitles were not correctly generated \"${working_dir}/processing/episode_intro.srt\"."
|
|
fi
|
|
|
|
|
|
# Time shift the media subtitles on by the duration of the intro wav file
|
|
# https://trac.ffmpeg.org/wiki/UnderstandingItsoffset
|
|
|
|
itsoffset_intro="$( mediainfo --full --Output=JSON "${working_dir}/processing/episode_intro.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )"
|
|
|
|
if [[ -z "${itsoffset_intro}" || "${itsoffset_intro}" == "null" ]]
|
|
then
|
|
echo_error "Could not retrieve the itsoffset_intro to correct the timing of the subtitles."
|
|
fi
|
|
|
|
ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_intro}" -i "${media%.*}.srt" -c copy "${working_dir}/processing/episode_body.srt"
|
|
|
|
# Timeshift the outro by the duration of the intro and the supplied media
|
|
|
|
itsoffset_body="$( mediainfo --full --Output=JSON "${working_dir}/processing/episode_body.flac" | jq --raw-output '.media.track | .[] | select(."@type"=="Audio") | .Duration' | awk -F '.' '{print $1}' )"
|
|
|
|
if [[ -z "${itsoffset_body}" || "${itsoffset_body}" == "null" ]]
|
|
then
|
|
echo_error "Could not retrieve the itsoffset_body to correct the timing of the subtitles."
|
|
fi
|
|
|
|
itsoffset_body=$((itsoffset_intro + $itsoffset_body))
|
|
|
|
ffmpeg -hide_banner -loglevel error -y -itsoffset "${itsoffset_body}" -i "${working_dir}/processing/episode_outro.srt" -c copy "${working_dir}/processing/episode_outro_shifted.srt"
|
|
|
|
# Combine the intro, timeshifted media subtitles, and the timeshifted outro subtitles.
|
|
|
|
cat "${working_dir}/processing/episode_intro.srt" "${working_dir}/processing/episode_body.srt" "${working_dir}/processing/episode_outro_shifted.srt" > "${working_dir}/processing/episode.srt"
|
|
|
|
# Parse the resulting subtitle file fixing the numberic counter
|
|
# https://en.wikipedia.org/wiki/SubRip
|
|
|
|
count=1
|
|
|
|
cat "${working_dir}/processing/episode.srt" | while read this_line
|
|
do
|
|
if [ "$( echo "${this_line}" | grep -c -P '^[0-9]+$' )" -eq "1" ]
|
|
then
|
|
echo "${count}"
|
|
count=$((count+1))
|
|
else
|
|
echo "${this_line}"
|
|
fi
|
|
done > "${working_dir}/hpr${ep_num}.srt"
|
|
|
|
# extract the txt version
|
|
|
|
grep -Pv -- '-->|^$|^[0-9]+$' "${working_dir}/hpr${ep_num}.srt" > "${working_dir}/hpr${ep_num}.txt"
|
|
|
|
if [[ ! -s "${working_dir}/hpr${ep_num}.srt" || ! -s "${working_dir}/hpr${ep_num}.txt" ]]
|
|
then
|
|
echo_error "The transcriptions files were not generated."
|
|
ls -al "${working_dir}/hpr${ep_num}.srt" "${working_dir}/hpr${ep_num}.txt"
|
|
fi
|
|
|
|
}
|
|
|
|
#################################################
|
|
## Generate Final Report
|
|
|
|
function generate_final_report() {
|
|
|
|
echo_debug "Generating the final report."
|
|
|
|
final_report="${working_dir}/processing/hpr${ep_num}_report.html"
|
|
|
|
for this_file_extension_to_check in flac mp3 ogg opus srt txt wav
|
|
do
|
|
if [[ ! -s "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" ]]
|
|
then
|
|
ls -al "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}"
|
|
echo_error "The generated media is missing \"${this_file_extension_to_check}\"."
|
|
fi
|
|
done
|
|
|
|
if [[ ! -s "${working_dir}/processing/${media_basename%.*}_media_report.html" ]]
|
|
then
|
|
ls -al "${working_dir}/processing/${media_basename%.*}_media_report.html"
|
|
echo_error "The initial report is not available.\"${working_dir}/processing/${media_basename%.*}_media_report.html\""
|
|
fi
|
|
|
|
grep -Pv '</body>|</html>' "${working_dir}/processing/${media_basename%.*}_media_report.html" > "${final_report}"
|
|
|
|
echo "<h3>Text To Speech</h3>
|
|
<p>
|
|
$( echo "${synopsis}" )
|
|
</p>
|
|
<p>
|
|
<audio controls=\"\" preload=\"none\" style=\"width:600px;\" >
|
|
<source src=\"${working_dir}/processing//episode_tts.flac\" type=\"audio/flac\">
|
|
</audio>
|
|
<br />
|
|
<a href=\"${working_dir}/processing//episode_tts.flac\">${working_dir}/processing//episode_tts.flac</a>
|
|
</p>
|
|
<hr />" >> "${final_report}"
|
|
|
|
|
|
for this_file_extension_to_check in flac mp3 ogg opus wav
|
|
do
|
|
ffmpeg -hide_banner -loglevel error -y -i "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" -lavfi "showspectrumpic=s=960x540" "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png"
|
|
|
|
audio2image.bash "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" && mv -v "${working_dir}/hpr${ep_num}.png" "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_waveform.png"
|
|
|
|
mediainfo "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt"
|
|
|
|
exiftool "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" > "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt"
|
|
|
|
ffprobe "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" 2>&1 | grep Audio: | sed 's/^.\s//g' > "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt"
|
|
file --brief --mime "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" >> "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt"
|
|
this_file_mime_type="$( file --brief --mime-type "${working_dir}/hpr${ep_num}.${this_file_extension_to_check}" )"
|
|
|
|
for this_file_to_check in spectrum.png waveform.png mediainfo.txt exiftool.txt ffprobe.txt
|
|
do
|
|
if [[ ! -s "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}" ]]
|
|
then
|
|
ls -al "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_${this_file_to_check}"
|
|
echo_error "The inital report information is missing \"${this_file_to_check}\"."
|
|
fi
|
|
done
|
|
|
|
echo "<h2>Report for derived media file \"hpr${ep_num}.${this_file_extension_to_check}\"</h2>
|
|
|
|
<h3>mediainfo report</h3>
|
|
<code>
|
|
<pre>
|
|
$( cat "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_mediainfo.txt" )
|
|
</pre>
|
|
</code>
|
|
|
|
<h3>exiftool report</h3>
|
|
<code>
|
|
<pre>
|
|
$( cat "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_exiftool.txt" )
|
|
</pre>
|
|
</code>
|
|
|
|
<h3>Audio Spectrum</h3>
|
|
<p>
|
|
<img src=\"${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_spectrum.png\" alt=\"Spectrum\" />
|
|
</p>
|
|
|
|
<h3>Audio Waveform</h3>
|
|
<p>
|
|
<img src=\"${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_waveform.png\" alt=\"Waveform\" />
|
|
</p>
|
|
|
|
<pre>
|
|
$( cat "${working_dir}/processing/hpr${ep_num}_${this_file_extension_to_check}_ffprobe.txt" )
|
|
</pre>
|
|
|
|
<p>
|
|
<audio controls=\"\" preload=\"none\" style=\"width:600px;\" >
|
|
<source src=\"${working_dir}/hpr${ep_num}.${this_file_extension_to_check}\" type=\"${this_file_mime_type}\">
|
|
</audio>
|
|
<br />
|
|
<a href=\"${working_dir}/hpr${ep_num}.${this_file_extension_to_check}\">${working_dir}/hpr${ep_num}.${this_file_extension_to_check}</a>
|
|
</p>" >> "${final_report}"
|
|
|
|
done
|
|
|
|
echo "
|
|
|
|
|
|
<h3>Rendered shownotes.html</h3>
|
|
|
|
<hr />
|
|
|
|
$(cat "${shownotes_html%.*}_edited.html" )
|
|
|
|
<hr />
|
|
|
|
<h3>Subtitle File</h3>
|
|
|
|
<hr />
|
|
<pre>
|
|
$(cat "${working_dir}/hpr${ep_num}.srt" )
|
|
</pre>
|
|
<hr />
|
|
|
|
<h3>Raw shownotes.html</h3>
|
|
|
|
<hr />
|
|
<pre>
|
|
$(cat "${working_dir}/hpr${ep_num}.txt" )
|
|
</pre>
|
|
<hr />
|
|
|
|
|
|
</body>
|
|
</html>" >> "${final_report}"
|
|
|
|
}
|
|
|
|
#################################################
|
|
## Manually edit the shownotes to fix issues
|
|
|
|
function manual_final_review() {
|
|
|
|
echo_debug "Validating the final report."
|
|
|
|
if [[ -z "${final_report}" || ! -s "${final_report}" ]]
|
|
then
|
|
ls -al "${final_report}"
|
|
echo_error "The files needed for to generate the final report information are not available."
|
|
fi
|
|
|
|
librewolf "${final_report}" >/dev/null 2>&1 &
|
|
# # # # bluefish "${shownotes_html%.*}_edited.html" >/dev/null 2>&1 &
|
|
# https://markdowntohtml.com/
|
|
|
|
read -p "Does the metadata 'look ok ? (N|y) ? " -n 1 -r
|
|
echo # (optional) move to a new line
|
|
if [[ ! $REPLY =~ ^[yY]$ ]]
|
|
then
|
|
echo "skipping...."
|
|
echo_error "The final review was not approved."
|
|
fi
|
|
|
|
}
|
|
|
|
#################################################
|
|
# Send the derived files to the server borg to be sent to the Internet Archive
|
|
|
|
function copy_derived_files_to_borg_for_the_internet_archive() {
|
|
|
|
echo_debug "Sending the derived files to the server borg to be sent to the Internet Archive"
|
|
|
|
for ext in flac mp3 ogg opus wav
|
|
do
|
|
if [[ ! -s "${working_dir}/hpr${ep_num}.${ext}" ]]
|
|
then
|
|
echo_error "The inital report information is missing \"${ext}\"."
|
|
ls -al "${working_dir}/hpr${ep_num}.${ext}"
|
|
fi
|
|
done
|
|
|
|
echo_debug "rsync -ave ssh --partial --progress \"${working_dir}/hpr${ep_num}.{flac,mp3,ogg,opus,wav}\" borg:/data/IA/uploads/"
|
|
rsync -ave ssh --partial --progress "${working_dir}/hpr${ep_num}".{flac,mp3,ogg,opus,wav} borg:/data/IA/uploads/
|
|
rsync_error="${?}"
|
|
if [ "${rsync_error}" -ne "0" ]
|
|
then
|
|
echo_error "rsync to \"borg:/data/IA/uploads/\" failed with error ${rsync_error}"
|
|
fi
|
|
|
|
rsync -ave ssh --partial --progress "${working_dir}/hpr${ep_num}".{txt,srt} borg:/data/IA/uploads/hpr${ep_num}/
|
|
rsync_error="${?}"
|
|
if [ "${rsync_error}" -ne "0" ]
|
|
then
|
|
echo_error "rsync to \"borg:/data/IA/uploads/hpr${ep_num}/\" failed with error ${rsync_error}"
|
|
fi
|
|
|
|
rsync -ave ssh --partial --progress "${shownotes_html%.*}_edited.html" borg:/data/IA/uploads/hpr${ep_num}/shownotes.html
|
|
rsync_error="${?}"
|
|
if [ "${rsync_error}" -ne "0" ]
|
|
then
|
|
echo_error "rsync to \"borg:/data/IA/uploads/hpr${ep_num}/shownotes.html\" failed with error ${rsync_error}"
|
|
fi
|
|
|
|
# Get the current status
|
|
if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${processing_dir}/status.csv" )" != 200 ]
|
|
then
|
|
echo_error "Could not get a list of the queue status from \"https://hub.hackerpublicradio.org/cms/status.php\""
|
|
fi
|
|
|
|
# Check the current status is correct SHOW_POSTED
|
|
response=$( grep -P ",${ep_num},.*SHOW_POSTED," "${processing_dir}/status.csv" | head -1 | sed 's/,/ /g' )
|
|
if [ -z "${response}" ]
|
|
then
|
|
grep -P ",${ep_num},.*SHOW_POSTED," "${processing_dir}/status.csv"
|
|
echo_debug "The show \"${ep_num}\" hasn't the correct status of \"SHOW_POSTED\" in the database."
|
|
fi
|
|
|
|
# Try and change the online db status to MEDIA_TRANSCODED
|
|
if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' "https://hub.hackerpublicradio.org/cms/status.php?ep_num=${ep_num}&status=MEDIA_TRANSCODED" )" != 200 ]
|
|
then
|
|
echo_error "Could not change the status of \"${ep_num}\" to \"MEDIA_TRANSCODED\""
|
|
fi
|
|
|
|
# Get the current status
|
|
if [ "$( curl --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${processing_dir}/status.csv" )" != 200 ]
|
|
then
|
|
echo_error "Could not get a list of the queue status from \"https://hub.hackerpublicradio.org/cms/status.php\""
|
|
fi
|
|
|
|
# Check the current status is correct MEDIA_TRANSCODED
|
|
response=$( grep -P ",${ep_num},.*MEDIA_TRANSCODED," "${processing_dir}/status.csv" | head -1 | sed 's/,/ /g' )
|
|
if [ -z "${response}" ]
|
|
then
|
|
grep -P ",${ep_num},.*MEDIA_TRANSCODED," "${processing_dir}/status.csv"
|
|
echo_error "The show \"${ep_num}\" hasn't the correct status of \"MEDIA_TRANSCODED\" in the database."
|
|
fi
|
|
|
|
echo_debug "The show \"${ep_num}\" has the correct status of \"MEDIA_TRANSCODED\" in the database."
|
|
|
|
#TODO images
|
|
|
|
# <img alt="Picture 1 shows the broken dog walking accessory." border="0" height="300" src="https://archive.org/download/hpr4283/hpr4283/hpr4283_1_tn.jpeg" width="400" />
|
|
|
|
}
|
|
|
|
#################################################
|
|
# Main exceution starts here
|
|
#
|
|
# This tool will process the HPR shows allowing the janitor to review the media and fix shownotes.
|
|
#
|
|
# TODO Add support for reserve queue - process validate and move to reserve dir
|
|
# TODO Add support to reprocess processed shows - when given onlay new media reprocess it, update duration on hub, generate and download shownotes.{html,json} from db
|
|
# TODO Add support for community news - reusing ^^^
|
|
# TODO Add support for stereo for some episodes that request it
|
|
# TODO Include links in extract_images_brute_force
|
|
# TODO run hpr_generator to genrate only the new episode
|
|
# TODO take screenshots of the rendered episode on the hpr website
|
|
# TODO audio_channels default to mono - stereo as an option
|
|
# TODO check the channels on the source audio and add a warning in the report to check it's ok to reduce to mono
|
|
# TODO Add chapter support
|
|
# TODO incorporate direct upload to the IA
|
|
# TODO change MEDIA_TRANSCODED
|
|
# TODO incorporate assets
|
|
# TODO copy the files to the backup disk
|
|
# TODO copy the derived files to the ccdn origin server
|
|
# TODO fix permissions on vger(two)
|
|
|
|
# TODO
|
|
|
|
program_checks # We know that all the programs and variables are set
|
|
|
|
get_working_dir $@ # We have a working directory and a valid json file
|
|
|
|
get_episode_metadata $@ # We have all the metadata we need to process the show.
|
|
|
|
extract_images_brute_force # Extract_images by brute force
|
|
|
|
media_checks #
|
|
|
|
generate_initial_report # Generate Initial Report for review by the Janitors
|
|
|
|
manual_shownotes_review # Janitors review audio and shownote. Skips if done.
|
|
|
|
post_show_to_hpr_db # Posts the episode to HPR. Skips if it is already posted.
|
|
|
|
create_tts_summary # Generate text to speech summary
|
|
|
|
generate_intro # Generate Intro from the intro theme with overlay of a lead in silence then the tts summary
|
|
|
|
generate_parent_audio # Combines the intro, the episode, and the outro to a final cut.
|
|
|
|
generate_derived_media # Generate the flac wav mp3 ogg opus files
|
|
|
|
generate_show_transcript
|
|
|
|
generate_final_report
|
|
|
|
manual_final_review
|
|
|
|
#TODO update the assets table
|
|
|
|
copy_derived_files_to_borg_for_the_internet_archive
|
|
|
|
echo_debug "here" ; exit # TODO
|