From 6150943cb3a20a37278b0b24d5448b254a782f32 Mon Sep 17 00:00:00 2001
From: Ken Fallon <ken@fallon.ie>
Date: Mon, 12 May 2025 15:31:12 +0200
Subject: [PATCH] Fix variable override

---
 workflow/process_episode.bash | 235 +++++++++++++++++-----------------
 1 file changed, 117 insertions(+), 118 deletions(-)

diff --git a/workflow/process_episode.bash b/workflow/process_episode.bash
index 51ce5e6..2b4323f 100755
--- a/workflow/process_episode.bash
+++ b/workflow/process_episode.bash
@@ -75,7 +75,7 @@ function program_checks() {
     done
   }
 
-  is_installed audio2image.bash awk base64 cat csvtojson curl date detox eval extract_images ffprobe file find grep head jq kate magick mediainfo mv realpath remove-image.pl rsync seamonkey sed sed sort sponge ssh touch touch wget
+  is_installed audio2image.bash awk base64 cat csvtojson curl date detox eval extract_images ffprobe file find grep head jq kate magick mediainfo mv realpath remove-image.pl rsync seamonkey sed sed sort sponge ssh touch touch wget hpr-check-ccdn-links
 
   for arg in $*
   do
@@ -114,8 +114,29 @@ function check_variable_is_correct() {
           echo_error "The \"ep_num\" variable is not a valid number between 1 and 9999."
         fi
       ;;
-      
-      
+      piper_bin)
+        if [ -z "${piper_bin}" ]
+        then
+          echo_error "The \"ep_num\" variable is missing."
+        fi
+      ;;
+      piper_voice)
+        if [ -z "${ep_num}" ]
+        then
+          echo_error "The \"ep_num\" variable is missing."
+        fi
+      ;;
+
+      shownotes_html)
+        if [[ ! -s "${shownotes_html}" || -z "${shownotes_html}" ]]
+        then
+          echo_error "The \"shownotes_html\" variable/file is missing."
+        fi
+        if [ "$( file --brief --mime-type "${shownotes_html}" | grep --count 'text/html' )" -ne "1" ]
+        then
+          echo_error "The \"shownotes_html\" variable has not a valid \"text/html\" mime type."
+        fi
+      ;;
       shownotes_edited)
         if [[ ! -s "${shownotes_edited}" || -z "${shownotes_edited}" ]]
         then
@@ -196,8 +217,8 @@ function check_variable_is_correct() {
           echo_error "The file \"${files_xml}\" is not valid xml."
         fi
       ;;
-      
-      
+
+
       album)
         if [[ -z "${album}" || "${album}" == "null" ]]
         then
@@ -507,27 +528,33 @@ function get_episode_metadata() {
   </html>'
   ) | sponge "${shownotes_html}"
 
-  variables=( shownotes_json shownotes_html hostid host_name email title summary series_id series_name explicit episode_license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised )
+  episode_variables=( shownotes_json shownotes_html hostid host_name email title summary series_id series_name explicit episode_license ep_date ep_num tags host_license host_profile remote_media shownotes_json_sanatised )
 
-  for variable in "${variables[@]}"
+  for episode_variable in "${episode_variables[@]}"
   do
-    if [[ -z ${!variable} && "${variable}" != "remote_media" ]]
+    if [[ -z ${!episode_variable} && "${episode_variable}" != "remote_media" ]]
     then   # indirect expansion here
-      echo_error "The variable \"${variable}\" is missing.";
+      echo_error "The episode_variable \"${episode_variable}\" is missing.";
     else
-      echo_debug "The variable \"${variable}\" is set to \"${!variable}\"";
+      echo_debug "The episode_variable \"${episode_variable}\" is set to \"${!episode_variable}\"";
     fi
   done
 
   # Argument Override
   if [ $# -gt 0 ]
   then
-    declare -A hash
     for argument
     do
       if [[ $argument =~ ^[^=]+=.*$ ]]
       then
         this_key="${argument%=*}"
+        echo "${this_key}"
+        echo "${episode_variables[*]}"
+        if [ "$( echo "${episode_variables[*]}" | grep --count "${this_key}" )" -eq "0" ]
+        then
+          echo_debug "Skipping replacing \"${this_key}\".";
+          continue
+        fi
         this_value="${argument#*=}"
         this_value="$( echo "${this_value}" | jq --slurp --raw-input @uri | sed -e 's/%0A"$//g' -e 's/^"//g'  )"
         eval "${this_key}=${this_value}"
@@ -1032,6 +1059,7 @@ function post_show_to_hpr_db() {
     echo_error "The Episode hpr${ep_num} has not been posted"
   fi
 
+  xdg-open https://archive.org/details/hpr${ep_num}
 }
 
 #################################################
@@ -1050,7 +1078,7 @@ function get_variables_from_episode_summary_json() {
 
   if [ ! -s "${episode_summary_json}" ]
   then
-    echo_debug "The \"episode_summary_json\" variable/file is missing."
+    echo_debug "The \"episode_summary_json\" variable/file is missing, getting a new version."
     if [ "$( curl --silent --netrc --write-out '%{http_code}'  https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output "${episode_summary_json}" )" != 200 ]
     then
       echo_error "The Episode hpr${ep_num} has not been posted"
@@ -1474,9 +1502,9 @@ function manual_final_review() {
 function register_assets() {
 
   echo_debug "Registering the assets with the hpr database. register_assets()"
-  
+
   check_variable_is_correct working_dir ep_num
-  
+
   assets_csv="${working_dir}/hpr${ep_num}_assets.csv"
   assets_json="${working_dir}/hpr${ep_num}_assets.json"
 
@@ -1529,15 +1557,15 @@ function register_assets() {
 
   if [ -s "${assets_csv}" ]
   then
-    cat "${assets_csv}" | csvtojson | jq '{"assets":[.[]]}' | tee "assets_json"
+    cat "${assets_csv}" | csvtojson | jq '{"assets":[.[]]}' | tee "${assets_json}"
   fi
 
-  if [ ! -s "assets_json" ]
+  if [ ! -s "${assets_json}" ]
   then
-    echo_error "The asset json file \"assets_json\" is missing.";
+    echo_error "The asset json file \"${assets_json}\" is missing.";
   fi
 
-  response="$( curl --silent --netrc-file $HOME/.netrc --write-out '%{http_code}' --output /dev/null --request POST https://hub.hackerpublicradio.org/cms/assets.php --data-ascii @"assets_json" --header "Content-Type: application/json" )"
+  response="$( curl --silent --netrc-file $HOME/.netrc --write-out '%{http_code}' --output /dev/null --request POST https://hub.hackerpublicradio.org/cms/assets.php --data-ascii @"${assets_json}" --header "Content-Type: application/json" )"
   if [[ -z "${response}" || "${response}" != "200" ]]
   then
     echo_error "The assets for episode hpr${ep_num} has not been registered. The response was \"${response}\""
@@ -1735,16 +1763,13 @@ function upload_files_to_the_internet_archive() {
     files_xml="${working_dir}/hpr${ep_num}_files.xml"
   fi
 
-  if [ ! -s "${files_xml}" ]
+  echo_debug "Getting the \"files_xml\" file from the IA."
+  if [ "$( curl --silent --netrc --location --write-out '%{http_code}' https://archive.org/download/hpr${ep_num}/hpr${ep_num}_files.xml --output "${files_xml}" )" != 200 ]
   then
-    echo_debug "Getting the \"files_xml\" variable/file from the HPR site."
-    if [ "$( curl --silent --netrc --location --write-out '%{http_code}' https://archive.org/download/hpr${ep_num}/hpr${ep_num}_files.xml --output "${files_xml}" )" != 200 ]
-    then
-      echo_error "Could not find file list \"https://archive.org/download/hpr${ep_num}/hpr${ep_num}_files.xml\""
-    fi
+    echo_error "Could not find file list \"https://archive.org/download/hpr${ep_num}/hpr${ep_num}_files.xml\""
   fi
   check_variable_is_correct files_xml
-  
+
   # Get the file list from the HPR db
   if [ -z "${files_json}" ]
   then
@@ -1753,14 +1778,14 @@ function upload_files_to_the_internet_archive() {
 
   if [ ! -s "${files_json}" ]
   then
-    echo_debug "Getting the \"files_json\" variable/file from the HPR site."
+    echo_debug "Getting the \"files_json\" file from the HPR site."
     if [ "$( curl --silent --netrc --write-out '%{http_code}'  https://hub.hackerpublicradio.org/cms/assets.php?id=${ep_num} --output "${files_json}" )" != 200 ]
     then
       echo_error "The Episode hpr${ep_num} has not been posted"
     fi
   fi
   check_variable_is_correct files_json
-  
+
   for this_file in $( jq --raw-output ".hpr${ep_num} | keys | @tsv" "${files_json}" )
   do
     db_file_info="$( jq --raw-output ".hpr${ep_num}.[] | select(.filename==\"${this_file}\") | [ .filename, .size, .sha1sum ] | @tsv" "${files_json}" )"
@@ -1779,19 +1804,19 @@ function upload_files_to_the_internet_archive() {
         echo_debug "The file \"${this_file}\" on HPR and on the IA do not match."
       fi
     fi
-    
+
     echo_debug "Transferring \"${this_file}\" to the IA"
-    
+
     this_file_path="${working_dir}/${this_file}"
-    
+
     if [ ! -s "${this_file_path}" ]
     then
       echo_error "The file \"${this_file}\" could not be found in the working directory \"${working_dir}\"."
     fi
-    
+
     this_file_path_sha1="$( sha1sum $this_file_path | awk '{print $1}' )"
     db_file_info_sha1="$( echo ${db_file_info} | awk '{print $NF}' )"
-    
+
     if [ "${this_file_path_sha1}" != "${db_file_info_sha1}" ]
     then
       echo_error "The sha1 sum of the local file \"${this_file}\" does not match the HPR database."
@@ -1799,51 +1824,16 @@ function upload_files_to_the_internet_archive() {
     else
       ia upload hpr${ep_num} "${this_file_path}" --header "x-archive-keep-old-version:0" --retries=5 --no-derive --no-backup
     fi
-    
+
   done
+  
+  wait_for_the_internet_archive_to_process
 
 }
 
 function hide_me() {
 
 echo "hidden"
-#   
-
-
-  # Compare assets_json to files_xml
-  # locate the different files and upload them
-  
-#   if [ "$( find ${working_dir} -mindepth 1 -maxdepth 1 -type f \( -iname "hpr${ep_num}.flac" -or -iname "hpr${ep_num}.mp3" -or -iname "hpr${ep_num}.ogg" -or -iname "hpr${ep_num}.opus" -or -iname "hpr${ep_num}_source.*" -or -iname "hpr${ep_num}.srt" -or -iname "hpr${ep_num}.txt" -or -iname "hpr${ep_num}.wav" \) | wc --lines )" -ne "8" ]
-#   then
-#     echo_error "There are not 8 derived files for the Internet Archive."
-#   fi
-# 
-#   while read this_ia_file
-#   do
-#     if [[ ! -s "${this_ia_file}" ]]
-#     then
-#       ls -al "${this_ia_file}"
-#       echo_error "The derived files to the Internet Archive are missing \"${this_ia_file}\"."
-#     fi
-#     
-#     echo_debug "Sending \"${this_ia_file}\" to the Internet Archive"
-#     
-#     
-# #     origin_sha1sum="$( echo $( ssh rsync.net "sha1 hpr/eps/hpr${ep_num}/${this_asset}" 2> /dev/null ) | awk '{print $NF}' )"
-# #     this_asset_sha1sum="$( sha1sum "${this_file}" | awk '{print $1}' )"
-# # 
-# #     if [[ -z "${origin_sha1sum}" || -z "${this_asset_sha1sum}" ]]
-# #     then
-# #       echo_error "Could not determine the local/origin sha1sum for file \"${this_file}\"."
-# #     fi
-# #     
-# #     ia download hpr_761c0f557b87090db3f8d4d9bce7fc70  hpr_761c0f557b87090db3f8d4d9bce7fc70_files.xml
-#     
-#     echo ia upload hpr${ep_num} "${this_ia_file}" --header "x-archive-keep-old-version:0" --retries=5 --no-derive --no-backup
-#     
-#     
-#     
-#   done < <( find ${working_dir} -mindepth 1 -maxdepth 1 -type f \( -iname "hpr${ep_num}.flac" -or -iname "hpr${ep_num}.mp3" -or -iname "hpr${ep_num}.ogg" -or -iname "hpr${ep_num}.opus" -or -iname "hpr${ep_num}_source.*" -or -iname "hpr${ep_num}.srt" -or -iname "hpr${ep_num}.txt" -or -iname "hpr${ep_num}.wav" \) )
 
 }
 
@@ -1880,58 +1870,67 @@ then
   done
 fi
 
+########################################################################################
+# Posting show
 
-
-# program_checks              # We know that all the programs and variables are set
-#
-# get_working_dir $@          # We have a working directory and a valid json file
-#
-# get_episode_metadata $@     # We have all the metadata we need to process the show.
-#
-# extract_images_brute_force  # We have extracted the images by brute force
-#
-# media_checks                #
-#
-# generate_initial_report     # Generate Initial Report for review by the Janitors
-#
-# manual_shownotes_review     # Janitors review audio and shownote. Skips if done.
-#
-# post_show_to_hpr_db         # Posts the episode to HPR. Skips if it is already posted.
-#
-
-#########################################################################################
-# Using DB info from here
-
-get_variables_from_episode_summary_json          # Get the episode from HPR.
-
-#
-# create_tts_summary          # Generate text to speech summary
-#
-# generate_intro              # Generate Intro from the intro theme with overlay of a lead in silence then the tts summary
-#
-# generate_parent_audio       # Combines the intro, the episode, and the outro to a final cut.
-#
-# generate_derived_media      # Generate the flac wav mp3 ogg opus files
-#
-# generate_show_transcript
-#
-# generate_final_report
-#
-# manual_final_review
-#
-# register_assets
-
-# copy_files_to_origin_server
-
-#check_variable_is_correct working_dir ep_num shownotes_edited
-
-# copy_derived_files_to_borg
-
-# create_item_on_the_internet_archive
+#TODO add logic to check where we are and what we need to do.
+# All functions should skip if their work is done. Except when mode=refresh
+# 
+# 
+# 
+# If there is a working dir then 
+# - is the show posted etc, shownotes_edited.html
+# - do the assets match hpr${ep_num}_assets.json
+# - 
+# # 
+# # program_checks              # We know that all the programs and variables are set
+# # 
+# # get_working_dir $@          # We have a working directory and a valid json file
+# # 
+# # get_episode_metadata $@     # We have all the metadata we need to process the show.
+# # 
+# # extract_images_brute_force  # We have extracted the images by brute force
+# # 
+# # media_checks                #
+# # 
+# # generate_initial_report     # Generate Initial Report for review by the Janitors
+# # 
+# # manual_shownotes_review     # Janitors review audio and shownote. Skips if done.
+# # 
+# # post_show_to_hpr_db         # Posts the episode to HPR. Skips if it is already posted.
+# # 
+# # ########################################################################################
+# # # Using DB info from here
+# # 
+# # get_variables_from_episode_summary_json          # Get the episode from HPR.
+# # 
+# # create_tts_summary          # Generate text to speech summary
+# # 
+# # generate_intro              # Generate Intro from the intro theme with overlay of a lead in silence then the tts summary
+# # 
+# # generate_parent_audio       # Combines the intro, the episode, and the outro to a final cut.
+# # 
+# # generate_derived_media      # Generate the flac wav mp3 ogg opus files
+# # 
+# # generate_show_transcript
+# # 
+# # generate_final_report
+# # 
+# # manual_final_review
+# # 
+# # register_assets
+# # 
+# # copy_files_to_origin_server
+# # 
+# # check_variable_is_correct working_dir ep_num shownotes_edited
+# # 
+# # copy_derived_files_to_borg
+# # 
+# # create_item_on_the_internet_archive
 
 upload_files_to_the_internet_archive
 
-#for i in {4301..4305};do echo ${i};/home/ken/sourcecode/personal/bin/hpr-check-ccdn-links.bash ${i};done
+hpr-check-ccdn-links
 
 echo_debug "The End"
 exit 0