Automating sql generation on db change

workflow/hpr_db_backup.bash

@@ -1,10 +1,38 @@
 #!/bin/bash
-sql_save_dir="$HOME/hpr/sql"
-credential_file="$HOME/.my.cnf"
-last_update_txt="${sql_save_dir}/last_update.txt"
-sync_delay="30 mins"
 
-last_update_query="SELECT update_time FROM information_schema.tables tab WHERE update_time > (current_timestamp() - interval 30 day) AND table_type = 'BASE TABLE' AND table_name not in ('reservations') AND table_schema not in ('information_schema', 'sys', 'performance_schema','mysql') ORDER BY update_time ASC LIMIT 1;"
+# 5,20,35,50 * * * * $HOME/userfiles/sql/hpr_db_backup.bash >> $HOME/userfiles/sql/cron.log 2>&1 &
+
+#TODO
+#add a rss feed with the latest updates. Linking to the changes on gittea.
+# run that every 5 minutes and then if there is a change sleep 5 and confirm there has been no change since.
+# Then download the db and regenerate the site.
+# While making sure to generate the site at least once a day 
+# check the skip-extended-insert export into of the sql into git, so that the small diffs show up.
+# generate a rss feed with the latest changes.
+# update all the other scripts reserve, stats etc.
+
+
+sql_save_dir="$HOME/userfiles/sql"
+credential_file="${sql_save_dir}/.my-hpr.cnf"
+last_update_txt="${sql_save_dir}/last_update.txt"
+hpr_full_sql="${sql_save_dir}/hpr_hpr_full.sql"
+hpr_site_sql="${sql_save_dir}/hpr.sql"
+full_mysqldump_sql="${sql_save_dir}/mysqldump.sql"
+
+sync_delay_seconds="300" # 5 minutes
+
+last_update_query="SELECT
+  update_time
+FROM
+  information_schema.tables tab
+WHERE
+  update_time > (current_timestamp() - interval 30 day)
+  AND table_type = 'BASE TABLE'
+  AND table_name not in ('reservations')
+  AND table_schema not in ('information_schema', 'sys', 'performance_schema', 'mysql')
+ORDER BY
+  update_time DESC
+LIMIT 1;"
 
 if [ ! -s "${credential_file}" ]
 then
@@ -12,37 +40,117 @@ then
   exit 
 fi
 
+local_db_last_update_epoch="0"
 if [ -s "${last_update_txt}" ]
 then
   echo "Found the last update file \"${last_update_txt}\""
   local_db_last_update_iso8601="$( \date --utc --date="$( cat ${last_update_txt} )" +%Y-%m-%dT%H:%M:%SZ )"
   local_db_last_update_epoch="$( \date --utc --date="$( cat ${last_update_txt} )" +%s )"
-  echo "Latest change saved locally is ${local_db_last_update_iso8601} (${local_db_last_update_epoch})"
+  echo -e "Local DB update time is\t${local_db_last_update_iso8601} (${local_db_last_update_epoch})"
 fi
 
-mysql --disable-column-names --batch --execute="${last_update_query}" | grep -v update_time | head -1 > ${last_update_txt}
+live_db_last_update_iso8601="$( mysql --defaults-file="${credential_file}" --disable-column-names --batch --execute="${last_update_query}" | sed -e 's/ /T/g' -e 's/$/Z/g' )"
-
+if [ -z "${live_db_last_update_iso8601}" ]
-if [ ! -s "${last_update_txt}" ]
 then
-  echo "The file \"${last_update_txt}\" is missing"
+  echo "The live db update time \"live_db_last_update_iso8601\" is missing"
-  exit 
+  exit 1
 fi
 
-hpr_db_last_update_iso8601="$( \date --utc --date="$( cat ${last_update_txt} )" +%Y-%m-%dT%H:%M:%SZ )"
+live_db_last_update_epoch="$( \date --utc --date="${live_db_last_update_iso8601}" +%s )"
-hpr_db_last_update_epoch="$( \date --utc --date="$( cat ${last_update_txt} )" +%s )"
+if [ -z "${live_db_last_update_epoch}" ]
-echo "Latest change on the HPR website database is ${hpr_db_last_update_iso8601} (${hpr_db_last_update_epoch})"
+then
+  echo "The live db update time \"live_db_last_update_epoch\" is missing"
+  exit 2
+fi
 
-#TODO check that the db is greater 
+echo -e "Live DB update time is\t${live_db_last_update_iso8601} (${live_db_last_update_epoch})"
+
+if [ "${local_db_last_update_epoch}" -eq "${live_db_last_update_epoch}" ]
+then
+  echo "No changes detected. Skipping export."
+  exit 0
+fi
+
+echo "Starting export full with -complete-insert."
+if [ -s "${hpr_full_sql}" ]
+then
+  hpr_full_sql_write_time_iso8601="$( \date --utc --date="$( ls -al --full-time "${hpr_full_sql}" | awk '{print $6, $7, $8}' )" +%Y-%m-%dT%H:%M:%SZ )"
+  hpr_full_sql_write_time_epoch="$( \date --utc --date="${hpr_full_sql_write_time_iso8601}" +%s )"
+  if [ -z "${hpr_full_sql_write_time_epoch}" ]
+  then
+    echo "The live db update time \"hpr_full_sql_write_time_epoch\" is missing"
+    exit 3
+  fi
+  echo -e "Full DB write time is\t${hpr_full_sql_write_time_iso8601} (${hpr_full_sql_write_time_epoch})"
+
+  hpr_full_sql_write_time_with_delay_epoch="$(( ${hpr_full_sql_write_time_epoch} + ${sync_delay_seconds} ))"
+  time_now_epoch="$( \date --utc +%s )"
+
+  if [ "${hpr_full_sql_write_time_with_delay_epoch}" -gt "${time_now_epoch}" ]
+  then
+    echo "Skipping export. The Database has been recently created \"${hpr_full_sql_write_time_iso8601}\". Try again after $( \date --utc --date="@${hpr_full_sql_write_time_with_delay_epoch}" +%Y-%m-%dT%H:%M:%SZ )."
+    exit 4
+  fi
+fi
+mysqldump --defaults-file="${credential_file}" --tz-utc --add-drop-database --extended-insert --complete-insert --skip-extended-insert --default-character-set=utf8 --single-transaction --skip-set-charset --databases hpr_hpr > "${hpr_full_sql}"
+tail "${hpr_full_sql}" | grep 'Dump completed on'
+
+echo "Starting export full for static site generation."
+if [ -s "${hpr_site_sql}" ]
+then
+  hpr_site_sql_write_time_iso8601="$( \date --utc --date="$( ls -al --full-time "${hpr_site_sql}" | awk '{print $6, $7, $8}' )" +%Y-%m-%dT%H:%M:%SZ )"
+  hpr_site_sql_write_time_epoch="$( \date --utc --date="${hpr_site_sql_write_time_iso8601}" +%s )"
+  if [ -z "${hpr_site_sql_write_time_epoch}" ]
+  then
+    echo "The live db update time \"hpr_site_sql_write_time_epoch\" is missing"
+    exit 5
+  fi
+  echo -e "Full DB write time is\t${hpr_site_sql_write_time_iso8601} (${hpr_site_sql_write_time_epoch})"
+
+  hpr_site_sql_write_time_with_delay_epoch="$(( ${hpr_site_sql_write_time_epoch} + ${sync_delay_seconds} ))"
+  time_now_epoch="$( \date --utc +%s )"
+
+  if [ "${hpr_site_sql_write_time_with_delay_epoch}" -gt "${time_now_epoch}" ]
+  then
+    echo "Skipping export. The Database has been recently created \"${hpr_site_sql_write_time_iso8601}\". Try again after $( \date --utc --date="@${hpr_site_sql_write_time_with_delay_epoch}" +%Y-%m-%dT%H:%M:%SZ )."
+    exit 6
+  fi
+fi
+mysqldump --defaults-file="${credential_file}" --tz-utc --add-drop-database --complete-insert --extended-insert --default-character-set=utf8 --single-transaction --skip-set-charset --databases hpr_hpr --ignore-table=hpr_hpr.reservations > "${hpr_site_sql}"
+tail "${hpr_site_sql}" | grep 'Dump completed on'
+
+echo "Starting export full for data recovery."
+if [ -s "${full_mysqldump_sql}" ]
+then
+  full_mysqldump_sql_write_time_iso8601="$( \date --utc --date="$( ls -al --full-time "${full_mysqldump_sql}" | awk '{print $6, $7, $8}' )" +%Y-%m-%dT%H:%M:%SZ )"
+  full_mysqldump_sql_write_time_epoch="$( \date --utc --date="${full_mysqldump_sql_write_time_iso8601}" +%s )"
+  if [ -z "${full_mysqldump_sql_write_time_epoch}" ]
+  then
+    echo "The live db update time \"full_mysqldump_sql_write_time_epoch\" is missing"
+    exit 5
+  fi
+  echo -e "Full DB write time is\t${full_mysqldump_sql_write_time_iso8601} (${full_mysqldump_sql_write_time_epoch})"
+
+  full_mysqldump_sql_write_time_with_delay_epoch="$(( ${full_mysqldump_sql_write_time_epoch} + ${sync_delay_seconds} ))"
+  time_now_epoch="$( \date --utc +%s )"
+
+  if [ "${full_mysqldump_sql_write_time_with_delay_epoch}" -gt "${time_now_epoch}" ]
+  then
+    echo "Skipping export. The Database has been recently created \"${full_mysqldump_sql_write_time_iso8601}\". Try again after $( \date --utc --date="@${full_mysqldump_sql_write_time_with_delay_epoch}" +%Y-%m-%dT%H:%M:%SZ )."
+    exit 6
+  fi
+fi
+mysqldump --defaults-file="${credential_file}" --tz-utc --add-drop-database  --databases hpr_hpr> "${full_mysqldump_sql}"
+tail "${full_mysqldump_sql}" | grep 'Dump completed on'
+
+if [ $HOSTNAME = "whp01.cloud-hosting.io" ]
+then
+  cp -v "${hpr_site_sql}" $HOME/hackerpublicradio.org/public_html/hpr.sql
+else
+  rsync -av --partial --progress ${hpr_site_sql} hpr:hackerpublicradio.org/public_html/hpr.sql
+fi
+  
+echo "${live_db_last_update_iso8601}" > "${last_update_txt}"
 
 
-# # # if [ "${hpr_db_last_update_epoch}" -eq "${local_db_last_update_epoch}" ]
-# # # then
-# # #   echo "The file \"${last_update_txt}\" is missing"
-# # #   exit 
-# # # fi
 
-mysqldump --tz-utc --add-drop-database --extended-insert --complete-insert --skip-extended-insert --default-character-set=utf8 --single-transaction --skip-set-charset --databases hpr_hpr > "${sql_save_dir}/hpr_hpr_full.sql"
-mysqldump --tz-utc --add-drop-database --complete-insert --extended-insert --default-character-set=utf8 --single-transaction --skip-set-charset --databases hpr_hpr --ignore-table=hpr_hpr.reservations > "${sql_save_dir}/hpr.sql"
-mysqldump --tz-utc --add-drop-database  --databases hpr_hpr> "${sql_save_dir}/mysqldump.sql"
-
-rsync -av --partial --progress "${sql_save_dir}/hpr.sql" hpr:/docker/users/hpr/hackerpublicradio.org/public_html/hpr.sql

workflow/process_episode.bash

@@ -91,7 +91,7 @@ function program_checks() {
     done
   }
 
-  is_installed audio2image.bash awk base64 cat csvtojson curl date detox eval extract_images ffprobe ffmpeg file find grep head jq kate magick mediainfo mv realpath remove-image.pl rsync seamonkey sed sed sort sponge ssh touch touch wget hpr-check-ccdn-links
+  is_installed audio2image.bash awk base64 cat csvtojson curl date eval extract_images ffprobe ffmpeg file find grep head jq kate magick mediainfo mv realpath remove-image.pl rsync seamonkey sed sed sort sponge ssh touch touch wget hpr-check-ccdn-links
 
   for arg in $*
   do
@@ -647,7 +647,6 @@ function get_next_show_from_hpr_hub() {
   echo_debug "Downloading hpr${ep_num} from ${email_unpadded}"
   echo_debug ""
   echo_debug "rsync -ave ssh --partial --progress ${source_dir}/ ${working_dir}/"
-  ssh hpr -t "detox -v ${hpr_upload_dir}/"
   rsync -ave ssh --partial --progress ${source_dir}/ ${working_dir}/
 
 }
@@ -1032,7 +1031,7 @@ function media_checks() {
   shownotes_srt="${media%.*}.srt"
   if [[ -z "${shownotes_srt}" || ! -s "${shownotes_srt}" ]]
   then
-    echo_error "Could not find the subtitles for media \"${media}\""
+    echo_error "Could not find the subtitles for media \"${media}\" in \"${shownotes_srt}\""
   fi
   
   #TODO fix close duration