Support last update date #20

workflow/check_feeds.bash

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 feed_dir="$HOME/tmp/hpr/rss"
 

workflow/export-jsons-stats.bash

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # 6,21,36,51 * * * * /docker/users/hpr/userfiles/bin/export-jsons-stats.bash > /dev/null 2>&1 &
 

workflow/hpr-get-and-transcribe.bash

@@ -1,8 +1,9 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 destination_dir="$HOME/tmp/hpr/processing/"
+reserve_dir="$HOME/tmp/hpr/reserve/"
 server_dir="hub.hackerpublicradio.org"
-
+debug=1
 #################################################
 # Display Error message, display help and exit
 
@@ -57,30 +58,39 @@ do
   
   if [ "${ep_num}" == "9999" ]
   then
-    source_dir="${server_dir}/reserve/"
+    hpr_dir="${server_dir}/reserve/"
+    local_dir="${reserve_dir}"
   else
-    source_dir="${server_dir}/upload/"
+    hpr_dir="${server_dir}/upload/"
+    local_dir="${destination_dir}"
   fi
-  if [ "$( find "${destination_dir}" -maxdepth 2 -mindepth 2 -type f -iname "*${key}*.srt" | wc --lines )" -ne "0" ]
+  if [ "$( find "${local_dir}" -maxdepth 2 -mindepth 2 -type f -iname "*${key}*.srt" | wc --lines )" -ne "0" ]
   then
     echo_debug "The episode \"${timestamp_epoc} ${ep_num} ${ep_date} ${key} ${status} ${email}\" has been processed."
     continue
   fi
   
-  show_path="$( ssh -n hpr -t "find \"${source_dir}\" -maxdepth 1 -mindepth 1 -type d -iname \"*${key}*\"" 2>/dev/null | sed "s#${source_dir}##g#" )"
+  
+  echo_debug "ssh -n hpr -t \"find \"${hpr_dir}\" -maxdepth 1 -mindepth 1 -type d -iname \"*${key}*\"\""
+  show_path="$( ssh -n hpr -t "find \"${hpr_dir}\" -maxdepth 1 -mindepth 1 -type d -iname \"*${key}*\"" 2>/dev/null | sed "s#${hpr_dir}##g#" )"
   if [ -z "${show_path}" ]
   then
-    echo_error "The show_path could not be found."
+    echo_error "The show_path could not be found. \"${show_path}\""
   fi
-  show_dir="${destination_dir}/${show_path}"
-  echo_debug "Copying \"hpr:${source_dir}/${show_path}/\" to \"${show_dir}/\""
-  rsync --archive --verbose --partial --progress hpr:${source_dir}/${show_path}/ ${show_dir}/
+  show_dir="${local_dir}/${show_path}"
+  if [ ! -d "${show_dir}" ]
+  then
+    mkdir -v "${show_dir}"
+  fi
+  echo_debug "Copying \"hpr:${hpr_dir}/${show_path}/\" to \"${show_dir}/\""
+  echo_debug "rsync --archive --verbose --partial --progress hpr:${hpr_dir}/${show_path}/ ${show_dir}/"
+  rsync --archive --verbose --partial --progress hpr:${hpr_dir}/${show_path}/ ${show_dir}/
   cd "${show_dir}"
   pwd
   find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media
   do
     whisper --output_format srt --model tiny --language en --output_dir "${show_dir}" "${this_media}"
   done
-  rsync --archive --verbose --partial --progress ${show_dir}/ hpr:${source_dir}/${show_path}/
+  rsync --archive --verbose --partial --progress ${show_dir}/ hpr:${hpr_dir}/${show_path}/
 
 done

workflow/hpr-prepare-images.bash

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+working_dir=$(pwd) 
+ep_num=$( echo ${working_dir} | awk -F '_' '{print $2}' ) 
+
+echo "Settting ep_num=\"${ep_num}\", and working_dir=\"${working_dir}\"" >&2
+
+image_count=0
+
+for found_image in $( find "${working_dir}" -maxdepth 1 -mindepth 1 -type f -exec file --mime-type {} \; | grep image | awk -F ': ' '{print $1}' )
+do
+  this_image="${working_dir}/hpr${ep_num}_image_${image_count}"
+  cp -v "${found_image}" "${this_image}" >&2
+  this_ext="$( file --mime-type ${this_image} | awk -F '/' '{print $NF}' )"
+  mv -v "${this_image}" "${this_image}.${this_ext}" >&2
+  this_width="$( mediainfo --Full --Output=XML "${this_image}.${this_ext}" | xmlstarlet sel --text --template --match '/_:MediaInfo/_:media/_:track[@type="Image"][1]' --value-of '_:Width' --nl - )"
+  if [ "${this_width}" -gt "400" ]
+  then
+    echo "Generating thumbnail for embedded image \"${this_image}.${this_ext}\"." >&2
+    magick "${this_image}.${this_ext}" -resize 400x "${this_image}_tn.${this_ext}"
+    
+    echo "<p><a href=\"$( basename "${this_image}.${this_ext}" )\"><img src=\"$( basename "${this_image}_tn.${this_ext}" )\" /></a></p>"
+    
+  fi
+  ((image_count=image_count+1))
+done

workflow/hpr_ccdn_stats.bash

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 yesterday="$( \date -u +%Y-%m-%d -d yesterday)" 
 echo -e "${yesterday}\t$( grep -Ec "${yesterday}T.*Sending request to" /home/hpr/logs/naughty-ip.txt )" >> /home/hpr/hub/hpr_ccdn_stats.tsv

workflow/hpr_db_backup.bash

@@ -1,9 +1,23 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # 5,20,35,50 * * * * $HOME/userfiles/sql/hpr_db_backup.bash >> $HOME/userfiles/sql/cron.log 2>&1 &
 
 # https://hackerpublicradio.org/hpr.sql.rss
 
+# Runs periodically (comment suggests every 15 minutes).
+# Gets the latest information_schema.tables.UPDATE_TIME.
+# Compares it with last_update.txt.
+# If unchanged, does nothing.
+# If changed, waits at least 5 minutes since the previous dump.
+# Creates three mysqldump files:
+#     full DB dump
+#     site dump excluding reservations
+#     another full recovery dump
+# Generates an RSS feed containing checksums of the site SQL.
+# Copies/rsyncs the site SQL and RSS to the HPR web host.
+# Records the detected DB timestamp in last_update.txt.
+
+
 #TODO
 #add a rss feed with the latest updates. Linking to the changes on gittea.
 # run that every 5 minutes and then if there is a change sleep 5 and confirm there has been no change since.
@@ -24,18 +38,11 @@ hpr_database_file="hackerpublicradio.org/public_html/hpr.sql"
 
 sync_delay_seconds="300" # 5 minutes
 
-last_update_query="SELECT
-  update_time
-FROM
-  information_schema.tables tab
-WHERE
-  update_time > (current_timestamp() - interval 30 day)
-  AND table_type = 'BASE TABLE'
-  AND table_name not in ('reservations')
-  AND table_schema not in ('information_schema', 'sys', 'performance_schema', 'mysql')
-ORDER BY
-  update_time DESC
-LIMIT 1;"
+last_update_query="USE hpr_hpr;
+SELECT GREATEST(
+    (SELECT MAX(last_changed) FROM eps),
+    (SELECT MAX(last_changed) FROM comments)
+) AS newest_last_changed;"
 
 if [ ! -s "${credential_file}" ]
 then

workflow/mdb.bash

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 pw=$(grep -E '^\$databasePassword = ' /home/hpr/php/credentials.php | awk -F "'" '{print $2}' )
 pw=${pw##* }
 mysql --host=localhost --user=hpr_hpr --password="$pw" hpr_hpr

workflow/run-speedtest.bash

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # * * * * * /usr/local/bin/run-speedtest.bash >/dev/null 2>&1
 

workflow/update-stats.bash

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 hub_dir="/home/hpr/hub"