Move HPR/hpr_hub#88

workflow/hpr-get-and-transcribe.bash

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+rsync -av --partial --progress hpr:hub.hackerpublicradio.org/upload/ $HOME/tmp/hpr/processing/
+
+find $HOME/tmp/hpr/processing/*_*_????-??-??_* -type d | sort -t _ -k 2 | while read show_dir
+do
+  echo "${show_dir}"
+  if [ "$( find "${show_dir}" -type f -iname "*srt" | wc -l )" -eq "0" ]
+  then
+    cd "${show_dir}"
+    ls -haltr
+    find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media
+    do
+      whisper --model tiny --language en --output_dir "${show_dir}" "${this_media}"
+    done
+    rsync -av --partial --progress "${show_dir}/" hpr:hub.hackerpublicradio.org/upload/$( basename "${show_dir}")/
+  fi
+done
+
+rsync -av --partial --progress hpr:hub.hackerpublicradio.org/reserve/ $HOME/tmp/hpr/reserve/
+
+find $HOME/tmp/hpr/reserve/*_*_* -type d | sort -t _ -k 2 | while read show_dir
+do
+  echo "${show_dir}"
+  if [ "$( find "${show_dir}" -type f -iname "*srt" | wc -l )" -eq "0" ]
+  then
+    cd "${show_dir}"
+    ls -haltr
+    find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media
+    do
+      whisper --model tiny --language en --output_dir "${show_dir}" "${this_media}"
+    done
+    rsync -av --partial --progress "${show_dir}/" hpr:hub.hackerpublicradio.org/reserve/$( basename "${show_dir}")/
+  fi
+done

workflow/hpr-publish.bash

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
+#============================================================
+
+git_dir="$HOME/tmp/hpr/hpr_generator/sourcecode"
+if [ ! -d "${git_dir}/.git" ]
+then
+  git clone gitea@repo.anhonesthost.net:HPR/hpr_generator.git "${git_dir}"
+fi
+
+cd "${git_dir}"
+
+git pull
+
+# ssh hpr -t "ls -al /home/hpr/www/hpr.sql;md5sum /home/hpr/www/hpr.sql"
+# ssh hpr -t "/home/hpr/bin/hpr_db_backup.bash"
+# ssh hpr -t "ls -al /home/hpr/www/hpr.sql;md5sum /home/hpr/www/hpr.sql"
+
+./utils/update-hpr-db.sh
+if [ $? -ne 0 ]
+then
+  echo 'Terminating...' >&2
+  exit 1
+fi
+
+./site-generator --all --verbose
+if [ $? -ne 0 ]
+then
+  echo 'Terminating...' >&2
+  exit 1
+fi
+
+rsync -av --partial --progress "${git_dir}/public_html/" hpr:hackerpublicradio.org/public_html
+#rsync -av --partial --progress "${git_dir}/public_html/" hobbypublicradio.org:hobbypublicradio.org/
+
+cd $HOME/sourcecode/hpr/hpr_hub/
+git pull 
+cd $HOME/sourcecode/hpr/hpr_hub/sql
+split --hex-suffixes --lines=1000 --additional-suffix=.sql hpr.sql hpr-db-part-
+cd $HOME/sourcecode/hpr/hpr_hub/
+git add $HOME/sourcecode/hpr/hpr_hub/sql/hpr*sql 
+git commit -m "$(\date -u +%Y-%m-%d_%H-%M-%SZ_%A ) database changed"
+git push
+#xdg-open https://hackerpublicradio.org/

workflow/hpr_ccdn_stats.bash

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+yesterday="$( \date -u +%Y-%m-%d -d yesterday)" 
+echo -e "${yesterday}\t$( grep -Ec "${yesterday}T.*Sending request to" /home/hpr/logs/naughty-ip.txt )" >> /home/hpr/hub/hpr_ccdn_stats.tsv

workflow/hpr_db_backup.bash

@@ -0,0 +1,48 @@
+#!/bin/bash
+sql_save_dir="$HOME/hpr/sql"
+credential_file="$HOME/.my.cnf"
+last_update_txt="${sql_save_dir}/last_update.txt"
+sync_delay="30 mins"
+
+last_update_query="SELECT update_time FROM information_schema.tables tab WHERE update_time > (current_timestamp() - interval 30 day) AND table_type = 'BASE TABLE' AND table_name not in ('reservations') AND table_schema not in ('information_schema', 'sys', 'performance_schema','mysql') ORDER BY update_time ASC LIMIT 1;"
+
+if [ ! -s "${credential_file}" ]
+then
+  echo "The file \"${credential_file}\" is missing"
+  exit 
+fi
+
+if [ -s "${last_update_txt}" ]
+then
+  echo "Found the last update file \"${last_update_txt}\""
+  local_db_last_update_iso8601="$( \date --utc --date="$( cat ${last_update_txt} )" +%Y-%m-%dT%H:%M:%SZ )"
+  local_db_last_update_epoch="$( \date --utc --date="$( cat ${last_update_txt} )" +%s )"
+  echo "Latest change saved locally is ${local_db_last_update_iso8601} (${local_db_last_update_epoch})"
+fi
+
+mysql --disable-column-names --batch --execute="${last_update_query}" | grep -v update_time | head -1 > ${last_update_txt}
+
+if [ ! -s "${last_update_txt}" ]
+then
+  echo "The file \"${last_update_txt}\" is missing"
+  exit 
+fi
+
+hpr_db_last_update_iso8601="$( \date --utc --date="$( cat ${last_update_txt} )" +%Y-%m-%dT%H:%M:%SZ )"
+hpr_db_last_update_epoch="$( \date --utc --date="$( cat ${last_update_txt} )" +%s )"
+echo "Latest change on the HPR website database is ${hpr_db_last_update_iso8601} (${hpr_db_last_update_epoch})"
+
+#TODO check that the db is greater 
+
+
+# # # if [ "${hpr_db_last_update_epoch}" -eq "${local_db_last_update_epoch}" ]
+# # # then
+# # #   echo "The file \"${last_update_txt}\" is missing"
+# # #   exit 
+# # # fi
+
+mysqldump --tz-utc --add-drop-database --extended-insert --complete-insert --skip-extended-insert --default-character-set=utf8 --single-transaction --skip-set-charset --databases hpr_hpr > "${sql_save_dir}/hpr_hpr_full.sql"
+mysqldump --tz-utc --add-drop-database --complete-insert --extended-insert --default-character-set=utf8 --single-transaction --skip-set-charset --databases hpr_hpr --ignore-table=hpr_hpr.reservations > "${sql_save_dir}/hpr.sql"
+mysqldump --tz-utc --add-drop-database  --databases hpr_hpr> "${sql_save_dir}/mysqldump.sql"
+
+rsync -av --partial --progress "${sql_save_dir}/hpr.sql" hpr:/docker/users/hpr/hackerpublicradio.org/public_html/hpr.sql

workflow/mdb.bash

@@ -0,0 +1,4 @@
+#!/bin/bash
+pw=$(grep -E '^\$databasePassword = ' /home/hpr/php/credentials.php | awk -F "'" '{print $2}' )
+pw=${pw##* }
+mysql --host=localhost --user=hpr_hpr --password="$pw" hpr_hpr

workflow/process_episode.bash

@@ -518,10 +518,10 @@ function check_variable_is_correct() {
         then
           echo_error "The \"shownotes_json\" file is missing."
         fi
-        if [ "$( file --brief --mime-type "${shownotes_json}" | grep --count 'application/json' )" -ne "1" ]
-        then
-          echo_error "The \"shownotes_json\" variable has not a valid \"application/json\" mime type."
-        fi
+#         if [ "$( file --brief --mime-type "${shownotes_json}" | grep --count 'application/json' )" -ne "1" ]
+#         then
+#           echo_error "The \"shownotes_json\" variable has not a valid \"application/json\" mime type."
+#         fi
         jq '.' "${shownotes_json}"  >/dev/null 2>&1
         if [ $? -ne 0 ]
         then
@@ -637,7 +637,7 @@ function get_next_show_from_hpr_hub() {
   email="$( echo ${response} | awk '{print $6}' )"
   email_unpadded="$( echo $email | sed 's/.nospam@nospam./@/g' )"
 
-  hpr_upload_dir="/home/hpr/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
+  hpr_upload_dir="hub.hackerpublicradio.org/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
   source_dir="hpr:${hpr_upload_dir}"
   dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
   working_dir="${processing_dir}/${dest_dir}"

workflow/run-speedtest.bash

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# * * * * * /usr/local/bin/run-speedtest.bash >/dev/null 2>&1
+
+speedtest-cli --json | jq '.' > /var/www/html/speedtest.json
+chown apache:apache /var/www/html/speedtest.json

workflow/update-stats.bash

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+hub_dir="/home/hpr/hub"
+
+for format in txt csv json xml
+do 
+  curl --silent --netrc-file /home/hpr/.netrc https://hub.hackerpublicradio.org/cms/stats.php?format=${format} --output ${hub_dir}/stats.${format} 
+done