diff --git a/workflow/check_feeds.bash b/workflow/check_feeds.bash index 44211be..039b8b7 100755 --- a/workflow/check_feeds.bash +++ b/workflow/check_feeds.bash @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash feed_dir="$HOME/tmp/hpr/rss" diff --git a/workflow/export-jsons-stats.bash b/workflow/export-jsons-stats.bash index a74d7bd..2f1196a 100755 --- a/workflow/export-jsons-stats.bash +++ b/workflow/export-jsons-stats.bash @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # 6,21,36,51 * * * * /docker/users/hpr/userfiles/bin/export-jsons-stats.bash > /dev/null 2>&1 & diff --git a/workflow/hpr-get-and-transcribe.bash b/workflow/hpr-get-and-transcribe.bash index fbc4447..564a325 100755 --- a/workflow/hpr-get-and-transcribe.bash +++ b/workflow/hpr-get-and-transcribe.bash @@ -1,8 +1,9 @@ -#!/bin/bash +#!/usr/bin/env bash destination_dir="$HOME/tmp/hpr/processing/" +reserve_dir="$HOME/tmp/hpr/reserve/" server_dir="hub.hackerpublicradio.org" - +debug=1 ################################################# # Display Error message, display help and exit @@ -57,30 +58,39 @@ do if [ "${ep_num}" == "9999" ] then - source_dir="${server_dir}/reserve/" + hpr_dir="${server_dir}/reserve/" + local_dir="${reserve_dir}" else - source_dir="${server_dir}/upload/" + hpr_dir="${server_dir}/upload/" + local_dir="${destination_dir}" fi - if [ "$( find "${destination_dir}" -maxdepth 2 -mindepth 2 -type f -iname "*${key}*.srt" | wc --lines )" -ne "0" ] + if [ "$( find "${local_dir}" -maxdepth 2 -mindepth 2 -type f -iname "*${key}*.srt" | wc --lines )" -ne "0" ] then echo_debug "The episode \"${timestamp_epoc} ${ep_num} ${ep_date} ${key} ${status} ${email}\" has been processed." continue fi - show_path="$( ssh -n hpr -t "find \"${source_dir}\" -maxdepth 1 -mindepth 1 -type d -iname \"*${key}*\"" 2>/dev/null | sed "s#${source_dir}##g#" )" + + echo_debug "ssh -n hpr -t \"find \"${hpr_dir}\" -maxdepth 1 -mindepth 1 -type d -iname \"*${key}*\"\"" + show_path="$( ssh -n hpr -t "find \"${hpr_dir}\" -maxdepth 1 -mindepth 1 -type d -iname \"*${key}*\"" 2>/dev/null | sed "s#${hpr_dir}##g#" )" if [ -z "${show_path}" ] then - echo_error "The show_path could not be found." + echo_error "The show_path could not be found. \"${show_path}\"" fi - show_dir="${destination_dir}/${show_path}" - echo_debug "Copying \"hpr:${source_dir}/${show_path}/\" to \"${show_dir}/\"" - rsync --archive --verbose --partial --progress hpr:${source_dir}/${show_path}/ ${show_dir}/ + show_dir="${local_dir}/${show_path}" + if [ ! -d "${show_dir}" ] + then + mkdir -v "${show_dir}" + fi + echo_debug "Copying \"hpr:${hpr_dir}/${show_path}/\" to \"${show_dir}/\"" + echo_debug "rsync --archive --verbose --partial --progress hpr:${hpr_dir}/${show_path}/ ${show_dir}/" + rsync --archive --verbose --partial --progress hpr:${hpr_dir}/${show_path}/ ${show_dir}/ cd "${show_dir}" pwd find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media do whisper --output_format srt --model tiny --language en --output_dir "${show_dir}" "${this_media}" done - rsync --archive --verbose --partial --progress ${show_dir}/ hpr:${source_dir}/${show_path}/ + rsync --archive --verbose --partial --progress ${show_dir}/ hpr:${hpr_dir}/${show_path}/ done \ No newline at end of file diff --git a/workflow/hpr-prepare-images.bash b/workflow/hpr-prepare-images.bash new file mode 100755 index 0000000..6ac99c1 --- /dev/null +++ b/workflow/hpr-prepare-images.bash @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +working_dir=$(pwd) +ep_num=$( echo ${working_dir} | awk -F '_' '{print $2}' ) + +echo "Settting ep_num=\"${ep_num}\", and working_dir=\"${working_dir}\"" >&2 + +image_count=0 + +for found_image in $( find "${working_dir}" -maxdepth 1 -mindepth 1 -type f -exec file --mime-type {} \; | grep image | awk -F ': ' '{print $1}' ) +do + this_image="${working_dir}/hpr${ep_num}_image_${image_count}" + cp -v "${found_image}" "${this_image}" >&2 + this_ext="$( file --mime-type ${this_image} | awk -F '/' '{print $NF}' )" + mv -v "${this_image}" "${this_image}.${this_ext}" >&2 + this_width="$( mediainfo --Full --Output=XML "${this_image}.${this_ext}" | xmlstarlet sel --text --template --match '/_:MediaInfo/_:media/_:track[@type="Image"][1]' --value-of '_:Width' --nl - )" + if [ "${this_width}" -gt "400" ] + then + echo "Generating thumbnail for embedded image \"${this_image}.${this_ext}\"." >&2 + magick "${this_image}.${this_ext}" -resize 400x "${this_image}_tn.${this_ext}" + + echo "

" + + fi + ((image_count=image_count+1)) +done diff --git a/workflow/hpr_ccdn_stats.bash b/workflow/hpr_ccdn_stats.bash index bbab7af..9ccfaea 100644 --- a/workflow/hpr_ccdn_stats.bash +++ b/workflow/hpr_ccdn_stats.bash @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash yesterday="$( \date -u +%Y-%m-%d -d yesterday)" echo -e "${yesterday}\t$( grep -Ec "${yesterday}T.*Sending request to" /home/hpr/logs/naughty-ip.txt )" >> /home/hpr/hub/hpr_ccdn_stats.tsv diff --git a/workflow/hpr_db_backup.bash b/workflow/hpr_db_backup.bash index 8757109..85eb17b 100755 --- a/workflow/hpr_db_backup.bash +++ b/workflow/hpr_db_backup.bash @@ -1,9 +1,23 @@ -#!/bin/bash +#!/usr/bin/env bash # 5,20,35,50 * * * * $HOME/userfiles/sql/hpr_db_backup.bash >> $HOME/userfiles/sql/cron.log 2>&1 & # https://hackerpublicradio.org/hpr.sql.rss +# Runs periodically (comment suggests every 15 minutes). +# Gets the latest information_schema.tables.UPDATE_TIME. +# Compares it with last_update.txt. +# If unchanged, does nothing. +# If changed, waits at least 5 minutes since the previous dump. +# Creates three mysqldump files: +# full DB dump +# site dump excluding reservations +# another full recovery dump +# Generates an RSS feed containing checksums of the site SQL. +# Copies/rsyncs the site SQL and RSS to the HPR web host. +# Records the detected DB timestamp in last_update.txt. + + #TODO #add a rss feed with the latest updates. Linking to the changes on gittea. # run that every 5 minutes and then if there is a change sleep 5 and confirm there has been no change since. @@ -24,18 +38,11 @@ hpr_database_file="hackerpublicradio.org/public_html/hpr.sql" sync_delay_seconds="300" # 5 minutes -last_update_query="SELECT - update_time -FROM - information_schema.tables tab -WHERE - update_time > (current_timestamp() - interval 30 day) - AND table_type = 'BASE TABLE' - AND table_name not in ('reservations') - AND table_schema not in ('information_schema', 'sys', 'performance_schema', 'mysql') -ORDER BY - update_time DESC -LIMIT 1;" +last_update_query="USE hpr_hpr; +SELECT GREATEST( + (SELECT MAX(last_changed) FROM eps), + (SELECT MAX(last_changed) FROM comments) +) AS newest_last_changed;" if [ ! -s "${credential_file}" ] then diff --git a/workflow/mdb.bash b/workflow/mdb.bash index 0bdf3eb..c09e691 100755 --- a/workflow/mdb.bash +++ b/workflow/mdb.bash @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash pw=$(grep -E '^\$databasePassword = ' /home/hpr/php/credentials.php | awk -F "'" '{print $2}' ) pw=${pw##* } mysql --host=localhost --user=hpr_hpr --password="$pw" hpr_hpr diff --git a/workflow/run-speedtest.bash b/workflow/run-speedtest.bash index f90885a..981ccf3 100644 --- a/workflow/run-speedtest.bash +++ b/workflow/run-speedtest.bash @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # * * * * * /usr/local/bin/run-speedtest.bash >/dev/null 2>&1 diff --git a/workflow/update-stats.bash b/workflow/update-stats.bash index 752c13e..d32a6da 100644 --- a/workflow/update-stats.bash +++ b/workflow/update-stats.bash @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash hub_dir="/home/hpr/hub"