renamed process episode

2026-05-13 12:56:34 +02:00
parent 8577444d5b
commit 79c643c938
3 changed files with 190 additions and 81 deletions
--- a/workflow/hpr-cn-generate-notes.bash
+++ b/workflow/hpr-cn-generate-notes.bash
@@ -47,7 +47,7 @@ fi
 ############################################################################################
 # Get previous show info

-row=$(sqlite3 hpr.db "
+row=$(sqlite3 "${hpr_db}" "
 SELECT notes
 FROM eps
 WHERE duration > 0
@@ -56,6 +56,13 @@ ORDER BY date DESC
 LIMIT 1;
 ")

+printf "\n%s\n" "SELECT notes
+FROM eps
+WHERE duration > 0
+AND series = 47
+ORDER BY date DESC
+LIMIT 1;" 1>&2
+
 eval $(
  echo "$row" |
  grep -o '<!-- HPRCN:[^>]*-->' |
@@ -63,14 +70,17 @@ eval $(
  tr ',' '\n'
 )

-echo_debug "The last show month was for ${month}, latest_comment: ${latest_comment}, latest_host: ${latest_host}, and latest_episode: ${latest_episode}"
+echo_debug "The last show month was for ${month}, previous_cn_latest_comment: ${latest_comment}, latest_host: ${latest_host}, and previous_cn_latest_episode: ${previous_cn_latest_episode}"
+
+previous_cn_latest_comment="${latest_comment}"
+previous_cn_latest_episode="${latest_episode}"

 ############################################################################################
 # Get next show info

 this_month=$( \date --date="${month}-01 +1 month" +%Y-%m )

-IFS='|' read -r eps_num eps_date <<< "$(sqlite3 hpr.db "
+IFS='|' read -r eps_num eps_date <<< "$(sqlite3 "${hpr_db}" "
 SELECT id, date
 FROM eps
 WHERE duration = 0
@@ -78,6 +88,12 @@ AND series = 47
 ORDER BY date ASC
 LIMIT 1;
 ")"
+printf "\n%s\n" "SELECT id, date
+FROM eps
+WHERE duration = 0
+AND series = 47
+ORDER BY date ASC
+LIMIT 1;" 1>&2

 echo_debug "This is going to be hpr${eps_num} for release on ${eps_date}"

@@ -94,72 +110,79 @@ shownotes_html="${eps_dir}/shownotes.html"
 ############################################################################################
 # Add New hosts to shownotes

-results=$(sqlite3 hpr.db "SELECT hostid, host FROM hosts WHERE hostid > ${latest_host};")
+results=$(sqlite3 "${hpr_db}" "SELECT hostid, host FROM hosts WHERE hostid > ${latest_host};")
+printf "\n%s\n" "SELECT hostid, host FROM hosts WHERE hostid > ${latest_host};" 1>&2

-echo "<h2>New hosts</h2>" | tee "${shownotes_html}"
-echo "<p>" | tee --append "${shownotes_html}"
+echo "<h2>New hosts</h2>" > "${shownotes_html}"
+echo "<p>" >> "${shownotes_html}"
 if [ -z "${results}" ]
 then
-  echo "There were no new hosts this month." | tee --append "${shownotes_html}"
+  echo "There were no new hosts this month." >> "${shownotes_html}"
  newest_host="${latest_host}"
 else
-  echo "Welcome to our new hosts: <br>" | tee --append "${shownotes_html}"
-  echo "<ul>" | tee --append "${shownotes_html}"
+  echo "Welcome to our new hosts: <br>" >> "${shownotes_html}"
+  echo "<ul>" >> "${shownotes_html}"
  while IFS="|" read -r hostid host
  do
    newest_host="${hostid}"
    printf -v hostid "%04d" "${hostid}"
-    echo "<li><a href=\"https://hackerpublicradio.org/correspondents/${hostid}.html\" target=\"_blank\">${host}</a></li>" | tee --append "${shownotes_html}"
+    echo "<li><a href=\"https://hackerpublicradio.org/correspondents/${hostid}.html\" target=\"_blank\">${host}</a></li>" >> "${shownotes_html}"
  done <<< "${results}"
-  echo "</ul>" | tee --append "${shownotes_html}"
+  echo "</ul>" >> "${shownotes_html}"
 fi
-echo "</p>" | tee --append "${shownotes_html}"
-echo "" | tee --append "${shownotes_html}"
+echo "</p>" >> "${shownotes_html}"
+echo "" >> "${shownotes_html}"

 ############################################################################################
 # Add Last Month's Shows to shownotes

-results=$(sqlite3 hpr.db "
+results=$(sqlite3 -separator '→' "${hpr_db}" "
 SELECT eps.id, hosts.host, eps.title, eps.date, eps.hostid
 FROM eps, hosts
-WHERE eps.id > ${latest_episode}
+WHERE eps.id > ${previous_cn_latest_episode}
 AND eps.date LIKE \"${this_month}%\"
 AND eps.valid = 1
 AND eps.hostid = hosts.hostid;
 ")

+printf "\n%s\n" "SELECT eps.id, hosts.host, eps.title, eps.date, eps.hostid
+FROM eps, hosts
+WHERE eps.id > ${previous_cn_latest_episode}
+AND eps.date LIKE \"${this_month}%\"
+AND eps.valid = 1
+AND eps.hostid = hosts.hostid;" 1>&2

-echo "<h2>Last Month's Shows</h2>" | tee --append "${shownotes_html}"
-echo "" | tee --append "${shownotes_html}"
+echo "<h2>Last Month's Shows</h2>" >> "${shownotes_html}"
+echo "" >> "${shownotes_html}"
 # [%# The id 't01' is in the HPR CSS but might give trouble on the IA -%]
-echo "<table id=\"t01\">" | tee --append "${shownotes_html}"
-echo "<tr>" | tee --append "${shownotes_html}"
-echo "<th>Id</th>" | tee --append "${shownotes_html}"
-echo "<th>Day</th>" | tee --append "${shownotes_html}"
-echo "<th>Date</th>" | tee --append "${shownotes_html}"
-echo "<th>Title</th>" | tee --append "${shownotes_html}"
-echo "<th>Host</th>" | tee --append "${shownotes_html}"
-echo "</tr>" | tee --append "${shownotes_html}"
+echo "<table id=\"t01\">" >> "${shownotes_html}"
+echo "<tr>" >> "${shownotes_html}"
+echo "<th>Id</th>" >> "${shownotes_html}"
+echo "<th>Day</th>" >> "${shownotes_html}"
+echo "<th>Date</th>" >> "${shownotes_html}"
+echo "<th>Title</th>" >> "${shownotes_html}"
+echo "<th>Host</th>" >> "${shownotes_html}"
+echo "</tr>" >> "${shownotes_html}"
 if [ -z "${results}" ]
 then
  echo_error "There were no shows found for this month."
 else
-  while IFS="|" read -r eps_id host title eps_date hostid
+  while IFS="→" read -r eps_id host title eps_date hostid
  do
    printf -v hostid "%04d" "${hostid}"
    printf -v eps_id "%04d" "${eps_id}"
-    echo "<tr>" | tee --append "${shownotes_html}"
-    echo "<td><strong><a href=\"https://hackerpublicradio.org/eps/hpr${eps_id}/index.html\" target=\"_blank\">${eps_id}</a></strong></td>" | tee --append "${shownotes_html}"
-    echo "<td>$( \date --date=${eps_date} +%a )</td>" | tee --append "${shownotes_html}"
-    echo "<td class=\"shrink\">${eps_date}</td>" | tee --append "${shownotes_html}"
-    echo "<td><a href=\"https://hackerpublicradio.org/eps/hpr${eps_id}/index.html\" target=\"_blank\">${title}</a></td>" | tee --append "${shownotes_html}"
-    echo "<td><a href=\"https://hackerpublicradio.org/correspondents/${hostid}.html\" target=\"_blank\">${host}</a></td>" | tee --append "${shownotes_html}"
+    echo "<tr>" >> "${shownotes_html}"
+    echo "<td><strong><a href=\"https://hackerpublicradio.org/eps/hpr${eps_id}/index.html\" target=\"_blank\">${eps_id}</a></strong></td>" >> "${shownotes_html}"
+    echo "<td>$( \date --date=${eps_date} +%a )</td>" >> "${shownotes_html}"
+    echo "<td class=\"shrink\">${eps_date}</td>" >> "${shownotes_html}"
+    echo "<td><a href=\"https://hackerpublicradio.org/eps/hpr${eps_id}/index.html\" target=\"_blank\">${title}</a></td>" >> "${shownotes_html}"
+    echo "<td><a href=\"https://hackerpublicradio.org/correspondents/${hostid}.html\" target=\"_blank\">${host}</a></td>" >> "${shownotes_html}"
    latest_episode=${eps_id}
-    echo "</tr>" | tee --append "${shownotes_html}"
+    echo "</tr>" >> "${shownotes_html}"
  done <<< "${results}"
 fi
-echo "</table>" | tee --append "${shownotes_html}"
-echo "" | tee --append "${shownotes_html}"
+echo "</table>" >> "${shownotes_html}"
+echo "" >> "${shownotes_html}"

 ############################################################################################
 # Add Comments to shownotes
@@ -175,40 +198,41 @@ function render_comments() {
  else
    this_comment="0"
    open_line="0"
-    echo "<ul>" | tee --append "${shownotes_html}"
-    while IFS="|" read -r comments_id comments_eps_id eps_date eps_title hosts_host hosts_hostid comments_comment_author_name comments_comment_timestamp comments_comment_title
+    echo "<ul>" >> "${shownotes_html}"
+    while IFS="→" read -r comments_id comments_eps_id eps_date eps_title hosts_host hosts_hostid comments_comment_author_name comments_comment_timestamp comments_comment_title
    do
      if [ "${this_comment}" -ne "${comments_eps_id}" ]
      then
        if [ "${open_line}" -ne "0" ]
        then
-          echo "</ul>" | tee --append "${shownotes_html}"
+          echo "</ul>" >> "${shownotes_html}"
        fi
        open_line="1"
        this_comment="${comments_eps_id}"
        printf -v hosts_hostid "%04d" "${hosts_hostid}"
        printf -v comments_eps_id "%04d" "${comments_eps_id}"
-        echo "<li><strong><a href=\"https://hackerpublicradio.org/eps/hpr${comments_eps_id}/index.html#comments\" target=\"_blank\">hpr${comments_eps_id}</a></strong> (${eps_date}) \"<em>${eps_title}</em>\" by <a href=\"https://hackerpublicradio.org/correspondents/${hosts_hostid}.html\" target=\"_blank\">${hosts_host}</a>.<br></li>" | tee --append "${shownotes_html}"
-        echo "<ul>" | tee --append "${shownotes_html}"
+        echo "<li><strong><a href=\"https://hackerpublicradio.org/eps/hpr${comments_eps_id}/index.html#comments\" target=\"_blank\">hpr${comments_eps_id}</a></strong> (${eps_date}) \"<em>${eps_title}</em>\" by <a href=\"https://hackerpublicradio.org/correspondents/${hosts_hostid}.html\" target=\"_blank\">${hosts_host}</a>.<br></li>" >> "${shownotes_html}"
+        echo "<ul>" >> "${shownotes_html}"
      fi
-      echo "<li style=\"list-style: none; display: inline\"><strong>${comments_comment_author_name}</strong> said: \"<em><a href=\"https://hackerpublicradio.org/eps/hpr${comments_eps_id}/index.html#comment_${comments_id}\" target=\"_blank\">${comments_comment_title}</a></em>\" (${comments_comment_timestamp})<br/></li>" | tee --append "${shownotes_html}"
+      echo "<li style=\"list-style: none; display: inline\"><strong>${comments_comment_author_name}</strong> said: \"<em><a href=\"https://hackerpublicradio.org/eps/hpr${comments_eps_id}/index.html#comment_${comments_id}\" target=\"_blank\">${comments_comment_title}</a></em>\" (${comments_comment_timestamp})<br/></li>" >> "${shownotes_html}"
      latest_comment="${comments_id}"
    done <<< "${results}"
-    echo "</ul>" | tee --append "${shownotes_html}"
-    echo "</ul>" | tee --append "${shownotes_html}"
+    echo "</ul>" >> "${shownotes_html}"
+    echo "</ul>" >> "${shownotes_html}"
  fi
 }

 # -------------------------------------------

-echo "<h2>Comments this month</h2>" | tee --append "${shownotes_html}"
-echo "" | tee --append "${shownotes_html}"
+echo "<h2>Comments this month</h2>" >> "${shownotes_html}"
+echo "" >> "${shownotes_html}"

-echo "<h3>Past shows</h3>" | tee --append "${shownotes_html}"
+echo "<h3>Past shows</h3>" >> "${shownotes_html}"

-latest_comment_last_changed=$(sqlite3 hpr.db "SELECT last_changed FROM comments WHERE comments.id = ${latest_comment};")
+previous_cn_latest_comment_last_changed=$(sqlite3 "${hpr_db}" "SELECT last_changed FROM comments WHERE comments.id = ${previous_cn_latest_comment};")
+printf "\n%s\n" "SELECT last_changed FROM comments WHERE comments.id = ${previous_cn_latest_comment};" 1>&2

-results=$(sqlite3 hpr.db "
+results=$(sqlite3 -separator '→' "${hpr_db}" "
 SELECT
  comments.id,
  comments.eps_id,
@@ -222,16 +246,34 @@ SELECT
 FROM comments
 JOIN eps   ON comments.eps_id = eps.id
 JOIN hosts ON eps.hostid = hosts.hostid
-WHERE comments.last_changed > \"${latest_comment_last_changed}\"
-  AND eps.date <= \"${this_month}-01\"
+WHERE comments.id > "${previous_cn_latest_comment}"
+  AND eps.id <= "${previous_cn_latest_episode}"
 ORDER BY eps.date;
 ")
+printf "\n%s\n" "
+SELECT
+  comments.id,
+  comments.eps_id,
+  eps.date,
+  eps.title,
+  hosts.host,
+  hosts.hostid,
+  comments.comment_author_name,
+  comments.comment_timestamp,
+  comments.comment_title
+FROM comments
+JOIN eps   ON comments.eps_id = eps.id
+JOIN hosts ON eps.hostid = hosts.hostid
+WHERE comments.id > "${previous_cn_latest_comment}"
+  AND eps.id <= "${previous_cn_latest_episode}"
+ORDER BY eps.date;
+" 1>&2

 render_comments "${results}"

-echo "<h3>This month's shows</h3>" | tee --append "${shownotes_html}"
+echo "<h3>This month's shows</h3>" >> "${shownotes_html}"

-results=$(sqlite3 hpr.db "
+results=$(sqlite3 -separator '→' "${hpr_db}" "
 SELECT
  comments.id,
  comments.eps_id,
@@ -248,6 +290,21 @@ JOIN hosts ON eps.hostid = hosts.hostid
 WHERE eps.date LIKE \"${this_month}-%\"
 ORDER BY eps.date;
 ")
+printf "\n%s\n" "SELECT
+  comments.id,
+  comments.eps_id,
+  eps.date,
+  eps.title,
+  hosts.host,
+  hosts.hostid,
+  comments.comment_author_name,
+  comments.comment_timestamp,
+  comments.comment_title
+FROM comments
+JOIN eps   ON comments.eps_id = eps.id
+JOIN hosts ON eps.hostid = hosts.hostid
+WHERE eps.date LIKE \"${this_month}-%\"
+ORDER BY eps.date;" 1>&2

 render_comments "${results}"

@@ -255,7 +312,7 @@ render_comments "${results}"
 # Add maillist and cal to shownotes

 mailman_link="https://lists.hackerpublicradio.com/pipermail/hpr/$( \date -d"${month}-01 next month" +%Y-%B )/thread.html"
-echo "<h2>Mailing List discussions</h2>" | tee --append "${shownotes_html}"
+echo "<h2>Mailing List discussions</h2>" >> "${shownotes_html}"
 echo "<p>Policy decisions surrounding HPR are taken by the community as a whole. This
 discussion takes place on the <a href=\"https://lists.hackerpublicradio.com/mailman/listinfo/hpr\" target=\"_blank\">Mailing List</a>
 which is open to all HPR listeners and contributors. The discussions are open
@@ -271,10 +328,11 @@ and available on the HPR server under <a href=\"https://lists.hackerpublicradio.
 <blockquote>This is the LWN.net community event calendar, where we track
 events of interest to people using and developing Linux and free software.
 Clicking on individual events will take you to the appropriate web
-page.</blockquote>" | tee --append "${shownotes_html}"
+page.</blockquote>" >> "${shownotes_html}"

-latest_comment=$(sqlite3 hpr.db "SELECT MAX(id) FROM comments;")
-echo "<!-- HPRCN:month=${this_month},latest_comment=${latest_comment},latest_host=${newest_host},latest_episode=${latest_episode} -->" | tee --append "${shownotes_html}"
+latest_comment=$(sqlite3 "${hpr_db}" "SELECT MAX(id) FROM comments;")
+printf "\n%s\n" "" 1>&2
+echo "<!-- HPRCN:month=${this_month},latest_comment=${latest_comment},latest_host=${newest_host},latest_episode=${latest_episode} -->" >> "${shownotes_html}"

 cp -v "${shownotes_html}" "${shownotes_html%.*}"_edited.html
 ls -al "${eps_dir}"
--- a/workflow/hpr-get-and-transcribe.bash
+++ b/workflow/hpr-get-and-transcribe.bash
@@ -1,35 +1,86 @@
 #!/bin/bash

-rsync -av --partial --progress hpr:hub.hackerpublicradio.org/upload/ $HOME/tmp/hpr/processing/
+destination_dir="$HOME/tmp/hpr/processing/"
+server_dir="hub.hackerpublicradio.org"

-find $HOME/tmp/hpr/processing/*_*_????-??-??_* -type d | sort -t _ -k 2 | while read show_dir
-do
-  echo "${show_dir}"
-  if [ "$( find "${show_dir}" -type f -iname "*srt" | wc -l )" -eq "0" ]
+#################################################
+# Display Error message, display help and exit
+
+function echo_error() {
+  echo -e "ERROR: $@" #1>&2
+  display_help_and_exit
+  exit 1
+}
+
+#################################################
+# Display Information message
+
+function echo_debug() {
+  if [ "${debug}" != "0"  ]
  then
+    echo -e "INFO: $@" #1>&2
+  fi
+}
+
+#################################################
+# Display Help
+
+function display_help_and_exit() {
+  echo_debug "For more information see https://repo.anhonesthost.net/HPR/hpr_documentation"
+  exit 1
+}
+
+#################################################
+# main
+
+if [ "$( curl --location --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${destination_dir}/status.csv" )" != 200 ]
+then
+  echo_error "Could not get a list of the queue status from \"https://hub.hackerpublicradio.org/cms/status.php\""
+fi
+
+if [ ! -s "${destination_dir}/status.csv" ]
+then
+  echo_error "Failed to retrieve \"${destination_dir}/status.csv\" from server."
+fi
+
+response=$( cat "${destination_dir}/status.csv" | grep --extended-regexp ',SHOW_SUBMITTED,|,RESERVE_SHOW_SUBMITTED,' | sort -t ',' -k 2 )
+if [ -z "${response}" ]
+then
+  echo_debug "Getting a list of all the reservations."
+  curl --location --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | sort -n
+  echo_error "There appear to be no more shows with the status \"SHOW_SUBMITTED\"."
+fi
+
+printf '%s\n' ${response} | while read this_line
+do
+  IFS=, read -r timestamp_epoc ep_num ep_date key status email <<< "$( printf '%s\n' "${this_line}"  )"
+  
+  if [ "${ep_num}" == "9999" ]
+  then
+    source_dir="${server_dir}/reserve/"
+  else
+    source_dir="${server_dir}/upload/"
+  fi
+  if [ "$( find "${destination_dir}" -maxdepth 2 -mindepth 2 -type f -iname "*${key}*.srt" | wc --lines )" -ne "0" ]
+  then
+    echo_debug "The episode \"${timestamp_epoc} ${ep_num} ${ep_date} ${key} ${status} ${email}\" has been processed."
+    continue
+  fi
+  
+  show_path="$( ssh -n hpr -t "find \"${source_dir}\" -maxdepth 1 -mindepth 1 -type d -iname \"*${key}*\"" 2>/dev/null | sed "s#${source_dir}##g#" )"
+  if [ -z "${show_path}" ]
+  then
+    echo_error "The show_path could not be found."
+  fi
+  show_dir="${destination_dir}/${show_path}"
+  echo_debug "Copying \"hpr:${source_dir}/${show_path}/\" to \"${show_dir}/\""
+  rsync --archive --verbose --partial --progress hpr:${source_dir}/${show_path}/ ${show_dir}/
  cd "${show_dir}"
-    ls -haltr
+  pwd
  find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media
  do
-      whisper --model tiny --language en --output_dir "${show_dir}" "${this_media}"
-    done
-    rsync -av --partial --progress "${show_dir}/" hpr:hub.hackerpublicradio.org/upload/$( basename "${show_dir}")/
-  fi
+    whisper --output_format srt --model tiny --language en --output_dir "${show_dir}" "${this_media}"
  done
+  rsync --archive --verbose --partial --progress ${show_dir}/ hpr:${source_dir}/${show_path}/

-rsync -av --partial --progress hpr:hub.hackerpublicradio.org/reserve/ $HOME/tmp/hpr/reserve/
-
-find $HOME/tmp/hpr/reserve/*_*_* -type d | sort -t _ -k 2 | while read show_dir
-do
-  echo "${show_dir}"
-  if [ "$( find "${show_dir}" -type f -iname "*srt" | wc -l )" -eq "0" ]
-  then
-    cd "${show_dir}"
-    ls -haltr
-    find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media
-    do
-      whisper --model tiny --language en --output_dir "${show_dir}" "${this_media}"
-    done
-    rsync -av --partial --progress "${show_dir}/" hpr:hub.hackerpublicradio.org/reserve/$( basename "${show_dir}")/
-  fi
 done
--- a/workflow/hpr-process_episode.bash
+++ b/workflow/hpr-process_episode.bash