HPR/hpr-tools
8
1
Fork 2
Code Issues 15 Pull Requests Packages Projects Releases Wiki Activity

renamed process episode

Browse Source
This commit is contained in:
Ken Fallon
2026-05-13 12:56:34 +02:00
parent 8577444d5b
commit 79c643c938
3 changed files with 190 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

170
workflow/hpr-cn-generate-notes.bash
View File

@@ -47,7 +47,7 @@ fi
############################################################################################ ############################################################################################
# Get previous show info # Get previous show info
row=$(sqlite3 hpr.db " row=$(sqlite3 "${hpr_db}" "
SELECT notes SELECT notes
FROM eps FROM eps
WHERE duration > 0 WHERE duration > 0
@@ -56,6 +56,13 @@ ORDER BY date DESC
LIMIT 1; LIMIT 1;
") ")
printf "\n%s\n" "SELECT notes
FROM eps
WHERE duration > 0
AND series = 47
ORDER BY date DESC
LIMIT 1;" 1>&2
eval $( eval $(
echo "$row" | echo "$row" |
grep -o '<!-- HPRCN:[^>]*-->' | grep -o '<!-- HPRCN:[^>]*-->' |
@@ -63,14 +70,17 @@ eval $(
tr ',' '\n' tr ',' '\n'
) )
echo_debug "The last show month was for ${month}, latest_comment: ${latest_comment}, latest_host: ${latest_host}, and latest_episode: ${latest_episode}" echo_debug "The last show month was for ${month}, previous_cn_latest_comment: ${latest_comment}, latest_host: ${latest_host}, and previous_cn_latest_episode: ${previous_cn_latest_episode}"
previous_cn_latest_comment="${latest_comment}"
previous_cn_latest_episode="${latest_episode}"
############################################################################################ ############################################################################################
# Get next show info # Get next show info
this_month=$( \date --date="${month}-01 +1 month" +%Y-%m ) this_month=$( \date --date="${month}-01 +1 month" +%Y-%m )
IFS='|' read -r eps_num eps_date <<< "$(sqlite3 hpr.db " IFS='|' read -r eps_num eps_date <<< "$(sqlite3 "${hpr_db}" "
SELECT id, date SELECT id, date
FROM eps FROM eps
WHERE duration = 0 WHERE duration = 0
@@ -78,6 +88,12 @@ AND series = 47
ORDER BY date ASC ORDER BY date ASC
LIMIT 1; LIMIT 1;
")" ")"
printf "\n%s\n" "SELECT id, date
FROM eps
WHERE duration = 0
AND series = 47
ORDER BY date ASC
LIMIT 1;" 1>&2
echo_debug "This is going to be hpr${eps_num} for release on ${eps_date}" echo_debug "This is going to be hpr${eps_num} for release on ${eps_date}"
@@ -94,72 +110,79 @@ shownotes_html="${eps_dir}/shownotes.html"
############################################################################################ ############################################################################################
# Add New hosts to shownotes # Add New hosts to shownotes
results=$(sqlite3 hpr.db "SELECT hostid, host FROM hosts WHERE hostid > ${latest_host};") results=$(sqlite3 "${hpr_db}" "SELECT hostid, host FROM hosts WHERE hostid > ${latest_host};")
printf "\n%s\n" "SELECT hostid, host FROM hosts WHERE hostid > ${latest_host};" 1>&2
echo "<h2>New hosts</h2>" | tee "${shownotes_html}" echo "<h2>New hosts</h2>" > "${shownotes_html}"
echo "<p>" | tee --append "${shownotes_html}" echo "<p>" >> "${shownotes_html}"
if [ -z "${results}" ] if [ -z "${results}" ]
then then
echo "There were no new hosts this month." | tee --append "${shownotes_html}" echo "There were no new hosts this month." >> "${shownotes_html}"
newest_host="${latest_host}" newest_host="${latest_host}"
else else
echo "Welcome to our new hosts: <br>" | tee --append "${shownotes_html}" echo "Welcome to our new hosts: <br>" >> "${shownotes_html}"
echo "<ul>" | tee --append "${shownotes_html}" echo "<ul>" >> "${shownotes_html}"
while IFS="|" read -r hostid host while IFS="|" read -r hostid host
do do
newest_host="${hostid}" newest_host="${hostid}"
printf -v hostid "%04d" "${hostid}" printf -v hostid "%04d" "${hostid}"
echo "<li><a href=\"https://hackerpublicradio.org/correspondents/${hostid}.html\" target=\"_blank\">${host}</a></li>" | tee --append "${shownotes_html}" echo "<li><a href=\"https://hackerpublicradio.org/correspondents/${hostid}.html\" target=\"_blank\">${host}</a></li>" >> "${shownotes_html}"
done <<< "${results}" done <<< "${results}"
echo "</ul>" | tee --append "${shownotes_html}" echo "</ul>" >> "${shownotes_html}"
fi fi
echo "</p>" | tee --append "${shownotes_html}" echo "</p>" >> "${shownotes_html}"
echo "" | tee --append "${shownotes_html}" echo "" >> "${shownotes_html}"
############################################################################################ ############################################################################################
# Add Last Month's Shows to shownotes # Add Last Month's Shows to shownotes
results=$(sqlite3 hpr.db " results=$(sqlite3 -separator '→' "${hpr_db}" "
SELECT eps.id, hosts.host, eps.title, eps.date, eps.hostid SELECT eps.id, hosts.host, eps.title, eps.date, eps.hostid
FROM eps, hosts FROM eps, hosts
WHERE eps.id > ${latest_episode} WHERE eps.id > ${previous_cn_latest_episode}
AND eps.date LIKE \"${this_month}%\" AND eps.date LIKE \"${this_month}%\"
AND eps.valid = 1 AND eps.valid = 1
AND eps.hostid = hosts.hostid; AND eps.hostid = hosts.hostid;
") ")
printf "\n%s\n" "SELECT eps.id, hosts.host, eps.title, eps.date, eps.hostid
FROM eps, hosts
WHERE eps.id > ${previous_cn_latest_episode}
AND eps.date LIKE \"${this_month}%\"
AND eps.valid = 1
AND eps.hostid = hosts.hostid;" 1>&2
echo "<h2>Last Month's Shows</h2>" | tee --append "${shownotes_html}" echo "<h2>Last Month's Shows</h2>" >> "${shownotes_html}"
echo "" | tee --append "${shownotes_html}" echo "" >> "${shownotes_html}"
# [%# The id 't01' is in the HPR CSS but might give trouble on the IA -%] # [%# The id 't01' is in the HPR CSS but might give trouble on the IA -%]
echo "<table id=\"t01\">" | tee --append "${shownotes_html}" echo "<table id=\"t01\">" >> "${shownotes_html}"
echo "<tr>" | tee --append "${shownotes_html}" echo "<tr>" >> "${shownotes_html}"
echo "<th>Id</th>" | tee --append "${shownotes_html}" echo "<th>Id</th>" >> "${shownotes_html}"
echo "<th>Day</th>" | tee --append "${shownotes_html}" echo "<th>Day</th>" >> "${shownotes_html}"
echo "<th>Date</th>" | tee --append "${shownotes_html}" echo "<th>Date</th>" >> "${shownotes_html}"
echo "<th>Title</th>" | tee --append "${shownotes_html}" echo "<th>Title</th>" >> "${shownotes_html}"
echo "<th>Host</th>" | tee --append "${shownotes_html}" echo "<th>Host</th>" >> "${shownotes_html}"
echo "</tr>" | tee --append "${shownotes_html}" echo "</tr>" >> "${shownotes_html}"
if [ -z "${results}" ] if [ -z "${results}" ]
then then
echo_error "There were no shows found for this month." echo_error "There were no shows found for this month."
else else
while IFS="|" read -r eps_id host title eps_date hostid while IFS="→" read -r eps_id host title eps_date hostid
do do
printf -v hostid "%04d" "${hostid}" printf -v hostid "%04d" "${hostid}"
printf -v eps_id "%04d" "${eps_id}" printf -v eps_id "%04d" "${eps_id}"
echo "<tr>" | tee --append "${shownotes_html}" echo "<tr>" >> "${shownotes_html}"
echo "<td><strong><a href=\"https://hackerpublicradio.org/eps/hpr${eps_id}/index.html\" target=\"_blank\">${eps_id}</a></strong></td>" | tee --append "${shownotes_html}" echo "<td><strong><a href=\"https://hackerpublicradio.org/eps/hpr${eps_id}/index.html\" target=\"_blank\">${eps_id}</a></strong></td>" >> "${shownotes_html}"
echo "<td>$( \date --date=${eps_date} +%a )</td>" | tee --append "${shownotes_html}" echo "<td>$( \date --date=${eps_date} +%a )</td>" >> "${shownotes_html}"
echo "<td class=\"shrink\">${eps_date}</td>" | tee --append "${shownotes_html}" echo "<td class=\"shrink\">${eps_date}</td>" >> "${shownotes_html}"
echo "<td><a href=\"https://hackerpublicradio.org/eps/hpr${eps_id}/index.html\" target=\"_blank\">${title}</a></td>" | tee --append "${shownotes_html}" echo "<td><a href=\"https://hackerpublicradio.org/eps/hpr${eps_id}/index.html\" target=\"_blank\">${title}</a></td>" >> "${shownotes_html}"
echo "<td><a href=\"https://hackerpublicradio.org/correspondents/${hostid}.html\" target=\"_blank\">${host}</a></td>" | tee --append "${shownotes_html}" echo "<td><a href=\"https://hackerpublicradio.org/correspondents/${hostid}.html\" target=\"_blank\">${host}</a></td>" >> "${shownotes_html}"
latest_episode=${eps_id} latest_episode=${eps_id}
echo "</tr>" | tee --append "${shownotes_html}" echo "</tr>" >> "${shownotes_html}"
done <<< "${results}" done <<< "${results}"
fi fi
echo "</table>" | tee --append "${shownotes_html}" echo "</table>" >> "${shownotes_html}"
echo "" | tee --append "${shownotes_html}" echo "" >> "${shownotes_html}"
############################################################################################ ############################################################################################
# Add Comments to shownotes # Add Comments to shownotes
@@ -175,40 +198,41 @@ function render_comments() {
else else
this_comment="0" this_comment="0"
open_line="0" open_line="0"
echo "<ul>" | tee --append "${shownotes_html}" echo "<ul>" >> "${shownotes_html}"
while IFS="|" read -r comments_id comments_eps_id eps_date eps_title hosts_host hosts_hostid comments_comment_author_name comments_comment_timestamp comments_comment_title while IFS="→" read -r comments_id comments_eps_id eps_date eps_title hosts_host hosts_hostid comments_comment_author_name comments_comment_timestamp comments_comment_title
do do
if [ "${this_comment}" -ne "${comments_eps_id}" ] if [ "${this_comment}" -ne "${comments_eps_id}" ]
then then
if [ "${open_line}" -ne "0" ] if [ "${open_line}" -ne "0" ]
then then
echo "</ul>" | tee --append "${shownotes_html}" echo "</ul>" >> "${shownotes_html}"
fi fi
open_line="1" open_line="1"
this_comment="${comments_eps_id}" this_comment="${comments_eps_id}"
printf -v hosts_hostid "%04d" "${hosts_hostid}" printf -v hosts_hostid "%04d" "${hosts_hostid}"
printf -v comments_eps_id "%04d" "${comments_eps_id}" printf -v comments_eps_id "%04d" "${comments_eps_id}"
echo "<li><strong><a href=\"https://hackerpublicradio.org/eps/hpr${comments_eps_id}/index.html#comments\" target=\"_blank\">hpr${comments_eps_id}</a></strong> (${eps_date}) \"<em>${eps_title}</em>\" by <a href=\"https://hackerpublicradio.org/correspondents/${hosts_hostid}.html\" target=\"_blank\">${hosts_host}</a>.<br></li>" | tee --append "${shownotes_html}" echo "<li><strong><a href=\"https://hackerpublicradio.org/eps/hpr${comments_eps_id}/index.html#comments\" target=\"_blank\">hpr${comments_eps_id}</a></strong> (${eps_date}) \"<em>${eps_title}</em>\" by <a href=\"https://hackerpublicradio.org/correspondents/${hosts_hostid}.html\" target=\"_blank\">${hosts_host}</a>.<br></li>" >> "${shownotes_html}"
echo "<ul>" | tee --append "${shownotes_html}" echo "<ul>" >> "${shownotes_html}"
fi fi
echo "<li style=\"list-style: none; display: inline\"><strong>${comments_comment_author_name}</strong> said: \"<em><a href=\"https://hackerpublicradio.org/eps/hpr${comments_eps_id}/index.html#comment_${comments_id}\" target=\"_blank\">${comments_comment_title}</a></em>\" (${comments_comment_timestamp})<br/></li>" | tee --append "${shownotes_html}" echo "<li style=\"list-style: none; display: inline\"><strong>${comments_comment_author_name}</strong> said: \"<em><a href=\"https://hackerpublicradio.org/eps/hpr${comments_eps_id}/index.html#comment_${comments_id}\" target=\"_blank\">${comments_comment_title}</a></em>\" (${comments_comment_timestamp})<br/></li>" >> "${shownotes_html}"
latest_comment="${comments_id}" latest_comment="${comments_id}"
done <<< "${results}" done <<< "${results}"
echo "</ul>" | tee --append "${shownotes_html}" echo "</ul>" >> "${shownotes_html}"
echo "</ul>" | tee --append "${shownotes_html}" echo "</ul>" >> "${shownotes_html}"
fi fi
} }
# ------------------------------------------- # -------------------------------------------
echo "<h2>Comments this month</h2>" | tee --append "${shownotes_html}" echo "<h2>Comments this month</h2>" >> "${shownotes_html}"
echo "" | tee --append "${shownotes_html}" echo "" >> "${shownotes_html}"
echo "<h3>Past shows</h3>" | tee --append "${shownotes_html}" echo "<h3>Past shows</h3>" >> "${shownotes_html}"
latest_comment_last_changed=$(sqlite3 hpr.db "SELECT last_changed FROM comments WHERE comments.id = ${latest_comment};") previous_cn_latest_comment_last_changed=$(sqlite3 "${hpr_db}" "SELECT last_changed FROM comments WHERE comments.id = ${previous_cn_latest_comment};")
printf "\n%s\n" "SELECT last_changed FROM comments WHERE comments.id = ${previous_cn_latest_comment};" 1>&2
results=$(sqlite3 hpr.db " results=$(sqlite3 -separator '→' "${hpr_db}" "
SELECT SELECT
comments.id, comments.id,
comments.eps_id, comments.eps_id,
@@ -222,16 +246,34 @@ SELECT
FROM comments FROM comments
JOIN eps ON comments.eps_id = eps.id JOIN eps ON comments.eps_id = eps.id
JOIN hosts ON eps.hostid = hosts.hostid JOIN hosts ON eps.hostid = hosts.hostid
WHERE comments.last_changed > \"${latest_comment_last_changed}\" WHERE comments.id > "${previous_cn_latest_comment}"
AND eps.date <= \"${this_month}-01\" AND eps.id <= "${previous_cn_latest_episode}"
ORDER BY eps.date; ORDER BY eps.date;
") ")
printf "\n%s\n" "
SELECT
comments.id,
comments.eps_id,
eps.date,
eps.title,
hosts.host,
hosts.hostid,
comments.comment_author_name,
comments.comment_timestamp,
comments.comment_title
FROM comments
JOIN eps ON comments.eps_id = eps.id
JOIN hosts ON eps.hostid = hosts.hostid
WHERE comments.id > "${previous_cn_latest_comment}"
AND eps.id <= "${previous_cn_latest_episode}"
ORDER BY eps.date;
" 1>&2
render_comments "${results}" render_comments "${results}"
echo "<h3>This month's shows</h3>" | tee --append "${shownotes_html}" echo "<h3>This month's shows</h3>" >> "${shownotes_html}"
results=$(sqlite3 hpr.db " results=$(sqlite3 -separator '→' "${hpr_db}" "
SELECT SELECT
comments.id, comments.id,
comments.eps_id, comments.eps_id,
@@ -248,6 +290,21 @@ JOIN hosts ON eps.hostid = hosts.hostid
WHERE eps.date LIKE \"${this_month}-%\" WHERE eps.date LIKE \"${this_month}-%\"
ORDER BY eps.date; ORDER BY eps.date;
") ")
printf "\n%s\n" "SELECT
comments.id,
comments.eps_id,
eps.date,
eps.title,
hosts.host,
hosts.hostid,
comments.comment_author_name,
comments.comment_timestamp,
comments.comment_title
FROM comments
JOIN eps ON comments.eps_id = eps.id
JOIN hosts ON eps.hostid = hosts.hostid
WHERE eps.date LIKE \"${this_month}-%\"
ORDER BY eps.date;" 1>&2
render_comments "${results}" render_comments "${results}"
@@ -255,7 +312,7 @@ render_comments "${results}"
# Add maillist and cal to shownotes # Add maillist and cal to shownotes
mailman_link="https://lists.hackerpublicradio.com/pipermail/hpr/$( \date -d"${month}-01 next month" +%Y-%B )/thread.html" mailman_link="https://lists.hackerpublicradio.com/pipermail/hpr/$( \date -d"${month}-01 next month" +%Y-%B )/thread.html"
echo "<h2>Mailing List discussions</h2>" | tee --append "${shownotes_html}" echo "<h2>Mailing List discussions</h2>" >> "${shownotes_html}"
echo "<p>Policy decisions surrounding HPR are taken by the community as a whole. This echo "<p>Policy decisions surrounding HPR are taken by the community as a whole. This
discussion takes place on the <a href=\"https://lists.hackerpublicradio.com/mailman/listinfo/hpr\" target=\"_blank\">Mailing List</a> discussion takes place on the <a href=\"https://lists.hackerpublicradio.com/mailman/listinfo/hpr\" target=\"_blank\">Mailing List</a>
which is open to all HPR listeners and contributors. The discussions are open which is open to all HPR listeners and contributors. The discussions are open
@@ -271,10 +328,11 @@ and available on the HPR server under <a href=\"https://lists.hackerpublicradio.
<blockquote>This is the LWN.net community event calendar, where we track <blockquote>This is the LWN.net community event calendar, where we track
events of interest to people using and developing Linux and free software. events of interest to people using and developing Linux and free software.
Clicking on individual events will take you to the appropriate web Clicking on individual events will take you to the appropriate web
page.</blockquote>" | tee --append "${shownotes_html}" page.</blockquote>" >> "${shownotes_html}"
latest_comment=$(sqlite3 hpr.db "SELECT MAX(id) FROM comments;") latest_comment=$(sqlite3 "${hpr_db}" "SELECT MAX(id) FROM comments;")
echo "<!-- HPRCN:month=${this_month},latest_comment=${latest_comment},latest_host=${newest_host},latest_episode=${latest_episode} -->" | tee --append "${shownotes_html}" printf "\n%s\n" "" 1>&2
echo "<!-- HPRCN:month=${this_month},latest_comment=${latest_comment},latest_host=${newest_host},latest_episode=${latest_episode} -->" >> "${shownotes_html}"
cp -v "${shownotes_html}" "${shownotes_html%.*}"_edited.html cp -v "${shownotes_html}" "${shownotes_html%.*}"_edited.html
ls -al "${eps_dir}" ls -al "${eps_dir}"

99
workflow/hpr-get-and-transcribe.bash
View File

@@ -1,35 +1,86 @@
#!/bin/bash #!/bin/bash
rsync -av --partial --progress hpr:hub.hackerpublicradio.org/upload/ $HOME/tmp/hpr/processing/ destination_dir="$HOME/tmp/hpr/processing/"
server_dir="hub.hackerpublicradio.org"
find $HOME/tmp/hpr/processing/*_*_????-??-??_* -type d | sort -t _ -k 2 | while read show_dir #################################################
do # Display Error message, display help and exit
echo "${show_dir}"
if [ "$( find "${show_dir}" -type f -iname "*srt" | wc -l )" -eq "0" ] function echo_error() {
echo -e "ERROR: $@" #1>&2
display_help_and_exit
exit 1
}
#################################################
# Display Information message
function echo_debug() {
if [ "${debug}" != "0" ]
then then
cd "${show_dir}" echo -e "INFO: $@" #1>&2
ls -haltr
find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media
do
whisper --model tiny --language en --output_dir "${show_dir}" "${this_media}"
done
rsync -av --partial --progress "${show_dir}/" hpr:hub.hackerpublicradio.org/upload/$( basename "${show_dir}")/
fi fi
done }
rsync -av --partial --progress hpr:hub.hackerpublicradio.org/reserve/ $HOME/tmp/hpr/reserve/ #################################################
# Display Help
find $HOME/tmp/hpr/reserve/*_*_* -type d | sort -t _ -k 2 | while read show_dir function display_help_and_exit() {
echo_debug "For more information see https://repo.anhonesthost.net/HPR/hpr_documentation"
exit 1
}
#################################################
# main
if [ "$( curl --location --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${destination_dir}/status.csv" )" != 200 ]
then
echo_error "Could not get a list of the queue status from \"https://hub.hackerpublicradio.org/cms/status.php\""
fi
if [ ! -s "${destination_dir}/status.csv" ]
then
echo_error "Failed to retrieve \"${destination_dir}/status.csv\" from server."
fi
response=$( cat "${destination_dir}/status.csv" | grep --extended-regexp ',SHOW_SUBMITTED,|,RESERVE_SHOW_SUBMITTED,' | sort -t ',' -k 2 )
if [ -z "${response}" ]
then
echo_debug "Getting a list of all the reservations."
curl --location --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | sort -n
echo_error "There appear to be no more shows with the status \"SHOW_SUBMITTED\"."
fi
printf '%s\n' ${response} | while read this_line
do do
echo "${show_dir}" IFS=, read -r timestamp_epoc ep_num ep_date key status email <<< "$( printf '%s\n' "${this_line}" )"
if [ "$( find "${show_dir}" -type f -iname "*srt" | wc -l )" -eq "0" ]
if [ "${ep_num}" == "9999" ]
then then
cd "${show_dir}" source_dir="${server_dir}/reserve/"
ls -haltr else
find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media source_dir="${server_dir}/upload/"
do
whisper --model tiny --language en --output_dir "${show_dir}" "${this_media}"
done
rsync -av --partial --progress "${show_dir}/" hpr:hub.hackerpublicradio.org/reserve/$( basename "${show_dir}")/
fi fi
if [ "$( find "${destination_dir}" -maxdepth 2 -mindepth 2 -type f -iname "*${key}*.srt" | wc --lines )" -ne "0" ]
then
echo_debug "The episode \"${timestamp_epoc} ${ep_num} ${ep_date} ${key} ${status} ${email}\" has been processed."
continue
fi
show_path="$( ssh -n hpr -t "find \"${source_dir}\" -maxdepth 1 -mindepth 1 -type d -iname \"*${key}*\"" 2>/dev/null | sed "s#${source_dir}##g#" )"
if [ -z "${show_path}" ]
then
echo_error "The show_path could not be found."
fi
show_dir="${destination_dir}/${show_path}"
echo_debug "Copying \"hpr:${source_dir}/${show_path}/\" to \"${show_dir}/\""
rsync --archive --verbose --partial --progress hpr:${source_dir}/${show_path}/ ${show_dir}/
cd "${show_dir}"
pwd
find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media
do
whisper --output_format srt --model tiny --language en --output_dir "${show_dir}" "${this_media}"
done
rsync --archive --verbose --partial --progress ${show_dir}/ hpr:${source_dir}/${show_path}/
done done

0
workflow/process_episode.bash → workflow/hpr-process_episode.bash
View File