Files
hpr-tools/workflow/hpr-cn-generate-notes.bash
T

520 lines
21 KiB
Bash
Executable File

#!/usr/bin/env bash
# https://brycebeattie.com/files/tts/
#piper_voice="/opt/bin/piper/piper/piper-voices/en/en_US/lessac/medium/en_US-lessac-medium.onnx"
piper_voice="/opt/bin/piper/piper/piper-voices/en/en_US/kristin/medium/en_US-kristin-medium.onnx"
############################################################################################
# Update db
source settings $1
git_dir="$HOME/tmp/hpr/hpr_generator"
if [ ! -d "${git_dir}/.git" ]
then
#git clone gitea@repo.anhonesthost.net:rho_n/hpr_generator.git "${git_dir}"
git clone gitea@repo.anhonesthost.net:HPR/hpr_generator.git "${git_dir}"
fi
if [ ! -d "${git_dir}" ]
then
echo_error "The git dir \"${git_dir}\" is missing"
fi
cd "${git_dir}"
git pull
# # # # ssh hpr -t "ls -al /docker/users/hpr/userfiles/sql/hpr.sql;md5sum /docker/users/hpr/userfiles/sql/hpr.sql"
# # # # ssh hpr -t "/docker/users/hpr/userfiles/sql/hpr_db_backup.bash"
# # # # ssh hpr -t "ls -al /docker/users/hpr/userfiles/sql/hpr.sql;md5sum /docker/users/hpr/userfiles/sql/hpr.sql"
# # # #
# # # # #$HOME/tmp/hpr/hpr_generator/utils/update-hpr-db.sh
# # # # ./utils/update-hpr-db.sh
# # # # if [ $? -ne 0 ]
# # # # then
# # # # echo 'Terminating...' >&2
# # # # exit 1
# # # # fi
hpr_db="${git_dir}/hpr.db"
if [[ ! -s "${hpr_db}" || -z "${hpr_db}" ]]
then
echo_error "The \"hpr_db\" variable/file is missing."
fi
if [ "$( file --brief --mime-type "${hpr_db}" | grep --count 'application/vnd.sqlite3' )" -ne "1" ]
then
echo_error "The \"hpr_db\" variable has not a valid \"application/vnd.sqlite3\" mime type."
fi
############################################################################################
# Get previous show info
row=$(sqlite3 "${hpr_db}" "
SELECT notes
FROM eps
WHERE duration > 0
AND series = 47
ORDER BY date DESC
LIMIT 1;
")
# <!-- HPRCN:month=2026-10,latest_comment=4952,latest_host=464,latest_episode=4760 -->
eval $(
echo "$row" |
grep -o '<!-- HPRCN:[^>]*-->' |
sed -E 's/<!-- HPRCN:(.*) -->/\1/' |
tr ',' '\n'
)
previous_month="${month}"
this_month="$( \date --date "${previous_month}-01 + 1 month" "+%Y-%m" )"
previous_cn_latest_comment="${latest_comment}"
previous_cn_latest_episode="${latest_episode}"
previous_latest_host="${latest_host}"
end_of_month_episode_date="$( \date --date "${this_month}-01 + 1 month - 1 day" "+%Y-%m-%d" )"
echo_debug "The last show month was for ${previous_month}, and this month is ${this_month}, previous_cn_latest_comment: ${previous_cn_latest_comment}, latest_host: ${previous_latest_host}, and previous_cn_latest_episode: ${previous_cn_latest_episode}, end_of_month_episode_date: ${end_of_month_episode_date}"
############################################################################################
# Get last show info
IFS='|' read -r end_of_month_episode_number <<< "$(sqlite3 "${hpr_db}" "
SELECT MAX(id) AS end_of_month_episode_number
FROM eps
WHERE date <= \"${end_of_month_episode_date}\"
LIMIT 1;
")"
echo_debug "Last episode this month is hpr${end_of_month_episode_number} released on ${end_of_month_episode_date}"
############################################################################################
# Get next show info
this_month=$( \date --date="${month}-01 +1 month" +%Y-%m )
IFS='|' read -r eps_num eps_date <<< "$(sqlite3 "${hpr_db}" "
SELECT id, date
FROM eps
WHERE duration = 0
AND series = 47
ORDER BY date ASC
LIMIT 1;
")"
echo_debug "This is going to be hpr${eps_num} for release on ${eps_date}"
eps_dir="$HOME/processing/x_${eps_num}_${eps_date}_x"
if [ ! -d "${eps_dir}" ]
then
mkdir -vp "${eps_dir}"
fi
echo_debug "The episode dir is \"${eps_dir}\""
shownotes_html="${eps_dir}/shownotes.html"
if [ -e "${shownotes_html}" ]
then
rm -v "${shownotes_html}"
fi
shownotes_html_edited="${shownotes_html%.*}"_edited.html
if [ -e "${shownotes_html_edited}" ]
then
rm -v "${shownotes_html_edited}"
fi
shownotes_recordings_dir="${eps_dir}/Recordings"
comment_script_md="${eps_dir}/script.md"
if [ -e "${comment_script_md}" ]
then
rm -v "${comment_script_md}"
fi
if [ ! -d "${shownotes_recordings_dir}" ]
then
mkdir -vp "${shownotes_recordings_dir}"
fi
############################################################################################
# Add New hosts to shownotes
results=$(sqlite3 "${hpr_db}" "SELECT hostid, host FROM hosts WHERE hostid > ${latest_host};")
echo "<h2>New hosts</h2>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<p>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
if [ -z "${results}" ]
then
echo "There were no new hosts this month." | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
newest_host="${latest_host}"
else
echo "Welcome to our new hosts: <br>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<ul>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
while IFS="|" read -r hostid host
do
newest_host="${hostid}"
echo "<li><a href=\"https://hackerpublicradio.org/correspondents/${hostid}.html\" target=\"_blank\">${host}</a></li>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
done <<< "${results}"
echo "</ul>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
fi
echo "</p>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
############################################################################################
# Add Last Month's Shows to shownotes
results=$(sqlite3 -separator '→' "${hpr_db}" "
SELECT eps.id, hosts.host, eps.title, eps.date, eps.hostid
FROM eps, hosts
WHERE eps.id > ${previous_cn_latest_episode}
AND eps.date LIKE \"${this_month}%\"
AND eps.valid = 1
AND eps.hostid = hosts.hostid;
")
echo "<h2>Last Month's Shows</h2>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
# [%# The id 't01' is in the HPR CSS but might give trouble on the IA -%]
echo "<table id=\"t01\">" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<tr>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<th>Id</th>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<th>Day</th>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<th>Date</th>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<th>Title</th>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<th>Host</th>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "</tr>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
if [ -z "${results}" ]
then
echo_error "There were no shows found for this month."
else
while IFS="→" read -r eps_id host title eps_date hostid
do
echo "<tr>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<td><strong><a href=\"https://hackerpublicradio.org/eps/hpr${eps_id}/index.html\" target=\"_blank\">${eps_id}</a></strong></td>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<td>$( \date --date=${eps_date} +%a )</td>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<td class=\"shrink\">${eps_date}</td>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<td><a href=\"https://hackerpublicradio.org/eps/hpr${eps_id}/index.html\" target=\"_blank\">${title}</a></td>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<td><a href=\"https://hackerpublicradio.org/correspondents/${hostid}.html\" target=\"_blank\">${host}</a></td>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
latest_episode=${eps_id}
echo "</tr>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
done <<< "${results}"
fi
echo "</table>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
############################################################################################
# Add Comments to shownotes
function render_comments() {
results=${1}
if [ -z "${results}" ]
then
echo_error "There were no comments found for this month."
else
this_comment="0"
open_line="0"
echo "<ul>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
while IFS="→" read -r comments_id comments_eps_id eps_date eps_title hosts_host hosts_hostid comments_comment_author_name comments_comment_timestamp comments_comment_title
do
echo "# Comment${comments_id}Vvvvvvvvvvvvvvvvvvvvvvvvvvvv" | tee --append "${comment_script_md}"
echo "${eps_title} by ${hosts_host}" | tee --append "${comment_script_md}"
echo "- https://hackerpublicradio.org/eps/hpr${comments_eps_id}/index.html#comment_${comments_id} (${eps_date})" | tee --append "${comment_script_md}"
echo "- https://hackerpublicradio.org/correspondents/${hosts_hostid}.html" | tee --append "${comment_script_md}"
comment_script_txt="${shownotes_recordings_dir}/Comment${comments_id}.txt"
comment_script_piper="${shownotes_recordings_dir}/Comment${comments_id}_piper.wav"
comment_script_espeak="${shownotes_recordings_dir}/Comment${comments_id}_espeak.wav"
echo "${comments_comment_author_name} said: «${comments_comment_title}»" | tee "${comment_script_txt}"
sqlite3 "${hpr_db}" "SELECT comments.comment_text AS comment_text FROM comments WHERE comments.id = ${comments_id};" | tee --append "${comment_script_txt}"
echo "${comments_comment_text}" | tee --append "${comment_script_txt}"
echo "" | tee --append "${comment_script_txt}"
cat "${comment_script_txt}" >> "${comment_script_md}"
if [ ! -e "${comment_script_piper}" ]
then
echo_debug "Genereating piper \"${comment_script_piper}\""
piper --model "${piper_voice}" --input-file "${comment_script_txt}" --output_file "${comment_script_piper}"
fi
if [ ! -e "${comment_script_espeak}" ]
then
echo_debug "Genereating espeak \"${comment_script_espeak}\""
espeak -f "${comment_script_txt}" -w "${comment_script_espeak}"
fi
if [ "${this_comment}" -ne "${comments_eps_id}" ]
then
if [ "${open_line}" -ne "0" ]
then
echo "</ul>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
fi
open_line="1"
this_comment="${comments_eps_id}"
echo "<li><strong><a href=\"https://hackerpublicradio.org/eps/hpr${comments_eps_id}/index.html#comments\" target=\"_blank\">hpr${comments_eps_id}</a></strong> (${eps_date}) \"<em>${eps_title}</em>\" by <a href=\"https://hackerpublicradio.org/correspondents/${hosts_hostid}.html\" target=\"_blank\">${hosts_host}</a>.<br></li>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<ul>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
fi
echo "<li style=\"list-style: none; display: inline\"><strong>${comments_comment_author_name}</strong> said: \"<em><a href=\"https://hackerpublicradio.org/eps/hpr${comments_eps_id}/index.html#comment_${comments_id}\" target=\"_blank\">${comments_comment_title}</a></em>\" (${comments_comment_timestamp})<br/></li>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
comment_script_txt="${shownotes_recordings_dir}/Comment${comments_id}.txt"
comment_script_piper="${shownotes_recordings_dir}/Comment${comments_id}_piper.wav"
comment_script_espeak="${shownotes_recordings_dir}/Comment${comments_id}_espeak.wav"
comment_script_narration="${shownotes_recordings_dir}/Comment${comments_id}Vvvvvvvvvvvvvvvvvvvvvvvvvvv.wav"
echo "<ul>" | tee --append "${shownotes_html_edited}"
if [ -e "${comment_script_narration}" ]
then
comment_script_narration_web="./Recordings/$( basename "${comment_script_narration}" )"
echo "<li><audio controls=\"\" preload=\"none\" >
<source src=\"${comment_script_narration_web}\" type=\"audio/x-wav\">
</audio>" | tee --append "${shownotes_html_edited}"
fi
if [ -e "${comment_script_piper}" ]
then
comment_script_piper_web="./Recordings/$( basename "${comment_script_piper}" )"
echo "<audio controls=\"\" preload=\"none\" >
<source src=\"${comment_script_piper_web}\" type=\"audio/x-wav\">
</audio>" | tee --append "${shownotes_html_edited}"
fi
if [ -e "${comment_script_espeak}" ]
then
comment_script_espeak_web="./Recordings/$( basename "${comment_script_espeak}" )"
echo "<audio controls=\"\" preload=\"none\" >
<source src=\"${comment_script_espeak_web}\" type=\"audio/x-wav\">
</audio></li>" | tee --append "${shownotes_html_edited}"
fi
echo "</ul>" | tee --append "${shownotes_html_edited}"
latest_comment="${comments_id}"
done <<< "${results}"
echo "</ul>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "</ul>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
fi
}
# -------------------------------------------
echo "<h2>Comments this month</h2>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<h3>Past shows</h3>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
previous_cn_latest_comment_last_changed=$(sqlite3 "${hpr_db}" "SELECT last_changed FROM comments WHERE comments.id = ${previous_cn_latest_comment};")
results=$(sqlite3 -separator '→' "${hpr_db}" "
SELECT
comments.id,
comments.eps_id,
eps.date,
eps.title,
hosts.host,
hosts.hostid,
comments.comment_author_name,
comments.comment_timestamp,
comments.comment_title
FROM comments
JOIN eps ON comments.eps_id = eps.id
JOIN hosts ON eps.hostid = hosts.hostid
WHERE comments.id > "${previous_cn_latest_comment}"
AND eps.id <= "${previous_cn_latest_episode}"
ORDER BY eps.date;
")
render_comments "${results}"
echo "<h3>This month's shows</h3>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
results=$(sqlite3 -separator '→' "${hpr_db}" "
SELECT
comments.id,
comments.eps_id,
eps.date,
eps.title,
hosts.host,
hosts.hostid,
comments.comment_author_name,
comments.comment_timestamp,
comments.comment_title
FROM comments
JOIN eps ON comments.eps_id = eps.id
JOIN hosts ON eps.hostid = hosts.hostid
WHERE eps.date LIKE \"${this_month}-%\"
ORDER BY eps.date;
")
render_comments "${results}"
############################################################################################
# Add maillist and cal to shownotes
mailman_base="https://lists.hackerpublicradio.com/pipermail/hpr/$( \date -d"${this_month}-01" +%Y-%B )"
mailman_link="${mailman_base}/thread.html"
wget "${mailman_link}" -O thread.html
grep --perl-regexp -- 'A HREF' thread.html | awk -F '"' '{print $2}' | while read this_post
do
this_post_html="${shownotes_recordings_dir}/Post${this_post%.*}.html"
if [ ! -e ${this_post_html} ]
then
wget ${mailman_base}/${this_post} --output-document=${this_post_html}
fi
this_post_txt="${shownotes_recordings_dir}/Post${this_post%.*}.txt"
if [ ! -e ${this_post_txt} ]
then
echo "# Post${this_post%.*}vvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" > "${this_post_txt}"
grep ' <B>' "${this_post_html}" | tee --append "${this_post_txt}"
sed --quiet --expression='/<!--beginarticle-->/,/<!--endarticle-->/{ /<!--beginarticle-->/d; /<!--endarticle-->/d; p; }' "${this_post_html}" | tee --append "${this_post_txt}"
sed -e 's#^ <B>##g' -e 's#</B># says:#g' -e 's#<PRE>##g' -e 's#</PRE>##g' -i "${this_post_txt}"
fi
done
for this_post in ${shownotes_recordings_dir}/Post*txt
do
this_post_name="$( basename "${this_post}" )"
if [ "$( find ${shownotes_recordings_dir} -type f -iname "${this_post_name%.*}*.wav" | wc -l )" -eq "0" ]
then
kate -n "${this_post}" >/dev/null 2>&1 &
fi
done
if [ "$( ps -ef | grep kate | grep Post | wc -l )" -ne "0" ]
then
read -p "Were the posts edited ? (N|y) ? " -n 1 -r
echo # (optional) move to a new line
if [[ ! $REPLY =~ ^[yY]$ ]]
then
echo_error "The final review was not approved."
fi
fi
for this_post in ${shownotes_recordings_dir}/Post*txt
do
echo " " >> ${comment_script_md}
cat ${this_post} >> ${comment_script_md}
post_id="$( basename "${this_post%.*}" )"
post_script_txt="${shownotes_recordings_dir}/${post_id}.txt"
post_script_piper="${shownotes_recordings_dir}/${post_id}_piper.wav"
post_script_espeak="${shownotes_recordings_dir}/${post_id}_espeak.wav"
if [ ! -e "${post_script_piper}" ]
then
echo_debug "Genereating piper \"${post_script_piper}\""
piper --model "${piper_voice}" --input-file "${post_script_txt}" --output_file "${post_script_piper}"
fi
if [ ! -e "${post_script_espeak}" ]
then
echo_debug "Genereating espeak \"${post_script_espeak}\""
espeak -f "${post_script_txt}" -w "${post_script_espeak}"
fi
done
echo "<h2>Mailing List discussions</h2>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<p>Policy decisions surrounding HPR are taken by the community as a whole. This
discussion takes place on the <a href=\"https://lists.hackerpublicradio.com/mailman/listinfo/hpr\" target=\"_blank\">Mailing List</a>
which is open to all HPR listeners and contributors. The discussions are open
and available on the HPR server under <a href=\"https://lists.hackerpublicradio.com/pipermail/hpr\">Mailman</a>.
</p>
<p>The threaded discussions this month can be found here:</p>
<a href=\"${mailman_link}\" target=\"_blank\">${mailman_link}</a>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<UL>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
sed --null-data "s/\n//g" thread.html | \
sed --expression='s/<LI>/\n<LI>/g' --expression='s#</LI>#</LI>\n#g' --expression='s#<UL>#\n<UL>\n#g' --expression='s#</UL>#\n</UL>\n#g' --expression='s#<ul>#\n<ul>\n#g' --expression='s#</ul>#\n</ul>\n#g' | \
grep --perl-regexp -- '<LI>|</LI>|<UL>|</UL>' | while read this_post_line
do
if [ "$( echo "${this_post_line}" | grep --count '<LI>' )" ]
then
echo "${this_post_line}</LI>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
### <LI><A HREF="005073.html">[Hpr] Community News Clone Voices</A><A NAME="5073">&nbsp;</A><I>candy</I><!--1 01793572759.5072-01793614515.5074- --></LI>
post_id="$( echo "${this_post_line}" | awk -F '"' '{print "Post"$2}' | sed 's/\.html//g' )"
post_script_narration="${shownotes_recordings_dir}/${post_id}Vvvvvvvvvvvvvvvvvvvvvvvvvvvv.wav"
post_script_piper="${shownotes_recordings_dir}/${post_id}_piper.wav"
post_script_espeak="${shownotes_recordings_dir}/${post_id}_espeak.wav"
echo "<ul>" | tee --append "${shownotes_html_edited}"
if [ -e "${post_script_narration}" ]
then
post_script_narration_web="./Recordings/$( basename "${post_script_narration}" )"
echo "<li><audio controls=\"\" preload=\"none\" >
<source src=\"${post_script_narration_web}\" type=\"audio/x-wav\">
</audio>" | tee --append "${shownotes_html_edited}"
fi
if [ -e "${post_script_piper}" ]
then
post_script_piper_web="./Recordings/$( basename "${post_script_piper}" )"
echo "<audio controls=\"\" preload=\"none\" >
<source src=\"${post_script_piper_web}\" type=\"audio/x-wav\">
</audio>" | tee --append "${shownotes_html_edited}"
fi
if [ -e "${post_script_espeak}" ]
then
post_script_espeak_web="./Recordings/$( basename "${post_script_espeak}" )"
echo "<audio controls=\"\" preload=\"none\" >
<source src=\"${post_script_espeak_web}\" type=\"audio/x-wav\">
</audio></li>" | tee --append "${shownotes_html_edited}"
fi
echo "</ul>" | tee --append "${shownotes_html_edited}"
else
echo "${this_post_line}" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
fi
done
echo "</UL>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
echo "<h2>Events Calendar</h2>
<p>With the kind permission of <strong>LWN.net</strong> we are linking to
<a href=\"https://lwn.net/Calendar/\" target=\"_blank\">The LWN.net Community Calendar</a>.</p>
<p>Quoting the site:</p>
<blockquote>This is the LWN.net community event calendar, where we track
events of interest to people using and developing Linux and free software.
Clicking on individual events will take you to the appropriate web
page.</blockquote>" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
latest_comment=$(sqlite3 "${hpr_db}" "SELECT MAX(id) FROM comments;")
echo "<!-- HPRCN:month=${this_month},latest_comment=${latest_comment},latest_host=${newest_host},latest_episode=${latest_episode} -->" | tee --append "${shownotes_html_edited}" >> "${shownotes_html}"
cd "${eps_dir}"
while [ "$( $HOME/sourcecode/hpr/solocast/solocast/solocast.py status | grep --count 'All Segments Recorded' )" == "0" ]
do
$HOME/sourcecode/hpr/solocast/solocast/solocast.py record
$HOME/sourcecode/hpr/solocast/solocast/solocast.py status
sleep 1
done
# # # for audio_wav in "${shownotes_recordings_dir}"/*.wav
# # # do
# # # if [ ! -e "${audio_wav%.*}.mp3" ]
# # # then
# # # ffmpeg -i "${audio_wav}" "${audio_wav%.*}.mp3"
# # # fi
# # # done
# # #
# # #