HPR/hpr-tools
8
0
Fork 2
Code Issues 13 Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: HPR:65d3a36818ae37d45649e40e98052feb1ce4769b
Branches Tags
HPR:main
...
compare: HPR:main
Branches Tags
HPR:main

7 Commits
65d3a36818 ... main

Author SHA1 Message Date
Ken Fallon
8efc6459c3 Correct location for the new server 2025-09-15 20:15:59 +02:00
Ken Fallon
2b4ef438b3 Added SQL rss feed, Fixed regex for image, and scp files to hpr server 2025-09-11 17:50:31 +02:00
Ken Fallon
16afecfb82 Automating sql generation on db change 2025-09-10 21:23:37 +02:00
Ken Fallon
27bb3be4af Move HPR/hpr_hub#88 2025-09-08 09:49:49 +02:00
Ken Fallon
1a69042b24 added check for ffmpeg 2025-08-29 12:21:07 +02:00
Dave Morriss
f56da62e05 Merge branch 'main' of repo.anhonesthost.net:HPR/hpr-tools 2025-08-20 23:20:14 +01:00
Dave Morriss
75d20cc081 Template update for 'make_shownotes' 
Community_News/make_email: minor POD changes

Community_News/make_shownotes: minor POD changes

Community_News/shownote_template.tpl: now a copy of the latest template
    rather than a symbolic linkl to it

Community_News/shownote_template13.tpl: new template which doesn't
    filter 'comment_author_name' with 'html_entity'
 2025-08-20 23:15:54 +01:00
12 changed files with 806 additions and 22 deletions
Expand all files Collapse all files

10
Community_News/make_email
View File

@@ -1617,24 +1617,24 @@ Reports the following (as well as the data for level 1):
=over 4
=item .
=item *
Details of the start date chosen
=item .
=item *
Details of the year, name of month, readable date, and recording start and end
times.
=item .
=item *
The subject line chosen for the email.
=item .
=item *
The date of the show being searched for in the database.
=item .
=item *
The number of the show found in the database.

13
Community_News/make_shownotes
View File

@@ -2115,7 +2115,7 @@ cache:
Here we need the last recording date for the show reviewing HPR shows in July
2025. The date and time for this recording was in early August (Friday before
the first Monday of September 2025-09-01), as shown. This combination will
the first Monday of August, 2025-08-01), as shown. This combination will
result in the addition of the following line to the cache file:
2025-07-01,2025-08-01 15:00:00
@@ -2311,7 +2311,7 @@ default template.
=item B<New Hosts>
These are formatted as a list of links to the B<hostid> with the host's name.
These are formatted as a list of links, each to the B<hostid> with the host's name.
=item B<Shows>
@@ -2321,10 +2321,11 @@ host name is a link to the host page on the website.
=item B<Comments>
These are formatted with <article> tags separated by horizontal lines.
A <header> shows the author name and title and a <footer> displays a link to
the show and the show's host and the show title is also included. Where
relevant, the body of the article contains the comment text with line breaks.
These are formatted with B<<article>> tags separated by horizontal lines.
A B<<header>> shows the author name and title and a B<<footer>> displays
a link to the show and the show's host and the show title is also included.
Where relevant, the body of the article contains the comment text with line
breaks.
=item B<Mailing list discussions>

1
Community_News/shownote_template.tpl
View File

@@ -1 +0,0 @@
shownote_template12.tpl

223
Community_News/shownote_template.tpl Normal file
View File

@@ -0,0 +1,223 @@
[%# shownote_template13.tpl Updated: 2025-08-20 -%]
[%# -------------------------------------------------------------------------------- -%]
[%# Makes either an HTML snippet for insertion into the database or a full -%]
[%# listing with full comments for circulation to the hosts recording the episode -%]
[%# This one uses the new format for the mailing list data, and partitions -%]
[%# comments into past and current. It also marks comments that don't need -%]
[%# to be read when mark_comments is true. It requires make_shownotes >= V0.0.30 -%]
[%# -------------------------------------------------------------------------------- -%]
[%- USE date -%]
[%- USE pad4 = format('%04d') -%]
[%- correspondents = "https://hackerpublicradio.org/correspondents"
mailinglist = "https://lists.hackerpublicradio.com/mailman/listinfo/hpr"
mailbase="https://lists.hackerpublicradio.com/pipermail/hpr"
mailthreads = "$mailbase/$review_year-$review_month/thread.html" -%]
[%- DEFAULT skip_comments = 0
mark_comments = 0
ctext = 0
ignore_count = 0
missed_count = 0
past_count = 0
-%]
[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
[%# Embedded CSS. The 'table' and 'hr' settings are always there but the rest is -%]
[%# only for when we are marking comments -%]
<style>
table td.shrink {
white-space:nowrap
}
hr.thin {
border: 0;
height: 0;
border-top: 1px solid rgba(0, 0, 0, 0.1);
border-bottom: 1px solid rgba(255, 255, 255, 0.3);
}
[%- IF mark_comments == 1 %]
p#ignore, li#ignore {
background-color: lightgreen;
color:maroon;
}
div#highlight {
border-style: solid;
border-color: red;
padding-right: 20px;
padding-left: 20px;
}
[%- END %]
</style>
[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
<h2>New hosts</h2>
<p>
[% IF hosts.size > 0 -%]
Welcome to our new host[%- hosts.size > 1 ? 's' : '' -%]: <br />
[%- count = 0 %]
[%# List the new hosts. If a name contains a comma quote it. -%]
[%- FOREACH row IN hosts %]
[%- count = count + 1 %]
[%- hostname = (row.host.search(',') ? row.host.replace('^(.*)$','"$1"') : row.host) %]
<a href="[% correspondents %]/[% pad4(row.hostid) %].html" target="_blank">[% hostname %]</a>
[%- count < hosts.size ? ', ' : '.' %]
[%- END %]
[% ELSE -%]
There were no new hosts this month.
[% END -%]
</p>
<h2>Last Month's Shows</h2>
[%# The id 't01' is in the HPR CSS but might give trouble on the IA -%]
<table id="t01">
<tr>
<th>Id</th>
<th>Day</th>
<th>Date</th>
<th>Title</th>
<th>Host</th>
</tr>
[%- FOREACH row IN shows %]
<tr>
<td><strong><a href="https://hackerpublicradio.org/eps/hpr[% pad4(row.eps_id) %]/index.html" target="_blank">[% row.eps_id %]</a></strong></td>
<td>[% date.format(row.date,'%a') %]</td>
<td class="shrink">[% date.format(row.date,'%Y-%m-%d') %]</td>
<td><a href="https://hackerpublicradio.org/eps/hpr[% pad4(row.eps_id) %]/index.html" target="_blank">[% row.title %]</a></td>
<td><a href="[% correspondents %]/[% pad4(row.ho_hostid) %].html" target="_blank">[% row.ho_host %]</a></td>
</tr>
[%- END %]
</table>
[%# ---------------------------------------------------------------------------------------- -%]
[%# Skip comments if told to by the caller -%]
[%- IF skip_comments == 0 -%]
[%# Handle any missed comments if mark_comments is true -%]
[%- IF (mark_comments == 1) AND (missed_count > 0) -%]
<br/><div id="highlight">
<h2>Missed comment[%- missed_comments.size > 1 ? 's' : '' -%] last month</h2>
<p><b>Note to Volunteers</b>: These are comments for shows last month that were not read in the last show because they arrived on or after the recording started. This section will be removed before these notes are released.</p>
<ul>
[%- FOREACH comment IN missed_comments -%]
<li><strong><a href="[% comment.identifier_url %]#comments" target="_blank">hpr[% pad4(comment.episode) %]</a></strong>
([% comment.date %]) "<em>[% comment.title %]</em>" by <a href="[% correspondents %]/[% pad4(comment.hostid) %].html" target="_blank">[% comment.host %]</a>.<br/>
<small>Summary: "<em>[% comment.summary %]</em>"</small><br/>
From: [% comment.comment_author_name %] on [% date.format(comment.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
[%- IF comment.comment_title.length > 0 %]
"[% comment.comment_title %]"
[%- ELSE -%]
"[no title]"
[%- END -%]
<br/><hr class="thin">[% comment.comment_text FILTER html_line_break %]
</li><br/>
[%- END -%]
</ul></div>
[%- END -%]
[%# ---------------------------------------------------------------------------------------- -%]
<h2>Comments this month</h2>
[% IF comment_count > 0 -%]
[%- IF (mark_comments == 1) AND (ignore_count > 0) -%]
<p id="ignore"><b>Note to Volunteers</b>: Comments marked in green were read in the last
Community News show and should be ignored in this one.</p>
[%- END -%]
<p>These are comments which have been made during the past month, either to shows released during the month or to past shows.
There [%- comment_count == 1 ? "is $comment_count comment" : "are $comment_count comments" -%] in total.</p>
[% IF past_count > 0 -%]
<h3>Past shows</h3>
<p>There [% past_count == 1 ? "is $past_count comment" : "are $past_count comments" %] on
[% past.size %] previous [% past.size == 1 ? "show" : "shows" %]:</p>
<ul>
[%# Loop through by episode then by comment relating to that episode -%]
[%- FOREACH ep IN past.keys.nsort -%]
[%- arr = past.$ep -%]
<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
([% arr.0.date %]) "<em>[% arr.0.title %]</em>"
by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.<br/>
[%- IF mark_comments == 1 || ctext == 1 -%]
<small>Summary: "<em>[% arr.0.summary %]</em>"</small></li>
[%- END %]
<li style="list-style: none; display: inline">
<ul>
[%- FOREACH row IN arr -%]
[%# IF mark_comments == 1 && ((row.comment_timestamp_ut <= last_recording) && (arr.0.date.substr(0,7) == last_month)) -%]
[%# IF mark_comments == 1 && ((row.comment_released_ut <= last_recording) && (arr.0.date.substr(0,7) == last_month)) -%]
[%- IF mark_comments == 1 && row.ignore == 1 -%]
<li id="ignore">
[%- ELSE %]
<li>
[%- END %]
<a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
[% row.comment_author_name -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
[%- IF row.comment_title.length > 0 %]
"[% row.comment_title %]"
[%- ELSE -%]
"[no title]"
[%- END -%]
[%# Add the comment body in too if ctext is true -%]
[%- IF ctext == 1 %]
<br/><hr class="thin">[% row.comment_text FILTER html_line_break %]
</li><br/>
[%- ELSE -%]
</li>
[%- END -%]
[%- END -%]
</ul><br/>
</li>
[%- END -%]
</ul>
[%- END %]
[%# ---------------------------------------------------------------------------------------- -%]
[% cc = (comment_count - past_count) -%]
[% IF cc > 0 -%]
<h3>This month's shows</h3>
<p>There [% cc == 1 ? "is $cc comment" : "are $cc comments" %] on [% current.size %] of this month's shows:</p>
<ul>
[%- FOREACH ep IN current.keys.nsort -%]
[%- arr = current.$ep -%]
<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
([% arr.0.date %]) "<em>[% arr.0.title %]</em>"
by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.</li>
<li style="list-style: none; display: inline">
<ul>
[%- FOREACH row IN arr -%]
<li><a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
[% row.comment_author_name -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
[%- IF row.comment_title.length > 0 %]
"[% row.comment_title %]"
[%- ELSE -%]
"[no title]"
[%- END -%]
</li>
[%- END -%]
</ul><br/>
</li>
[%- END -%]
</ul>
[%- END %]
[%- ELSE %]
There were no comments this month.
[%- END %]
[%- END %]
[%# ---------------------------------------------------------------------------------------- -%]
[%- IF mailnotes == 1 -%]
<h2>Mailing List discussions</h2>
<p>
Policy decisions surrounding HPR are taken by the community as a whole. This
discussion takes place on the <a href="[% mailinglist %]" target="_blank">Mailing List</a>
which is open to all HPR listeners and contributors. The discussions are open
and available on the HPR server under <a href="[% mailbase %]">Mailman</a>.
</p>
<p>The threaded discussions this month can be found here:</p>
<a href="[% mailthreads %]" target="_blank">[% mailthreads %]</a>
[%- END %]
[%# ---------------------------------------------------------------------------------------- -%]
<h2>Events Calendar</h2>
<p>With the kind permission of <strong>LWN.net</strong> we are linking to
<a href="https://lwn.net/Calendar/" target="_blank">The LWN.net Community Calendar</a>.</p>
<p>Quoting the site:</p>
<blockquote>This is the LWN.net community event calendar, where we track
events of interest to people using and developing Linux and free software.
Clicking on individual events will take you to the appropriate web
page.</blockquote>
[%# ---------------------------------------------------------------------------------------- -%]
[%#
# vim: syntax=tt2:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
-%]

223
Community_News/shownote_template13.tpl Normal file
View File

@@ -0,0 +1,223 @@
[%# shownote_template13.tpl Updated: 2025-08-20 -%]
[%# -------------------------------------------------------------------------------- -%]
[%# Makes either an HTML snippet for insertion into the database or a full -%]
[%# listing with full comments for circulation to the hosts recording the episode -%]
[%# This one uses the new format for the mailing list data, and partitions -%]
[%# comments into past and current. It also marks comments that don't need -%]
[%# to be read when mark_comments is true. It requires make_shownotes >= V0.0.30 -%]
[%# -------------------------------------------------------------------------------- -%]
[%- USE date -%]
[%- USE pad4 = format('%04d') -%]
[%- correspondents = "https://hackerpublicradio.org/correspondents"
mailinglist = "https://lists.hackerpublicradio.com/mailman/listinfo/hpr"
mailbase="https://lists.hackerpublicradio.com/pipermail/hpr"
mailthreads = "$mailbase/$review_year-$review_month/thread.html" -%]
[%- DEFAULT skip_comments = 0
mark_comments = 0
ctext = 0
ignore_count = 0
missed_count = 0
past_count = 0
-%]
[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
[%# Embedded CSS. The 'table' and 'hr' settings are always there but the rest is -%]
[%# only for when we are marking comments -%]
<style>
table td.shrink {
white-space:nowrap
}
hr.thin {
border: 0;
height: 0;
border-top: 1px solid rgba(0, 0, 0, 0.1);
border-bottom: 1px solid rgba(255, 255, 255, 0.3);
}
[%- IF mark_comments == 1 %]
p#ignore, li#ignore {
background-color: lightgreen;
color:maroon;
}
div#highlight {
border-style: solid;
border-color: red;
padding-right: 20px;
padding-left: 20px;
}
[%- END %]
</style>
[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
<h2>New hosts</h2>
<p>
[% IF hosts.size > 0 -%]
Welcome to our new host[%- hosts.size > 1 ? 's' : '' -%]: <br />
[%- count = 0 %]
[%# List the new hosts. If a name contains a comma quote it. -%]
[%- FOREACH row IN hosts %]
[%- count = count + 1 %]
[%- hostname = (row.host.search(',') ? row.host.replace('^(.*)$','"$1"') : row.host) %]
<a href="[% correspondents %]/[% pad4(row.hostid) %].html" target="_blank">[% hostname %]</a>
[%- count < hosts.size ? ', ' : '.' %]
[%- END %]
[% ELSE -%]
There were no new hosts this month.
[% END -%]
</p>
<h2>Last Month's Shows</h2>
[%# The id 't01' is in the HPR CSS but might give trouble on the IA -%]
<table id="t01">
<tr>
<th>Id</th>
<th>Day</th>
<th>Date</th>
<th>Title</th>
<th>Host</th>
</tr>
[%- FOREACH row IN shows %]
<tr>
<td><strong><a href="https://hackerpublicradio.org/eps/hpr[% pad4(row.eps_id) %]/index.html" target="_blank">[% row.eps_id %]</a></strong></td>
<td>[% date.format(row.date,'%a') %]</td>
<td class="shrink">[% date.format(row.date,'%Y-%m-%d') %]</td>
<td><a href="https://hackerpublicradio.org/eps/hpr[% pad4(row.eps_id) %]/index.html" target="_blank">[% row.title %]</a></td>
<td><a href="[% correspondents %]/[% pad4(row.ho_hostid) %].html" target="_blank">[% row.ho_host %]</a></td>
</tr>
[%- END %]
</table>
[%# ---------------------------------------------------------------------------------------- -%]
[%# Skip comments if told to by the caller -%]
[%- IF skip_comments == 0 -%]
[%# Handle any missed comments if mark_comments is true -%]
[%- IF (mark_comments == 1) AND (missed_count > 0) -%]
<br/><div id="highlight">
<h2>Missed comment[%- missed_comments.size > 1 ? 's' : '' -%] last month</h2>
<p><b>Note to Volunteers</b>: These are comments for shows last month that were not read in the last show because they arrived on or after the recording started. This section will be removed before these notes are released.</p>
<ul>
[%- FOREACH comment IN missed_comments -%]
<li><strong><a href="[% comment.identifier_url %]#comments" target="_blank">hpr[% pad4(comment.episode) %]</a></strong>
([% comment.date %]) "<em>[% comment.title %]</em>" by <a href="[% correspondents %]/[% pad4(comment.hostid) %].html" target="_blank">[% comment.host %]</a>.<br/>
<small>Summary: "<em>[% comment.summary %]</em>"</small><br/>
From: [% comment.comment_author_name %] on [% date.format(comment.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
[%- IF comment.comment_title.length > 0 %]
"[% comment.comment_title %]"
[%- ELSE -%]
"[no title]"
[%- END -%]
<br/><hr class="thin">[% comment.comment_text FILTER html_line_break %]
</li><br/>
[%- END -%]
</ul></div>
[%- END -%]
[%# ---------------------------------------------------------------------------------------- -%]
<h2>Comments this month</h2>
[% IF comment_count > 0 -%]
[%- IF (mark_comments == 1) AND (ignore_count > 0) -%]
<p id="ignore"><b>Note to Volunteers</b>: Comments marked in green were read in the last
Community News show and should be ignored in this one.</p>
[%- END -%]
<p>These are comments which have been made during the past month, either to shows released during the month or to past shows.
There [%- comment_count == 1 ? "is $comment_count comment" : "are $comment_count comments" -%] in total.</p>
[% IF past_count > 0 -%]
<h3>Past shows</h3>
<p>There [% past_count == 1 ? "is $past_count comment" : "are $past_count comments" %] on
[% past.size %] previous [% past.size == 1 ? "show" : "shows" %]:</p>
<ul>
[%# Loop through by episode then by comment relating to that episode -%]
[%- FOREACH ep IN past.keys.nsort -%]
[%- arr = past.$ep -%]
<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
([% arr.0.date %]) "<em>[% arr.0.title %]</em>"
by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.<br/>
[%- IF mark_comments == 1 || ctext == 1 -%]
<small>Summary: "<em>[% arr.0.summary %]</em>"</small></li>
[%- END %]
<li style="list-style: none; display: inline">
<ul>
[%- FOREACH row IN arr -%]
[%# IF mark_comments == 1 && ((row.comment_timestamp_ut <= last_recording) && (arr.0.date.substr(0,7) == last_month)) -%]
[%# IF mark_comments == 1 && ((row.comment_released_ut <= last_recording) && (arr.0.date.substr(0,7) == last_month)) -%]
[%- IF mark_comments == 1 && row.ignore == 1 -%]
<li id="ignore">
[%- ELSE %]
<li>
[%- END %]
<a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
[% row.comment_author_name -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
[%- IF row.comment_title.length > 0 %]
"[% row.comment_title %]"
[%- ELSE -%]
"[no title]"
[%- END -%]
[%# Add the comment body in too if ctext is true -%]
[%- IF ctext == 1 %]
<br/><hr class="thin">[% row.comment_text FILTER html_line_break %]
</li><br/>
[%- ELSE -%]
</li>
[%- END -%]
[%- END -%]
</ul><br/>
</li>
[%- END -%]
</ul>
[%- END %]
[%# ---------------------------------------------------------------------------------------- -%]
[% cc = (comment_count - past_count) -%]
[% IF cc > 0 -%]
<h3>This month's shows</h3>
<p>There [% cc == 1 ? "is $cc comment" : "are $cc comments" %] on [% current.size %] of this month's shows:</p>
<ul>
[%- FOREACH ep IN current.keys.nsort -%]
[%- arr = current.$ep -%]
<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
([% arr.0.date %]) "<em>[% arr.0.title %]</em>"
by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.</li>
<li style="list-style: none; display: inline">
<ul>
[%- FOREACH row IN arr -%]
<li><a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
[% row.comment_author_name -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
[%- IF row.comment_title.length > 0 %]
"[% row.comment_title %]"
[%- ELSE -%]
"[no title]"
[%- END -%]
</li>
[%- END -%]
</ul><br/>
</li>
[%- END -%]
</ul>
[%- END %]
[%- ELSE %]
There were no comments this month.
[%- END %]
[%- END %]
[%# ---------------------------------------------------------------------------------------- -%]
[%- IF mailnotes == 1 -%]
<h2>Mailing List discussions</h2>
<p>
Policy decisions surrounding HPR are taken by the community as a whole. This
discussion takes place on the <a href="[% mailinglist %]" target="_blank">Mailing List</a>
which is open to all HPR listeners and contributors. The discussions are open
and available on the HPR server under <a href="[% mailbase %]">Mailman</a>.
</p>
<p>The threaded discussions this month can be found here:</p>
<a href="[% mailthreads %]" target="_blank">[% mailthreads %]</a>
[%- END %]
[%# ---------------------------------------------------------------------------------------- -%]
<h2>Events Calendar</h2>
<p>With the kind permission of <strong>LWN.net</strong> we are linking to
<a href="https://lwn.net/Calendar/" target="_blank">The LWN.net Community Calendar</a>.</p>
<p>Quoting the site:</p>
<blockquote>This is the LWN.net community event calendar, where we track
events of interest to people using and developing Linux and free software.
Clicking on individual events will take you to the appropriate web
page.</blockquote>
[%# ---------------------------------------------------------------------------------------- -%]
[%#
# vim: syntax=tt2:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
-%]

35
workflow/hpr-get-and-transcribe.bash Executable file
View File

@@ -0,0 +1,35 @@
#!/bin/bash
rsync -av --partial --progress hpr:hub.hackerpublicradio.org/upload/ $HOME/tmp/hpr/processing/
find $HOME/tmp/hpr/processing/*_*_????-??-??_* -type d | sort -t _ -k 2 | while read show_dir
do
echo "${show_dir}"
if [ "$( find "${show_dir}" -type f -iname "*srt" | wc -l )" -eq "0" ]
then
cd "${show_dir}"
ls -haltr
find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media
do
whisper --model tiny --language en --output_dir "${show_dir}" "${this_media}"
done
rsync -av --partial --progress "${show_dir}/" hpr:hub.hackerpublicradio.org/upload/$( basename "${show_dir}")/
fi
done
rsync -av --partial --progress hpr:hub.hackerpublicradio.org/reserve/ $HOME/tmp/hpr/reserve/
find $HOME/tmp/hpr/reserve/*_*_* -type d | sort -t _ -k 2 | while read show_dir
do
echo "${show_dir}"
if [ "$( find "${show_dir}" -type f -iname "*srt" | wc -l )" -eq "0" ]
then
cd "${show_dir}"
ls -haltr
find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media
do
whisper --model tiny --language en --output_dir "${show_dir}" "${this_media}"
done
rsync -av --partial --progress "${show_dir}/" hpr:hub.hackerpublicradio.org/reserve/$( basename "${show_dir}")/
fi
done

44
workflow/hpr-publish.bash Executable file
View File

@@ -0,0 +1,44 @@
#!/usr/bin/env bash
# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/
#============================================================
git_dir="$HOME/tmp/hpr/hpr_generator/sourcecode"
if [ ! -d "${git_dir}/.git" ]
then
git clone gitea@repo.anhonesthost.net:HPR/hpr_generator.git "${git_dir}"
fi
cd "${git_dir}"
git pull
# ssh hpr -t "ls -al /home/hpr/www/hpr.sql;md5sum /home/hpr/www/hpr.sql"
# ssh hpr -t "/home/hpr/bin/hpr_db_backup.bash"
# ssh hpr -t "ls -al /home/hpr/www/hpr.sql;md5sum /home/hpr/www/hpr.sql"
./utils/update-hpr-db.sh
if [ $? -ne 0 ]
then
echo 'Terminating...' >&2
exit 1
fi
./site-generator --all --verbose
if [ $? -ne 0 ]
then
echo 'Terminating...' >&2
exit 1
fi
rsync -av --partial --progress "${git_dir}/public_html/" hpr:hackerpublicradio.org/public_html
#rsync -av --partial --progress "${git_dir}/public_html/" hobbypublicradio.org:hobbypublicradio.org/
cd $HOME/sourcecode/hpr/hpr_hub/
git pull
cd $HOME/sourcecode/hpr/hpr_hub/sql
split --hex-suffixes --lines=1000 --additional-suffix=.sql hpr.sql hpr-db-part-
cd $HOME/sourcecode/hpr/hpr_hub/
git add $HOME/sourcecode/hpr/hpr_hub/sql/hpr*sql
git commit -m "$(\date -u +%Y-%m-%d_%H-%M-%SZ_%A ) database changed"
git push
#xdg-open https://hackerpublicradio.org/

4
workflow/hpr_ccdn_stats.bash Normal file
View File

@@ -0,0 +1,4 @@
#!/bin/bash
yesterday="$( \date -u +%Y-%m-%d -d yesterday)"
echo -e "${yesterday}\t$( grep -Ec "${yesterday}T.*Sending request to" /home/hpr/logs/naughty-ip.txt )" >> /home/hpr/hub/hpr_ccdn_stats.tsv

194
workflow/hpr_db_backup.bash Executable file
View File

@@ -0,0 +1,194 @@
#!/bin/bash
# 5,20,35,50 * * * * $HOME/userfiles/sql/hpr_db_backup.bash >> $HOME/userfiles/sql/cron.log 2>&1 &
#TODO
#add a rss feed with the latest updates. Linking to the changes on gittea.
# run that every 5 minutes and then if there is a change sleep 5 and confirm there has been no change since.
# Then download the db and regenerate the site.
# While making sure to generate the site at least once a day
# check the skip-extended-insert export into of the sql into git, so that the small diffs show up.
# generate a rss feed with the latest changes.
# update all the other scripts reserve, stats etc.
sql_save_dir="$HOME/userfiles/sql"
credential_file="${sql_save_dir}/.my-hpr.cnf"
last_update_txt="${sql_save_dir}/last_update.txt"
hpr_full_sql="${sql_save_dir}/hpr_hpr_full.sql"
hpr_site_sql="${sql_save_dir}/hpr.sql"
full_mysqldump_sql="${sql_save_dir}/mysqldump.sql"
hpr_database_file="hackerpublicradio.org/public_html/hpr.sql"
sync_delay_seconds="300" # 5 minutes
last_update_query="SELECT
update_time
FROM
information_schema.tables tab
WHERE
update_time > (current_timestamp() - interval 30 day)
AND table_type = 'BASE TABLE'
AND table_name not in ('reservations')
AND table_schema not in ('information_schema', 'sys', 'performance_schema', 'mysql')
ORDER BY
update_time DESC
LIMIT 1;"
if [ ! -s "${credential_file}" ]
then
echo "The file \"${credential_file}\" is missing"
exit
fi
local_db_last_update_epoch="0"
if [ -s "${last_update_txt}" ]
then
echo "Found the last update file \"${last_update_txt}\""
local_db_last_update_iso8601="$( \date --utc --date="$( cat ${last_update_txt} )" +%Y-%m-%dT%H:%M:%SZ )"
local_db_last_update_epoch="$( \date --utc --date="$( cat ${last_update_txt} )" +%s )"
echo -e "Local DB update time is\t${local_db_last_update_iso8601} (${local_db_last_update_epoch})"
fi
live_db_last_update_iso8601="$( mysql --defaults-file="${credential_file}" --disable-column-names --batch --execute="${last_update_query}" | sed -e 's/ /T/g' -e 's/$/Z/g' )"
if [ -z "${live_db_last_update_iso8601}" ]
then
echo "The live db update time \"live_db_last_update_iso8601\" is missing"
exit 1
fi
live_db_last_update_epoch="$( \date --utc --date="${live_db_last_update_iso8601}" +%s )"
if [ -z "${live_db_last_update_epoch}" ]
then
echo "The live db update time \"live_db_last_update_epoch\" is missing"
exit 2
fi
echo -e "Live DB update time is\t${live_db_last_update_iso8601} (${live_db_last_update_epoch})"
if [ "${local_db_last_update_epoch}" -eq "${live_db_last_update_epoch}" ]
then
echo "No changes detected. Skipping export."
exit 0
fi
echo "Starting export full with -complete-insert."
if [ -s "${hpr_full_sql}" ]
then
hpr_full_sql_write_time_iso8601="$( \date --utc --date="$( ls -al --full-time "${hpr_full_sql}" | awk '{print $6, $7, $8}' )" +%Y-%m-%dT%H:%M:%SZ )"
hpr_full_sql_write_time_epoch="$( \date --utc --date="${hpr_full_sql_write_time_iso8601}" +%s )"
if [ -z "${hpr_full_sql_write_time_epoch}" ]
then
echo "The live db update time \"hpr_full_sql_write_time_epoch\" is missing"
exit 3
fi
echo -e "Full DB write time is\t${hpr_full_sql_write_time_iso8601} (${hpr_full_sql_write_time_epoch})"
hpr_full_sql_write_time_with_delay_epoch="$(( ${hpr_full_sql_write_time_epoch} + ${sync_delay_seconds} ))"
time_now_epoch="$( \date --utc +%s )"
if [ "${hpr_full_sql_write_time_with_delay_epoch}" -gt "${time_now_epoch}" ]
then
echo "Skipping export. The Database has been recently created \"${hpr_full_sql_write_time_iso8601}\". Try again after $( \date --utc --date="@${hpr_full_sql_write_time_with_delay_epoch}" +%Y-%m-%dT%H:%M:%SZ )."
exit 4
fi
fi
mysqldump --defaults-file="${credential_file}" --tz-utc --add-drop-database --extended-insert --complete-insert --skip-extended-insert --default-character-set=utf8 --single-transaction --skip-set-charset --databases hpr_hpr > "${hpr_full_sql}"
tail "${hpr_full_sql}" | grep 'Dump completed on'
echo "Starting export full for static site generation."
if [ -s "${hpr_site_sql}" ]
then
hpr_site_sql_write_time_iso8601="$( \date --utc --date="$( ls -al --full-time "${hpr_site_sql}" | awk '{print $6, $7, $8}' )" +%Y-%m-%dT%H:%M:%SZ )"
hpr_site_sql_write_time_epoch="$( \date --utc --date="${hpr_site_sql_write_time_iso8601}" +%s )"
if [ -z "${hpr_site_sql_write_time_epoch}" ]
then
echo "The live db update time \"hpr_site_sql_write_time_epoch\" is missing"
exit 5
fi
echo -e "Full DB write time is\t${hpr_site_sql_write_time_iso8601} (${hpr_site_sql_write_time_epoch})"
hpr_site_sql_write_time_with_delay_epoch="$(( ${hpr_site_sql_write_time_epoch} + ${sync_delay_seconds} ))"
time_now_epoch="$( \date --utc +%s )"
if [ "${hpr_site_sql_write_time_with_delay_epoch}" -gt "${time_now_epoch}" ]
then
echo "Skipping export. The Database has been recently created \"${hpr_site_sql_write_time_iso8601}\". Try again after $( \date --utc --date="@${hpr_site_sql_write_time_with_delay_epoch}" +%Y-%m-%dT%H:%M:%SZ )."
exit 6
fi
fi
mysqldump --defaults-file="${credential_file}" --tz-utc --add-drop-database --complete-insert --extended-insert --default-character-set=utf8 --single-transaction --skip-set-charset --databases hpr_hpr --ignore-table=hpr_hpr.reservations > "${hpr_site_sql}"
tail "${hpr_site_sql}" | grep 'Dump completed on'
echo "Starting export full for data recovery."
if [ -s "${full_mysqldump_sql}" ]
then
full_mysqldump_sql_write_time_iso8601="$( \date --utc --date="$( ls -al --full-time "${full_mysqldump_sql}" | awk '{print $6, $7, $8}' )" +%Y-%m-%dT%H:%M:%SZ )"
full_mysqldump_sql_write_time_epoch="$( \date --utc --date="${full_mysqldump_sql_write_time_iso8601}" +%s )"
if [ -z "${full_mysqldump_sql_write_time_epoch}" ]
then
echo "The live db update time \"full_mysqldump_sql_write_time_epoch\" is missing"
exit 5
fi
echo -e "Full DB write time is\t${full_mysqldump_sql_write_time_iso8601} (${full_mysqldump_sql_write_time_epoch})"
full_mysqldump_sql_write_time_with_delay_epoch="$(( ${full_mysqldump_sql_write_time_epoch} + ${sync_delay_seconds} ))"
time_now_epoch="$( \date --utc +%s )"
if [ "${full_mysqldump_sql_write_time_with_delay_epoch}" -gt "${time_now_epoch}" ]
then
echo "Skipping export. The Database has been recently created \"${full_mysqldump_sql_write_time_iso8601}\". Try again after $( \date --utc --date="@${full_mysqldump_sql_write_time_with_delay_epoch}" +%Y-%m-%dT%H:%M:%SZ )."
exit 6
fi
fi
mysqldump --defaults-file="${credential_file}" --tz-utc --add-drop-database --databases hpr_hpr> "${full_mysqldump_sql}"
tail "${full_mysqldump_sql}" | grep 'Dump completed on'
echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>
<rss xmlns:atom=\"https://www.w3.org/2005/Atom\" version=\"2.0\">
<channel>
<title>Hacker Public Radio ~ Database Feed</title>
<link>http://hackerpublicradio.org/about.html</link>
<description>This Feed provides information the latest version of the HPR database.</description>
<language>en-us</language>
<copyright>Creative Commons Attribution-ShareAlike 4.0 International (CC BY-SA 4.0) License</copyright>
<managingEditor>feedback.nospam@nospam.hackerpublicradio.org (HPR Feedback)</managingEditor>
<webMaster>admin.nospam@nospam.hackerpublicradio.org (HPR Webmaster)</webMaster>
<generator>https://repo.anhonesthost.net/HPR/hpr-tools/src/branch/main/workflow/hpr_db_backup.bash</generator>
<docs>https://www.rssboard.org/rss-specification</docs>
<ttl>15</ttl>
<image>
<url>http://hackerpublicradio.org/images/hpr_feed_small.png</url>
<title>Hacker Public Radio ~ Database Feed</title>
<link>http://hackerpublicradio.org/about.html</link>
<description>The Hacker Public Radio Old Microphone Logo</description>
<height>164</height>
<width>144</width>
</image>
<atom:link href=\"https://hackerpublicradio.org/hpr.sql.rss\" rel=\"self\" type=\"application/rss+xml\"/>
<pubDate>$( date --utc --rfc-email )</pubDate>
<item>
<title>Export of the Public mariadb SQL for ${live_db_last_update_iso8601}</title>
<author>admin.nospam@nospam.hackerpublicradio.org (Janitors)</author>
<link>http://hackerpublicradio.org/hpr.sql#${live_db_last_update_iso8601}</link>
<description/>
<pubDate>$( date --utc --rfc-email )</pubDate>
<enclosure url=\"http://hackerpublicradio.org/hpr.sql#${live_db_last_update_iso8601}\" length=\"$( ls -al "${hpr_site_sql}" | awk '{print $5}' )\" type=\"application/sql\"/>
<guid isPermaLink=\"false\">sha1sum:$( sha1sum "${hpr_site_sql}" | awk '{print $1}' ),md5sum:$( md5sum "${hpr_site_sql}" | awk '{print $1}' )</guid>
</item>
</channel>
</rss>" > "${hpr_site_sql}.rss"
if [ $HOSTNAME = "whp01.cloud-hosting.io" ]
then
cp -v "${hpr_site_sql}" "$HOME/${hpr_database_file}"
cp -v "${hpr_site_sql}.rss" "$HOME/${hpr_database_file}.rss"
else
rsync -av --partial --progress ${hpr_site_sql} hpr:${hpr_database_file}
rsync -av --partial --progress ${hpr_site_sql}.rss hpr:${hpr_database_file}.rss
fi
echo "${live_db_last_update_iso8601}" > "${last_update_txt}"
echo "Finished export of \"${live_db_last_update_iso8601}\""

4
workflow/mdb.bash Executable file
View File

@@ -0,0 +1,4 @@
#!/bin/bash
pw=$(grep -E '^\$databasePassword = ' /home/hpr/php/credentials.php | awk -F "'" '{print $2}' )
pw=${pw##* }
mysql --host=localhost --user=hpr_hpr --password="$pw" hpr_hpr

63
workflow/process_episode.bash
View File

@@ -91,7 +91,7 @@ function program_checks() {
done
}
is_installed audio2image.bash awk base64 cat csvtojson curl date detox eval extract_images ffprobe file find grep head jq kate magick mediainfo mv realpath remove-image.pl rsync seamonkey sed sed sort sponge ssh touch touch wget hpr-check-ccdn-links
is_installed audio2image.bash awk base64 cat csvtojson curl date eval extract_images ffprobe ffmpeg file find grep head jq kate magick mediainfo mv realpath remove-image.pl rsync seamonkey sed sed sort sponge ssh touch touch wget hpr-check-ccdn-links
for arg in $*
do
@@ -518,10 +518,10 @@ function check_variable_is_correct() {
then
echo_error "The \"shownotes_json\" file is missing."
fi
if [ "$( file --brief --mime-type "${shownotes_json}" | grep --count 'application/json' )" -ne "1" ]
then
echo_error "The \"shownotes_json\" variable has not a valid \"application/json\" mime type."
fi
# if [ "$( file --brief --mime-type "${shownotes_json}" | grep --count 'application/json' )" -ne "1" ]
# then
# echo_error "The \"shownotes_json\" variable has not a valid \"application/json\" mime type."
# fi
jq '.' "${shownotes_json}" >/dev/null 2>&1
if [ $? -ne 0 ]
then
@@ -637,7 +637,7 @@ function get_next_show_from_hpr_hub() {
email="$( echo ${response} | awk '{print $6}' )"
email_unpadded="$( echo $email | sed 's/.nospam@nospam./@/g' )"
hpr_upload_dir="/home/hpr/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
hpr_upload_dir="hub.hackerpublicradio.org/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
source_dir="hpr:${hpr_upload_dir}"
dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}"
working_dir="${processing_dir}/${dest_dir}"
@@ -647,7 +647,6 @@ function get_next_show_from_hpr_hub() {
echo_debug "Downloading hpr${ep_num} from ${email_unpadded}"
echo_debug ""
echo_debug "rsync -ave ssh --partial --progress ${source_dir}/ ${working_dir}/"
ssh hpr -t "detox -v ${hpr_upload_dir}/"
rsync -ave ssh --partial --progress ${source_dir}/ ${working_dir}/
}
@@ -870,7 +869,7 @@ function extract_images_brute_force() {
# Download referenced images
image_count_external="1"
for image in $( grep --color=never --perl-regexp --only-matching '<img.*src.*http.*>' "${shownotes_html}" | awk -F 'src=' '{print $2}' | awk -F '"' '{print $2}' )
for image in $( grep --color=never --perl-regexp --only-matching '<img.*src.*http.*://.*>' "${shownotes_html}" | awk -F 'src=' '{print $2}' | awk -F '"' '{print $2}' )
do
this_image="${working_dir}/hpr${ep_num}_image_ext_${image_count_external}"
wget "${image}" --output-document=${this_image}
@@ -1032,7 +1031,7 @@ function media_checks() {
shownotes_srt="${media%.*}.srt"
if [[ -z "${shownotes_srt}" || ! -s "${shownotes_srt}" ]]
then
echo_error "Could not find the subtitles for media \"${media}\""
echo_error "Could not find the subtitles for media \"${media}\" in \"${shownotes_srt}\""
fi
#TODO fix close duration
@@ -1968,7 +1967,7 @@ function register_assets() {
}
#################################################
# Register the assets with the hpr database
# Copy the files to the origin server
function copy_files_to_origin_server() {
@@ -2014,6 +2013,48 @@ function copy_files_to_origin_server() {
done
}
#################################################
# Copy the images to the hpr server
function copy_files_to_hpr_server() {
echo_debug "Copying the files to the hpr server. copy_files_to_hpr_server()"
check_variable_is_correct working_dir ep_num
# Get the file list from the HPR db
if [ -z "${files_json}" ]
then
files_json="${working_dir}/hpr${ep_num}_files.json"
fi
if [ -s "${files_json}" ]
then
echo_debug "Removing stale \"files_json\" file \"${files_json}\"."
rm -v "${files_json}"
fi
echo_debug "Getting the \"files_json\" file from the HPR site."
if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/assets.php?id=${ep_num} --output "${files_json}" )" != 200 ]
then
echo_error "The Episode hpr${ep_num} has not been posted"
fi
check_variable_is_correct files_json
ssh hpr -t "mkdir -v hackerpublicradio.org/public_html/eps/hpr${ep_num}"
for this_file in $( jq --raw-output ".hpr${ep_num} | keys | @tsv" "${files_json}" )
do
skip_file="$( echo "$this_file" | grep --count --perl-regexp "hpr${ep_num}.flac|hpr${ep_num}.mp3|hpr${ep_num}.ogg|hpr${ep_num}.opus|hpr${ep_num}.spx|hpr${ep_num}.srt|hpr${ep_num}.txt|hpr${ep_num}.wav" )"
if [ "${skip_file}" -eq "1" ]
then
continue
fi
scp "${working_dir}/$this_file" hpr:hackerpublicradio.org/public_html/eps/hpr${ep_num}/
done
}
#################################################
# Send the derived files to the server borg to be sent to borg
@@ -2335,6 +2376,8 @@ copy_files_to_origin_server
check_variable_is_correct working_dir ep_num shownotes_edited
copy_files_to_hpr_server
copy_derived_files_to_borg
create_item_on_the_internet_archive

6
workflow/run-speedtest.bash Normal file
View File

@@ -0,0 +1,6 @@
#!/bin/bash
# * * * * * /usr/local/bin/run-speedtest.bash >/dev/null 2>&1
speedtest-cli --json | jq '.' > /var/www/html/speedtest.json
chown apache:apache /var/www/html/speedtest.json

8
workflow/update-stats.bash Normal file
View File

@@ -0,0 +1,8 @@
#!/bin/bash
hub_dir="/home/hpr/hub"
for format in txt csv json xml
do
curl --silent --netrc-file /home/hpr/.netrc https://hub.hackerpublicradio.org/cms/stats.php?format=${format} --output ${hub_dir}/stats.${format}
done