Move HPR/hpr_hub#88

Community_News/make_email: minor POD changes

Community_News/make_shownotes: minor POD changes

Community_News/shownote_template.tpl: now a copy of the latest template
    rather than a symbolic linkl to it

Community_News/shownote_template13.tpl: new template which doesn't
    filter 'comment_author_name' with 'html_entity'

Community_News/README.md

@@ -0,0 +1,685 @@
+# NAME
+
+make\_shownotes - Make show notes for the Hacker Public Radio Community News show
+
+# VERSION
+
+This documentation refers to **make\_shownotes** version 0.4.5
+
+# USAGE
+
+    make_shownotes [-help] [-documentation|-man] [-config=FILE]
+        [-from=DATE] [-[no]comments] [-[no]silent] [-[no]mailnotes]
+        [-lastrecording=DATETIME]
+        [-full-html=FILE] [-html=FILE] [-json=FILE]
+        [-debug=N]
+
+# OPTIONS
+
+- **-help**
+
+    Displays a brief help message describing the usage of the program, and then exits.
+
+- **-documentation** or **-man**
+
+    Displays the entirety of the documentation (using a pager), and then exits.
+
+    To generate a PDF version use:
+
+        pod2pdf make_shownotes --out=make_shownotes.pdf
+
+- **-config=FILE**
+
+    The script uses a configuration file to hold the various parameters it needs
+    to run. This option allows an alternative configuration file to be used. This file
+    defines many settings including the location of the database.
+
+    See the CONFIGURATION AND ENVIRONMENT section below for the file format.
+
+    If the option is omitted the default file is used: **.make\_shownotes.cfg**,
+    which is expected to be in the same directory as the script itself.
+
+- **-from=DATE**
+
+    This option is used to indicate the month for which the shownotes are to be
+    generated. The script is able to parse a variety of date formats, but it is
+    recommended that ISO8601 **YYYY-MM-DD** format be used (for example 2014-06-30).
+
+    The day part of the date must be present but is ignored and only the month and
+    year parts are used (to internally denote the first day of the month).
+
+    If this option is omitted the current month is used. Of course, this may cause
+    problems if the notes are to generated for an earlier (or later) month, which
+    is why this option exists.
+
+- **-\[no\]comments**
+
+    This option controls whether the comments pertaining to the selected month are
+    included in the output. If the option is omitted then no comments are included
+    (**-nocomments**).
+
+- _Output file options_: **-html=FILE**, **-full-html=FILE**, **-json=FILE**
+
+    There are three output file types that can be generated by the script. At
+    least one must be present:
+
+    - _HTML fragment_ (**-html=FILE**)
+
+        This file will contain the HTML to be added to the HPR database. The page for
+        the show, when it is released, will be a full web page, with standard header
+        and footer, and the contents will come from this HTML fragment in the database.
+
+        Action will be needed in addition to the script to add this file to the
+        database, but how this is done is outside the scope of this documentation.
+
+    - _Standalone HTML_ (**-full-html=FILE**)
+
+        The file created in this case will contain a full, stand-alone HTML page. It
+        is intended to be circulated to the co-hosts recording the episode to make it
+        easier to access various information sources during the recording.
+
+        In the file the comments relating to past shows will show the full text, and
+        there will be indications of comments that were read in the last recording,
+        and any that were missed.
+
+        In order to highlight comments read, and those missed in the previous
+        recording the script needs to know the date and time of the recording. This
+        information should be in a date cache file referenced in the configuration
+        file (usually **recording\_dates.dat**). This file is updated when the monthly
+        mail message is generated (see **make\_email**). If, for any reason, this has
+        not happened, the information can be provided with the
+        **-lastrecording=DATETIME** option (alternatively written as **-lr=DATETIME**).
+        See below for more information.
+
+    - _JSON details_ (**-json=FILE**)
+
+        This file will contain JSON data which is intended to be used to upload the
+        episode to the database. How this is done is outside the scope of this
+        document. The format used is very close to that used in the workflow which is
+        used to upload episodes submitted through the upload forms.
+
+    In all cases the output file name may contain the characters '**%s**'. This
+    denotes the point at which the year and month in the format **YYYY-MM** are
+    inserted. For example if the script is being run for July 2014 the option:
+
+        -html=shownotes_%s.html
+
+    This will cause the generation of the file:
+
+        shownotes_2014-07.html
+
+- **-lastrecording=DATETIME** or **-lr=DATETIME**
+
+    As mentioned for **-full-html=FILE**, and later in the _MARKING COMMENTS_
+    section, the script needs the date of the last recording when marking
+    comments. This can be extracted from the file referenced in the configuration
+    data using the setting **cache**. By default the name of this file is
+    **recording\_dates.dat**, and its contents are managed when the script
+    **make\_email** is run and by this script.
+
+    If for any reason the date and time of the last recording is missing, these
+    values can be defined with this option, and these values will be written to
+    the cache file (or modified, if necessary).
+
+    The format can be an ISO 8601 date followed by a 24-hour time, such as
+    '2020-01-25 15:00:00'. If the time is omitted it defaults to the value of
+    _starttime_ in the configuration file.
+
+    The script will update the cache file with the date and time used in this
+    option if the relevant entry is missing. Also, if an entry is present but the
+    values are different from those provided with the option, the relevant entry
+    will be updated.
+
+    Note that the **DATETIME** value must contain the date of the last recording.
+    This will be checked, and written to the cache file prefixed by a "key"
+    consisting of the first day of the month _BEFORE_ the month being reviewed.
+
+    For example, when generating the notes for August 2025 the following command
+    will be needed if there is no last recording date (for July 2025) in the
+    cache:
+
+        ./make_shownotes -from=2025-08-01 -full-html=full_shownotes_%s.html \
+            -mail -comments -lr="2025-08-01 15:00:00"
+
+    Here we need the last recording date for the show reviewing HPR shows in July
+    2025\. The date and time for this recording was in early August (Friday before
+    the first Monday of September 2025-09-01), as shown. This combination will
+    result in the addition of the following line to the cache file:
+
+        2025-07-01,2025-08-01 15:00:00
+
+    As mentioned, the addition of such date and time information to the cache will
+    normally be performed by **make\_email**, which performs the date computations
+    itself, unlike this script. This feature in this script is an alternative for
+    special cases.
+
+- **-\[no\]silent**
+
+    This option controls whether the script reports details of its progress
+    to STDERR. If the option is omitted the report is generated (**-nosilent**).
+
+    The script reports: the month it is working on, the name of the requested output files
+    and details of the process of generating these files.
+
+- **-\[no\]mailnotes**
+
+    If desired, the show notes may include a section linking to recent discussions
+    on the HPR Mailman mailing list.
+
+    The current template (defined in the configuration file by the variable
+    **main\_template**, **shownote\_template12.tpl**) simply contains a section
+    like the following:
+
+        [%- IF mailnotes == 1 -%]
+        <h2>Mailing List discussions</h2>
+        <p>
+        Policy decisions surrounding HPR are taken by the community as a whole.
+        This discussion takes place on the <a href="[% mailinglist %]"
+        target="_blank">Mailing List</a> which is open to all HPR listeners and
+        contributors. The discussions are open and available on the HPR server
+        under <a href="[% mailbase %]">Mailman</a>.
+        </p>
+        <p>The threaded discussions this month can be found here:</p>
+        <a href="[% mailthreads %]" target="_blank">[% mailthreads %]</a>
+        [%- END %]
+
+    The _TT2_ variables such as **mailinglist** and **mailthreads** are defined earlier in
+    the template.
+
+- **-debug=N**
+
+    Enables debugging mode when N > 0 (zero is the default, no debugging output).
+    The levels are:
+
+    Values are:
+
+    1. TBA
+    2. Reports the following (as well as the data for level 1):
+
+        >         Details of the last recording data (and time)
+
+    3. Reports the following (as well as the data for level 2):
+
+        >         The generation of comment indexes needed in the comment lists. These are
+        >         computed after the query has been run.
+
+    4. See the **DESCRIPTION** section for an explanation of the data structures
+    mentioned here.
+
+        Reports the following (as well as the data for level 3):
+
+        >         A dump of the '%past' hash which contains details of comments on past shows.
+        >
+        >         A dump of the '%current' hash which contains details of comments on this
+        >         month's shows.
+        >
+        >         A dump of the '@missed\_comments' array containing comments that arrived after
+        >         the last recording.
+        >
+        >         A list of the duplicated episode numbers in '@missed\_episodes'
+        >
+        >         Another dump of '%past' after it has been cleaned up. Also the count of
+        >         comments to past shows and the comment count.
+
+# MARKING COMMENTS
+
+Explaining the marking of comments in the full HTML file:
+
+> This is only relevant when generating the full stand-alone HTML ("handout")
+> for circulation to the volunteers recording the Community News episode. In
+> this output the comments sent in to past shows (those before the review month)
+> include their full texts in order to make reading them easier.
+>
+> Normally comments (to shows in the reviewed month) are read as the shows
+> themselves are reviewed, so the full texts are not needed in this handout.
+>
+> In addition to this, there is a possibility that certain comments relating to
+> past shows were already discussed last month, because they were made before
+> that show was recorded. We don't want to read them again during this show, so
+> a means of marking them is needed.
+>
+> As well as this, some comments may have been missed in the last month because
+> the recording was made before the end of the review month. On some months of
+> the year the recording is made during the month itself because the first
+> Monday of the next month is in the first few days of the next month. For
+> example, in March 2019 the date of recording is the 30th, and the show is
+> released on April 1st. Between the recording and the release of the show there
+> is time during which more comments could be submitted.
+>
+> Such comments should be in the notes for March (and these can be regenerated
+> to make sure this is so) but they will not have been read on the March
+> recording. The **make\_shownotes** script detects this problem and, if
+> generating full notes will show a list of any eligible comments in a red
+> highlighted box. This is so that the volunteers recording the show can ensure
+> they read comments that have slipped through this loophole.
+>
+> The script extracts the date of the last recording  from the _cache_ file (or
+> it can be specified with the **-lastrecording=DATETIME** option, or its
+> abbreviation **-lr=DATETIME**) and passes it to the template. The template can
+> then compare this date with the dates of the relevant comments and take action
+> to highlight those that don't want to be re-read or were missed last month. It
+> is up to the template to do what is necessary to highlight them.
+
+# DESCRIPTION
+
+## Overview
+
+This script generates notes for the next Hacker Public Radio _Community News_
+show. It does this by collecting various details of activity from the HPR
+database and passing them to a template. The default template is called
+**shownote\_template.tpl** (defined in the configuration file) and this
+generates HTML, but any suitable textual format could be generated if
+required, by using a different template.
+
+## Data Gathering
+
+Four types of information are collected by the script:
+
+- **Host details**
+
+    Details of new hosts who have released new shows in the selected month
+
+- **Show details**
+
+    Details of shows which have been released in the selected month
+
+- **Comments**
+
+    Comments which have been submitted to the HPR website in the selected month.
+    These need to be related to shows in the current period or in the past.
+    Comments made about shows which have not yet been released (but are visible on
+    the website) are not included even though they are made in the current month.
+
+- **Mailing list threads**
+
+    A link to the current threads on the mailing list in the past month can be included. This
+    is done by default but can be skipped if the **-nomailnotes** option is used.
+
+## Report Generation
+
+The four components listed above are formatted in the following way by the
+default template.
+
+- **New Hosts**
+
+    These are formatted as a list of links to the **hostid** with the host's name.
+
+- **Shows**
+
+    These are formatted into an HTML table containing the show number, title and
+    host name. The show title is a link to the show page on the HPR website. The
+    host name is a link to the host page on the website.
+
+- **Comments**
+
+    These are formatted with &lt;article> tags separated by horizontal lines.
+    A &lt;header> shows the author name and title and a &lt;footer> displays a link to
+    the show and the show's host and the show title is also included. Where
+    relevant, the body of the article contains the comment text with line breaks.
+
+- **Mailing list discussions**
+
+    A link to the mail threads on the mailing list is included if the
+    **-mailnotes** option is chosen or defaulted.
+
+    See the explanation of the **-mailnotes** option for more details.
+
+## Variable, Field and Hash names
+
+If you wish to write your own template refer to the following lists for the
+names of items. Also refer to the default template **shownote\_template.tpl**
+for the techniques used there. (Note that **shownote\_template.tpl** is a soft
+link to the current default template, such as **shownote\_template12.tpl**, the
+soft link name is currently used in the configuration file).
+
+The hash and field names available to the template are as follows
+
+- **Global variables**
+
+        Variable Name          Details
+        -------------          -------
+        review_month           The month name of the report date
+        review_year            The year of the report date
+        comment_count          The number of comments in total
+        past_count             The number of comments on old shows
+        ignore_count           The number of past comments to ignore
+        missed_count           The number of comments missed last time
+        skip_comments          Set when -comments is omitted
+        mark_comments          Set when comments are being marked
+        ctext                  Set when the comment texts in the 'Past shows'
+                               section are to be shown
+        last_recording         The date the last recording was made
+                               in Unixtime format
+        last_month             The month prior to the month for which the notes are
+                               being generated (computed if comments are being
+                               marked) in 'YYYY-MM' format
+        mailnotes              Set when a mail link is required
+
+- **New Hosts**
+
+    The name of the hash in the template is **hosts**. The hash might be empty if
+    there are no new hosts in the month. See the default template for how to
+    handle this.
+
+        Field Name     Details
+        ----------     -------
+        host           Name of host
+        hostid         Host id number
+
+- **Show Details**
+
+    The name of the hash in the template is **shows**. Note that there are more
+    fields available than are used in the default template. Note also that certain
+    field names are aliases to avoid clashes (e.g. _eps\_hostid_ and _ho\_hostid_).
+
+        Field Name     Details
+        ----------     -------
+        eps_id         Episode number
+        date           Episode date
+        title          Episode title
+        length         Episode duration
+        summary        Episode summary
+        notes          Episode show notes
+        eps_hostid     The numerical host id from the 'eps' table
+        series         The series number from the 'eps' table
+        explicit       The explicit marker for the show
+        eps_license    The license for the show
+        tags           The show's tags as a comma-delimited string
+        version        ?Obsolete?
+        eps_valid      The valid value from the 'eps' table
+        ho_hostid      The host id number from the 'hosts' table
+        ho_host        The host name
+        email          The hosts's email address (protected)
+        profile        The host's profile
+        ho_license     The default license for the host
+        ho_valid       The valid value from the 'hosts' table
+
+- **Comment Details**
+
+    Two hashes are created for comments. The hash named **past** contains comments
+    made in the review month to shows before this month, and **current** contains
+    comments to this month's shows. Note that these hashes are only populated if
+    the **-comments** option is provided or defaulted. Both hashes have the same
+    structure.
+
+        Field Name             Details
+        ----------             -------
+        episode                Episode number
+        identifier_url         Full show URL
+        title                  Episode title
+        date                   Episode date
+        host                   Host name
+        hostid                 Host id number
+        timestamp              Comment timestamp in ISO8601 format
+        comment_author_name    Name of the commenter
+        comment_title          Title of comment
+        comment_text           Text of the comment
+        comment_timestamp_ut   Comment timestamp in Unixtime format
+        in_range               Boolean (0/1) denoting whether the comment was made
+                               in the target month
+        index                  The numerical index of the comment for a given show
+        ignore                 Boolean (0/1), set if the comment is to be ignored
+
+    The purpose of the **in\_range** value is to denote whether a comment was made
+    in the target month. This is used in the script to split the comments into the
+    **past** and **current** hashes. It is therefore of little use in the template,
+    but is retained in case it might be useful. The **index** value can be used in
+    the template to refer to the comment, make linking URLs etc. It is generated
+    by the SQL.
+
+- **Mailing List Link**
+
+    The variable **mailnotes** contains 0 or 1 depending on whether the link and
+    accompanying text are required.
+
+## Filters
+
+A filter called **decode\_entities** is available to the template. The reason
+for creating this was when the HTML of a comment is being listed as text
+(Unicode actually). Since comment text is stored in the database as HTML with
+entities when appropriate this is needed to prevent the plain text showing
+_&amp;amp;_ and the like verbatim. It is not currently used, but has been left in
+case it might be of use in future.
+
+# DIAGNOSTICS
+
+- **Unable to find configuration file ...**
+
+    The nominated configuration file in **-config=FILE** (or the default file)
+    cannot be found.
+
+- **Error: Unable to find AOB file ...**
+
+    The AOB file referred to in the error message is missing. Th
+
+- **Use -lastrecording=DATETIME only with -full-html=FILE**
+
+    The **-lastrecording=DATETIME** option is only relevant when the full
+    stand-alone HTML is being generated. This is a fatal error.
+
+- **At least one of -html=FILE, -full-html=FILE and -json=FILE must be present**
+
+    The script writes up to three output files, as explained above. At least one
+    must be present otherwise there is no point in running it!
+
+- **Error: Unable to find ... file ...**
+
+    The type of mandatory file with the name referred to in the error message is missing.
+
+- **The date and time of the last recording is not in the cache**
+
+    Followed by:
+
+        Use option -lastrecording=DATETIME (or -lr=DATETIME) instead
+        Can't continue
+
+    This means that the date and time expected in the date cache cannot be found,
+    so the script needs to be run again with the information presented in the
+    option mentioned.
+
+- **Unable to find database**
+
+    The SQLite database referenced in the configuration file has not been found.
+
+- **Trying to overwrite an existing show. Aborting**
+
+    After a check on the database, the computed episode number matches a slot that
+    has already been allocated. A check has been made for an old-style placeholder
+    for Community News episodes, but no match was found, so the script has aborted
+    in case an existing episode will be overwritten.
+
+- **Error: show ... has a date in the past**
+
+    The date computed for the Community News episode is in the past. Perhaps the
+    wrong date or month was specified in an option?
+
+- **Problem with constructing JSON**
+
+    A request for JSON output has been made. The script is attempting to collect
+    information about the host who is preparing the show (_HPR Volunteers_) but
+    the database query has failed.
+
+    It is possible the configuration file entry _hostid_ contains the wrong host
+    id number.
+
+- **Unable to find a reference show**
+
+    The script is attempting to find the release date (and episode number) for the
+    Community News show in the database. It does this by finding a reference
+    episode and stepping forward, incrementing the episode number and date. The
+    reference date is the earliest in the target month, but for some unexpected
+    reason this has not been found.
+
+- **Invalid DATE or DATETIME '...' in parse\_to\_dc**
+
+    It is likely that the date provided through the **-from=DATE** option is
+    invalid. Use an ISO8601 date in the format _YYYY-MM-DD_.
+
+- **... failed to open '...': ...**
+
+    There was a problem opening the date cache file. Check the details in the
+    configuration file. This file is expected to be located in the same directory
+    as the script.
+
+- **... failed to close '...': ...**
+
+    There was a problem closing the date cache file.
+
+- **Invalid Date::Calc date and time (...) in dc\_to\_dt**
+
+    There was a problem processing a date (and time). A likely cause is an invalid
+    date in one of options which requires and date or date and time. The script
+    will report more than usual on this error to try and aid with debugging.
+
+# CONFIGURATION AND ENVIRONMENT
+
+The script obtains the details and settings it required from a configuration
+file. The name of the file it expects is **.make\_shownotes.cfg**, but this can
+be changed with the **-config=FILE** option.
+
+The configuration file is expected to be in the directory holding the
+script. The script determines its location dynamically for this purpose.
+
+The configuration file contains the following settings, which are explained
+at the end.
+
+    #
+    # .make_shownotes.cfg (2025-04-13)
+    # Configuration file for make_shownotes version >= 4
+    #
+    <settings>
+        # Format strings (using 'printf' formatting) for building certain required strings
+        title_template   = HPR Community News for %s %s
+        summary_template = HPR Volunteers talk about shows released and comments posted in %s %s
+    
+        # Repeat the following line with each of the desired tags to make an
+        # array-like structure
+        tags             = Community News
+    
+        # Host id for HPR Volunteers
+        # Series id for HPR Community News series
+        hostid = 159
+        series_id = 47
+    
+        # Day the Community News show is released
+        releaseday = Monday
+    
+        # Default day of the week for the recording
+        recordingday = Friday
+    
+        # Recording times are UTC
+        starttime = 16:00:00
+        endtime = 17:00:00
+    
+        # cache of previous recording dates and times
+        cache = recording_dates.dat
+    
+        # Template Toolkit templates
+        # --------------------------
+    
+        # Main note template (actually a soft link)
+        main_template = shownote_template.tpl
+    
+        # Used to make a stand-alone HTML file from the default HTML
+        # fragment
+        container_template = shownotes_container.tpl
+    
+    </settings>
+    
+    <database>
+        # Assume a local file
+        name = hpr.db
+    </database>
+
+## Configuration file details
+
+If any mandatory elements are omitted from the configuration file they are
+given default values in the script.
+
+- **Section "settings"**
+    - **title\_template**, **summary\_template**
+
+        These are (_sprintf_) templates for these database fields, which are added to the full HTML
+        and the JSON. They are filled in (using _sprintf_) in the script.
+
+    - **tags**
+
+        Up to now, Community News episodes have used the tag **"Community News"**, so
+        this is now used in the configuration file.
+
+    - **hostid**, **series\_id**
+
+        As documented in the file, these are the internal numbers from the database
+        for the host and series associated with the show.
+
+    - **releaseday**, **recordingday**, **starttime**, **endtime**
+
+        These values define default dates and times for the recording of the show and
+        the date of release.
+
+    - **cache**
+
+        This is the name of the file holding recording dates and times for Community
+        News shows. The file is updated when **make\_email** is run since this script
+        defines the recording date and time. The script expects this file to be in the
+        directory holding the script itself, though an absolute path could be used if
+        needed.
+
+    - **main\_template**, **container\_template**
+
+        These are _TT2_ templates to be used by the script. The first is the template
+        used to generate the two types of HTML, and the second is used to generate
+        stand-alone HTML when the **-full-html=FILE** option is used. The script uses
+        both templates when generating the full HTML.
+
+        The **container\_template** contains a snapshot of an HPR HTML page with the body
+        removed and the header and footer retained. The generated notes are added by
+        the script, giving the HTML which can be handed out to the volunteers hosting
+        the recording.
+- **Section "database"**
+    - **name**
+
+        This contains the name of the SQLite database. This is a file which is
+        expected to be in the same directory as the script.
+
+# DEPENDENCIES
+
+Modules used:
+
+    Carp
+    Config::General
+    Cwd
+    DBI
+    Data::Dumper
+    Date::Calc
+    Date::Parse
+    DateTime::Duration
+    DateTime
+    File::Copy
+    Getopt::Long
+    HTML::Entities
+    JSON
+    Pod::Usage
+    Template::Filters
+    Template
+
+# BUGS AND LIMITATIONS
+
+There are no known bugs in this module.
+Please report problems to Dave Morriss (Dave.Morriss@gmail.com)
+Patches are welcome.
+
+# AUTHOR
+
+Dave Morriss  (Dave.Morriss@gmail.com)
+
+# LICENCE AND COPYRIGHT
+
+Copyright (c) 2014-2025 Dave Morriss (Dave.Morriss@gmail.com). All rights reserved.
+
+This module is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself. See perldoc perlartistic.
+
+This program is distributed in the hope that it will be useful
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

Community_News/make_email

@@ -87,7 +87,7 @@ our $VERSION = '0.3.4';
 ( my $basedir = abs_path($0) ) =~ s|/?[^/]*$||mx;
 my $configfile = "$basedir/.${PROG}.cfg";
 
-my ( %recdates, $rdfh );
+my ( %recdates );
 
 #
 # Run in the script's directory
@@ -627,7 +627,7 @@ elsif ( ! -e $cfg_recdatefile) {
 # Load the recording dates
 #
 if ($cfg_recdatefile) {
-    %recdates = load_cache($cfg_recdatefile, $rdfh);
+    %recdates = load_cache($cfg_recdatefile);
 }
 
 _debug($DEBUG >= 2,
@@ -1617,24 +1617,24 @@ Reports the following (as well as the data for level 1):
 
 =over 4
 
-=item .
+=item *
 
 Details of the start date chosen
 
-=item .
+=item *
 
 Details of the year, name of month, readable date, and recording start and end
 times.
 
-=item .
+=item *
 
 The subject line chosen for the email.
 
-=item .
+=item *
 
 The date of the show being searched for in the database.
 
-=item .
+=item *
 
 The number of the show found in the database.
 

Community_News/make_shownotes

@@ -29,10 +29,10 @@
 #               Where pod2pdf comes from App::pod2pdf
 #
 #       AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
-#      VERSION: 0.4.4
+#      VERSION: 0.4.5
 #     ORIGINAL: 2014-04-24 16:08:30
 #      CREATED: 2025-03-13 15:07:35
-#     REVISION: 2025-04-17 15:12:27
+#     REVISION: 2025-08-09 11:28:22
 #
 #===============================================================================
 
@@ -51,6 +51,8 @@ use Pod::Usage qw(pod2usage);           # Use colour-capable Pod::Text
 
 use Config::General;
 
+use File::Copy;
+
 use Date::Parse;
 use Date::Calc qw{:all};
 use DateTime;
@@ -71,7 +73,7 @@ use Data::Dumper;
 #
 # Version number (manually incremented)
 #
-our $VERSION = '0.4.4';
+our $VERSION = '0.4.5';
 
 #
 # Various constants
@@ -103,14 +105,13 @@ my $defseries_id = 47;
 my $defcache     = "$basedir/recording_dates.dat";
 my $defcontainer = "$basedir/shownotes_container.tpl";
 
-my ( $dbh,           $sth1, $h1 );
+my ( $dbh,          $sth1,        $h1 );
-my ( @review_month,  $releasedate, @releasedate, $hosts, $shows, $episode );
+my ( @review_month, $releasedate, @releasedate, $hosts, $shows, $episode );
-my ( @dc_lr,         $dt_lr,       @dc_lm,       $dt_lm, @dc_rd, $dt_rd );
+my ( @dc_lr,        $dt_lr,       @dc_lm,       $dt_lm, @dc_rd, $dt_rd );
-my ( %attributes );
+my ( %attributes,   %date_cache,  $lr_option_status, $date_offset, @deftime );
-my ( %date_cache, $date_offset,     @deftime );
+my ( $t_time,       $missed_comments, $missed_count );
-my ( $t_time,     $missed_comments, $missed_count );
+my ( $comments,     $comment_count,   $past_count, $ignore_count );
-my ( $comments,   $comment_count,   $past_count, $ignore_count );
+my ( %past,         %current );
-my ( %past,       %current );
 
 #-------------------------------------------------------------------------------
 # The structure of the JSON to be sent to the HPR server
@@ -195,8 +196,8 @@ my $json_outfile      = $options{json};
 die "Unable to find configuration file $cfgfile\n" unless ( -e $cfgfile );
 
 #
-# We're receiving the datetime for the last recording (that's the recording
+# If we're receiving the datetime for the last recording (that's the recording
-# for the previous month), which isn't appropriate unless we're marking
+# for the previous month), it isn't appropriate unless we're marking
 # comments.
 #
 if (defined($lastrecording)) {
@@ -204,6 +205,11 @@ if (defined($lastrecording)) {
         unless defined($full_html_outfile);
 }
 
+#
+# Record whether or not we got a -lastrecording=DATETIME option
+#
+$lr_option_status = defined($lastrecording);
+
 #
 # One at least of the output files must be present
 #
@@ -213,7 +219,7 @@ unless ($full_html_outfile || $html_outfile || $json_outfile) {
     die "Missing output file option\n";
 }
 
-say "DEBUG level is %d", $DEBUG if $DEBUG > 0;
+say "DEBUG level is $DEBUG" if $DEBUG > 0;
 
 #-------------------------------------------------------------------------------
 # Use the date provided for the review month or use today's date as the default
@@ -231,6 +237,7 @@ else {
     @review_month = Today();
 }
 $review_month[2] = 1;
+_debug( $DEBUG > 1, '@review_month = ' . Dumper( \@review_month ) );
 @dc_lm = @review_month; # TODO: Is this right?
 
 #-------------------------------------------------------------------------------
@@ -238,9 +245,9 @@ $review_month[2] = 1;
 #-------------------------------------------------------------------------------
 emit( $silent, "Configuration file: ", $cfgfile, "\n" );
 my $conf = Config::General->new(
-    -ConfigFile      => $cfgfile,
+    -ConfigFile       => $cfgfile,
-    -InterPolateVars => 1,
+    -InterPolateVars  => 1,
-    -ExtendedAccess  => 1,
+    -ExtendedAccess   => 1,
     -UseApacheInclude => 1,
 );
 my %config = $conf->getall();
@@ -291,6 +298,11 @@ $tags = [$tags] unless ( ref($tags) eq 'ARRAY' );
 @deftime = split( ':', $start_time );
 my $release_dow = Decode_Day_of_Week($release_day);
 
+#
+# Report the date cache name before working on it.
+#
+emit( $silent, "Date cache: ", $date_cache_name, "\n" );
+
 #-------------------------------------------------------------------------------
 # Set last recording date and time from option or cache
 #-------------------------------------------------------------------------------
@@ -298,18 +310,38 @@ my $release_dow = Decode_Day_of_Week($release_day);
 # recording for the previous month) as an option, or we'll check the
 # cache.
 #
-if (defined($lastrecording)) {
+if ($lr_option_status) {
     #
-    # Parse and perform rudimentary validation on the -lastrecording option
+    # The -lastrecording option is present, so parse and perform rudimentary
+    # validation on the values.
     #
     emit( $silent, "Last recording from option: ", $lastrecording, "\n" );
     _debug( $DEBUG > 1, '$lastrecording = ' . $lastrecording );
 
     # Compute the last month, with a date
     @dc_lm = find_last_month(\@review_month);
+    _debug( $DEBUG > 1, '@dc_lm = ' . Dumper( \@dc_lm ) );
 
     @dc_lr = parse_to_dc( $lastrecording, \@deftime );
     _debug( $DEBUG > 1, '@dc_lr = ' . Dumper( \@dc_lr ) );
+
+    #
+    # Validate the provided lastrecording date which should be a few days
+    # before the start of the review month or into the next month. This will
+    # make it between 26 and 35 days from the start of the previous month.
+    #
+    my $lr_delta_days
+        = abs( Delta_Days( @dc_lm[ 0 .. 2 ], @dc_lr[ 0 .. 2 ] ) );
+    _debug(
+        $DEBUG > 1,
+        'Difference between @dc_lm and @dc_lr: ' . $lr_delta_days
+    );
+    if ($lr_delta_days < 26 || $lr_delta_days > 35) {
+        say "Problem with -lastrecording=DATETIME specification.";
+        say "Difference between given date and start of the month before ",
+            "the review month is $lr_delta_days days";
+        die "Can't continue\n";
+    }
 }
 else {
     emit( $silent, "Getting last recording from cache\n" );
@@ -317,7 +349,6 @@ else {
     #
     # Load the cache
     #
-    emit( $silent, "Date cache: ", $date_cache_name, "\n" );
     %date_cache = load_cache($date_cache_name);
     #_debug( $DEBUG > 1, '%date_cache = ' . Dumper(\%date_cache) );
 
@@ -331,8 +362,8 @@ else {
     # Abort if the cache didn't have the date
     #
     unless (defined($lastrecording)) {
-        say "The date and time of the last recording is not in the cache";
+        say "The date and time of the last recording is not in the cache. Use";
-        say "Use option -lastrecording=DATETIME (or -lr=DATETIME) instead";
+        say "option -lastrecording=DATETIME (or -lr=DATETIME) to define them.";
         die "Can't continue\n";
     }
 
@@ -752,7 +783,7 @@ if ($show_comments) {
             #
             if ( $full_html_outfile ) {
                 {
-                    $/ = '';
+                    local $/ = '';
                     chomp($row->{comment_text}); # NOTE: experimental
                 }
             }
@@ -909,6 +940,48 @@ my $tt = Template->new(
     }
 );
 
+#-------------------------------------------------------------------------------
+# Update the cache from the -lastrecording=DATETIME option if needed
+#-------------------------------------------------------------------------------
+if ($lr_option_status) {
+    #
+    # We were given the last recording as an option. The most likely reason is
+    # that it's not in the cache, but it may contain a correction.  Look to
+    # see if it is in the cache, and if so, whether it's the same. Add it if
+    # it's missing or update it unless it agrees.
+    #
+    emit( $silent, "Loading recording date cache\n" );
+
+    #
+    # Load the cache
+    #
+    %date_cache = load_cache($date_cache_name);
+    #_debug( $DEBUG > 1, '%date_cache = ' . Dumper(\%date_cache) );
+
+    #
+    # Create the month key from the month before the review month, then see if
+    # it's already in the cache. The date of the start of last month is in
+    # $dt_lm.
+    #
+    my $monthkey = sprintf( "%d-%02d-01", $dt_lm->year, $dt_lm->month );
+
+    #
+    # If the key is not in the %date_cache OR if it's already there but the
+    # value part doesn't match the last recording specification run
+    # update_cache to make changes. The tests are made in this order so we
+    # don't try and reference a non-existent element.
+    #
+    if (!exists( $date_cache{$monthkey} )
+        || ( exists( $date_cache{$monthkey} )
+            && ( $date_cache{$monthkey} ne $lastrecording ) )
+        )
+    {
+        $date_cache{$monthkey} = $lastrecording;
+        update_cache( $date_cache_name, \%date_cache );
+        emit( $silent, "Updated date cache\n" );
+    }
+}
+
 #-------------------------------------------------------------------------------
 # Generate the HTML fragment and add it to the JSON if that output is requested
 #-------------------------------------------------------------------------------
@@ -1307,7 +1380,14 @@ sub dc_to_dt {
 #      PURPOSE: Load the date cache into a hash
 #   PARAMETERS: $cache_name     Name of file holding the cache
 #      RETURNS: Contents of cache as a hash
-#  DESCRIPTION:
+#  DESCRIPTION: Opens the nominated file, parses each record, and adds the
+#               data to a hash. The record should contain the following:
+#               * 'YYYY-MM-01' the month for which the details are being
+#               recorded
+#               * ',' comma field separator
+#               * 'YYYY-MM-DD HH:MM:SS' timestamp of the recording
+#               The file is closed once it has been scanned.  The function
+#               returns the completed hash.
 #       THROWS: No exceptions
 #     COMMENTS: None
 #     SEE ALSO: N/A
@@ -1317,24 +1397,102 @@ sub load_cache {
 
     my ( $month, $datetime, %result );
 
-    open( my $dc, '<', $cache_name )
+    #
-        or die "$0 : failed to open '$cache_name': $!\n";
+    # Open the file in read mode
+    #
+    open( my $dcfh, '<', $cache_name )
+        or die "$PROG: failed to open '$cache_name': $!\n";
 
-    while ( my $line = <$dc> ) {
+    while ( my $line = <$dcfh> ) {
         chomp($line);
         if ( ( $month, $datetime )
             = ( $line =~ /^(\d{4}-\d{2}-\d{2}),(.*)$/ ) )
         {
             $result{$month} = $datetime;
         }
+        # TODO: Report any errors found in the file
     }
 
-    close($dc)
+    close($dcfh)
-        or warn "$0 : failed to close '$cache_name': $!\n";
+        or warn "${PROG}: failed to close '$cache_name': $!\n";
 
     return %result;
 }
 
+#===  FUNCTION  ================================================================
+#         NAME: append_cache
+#      PURPOSE: Append a new line to the cache
+#   PARAMETERS: $cache_name     Name of file holding the cache
+#               $line           New record to add
+#      RETURNS: Nothing
+#  DESCRIPTION: Opens the nominated file and appends the new record in $line.
+#               The file is closed once it has been updated.
+#       THROWS: No exceptions
+#     COMMENTS: None
+#     SEE ALSO: N/A
+#===============================================================================
+sub append_cache {
+    my ( $cache_name, $line ) = @_;
+
+    #
+    # Open the file in append mode
+    #
+    open( my $dcfh, '>>', $cache_name )
+        or die "$PROG: failed to open '$cache_name': $!\n";
+
+    say $dcfh $line;
+
+    close($dcfh)
+        or warn "${PROG}: failed to close '$cache_name': $!\n";
+}
+
+#===  FUNCTION  ================================================================
+#         NAME: update_cache
+#      PURPOSE: Make changes to an existing line in the cache
+#   PARAMETERS: $cache_name     Name of file holding the cache
+#               $rhash          Hashref holding the updated cache contents
+#      RETURNS: Nothing
+#  DESCRIPTION: Makes a backup of the nominated file. Opens, truncates it and
+#               positions for writing (using 'seek'). The now empty file is
+#               filled with data from the hash and closed.
+#       THROWS: No exceptions
+#     COMMENTS: Uses 'copy' from File::Copy
+#     SEE ALSO: N/A
+#===============================================================================
+sub update_cache {
+    my ( $cache_name, $rhash ) = @_;
+
+    #
+    # Copy the cache file to a backup
+    #
+    copy($cache_name,"${cache_name}~")
+        or die "Unable to back up '$cache_name'\n";
+
+    #
+    # Open the original file in write mode
+    #
+    open( my $dcfh, '>', $cache_name )
+        or die "${PROG}: failed to open '$cache_name': $!\n";
+
+    #
+    # Truncate the file and seek to the start again
+    #
+    truncate($dcfh,0)
+        or die "${PROG}: failed to truncate '$cache_name': $!\n";
+    seek($dcfh,0,0)
+        or die "$PROG: failed to seek in '$cache_name': $!\n";
+
+    #
+    # Write the cache data to the file
+    #
+    for my $key (sort(keys(%$rhash))) {
+        say $dcfh sprintf("%s,%s",$key, $rhash->{$key});
+    }
+
+    close($dcfh)
+        or warn "${PROG}: failed to close '$cache_name': $!\n";
+}
+
 #===  FUNCTION  ================================================================
 #         NAME: find_last_month
 #      PURPOSE: Finds the previous month for working out marks
@@ -1811,7 +1969,7 @@ make_shownotes - Make show notes for the Hacker Public Radio Community News show
 
 =head1 VERSION
 
-This documentation refers to B<make_shownotes> version 0.4.4
+This documentation refers to B<make_shownotes> version 0.4.5
 
 
 =head1 USAGE
@@ -1924,19 +2082,49 @@ This will cause the generation of the file:
 
 =item B<-lastrecording=DATETIME> or B<-lr=DATETIME>
 
-As mentioned for B<-full-html=FILE>, the script needs the date of the last
+As mentioned for B<-full-html=FILE>, and later in the I<MARKING COMMENTS>
-recording when marking comments. This can be extracted from the file
+section, the script needs the date of the last recording when marking
-referenced in the configuration data using the setting B<cache>. By default
+comments. This can be extracted from the file referenced in the configuration
-the name of this file is B<recording_dates.dat>, and its contents are managed
+data using the setting B<cache>. By default the name of this file is
-when the script B<make_email> is run.
+B<recording_dates.dat>, and its contents are managed when the script
+B<make_email> is run and by this script.
 
 If for any reason the date and time of the last recording is missing, these
-values can be defined with this option.
+values can be defined with this option, and these values will be written to
+the cache file (or modified, if necessary).
 
 The format can be an ISO 8601 date followed by a 24-hour time, such as
-'2020-01-25 15:00'. If the time is omitted it defaults to the value of
+'2020-01-25 15:00:00'. If the time is omitted it defaults to the value of
 I<starttime> in the configuration file.
 
+The script will update the cache file with the date and time used in this
+option if the relevant entry is missing. Also, if an entry is present but the
+values are different from those provided with the option, the relevant entry
+will be updated.
+
+Note that the B<DATETIME> value must contain the date of the last recording.
+This will be checked, and written to the cache file prefixed by a "key"
+consisting of the first day of the month I<BEFORE> the month being reviewed.
+
+For example, when generating the notes for August 2025 the following command
+will be needed if there is no last recording date (for July 2025) in the
+cache:
+
+    ./make_shownotes -from=2025-08-01 -full-html=full_shownotes_%s.html \
+        -mail -comments -lr="2025-08-01 15:00:00"
+
+Here we need the last recording date for the show reviewing HPR shows in July
+2025. The date and time for this recording was in early August (Friday before
+the first Monday of August, 2025-08-01), as shown. This combination will
+result in the addition of the following line to the cache file:
+
+    2025-07-01,2025-08-01 15:00:00
+
+As mentioned, the addition of such date and time information to the cache will
+normally be performed by B<make_email>, which performs the date computations
+itself, unlike this script. This feature in this script is an alternative for
+special cases.
+
 =item B<-[no]silent>
 
 This option controls whether the script reports details of its progress
@@ -2123,7 +2311,7 @@ default template.
 
 =item B<New Hosts>
 
-These are formatted as a list of links to the B<hostid> with the host's name.
+These are formatted as a list of links, each to the B<hostid> with the host's name.
 
 =item B<Shows>
 
@@ -2133,10 +2321,11 @@ host name is a link to the host page on the website.
 
 =item B<Comments>
 
-These are formatted with <article> tags separated by horizontal lines.
+These are formatted with B<<article>> tags separated by horizontal lines.
-A <header> shows the author name and title and a <footer> displays a link to
+A B<<header>> shows the author name and title and a B<<footer>> displays
-the show and the show's host and the show title is also included. Where
+a link to the show and the show's host and the show title is also included.
-relevant, the body of the article contains the comment text with line breaks.
+Where relevant, the body of the article contains the comment text with line
+breaks.
 
 =item B<Mailing list discussions>
 
@@ -2501,6 +2690,7 @@ Modules used:
  Date::Parse
  DateTime::Duration
  DateTime
+ File::Copy
  Getopt::Long
  HTML::Entities
  JSON

Community_News/recording_dates.dat

@@ -31,4 +31,8 @@
 2025-01-01,2025-01-31 15:00:00
 2025-02-01,2025-02-28 16:00:00
 2025-03-01,2025-04-04 16:00:00
-2025-04-01,2025-05-02 16:00:00
+2025-04-01,2025-05-02 15:00:00
+2025-05-01,2025-05-30 15:00:00
+2025-06-01,2025-07-04 15:00:00
+2025-07-01,2025-08-01 15:00:00
+2025-08-01,2025-08-29 15:00:00

Community_News/shownote_template.tpl

@@ -1 +0,0 @@
-shownote_template12.tpl

Community_News/shownote_template.tpl

@@ -0,0 +1,223 @@
+[%# shownote_template13.tpl Updated: 2025-08-20 -%]
+[%# -------------------------------------------------------------------------------- -%]
+[%# Makes either an HTML snippet for insertion into the database or a full           -%]
+[%# listing with full comments for circulation to the hosts recording the episode    -%]
+[%# This one uses the new format for the mailing list data, and partitions           -%]
+[%# comments into past and current. It also marks comments that don't need           -%]
+[%# to be read when mark_comments is true. It requires make_shownotes >= V0.0.30     -%]
+[%# -------------------------------------------------------------------------------- -%]
+[%- USE date -%]
+[%- USE pad4 = format('%04d') -%]
+[%- correspondents = "https://hackerpublicradio.org/correspondents"
+    mailinglist = "https://lists.hackerpublicradio.com/mailman/listinfo/hpr"
+    mailbase="https://lists.hackerpublicradio.com/pipermail/hpr"
+    mailthreads = "$mailbase/$review_year-$review_month/thread.html" -%]
+[%- DEFAULT skip_comments = 0
+    mark_comments = 0
+    ctext = 0
+    ignore_count = 0
+    missed_count = 0
+    past_count = 0
+-%]
+[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
+[%# Embedded CSS. The 'table' and 'hr' settings are always there but the rest is     -%]
+[%# only for when we are marking comments                                            -%]
+<style>
+table td.shrink {
+    white-space:nowrap
+}
+hr.thin {
+    border: 0;
+    height: 0;
+    border-top: 1px solid rgba(0, 0, 0, 0.1);
+    border-bottom: 1px solid rgba(255, 255, 255, 0.3);
+}
+[%- IF mark_comments == 1 %]
+p#ignore, li#ignore {
+    background-color: lightgreen;
+    color:maroon;
+}
+div#highlight {
+  border-style: solid;
+  border-color: red;
+  padding-right: 20px;
+  padding-left: 20px;
+}
+[%- END %]
+</style>
+[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
+<h2>New hosts</h2>
+<p>
+[% IF hosts.size > 0 -%]
+Welcome to our new host[%- hosts.size > 1 ? 's' : '' -%]: <br />
+[%- count = 0 %]
+[%# List the new hosts. If a name contains a comma quote it.                  -%]
+[%- FOREACH row IN hosts %]
+    [%- count = count + 1 %]
+    [%- hostname = (row.host.search(',') ? row.host.replace('^(.*)$','"$1"') : row.host) %]
+    <a href="[% correspondents %]/[% pad4(row.hostid) %].html" target="_blank">[% hostname %]</a>
+    [%- count < hosts.size ? ', ' : '.' %]
+[%- END %]
+[% ELSE -%]
+There were no new hosts this month.
+[% END -%]
+</p>
+
+<h2>Last Month's Shows</h2>
+[%# The id 't01' is in the HPR CSS but might give trouble on the IA -%]
+<table id="t01">
+<tr>
+<th>Id</th>
+<th>Day</th>
+<th>Date</th>
+<th>Title</th>
+<th>Host</th>
+</tr>
+[%- FOREACH row IN shows %]
+<tr>
+<td><strong><a href="https://hackerpublicradio.org/eps/hpr[% pad4(row.eps_id) %]/index.html" target="_blank">[% row.eps_id %]</a></strong></td>
+<td>[% date.format(row.date,'%a') %]</td>
+<td class="shrink">[% date.format(row.date,'%Y-%m-%d') %]</td>
+<td><a href="https://hackerpublicradio.org/eps/hpr[% pad4(row.eps_id) %]/index.html" target="_blank">[% row.title %]</a></td>
+<td><a href="[% correspondents %]/[% pad4(row.ho_hostid) %].html" target="_blank">[% row.ho_host %]</a></td>
+</tr>
+[%- END %]
+</table>
+
+[%# ---------------------------------------------------------------------------------------- -%]
+[%# Skip comments if told to by the caller -%]
+[%- IF skip_comments == 0 -%]
+[%# Handle any missed comments if mark_comments is true -%]
+[%- IF (mark_comments == 1) AND (missed_count > 0) -%]
+<br/><div id="highlight">
+<h2>Missed comment[%- missed_comments.size > 1 ? 's' : '' -%] last month</h2>
+<p><b>Note to Volunteers</b>: These are comments for shows last month that were not read in the last show because they arrived on or after the recording started. This section will be removed before these notes are released.</p>
+<ul>
+[%- FOREACH comment IN missed_comments -%]
+<li><strong><a href="[% comment.identifier_url %]#comments" target="_blank">hpr[% pad4(comment.episode) %]</a></strong>
+([% comment.date %]) "<em>[% comment.title %]</em>" by <a href="[% correspondents %]/[% pad4(comment.hostid) %].html" target="_blank">[% comment.host %]</a>.<br/>
+<small>Summary: "<em>[% comment.summary %]</em>"</small><br/>
+From: [% comment.comment_author_name %] on [% date.format(comment.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
+[%- IF comment.comment_title.length > 0 %]
+"[% comment.comment_title %]"
+[%- ELSE -%]
+"[no title]"
+[%- END -%]
+<br/><hr class="thin">[% comment.comment_text FILTER html_line_break %]
+</li><br/>
+[%- END -%]
+</ul></div>
+[%- END -%]
+[%# ---------------------------------------------------------------------------------------- -%]
+<h2>Comments this month</h2>
+
+[% IF comment_count > 0 -%]
+[%- IF (mark_comments == 1) AND (ignore_count > 0) -%]
+<p id="ignore"><b>Note to Volunteers</b>: Comments marked in green were read in the last
+Community News show and should be ignored in this one.</p>
+[%- END -%]
+<p>These are comments which have been made during the past month, either to shows released during the month or to past shows.
+There [%- comment_count == 1 ? "is $comment_count comment" : "are $comment_count comments" -%] in total.</p>
+[% IF past_count > 0 -%]
+<h3>Past shows</h3>
+<p>There [% past_count == 1 ? "is $past_count comment" : "are $past_count comments" %] on
+[% past.size %] previous [% past.size == 1 ? "show" : "shows" %]:</p>
+<ul>
+[%# Loop through by episode then by comment relating to that episode -%]
+[%-    FOREACH ep IN past.keys.nsort -%]
+[%-       arr = past.$ep -%]
+<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
+([%       arr.0.date %]) "<em>[% arr.0.title %]</em>"
+by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.<br/>
+[%-       IF mark_comments == 1 || ctext == 1 -%]
+<small>Summary: "<em>[% arr.0.summary %]</em>"</small></li>
+[%-       END %]
+<li style="list-style: none; display: inline">
+<ul>
+[%-       FOREACH row IN arr -%]
+[%#          IF mark_comments == 1 && ((row.comment_timestamp_ut <= last_recording) && (arr.0.date.substr(0,7) == last_month)) -%]
+[%#          IF mark_comments == 1 && ((row.comment_released_ut <= last_recording) && (arr.0.date.substr(0,7) == last_month)) -%]
+[%-          IF mark_comments == 1 && row.ignore == 1 -%]
+<li id="ignore">
+[%-          ELSE %]
+<li>
+[%-          END %]
+<a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
+[%        row.comment_author_name -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
+[%-          IF row.comment_title.length > 0 %]
+"[%             row.comment_title %]"
+[%-          ELSE -%]
+"[no title]"
+[%-          END -%]
+[%# Add the comment body in too if ctext is true -%]
+[%-          IF ctext == 1 %]
+<br/><hr class="thin">[% row.comment_text FILTER html_line_break %]
+</li><br/>
+[%-          ELSE -%]
+</li>
+[%-          END -%]
+[%-       END -%]
+</ul><br/>
+</li>
+[%-    END -%]
+</ul>
+[%- END %]
+[%# ---------------------------------------------------------------------------------------- -%]
+[% cc = (comment_count - past_count) -%]
+[% IF cc > 0 -%]
+<h3>This month's shows</h3>
+<p>There [% cc == 1 ? "is $cc comment" : "are $cc comments" %] on [% current.size %] of this month's shows:</p>
+<ul>
+[%- FOREACH ep IN current.keys.nsort -%]
+[%- arr = current.$ep -%]
+<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
+([% arr.0.date %]) "<em>[% arr.0.title %]</em>"
+by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.</li>
+<li style="list-style: none; display: inline">
+<ul>
+[%- FOREACH row IN arr -%]
+<li><a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
+[% row.comment_author_name -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
+[%- IF row.comment_title.length > 0 %]
+"[% row.comment_title %]"
+[%- ELSE -%]
+"[no title]"
+[%- END -%]
+</li>
+[%- END -%]
+</ul><br/>
+</li>
+[%- END -%]
+</ul>
+[%- END %]
+[%- ELSE %]
+There were no comments this month.
+[%- END %]
+[%- END %]
+
+[%# ---------------------------------------------------------------------------------------- -%]
+[%- IF mailnotes == 1 -%]
+<h2>Mailing List discussions</h2>
+<p>
+Policy decisions surrounding HPR are taken by the community as a whole. This
+discussion takes place on the <a href="[% mailinglist %]" target="_blank">Mailing List</a>
+which is open to all HPR listeners and contributors. The discussions are open
+and available on the HPR server under <a href="[% mailbase %]">Mailman</a>.
+</p>
+<p>The threaded discussions this month can be found here:</p>
+<a href="[% mailthreads %]" target="_blank">[% mailthreads %]</a>
+[%- END %]
+
+[%# ---------------------------------------------------------------------------------------- -%]
+<h2>Events Calendar</h2>
+<p>With the kind permission of <strong>LWN.net</strong> we are linking to
+<a href="https://lwn.net/Calendar/" target="_blank">The LWN.net Community Calendar</a>.</p>
+<p>Quoting the site:</p>
+<blockquote>This is the LWN.net community event calendar, where we track
+events of interest to people using and developing Linux and free software.
+Clicking on individual events will take you to the appropriate web
+page.</blockquote>
+[%# ---------------------------------------------------------------------------------------- -%]
+[%#
+  # vim: syntax=tt2:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
+-%]

Community_News/shownote_template12.tpl

@@ -161,9 +161,6 @@ by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">
 </li>
 [%-    END -%]
 </ul>
-[%-    IF mark_comments == 1 || ctext == 1 -%]
-<p><small><small><em>Updated on [% date.format(date.now,'%Y-%m-%d %H:%M:%S') %]</em></small></small></p>
-[%-    END -%]
 [%- END %]
 [%# ---------------------------------------------------------------------------------------- -%]
 [% cc = (comment_count - past_count) -%]

Community_News/shownote_template13.tpl

@@ -0,0 +1,223 @@
+[%# shownote_template13.tpl Updated: 2025-08-20 -%]
+[%# -------------------------------------------------------------------------------- -%]
+[%# Makes either an HTML snippet for insertion into the database or a full           -%]
+[%# listing with full comments for circulation to the hosts recording the episode    -%]
+[%# This one uses the new format for the mailing list data, and partitions           -%]
+[%# comments into past and current. It also marks comments that don't need           -%]
+[%# to be read when mark_comments is true. It requires make_shownotes >= V0.0.30     -%]
+[%# -------------------------------------------------------------------------------- -%]
+[%- USE date -%]
+[%- USE pad4 = format('%04d') -%]
+[%- correspondents = "https://hackerpublicradio.org/correspondents"
+    mailinglist = "https://lists.hackerpublicradio.com/mailman/listinfo/hpr"
+    mailbase="https://lists.hackerpublicradio.com/pipermail/hpr"
+    mailthreads = "$mailbase/$review_year-$review_month/thread.html" -%]
+[%- DEFAULT skip_comments = 0
+    mark_comments = 0
+    ctext = 0
+    ignore_count = 0
+    missed_count = 0
+    past_count = 0
+-%]
+[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
+[%# Embedded CSS. The 'table' and 'hr' settings are always there but the rest is     -%]
+[%# only for when we are marking comments                                            -%]
+<style>
+table td.shrink {
+    white-space:nowrap
+}
+hr.thin {
+    border: 0;
+    height: 0;
+    border-top: 1px solid rgba(0, 0, 0, 0.1);
+    border-bottom: 1px solid rgba(255, 255, 255, 0.3);
+}
+[%- IF mark_comments == 1 %]
+p#ignore, li#ignore {
+    background-color: lightgreen;
+    color:maroon;
+}
+div#highlight {
+  border-style: solid;
+  border-color: red;
+  padding-right: 20px;
+  padding-left: 20px;
+}
+[%- END %]
+</style>
+[%# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -%]
+<h2>New hosts</h2>
+<p>
+[% IF hosts.size > 0 -%]
+Welcome to our new host[%- hosts.size > 1 ? 's' : '' -%]: <br />
+[%- count = 0 %]
+[%# List the new hosts. If a name contains a comma quote it.                  -%]
+[%- FOREACH row IN hosts %]
+    [%- count = count + 1 %]
+    [%- hostname = (row.host.search(',') ? row.host.replace('^(.*)$','"$1"') : row.host) %]
+    <a href="[% correspondents %]/[% pad4(row.hostid) %].html" target="_blank">[% hostname %]</a>
+    [%- count < hosts.size ? ', ' : '.' %]
+[%- END %]
+[% ELSE -%]
+There were no new hosts this month.
+[% END -%]
+</p>
+
+<h2>Last Month's Shows</h2>
+[%# The id 't01' is in the HPR CSS but might give trouble on the IA -%]
+<table id="t01">
+<tr>
+<th>Id</th>
+<th>Day</th>
+<th>Date</th>
+<th>Title</th>
+<th>Host</th>
+</tr>
+[%- FOREACH row IN shows %]
+<tr>
+<td><strong><a href="https://hackerpublicradio.org/eps/hpr[% pad4(row.eps_id) %]/index.html" target="_blank">[% row.eps_id %]</a></strong></td>
+<td>[% date.format(row.date,'%a') %]</td>
+<td class="shrink">[% date.format(row.date,'%Y-%m-%d') %]</td>
+<td><a href="https://hackerpublicradio.org/eps/hpr[% pad4(row.eps_id) %]/index.html" target="_blank">[% row.title %]</a></td>
+<td><a href="[% correspondents %]/[% pad4(row.ho_hostid) %].html" target="_blank">[% row.ho_host %]</a></td>
+</tr>
+[%- END %]
+</table>
+
+[%# ---------------------------------------------------------------------------------------- -%]
+[%# Skip comments if told to by the caller -%]
+[%- IF skip_comments == 0 -%]
+[%# Handle any missed comments if mark_comments is true -%]
+[%- IF (mark_comments == 1) AND (missed_count > 0) -%]
+<br/><div id="highlight">
+<h2>Missed comment[%- missed_comments.size > 1 ? 's' : '' -%] last month</h2>
+<p><b>Note to Volunteers</b>: These are comments for shows last month that were not read in the last show because they arrived on or after the recording started. This section will be removed before these notes are released.</p>
+<ul>
+[%- FOREACH comment IN missed_comments -%]
+<li><strong><a href="[% comment.identifier_url %]#comments" target="_blank">hpr[% pad4(comment.episode) %]</a></strong>
+([% comment.date %]) "<em>[% comment.title %]</em>" by <a href="[% correspondents %]/[% pad4(comment.hostid) %].html" target="_blank">[% comment.host %]</a>.<br/>
+<small>Summary: "<em>[% comment.summary %]</em>"</small><br/>
+From: [% comment.comment_author_name %] on [% date.format(comment.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
+[%- IF comment.comment_title.length > 0 %]
+"[% comment.comment_title %]"
+[%- ELSE -%]
+"[no title]"
+[%- END -%]
+<br/><hr class="thin">[% comment.comment_text FILTER html_line_break %]
+</li><br/>
+[%- END -%]
+</ul></div>
+[%- END -%]
+[%# ---------------------------------------------------------------------------------------- -%]
+<h2>Comments this month</h2>
+
+[% IF comment_count > 0 -%]
+[%- IF (mark_comments == 1) AND (ignore_count > 0) -%]
+<p id="ignore"><b>Note to Volunteers</b>: Comments marked in green were read in the last
+Community News show and should be ignored in this one.</p>
+[%- END -%]
+<p>These are comments which have been made during the past month, either to shows released during the month or to past shows.
+There [%- comment_count == 1 ? "is $comment_count comment" : "are $comment_count comments" -%] in total.</p>
+[% IF past_count > 0 -%]
+<h3>Past shows</h3>
+<p>There [% past_count == 1 ? "is $past_count comment" : "are $past_count comments" %] on
+[% past.size %] previous [% past.size == 1 ? "show" : "shows" %]:</p>
+<ul>
+[%# Loop through by episode then by comment relating to that episode -%]
+[%-    FOREACH ep IN past.keys.nsort -%]
+[%-       arr = past.$ep -%]
+<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
+([%       arr.0.date %]) "<em>[% arr.0.title %]</em>"
+by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.<br/>
+[%-       IF mark_comments == 1 || ctext == 1 -%]
+<small>Summary: "<em>[% arr.0.summary %]</em>"</small></li>
+[%-       END %]
+<li style="list-style: none; display: inline">
+<ul>
+[%-       FOREACH row IN arr -%]
+[%#          IF mark_comments == 1 && ((row.comment_timestamp_ut <= last_recording) && (arr.0.date.substr(0,7) == last_month)) -%]
+[%#          IF mark_comments == 1 && ((row.comment_released_ut <= last_recording) && (arr.0.date.substr(0,7) == last_month)) -%]
+[%-          IF mark_comments == 1 && row.ignore == 1 -%]
+<li id="ignore">
+[%-          ELSE %]
+<li>
+[%-          END %]
+<a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
+[%        row.comment_author_name -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
+[%-          IF row.comment_title.length > 0 %]
+"[%             row.comment_title %]"
+[%-          ELSE -%]
+"[no title]"
+[%-          END -%]
+[%# Add the comment body in too if ctext is true -%]
+[%-          IF ctext == 1 %]
+<br/><hr class="thin">[% row.comment_text FILTER html_line_break %]
+</li><br/>
+[%-          ELSE -%]
+</li>
+[%-          END -%]
+[%-       END -%]
+</ul><br/>
+</li>
+[%-    END -%]
+</ul>
+[%- END %]
+[%# ---------------------------------------------------------------------------------------- -%]
+[% cc = (comment_count - past_count) -%]
+[% IF cc > 0 -%]
+<h3>This month's shows</h3>
+<p>There [% cc == 1 ? "is $cc comment" : "are $cc comments" %] on [% current.size %] of this month's shows:</p>
+<ul>
+[%- FOREACH ep IN current.keys.nsort -%]
+[%- arr = current.$ep -%]
+<li><strong><a href="[% arr.0.identifier_url %]#comments" target="_blank">hpr[% pad4(arr.0.episode) %]</a></strong>
+([% arr.0.date %]) "<em>[% arr.0.title %]</em>"
+by <a href="[% correspondents %]/[% pad4(arr.0.hostid) %].html" target="_blank">[% arr.0.host %]</a>.</li>
+<li style="list-style: none; display: inline">
+<ul>
+[%- FOREACH row IN arr -%]
+<li><a href="[% row.identifier_url %]#comment_[% row.comment_id %]" target="_blank">Comment [% row.comment_number %]</a>:
+[% row.comment_author_name -%] on [% date.format(row.comment_timestamp_ut,'%Y-%m-%d','UTC') -%]:
+[%- IF row.comment_title.length > 0 %]
+"[% row.comment_title %]"
+[%- ELSE -%]
+"[no title]"
+[%- END -%]
+</li>
+[%- END -%]
+</ul><br/>
+</li>
+[%- END -%]
+</ul>
+[%- END %]
+[%- ELSE %]
+There were no comments this month.
+[%- END %]
+[%- END %]
+
+[%# ---------------------------------------------------------------------------------------- -%]
+[%- IF mailnotes == 1 -%]
+<h2>Mailing List discussions</h2>
+<p>
+Policy decisions surrounding HPR are taken by the community as a whole. This
+discussion takes place on the <a href="[% mailinglist %]" target="_blank">Mailing List</a>
+which is open to all HPR listeners and contributors. The discussions are open
+and available on the HPR server under <a href="[% mailbase %]">Mailman</a>.
+</p>
+<p>The threaded discussions this month can be found here:</p>
+<a href="[% mailthreads %]" target="_blank">[% mailthreads %]</a>
+[%- END %]
+
+[%# ---------------------------------------------------------------------------------------- -%]
+<h2>Events Calendar</h2>
+<p>With the kind permission of <strong>LWN.net</strong> we are linking to
+<a href="https://lwn.net/Calendar/" target="_blank">The LWN.net Community Calendar</a>.</p>
+<p>Quoting the site:</p>
+<blockquote>This is the LWN.net community event calendar, where we track
+events of interest to people using and developing Linux and free software.
+Clicking on individual events will take you to the appropriate web
+page.</blockquote>
+[%# ---------------------------------------------------------------------------------------- -%]
+[%#
+  # vim: syntax=tt2:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
+-%]

Community_News/shownotes_container.tpl

@@ -1,5 +1,6 @@
 [%# /home/cendjm/HPR/Community_News/shownotes_container.tpl 2024-06-22 -%]
 [%# Container to display Community News shownotes as an HTML page -%]
+[%- USE date -%]
 [% DEFAULT shownotes = ""
    episode = ""
    month_year = ""
@@ -52,7 +53,8 @@
       </h1>
     <h2>Your ideas, projects, opinions - podcasted.</h2>
     <h3>New episodes every weekday Monday through Friday.<br />
-    <em><small>Temporary version of the Community News notes for hosts to use when recording</small></em></h3>
+    <em><small>Temporary version of the Community News notes for hosts to use when recording<br />
+    Updated on [% date.format(date.now,'%Y-%m-%d %H:%M:%S') %]</small></em></h3>
 
   </div>
   <hr />

workflow/duration.bash

@@ -2,7 +2,14 @@
 # Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
 #============================================================
 
-find ${HOME}/processing/ -type f | egrep -v '/sponsor-anhonesthost.com-hpr15.flac|/outro.flac|/intro.flac|/sponsor-archive.org.flac' | while read mediafile
+search_dir="${HOME}/processing/"
+
+if [ -d "${1}" ]
+then
+  search_dir="${1}"
+fi
+
+find ${search_dir} -type f | grep -vP '/sponsor-anhonesthost.com-hpr15.flac|/outro.flac|/intro.flac|/sponsor-archive.org.flac' | while read mediafile
 do 
   duration=$( mediainfo --full --Output=XML "${mediafile}" | xmlstarlet sel -T -t -m "_:MediaInfo/_:media/_:track[@type='Audio']/_:Duration[1]" -v "." -n - | awk -F '.' '{print $1}' )
   if [ "${duration}" != "" ]

workflow/hpr-get-and-transcribe.bash

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+rsync -av --partial --progress hpr:hub.hackerpublicradio.org/upload/ $HOME/tmp/hpr/processing/
+
+find $HOME/tmp/hpr/processing/*_*_????-??-??_* -type d | sort -t _ -k 2 | while read show_dir
+do
+  echo "${show_dir}"
+  if [ "$( find "${show_dir}" -type f -iname "*srt" | wc -l )" -eq "0" ]
+  then
+    cd "${show_dir}"
+    ls -haltr
+    find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media
+    do
+      whisper --model tiny --language en --output_dir "${show_dir}" "${this_media}"
+    done
+    rsync -av --partial --progress "${show_dir}/" hpr:hub.hackerpublicradio.org/upload/$( basename "${show_dir}")/
+  fi
+done
+
+rsync -av --partial --progress hpr:hub.hackerpublicradio.org/reserve/ $HOME/tmp/hpr/reserve/
+
+find $HOME/tmp/hpr/reserve/*_*_* -type d | sort -t _ -k 2 | while read show_dir
+do
+  echo "${show_dir}"
+  if [ "$( find "${show_dir}" -type f -iname "*srt" | wc -l )" -eq "0" ]
+  then
+    cd "${show_dir}"
+    ls -haltr
+    find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media
+    do
+      whisper --model tiny --language en --output_dir "${show_dir}" "${this_media}"
+    done
+    rsync -av --partial --progress "${show_dir}/" hpr:hub.hackerpublicradio.org/reserve/$( basename "${show_dir}")/
+  fi
+done

workflow/hpr-publish.bash

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ 
+#============================================================
+
+git_dir="$HOME/tmp/hpr/hpr_generator/sourcecode"
+if [ ! -d "${git_dir}/.git" ]
+then
+  git clone gitea@repo.anhonesthost.net:HPR/hpr_generator.git "${git_dir}"
+fi
+
+cd "${git_dir}"
+
+git pull
+
+# ssh hpr -t "ls -al /home/hpr/www/hpr.sql;md5sum /home/hpr/www/hpr.sql"
+# ssh hpr -t "/home/hpr/bin/hpr_db_backup.bash"
+# ssh hpr -t "ls -al /home/hpr/www/hpr.sql;md5sum /home/hpr/www/hpr.sql"
+
+./utils/update-hpr-db.sh
+if [ $? -ne 0 ]
+then
+  echo 'Terminating...' >&2
+  exit 1
+fi
+
+./site-generator --all --verbose
+if [ $? -ne 0 ]
+then
+  echo 'Terminating...' >&2
+  exit 1
+fi
+
+rsync -av --partial --progress "${git_dir}/public_html/" hpr:hackerpublicradio.org/public_html
+#rsync -av --partial --progress "${git_dir}/public_html/" hobbypublicradio.org:hobbypublicradio.org/
+
+cd $HOME/sourcecode/hpr/hpr_hub/
+git pull 
+cd $HOME/sourcecode/hpr/hpr_hub/sql
+split --hex-suffixes --lines=1000 --additional-suffix=.sql hpr.sql hpr-db-part-
+cd $HOME/sourcecode/hpr/hpr_hub/
+git add $HOME/sourcecode/hpr/hpr_hub/sql/hpr*sql 
+git commit -m "$(\date -u +%Y-%m-%d_%H-%M-%SZ_%A ) database changed"
+git push
+#xdg-open https://hackerpublicradio.org/

workflow/hpr_ccdn_stats.bash

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+yesterday="$( \date -u +%Y-%m-%d -d yesterday)" 
+echo -e "${yesterday}\t$( grep -Ec "${yesterday}T.*Sending request to" /home/hpr/logs/naughty-ip.txt )" >> /home/hpr/hub/hpr_ccdn_stats.tsv

workflow/hpr_db_backup.bash

@@ -0,0 +1,48 @@
+#!/bin/bash
+sql_save_dir="$HOME/hpr/sql"
+credential_file="$HOME/.my.cnf"
+last_update_txt="${sql_save_dir}/last_update.txt"
+sync_delay="30 mins"
+
+last_update_query="SELECT update_time FROM information_schema.tables tab WHERE update_time > (current_timestamp() - interval 30 day) AND table_type = 'BASE TABLE' AND table_name not in ('reservations') AND table_schema not in ('information_schema', 'sys', 'performance_schema','mysql') ORDER BY update_time ASC LIMIT 1;"
+
+if [ ! -s "${credential_file}" ]
+then
+  echo "The file \"${credential_file}\" is missing"
+  exit 
+fi
+
+if [ -s "${last_update_txt}" ]
+then
+  echo "Found the last update file \"${last_update_txt}\""
+  local_db_last_update_iso8601="$( \date --utc --date="$( cat ${last_update_txt} )" +%Y-%m-%dT%H:%M:%SZ )"
+  local_db_last_update_epoch="$( \date --utc --date="$( cat ${last_update_txt} )" +%s )"
+  echo "Latest change saved locally is ${local_db_last_update_iso8601} (${local_db_last_update_epoch})"
+fi
+
+mysql --disable-column-names --batch --execute="${last_update_query}" | grep -v update_time | head -1 > ${last_update_txt}
+
+if [ ! -s "${last_update_txt}" ]
+then
+  echo "The file \"${last_update_txt}\" is missing"
+  exit 
+fi
+
+hpr_db_last_update_iso8601="$( \date --utc --date="$( cat ${last_update_txt} )" +%Y-%m-%dT%H:%M:%SZ )"
+hpr_db_last_update_epoch="$( \date --utc --date="$( cat ${last_update_txt} )" +%s )"
+echo "Latest change on the HPR website database is ${hpr_db_last_update_iso8601} (${hpr_db_last_update_epoch})"
+
+#TODO check that the db is greater 
+
+
+# # # if [ "${hpr_db_last_update_epoch}" -eq "${local_db_last_update_epoch}" ]
+# # # then
+# # #   echo "The file \"${last_update_txt}\" is missing"
+# # #   exit 
+# # # fi
+
+mysqldump --tz-utc --add-drop-database --extended-insert --complete-insert --skip-extended-insert --default-character-set=utf8 --single-transaction --skip-set-charset --databases hpr_hpr > "${sql_save_dir}/hpr_hpr_full.sql"
+mysqldump --tz-utc --add-drop-database --complete-insert --extended-insert --default-character-set=utf8 --single-transaction --skip-set-charset --databases hpr_hpr --ignore-table=hpr_hpr.reservations > "${sql_save_dir}/hpr.sql"
+mysqldump --tz-utc --add-drop-database  --databases hpr_hpr> "${sql_save_dir}/mysqldump.sql"
+
+rsync -av --partial --progress "${sql_save_dir}/hpr.sql" hpr:/docker/users/hpr/hackerpublicradio.org/public_html/hpr.sql

workflow/mdb.bash

@@ -0,0 +1,4 @@
+#!/bin/bash
+pw=$(grep -E '^\$databasePassword = ' /home/hpr/php/credentials.php | awk -F "'" '{print $2}' )
+pw=${pw##* }
+mysql --host=localhost --user=hpr_hpr --password="$pw" hpr_hpr

workflow/run-speedtest.bash

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# * * * * * /usr/local/bin/run-speedtest.bash >/dev/null 2>&1
+
+speedtest-cli --json | jq '.' > /var/www/html/speedtest.json
+chown apache:apache /var/www/html/speedtest.json

workflow/update-stats.bash

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+hub_dir="/home/hpr/hub"
+
+for format in txt csv json xml
+do 
+  curl --silent --netrc-file /home/hpr/.netrc https://hub.hackerpublicradio.org/cms/stats.php?format=${format} --output ${hub_dir}/stats.${format} 
+done