253 lines
6.4 KiB
Bash
Executable File
253 lines
6.4 KiB
Bash
Executable File
#!/bin/bash -
|
|
#===============================================================================
|
|
#
|
|
# FILE: build_AOB
|
|
#
|
|
# USAGE: ./build_AOB [date]
|
|
#
|
|
# DESCRIPTION: Build the AOB files for a particular month
|
|
#
|
|
# OPTIONS: ---
|
|
# REQUIREMENTS: ---
|
|
# BUGS: ---
|
|
# NOTES: ---
|
|
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
|
|
# VERSION: 0.0.12
|
|
# CREATED: 2021-04-15 17:36:22
|
|
# REVISION: 2024-03-15 09:50:02
|
|
#
|
|
#===============================================================================
|
|
|
|
set -o nounset # Treat unset variables as an error
|
|
|
|
SCRIPT=${0##*/}
|
|
BASEDIR=${0%/*}
|
|
|
|
VERSION="0.0.12"
|
|
|
|
STDOUT="/dev/fd/2"
|
|
|
|
#
|
|
# Make sure we're in the working directory
|
|
#
|
|
cd "$BASEDIR" || exit 1
|
|
|
|
#
|
|
# Load library functions
|
|
#
|
|
LIB="$HOME/bin/function_lib.sh"
|
|
[ -e "$LIB" ] || { echo "$SCRIPT: Unable to source functions"; exit 1; }
|
|
# shellcheck disable=SC1090
|
|
source "$LIB"
|
|
|
|
# {{{ -- Functions usage and _DEBUG
|
|
#=== FUNCTION ================================================================
|
|
# NAME: _usage
|
|
# DESCRIPTION: Report usage
|
|
# PARAMETERS: None
|
|
# RETURNS: Nothing
|
|
#===============================================================================
|
|
_usage () {
|
|
cat >$STDOUT <<-endusage
|
|
Usage: ./${SCRIPT} [-h] [-D] [date]
|
|
|
|
Version: $VERSION
|
|
|
|
Converts the AOB in Markdown format for a particular month to HTML and to text
|
|
|
|
Options:
|
|
-h Print this help
|
|
-D Select debug mode (works the same; more output)
|
|
|
|
Arguments (optional):
|
|
date Specifies the month to build the AOB for. The default
|
|
is the current month. The format can be YYYY-MM (e.g.
|
|
2022-05) or any date format that the 'date' command
|
|
can parse, so 2022-04-01 or 01-Apr-2022 and so on. If
|
|
the date cannot be parsed an error will be reported.
|
|
|
|
Examples
|
|
./${SCRIPT} -h
|
|
./${SCRIPT} -D 01-February-2021
|
|
./${SCRIPT} 2021-02
|
|
|
|
endusage
|
|
exit
|
|
}
|
|
|
|
#=== FUNCTION ================================================================
|
|
# NAME: _DEBUG
|
|
# DESCRIPTION: Writes a message if in DEBUG mode
|
|
# PARAMETERS: List of messages
|
|
# RETURNS: Nothing
|
|
#===============================================================================
|
|
_DEBUG () {
|
|
[ "$DEBUG" == 0 ] && return
|
|
for msg in "$@"; do
|
|
printf 'D> %s\n' "$msg"
|
|
done
|
|
}
|
|
|
|
# }}}
|
|
|
|
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
|
|
#
|
|
# Base and database directories
|
|
#
|
|
PARENT="$HOME/HPR"
|
|
BASEDIR="$PARENT/Community_News"
|
|
cd "$BASEDIR" || {
|
|
echo "Failed to cd to $BASEDIR";
|
|
exit 1;
|
|
}
|
|
# IADIR="$PARENT/InternetArchive"
|
|
|
|
#
|
|
# Option defaults
|
|
#
|
|
DEBUG=0
|
|
|
|
#
|
|
# Process options
|
|
#
|
|
while getopts :hdD opt
|
|
do
|
|
case "${opt}" in
|
|
h) _usage;;
|
|
D) DEBUG=1;;
|
|
?) echo "$SCRIPT: Invalid option; aborting"; exit 1;;
|
|
esac
|
|
done
|
|
shift $((OPTIND - 1))
|
|
|
|
#
|
|
# Handle the optional argument
|
|
#
|
|
if [[ $# -eq 1 ]]; then
|
|
startdate="$1"
|
|
# Normalise a YYYY-MM date so 'date' will not complain
|
|
if [[ $startdate =~ ^[0-9]{4}-[0-9]{2}$ ]]; then
|
|
startdate+='-01'
|
|
fi
|
|
# Validate the date and standardise it if it's OK
|
|
tmp="$(date -d "$startdate" +%Y-%m)" || {
|
|
echo "Use a date such as $(date +%Y-%m)"
|
|
exit 1
|
|
}
|
|
startdate="$tmp"
|
|
else
|
|
startdate="$(date +%Y-%m)"
|
|
fi
|
|
|
|
_DEBUG "Date used: $startdate"
|
|
|
|
#
|
|
# We added a new field in 2022, 'item_last_updated' which is taken from the IA
|
|
# (which we discovered was being maintained). It is a Unix date field, but the
|
|
# view 'episodes_view' converts it.
|
|
#
|
|
# TODO: Since query3 was added it has made query1 and query2 obsolete. We
|
|
# generate a per-month table with query3 which is turned into HTML using awk
|
|
# and used in the AOB report. The code below that uses these queries and their
|
|
# results could now be removed (or commented out).
|
|
#
|
|
#query1="select count(*) from episodes where id between 871 and 2429 and with_derived = 1"
|
|
|
|
##query1="select count(*) from episodes_view where id between 871 and 2429 and \
|
|
##item_last_updated between '${startdate}-01' and \
|
|
##date('${startdate}-01','+1 month') and with_derived = 1"
|
|
##
|
|
##query2='select count(*) from episodes where id between 871 and 2429 and with_derived = 0'
|
|
##
|
|
##query3=$(cat <<ENDOFQ3
|
|
##SELECT
|
|
## strftime('%Y-%m',item_last_updated) AS month,
|
|
## count(*) AS count
|
|
##FROM episodes_view
|
|
##WHERE id BETWEEN 871 AND 2429
|
|
##AND item_last_updated IS NOT NULL
|
|
##AND item_last_updated < date('${startdate}-01','+1 month')
|
|
##GROUP BY strftime('%Y-%m',item_last_updated);
|
|
##ENDOFQ3
|
|
##)
|
|
##
|
|
##_DEBUG "Query used (1): $query1"
|
|
##_DEBUG "Query used (2): $query2"
|
|
##_DEBUG "Query used (3): $query3"
|
|
##
|
|
#
|
|
# The database
|
|
#
|
|
##IADB="$IADIR/ia.db"
|
|
##
|
|
#
|
|
# Collect the values
|
|
#
|
|
##UPLOADS=$(echo "$query1" | sqlite3 -list "$IADB")
|
|
##REMAINING=$(echo "$query2" | sqlite3 -list "$IADB")
|
|
##TABLE=$(echo "$query3" | sqlite3 -list "$IADB")
|
|
##
|
|
##_DEBUG "Uploads=$UPLOADS (query 1)"
|
|
##_DEBUG "Remaining=$REMAINING (query 2)"
|
|
##_DEBUG "Table=$TABLE"
|
|
|
|
#
|
|
# Files we need to build the AOB
|
|
#
|
|
AOBMKD="$BASEDIR/aob_$startdate.mkd"
|
|
|
|
#
|
|
# Sanity check
|
|
#
|
|
[ -e "$AOBMKD" ] || { echo "Unable to find $AOBMKD"; exit 1; }
|
|
|
|
#
|
|
# Make temporary files and set traps to delete them
|
|
#
|
|
##TMP1=$(mktemp) || { echo "$SCRIPT: creation of temporary file failed!"; exit 1; }
|
|
##trap 'cleanup_temp $TMP1' SIGHUP SIGINT SIGPIPE SIGTERM EXIT
|
|
##
|
|
#
|
|
# Use Awk to process the table we failed to generate in SQL :-(
|
|
#
|
|
##awk -F '|' -f /dev/fd/7 <<<"$TABLE" >"$TMP1" 7<<'ENDAWK'
|
|
##BEGIN{
|
|
## total = 0
|
|
## remainder = (2429 - 871 + 1)
|
|
## print "<table>"
|
|
## print "<tr><th>Month</th><th>Month count</th>"\
|
|
## "<th>Running total</th><th>Remainder</th></tr>"
|
|
##}
|
|
##{
|
|
## total = total + $2
|
|
## remainder = remainder - $2
|
|
## printf "<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>\n",
|
|
## $1,$2,total,remainder
|
|
##}
|
|
##END{
|
|
## print "</table>"
|
|
##}
|
|
##ENDAWK
|
|
##cat >>"$TMP1" <<ENDDATE
|
|
##<p><small><small>Table updated: $(date --utc +'%F %T')</small></small></p>
|
|
##ENDDATE
|
|
##_DEBUG "Table" "$(cat "$TMP1")"
|
|
|
|
#
|
|
# Build the output files
|
|
#
|
|
# if tpage --define "uploads=$UPLOADS" --define "remaining=$REMAINING" \
|
|
# --define "table=$TMP1" "$AOBMKD" |\
|
|
# pandoc -f markdown-smart -t html5 -o "${AOBMKD%mkd}html"; then
|
|
#
|
|
if pandoc -f markdown-smart -t html5 "$AOBMKD" -o "${AOBMKD%mkd}html"; then
|
|
echo "Converted $AOBMKD to HTML"
|
|
else
|
|
echo "Conversion of $AOBMKD to HTML failed"
|
|
fi
|
|
|
|
exit
|
|
|
|
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21:fdm=marker
|