diff --git a/workflow/hpr-cn-generate-notes.bash b/workflow/hpr-cn-generate-notes.bash index 1b20ef4..8e1ef94 100755 --- a/workflow/hpr-cn-generate-notes.bash +++ b/workflow/hpr-cn-generate-notes.bash @@ -47,7 +47,7 @@ fi ############################################################################################ # Get previous show info -row=$(sqlite3 hpr.db " +row=$(sqlite3 "${hpr_db}" " SELECT notes FROM eps WHERE duration > 0 @@ -56,6 +56,13 @@ ORDER BY date DESC LIMIT 1; ") +printf "\n%s\n" "SELECT notes +FROM eps +WHERE duration > 0 +AND series = 47 +ORDER BY date DESC +LIMIT 1;" 1>&2 + eval $( echo "$row" | grep -o '' | @@ -63,14 +70,17 @@ eval $( tr ',' '\n' ) -echo_debug "The last show month was for ${month}, latest_comment: ${latest_comment}, latest_host: ${latest_host}, and latest_episode: ${latest_episode}" +echo_debug "The last show month was for ${month}, previous_cn_latest_comment: ${latest_comment}, latest_host: ${latest_host}, and previous_cn_latest_episode: ${previous_cn_latest_episode}" + +previous_cn_latest_comment="${latest_comment}" +previous_cn_latest_episode="${latest_episode}" ############################################################################################ # Get next show info this_month=$( \date --date="${month}-01 +1 month" +%Y-%m ) -IFS='|' read -r eps_num eps_date <<< "$(sqlite3 hpr.db " +IFS='|' read -r eps_num eps_date <<< "$(sqlite3 "${hpr_db}" " SELECT id, date FROM eps WHERE duration = 0 @@ -78,6 +88,12 @@ AND series = 47 ORDER BY date ASC LIMIT 1; ")" +printf "\n%s\n" "SELECT id, date +FROM eps +WHERE duration = 0 +AND series = 47 +ORDER BY date ASC +LIMIT 1;" 1>&2 echo_debug "This is going to be hpr${eps_num} for release on ${eps_date}" @@ -94,72 +110,79 @@ shownotes_html="${eps_dir}/shownotes.html" ############################################################################################ # Add New hosts to shownotes -results=$(sqlite3 hpr.db "SELECT hostid, host FROM hosts WHERE hostid > ${latest_host};") +results=$(sqlite3 "${hpr_db}" "SELECT hostid, host FROM hosts WHERE hostid > ${latest_host};") +printf "\n%s\n" "SELECT hostid, host FROM hosts WHERE hostid > ${latest_host};" 1>&2 -echo "

New hosts

" | tee "${shownotes_html}" -echo "

" | tee --append "${shownotes_html}" +echo "

New hosts

" > "${shownotes_html}" +echo "

" >> "${shownotes_html}" if [ -z "${results}" ] then - echo "There were no new hosts this month." | tee --append "${shownotes_html}" + echo "There were no new hosts this month." >> "${shownotes_html}" newest_host="${latest_host}" else - echo "Welcome to our new hosts:
" | tee --append "${shownotes_html}" - echo "

" >> "${shownotes_html}" fi -echo "

" | tee --append "${shownotes_html}" -echo "" | tee --append "${shownotes_html}" +echo "

" >> "${shownotes_html}" +echo "" >> "${shownotes_html}" ############################################################################################ # Add Last Month's Shows to shownotes -results=$(sqlite3 hpr.db " +results=$(sqlite3 -separator '→' "${hpr_db}" " SELECT eps.id, hosts.host, eps.title, eps.date, eps.hostid FROM eps, hosts -WHERE eps.id > ${latest_episode} +WHERE eps.id > ${previous_cn_latest_episode} AND eps.date LIKE \"${this_month}%\" AND eps.valid = 1 AND eps.hostid = hosts.hostid; ") +printf "\n%s\n" "SELECT eps.id, hosts.host, eps.title, eps.date, eps.hostid +FROM eps, hosts +WHERE eps.id > ${previous_cn_latest_episode} +AND eps.date LIKE \"${this_month}%\" +AND eps.valid = 1 +AND eps.hostid = hosts.hostid;" 1>&2 -echo "

Last Month's Shows

" | tee --append "${shownotes_html}" -echo "" | tee --append "${shownotes_html}" +echo "

Last Month's Shows

" >> "${shownotes_html}" +echo "" >> "${shownotes_html}" # [%# The id 't01' is in the HPR CSS but might give trouble on the IA -%] -echo "" | tee --append "${shownotes_html}" -echo "" | tee --append "${shownotes_html}" -echo "" | tee --append "${shownotes_html}" -echo "" | tee --append "${shownotes_html}" -echo "" | tee --append "${shownotes_html}" -echo "" | tee --append "${shownotes_html}" -echo "" | tee --append "${shownotes_html}" -echo "" | tee --append "${shownotes_html}" +echo "
IdDayDateTitleHost
" >> "${shownotes_html}" +echo "" >> "${shownotes_html}" +echo "" >> "${shownotes_html}" +echo "" >> "${shownotes_html}" +echo "" >> "${shownotes_html}" +echo "" >> "${shownotes_html}" +echo "" >> "${shownotes_html}" +echo "" >> "${shownotes_html}" if [ -z "${results}" ] then echo_error "There were no shows found for this month." else - while IFS="|" read -r eps_id host title eps_date hostid + while IFS="→" read -r eps_id host title eps_date hostid do printf -v hostid "%04d" "${hostid}" printf -v eps_id "%04d" "${eps_id}" - echo "" | tee --append "${shownotes_html}" - echo "" | tee --append "${shownotes_html}" - echo "" | tee --append "${shownotes_html}" - echo "" | tee --append "${shownotes_html}" - echo "" | tee --append "${shownotes_html}" - echo "" | tee --append "${shownotes_html}" + echo "" >> "${shownotes_html}" + echo "" >> "${shownotes_html}" + echo "" >> "${shownotes_html}" + echo "" >> "${shownotes_html}" + echo "" >> "${shownotes_html}" + echo "" >> "${shownotes_html}" latest_episode=${eps_id} - echo "" | tee --append "${shownotes_html}" + echo "" >> "${shownotes_html}" done <<< "${results}" fi -echo "
IdDayDateTitleHost
${eps_id}$( \date --date=${eps_date} +%a )${eps_date}${title}${host}
${eps_id}$( \date --date=${eps_date} +%a )${eps_date}${title}${host}
" | tee --append "${shownotes_html}" -echo "" | tee --append "${shownotes_html}" +echo "" >> "${shownotes_html}" +echo "" >> "${shownotes_html}" ############################################################################################ # Add Comments to shownotes @@ -175,40 +198,41 @@ function render_comments() { else this_comment="0" open_line="0" - echo "" >> "${shownotes_html}" fi open_line="1" this_comment="${comments_eps_id}" printf -v hosts_hostid "%04d" "${hosts_hostid}" printf -v comments_eps_id "%04d" "${comments_eps_id}" - echo "
  • hpr${comments_eps_id} (${eps_date}) \"${eps_title}\" by ${hosts_host}.
  • " | tee --append "${shownotes_html}" - echo "" | tee --append "${shownotes_html}" + echo "" >> "${shownotes_html}" + echo "" >> "${shownotes_html}" fi } # ------------------------------------------- -echo "

    Comments this month

    " | tee --append "${shownotes_html}" -echo "" | tee --append "${shownotes_html}" +echo "

    Comments this month

    " >> "${shownotes_html}" +echo "" >> "${shownotes_html}" -echo "

    Past shows

    " | tee --append "${shownotes_html}" +echo "

    Past shows

    " >> "${shownotes_html}" -latest_comment_last_changed=$(sqlite3 hpr.db "SELECT last_changed FROM comments WHERE comments.id = ${latest_comment};") +previous_cn_latest_comment_last_changed=$(sqlite3 "${hpr_db}" "SELECT last_changed FROM comments WHERE comments.id = ${previous_cn_latest_comment};") +printf "\n%s\n" "SELECT last_changed FROM comments WHERE comments.id = ${previous_cn_latest_comment};" 1>&2 -results=$(sqlite3 hpr.db " +results=$(sqlite3 -separator '→' "${hpr_db}" " SELECT comments.id, comments.eps_id, @@ -222,16 +246,34 @@ SELECT FROM comments JOIN eps ON comments.eps_id = eps.id JOIN hosts ON eps.hostid = hosts.hostid -WHERE comments.last_changed > \"${latest_comment_last_changed}\" - AND eps.date <= \"${this_month}-01\" +WHERE comments.id > "${previous_cn_latest_comment}" + AND eps.id <= "${previous_cn_latest_episode}" ORDER BY eps.date; ") +printf "\n%s\n" " +SELECT + comments.id, + comments.eps_id, + eps.date, + eps.title, + hosts.host, + hosts.hostid, + comments.comment_author_name, + comments.comment_timestamp, + comments.comment_title +FROM comments +JOIN eps ON comments.eps_id = eps.id +JOIN hosts ON eps.hostid = hosts.hostid +WHERE comments.id > "${previous_cn_latest_comment}" + AND eps.id <= "${previous_cn_latest_episode}" +ORDER BY eps.date; +" 1>&2 render_comments "${results}" -echo "

    This month's shows

    " | tee --append "${shownotes_html}" +echo "

    This month's shows

    " >> "${shownotes_html}" -results=$(sqlite3 hpr.db " +results=$(sqlite3 -separator '→' "${hpr_db}" " SELECT comments.id, comments.eps_id, @@ -248,6 +290,21 @@ JOIN hosts ON eps.hostid = hosts.hostid WHERE eps.date LIKE \"${this_month}-%\" ORDER BY eps.date; ") +printf "\n%s\n" "SELECT + comments.id, + comments.eps_id, + eps.date, + eps.title, + hosts.host, + hosts.hostid, + comments.comment_author_name, + comments.comment_timestamp, + comments.comment_title +FROM comments +JOIN eps ON comments.eps_id = eps.id +JOIN hosts ON eps.hostid = hosts.hostid +WHERE eps.date LIKE \"${this_month}-%\" +ORDER BY eps.date;" 1>&2 render_comments "${results}" @@ -255,7 +312,7 @@ render_comments "${results}" # Add maillist and cal to shownotes mailman_link="https://lists.hackerpublicradio.com/pipermail/hpr/$( \date -d"${month}-01 next month" +%Y-%B )/thread.html" -echo "

    Mailing List discussions

    " | tee --append "${shownotes_html}" +echo "

    Mailing List discussions

    " >> "${shownotes_html}" echo "

    Policy decisions surrounding HPR are taken by the community as a whole. This discussion takes place on the Mailing List which is open to all HPR listeners and contributors. The discussions are open @@ -271,10 +328,11 @@ and available on the HPR server under This is the LWN.net community event calendar, where we track events of interest to people using and developing Linux and free software. Clicking on individual events will take you to the appropriate web -page." | tee --append "${shownotes_html}" +page." >> "${shownotes_html}" -latest_comment=$(sqlite3 hpr.db "SELECT MAX(id) FROM comments;") -echo "" | tee --append "${shownotes_html}" +latest_comment=$(sqlite3 "${hpr_db}" "SELECT MAX(id) FROM comments;") +printf "\n%s\n" "" 1>&2 +echo "" >> "${shownotes_html}" cp -v "${shownotes_html}" "${shownotes_html%.*}"_edited.html ls -al "${eps_dir}" diff --git a/workflow/hpr-get-and-transcribe.bash b/workflow/hpr-get-and-transcribe.bash index 9ae03ca..fbc4447 100755 --- a/workflow/hpr-get-and-transcribe.bash +++ b/workflow/hpr-get-and-transcribe.bash @@ -1,35 +1,86 @@ #!/bin/bash -rsync -av --partial --progress hpr:hub.hackerpublicradio.org/upload/ $HOME/tmp/hpr/processing/ +destination_dir="$HOME/tmp/hpr/processing/" +server_dir="hub.hackerpublicradio.org" -find $HOME/tmp/hpr/processing/*_*_????-??-??_* -type d | sort -t _ -k 2 | while read show_dir -do - echo "${show_dir}" - if [ "$( find "${show_dir}" -type f -iname "*srt" | wc -l )" -eq "0" ] +################################################# +# Display Error message, display help and exit + +function echo_error() { + echo -e "ERROR: $@" #1>&2 + display_help_and_exit + exit 1 +} + +################################################# +# Display Information message + +function echo_debug() { + if [ "${debug}" != "0" ] then - cd "${show_dir}" - ls -haltr - find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media - do - whisper --model tiny --language en --output_dir "${show_dir}" "${this_media}" - done - rsync -av --partial --progress "${show_dir}/" hpr:hub.hackerpublicradio.org/upload/$( basename "${show_dir}")/ + echo -e "INFO: $@" #1>&2 fi -done +} -rsync -av --partial --progress hpr:hub.hackerpublicradio.org/reserve/ $HOME/tmp/hpr/reserve/ +################################################# +# Display Help -find $HOME/tmp/hpr/reserve/*_*_* -type d | sort -t _ -k 2 | while read show_dir +function display_help_and_exit() { + echo_debug "For more information see https://repo.anhonesthost.net/HPR/hpr_documentation" + exit 1 +} + +################################################# +# main + +if [ "$( curl --location --silent --netrc-file ${HOME}/.netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/status.php --output "${destination_dir}/status.csv" )" != 200 ] +then + echo_error "Could not get a list of the queue status from \"https://hub.hackerpublicradio.org/cms/status.php\"" +fi + +if [ ! -s "${destination_dir}/status.csv" ] +then + echo_error "Failed to retrieve \"${destination_dir}/status.csv\" from server." +fi + +response=$( cat "${destination_dir}/status.csv" | grep --extended-regexp ',SHOW_SUBMITTED,|,RESERVE_SHOW_SUBMITTED,' | sort -t ',' -k 2 ) +if [ -z "${response}" ] +then + echo_debug "Getting a list of all the reservations." + curl --location --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | sort -n + echo_error "There appear to be no more shows with the status \"SHOW_SUBMITTED\"." +fi + +printf '%s\n' ${response} | while read this_line do - echo "${show_dir}" - if [ "$( find "${show_dir}" -type f -iname "*srt" | wc -l )" -eq "0" ] + IFS=, read -r timestamp_epoc ep_num ep_date key status email <<< "$( printf '%s\n' "${this_line}" )" + + if [ "${ep_num}" == "9999" ] then - cd "${show_dir}" - ls -haltr - find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media - do - whisper --model tiny --language en --output_dir "${show_dir}" "${this_media}" - done - rsync -av --partial --progress "${show_dir}/" hpr:hub.hackerpublicradio.org/reserve/$( basename "${show_dir}")/ + source_dir="${server_dir}/reserve/" + else + source_dir="${server_dir}/upload/" fi -done + if [ "$( find "${destination_dir}" -maxdepth 2 -mindepth 2 -type f -iname "*${key}*.srt" | wc --lines )" -ne "0" ] + then + echo_debug "The episode \"${timestamp_epoc} ${ep_num} ${ep_date} ${key} ${status} ${email}\" has been processed." + continue + fi + + show_path="$( ssh -n hpr -t "find \"${source_dir}\" -maxdepth 1 -mindepth 1 -type d -iname \"*${key}*\"" 2>/dev/null | sed "s#${source_dir}##g#" )" + if [ -z "${show_path}" ] + then + echo_error "The show_path could not be found." + fi + show_dir="${destination_dir}/${show_path}" + echo_debug "Copying \"hpr:${source_dir}/${show_path}/\" to \"${show_dir}/\"" + rsync --archive --verbose --partial --progress hpr:${source_dir}/${show_path}/ ${show_dir}/ + cd "${show_dir}" + pwd + find "${show_dir}/" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' | while read this_media + do + whisper --output_format srt --model tiny --language en --output_dir "${show_dir}" "${this_media}" + done + rsync --archive --verbose --partial --progress ${show_dir}/ hpr:${source_dir}/${show_path}/ + +done \ No newline at end of file diff --git a/workflow/process_episode.bash b/workflow/hpr-process_episode.bash similarity index 100% rename from workflow/process_episode.bash rename to workflow/hpr-process_episode.bash