diff --git a/workflow/duration.bash b/workflow/duration.bash new file mode 100755 index 0000000..4f812be --- /dev/null +++ b/workflow/duration.bash @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ ] + + +if [ -f "${1}" ] +then + mediainfo --Output=XML --Full "${1}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | sed 's/0$//g' + exit 0 +fi + +find ./ -type f | grep -E -v '/sponsor-anhonesthost.com-hpr15.flac|/outro.flac|/intro.flac|/sponsor-archive.org.flac' | while read mediafile +do + if [ "$( file "${mediafile}" | grep -c audio )" == "0" ] + then + continue + fi + duration=$( mediainfo --full --Output=XML "${mediafile}" | xmlstarlet sel -T -t -m "_:MediaInfo/_:media/_:track[@type='Audio']/_:Duration[1]" -v "." -n - | sed 's/0$//g') + if [ "${duration}" != "" ] + then + echo "${mediafile}: ${duration}" + continue + fi + duration=$( /bin/date -ud "1970-01-01 $( ffprobe -i "${mediafile}" 2>&1| awk -F ': |, ' '/Duration:/ { print $2 }' )" +%s ) + if [ "${duration}" != 0 ] + then + echo "${mediafile}: ${duration}" + continue + fi +done + diff --git a/workflow/fix_tags b/workflow/fix_tags new file mode 100755 index 0000000..5468ae2 --- /dev/null +++ b/workflow/fix_tags @@ -0,0 +1,641 @@ +#!/usr/bin/perl +#=============================================================================== +# +# FILE: fix_tags +# +# USAGE: ./fix_tags [-help] [-album=ALBUMSTRING] [-artist=ARTISTSTRING] +# [-comment=COMMENTSTRING] [-genre=GENRESTRING] +# [-title=TITLESTRING] [-track=TRACKNUMBER] [-year=YEAR] +# [-filter TAGNAME=FILTERNAME] audio_file ... +# +# DESCRIPTION: A tool for reporting and altering tags in audio files of +# a variety of formats +# +# OPTIONS: --- +# REQUIREMENTS: --- +# BUGS: --- +# NOTES: --- +# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com +# LICENCE: Copyright (c) year 2011, 2012, 2013, 2014, Dave Morriss +# VERSION: 1.3.4 +# CREATED: 2011-12-12 22:00:34 +# REVISION: 2014-06-25 15:10:17 +# +#=============================================================================== + +use Modern::Perl '2011'; + +use Getopt::Long; +use Pod::Usage; + +use Data::Dumper; + +use File::stat; +use Date::Manip::Delta; +use Date::Manip::TZ; +use Audio::TagLib; + +use TryCatch; + +use HTML::Restrict; + +# +# Version number (manually incremented) +# +our $VERSION = '1.3.4'; + +# +# Script name +# +( my $PROG = $0 ) =~ s|.*/||mx; + +# +# Declarations +# +my ( $sb, $fyear, $ref, $tag, $aprop, $changed ); +my ( %tags, @errors ); + +# +# The Audio::TagLib methods to call for each tag manipulated by the script. +# The number after the method name is 1 if the value being set is a string, +# and zero otherwise. +# +my %tagmethods = ( + album => [ 'setAlbum', 1 ], + artist => [ 'setArtist', 1 ], + comment => [ 'setComment', 1 ], + genre => [ 'setGenre', 1 ], + title => [ 'setTitle', 1 ], + track => [ 'setTrack', 0 ], + year => [ 'setYear', 0 ], +); + +# +# Internal routines to invoke to perform filtering tasks +# +my %filtermethods = ( + clean => \&clean_string, + underscore => \&replace_underscores, + HTML => \&remove_HTML, +); + +# +# Ensure STDOUT and STDERR are in UTF8 mode +# +binmode( STDOUT, ":encoding(utf8)" ); +binmode( STDERR, ":encoding(utf8)" ); + +# +# Options and arguments +# +my ( %options, %filter ); +Options( \%options, \%filter ); + +# +# Default help +# +pod2usage( -msg => "Version $VERSION\n", -exitval => 1 ) + if ( $options{'help'} ); + +# +# Collect options +# +my $album = $options{album}; +my $artist = $options{artist}; +my $comment = $options{comment}; +my $genre = $options{genre}; +my $title = $options{title}; +my $track = $options{track}; +my $year = $options{year}; + +# +# Check the filter options +# +unless ( check_filters( \%filter, \@errors ) ) { + print STDERR join( "\n", @errors ), "\n"; + exit(1); +} + +my @files = @ARGV; + +pod2usage( + -msg => "Missing arguments\n\nVersion $VERSION\n", + -exitval => 1 +) unless @files; + +foreach my $file (@files) { + unless ( -e $file ) { + warn "$file does not exist\n"; + next; + } + + # + # Report the file name + # + print "$file\n"; + + # + # If the file is empty report it and skip it + # + if ( -z $file ) { + warn "File $file is empty\n"; + next; + } + + $sb = stat($file); + $fyear = ( localtime( $sb->mtime ) )[5] + 1900; + + # + # Catch errors if someone tries to use this tool on a file that + # Audio::TagLib doesn't know about + # + try { + $ref = Audio::TagLib::FileRef->new($file); + $tag = $ref->tag(); + $aprop = $ref->audioProperties(); + + %tags = ( + title => $tag->title()->toCString(), + artist => $tag->artist()->toCString(), + album => $tag->album()->toCString(), + comment => $tag->comment()->toCString(), + genre => $tag->genre()->toCString(), + year => $tag->year(), + track => $tag->track(), + length => sprintf( "%s (%d sec)", + interval( $aprop->length() ), + $aprop->length() ), + ); + } + + # + # We choked on something nasty + # + catch { + warn "File $file apparently does not contain tags\n"; + next; + } + + # + # Report current tags + # + for my $key ( sort( keys(%tags) ) ) { + printf "%-10s: %s\n", $key, $tags{$key}; + } + + $changed = 0; + + # + # Change album, artist name, comment, genre, track number or year if + # requested + # + $changed + += changeTag( $tag, 'album', $tags{album}, $album, 'setAlbum', 1 ); + $changed + += changeTag( $tag, 'artist', $tags{artist}, $artist, 'setArtist', + 1 ); + $changed + += changeTag( $tag, 'comment', $tags{comment}, $comment, + 'setComment', 1 ); + $changed + += changeTag( $tag, 'genre', $tags{genre}, $genre, 'setGenre', 1 ); + $changed + += changeTag( $tag, 'title', $tags{title}, $title, 'setTitle', 1 ); + $changed + += changeTag( $tag, 'track', $tags{track}, $track, 'setTrack', 0 ); + $changed += changeTag( $tag, 'year', $tags{year}, $year, 'setYear', 0 ); + + # + # Do some filtering + # + $changed += apply_filters( $tag, \%tags, \%tagmethods, \%filter, + \%filtermethods ); + + # + # Update if there are changes + # + if ($changed) { + $ref->save(); + } + +} +continue { + print "\n"; +} + +exit; + +#=== FUNCTION ================================================================ +# NAME: changeTag +# PURPOSE: Changes a tag to a new value if appropriate +# PARAMETERS: $tag Tag object +# $tagname Name of tag +# $oldValue Current value of tag +# $newValue New value of tag or undefined +# $setFunc String containing the name of the 'set' +# function +# $isString True if the value being set is a string +# RETURNS: 1 if a change has been made, 0 otherwise +# DESCRIPTION: Checks that $newValue is defined (it can be an empty string) +# and that the new value differs from the old one, returning if +# not. The $isString value defaults to zero. Ensures that a null +# $newValue is replaced by a zero if the tag is numeric. Reports +# what change has been requested then makes the change. +# THROWS: No exceptions +# COMMENTS: None +# SEE ALSO: +#=============================================================================== +sub changeTag { + my ( $tag, $tagname, $oldValue, $newValue, $setFunc, $isString ) = @_; + + return 0 unless defined($newValue); + return 0 if $oldValue eq $newValue; + + $isString = 0 unless defined($isString); + + $newValue = 0 if ( $newValue eq '' && !$isString ); + + print "Changing $tagname to '$newValue'\n"; + $tag->$setFunc( + ( $isString + ? Audio::TagLib::String->new($newValue) + : $newValue + ) + ); + return 1; +} + +#=== FUNCTION ================================================================ +# NAME: check_filters +# PURPOSE: Check that the filter hash contains valid settings +# PARAMETERS: $filter Hashref containing the filter options +# $errors Arrayref to contain error messages +# RETURNS: 1 (true) if all is well, otherwise 0 (false) +# DESCRIPTION: The check returns true if there are no filters. It returns +# false if there are any unknown tag names or any unknown filter +# names. +# THROWS: No exceptions +# COMMENTS: The knowledge of what is a vlaid tag name or filter name is +# within this function, which is not ideal for maintenance. +# SEE ALSO: N/A +#=============================================================================== +sub check_filters { + my ( $filters, $errors ) = @_; + + my @filterable_tags = (qw{ album artist comment genre title }); + my @valid_filters = (qw{clean underscore html}); + my @unknown; + + # + # Nothing is wrong if there are no filters + # + return 1 unless defined($filters); + + # + # Are any tag names unknown? + # + my %filterable = map { $_ => 1 } @filterable_tags; + @unknown = grep { !defined( $filterable{$_} ) } + map { lc($_) } keys( %{$filters} ); + if (@unknown) { + push( @{$errors}, "Error: Invalid filter(s)" ) + unless defined($errors); + push( @{$errors}, "Unknown tag names: " . join( ", ", @unknown ) ); + } + + # + # Are any filter names unknown? + # + my %valid = map { $_ => 1 } @valid_filters; + my @names = map { @{ $filters->{$_} } } keys( %{$filters} ); + @unknown = grep { !defined( $valid{$_} ) } map { lc($_) } @names; + if (@unknown) { + push( @{$errors}, "Error: Invalid filter(s)" ) + unless defined($errors); + push( @{$errors}, "Unknown filter names: " . join( ", ", @unknown ) ); + } + + # + # All tests passed if no errors + # + return scalar( @{$errors} ) == 0; +} + +#=== FUNCTION ================================================================ +# NAME: apply_filters +# PURPOSE: Looks for requested filters and the tags they are to be +# applied to and performs the necessary filtering +# PARAMETERS: $tag Tag object +# $tags Hashref containing the converted tags from the +# current file +# $tagmethods Hashref containing the Audio::TagLib method +# names per tag +# $filter Hashref containing filter options +# $filtermethods Hashref containing filter names and the +# routines that handle them +# RETURNS: Number of changes made +# DESCRIPTION: The $filter hash contains tag names as keys (lower- or +# upper-case). The value is an array of filter names (lower- or +# upper-case). We loop through the tag names looking for filter +# names and applying the filters we find. +# THROWS: No exceptions +# COMMENTS: The sorting should be case-insensitive. +# SEE ALSO: N/A +#=============================================================================== +sub apply_filters { + my ( $tag, $tags, $tagmethods, $filter, $filtermethods ) = @_; + + my $lc_t; + my $newtag; + my $changes = 0; + + # + # Loop through the tags we are to filter in sorted order + # + for my $t ( sort( keys( %{$filter} ) ) ) { + + # + # We need a lowercase key to access the tag + # + $lc_t = lc($t); + + # + # Loop through all available methods and apply them if requested + # + for my $f ( sort( keys( %{$filtermethods} ) ) ) { + if ( grep( /^$f$/i, @{ $filter->{$t} } ) ) { + $newtag = &{ $filtermethods->{$f} }( $tags->{$lc_t} ); + $changes += changeTag( $tag, $lc_t, $tags->{$lc_t}, + $newtag, @{ $tagmethods->{$lc_t} } ); + } + + } + } + + return $changes; +} + +#=== FUNCTION ================================================================ +# NAME: interval +# PURPOSE: Convert a 'MM:SS' string into an acceptable PostgreSQL +# interval where the minutes portion may exceed 60. +# PARAMETERS: +# RETURNS: The interval string in the format 'HH:MM:SS' +# DESCRIPTION: +# THROWS: No exceptions +# COMMENTS: None +# SEE ALSO: +#=============================================================================== +sub interval { + my ($time) = @_; + + return undef unless $time; + + my $date = new Date::Manip::Delta; + unless ( $date->parse($time) ) { + return $date->printf("%02hv:%02mv:%02sv"); + } + else { + warn "Invalid time $time\n"; + return undef; + } + +} + +#=== FUNCTION ================================================================ +# NAME: clean_string +# PURPOSE: Clean a string of non-printables, newlines, multiple spaces +# PARAMETERS: $str The string to process +# RETURNS: The processed string +# DESCRIPTION: Removes leading and trailing spaces. Removes all non-printable +# characters. Removes all CR/LF sequences. Replaces multiple +# spaces with a single space. +# THROWS: No exceptions +# COMMENTS: None +# SEE ALSO: +#=============================================================================== +sub clean_string { + my ($str) = @_; + + $str =~ s/(^\s+|\s+$)//g; + $str =~ tr/[[:graph:]]//c; + $str =~ tr/\x0A\x0D/ /s; + $str =~ tr/ \t/ /s; + + return $str; +} + +#=== FUNCTION ================================================================ +# NAME: replace_underscores +# PURPOSE: Replaces underscores in a string by spaces +# PARAMETERS: $str The string to process +# RETURNS: The processed string +# DESCRIPTION: +# THROWS: No exceptions +# COMMENTS: None +# SEE ALSO: N/A +#=============================================================================== +sub replace_underscores { + my ($str) = @_; + + $str =~ s/_/ /g; + + return $str; +} + +#=== FUNCTION ================================================================ +# NAME: remove_HTML +# PURPOSE: Clean a string of HTML tags +# PARAMETERS: $str The string to process +# RETURNS: The processed string +# DESCRIPTION: +# THROWS: No exceptions +# COMMENTS: None +# SEE ALSO: N/A +#=============================================================================== +sub remove_HTML { + my ($str) = @_; + + my $hr = HTML::Restrict->new(); + my $processed = $hr->process($str); + + return $processed; +} + +#=== FUNCTION ================================================================ +# NAME: Options +# PURPOSE: Processes command-line options +# PARAMETERS: $optref Hash reference to hold the options +# RETURNS: Undef +# DESCRIPTION: +# THROWS: no exceptions +# COMMENTS: none +# SEE ALSO: n/a +#=============================================================================== +sub Options { + my ( $optref, $filter ) = @_; + + my @options = ( + "help", "album:s", "artist:s", "comment:s", + "genre:s", "title:s", "track:s", "year:s", + ); + + # + # Implement '-filter=TAGNAME=FILTERNAME' (from the Getopt::Long manpage) + # + my %opthash + = ( "filter=s%" => sub { push( @{ $filter->{ $_[1] } }, $_[2] ) }, ); + + if ( !GetOptions( $optref, @options, %opthash ) ) { + pod2usage( -msg => "Version $VERSION\n", -exitval => 1 ); + } + + return; +} + +__END__ + +#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +# Application Documentation +#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +#{{{ + +=head1 NAME + +fix_tags - manipulate ID3 tags + +=head1 VERSION + +This documentation refers to I version 1.3.4 + +=head1 USAGE + + fix_tags [ -help ] [-album=ALBUMSTRING] [-artist=ARTISTSTRING] + [-comment=COMMENTSTRING] [-genre=GENRESTRING] [-title=TITLESTRING] + [-track=TRACKNUMBER] [-year=YEAR] [-filter TAGNAME=FILTERNAME] audio_file ... + +=head1 OPTIONS + +=over 8 + +=item B<-help> + +Prints a brief help message describing the usage of the program, and then exits. + +=item B<-album=ALBUMSTRING> + +Sets the album tag to the string defined by the option. Use B<-album=> to +clear the tag. + +=item B<-artist=ARTISTSTRING> + +Sets the artist tag to the string defined by the option. Use B<-artist=> to +clear the tag. + +=item B<-comment=COMMENTSTRING> + +Sets the comment tag to the string defined by the option. Use B<-comment=> to +clear the tag. + +=item B<-genre=GENRESTRING> + +Sets the genre tag to the string defined by the option. Use B<-genre=> to +clear the tag. + +=item B<-title=TITLESTRING> + +Sets the title tag to the string defined by the option. Use B<-title=> to +clear the tag. + +=item B<-track=TRACKNUMBER> + +Sets the track tag to the number defined by the option. Use B<-track=> to +set the tag to zero. + +=item B<-year=YEAR> + +Sets the year tag to the number defined by the option. Use B<-year=> to +set the tag to zero. + +=item B<-filter TAGNAME=FILTERNAME> or B<-filter=TAGNAME=FILTERNAME> + +This option provides an interface to the filtering capability of the script. +Here B denotes the full name of one of the text tags (album, artist, +comment, genre or title - not track or year), and B is one of the +built-in filters: + +=over 4 + +=item B + +The tag string has leading and trailing spaces removed. Multiple internal +spaces and tabs are replaced by a single space. CR/LF sequences are removed, +as are all non-graphic characters. + +=item B + +All underscores in the tag string are replaced by spaces. No space compression +takes place. + +=item B + +The tag string is fed through the I module and all HTML tags +are removed. + +=back + +Neither the B nor the B parts of the option may be +abbreviated, but neither is case-sensitive. + +Multiple filters may be specified by repeating the complete option sequence. +For example: + + fix_tags -filter comment=clean -fil comment=underscore FILE + +The script processes the tags specified in the B<-filter> option in alphabetic +order, and for a given tag it also processes the filters in alphabetic order. + +=back + +=head1 DESCRIPTION + +This script manipulates ID3 tags (or their equivalents) in FLAC, MP3, OGG, SPX +and WAV files. + +=head1 DEPENDENCIES + + Audio::TagLib + Data::Dumper + Date::Manip::Delta + Date::Manip::TZ + File::stat + Getopt::Long + HTML::Restrict + Pod::Usage + TryCatch + + +=head1 BUGS AND LIMITATIONS + +There are no known bugs in this module. +Please report problems to Dave Morriss (Dave.Morriss@gmail.com) +Patches are welcome. + +=head1 AUTHOR + +Dave Morriss (Dave.Morriss@gmail.com) 2011, 2012, 2013, 2014 + +=head1 LICENCE AND COPYRIGHT + +Copyright (c) Dave Morriss (Dave.Morriss@gmail.com). All rights reserved. + +This program is free software. You can redistribute it and/or modify it under +the same terms as perl itself. + +=cut + +#}}} + +# [zo to open fold, zc to close] + +# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker diff --git a/workflow/hpr-logo.jpg b/workflow/hpr-logo.jpg new file mode 100644 index 0000000..c8985dd Binary files /dev/null and b/workflow/hpr-logo.jpg differ diff --git a/workflow/hprid b/workflow/hprid new file mode 100755 index 0000000..477da15 --- /dev/null +++ b/workflow/hprid @@ -0,0 +1,268 @@ +#!/bin/bash +# hprid + +################################################################################ +# +# script to prepare audio files for HPR shows +# +# input: mp3 or ogg file +# result: mp3, ogg in 44100 Hz, spx files 16000Hz with intro and outro +# provides 3 interactive checks for audio quality, intro and outro +# +################################################################################ +# This file is part of the HPR Tool set +# +# HPR Tool set is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# HPR Tool set is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with HPR Tool set. If not, see . +# http://www.gnu.org/licenses/agpl-3.0.html +# ######################################################################### + +################################################################################ +# +# PREREQUISITS +# - current folder has to be writable +# - there should NOT be a temp.ogg or temp.mp3 file +# - intro.mp3 and outro.mp3 have to be present +# - IMPORTANT: sox compiled with mp3 support +# see http://a0u.xanga.com/700438974/howto-soc-installation +# +# +# IMPORTANT +# Backup the files before feeding them to this script, no guarantees here +# Handling of .wav not yet tested, but it should work +# +# code.cruncher, May 2011 +# +################################################################################ + +################################################################################ +# +# TODO +# +# test handling of wav files +# add play final files or open them in specific player(s) +# add handling of ID3 tags +# create html interface for standardized info gathering +# +################################################################################ + + +#============================================================ +# Check input +usage="usage: $(basename $0 ) [ -i ] [ -o ] , -i to add intro and -o outro, fname is a file with audio for HPR" + +CHANNELS="1" +ADDINTRO="n" +ADDOUTRO="n" + + +while getopts "io" opt; do + case $opt in + i ) + ADDINTRO="y" + ;; + o ) + ADDOUTRO="y" + ;; + esac +done +shift $(($OPTIND - 1)) + +# if not ${mediafile} return usage +if [ $# -lt 1 ]; then + echo $usage + exit 1 +fi + +mediafile=${1} + +# test if file exists +if [ ! -f "intro.flac" ]; then + echo "sorry, file \"intro.flac\" does not exist" + echo "To download it run the command:" + echo " wget http://hackerpublicradio.org/media/theme-music/intro.flac" + exit 1 +fi + +if [ ! -f "outro.flac" ]; then + echo "sorry, file \"outro.flac\" does not exist" + echo "To download it run the command:" + echo " wget http://hackerpublicradio.org/media/theme-music/outro.flac" + exit 1 +fi + +for mediafile in "$@" +do + echo $var + + if [ ! -f "${mediafile}" ]; then + echo "sorry, file \"${mediafile}\" does not exist" + continue + fi + + # test if file exists + if [ ! -r "${mediafile}" ]; then + echo "sorry, file \"${mediafile}\" is not readable" + continue + fi + + if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ]; then + echo "sorry, file \"${mediafile}\" has no audio track" + continue + fi + + # extract file name and extension + fname=${mediafile%.*} + ext=${mediafile/*./} + + #Make a backup + # mediafilebackup=${mediafile}_$(md5sum ${mediafile} | cut -c -32 )_orig.${ext} + # cp -v ${mediafile} ${mediafilebackup} + # + # if [[ ! -e ${mediafilebackup} ]]; then + # echo "Backup not made: ${mediafilebackup}" + # exit + # fi + + # check audio quality + dur=7 # playtime of sample in seconds + go=1 # variable to repeat playing of sample + from=180 # start sample at 3 minutes in + + #============================================================ + # Question Time + + # # # while [ $go -ne 0 ] + # # # do + # # # echo + # # # echo "--------------------------------------------------------------------------------" + # # # echo "1/4 AUDIO TEST: check audio quality: ... playing $dur seconds ..." + # # # play "${mediafile}" trim $from $dur + # # # ((from+=180)) # next sample will be 3 minutes later + # # # read -s -n1 -p "sound quality ok?[y,n] ... or play another sample[a] ... [y,n,a]" + # # # echo + # # # case "$REPLY" in + # # # n) echo "aborting ... get better quality sound file ... good bye!"; exit 0;; + # # # y) go=0;; + # # # esac + # # # done + # # # + # # # # Check for intro + # # # echo + # # # echo "--------------------------------------------------------------------------------" + # # # echo "2/4 ADDINTRO TEST: Is the intro playing? " + # # # play "${mediafile}" trim 1 5 # play 5 seconds at beginning of file + # # # read -s -n1 -p "Is there a intro? [y, n]" -i "n"; echo + # # # if [ "$REPLY" = 'y' ]; then + # # # echo "Will add the intro" + # # # ADDINTRO="y" + # # # fi + # # # + # # # # Check for outro + # # # echo + # # # echo "--------------------------------------------------------------------------------" + # # # echo "3/4 ADDOUTRO TEST: Is the outro playing? " + # # # len=$(eval "soxi -D \"${mediafile}\"") + # # # len=$(echo "scale=0; $len - 50" | bc) + # # # play "${mediafile}" trim $len 5 + # # # read -s -n1 -p "Is there a outro ? [y, n]" -i "n"; echo + # # # if [ "$REPLY" = 'y' ]; then + # # # echo "Will add the outro" + # # # ADDOUTRO="y" + # # # fi + # # # + # # # + # # # echo + # # # echo "--------------------------------------------------------------------------------" + # # # echo "4/4 STEREO TEST: Should this be mono or stereo [m,s] ? " + # # # CHANNELS="1" + # # # read -s -n1 -p "intro ok? [m, s]" -i "m" ; echo + # # # if [ "$REPLY" = 's' ]; then + # # # echo "Will convert to stereo" + # # # CHANNELS="2" + # # # fi + + #============================================================ + # Preprocess the source file + + echo "Convert from ${mediafile} to known wav format ${fname}_tmp.wav" + ffmpeg -i ${mediafile} -ar 44100 -ac $CHANNELS ${fname}_tmp.wav > ${fname}_tmp.log 2>&1 + + # echo "Normalising the audio" + # normalize -a 0.5 ${fname}_tmp.wav >> ${fname}_tmp.log 2>&1 + + # TODO Compressor ! + + # TODO add a little speed up + + # TODO little overlap in fade in of intro + + # echo "Truncating the silence" + # sox ${fname}_tmp.wav ${fname}_sox.wav silence -l 1 0.1 1.6% -1 0.6 1.6% >> ${fname}_tmp.log 2>&1 + cp -v ${fname}_tmp.wav ${fname}_sox.wav + + echo "Add the intro if it is missing and add it to the temp pcm file" + if [ "$ADDINTRO" = 'y' ]; then + ffmpeg -i intro.flac -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - > ${fname}_tmp.pcm 2>> ${fname}_tmp.log + fi + + echo "convert the uploaded episode and add it to the temp pcm file" + ffmpeg -i ${fname}_sox.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp.pcm 2>> ${fname}_tmp.log + + echo "Add the outro if it is missing and add it to the temp pcm file" + if [ "$ADDOUTRO" = 'y' ]; then + ffmpeg -i outro.flac -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp.pcm 2>> ${fname}_tmp.log + fi + + echo "Convert the pcm file to a know wav format" + ffmpeg -f s16le -ar 44100 -ac 1 -acodec pcm_s16le -i ${fname}_tmp.pcm ${fname}_mez.wav 2>> ${fname}_tmp.log + + # echo "Get an image of the converted audio" + # sox ${fname}_mez.wav -n spectrogram -x 800 -y 100 -o ${fname}_mez.png + + echo "--------------------------------------------------------------------------------" + echo "File information" + ffprobe ${fname}_mez.wav 2>&1 | grep Audio: + mediainfo ${fname}_mez.wav + + echo "--------------------------------------------------------------------------------" + # display ${fname}_mez.png & + # read -s -n1 -p "Spectrogram check: Everything look ok [y,n] ? " -i "y" ; echo + # if [ "$REPLY" = 'n' ]; then + # echo "Something went w rong. Aborting." + # exit + # fi + + # echo "--------------------------------------------------------------------------------" + # vlc ${fname}_mez.wav >> ${fname}_tmp.log 2>&1 & + # read -s -n1 -p "VLC check: Everything look ok [y,n] ? " -i "y" ; echo + # if [ "$REPLY" = 'n' ]; then + # echo "Something went wrong. Aborting." + # exit + # fi + + echo "Remove temp files" + rm -v ${fname}_tmp.wav ${fname}_sox.wav ${fname}_tmp.pcm ${fname}_tmp.log + + echo "Convert to mp3" # TODO and add tags" + sox -S ${fname}_mez.wav ${fname}_mez.mp3 + + echo "Convert to ogg" # TODO and add tags" + sox -S ${fname}_mez.wav ${fname}_mez.ogg + + echo "Convert to spx" # TODO and add tags" + sox -S ${fname}_mez.wav -c 1 -r 16000 -t wav - | speexenc - ${fname}_mez.spx + + echo "Changing the file dates to the time of upload" + touch -r ${mediafile} ${fname}* +done diff --git a/workflow/hprid.sh b/workflow/hprid.sh new file mode 100755 index 0000000..477da15 --- /dev/null +++ b/workflow/hprid.sh @@ -0,0 +1,268 @@ +#!/bin/bash +# hprid + +################################################################################ +# +# script to prepare audio files for HPR shows +# +# input: mp3 or ogg file +# result: mp3, ogg in 44100 Hz, spx files 16000Hz with intro and outro +# provides 3 interactive checks for audio quality, intro and outro +# +################################################################################ +# This file is part of the HPR Tool set +# +# HPR Tool set is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# HPR Tool set is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with HPR Tool set. If not, see . +# http://www.gnu.org/licenses/agpl-3.0.html +# ######################################################################### + +################################################################################ +# +# PREREQUISITS +# - current folder has to be writable +# - there should NOT be a temp.ogg or temp.mp3 file +# - intro.mp3 and outro.mp3 have to be present +# - IMPORTANT: sox compiled with mp3 support +# see http://a0u.xanga.com/700438974/howto-soc-installation +# +# +# IMPORTANT +# Backup the files before feeding them to this script, no guarantees here +# Handling of .wav not yet tested, but it should work +# +# code.cruncher, May 2011 +# +################################################################################ + +################################################################################ +# +# TODO +# +# test handling of wav files +# add play final files or open them in specific player(s) +# add handling of ID3 tags +# create html interface for standardized info gathering +# +################################################################################ + + +#============================================================ +# Check input +usage="usage: $(basename $0 ) [ -i ] [ -o ] , -i to add intro and -o outro, fname is a file with audio for HPR" + +CHANNELS="1" +ADDINTRO="n" +ADDOUTRO="n" + + +while getopts "io" opt; do + case $opt in + i ) + ADDINTRO="y" + ;; + o ) + ADDOUTRO="y" + ;; + esac +done +shift $(($OPTIND - 1)) + +# if not ${mediafile} return usage +if [ $# -lt 1 ]; then + echo $usage + exit 1 +fi + +mediafile=${1} + +# test if file exists +if [ ! -f "intro.flac" ]; then + echo "sorry, file \"intro.flac\" does not exist" + echo "To download it run the command:" + echo " wget http://hackerpublicradio.org/media/theme-music/intro.flac" + exit 1 +fi + +if [ ! -f "outro.flac" ]; then + echo "sorry, file \"outro.flac\" does not exist" + echo "To download it run the command:" + echo " wget http://hackerpublicradio.org/media/theme-music/outro.flac" + exit 1 +fi + +for mediafile in "$@" +do + echo $var + + if [ ! -f "${mediafile}" ]; then + echo "sorry, file \"${mediafile}\" does not exist" + continue + fi + + # test if file exists + if [ ! -r "${mediafile}" ]; then + echo "sorry, file \"${mediafile}\" is not readable" + continue + fi + + if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ]; then + echo "sorry, file \"${mediafile}\" has no audio track" + continue + fi + + # extract file name and extension + fname=${mediafile%.*} + ext=${mediafile/*./} + + #Make a backup + # mediafilebackup=${mediafile}_$(md5sum ${mediafile} | cut -c -32 )_orig.${ext} + # cp -v ${mediafile} ${mediafilebackup} + # + # if [[ ! -e ${mediafilebackup} ]]; then + # echo "Backup not made: ${mediafilebackup}" + # exit + # fi + + # check audio quality + dur=7 # playtime of sample in seconds + go=1 # variable to repeat playing of sample + from=180 # start sample at 3 minutes in + + #============================================================ + # Question Time + + # # # while [ $go -ne 0 ] + # # # do + # # # echo + # # # echo "--------------------------------------------------------------------------------" + # # # echo "1/4 AUDIO TEST: check audio quality: ... playing $dur seconds ..." + # # # play "${mediafile}" trim $from $dur + # # # ((from+=180)) # next sample will be 3 minutes later + # # # read -s -n1 -p "sound quality ok?[y,n] ... or play another sample[a] ... [y,n,a]" + # # # echo + # # # case "$REPLY" in + # # # n) echo "aborting ... get better quality sound file ... good bye!"; exit 0;; + # # # y) go=0;; + # # # esac + # # # done + # # # + # # # # Check for intro + # # # echo + # # # echo "--------------------------------------------------------------------------------" + # # # echo "2/4 ADDINTRO TEST: Is the intro playing? " + # # # play "${mediafile}" trim 1 5 # play 5 seconds at beginning of file + # # # read -s -n1 -p "Is there a intro? [y, n]" -i "n"; echo + # # # if [ "$REPLY" = 'y' ]; then + # # # echo "Will add the intro" + # # # ADDINTRO="y" + # # # fi + # # # + # # # # Check for outro + # # # echo + # # # echo "--------------------------------------------------------------------------------" + # # # echo "3/4 ADDOUTRO TEST: Is the outro playing? " + # # # len=$(eval "soxi -D \"${mediafile}\"") + # # # len=$(echo "scale=0; $len - 50" | bc) + # # # play "${mediafile}" trim $len 5 + # # # read -s -n1 -p "Is there a outro ? [y, n]" -i "n"; echo + # # # if [ "$REPLY" = 'y' ]; then + # # # echo "Will add the outro" + # # # ADDOUTRO="y" + # # # fi + # # # + # # # + # # # echo + # # # echo "--------------------------------------------------------------------------------" + # # # echo "4/4 STEREO TEST: Should this be mono or stereo [m,s] ? " + # # # CHANNELS="1" + # # # read -s -n1 -p "intro ok? [m, s]" -i "m" ; echo + # # # if [ "$REPLY" = 's' ]; then + # # # echo "Will convert to stereo" + # # # CHANNELS="2" + # # # fi + + #============================================================ + # Preprocess the source file + + echo "Convert from ${mediafile} to known wav format ${fname}_tmp.wav" + ffmpeg -i ${mediafile} -ar 44100 -ac $CHANNELS ${fname}_tmp.wav > ${fname}_tmp.log 2>&1 + + # echo "Normalising the audio" + # normalize -a 0.5 ${fname}_tmp.wav >> ${fname}_tmp.log 2>&1 + + # TODO Compressor ! + + # TODO add a little speed up + + # TODO little overlap in fade in of intro + + # echo "Truncating the silence" + # sox ${fname}_tmp.wav ${fname}_sox.wav silence -l 1 0.1 1.6% -1 0.6 1.6% >> ${fname}_tmp.log 2>&1 + cp -v ${fname}_tmp.wav ${fname}_sox.wav + + echo "Add the intro if it is missing and add it to the temp pcm file" + if [ "$ADDINTRO" = 'y' ]; then + ffmpeg -i intro.flac -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - > ${fname}_tmp.pcm 2>> ${fname}_tmp.log + fi + + echo "convert the uploaded episode and add it to the temp pcm file" + ffmpeg -i ${fname}_sox.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp.pcm 2>> ${fname}_tmp.log + + echo "Add the outro if it is missing and add it to the temp pcm file" + if [ "$ADDOUTRO" = 'y' ]; then + ffmpeg -i outro.flac -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp.pcm 2>> ${fname}_tmp.log + fi + + echo "Convert the pcm file to a know wav format" + ffmpeg -f s16le -ar 44100 -ac 1 -acodec pcm_s16le -i ${fname}_tmp.pcm ${fname}_mez.wav 2>> ${fname}_tmp.log + + # echo "Get an image of the converted audio" + # sox ${fname}_mez.wav -n spectrogram -x 800 -y 100 -o ${fname}_mez.png + + echo "--------------------------------------------------------------------------------" + echo "File information" + ffprobe ${fname}_mez.wav 2>&1 | grep Audio: + mediainfo ${fname}_mez.wav + + echo "--------------------------------------------------------------------------------" + # display ${fname}_mez.png & + # read -s -n1 -p "Spectrogram check: Everything look ok [y,n] ? " -i "y" ; echo + # if [ "$REPLY" = 'n' ]; then + # echo "Something went w rong. Aborting." + # exit + # fi + + # echo "--------------------------------------------------------------------------------" + # vlc ${fname}_mez.wav >> ${fname}_tmp.log 2>&1 & + # read -s -n1 -p "VLC check: Everything look ok [y,n] ? " -i "y" ; echo + # if [ "$REPLY" = 'n' ]; then + # echo "Something went wrong. Aborting." + # exit + # fi + + echo "Remove temp files" + rm -v ${fname}_tmp.wav ${fname}_sox.wav ${fname}_tmp.pcm ${fname}_tmp.log + + echo "Convert to mp3" # TODO and add tags" + sox -S ${fname}_mez.wav ${fname}_mez.mp3 + + echo "Convert to ogg" # TODO and add tags" + sox -S ${fname}_mez.wav ${fname}_mez.ogg + + echo "Convert to spx" # TODO and add tags" + sox -S ${fname}_mez.wav -c 1 -r 16000 -t wav - | speexenc - ${fname}_mez.spx + + echo "Changing the file dates to the time of upload" + touch -r ${mediafile} ${fname}* +done diff --git a/workflow/hprtranscode-archive.bash b/workflow/hprtranscode-archive.bash new file mode 100644 index 0000000..659a390 --- /dev/null +++ b/workflow/hprtranscode-archive.bash @@ -0,0 +1,422 @@ +#!/bin/bash +# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ +#============================================================ +# Check input +usage="usage: $(basename $0 ) [ -i ] [ -o ] , -i to add intro and -o outro, fname is a file with audio for HPR" + +# Argument = -t test -r server -p password -v + +echoerr() +{ +echo "$@" 1>&2; +} + +usage() +{ +cat << EOF +usage: $0 [options] {media file to encode} {episode number} + +OPTIONS: + -h Show this message + -p add a promo + -i add the intro + -s Automatically generate the summary + -o add the outro + -b add the intro and outro ( and summary default) + -2 encode to 2 channels + -n no audio normalization + -x eXclude sponsor mention + +EOF +} + +TEMP_DIR="/var/tmp/" +CHANNELS="1" +FIXAUDIO="1" +ADDSUMMARY="n" +ADDINTRO="n" +ADDPROMO="n" +ADDOUTRO="n" +ADDSPONSOR="y" +ARTIST="EMPTY" +TITLE="EMPTY" +YEAR="EMPTY" +SLOT="EMPTY" +basedir="/var/IA" +intro="${basedir}/intro.flac" +promo="${basedir}/promo.flac" +outro="${basedir}/outro.flac" +anhonesthost="${basedir}/sponsor-anhonesthost.com-hpr15.flac" +internetarchive="${basedir}/sponsor-archive.org.flac" + +while getopts "hipsob2nx" OPTION +do + case $OPTION in + h) + usage + exit 1 + ;; + s) + ADDSUMMARY="y" + ;; + i) + ADDINTRO="y" + ;; + p) + ADDPROMO="y" + ;; + o) + ADDOUTRO="y" + ;; + b) + ADDINTRO="y" + ADDOUTRO="y" + ;; + 2) + CHANNELS="2" + ;; + n) + FIXAUDIO="0" + ;; + x) + ADDSPONSOR="n" + ;; + ?) + usage + exit + ;; + esac +done + +shift $(($OPTIND - 1)) + +if [ "$#" -ne 2 ]; then + echoerr "Please enter the source file and episode number" + exit +fi + +mediafile=${1} +ep_num=${2} + +ep_num=$(echo $ep_num | sed 's/hpr//g') +re='^[0-9]+$' +if ! [[ $ep_num =~ $re ]] ; then + echoerr "error: episode \"${ep_num}\" is not a number" + exit 1 +fi + +if [ ! -f "${anhonesthost}" ]; then + echoerr "sorry, file \"${anhonesthost}\" does not exist" + exit 1 +fi + +if [ ! -f "${internetarchive}" ]; then + echoerr "sorry, file \"${internetarchive}\" does not exist" + exit 1 +fi + +if [ ! -f "${intro}" ]; then + echoerr "sorry, file \"intro.flac\" does not exist" + exit 1 +fi + +if [ "$ADDPROMO" = 'y' ]; then + if [ ! -f "${promo}" ]; then + echoerr "sorry, file \"promo.flac\" does not exist" + exit 1 + fi +fi + +if [ ! -f "${outro}" ]; then + echoerr "sorry, file \"outro.flac\" does not exist" + exit 1 +fi + +if [ ! -f "${mediafile}" ]; then + echoerr "sorry, media file \"${mediafile}\" does not exist" + exit +fi + +if [ ! -r "${mediafile}" ]; then + echoerr "sorry, media file \"${mediafile}\" is not readable" + exit +fi + +if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ]; then + echoerr "sorry, media file \"${mediafile}\" has no audio track" + exit +fi + +# extract file name and extension +media_dir=$(dirname ${mediafile}) +fname=${mediafile%.*} +ext=${mediafile/*./} + +if [ "$ADDSUMMARY" = 'n' ]; then + if [ ! -e "${media_dir}/summary.wav" ] + then + echoerr "ERROR: Can not find the summary file \"${media_dir}/summary.wav\"" + wget -O- --timeout=10 --tries=1 --quiet http://hackerpublicradio.org/say.php?id=${ep_num} | grep HPR_summary + exit 1 + fi +fi + +if [[ -e hpr${ep_num}.wav ]] || [[ -e hpr${ep_num}.mp3 ]] || [[ -e hpr${ep_num}.ogg ]] || [[ -e hpr${ep_num}.spx ]] || [[ -e hpr${ep_num}_summary.wav ]] || [[ -e ${fname}_mez_norm.wav ]] || [[ -e ${fname}_mez.wav ]] || [[ -e ${fname}_sox_norm.wav ]] || [[ -e ${fname}_sox.wav ]] || [[ -e ${fname}_tmp_hh.pcm ]] || [[ -e ${fname}_tmp_ia.pcm ]] || [[ -e ${fname}_tmp.log ]] +then + echoerr "Files for this episode already exist." + ls -1 hpr${ep_num}* ${fname}_* 2>/dev/null + exit 1 +fi + +echo "--------------------------------------------------------------------------------" +echo "Geting metadata for hpr${ep_num}" + +while read -r line +do + field=$(echo $line | awk -F ':' '{print $1}') + case $field in + "HPR_summary") + HPR_summary=$(echo $line | grep "HPR_summary: " | cut -c 14- ) + continue + ;; + "HPR_album") + HPR_album=$(echo $line | grep "HPR_album: " | cut -c 12- ) + continue + ;; + "HPR_artist") + HPR_artist=$(echo $line | grep "HPR_artist: " | cut -c 13- ) + continue + ;; + "HPR_comment") + HPR_comment=$(echo $line | grep "HPR_comment: " | cut -c 14- ) + continue + ;; + "HPR_genre") + HPR_genre=$(echo $line | grep "HPR_genre: " | cut -c 12- ) + continue + ;; + "HPR_title") + HPR_title=$(echo $line | grep "HPR_title: " | cut -c 12- ) + continue + ;; + "HPR_track") + HPR_track=$(echo $line | grep "HPR_track: " | cut -c 12- ) + continue + ;; + "HPR_year") + HPR_year=$(echo $line | grep "HPR_year: " | cut -c 11- ) + continue + ;; + "HPR_duration") + HPR_duration=$(echo $line | grep "HPR_duration: " | cut -c 15- ) + continue + ;; + "HPR_explicit") + HPR_explicit=$(echo $line | grep "HPR_explicit: " | cut -c 15- ) + continue + ;; + "HPR_license") + HPR_license=$(echo $line | grep "HPR_license: " | cut -c 14- ) + continue + ;; + esac +done < <( wget --timeout=10 --tries=1 --quiet http://hackerpublicradio.org/say.php?id=${ep_num} -O - ) + +if [[ -z "$ADDINTRO" || -z "$ADDPROMO" || -z "$ADDSUMMARY" || -z "$ADDOUTRO" || -z "$HPR_album" || -z "$HPR_artist" || -z "$HPR_comment" || -z "$HPR_genre" || -z "$HPR_title" || -z "$HPR_track" || -z "$HPR_year" || -z "$HPR_summary" || -z "$HPR_duration" || -z "$HPR_explicit" || -z "$HPR_license" ]] +then + echoerr "Could not find information on ${ep_num}. Has the show been posted ?" + exit; +fi +echo "--------------------------------------------------------------------------------" +echo "Add intro : $ADDINTRO" +echo "Add promo : $ADDPROMO" +echo "Add Summary : $ADDSUMMARY" +echo "Add outro : $ADDOUTRO" +echo "album : $HPR_album" +echo "artist : $HPR_artist" +echo "comment : $HPR_comment" +echo "genre : $HPR_genre" +echo "title : $HPR_title" +echo "track : $HPR_track" +echo "year : $HPR_year" +echo "summary : $HPR_summary" +echo "duration : $HPR_duration" +echo "explicit : $HPR_explicit" +echo "license : $HPR_license" + +if [[ $HPR_duration == "0" ]] +then + echoerr "The duration is set to 0. Please update the show with the correct time." + exit; +fi + +#============================================================ +# Preproc`s the source file + +echo "--------------------------------------------------------------------------------" +echo "Prepare mezzanine file" +ffmpeg -i ${mediafile} -ar 44100 -ac $CHANNELS ${fname}_sox.wav > ${fname}_tmp.log 2>&1 + +if [ "$FIXAUDIO" = "1" ];then + echo "normalizing audio" + sox --temp "${TEMP_DIR}" --norm ${fname}_sox.wav ${fname}_sox_norm.wav + mv -v ${fname}_sox_norm.wav ${fname}_sox.wav >> ${fname}_tmp.log 2>&1 +# normalize -a 0.5 ${fname}_sox.wav >> ${fname}_tmp.log 2>&1 +fi + +echo "--------------------------------------------------------------------------------" +echo "Add HPR Branding" + +if [ "$ADDSUMMARY" = 'y' ]; then + echo "Creating the summary" + #echo "$HPR_summary" - | espeak -w hpr${ep_num}_summary.wav + echo "$HPR_summary" - | text2wave - -o hpr${ep_num}_summary.wav #festival --tts + #echo "${HPR_summary}" | gtts-cli - --output hpr${ep_num}_summary.mp3 + #ffmpeg -i hpr${ep_num}_summary.mp3 hpr${ep_num}_summary.wav + #rm -v hpr${ep_num}_summary.mp3 +else + echo "Copying the supplied summary" + if [ ! -e "${media_dir}/summary.wav" ] + then + echoerr "ERROR: Can not find the summary file \"${media_dir}/summary.wav\"" + exit 1 + fi + cp -v "${media_dir}/summary.wav" hpr${ep_num}_summary.wav +fi + +ffmpeg -i hpr${ep_num}_summary.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log +ffmpeg -i hpr${ep_num}_summary.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log +rm hpr${ep_num}_summary.wav + +if [ "$ADDSPONSOR" = 'y' ]; then + echo "Adding the sponsor" + ffmpeg -i "$anhonesthost" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log + ffmpeg -i "$internetarchive" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log +else + echo "NOT Adding the sponsor" +fi + +if [ "$ADDPROMO" = 'y' ]; then + echo "Adding the promo" + ffmpeg -i "$promo" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log + ffmpeg -i "$promo" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log +fi + +if [ "$ADDINTRO" = 'y' ]; then + echo "Adding the intro" + ffmpeg -i "$intro" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log + ffmpeg -i "$intro" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log +fi + +echo "convert the uploaded episode and add it to the temp pcm file" +ffmpeg -i ${fname}_sox.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log +ffmpeg -i ${fname}_sox.wav -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log + +if [ "$ADDOUTRO" = 'y' ]; then + echo "Adding the outro" + ffmpeg -i "$outro" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_hh.pcm 2>> ${fname}_tmp.log + ffmpeg -i "$outro" -ar 44100 -ac $CHANNELS -acodec pcm_s16le -f s16le - >> ${fname}_tmp_ia.pcm 2>> ${fname}_tmp.log +fi + +echo "Convert the pcm file to a know wav format" +ffmpeg -f s16le -ar 44100 -ac $CHANNELS -acodec pcm_s16le -i ${fname}_tmp_hh.pcm ${fname}_mez.wav 2>> ${fname}_tmp.log +ffmpeg -f s16le -ar 44100 -ac $CHANNELS -acodec pcm_s16le -i ${fname}_tmp_ia.pcm hpr${ep_num}.wav 2>> ${fname}_tmp.log + +echo "Normalizing the wav files" +sox --temp "${TEMP_DIR}" --norm ${fname}_mez.wav ${fname}_mez_norm.wav +mv -v ${fname}_mez_norm.wav ${fname}_mez.wav >> ${fname}_tmp.log 2>&1 +# normalize -a 0.5 ${fname}_mez.wav >> ${fname}_tmp.log 2>&1 +sox --temp "${TEMP_DIR}" --norm hpr${ep_num}.wav hpr${ep_num}_norm.wav +mv -v hpr${ep_num}_norm.wav /var/IA/uploads/hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1 +# normalize -a 0.5 hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1 + +echo "--------------------------------------------------------------------------------" +echo "File information" +ffprobe ${fname}_mez.wav 2>&1 | grep Audio: +mediainfo ${fname}_mez.wav + +echo "--------------------------------------------------------------------------------" +echo "Convert to opus for IA" +opusenc hpr${ep_num}.wav /var/IA/uploads/hpr${ep_num}.opus + +echo "--------------------------------------------------------------------------------" +echo "Convert to flac for IA" +sox --temp "${TEMP_DIR}" -S hpr${ep_num}.wav /var/IA/uploads/hpr${ep_num}.flac + +echo "--------------------------------------------------------------------------------" +echo "Convert to mp3 for HPR" +sox --temp "${TEMP_DIR}" -S ${fname}_mez.wav hpr${ep_num}.mp3 +echo "Convert to mp3 for IA" +sox --temp "${TEMP_DIR}" -S hpr${ep_num}.wav /var/IA/uploads/hpr${ep_num}.mp3 + +echo "--------------------------------------------------------------------------------" +echo "Convert to ogg for HPR" +sox --temp "${TEMP_DIR}" -S ${fname}_mez.wav hpr${ep_num}.ogg +echo "Convert to ogg for IA" +sox --temp "${TEMP_DIR}" -S hpr${ep_num}.wav /var/IA/uploads/hpr${ep_num}.ogg + +echo "--------------------------------------------------------------------------------" +echo "Convert to spx for HPR" +sox --temp "${TEMP_DIR}" -S ${fname}_mez.wav -c 1 -r 16000 -t wav - | speexenc - hpr${ep_num}.spx +echo "Convert to spx for IA" +sox --temp "${TEMP_DIR}" -S hpr${ep_num}.wav -c 1 -r 16000 -t wav - | speexenc - /var/IA/uploads/hpr${ep_num}.spx + + +if [[ ! -s /var/IA/uploads/hpr${ep_num}.wav ]] || [[ ! -s /var/IA/uploads/hpr${ep_num}.mp3 ]] || [[ ! -s /var/IA/uploads/hpr${ep_num}.ogg ]] || [[ ! -s /var/IA/uploads/hpr${ep_num}.spx ]] || [[ ! -s hpr${ep_num}.mp3 ]] || [[ ! -s hpr${ep_num}.ogg ]] || [[ ! -s hpr${ep_num}.spx ]] +then + echoerr "ERROR: Something went wrong encoding the files" + exit 1 +fi + +fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="${HPR_comment} The license is ${HPR_license}" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2 +fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="$HPR_comment" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" /var/IA/uploads/hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2 +fix_tags hpr${ep_num}* +fix_tags /var/IA/uploads/hpr${ep_num}* + +#echo "Changing the file dates to the time of upload" +touch -r ${mediafile} hpr${ep_num}* +touch -r ${mediafile} /var/IA/uploads/hpr${ep_num}* + + +rsync -ave ssh --partial --progress --ignore-existing hpr${ep_num}.mp3 hpr${ep_num}.ogg hpr${ep_num}.spx hpr:www/eps/ + +firefox http://hackerpublicradio.org/local/hpr${ep_num}.mp3 +firefox http://hackerpublicradio.org/local/hpr${ep_num}.ogg +firefox file:///var/IA/uploads/hpr${ep_num}.mp3 +firefox file:///var/IA/uploads/hpr${ep_num}.ogg + +echo "Source: $( mediainfo --Output=XML --Full "${mediafile}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" +mediainfo --Output=XML --Full hpr${ep_num}* /var/IA/uploads/hpr${ep_num}* | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' | sort | uniq -c + +read -p "Remove files for \"${fname}\" (y|N) ? " -n 1 -r +echo # (optional) move to a new line +if [[ $REPLY =~ ^[Yy]$ ]] +then + mediafilename=$(basename "${mediafile}") + mediaextension="${mediafilename##*.}" + + ssh hpr -t "mkdir /home/hpr/www/eps/hpr${ep_num}" >/dev/null 2>&1 + rsync -ave ssh --partial --progress --ignore-existing "${mediafile}" hpr:www/eps/hpr${ep_num}/hpr${ep_num}_source.${mediaextension} + ssh hpr -t "ls -al /home/hpr/www/eps/hpr${ep_num}*" + cp -v "${mediafile}" "/var/IA/uploads/hpr${ep_num}_source.${mediaextension}" + + #echo "Remove temp files" + rm -v ${fname}_sox.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav + mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/ + wget --timeout=0 -q "http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1" -O - | xmlstarlet val --err - + wget --timeout=0 -q "http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1" -O - | xmlstarlet val --err - + wget --timeout=0 -q "http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1" -O - | xmlstarlet val --err - + + #rsync -ave ssh --partial --progress /var/IA/uploads/ hpr:/home/hpr/upload/processed/ + rsync -ave ssh --partial --progress /var/IA/uploads/ borg:/data/IA/uploads/ + find /var/IA/done/ -empty -delete +else + echo "skipping...." + echo "cp -v \"${mediafile}\" \"/var/IA/uploads/hpr${ep_num}_source.${mediaextension}\"" + echo "rm -v ${fname}_sox.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav" + echo "mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/" + echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1\" -O - | xmlstarlet val --err -" + echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1\" -O - | xmlstarlet val --err -" + echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1\" -O - | xmlstarlet val --err -" + echo "rsync -ave ssh --partial --progress /var/IA/uploads/ borg:/data/IA/uploads/" +fi diff --git a/workflow/hprtranscode-just-transcode.bash b/workflow/hprtranscode-just-transcode.bash new file mode 100755 index 0000000..e14ecbb --- /dev/null +++ b/workflow/hprtranscode-just-transcode.bash @@ -0,0 +1,238 @@ +#!/bin/bash +# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ +#============================================================ + +echoerr() +{ +echo "$@" 1>&2; +} + +TEMP_DIR="/var/tmp/" +CHANNELS="1" +FIXAUDIO="1" +ARTIST="EMPTY" +TITLE="EMPTY" +YEAR="EMPTY" +SLOT="EMPTY" +basedir="/var/IA" +upload_dir="${basedir}/uploads" +intro="${basedir}/short.flac" + + +if [ "$#" -ne 2 ]; then + echoerr "Please enter the source file and episode number" + exit +fi + +mediafile=${1} +ep_num=${2} + +ep_num=$(echo $ep_num | sed 's/hpr//g') +re='^[0-9]+$' +if ! [[ $ep_num =~ $re ]] ; then + echoerr "error: episode \"${ep_num}\" is not a number" + exit 1 +fi + +if [ ! -f "${mediafile}" ]; then + echoerr "sorry, media file \"${mediafile}\" does not exist" + exit +fi + +if [ ! -r "${mediafile}" ]; then + echoerr "sorry, media file \"${mediafile}\" is not readable" + exit +fi + +if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ]; then + echoerr "sorry, media file \"${mediafile}\" has no audio track" + exit +fi + +# extract file name and extension +media_dir=$(dirname ${mediafile}) +fname=${mediafile%.*} +ext=${mediafile/*./} + +if [[ -e hpr${ep_num}.wav ]] || [[ -e hpr${ep_num}.mp3 ]] || [[ -e hpr${ep_num}.ogg ]] || [[ -e hpr${ep_num}.spx ]] || [[ -e hpr${ep_num}_summary.wav ]] || [[ -e hpr${ep_num}.wav ]] || [[ -e ${fname}_mez.wav ]] || [[ -e ${fname}_sox_norm.wav ]] || [[ -e ${fname}_mezzanine.wav ]] || [[ -e ${fname}_tmp.pcm ]] || [[ -e ${fname}_tmp.log ]] +then + echoerr "Files for this episode already exist." + ls -1 hpr${ep_num}* ${fname}_* 2>/dev/null + exit 1 +fi + +echo "--------------------------------------------------------------------------------" +echo "Geting metadata for hpr${ep_num}" + +while read -r line +do + field=$(echo $line | awk -F ':' '{print $1}') + case $field in + "HPR_summary") + HPR_summary=$(echo $line | grep "HPR_summary: " | cut -c 14- ) + continue + ;; + "HPR_album") + HPR_album=$(echo $line | grep "HPR_album: " | cut -c 12- ) + continue + ;; + "HPR_artist") + HPR_artist=$(echo $line | grep "HPR_artist: " | cut -c 13- ) + continue + ;; + "HPR_comment") + HPR_comment=$(echo $line | grep "HPR_comment: " | cut -c 14- ) + continue + ;; + "HPR_genre") + HPR_genre=$(echo $line | grep "HPR_genre: " | cut -c 12- ) + continue + ;; + "HPR_title") + HPR_title=$(echo $line | grep "HPR_title: " | cut -c 12- ) + continue + ;; + "HPR_track") + HPR_track=$(echo $line | grep "HPR_track: " | cut -c 12- ) + continue + ;; + "HPR_year") + HPR_year=$(echo $line | grep "HPR_year: " | cut -c 11- ) + continue + ;; + "HPR_duration") + HPR_duration=$(echo $line | grep "HPR_duration: " | cut -c 15- ) + continue + ;; + "HPR_explicit") + HPR_explicit=$(echo $line | grep "HPR_explicit: " | cut -c 15- ) + continue + ;; + "HPR_license") + HPR_license=$(echo $line | grep "HPR_license: " | cut -c 14- ) + continue + ;; + esac +done < <( wget --timeout=10 --tries=1 --quiet http://hackerpublicradio.org/say.php?id=${ep_num} -O - ) + +if [[ -z "$HPR_album" || -z "$HPR_artist" || -z "$HPR_comment" || -z "$HPR_genre" || -z "$HPR_title" || -z "$HPR_track" || -z "$HPR_year" || -z "$HPR_summary" || -z "$HPR_duration" || -z "$HPR_explicit" || -z "$HPR_license" ]] +then + echoerr "Could not find information on ${ep_num}. Has the show been posted ?" + exit; +fi +echo "--------------------------------------------------------------------------------" +echo "album : $HPR_album" +echo "artist : $HPR_artist" +echo "comment : $HPR_comment" +echo "genre : $HPR_genre" +echo "title : $HPR_title" +echo "track : $HPR_track" +echo "year : $HPR_year" +echo "summary : $HPR_summary" +echo "duration : $HPR_duration" +echo "explicit : $HPR_explicit" +echo "license : $HPR_license" + +if [[ $HPR_duration == "0" ]] +then + echoerr "The duration is set to 0. Please update the show with the correct time." + exit; +fi + +#============================================================ +# Preproc`s the source file + +echo "--------------------------------------------------------------------------------" +echo "Prepare mezzanine file" +ffmpeg -i ${mediafile} -ar 44100 -ac $CHANNELS hpr${ep_num}.wav > ${fname}_tmp.log 2>&1 + +echo "Normalizing the wav files" +sox --temp "${TEMP_DIR}" --norm hpr${ep_num}.wav hpr${ep_num}_norm.wav +mv -v hpr${ep_num}_norm.wav ${upload_dir}/hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1 + +echo "--------------------------------------------------------------------------------" +echo "File information" +ffprobe ${upload_dir}/hpr${ep_num}.wav 2>&1 | grep Audio: +mediainfo ${upload_dir}/hpr${ep_num}.wav + +echo "--------------------------------------------------------------------------------" +echo "Convert to opus" +opusenc ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.opus 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" +echo "Convert to flac" +sox --temp "${TEMP_DIR}" -S ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.flac 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" +echo "Convert to mp3" +sox --temp "${TEMP_DIR}" -S ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.mp3 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" +echo "Convert to ogg" +sox --temp "${TEMP_DIR}" -S ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.ogg 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" +echo "Convert to spx" +sox --temp "${TEMP_DIR}" -S ${upload_dir}/hpr${ep_num}.wav -c 1 -r 16000 -t wav - | speexenc - ${upload_dir}/hpr${ep_num}.spx 2>> ${fname}_tmp.log 1>&2 + +if [[ ! -s ${upload_dir}/hpr${ep_num}.wav ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.opus ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.flac ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.mp3 ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.ogg ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.spx ]] +then + echoerr "ERROR: Something went wrong encoding the files" + exit 1 +fi + +fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="${HPR_comment} The license is ${HPR_license}" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" ${upload_dir}/hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2 + +fix_tags ${upload_dir}/hpr${ep_num}* + +#echo "Changing the file dates to the time of upload" +touch -r ${mediafile} hpr${ep_num}* +touch -r ${mediafile} ${upload_dir}/hpr${ep_num}* + +ls -al hpr${ep_num}* ${upload_dir}/hpr${ep_num}* + +rsync -ave ssh --partial --progress --ignore-existing ${upload_dir}/hpr${ep_num}.mp3 ${upload_dir}/hpr${ep_num}.ogg ${upload_dir}/hpr${ep_num}.spx hpr:www/eps/ + +firefox http://hackerpublicradio.org/local/hpr${ep_num}.mp3 +firefox http://hackerpublicradio.org/local/hpr${ep_num}.ogg +firefox file://${upload_dir}/hpr${ep_num}.mp3 +firefox file://${upload_dir}/hpr${ep_num}.ogg +mpv "http://hackerpublicradio.org/local/hpr${ep_num}.spx" "http://hackerpublicradio.org/local/hpr${ep_num}.ogg" "http://hackerpublicradio.org/local/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.spx" "file://${upload_dir}/hpr${ep_num}.opus" "file://${upload_dir}/hpr${ep_num}.ogg" "file://${upload_dir}/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.flac" + +echo "Source: $( mediainfo --Output=XML --Full "${mediafile}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" +mediainfo --Output=XML --Full hpr${ep_num}* ${upload_dir}/hpr${ep_num}* | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' | sort | uniq -c + +read -p "Remove files for \"${fname}\" (y|N) ? " -n 1 -r +echo # (optional) move to a new line +if [[ $REPLY =~ ^[Yy]$ ]] +then + mediafilename=$(basename "${mediafile}") + mediaextension="${mediafilename##*.}" + + ssh hpr -t "mkdir /home/hpr/www/eps/hpr${ep_num}" >/dev/null 2>&1 + rsync -ave ssh --partial --progress --ignore-existing "${mediafile}" hpr:www/eps/hpr${ep_num}/hpr${ep_num}_source.${mediaextension} + ssh hpr -t "ls -al /home/hpr/www/eps/hpr${ep_num}*" + cp -v "${mediafile}" "${upload_dir}/hpr${ep_num}_source.${mediaextension}" + + #echo "Remove temp files" + rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav + mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/ + wget --timeout=0 -q "http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1" -O - | xmlstarlet val --err - + wget --timeout=0 -q "http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1" -O - | xmlstarlet val --err - + wget --timeout=0 -q "http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1" -O - | xmlstarlet val --err - + wget --timeout=0 -q "http://hackerpublicradio.org/rss-future.php" -O - | xmlstarlet val --err - + + #rsync -ave ssh --partial --progress ${upload_dir}/ hpr:/home/hpr/upload/processed/ + rsync -ave ssh --partial --progress ${upload_dir}/ borg:/data/IA/uploads/ + echo "$( ssh borg -t "ls -al /data/IA/uploads/hpr${ep_num}*" ; ls -al ${upload_dir}/hpr${ep_num}* )" | grep "hpr${ep_num}" | awk '{print $5, $NF}' | sort + find /var/IA/done/ -empty -delete +else + echo "skipping...." + echo "cp -v \"${mediafile}\" \"${upload_dir}/hpr${ep_num}_source.${mediaextension}\"" + echo "rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav" + echo "mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/" + echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1\" -O - | xmlstarlet val --err -" + echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1\" -O - | xmlstarlet val --err -" + echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1\" -O - | xmlstarlet val --err -" + echo "rsync -ave ssh --partial --progress ${upload_dir}/ borg:/data/IA/uploads/" +fi diff --git a/workflow/hprtranscode-simple.bash b/workflow/hprtranscode-simple.bash new file mode 100755 index 0000000..191f1ae --- /dev/null +++ b/workflow/hprtranscode-simple.bash @@ -0,0 +1,603 @@ +#!/usr/bin/env bash +# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ +#============================================================ +TEMP_DIR="/var/tmp/" +CHANNELS="1" +FIXAUDIO="1" +ARTIST="EMPTY" +TITLE="EMPTY" +YEAR="EMPTY" +SLOT="EMPTY" +basedir="/var/IA" +upload_dir="${basedir}/uploads" +theme="${basedir}/theme.wav" +outro="${basedir}/2022-03-07-outro.wav" +ttsserver="http://localhost:5500" +processing_dir="/home/ken/tmp/hpr/processing" +git_image_dir="/home/ken/sourcecode/hpr/HPR_Public_Code/www/images/hosts" +acceptable_duration_difference="2" # Seconds +thedate=$(/usr/bin/date -u +%Y-%m-%d_%H-%M-%SZ_%A) +echo "Processing the next HPR Show in the queue" + + +if [ $( curl -s -o /dev/null -w "%{http_code}" -X 'GET' "${ttsserver}/api/voices" -H 'accept: */*' ) != "200" ] +then + echo "Please start the tts-server \"podman run -it -p 5500:5500 synesthesiam/opentts:en\"" + exit +fi + +function abs_diff { + echo $(($1 >= $2 ? $1 - $2 : $2 - $1)) +} + +function create_tts_summary { + HPR_summary="$( cat "hpr${ep_num}_summary.txt" )" + echo "INFO: Converting text \"${HPR_summary}\" to speech." + curl -X 'GET' -G --data-urlencode "voice=coqui-tts:en_ljspeech" --data-urlencode "text=${HPR_summary}" --data-urlencode "vocoder=high" --data-urlencode "denoiserStrength=0.03" --data-urlencode "cache=false" ${ttsserver}/api/tts -H 'accept: */*' --output ~hpr${ep_num}_summary.wav +} + +################### +# Locate media directory +# +# +show_type="" +if [[ -f "${1}" && -n "${2}" ]] +then + show_type="local" + mediafile="${1}" + media_dir="$( dirname "${mediafile}" )" + ep_num="${2}" + echo "The duration is \"$( \date -ud "1970-01-01 $( ffprobe -i "${mediafile}" 2>&1| awk -F ': |, ' '/Duration:/ { print $2 }' )" +%s )\"." +else + show_type="remote" + response=$( curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" | \ + grep 'SHOW_POSTED' | \ + head -1 | \ + sed 's/,/ /g' ) + + if [ -z "${response}" ] + then + echo "INFO: There appear to be no more shows with the status \"SHOW_POSTED\"." + echo "Getting a list of all the reservations." + curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" + exit 3 + fi + + timestamp_epoc="$( echo ${response} | awk '{print $1}' )" + ep_num="$( echo ${response} | awk '{print $2}' )" + ep_date="$( echo ${response} | awk '{print $3}' )" + key="$( echo ${response} | awk '{print $4}' )" + status="$( echo ${response} | awk '{print $5}' )" + email="$( echo ${response} | awk '{print $6}' )" + + #source_dir="hpr:/home/hpr/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}" + dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}" + media_dir="${processing_dir}/${timestamp_epoc}_${ep_num}_${ep_date}_${key}" +fi + +if [ -z "${show_type}" ] +then + echo "sorry, variable \"show_type\" is not set." + exit +fi + +if [ -z "${media_dir}" ] +then + echo "sorry, variable \"media_dir\" is not set." + exit +fi + +if [ ! -d "${media_dir}" ] +then + echo "sorry, directory \"media_dir: ${media_dir}\" does not exist" + exit 1 +fi + +echo detox -v "${media_dir}/" +detox -vr "${media_dir}/" + +if [[ "$( find "${media_dir}" \( -iname "hpr${ep_num}.wav" -o -iname "hpr${ep_num}.mp3" -o -iname "hpr${ep_num}.ogg" -o -iname "hpr${ep_num}.spx" -o -iname "hpr${ep_num}_summary.txt" -o -iname "hpr${ep_num}_summary.wav" -o -iname "hpr${ep_num}.wav" -o -iname "*_mez.wav" -o -iname "*_sox_norm.wav" -o -iname "*_mezzanine.wav" -o -iname "*_tmp.pcm" -o -iname "hpr${ep_num}_intro.wav" -o -iname "~hpr${ep_num}_summary.wav" -o -iname "~~hpr${ep_num}_summary.wav" -o -iname "*_tmp.log" -o -iname "silence.wav" -o -iname "hpr${ep_num}_norm.wav" \) | wc -l )" -ne 0 ]] +then + echo "Files for this episode already exist." + find "${media_dir}" \( -iname "hpr${ep_num}.wav" -o -iname "hpr${ep_num}.mp3" -o -iname "hpr${ep_num}.ogg" -o -iname "hpr${ep_num}.spx" -o -iname "hpr${ep_num}_summary.txt" -o -iname "hpr${ep_num}_summary.wav" -o -iname "hpr${ep_num}.wav" -o -iname "*_mez.wav" -o -iname "*_sox_norm.wav" -o -iname "*_mezzanine.wav" -o -iname "*_tmp.pcm" -o -iname "hpr${ep_num}_intro.wav" -o -iname "~hpr${ep_num}_summary.wav" -o -iname "~~hpr${ep_num}_summary.wav" -o -iname "*_tmp.log" -o -iname "silence.wav" -o -iname "hpr${ep_num}_norm.wav" \) + read -p "Shall I remove them ? (N|y) ? " -n 1 -r + echo # (optional) move to a new line + if [[ ! $REPLY =~ ^[yY]$ ]] + then + echo "skipping...." + exit + else + echo "Removing old files ...." + find "${media_dir}" \( -iname "hpr${ep_num}.wav" -o -iname "hpr${ep_num}.mp3" -o -iname "hpr${ep_num}.ogg" -o -iname "hpr${ep_num}.spx" -o -iname "hpr${ep_num}_summary.txt" -o -iname "hpr${ep_num}_summary.wav" -o -iname "hpr${ep_num}.wav" -o -iname "*_mez.wav" -o -iname "*_sox_norm.wav" -o -iname "*_mezzanine.wav" -o -iname "*_tmp.pcm" -o -iname "hpr${ep_num}_intro.wav" -o -iname "~hpr${ep_num}_summary.wav" -o -iname "~~hpr${ep_num}_summary.wav" -o -iname "*_tmp.log" -o -iname "silence.wav" -o -iname "hpr${ep_num}_norm.wav" \) -delete -print + fi +fi + + +##################################################### +# Process media + +media=$( find "${media_dir}" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' ) +if [ -z "${media}" ] +then + echo "ERROR: Can't find any media in \"${media_dir}/\"" + find "${media_dir}/" -type f + exit 1 +fi + +mediafile="" + +echo "Found more than one media file. Please select which one to use ?" +if [ "$( echo "${media}" | wc -l )" -ne 1 ] +then + select this_media in $( echo "${media}" ) + do + echo "INFO: You selected \"${this_media}\"." + ls -al "${this_media}" + mediafile="${this_media}" + break + done +else + echo "INFO: Selecting media as \"${media}\"." + mediafile="${media}" +fi + +# extract file name and extension +fname="$( basename "${mediafile%.*}" )" +ext="${mediafile/*./}" + +cd "${media_dir}/" +pwd + +echo "INFO: Processing hpr${ep_num} from ${email}" +echo "INFO: Working directory is \"${media_dir}/\"" +echo "" + + +if [ -z "${mediafile}" ] +then + echo "sorry, variable \"mediafile\" is not set." + exit +fi + +if [ -z "${fname}" ] +then + echo "sorry, variable \"fname\" is not set." + exit +fi + +if [ -z "${ext}" ] +then + echo "sorry, variable \"ext\" is not set." + exit +fi + +if [ ! -f "${mediafile}" ] +then + echo "sorry, media file \"${mediafile}\" does not exist" + exit +fi + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "File information for \"${mediafile}\"" | tee -a ${fname}_tmp.log +ffprobe ${mediafile} 2>&1 | grep Audio: +mediainfo ${mediafile} +audio2image.bash ${mediafile} +xdg-open ${mediafile%.*}.png >/dev/null 2>&1 & +unset REPLY +until [[ $REPLY =~ ^[yYnN]$ ]] +do + read -p "Source Waveform look ok ? (N|y) ? " -n 1 -r + echo "" +done + +if [[ ! $REPLY =~ ^[yY]$ ]] +then + echo "skipping.... $REPLY" + exit +fi + + +re='^[0-9]+$' +if ! [[ $ep_num =~ $re ]] +then + echo "error: episode \"${ep_num}\" is not a number" + exit 1 +fi + +if [ ! -f "${outro}" ] +then + echo "sorry, file \"${outro}\" does not exist" + exit 1 +fi + +if [ ! -f "${theme}" ] +then + echo "sorry, file \"${theme}\" does not exist" + exit 1 +fi + +if [ ! -r "${mediafile}" ] +then + echo "sorry, media file \"${mediafile}\" is not readable" + exit +fi + +if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ] +then + echo "sorry, media file \"${mediafile}\" has no audio track" + exit +fi + +xdg-open "https://hackerpublicradio.org/eps/hpr${ep_num}/index.html" >/dev/null 2>&1 & + +mpv -vo=null "${mediafile}" + +unset REPLY +until [[ $REPLY =~ ^[yYnN]$ ]] +do + read -p "Is the audio ok (n|Y) ? " -n 1 -r + echo "" +done + +if [[ ! $REPLY =~ ^[yY]$ ]] +then + echo "skipping.... $REPLY" + exit +fi + +echo "--------------------------------------------------------------------------------" +echo "Geting metadata for hpr${ep_num}" + +while read -r line +do + field=$(echo $line | awk -F ':' '{print $1}') + case $field in + "HPR_summary") + HPR_summary=$(echo $line | grep "HPR_summary: " | cut -c 14- ) + continue + ;; + "HPR_album") + HPR_album=$(echo $line | grep "HPR_album: " | cut -c 12- ) + continue + ;; + "HPR_artist") + HPR_artist=$(echo $line | grep "HPR_artist: " | cut -c 13- ) + continue + ;; + "HPR_comment") + HPR_comment=$(echo $line | grep "HPR_comment: " | cut -c 14- ) + continue + ;; + "HPR_genre") + HPR_genre=$(echo $line | grep "HPR_genre: " | cut -c 12- ) + continue + ;; + "HPR_title") + HPR_title=$(echo $line | grep "HPR_title: " | cut -c 12- ) + continue + ;; + "HPR_track") + HPR_track=$(echo $line | grep "HPR_track: " | cut -c 12- ) + continue + ;; + "HPR_year") + HPR_year=$(echo $line | grep "HPR_year: " | cut -c 11- ) + continue + ;; + "HPR_duration") + HPR_duration=$(echo $line | grep "HPR_duration: " | cut -c 15- ) + continue + ;; + "HPR_explicit") + HPR_explicit=$(echo $line | grep "HPR_explicit: " | cut -c 15- ) + continue + ;; + "HPR_license") + HPR_license=$(echo $line | grep "HPR_license: " | cut -c 14- ) + continue + ;; + esac +done < <( curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num}" ) + +if [[ -z "$HPR_album" || -z "$HPR_artist" || -z "$HPR_comment" || -z "$HPR_genre" || -z "$HPR_title" || -z "$HPR_track" || -z "$HPR_year" || -z "$HPR_summary" || -z "$HPR_duration" || -z "$HPR_explicit" || -z "$HPR_license" ]] +then + echo "Could not find information on ${ep_num}. Has the show been posted ?" + exit; +fi +echo "--------------------------------------------------------------------------------" +echo "album : $HPR_album" +echo "artist : $HPR_artist" +echo "comment : $HPR_comment" +echo "genre : $HPR_genre" +echo "title : $HPR_title" +echo "track : $HPR_track" +echo "year : $HPR_year" +echo "summary : $HPR_summary" +echo "duration : $HPR_duration" +echo "explicit : $HPR_explicit" +echo "license : $HPR_license" +echo "media_dir : ${media_dir}" +echo "mediafile : ${mediafile}" +echo "fname : ${fname}" +echo "ext : ${ext}" + +if [[ $HPR_duration == "0" ]] +then + echo "The duration is set to 0. Please update the show with the correct time." + exit; +fi + +#============================================================ +# Preproc`s the source file + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Prepare mezzanine file" | tee -a ${fname}_tmp.log +ffmpeg -y -i ${mediafile} -ar 44100 -ac $CHANNELS ${fname}_mezzanine.wav > ${fname}_tmp.log 2>&1 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Add HPR Branding" | tee -a ${fname}_tmp.log + +echo "Creating the summary" | tee -a ${fname}_tmp.log +#echo "$HPR_summary" - | text2wave -F 44100 - -o hpr${ep_num}_summary.wav #festival --tts +#echo "$HPR_summary" - | espeak -w ~hpr${ep_num}_summary.wav + +echo "$HPR_summary" > "hpr${ep_num}_summary.txt" + +create_tts_summary_ok="not-ok" + +while [ "${create_tts_summary_ok}" != "OK" ] +do + create_tts_summary + + xdg-open "hpr${ep_num}_summary.txt" 2>&1 & + mpv --speed=1.8 ~hpr${ep_num}_summary.wav + + read -p "Is the text to speech correct (y|N) ? " -n 1 -r + echo # (optional) move to a new line + if [[ $REPLY =~ ^[Yy]$ ]] + then + create_tts_summary_ok="OK" + else + echo "WARN: Please correct the text and try again." + inotifywait --event modify "hpr${ep_num}_summary.txt" + fi +done +echo "INFO: TTS complete." + +ffmpeg -y -i ~hpr${ep_num}_summary.wav -ar 44100 ~~hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1 + +sox -V2 -n -r 44100 -c 1 silence.wav trim 0.0 6.0 >> ${fname}_tmp.log 2>&1 + +sox -V2 silence.wav ~~hpr${ep_num}_summary.wav hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1 + +echo "Adding the theme" | tee -a ${fname}_tmp.log +sox -V2 -m "hpr${ep_num}_summary.wav" "${theme}" "hpr${ep_num}_intro.wav" >> ${fname}_tmp.log 2>&1 + +echo "Creating the sandwitch: \"hpr${ep_num}_intro.wav\" \"${fname}_mezzanine.wav\" \"${outro}\" \"hpr${ep_num}.wav\" " | tee -a ${fname}_tmp.log + +sox -V2 "hpr${ep_num}_intro.wav" "${fname}_mezzanine.wav" "${outro}" "hpr${ep_num}.wav" >> ${fname}_tmp.log 2>&1 + +echo "Normalizing the wav files" | tee -a ${fname}_tmp.log +#sox --temp "${TEMP_DIR}" --norm hpr${ep_num}.wav hpr${ep_num}_norm.wav +ffmpeg -y -i hpr${ep_num}.wav -af loudnorm=I=-16:LRA=11:TP=-1.5 hpr${ep_num}_norm.wav >> ${fname}_tmp.log 2>&1 + +mv -v hpr${ep_num}_norm.wav ${upload_dir}/hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "File information" | tee -a ${fname}_tmp.log +ffprobe ${upload_dir}/hpr${ep_num}.wav 2>&1 | grep Audio: +mediainfo ${upload_dir}/hpr${ep_num}.wav +audio2image.bash ${upload_dir}/hpr${ep_num}.wav +xdg-open ${upload_dir}/hpr${ep_num}.png >/dev/null 2>&1 & +read -p "Processed Waveform look ok ? (N|y) ? " -n 1 -r +echo # (optional) move to a new line +if [[ ! $REPLY =~ ^[yY]$ ]] +then + echo "skipping...." + exit +fi +rm -v ${upload_dir}/hpr${ep_num}.png + + +echo "--------------------------------------------------------------------------------" +echo "Geting transcript of hpr${ep_num}" + +whisper --model tiny --language en --output_dir "${media_dir}" "${upload_dir}/hpr${ep_num}.wav" | tee -a ${fname}_tmp.log + +ls -al "${media_dir}/hpr${ep_num}".* +xdg-open "${media_dir}/hpr${ep_num}".txt 2>&1 & +echo mpv --no-audio-display --audio-channels=stereo --speed="3.5" "${media_dir}/hpr${ep_num}".txt +unset REPLY +until [[ $REPLY =~ ^[yYnN]$ ]] +do + read -p "Processed transcript look ok ? (N|y) ? " -n 1 -r + echo "" +done + +if [[ ! $REPLY =~ ^[yY]$ ]] +then + echo "skipping.... $REPLY" + exit +fi + +mkdir "${upload_dir}/hpr${ep_num}" >/dev/null 2>&1 + +cp -v "${media_dir}/hpr${ep_num}".vtt "${upload_dir}/hpr${ep_num}/hpr${ep_num}.vtt" | tee -a ${fname}_tmp.log +cp -v "${media_dir}/hpr${ep_num}".srt "${upload_dir}/hpr${ep_num}/hpr${ep_num}.srt" | tee -a ${fname}_tmp.log +cp -v "${media_dir}/hpr${ep_num}".txt "${upload_dir}/hpr${ep_num}/hpr${ep_num}.txt" | tee -a ${fname}_tmp.log +cp -v "${media_dir}/hpr${ep_num}".tsv "${upload_dir}/hpr${ep_num}/hpr${ep_num}.tsv" | tee -a ${fname}_tmp.log +cp -v "${media_dir}/hpr${ep_num}".json "${upload_dir}/hpr${ep_num}/hpr${ep_num}.json" | tee -a ${fname}_tmp.log + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Convert to opus" | tee -a ${fname}_tmp.log +ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.opus 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Convert to flac" | tee -a ${fname}_tmp.log +ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.flac 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Convert to mp3" | tee -a ${fname}_tmp.log +ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.mp3 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Convert to ogg" | tee -a ${fname}_tmp.log +#ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.ogg 2>> ${fname}_tmp.log 1>&2 +ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav -acodec libopus -f ogg ${upload_dir}/hpr${ep_num}.ogg 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Convert to spx" | tee -a ${fname}_tmp.log +ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.spx 2>> ${fname}_tmp.log 1>&2 + +### End conversion + +intro_duration="$( mediainfo --Output=XML --Full "hpr${ep_num}_intro.wav" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" +outro_duration="$( mediainfo --Output=XML --Full "${outro}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" +source_duration="$( mediainfo --Output=XML --Full "${mediafile}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" +expected_duration=$(( ${intro_duration} + ${HPR_duration} + ${outro_duration} )) + +echo "Intro(${intro_duration}) + Show(${HPR_duration}) + Outro(${outro_duration}) = ${expected_duration}" + +media_error="0" + +for check_file in \ +${upload_dir}/hpr${ep_num}.wav \ +${upload_dir}/hpr${ep_num}.opus \ +${upload_dir}/hpr${ep_num}.flac \ +${upload_dir}/hpr${ep_num}.mp3 \ +${upload_dir}/hpr${ep_num}.spx \ +${upload_dir}/hpr${ep_num}.ogg +do + # ${upload_dir}/hpr${ep_num}.spx + echo "INFO: Processing the file \"${check_file}\"" + if [[ ! -s "${check_file}" ]] + then + echo "ERROR: Something went wrong encoding of the file \"${check_file}\"" + exit + else + mediainfo --Output=XML --Full "${check_file}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' + this_duration=$( mediainfo --Output=XML --Full "${check_file}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' ) + if [[ $(abs_diff "${this_duration}" "${expected_duration}" ) -le "${acceptable_duration_difference}" ]] + then + echo "INFO: The file \"${check_file}\" duration of ${this_duration} () is close enough to ${expected_duration}" + else + echo "ERROR: The file \"${check_file}\" actual duration of ${this_duration} is not close enough to posted duration of ${expected_duration}." + echo " Fix or update the posted duration to ${source_duration}." + media_error="1" + fi + #${expected_duration} + fi +done +echo "Source: ${source_duration}" + +if [[ "${media_error}" -eq "1" ]] +then + echo "ERROR: Media is not encoded correctly" + exit +else + echo "INFO: Media duration is correct" +fi + + +if [[ ! -s ${upload_dir}/hpr${ep_num}.wav ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.opus ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.flac ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.mp3 ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.ogg ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.spx ]] +then + echo "ERROR: Something went wrong encoding the files" + exit 1 +fi + + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Fixing Tags" | tee -a ${fname}_tmp.log + +fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="${HPR_comment} The license is ${HPR_license}" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" ${upload_dir}/hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2 + +fix_tags ${upload_dir}/hpr${ep_num}* + +#echo "Changing the file dates to the time of upload" +touch -r ${mediafile} hpr${ep_num}* +touch -r ${mediafile} ${upload_dir}/hpr${ep_num}* + +ls -al hpr${ep_num}* ${upload_dir}/hpr${ep_num}* | tee -a ${fname}_tmp.log + +# # echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +# # echo "Getting info for the asset table" | tee -a ${fname}_tmp.log +# # +# # echo "INSERT INTO assets (episode_id,filename,extension,size,sha1sum,mime_type,file_type) VALUES" +# # +# # for asset_file in \ +# # ${upload_dir}/hpr${ep_num}.wav \ +# # ${upload_dir}/hpr${ep_num}.opus \ +# # ${upload_dir}/hpr${ep_num}.flac \ +# # ${upload_dir}/hpr${ep_num}.mp3 \ +# # ${upload_dir}/hpr${ep_num}.spx \ +# # ${upload_dir}/hpr${ep_num}.ogg +# # do +# # size="$( ls -al "${asset_file}" | awk '{print $5}' )" +# # sha1sum="$( sha1sum "${asset_file}" | awk '{print $1}' )" +# # mime_type=$( file --dereference --brief --mime "${asset_file}" ) +# # file_type=$( file --dereference --brief "${asset_file}" ) +# # echo "(${ep_num},'${filename}','${extension}','${size}','${sha1sum}','${mime_type}','${file_type}')," +# # done + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Transferring files to Servers" | tee -a ${fname}_tmp.log + + +#rsync -ave ssh --partial --progress --ignore-existing ${upload_dir}/hpr${ep_num}.mp3 ${upload_dir}/hpr${ep_num}.ogg ${upload_dir}/hpr${ep_num}.spx hpr:www/eps/ + +#firefox "https://hackerpublicradio.org/local/hpr${ep_num}.mp3" >/dev/null 2>&1 & +#firefox "https://hackerpublicradio.org/local/hpr${ep_num}.ogg" >/dev/null 2>&1 & +firefox "file://${upload_dir}/hpr${ep_num}.mp3" >/dev/null 2>&1 & +firefox "file://${upload_dir}/hpr${ep_num}.ogg" >/dev/null 2>&1 & +#firefox "https://hackerpublicradio.org/eps.php?id=${ep_num}" >/dev/null 2>&1 & + +#mpv "http://hackerpublicradio.org/local/hpr${ep_num}.spx" "http://hackerpublicradio.org/local/hpr${ep_num}.ogg" "http://hackerpublicradio.org/local/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.spx" "file://${upload_dir}/hpr${ep_num}.opus" "file://${upload_dir}/hpr${ep_num}.ogg" "file://${upload_dir}/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.flac" +mpv "file://${upload_dir}/hpr${ep_num}.spx" "file://${upload_dir}/hpr${ep_num}.opus" "file://${upload_dir}/hpr${ep_num}.ogg" "file://${upload_dir}/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.flac" + +read -p "Remove files for \"${fname}\" (y|N) ? " -n 1 -r +echo # (optional) move to a new line +if [[ $REPLY =~ ^[Yy]$ ]] +then + mediafilename=$(basename "${mediafile}") + mediaextension="${mediafilename##*.}" # " +# ssh hpr -t "mkdir /home/hpr/www/eps/hpr${ep_num}" >/dev/null 2>&1 +# rsync -ave ssh --partial --progress --ignore-existing "${mediafile}" hpr:www/eps/hpr${ep_num}/hpr${ep_num}_source.${mediaextension} | tee -a ${fname}_tmp.log +# +# rsync -ave ssh --partial --progress --ignore-existing "${media_dir}/hpr${ep_num}.wav".vtt hpr:www/eps/hpr${ep_num}/hpr${ep_num}.vtt | tee -a ${fname}_tmp.log +# rsync -ave ssh --partial --progress --ignore-existing "${media_dir}/hpr${ep_num}.wav".srt hpr:www/eps/hpr${ep_num}/hpr${ep_num}.srt | tee -a ${fname}_tmp.log +# rsync -ave ssh --partial --progress --ignore-existing "${media_dir}/hpr${ep_num}.wav".txt hpr:www/eps/hpr${ep_num}/hpr${ep_num}.txt | tee -a ${fname}_tmp.log +# +# ssh hpr -t "ls -al /home/hpr/www/eps/hpr${ep_num}*" +# cp -v "${mediafile}" "${upload_dir}/hpr${ep_num}_source.${mediaextension}" + + #echo "Remove temp files" + rm -v ~hpr${ep_num}_summary.wav ~~hpr${ep_num}_summary.wav silence.wav + rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav + #mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/ + +# wget --timeout=0 -q "http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1" -O - | xmlstarlet val --err - +# wget --timeout=0 -q "http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1" -O - | xmlstarlet val --err - +# wget --timeout=0 -q "http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1" -O - | xmlstarlet val --err - +# wget --timeout=0 -q "http://hackerpublicradio.org/rss-future.php" -O - | xmlstarlet val --err - + + echo "INFO: rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* borg:/data/IA/uploads/" + rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* borg:/data/IA/uploads/ + echo "$( ssh borg -t "ls -al /data/IA/uploads/hpr${ep_num}*" ; ls -al ${upload_dir}/hpr${ep_num}* )" | grep "hpr${ep_num}" | awk '{print $5, $NF}' | sort + + if [ ${show_type} == "remote" ] + then + echo "INFO: Setting the status" + # SHOW_SUBMITTED → METADATA_PROCESSED → SHOW_POSTED → MEDIA_TRANSCODED → UPLOADED_TO_IA → UPLOADED_TO_RSYNC_NET + curl --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php?ep_num=${ep_num}&status=MEDIA_TRANSCODED" + curl --silent --netrc-file ${HOME}/.netrc "https://hub.hackerpublicradio.org/cms/status.php" + fi +else + echo "skipping...." + echo "cp -v \"${mediafile}\" \"${upload_dir}/hpr${ep_num}_source.${mediaextension}\"" + echo "rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav" + #echo "mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/" + echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1\" -O - | xmlstarlet val --err -" + echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1\" -O - | xmlstarlet val --err -" + echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1\" -O - | xmlstarlet val --err -" + echo "rsync -ave ssh --partial --progress ${upload_dir}/ borg:/data/IA/uploads/" +fi diff --git a/workflow/hprtranscode-simple.bash.2022-07-31 b/workflow/hprtranscode-simple.bash.2022-07-31 new file mode 100644 index 0000000..eada88e --- /dev/null +++ b/workflow/hprtranscode-simple.bash.2022-07-31 @@ -0,0 +1,397 @@ +#!/usr/bin/env bash +# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ +#============================================================ +TEMP_DIR="/var/tmp/" +CHANNELS="1" +FIXAUDIO="1" +ARTIST="EMPTY" +TITLE="EMPTY" +YEAR="EMPTY" +SLOT="EMPTY" +basedir="/var/IA" +upload_dir="${basedir}/uploads" +theme="${basedir}/theme.wav" +outro="${basedir}/2022-03-07-outro.wav" +ttsserver="http://localhost:5500" +processing_dir="/home/ken/tmp/hpr/processing" +git_image_dir="/home/ken/sourcecode/hpr/HPR_Public_Code/www/images/hosts" +thedate=$(/usr/bin/date -u +%Y-%m-%d_%H-%M-%SZ_%A) +echo "Processing the next HPR Show in the queue" + + +if [ $( curl -s -o /dev/null -w "%{http_code}" -X 'GET' "${ttsserver}/api/voices" -H 'accept: */*' ) != "200" ] +then + echo "Please start the tts-server \"podman run -it -p 5500:5500 synesthesiam/opentts:en\"" + exit +fi + +################### +# Get the show +# +# +if [[ -f "${1}" && -n "${2}" ]] +then + mediafile="${1}" + ep_num="${2}" + echo "The duration is \"$( \date -ud "1970-01-01 $( ffprobe -i "${mediafile}" 2>&1| awk -F ': |, ' '/Duration:/ { print $2 }' )" +%s )\"." +else + response=$( curl --silent --netrc-file ${HOME}/.netrc "https://hackerpublicradio.org/cms/status.php" | \ + grep 'SHOW_POSTED' | \ + head -1 | \ + sed 's/,/ /g' ) + + if [ -z "${response}" ] + then + echo "INFO: There appear to be no more shows with the status \"SHOW_POSTED\"." + echo "Getting a list of all the reservations." + curl --silent --netrc-file ${HOME}/.netrc "https://hackerpublicradio.org/cms/status.php" + exit 3 + fi + + timestamp_epoc="$( echo ${response} | awk '{print $1}' )" + ep_num="$( echo ${response} | awk '{print $2}' )" + ep_date="$( echo ${response} | awk '{print $3}' )" + key="$( echo ${response} | awk '{print $4}' )" + status="$( echo ${response} | awk '{print $5}' )" + email="$( echo ${response} | awk '{print $6}' )" + + #source_dir="hpr:/home/hpr/upload/${timestamp_epoc}_${ep_num}_${ep_date}_${key}" + dest_dir="${timestamp_epoc}_${ep_num}_${ep_date}_${key}" + echo detox -v "${processing_dir}/${dest_dir}/" + detox -vr "${processing_dir}/${dest_dir}/" + cd "${processing_dir}/${dest_dir}/" + pwd + + echo "INFO: Processing hpr${ep_num} from ${email}" + echo "INFO: Working directory is \"${processing_dir}/${dest_dir}/\"" + echo "" + + media=$( find "${processing_dir}/${timestamp_epoc}_${ep_num}_${ep_date}_${key}" -type f -exec file {} \; | grep -Ei 'audio|mpeg|video|MP4' | awk -F ': ' '{print $1}' ) + if [ -z "${media}" ] + then + echo "ERROR: Can't find any media in \"${processing_dir}/${dest_dir}/\"" + find "${processing_dir}/${dest_dir}/" -type f + exit 1 + fi + + mediafile="" + + if [ "$( echo "${media}" | wc -l )" -ne 1 ] + then + select this_media in $( echo "${media}" ) + do + echo "INFO: You selected \"${this_media}\"." + ls -al "${this_media}" + mediafile="${this_media}" + break + done + else + echo "INFO: Selecting media as \"${media}\"." + mediafile="${media}" + fi + echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log + echo "File information for \"${mediafile}\"" | tee -a ${fname}_tmp.log + ffprobe ${mediafile} 2>&1 | grep Audio: + mediainfo ${mediafile} + audio2image.bash ${mediafile} + xdg-open ${mediafile%.*}.png >/dev/null 2>&1 & + read -p "Source Waveform look ok ? (N|y) ? " -n 1 -r + echo # (optional) move to a new line + if [[ ! $REPLY =~ ^[yY]$ ]] + then + echo "skipping...." + exit + fi + +fi + +re='^[0-9]+$' +if ! [[ $ep_num =~ $re ]] +then + echo "error: episode \"${ep_num}\" is not a number" + exit 1 +fi + +if [ ! -f "${outro}" ] +then + echo "sorry, file \"${outro}\" does not exist" + exit 1 +fi + +if [ ! -f "${theme}" ] +then + echo "sorry, file \"${theme}\" does not exist" + exit 1 +fi + +if [ ! -f "${mediafile}" ] +then + echo "sorry, media file \"${mediafile}\" does not exist" + exit +fi + +if [ ! -r "${mediafile}" ] +then + echo "sorry, media file \"${mediafile}\" is not readable" + exit +fi + +if [ $(ffprobe "${mediafile}" 2>&1 | grep "Audio:" | wc -l ) -eq 0 ] +then + echo "sorry, media file \"${mediafile}\" has no audio track" + exit +fi + +mpv -vo=null "${mediafile}" +read -p "Is there any problems with the audio (N|y) ? " -n 1 -r +echo # (optional) move to a new line +if [[ $REPLY =~ ^[yY]$ ]] +then + echo "skipping...." + exit 1 +fi + +# extract file name and extension +media_dir=$(dirname ${mediafile}) +fname=${mediafile%.*} +ext=${mediafile/*./} + +if [[ -e hpr${ep_num}.wav ]] || [[ -e hpr${ep_num}.mp3 ]] || [[ -e hpr${ep_num}.ogg ]] || [[ -e hpr${ep_num}.spx ]] || [[ -e hpr${ep_num}_summary.wav ]] || [[ -e hpr${ep_num}.wav ]] || [[ -e ${fname}_mez.wav ]] || [[ -e ${fname}_sox_norm.wav ]] || [[ -e ${fname}_mezzanine.wav ]] || [[ -e ${fname}_tmp.pcm ]] || [[ -e ${fname}_tmp.log ]] || [[ -e hpr${ep_num}_intro.wav ]] || [[ -e ~hpr${ep_num}_summary.wav ]] || [[ -e ~~hpr${ep_num}_summary.wav ]] +then + echo "Files for this episode already exist." + ls -1 *hpr${ep_num}* ${fname}_* 2>/dev/null + exit 1 +fi + +echo "--------------------------------------------------------------------------------" +echo "Geting metadata for hpr${ep_num}" + +while read -r line +do + field=$(echo $line | awk -F ':' '{print $1}') + case $field in + "HPR_summary") + HPR_summary=$(echo $line | grep "HPR_summary: " | cut -c 14- ) + continue + ;; + "HPR_album") + HPR_album=$(echo $line | grep "HPR_album: " | cut -c 12- ) + continue + ;; + "HPR_artist") + HPR_artist=$(echo $line | grep "HPR_artist: " | cut -c 13- ) + continue + ;; + "HPR_comment") + HPR_comment=$(echo $line | grep "HPR_comment: " | cut -c 14- ) + continue + ;; + "HPR_genre") + HPR_genre=$(echo $line | grep "HPR_genre: " | cut -c 12- ) + continue + ;; + "HPR_title") + HPR_title=$(echo $line | grep "HPR_title: " | cut -c 12- ) + continue + ;; + "HPR_track") + HPR_track=$(echo $line | grep "HPR_track: " | cut -c 12- ) + continue + ;; + "HPR_year") + HPR_year=$(echo $line | grep "HPR_year: " | cut -c 11- ) + continue + ;; + "HPR_duration") + HPR_duration=$(echo $line | grep "HPR_duration: " | cut -c 15- ) + continue + ;; + "HPR_explicit") + HPR_explicit=$(echo $line | grep "HPR_explicit: " | cut -c 15- ) + continue + ;; + "HPR_license") + HPR_license=$(echo $line | grep "HPR_license: " | cut -c 14- ) + continue + ;; + esac +done < <( wget --timeout=10 --tries=1 --quiet http://hackerpublicradio.org/say.php?id=${ep_num} -O - ) + +if [[ -z "$HPR_album" || -z "$HPR_artist" || -z "$HPR_comment" || -z "$HPR_genre" || -z "$HPR_title" || -z "$HPR_track" || -z "$HPR_year" || -z "$HPR_summary" || -z "$HPR_duration" || -z "$HPR_explicit" || -z "$HPR_license" ]] +then + echo "Could not find information on ${ep_num}. Has the show been posted ?" + exit; +fi +echo "--------------------------------------------------------------------------------" +echo "album : $HPR_album" +echo "artist : $HPR_artist" +echo "comment : $HPR_comment" +echo "genre : $HPR_genre" +echo "title : $HPR_title" +echo "track : $HPR_track" +echo "year : $HPR_year" +echo "summary : $HPR_summary" +echo "duration : $HPR_duration" +echo "explicit : $HPR_explicit" +echo "license : $HPR_license" + +if [[ $HPR_duration == "0" ]] +then + echo "The duration is set to 0. Please update the show with the correct time." + exit; +fi + +#============================================================ +# Preproc`s the source file + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Prepare mezzanine file" | tee -a ${fname}_tmp.log +ffmpeg -i ${mediafile} -ar 44100 -ac $CHANNELS ${fname}_mezzanine.wav > ${fname}_tmp.log 2>&1 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Add HPR Branding" | tee -a ${fname}_tmp.log + +echo "Creating the summary" | tee -a ${fname}_tmp.log +#echo "$HPR_summary" - | text2wave -F 44100 - -o hpr${ep_num}_summary.wav #festival --tts +#echo "$HPR_summary" - | espeak -w ~hpr${ep_num}_summary.wav +echo "$HPR_summary" - | curl -X 'GET' -G --data-urlencode "voice=coqui-tts:en_ljspeech" --data-urlencode "text=${HPR_summary}" --data-urlencode "vocoder=high" --data-urlencode "denoiserStrength=0.03" --data-urlencode "cache=false" ${ttsserver}/api/tts -H 'accept: */*' --output ~hpr${ep_num}_summary.wav + +ffmpeg -i ~hpr${ep_num}_summary.wav -ar 44100 ~~hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1 + +sox -n -r 44100 -c 1 silence.wav trim 0.0 6.0 >> ${fname}_tmp.log 2>&1 + +sox silence.wav ~~hpr${ep_num}_summary.wav hpr${ep_num}_summary.wav >> ${fname}_tmp.log 2>&1 + +echo "Adding the theme" | tee -a ${fname}_tmp.log +sox -m "hpr${ep_num}_summary.wav" "${theme}" "hpr${ep_num}_intro.wav" >> ${fname}_tmp.log 2>&1 + +echo "Creating the sandwitch" | tee -a ${fname}_tmp.log + +sox "hpr${ep_num}_intro.wav" "${fname}_mezzanine.wav" "${outro}" "hpr${ep_num}.wav" >> ${fname}_tmp.log 2>&1 + +echo "Normalizing the wav files" | tee -a ${fname}_tmp.log +#sox --temp "${TEMP_DIR}" --norm hpr${ep_num}.wav hpr${ep_num}_norm.wav +ffmpeg -i hpr${ep_num}.wav -af loudnorm=I=-16:LRA=11:TP=-1.5 hpr${ep_num}_norm.wav >> ${fname}_tmp.log 2>&1 + +mv -v hpr${ep_num}_norm.wav ${upload_dir}/hpr${ep_num}.wav >> ${fname}_tmp.log 2>&1 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "File information" | tee -a ${fname}_tmp.log +ffprobe ${upload_dir}/hpr${ep_num}.wav 2>&1 | grep Audio: +mediainfo ${upload_dir}/hpr${ep_num}.wav +audio2image.bash ${upload_dir}/hpr${ep_num}.wav +xdg-open ${upload_dir}/hpr${ep_num}.png >/dev/null 2>&1 & +read -p "Processed Waveform look ok ? (N|y) ? " -n 1 -r +echo # (optional) move to a new line +if [[ ! $REPLY =~ ^[yY]$ ]] +then + echo "skipping...." + exit +fi +rm -v ${upload_dir}/hpr${ep_num}.png + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Convert to opus" | tee -a ${fname}_tmp.log +ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.opus 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Convert to flac" | tee -a ${fname}_tmp.log +sox --temp "${TEMP_DIR}" -S ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.flac 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Convert to mp3" | tee -a ${fname}_tmp.log +sox --temp "${TEMP_DIR}" -S ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.mp3 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Convert to ogg" | tee -a ${fname}_tmp.log +ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.ogg 2>> ${fname}_tmp.log 1>&2 + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Convert to spx" | tee -a ${fname}_tmp.log +ffmpeg -y -i ${upload_dir}/hpr${ep_num}.wav ${upload_dir}/hpr${ep_num}.spx 2>> ${fname}_tmp.log 1>&2 + +for check_file in \ +${upload_dir}/hpr${ep_num}.wav \ +${upload_dir}/hpr${ep_num}.opus \ +${upload_dir}/hpr${ep_num}.flac \ +${upload_dir}/hpr${ep_num}.mp3 \ +${upload_dir}/hpr${ep_num}.ogg \ +${upload_dir}/hpr${ep_num}.spx +do + if [[ ! -s "${check_file}" ]] + then + echo "ERROR: Something went wrong encoding of the file \"${check_file}\"" + fi +done + +if [[ ! -s ${upload_dir}/hpr${ep_num}.wav ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.opus ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.flac ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.mp3 ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.ogg ]] || [[ ! -s ${upload_dir}/hpr${ep_num}.spx ]] +then + echo "ERROR: Something went wrong encoding the files" + exit 1 +fi + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Fixing Tags" | tee -a ${fname}_tmp.log + +fix_tags -album="$HPR_album" -artist="$HPR_artist" -comment="${HPR_comment} The license is ${HPR_license}" -genre="$HPR_genre" -title="$HPR_title" -track="$HPR_track" -year="$HPR_year" ${upload_dir}/hpr${ep_num}* 2>> ${fname}_tmp.log 1>&2 + +fix_tags ${upload_dir}/hpr${ep_num}* + +#echo "Changing the file dates to the time of upload" +touch -r ${mediafile} hpr${ep_num}* +touch -r ${mediafile} ${upload_dir}/hpr${ep_num}* + +ls -al hpr${ep_num}* ${upload_dir}/hpr${ep_num}* | tee -a ${fname}_tmp.log + + +echo "--------------------------------------------------------------------------------" | tee -a ${fname}_tmp.log +echo "Transferring files to Servers" | tee -a ${fname}_tmp.log + + +rsync -ave ssh --partial --progress --ignore-existing ${upload_dir}/hpr${ep_num}.mp3 ${upload_dir}/hpr${ep_num}.ogg ${upload_dir}/hpr${ep_num}.spx hpr:www/eps/ + +firefox "https://hackerpublicradio.org/local/hpr${ep_num}.mp3" >/dev/null 2>&1 & +firefox "https://hackerpublicradio.org/local/hpr${ep_num}.ogg" >/dev/null 2>&1 & +firefox "file://${upload_dir}/hpr${ep_num}.mp3" >/dev/null 2>&1 & +firefox "file://${upload_dir}/hpr${ep_num}.ogg" >/dev/null 2>&1 & +firefox "https://hackerpublicradio.org/eps.php?id=${ep_num}" >/dev/null 2>&1 & + +mpv "http://hackerpublicradio.org/local/hpr${ep_num}.spx" "http://hackerpublicradio.org/local/hpr${ep_num}.ogg" "http://hackerpublicradio.org/local/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.spx" "file://${upload_dir}/hpr${ep_num}.opus" "file://${upload_dir}/hpr${ep_num}.ogg" "file://${upload_dir}/hpr${ep_num}.mp3" "file://${upload_dir}/hpr${ep_num}.flac" + +echo "Source: $( mediainfo --Output=XML --Full "${mediafile}" | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' )" +mediainfo --Output=XML --Full hpr${ep_num}* ${upload_dir}/hpr${ep_num}* | xmlstarlet sel -T -t -m '/_:MediaInfo/_:media/_:track[@type="Audio"]' -v '_:Duration' -n | awk -F '.' '{print $1}' | sort | uniq -c + +read -p "Remove files for \"${fname}\" (y|N) ? " -n 1 -r +echo # (optional) move to a new line +if [[ $REPLY =~ ^[Yy]$ ]] +then + mediafilename=$(basename "${mediafile}") + mediaextension="${mediafilename##*.}" # " + ssh hpr -t "mkdir /home/hpr/www/eps/hpr${ep_num}" >/dev/null 2>&1 + rsync -ave ssh --partial --progress --ignore-existing "${mediafile}" hpr:www/eps/hpr${ep_num}/hpr${ep_num}_source.${mediaextension} + ssh hpr -t "ls -al /home/hpr/www/eps/hpr${ep_num}*" + cp -v "${mediafile}" "${upload_dir}/hpr${ep_num}_source.${mediaextension}" + + #echo "Remove temp files" + rm -v ~hpr${ep_num}_summary.wav ~~hpr${ep_num}_summary.wav silence.wav + rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav + #mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/ + wget --timeout=0 -q "http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1" -O - | xmlstarlet val --err - + wget --timeout=0 -q "http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1" -O - | xmlstarlet val --err - + wget --timeout=0 -q "http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1" -O - | xmlstarlet val --err - + wget --timeout=0 -q "http://hackerpublicradio.org/rss-future.php" -O - | xmlstarlet val --err - + + rsync -ave ssh --partial --progress ${upload_dir}/hpr${ep_num}* borg:/data/IA/uploads/ + echo "$( ssh borg -t "ls -al /data/IA/uploads/hpr${ep_num}*" ; ls -al ${upload_dir}/hpr${ep_num}* )" | grep "hpr${ep_num}" | awk '{print $5, $NF}' | sort + # SHOW_SUBMITTED → METADATA_PROCESSED → SHOW_POSTED → MEDIA_TRANSCODED → UPLOADED_TO_IA → UPLOADED_TO_RSYNC_NET + curl --netrc-file ${HOME}/.netrc "https://hackerpublicradio.org/cms/status.php?ep_num=${ep_num}&status=MEDIA_TRANSCODED" +else + echo "skipping...." + echo "cp -v \"${mediafile}\" \"${upload_dir}/hpr${ep_num}_source.${mediaextension}\"" + echo "rm -v ${fname}_mezzanine.wav ${fname}_tmp*.pcm ${fname}_tmp.log ${fname}_mez.wav" + #echo "mv -v ${fname}* hpr${ep_num}* *_${ep_num}_* /var/IA/done/" + echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_ogg_rss.php?gomax=1\" -O - | xmlstarlet val --err -" + echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_mp3_rss.php?gomax=1\" -O - | xmlstarlet val --err -" + echo "wget --timeout=0 -q \"http://hackerpublicradio.org/hpr_spx_rss.php?gomax=1\" -O - | xmlstarlet val --err -" + echo "rsync -ave ssh --partial --progress ${upload_dir}/ borg:/data/IA/uploads/" +fi diff --git a/workflow/rss-2_0.xsd b/workflow/rss-2_0.xsd new file mode 100644 index 0000000..b36ff26 --- /dev/null +++ b/workflow/rss-2_0.xsd @@ -0,0 +1,501 @@ + + + + + XML Schema for RSS v2.0 feed files. + Project home: http://www.codeplex.com/rss2schema/ + Based on the RSS 2.0 specification document at http://cyber.law.harvard.edu/rss/rss.html + Author: Jorgen Thelin + Revision: 16 + Date: 01-Nov-2008 + Feedback to: http://www.codeplex.com/rss2schema/WorkItem/List.aspx + + + + + + + + + + + + + + An item may represent a "story" -- much like a story in a newspaper or magazine; if so its description is a synopsis of the story, and the link points to the full story. An item may also be complete in itself, if so, the description contains the text (entity-encoded HTML is allowed), and the link and title may be omitted. + + + + + + The title of the item. + + + + + The item synopsis. + + + + + The URL of the item. + + + + + Email address of the author of the item. + + + + + Includes the item in one or more categories. + + + + + URL of a page for comments relating to the item. + + + + + Describes a media object that is attached to the item. + + + + + guid or permalink URL for this entry + + + + + Indicates when the item was published. + + + + + The RSS channel that the item came from. + + + + + Extensibility element. + + + + + + + + + + + + The name of the channel. It's how people refer to your service. If you have an HTML website that contains the same information as your RSS file, the title of your channel should be the same as the title of your website. + + + + + The URL to the HTML website corresponding to the channel. + + + + + Phrase or sentence describing the channel. + + + + + The language the channel is written in. This allows aggregators to group all Italian language sites, for example, on a single page. A list of allowable values for this element, as provided by Netscape, is here. You may also use values defined by the W3C. + + + + + Copyright notice for content in the channel. + + + + + Email address for person responsible for editorial content. + + + + + Email address for person responsible for technical issues relating to channel. + + + + + The publication date for the content in the channel. All date-times in RSS conform to the Date and Time Specification of RFC 822, with the exception that the year may be expressed with two characters or four characters (four preferred). + + + + + The last time the content of the channel changed. + + + + + Specify one or more categories that the channel belongs to. + + + + + A string indicating the program used to generate the channel. + + + + + A URL that points to the documentation for the format used in the RSS file. It's probably a pointer to this page. It's for people who might stumble across an RSS file on a Web server 25 years from now and wonder what it is. + + + + + Allows processes to register with a cloud to be notified of updates to the channel, implementing a lightweight publish-subscribe protocol for RSS feeds. + + + + + ttl stands for time to live. It's a number of minutes that indicates how long a channel can be cached before refreshing from the source. + + + + + Specifies a GIF, JPEG or PNG image that can be displayed with the channel. + + + + + The PICS rating for the channel. + + + + + Specifies a text input box that can be displayed with the channel. + + + + + A hint for aggregators telling them which hours they can skip. + + + + + A hint for aggregators telling them which days they can skip. + + + + + Extensibility element. + + + + + + + + + Extensibility element. + + + + + + + + A time in GMT when aggregators should not request the channel data. The hour beginning at midnight is hour zero. + + + + + + + + + + + + + + A day when aggregators should not request the channel data. + + + + + + + + + + + + + + + + A time in GMT, when aggregators should not request the channel data. The hour beginning at midnight is hour zero. + + + + + + + + + + + + + + + + The URL of the image file. + + + + + Describes the image, it's used in the ALT attribute of the HTML <img> tag when the channel is rendered in HTML. + + + + + The URL of the site, when the channel is rendered, the image is a link to the site. (Note, in practice the image <title> and <link> should have the same value as the channel's <title> and <link>. + + + + + The width of the image in pixels. + + + + + The height of the image in pixels. + + + + + Text that is included in the TITLE attribute of the link formed around the image in the HTML rendering. + + + + + + + The height of the image in pixels. + + + + + + + + The width of the image in pixels. + + + + + + + + Specifies a web service that supports the rssCloud interface which can be implemented in HTTP-POST, XML-RPC or SOAP 1.1. Its purpose is to allow processes to register with a cloud to be notified of updates to the channel, implementing a lightweight publish-subscribe protocol for RSS feeds. + + + + + + + + + + + + + + + + + The purpose of this element is something of a mystery! You can use it to specify a search engine box. Or to allow a reader to provide feedback. Most aggregators ignore it. + + + + + The label of the Submit button in the text input area. + + + + + Explains the text input area. + + + + + The name of the text object in the text input area. + + + + + The URL of the CGI script that processes text input requests. + + + + + + + Using the regexp definiton of E-Mail Address by Lucadean from the .NET RegExp Pattern Repository at http://www.3leaf.com/default/NetRegExpRepository.aspx + + + + + + + + + A date-time displayed in RFC-822 format. + Using the regexp definiton of rfc-822 date by Sam Ruby at http://www.intertwingly.net/blog/1360.html + + + + + + + + + + + + + + + + + + URL where the enclosure is located + + + + + Size in bytes + + + + + MIME media-type of the enclosure + + + + + + + + + + + + + + + + + + diff --git a/workflow/show2youtube.bash b/workflow/show2youtube.bash new file mode 100755 index 0000000..f72c08d --- /dev/null +++ b/workflow/show2youtube.bash @@ -0,0 +1,60 @@ +#!/bin/bash +# http://eddmann.com/posts/uploading-podcast-audio-to-youtube/ +# http://cutycapt.sourceforge.net/ xvfb-run +# https://el-tramo.be/blog/ken-burns-ffmpeg/ + +#hpr${ep_num}.wav + +ep_num=2463 + +xvfb-run --server-args="-screen 0, 1024x768x24" CutyCapt --url="http://hackerpublicradio.org/eps.php?id=${ep_num}" --out=${ep_num}.png --insecure + +ffmpeg -i input.wav -filter_complex "[0:a]showwaves=s=800x600:mode=line:rate=25,format=yuv420p[v]" -map "[v]" -map 0:a output-showwaves.mp4 + + +# ffmpeg -start_number n -i test_%d.jpg -vcodec mpeg4 test.avi +# ffmpeg -loop 1 -r 2 -i image.jpg -i input.mp3 -vf scale=-1:380 -c:v libx264 -preset slow -tune stillimage -crf 18 -c:a copy -shortest -pix_fmt yuv420p -threads 0 output.mkv + +# https://video.stackexchange.com/questions/9644/how-do-i-turn-audio-into-video-that-is-show-the-waveforms-in-a-video + +#this +# +# rm out.mp4 output.mp4 ; ffmpeg -i 2463.png -filter_complex "pad=w=9600:h=6000:x='(ow-iw)/2':y='(oh-ih)/2',zoompan=x='(iw-0.625*ih)/2':y='(1-on/(25*4))*(ih-ih/zoom)':z='if(eq(on,1),2.56,zoom+0.002)':d=25*4:s=1280x800" -pix_fmt yuv420p -c:v libx264 out.mp4 +# +# rm out.mp4 output.mp4; ffmpeg -i 2463.png -filter_complex "pad=w=9600:h=6000:x='(ow-iw)/2':y='(oh-ih)/2',zoompan=z='zoom+0':d=25*4:s=1280x2048,crop=w=1280:h=800:x='(iw-ow)/2':y='(ih-oh)/2' " -pix_fmt yuv420p -c:v libx264 out.mp4 +# +# ffmpeg -i in.jpg +# -filter_complex +# "zoompan=z='zoom+0.002':d=25*4:s=1280x800" +# -pix_fmt yuv420p -c:v libx264 out.mp4 +# +# +# +# ffmpeg -i input.flac -filter_complex "[0:a]ahistogram,format=yuv420p[v]" -map "[v]" -map 0:a output.mp4 +# +# +# ffmpeg -i input -i background.png -filter_complex "[0:a]showwavespic=s=640x240[fg];[1:v][top]overlay=format=auto" -frames:v 1 output.png + + +ffmpeg -loop 1 -i background.png -i video1.mp4 -i video2.mp4 -filter_complex \ +"[1:v]scale=(iw/2)-20:-1[a]; \ + [2:v]scale=(iw/2)-20:-1[b]; \ + [0:v][a]overlay=10:(main_h/2)-(overlay_h/2):shortest=1[c]; \ + [c][b]overlay=main_w-overlay_w-10:(main_h/2)-(overlay_h/2)[video]" \ +-map "[video]" output.mkv + + +https://stackoverflow.com/questions/13390714/superimposing-two-videos-onto-a-static-image +https://video.stackexchange.com/questions/14519/add-image-under-the-video-with-ffmepg + + +ffmpeg \ + -loop 1 -i hprback.png \ + -i output-showwaves-320x240.mp4 \ + -filter_complex "overlay=0:0:shortest=1" \ + out.m4v + + +# ffmpeg -i input.wav -filter_complex "[0:a]showfreqs=s=320x266:mode=line:fscale=log,format=yuv420p[v]" -map "[v]" -map 0:a output-showfreqs-319x266.mp4 +# ffmpeg -loop 1 -i hprback.png -i output-showfreqs-319x266.mp4 -filter_complex "overlay=663:143:shortest=1" out1.m4v + diff --git a/workflow/status_lighttower.py b/workflow/status_lighttower.py new file mode 100644 index 0000000..9ac21e3 --- /dev/null +++ b/workflow/status_lighttower.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2017 Ken Fallon ken@fallon.ie +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# UTF-8 Test >> ÇirçösÚáóíéőöÓÁśł << +# 213.46.252.136 gateway + +import sys, getopt, json, os, logging, sys, requests, datetime, re, dateutil.parser, base64, xmltodict, random, psycopg2, time, automationhat + +from pprint import pprint +#import traceback +logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') +#logging.disable(logging.DEBUG) +settings = dict() + +def usage(exitcode, message = None): + ''' + prints usage and exits + ''' + if message is not None: + logging.error( message ) + print( os.path.basename(sys.argv[0]) + ' [--help -l -c ]') + sys.exit(exitcode) + +def get_lab_settings(settings): + ''' + Get configuration for the lab, usually from http://172.30.218.244/autotest/settings.json + ''' + if os.path.isfile( settings[ 'config_url' ] ): + with open( settings[ 'config_url' ] ) as json_file: + info_dict = json.load(json_file) + return info_dict + else: + session = requests.session() + resp = session.get(url=settings[ 'config_url' ], headers={'Accept-Encoding': 'gzip'}) + if resp.status_code == 200: + info_dict = json.loads(resp.text) + return info_dict + return None + +def argumentTest(): + ''' + Gets the required paramaters that the program needs in order to run. These can either be provided on the command line, or as environemental variables, or both. + + export LAB="lab4b" + export config_url="http://172.30.218.244/autotest/settings.json" + export service="purcha#!/usr/bin/env python + +import automationhat +import time +import os +import sys + +counter = -1 +lightPattern = [ + [1,0,0], + [0,1,0], + [0,0,1], + [0,1,1], + [0,1,0], + [1,1,0], + [1,0,0], + [1,1,0], + [0,1,0], + [0,1,1], + [0,0,1], + [0,0,0], + [1,0,1], + [0,1,0], + [0,0,0], +] + +def increment(): + global counter + counter+=1 + if counter==len(lightPattern): + counter = 0 + +def doCurrentCycle(): + currentCycle = lightPattern[counter] + print currentCycle + if (currentCycle[0]==1) : + automationhat.output.one.on() + else: + automationhat.output.one.off() + + if (currentCycle[1] == 1): + automationhat.output.two.on() + else: + automationhat.output.two.off() + + if (currentCycle[2] == 1): + automationhat.output.three.on() + else: + automationhat.output.three.off() + + time.sleep(2) + + + +while True: + increment() + doCurrentCycle()se-service" + export customers="customers.json" + + ''' + lab = str() + config_url = str() + service = str() + settings = dict() + + for evariable in [ 'LAB', 'config_url' ]: + if os.getenv( evariable ) is not None: + settings[ evariable ] = os.environ[ evariable ] + logging.debug( "Setting \"" + evariable + "\" set to \"" + settings[ evariable ] + "\" from an enviroment variable.") + + try: + options, remainder = getopt.getopt(sys.argv[1:], 'hl:c:', [ 'help', 'lab=', 'config=' ]) + + except getopt.GetoptError: + usage(1, "Unrecognized option was found or missing argument." ) + + for opt, arg in options: + if opt in ('-h', '--help'): + usage(2) + elif opt in ('-l', '--lab'): + logging.debug( "Setting \"LAB\" set to \"" + arg + "\" from the \"" + opt + "\" command line argument.") + settings[ 'LAB' ] = arg + elif opt in ('-c', '--config'): + logging.debug( "Setting \"config_url\" set to \"" + arg + "\" from the \"" + opt + "\" command line argument.") + settings[ 'config_url' ] = arg + + for param in [ 'LAB', 'config_url' ]: + if not param in settings: + usage( 3, "Cant find value for \"" + param +"\"") + + try: + settings[ 'config' ] = get_lab_settings( settings )[ settings[ 'LAB' ] ] + if 'jenkins' in settings.keys(): + logging.debug('Updating TraxIS Customer file to include the Purchase Service status.') + + except Exception as e: + print('Error: Could not get Lab Configuration for "%s".' % settings[ 'LAB' ] ) + traceback.print_exc() + exit(1) + + return settings + + +#def get_server_instance(): + #jenkins_url = 'http://172.22.137.160:8080' + #server = Jenkins(jenkins_url, username='autotest', password='autotest') + #return server + +def get_server_instance(settings): + jenkins_url = 'http://%s:%s' % ( settings[ 'config' ][ 'jenkins' ][ 'host' ] , settings[ 'config' ][ 'jenkins' ][ 'port' ] ) + server = Jenkins(jenkins_url, username = settings[ 'config' ][ 'jenkins' ][ 'user' ], password = settings[ 'config' ][ 'jenkins' ][ 'password' ]) + return server + +def alloff(): + automationhat.relay.one.off() + automationhat.output.one.off() + automationhat.output.two.off() + automationhat.output.three.off() + +# Set according to status +def setstatus(status): + if status != "": + alloff() + if status == "SUCCESS": + automationhat.output.three.on() + print( "SUCCESS" ) + + if status == "UNSTABLE": + automationhat.output.two.on() + print( "UNSTABLE" ) + + if status == "FAILURE": + automationhat.output.one.on() + print( "FAILURE" ) + return + +def get_job_details( settings ): + # Refer Example #1 for definition of function 'get_server_instance' + server = get_server_instance( settings ) + settings[ 'teststatus' ] = str() + for job in settings[ 'config' ][ 'jenkins' ][ 'jobs' ]: + logging.debug( "Checking job %s" % job ) + job_instance = server.get_job( job ) + this_status = job_instance.get_last_build().get_status() + if not ( settings[ 'teststatus' ] == "FAILURE" or (settings[ 'teststatus' ] == "UNSTABLE" and this_status == "SUCCESS") ): + settings[ 'teststatus' ] = this_status + return settings + +if __name__ == "__main__": + logging.debug( "start" ) + try: + while True: + alloff() + automationhat.relay.one.on() + settings = argumentTest() + #pprint( settings[ 'config' ][ 'jenkins' ] ) + #print( "Hello" ) + #pprint ( get_server_instance( settings ).version ) + settings = get_job_details( settings ) + status = settings[ 'teststatus' ] + setstatus(status) + #print( get_server_instance().version ) + time.sleep(300) + + except Exception as e: + print("Error: " + str(e)) + traceback.print_exc() + exit(1) + logging.debug('done') + +''' +export LAB="lab4b" +export config_url="http://172.30.218.244/autotest/settings.json" +''' diff --git a/workflow/this_duration.bash b/workflow/this_duration.bash new file mode 100755 index 0000000..5245ac2 --- /dev/null +++ b/workflow/this_duration.bash @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ +find ${pwd} -type f | while read mediafile +do + duration=$( mediainfo --full --Output=XML "${mediafile}" | xmlstarlet sel -T -t -m "_:MediaInfo/_:media/_:track[@type='Audio']/_:Duration[1]" -v "." -n - | awk -F '.' '{print $1}' ) + if [ "${duration}" != "" ] + then + echo "${duration} ${mediafile}" + continue + fi + duration=$( /bin/date -ud "1970-01-01 $( ffprobe -i "${mediafile}" 2>&1| awk -F ': |, ' '/Duration:/ { print $2 }' )" +%s ) + if [ "${duration}" != 0 ] + then + echo "${duration} ${mediafile}" + continue + fi +done +find -type f -exec file {} \; | grep -vEi 'audio|mpeg|video' diff --git a/workflow/transfer_tags b/workflow/transfer_tags new file mode 100755 index 0000000..f8cb95a --- /dev/null +++ b/workflow/transfer_tags @@ -0,0 +1,638 @@ +#!/usr/bin/perl +#=============================================================================== +# +# FILE: transfer_tags +# +# USAGE: ./transfer_tags masterfile +# +# DESCRIPTION: Transfer ID3 (or equivalent) tags from a base file to +# whichever of FLAC, MP3, OGG, SPX and WAV versions are found. +# +# OPTIONS: --- +# REQUIREMENTS: --- +# BUGS: --- +# NOTES: --- +# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com +# VERSION: 1.4.2 +# CREATED: 31/03/2013 14:18:55 +# REVISION: 24/05/2013 15:54:36 +# +#=============================================================================== + +use 5.010; +use strict; +use warnings; + +use File::Basename; +use File::Find::Rule; +use Audio::TagLib; + +# +# Version number (manually incremented) +# +our $VERSION = '1.4.2'; + +# +# Declarations +# +my ( $ref, $tag, $tags, %mastertags, %slavetags, $changed ); +my ( $directories, $filename, $suffix ); + +# +# The extensions the various codings are expected to have (it looks both for +# the lower- and upper-case versions, but only the lower-case ones are +# required here) +# +my @exts = qw{ flac mp3 ogg spx wav }; + +# +# The additional name variants we'll accept +# +my @variants = qw{ _mez }; + +# +# Used to test tag conformity. The key is the tagname, the value is a hashref +# containing a regex (key 're') for detecting conformity and the correct value +# (key 'ok'). +# To be passed to subroutine 'checkConformity', but kept here for ease of +# maintenance. +# +my %tag_control = ( + album => { re => qr{^Hacker Public Radio$}, ok => 'Hacker Public Radio' }, + comment => { + re => qr{^http://hackerpublicradio\.org/?}, + ok => 'http://hackerpublicradio.org' + }, + genre => { re => qr{(?i)^Podcast$}, ok => 'Podcast' }, +); + +# +# The Audio::TagLib methods to call for each tag manipulated by the script. +# The number after the method name is 1 if the value being set is a string, +# and zero otherwise. +# +my %tagmethods = ( + album => [ 'setAlbum', 1 ], + artist => [ 'setArtist', 1 ], + comment => [ 'setComment', 1 ], + genre => [ 'setGenre', 1 ], + title => [ 'setTitle', 1 ], + track => [ 'setTrack', 0 ], + year => [ 'setYear', 0 ], +); + +# +# Because Audio::TagLib::FileRef does not seem to commit the tag update until +# very late (during the DESTROY?) it's very difficult to update file times +# _after_ the tags have been written. The solution is to save all of the tag +# hashes (which contain the file path and times) and process then in the END{} +# block. Dirty, but very Perl-ish. +# +my @tag_stash; + +# +# Script name +# +( my $PROG = $0 ) =~ s|.*/||mx; + +# +# Ensure STDOUT is in UTF8 mode +# +binmode( STDOUT, ":encoding(utf8)" ); + +# +# Get the argument, the "master" file +# +my $masterfile = shift; +die "Usage: $PROG masterfilename\n" unless $masterfile; + +# +# Check the file exists +# +die "$masterfile does not exist\n" unless ( -e $masterfile ); + +# +# Assume there are other versions of the same file with suffixes in the set +# { flac, mp3, ogg, spx, wav } so build these names from the master file name. +# Start by parsing the filename into its directories, filename and suffix. +# Remove the leading dot from the suffix. +# +($filename,$directories,$suffix) = fileparse($masterfile,qr/\.[^.]*/); +$suffix =~ s/^\.//; + +# +# Reject the file if it doesn't have an expected suffix (we're not +# case-sensitive here) +# +die "$masterfile does not have a recognised suffix (expecting " + . join( ", ", @exts ) . ")\n" + unless ( grep {/$suffix/i} @exts ); + +# +# Use File::Find::Rule to find all the files that match a regular expression. +# This is built from the parsed file, with an optional list of variants and +# all of the extensions (case insensitively). We then remove the master file +# from the list and we're done. +# +my $re + = "^${filename}(" + . join( '|', @variants ) . ")?\.(?i:" + . join( '|', @exts ) . ')$'; +my @files = grep { !/^$masterfile$/ } + File::Find::Rule->file()->name(qr{$re})->in($directories); + +# +# Log the file +# +print "$masterfile\n"; + +# +# Collect the tags from the master file (& stash them for later too) +# +( $ref, $tag, $tags ) = collectTags($masterfile); +%mastertags = %$tags; +push( @tag_stash, $tags ); + +# +# Report the tags found in the master file +# +reportTags( \%mastertags ); + +# +# Check that the master file conforms to the HPR standards, ensuring that any +# changes are returned from the routine +# +( $ref, $tag, $tags ) = checkConformity( $ref, $tag, $tags, \%tag_control ); +%mastertags = %$tags; +print '=' x 80, "\n"; + +# +# Now process the "slave" files +# +foreach my $file (@files) { + # + # Check the "slave" file exists and report it if so + # + if ( -r $file ) { + print "$file\n"; + } + else { + warn "$file is not readable\n"; + next; + } + + # + # Get the "slave" file's tags (& keep a copy in the stash) + # + ( $ref, $tag, $tags ) = collectTags($file); + %slavetags = %$tags; + push( @tag_stash, $tags ); + + # + # Report the tags + # + reportTags( \%slavetags ); + + # + # Change the tags to match the "master" file's tags + # + $changed = 0; + for my $t ( sort( grep { !/^_/ } keys(%mastertags) ) ) { + $changed += changeTag( $tag, $t, $slavetags{$t}, $mastertags{$t}, + @{ $tagmethods{$t} } ); + } + print '-' x 80, "\n"; + + # + # Save any changes + # + if ($changed) { + $ref->save(); + } +} + +exit; + +#------------------------------------------------------------------------------- +# Post-processing of file times +# +# Process all files we've visited (whether their tags were changed or not) +# and force the 'atime' and 'mtime' back to their starting values. This will +# be done after the rest of the script runs, when we know that all of the +# Audio::TagLib::FileRef objects have been destroyed and have done their lazy +# updates. +#------------------------------------------------------------------------------- +END { + for my $t (@tag_stash) { + warn "Time restoration failed on $t->{_path}\n" + unless restoreTimes($t); + } +} + +#=== FUNCTION ================================================================ +# NAME: collectTags +# PURPOSE: Collects tags from a media file +# PARAMETERS: $filepath Path to the file +# RETURNS: A list containing an Audio::TagLib::FileRef object, an +# Audio::TagLib::Tag object (containing the actual tags) and +# a hashref containing the converted tag values (along with +# a few other attributes). +# DESCRIPTION: Collects the tags and the timestamps from the file. Then the +# various tags and other attributes are placed in a hash which +# will be returned to the caller. The non-tag keys begin with +# '_' to differentiate them. +# THROWS: No exceptions +# COMMENTS: None +# SEE ALSO: +#=============================================================================== +sub collectTags { + my ($filepath) = @_; + + my ( $atime, $mtime ) = ( stat($filepath) )[ 8, 9 ]; + + my $fileref = Audio::TagLib::FileRef->new($filepath); + my $ftag = $fileref->tag(); + + my %tags = ( + album => $ftag->album()->toCString(), + artist => $ftag->artist()->toCString(), + comment => $ftag->comment()->toCString(), + genre => $ftag->genre()->toCString(), + title => $ftag->title()->toCString(), + track => $ftag->track(), + year => $ftag->year(), + _path => $filepath, + _atime => $atime, + _mtime => $mtime, + ); + + return ( $fileref, $ftag, \%tags ); +} + +#=== FUNCTION ================================================================ +# NAME: reportTags +# PURPOSE: Print the tags in a hash +# PARAMETERS: $tags Hashref keyed by tagname and containing tag +# contents from a media file +# RETURNS: Nothing +# DESCRIPTION: Just prints all the "proper" tags held in the hash argument in +# alphabetical order of the keys. Note that the "secret" keys, +# those begining with '_', are skipped. See 'collectTags' for +# what they are. +# THROWS: No exceptions +# COMMENTS: None +# SEE ALSO: +#=============================================================================== +sub reportTags { + my ($tags) = @_; + + my @keys = sort( grep { !/^_/ } keys(%$tags) ); + + for my $key (@keys) { + printf "%-10s: %s\n", $key, $tags->{$key}; + } + + return; +} + +#=== FUNCTION ================================================================ +# NAME: changeTag +# PURPOSE: Changes a tag to a new value if appropriate +# PARAMETERS: $tag Audio::TagLib::Tag object +# $tagname Name of tag +# $oldValue Current value of tag +# $newValue New value of tag or undefined +# $setFunc String containing the name of the 'set' +# function +# $isString True if the value being set is a string +# RETURNS: 1 if a change has been made, 0 otherwise +# DESCRIPTION: Performs some argument checks, returning on a missing new +# value, or if the old and new values are the same. The old and +# new values may be encoded integers, so we look for this +# eventuality. After all of this we know there's a change to be +# made and perform the appropriate steps to make it. +# THROWS: No exceptions +# COMMENTS: None +# SEE ALSO: +#=============================================================================== +sub changeTag { + my ( $tag, $tagname, $oldValue, $newValue, $setFunc, $isString ) = @_; + + return 0 unless defined($newValue); + return 0 if $oldValue eq $newValue; + + $isString = 0 unless defined($isString); + + if ( !$isString ) { + return 0 if int($oldValue) == int($newValue); + } + + print ">> Changing $tagname to '$newValue'\n"; + $tag->$setFunc( + ( $isString + ? Audio::TagLib::String->new($newValue) + : $newValue + ) + ); + return 1; +} + +#=== FUNCTION ================================================================ +# NAME: restoreTimes +# PURPOSE: Restore the original times to a file which has had its tags +# changed +# PARAMETERS: $tags Hashref keyed by tagname and containing tag +# contents (and file attributes) from a media +# file. The file details have keys beginning +# with '_'. +# RETURNS: Number of files changed (see 'utime') +# DESCRIPTION: Uses the Perl 'utime' function to change the file's access +# time and modification time to whatever is in the hash. These +# are expected to be the times the file had when it was first +# encountered. +# THROWS: No exceptions +# COMMENTS: None +# SEE ALSO: +#=============================================================================== +sub restoreTimes { + my ($tags) = @_; + + return utime( $tags->{_atime}, $tags->{_mtime}, $tags->{_path} ); +} + +#=== FUNCTION ================================================================ +# NAME: checkConformity +# PURPOSE: Check that the master file has conforming tags, fixing them if +# not +# PARAMETERS: $ref Audio::TagLib::FileRef relating to the master +# file +# $tag Audio::TagLib::Tag containing the tags of the +# master file +# $tags Hashref containing the converted tags (and +# a few other odds and sods) +# $kosher Hashref containing the checking values (see +# %tag_control in the main program) +# RETURNS: A list containing $ref, $tag and $tags as described above +# DESCRIPTION: Implements a number of complex rules. Firstly the 'genre' tag +# is expected to contain 'Podcast'. Secondly the 'album' tag +# must contain 'Hacker Public Radio'. If it does not then the +# value is stored for later then replaced. Finally the +# 'comment' tag must begin with 'http://hackerpublicradio.org'. +# If it does not its current contents are stored and replaced +# with the required URL. However, the comment tag will also +# contain the saved album tag (if any) and the saved comment, +# and these will be placed at the end. +# THROWS: No exceptions +# COMMENTS: This code is ugly and difficult to extend and maintain. +# TODO look into ways of improving it! +# SEE ALSO: +#=============================================================================== +sub checkConformity { + my ( $ref, $tag, $tags, $kosher ) = @_; + + my $changed = 0; + my %saved; + my ( $t, $commentOK, $newval ); + + # + # The 'genre' tag + # + $t = 'genre'; + unless ( $tags->{$t} =~ /$kosher->{$t}->{re}/ ) { + $changed += changeTag( + $tag, $t, $tags->{$t}, + $kosher->{$t}->{ok}, + @{ $tagmethods{$t} } + ); + $tags->{genre} = $tag->genre()->toCString(); + } + + # + # The 'album' tag. We save this one for adding to the comment + # + $t = 'album'; + unless ( $tags->{$t} =~ /$kosher->{$t}->{re}/ ) { + ( $saved{$t} = $tags->{$t} ) =~ s/(^\s+|\s+$)//g; + $changed += changeTag( + $tag, $t, $tags->{$t}, + $kosher->{$t}->{ok}, + @{ $tagmethods{$t} } + ); + $tags->{album} = $tag->album()->toCString(); + } + + # + # If the 'comment' is non-standard *or* if the 'album' was changed we want + # to do stuff here. We make sure the 'comment' is good and append the + # original 'album' and 'comment' as appropriate. + # + $t = 'comment'; + $commentOK = $tags->{$t} =~ /$kosher->{$t}->{re}/; + unless ( !$changed && $commentOK ) { + ( $saved{$t} = $tags->{$t} ) =~ s/(^\s+|\s+$)//g; + + if ($changed) { + if ($commentOK) { + # Album had errors, comment is OK + $newval = concat( ", ", $saved{comment}, $saved{album} ); + } + else { + # Album had errors, comment also + $newval = concat( ", ", $kosher->{$t}->{ok}, + $saved{album}, $saved{comment} ); + } + } + else { + # Comment had errors, album OK + $newval = concat( ", ", $kosher->{$t}->{ok}, $saved{comment} ); + } + + $changed += changeTag( $tag, $t, $tags->{$t}, + $newval, @{ $tagmethods{$t} } ); + $tags->{comment} = $tag->comment()->toCString(); + } + + # + # Save any changes + # + if ($changed) { + $ref->save(); + } + + # + # Return tag-related stuff so the caller can get the benefit + # + return ( $ref, $tag, $tags ); + +} + +#=== FUNCTION ================================================================ +# NAME: concat +# PURPOSE: Reimplementation of join but with any undefined or empty +# arguments removed +# PARAMETERS: $sep The string to be used to separate elements in +# the result +# [variable args] Any number of arguments to be joined together +# with the separator +# RETURNS: The concatenated arguments +# DESCRIPTION: Giving 'join' an array that may contain undefined elements will +# result in empty results in the output string and error +# messages as the undefined elements are processed. Giving it +# empty string elements will result in dangling separators in +# the output. This routine removes the undefined and empty +# elements before joining the rest. +# THROWS: No exceptions +# COMMENTS: None +# SEE ALSO: +#=============================================================================== +sub concat { + my $sep = shift; + + my @args = grep { defined($_) && length($_) > 0 } @_; + + return join( $sep, @args ); +} + +__END__ + +#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +# Application Documentation +#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +#{{{ + +=head1 NAME + +transfer_tags - standardise and transfer tags between HPR audio files + +=head1 VERSION + +This documentation refers to I version 1.4.2 + + +=head1 USAGE + + transfer_tags masterfile + + +=head1 REQUIRED ARGUMENTS + +=over 4 + +=item B + +This is the name of the audio file, which contains the definitive tags which +are to be copied to all of the other files of the same name but different +extensions. + +=back + + +=head1 DESCRIPTION + +The script transfers ID3 (or equivalent) tags from a base file to whichever of +FLAC, MP3, OGG, SPX and WAV versions are found. The tags copied are: B, +B, B, B, B, B<track> and B<year>. The target +files are determined by taking the name of the B<master file> without its +extension and appending all of the remaining extensions in the list. Files +with the string "B<_mez>" between the filename and the extension are also +included. + +For example: if the B<master file> is called B<hpr1234.flac> and FLAC, MP3, +OGG, SPX and WAV versions exist, the tags found in the file B<hpr1234.flac> +are copied to B<hpr1234.mp3>, B<hpr1234.ogg>, B<hpr1234.spx> and +B<hpr1234.wav>. If B<hpr1234_mez.mp3> or any other variant existed it would +also receive a copy of the tags. + +A certain amount of manipulation is performed before the tags are propagated. +The changes made conform to certain rules, which are: + +=over 4 + +=item . + +The B<genre> tag must contain the string "I<Podcast>". + +=item . + +The B<album> tag must contain the string "I<Hacker Public Radio>". If it does +not then the existing value is stored for later and is then replaced. + +=item . + +The B<comment> tag must begin with the string +"I<http://hackerpublicradio.org>". If it does not its current contents are +stored and replaced with the required URL. However, the comment tag will also +contain the saved album tag (if any) and the saved comment (if any), and these +will be placed at the end, separated by commas. + +=back + +The script saves the access time and modification time of all of the media +files it processes. It then restores these times at the end of its run. This +prevents any external processes which depend on these file times from being +confused by the tag changes. + +=head1 DIAGNOSTICS + +=over 4 + +=item B<Usage: transfer_tags masterfile> + +This error is produced if the script is called without the mandatory argument. +The error is fatal. + +=item B<... does not exist> + +The master file specified as the argument does not exist. The error is fatal. + +=item B<... does not have a recognised suffix (expecting ...)> + +The master file specified as the argument does not have one of the expected +extensions (flac, mp3, ogg, spx, wav). The error is fatal. + +=item B<... is not readable> + +One of the target files was found not to be readable (probably due to file +permissions). The script will ignore this file. + +=item B<Time restoration failed on ...> + +The master file or one of the target files could not have its time restored. +The script will ignore this file. + +=back + + +=head1 DEPENDENCIES + + Audio::TagLib + File::Basename + File::Find::Rule + + +=head1 BUGS AND LIMITATIONS + +There are no known bugs in this module. +Please report problems to Dave Morriss (Dave.Morriss@gmail.com). +Patches are welcome. + + +=head1 AUTHOR + +Dave Morriss (Dave.Morriss@gmail.com) 2013 + + +=head1 LICENCE AND COPYRIGHT + +Copyright (c) 2013 Dave Morriss (Dave.Morriss@gmail.com). All rights reserved. + +This program is free software; you can redistribute it and/or +modify it under the same terms as Perl itself. See perldoc perlartistic. + +=cut + +#}}} + +# [zo to open fold, zc to close] + +# vim: syntax=perl:ts=8:sw=4:et:ai:tw=78:fo=tcrqn21:fdm=marker diff --git a/workflow/unpack.bash b/workflow/unpack.bash new file mode 100644 index 0000000..6cfeaa3 --- /dev/null +++ b/workflow/unpack.bash @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright Ken Fallon - Released into the public domain. http://creativecommons.org/publicdomain/ +intro_clip_end="00:01:30" +quality_clip_duration="90" +outro_clip="90" +#DEBUG="echo " +cd /mnt/DUMP/hpr-for-archive.org/todo/ +for FILEX in * +do + if [[ "$(file "$FILEX" | grep -i audio | wc -l )" -ne 1 && "$(mediainfo "$FILEX" | grep -i audio | wc -l )" -ne 1 ]] + then + echo "$FILEX is not an audio file" + else + echo "Processing $FILEX" + fname=${FILEX%.*} + ext=${FILEX/*./} + # We get the druation from mediainfo in miliseconds and remove the last three digits to convert it to seconds + # Then calculate the times of the end clip working back from the end of the file + # We then pick a random segment between the intro and outro. + # Finally we convert to hour minute second format and the date command is the easiest way to do that + duration=$( mediainfo --full "${fname}.${ext}" | grep Duration | egrep -v ".*:.*:|ms|s|mn" | head -1 | awk -F ': ' '{print $2}' ) + duration="${duration:0:${#duration}-3}" + outro_clip_start="$(($duration-$outro_clip))" + quality_clip_end="$(($outro_clip_start-$quality_clip_duration))" + quality_clip_start=$(shuf -i ${intro_clip_end}-${quality_clip_end} -n 1) + quality_clip_end="$(($quality_clip_start+$quality_clip_duration))" + duration=$(\date -d@${duration} -u +%H:%M:%S) + quality_clip_start=$(\date -d@${quality_clip_start} -u +%H:%M:%S) + quality_clip_end=$(\date -d@${quality_clip_end} -u +%H:%M:%S) + outro_clip_start=$(\date -d@${outro_clip_start} -u +%H:%M:%S) + echo -e "\tExtracting spectrogram" + ${DEBUG} sox "${fname}.${ext}" -n spectrogram -Y 130 -l -r -t "${fname}.${ext}" -o "${fname}_spectrogram.png" + echo -e "\tExtracting intro clip from 0 to ${intro_clip_end}" + ${DEBUG} ffmpeg -y -ss 0 -t ${intro_clip_end} -i "${fname}.${ext}" "${fname}_intro_clip".wav 2>/dev/null + echo -e "\tExtracting quality clip from ${quality_clip_start} to ${quality_clip_end}" + ${DEBUG} ffmpeg -y -ss ${quality_clip_start} -t ${quality_clip_end} -i "${fname}.${ext}" "${fname}_quality_clip".wav 2>/dev/null + echo -e "\tExtracting outro clip from ${outro_clip_start} ${duration}" + ${DEBUG} ffmpeg -y -ss ${outro_clip_start} -t ${duration} -i "${fname}.${ext}" "${fname}_outro_clip".wav 2>/dev/null + echo -e "\tCreating low fidelity version" + ${DEBUG} ffmpeg -y -i "${fname}.${ext}" temp.wav 2>/dev/null + ${DEBUG} sox "temp.wav" -c 1 -r 16000 -t wav - 2>/dev/null | speexenc - "${fname}_low_fidelity.spx" 2>/dev/null + ${DEBUG} rm temp.wav + fi +done diff --git a/workflow/x.bash b/workflow/x.bash new file mode 100755 index 0000000..27abb48 --- /dev/null +++ b/workflow/x.bash @@ -0,0 +1,50 @@ +#cd /home/ken/processing/1659905303_3664_2022-08-18_037b55af6b77191b2610f5cedb5d2bd962f0251720bf0 +ttsserver="http://localhost:5500" +ep_num="3664" + +echo "This is Hacker Public Radio " > "hpr${ep_num}_summary.txt" + +## Jump to encoding + +function create_tts_summary { + HPR_summary="$( cat "hpr${ep_num}_summary.txt" )" + echo "INFO: Converting text \"${HPR_summary}\" to speech." + curl -X 'GET' -G --data-urlencode "voice=coqui-tts:en_ljspeech" --data-urlencode "text=${HPR_summary}" --data-urlencode "vocoder=high" --data-urlencode "denoiserStrength=0.03" --data-urlencode "cache=false" ${ttsserver}/api/tts -H 'accept: */*' --output ~hpr${ep_num}_summary.wav +} + +create_tts_summary_ok="not-ok" + +while [ "${create_tts_summary_ok}" != "OK" ] +do + create_tts_summary + + mpv ~hpr${ep_num}_summary.wav + + read -p "Is the text to speech correct (y|N) ? " -n 1 -r + echo # (optional) move to a new line + if [[ $REPLY =~ ^[Yy]$ ]] + then + create_tts_summary_ok="OK" + else + echo "WARN: Please correct the text and try again." + xdg-open "hpr${ep_num}_summary.txt" 2>&1 & + inotifywait --event modify "hpr${ep_num}_summary.txt" + fi +done +echo "INFO: TTS complete." +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# exit 9999