davmo
/
Free_Culture_Podcasts
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Additions to the database and feedWatcher 

feedWatcher: added the parsing of HTML feeds to get the title tag in the
    <head> area; new database fields relating to the copyright check
    done, and why the feed was allowed in if done so manually; added dry
    run mode; changed the way -load and -delete work so each can be
    given URLs on the command line; starting to report settings at start
    time (needs work); -load and -delete not allowed together; more
    logging; addition of a _debug function; enhancement of reportFeed to
    show one feed and a summary of relevant details (more useful than
    dumping the entire database this way); added getHTMLTitle for
    parsing out the HTML title; enhanced checkCopyright to get a reason
    if in manual mode and a feed is allowed in; needs a lot of
    clean-up!

feedWatcher.{html,json,mkd,opml,pdf}: various reports.

feedWatcher_3.tpl: For making Markdown which is turned into PDF.
    'Licence' becomes 'Copyright'

feedWatcher_5.tpl: for dumping all the URLs in the database
    & regenerating everything

feedWatcher_schema.sql: new fields added
This commit is contained in:
Dave Morriss 2023-01-14 23:13:49 +00:00
parent 01ec2cf92f
commit db39655199
10 changed files with 1047 additions and 393 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

513
feedWatcher
View File

@ -48,8 +48,8 @@ use feature qw{ postderef say signatures state };
no warnings qw{ experimental::postderef experimental::signatures } ;
#
# There's an issue in XML::RSS, so we're using a loocal version with a hack.
# It's in ./lib/ and FiindBin::libs looks there to find it.
# There's an issue in XML::RSS, so we're using a local version with a hack.
# It's in ./lib/ and FindBin::libs looks there to find it.
#
use FindBin::libs;
use XML::RSS;
@ -77,6 +77,7 @@ use Template::Filters;
Template::Filters->use_html_entities; # Use HTML::Entities in the template
use HTML::Entities;
use HTML::Parser ();
use IO::Prompter;
@ -102,14 +103,17 @@ our $VERSION = '0.1.2';
#
# Declarations
#
my ( @new_urls, @deletions );
my ( $action_mode, @urls, @deletions );
my ( $rules, $robot_name ) = ( undef, "$PROG/$VERSION" );
my ($search_target);
my ( $sth1, $h1, $rv );
my ($rejectcount);
my ( $sth1, $h1, $rv, $search_target, $rejectcount );
my $feeds;
#
# To be written by a handler subroutine called by HTML::Parser
#
our $html_title;
#
# File and directory paths
#
@ -127,6 +131,7 @@ my $deftemplate = "$basedir/${PROG}.tpl";
my %keymap = (
AUTHOR => 'author',
COPYRIGHT => 'copyright',
CHECKTYPE => 'check_type',
DESCRIPTION => 'description',
DNS => 'dns',
# ENCLOSURES => undef,
@ -142,6 +147,7 @@ my %keymap = (
LINK => 'link',
MODIFIED => 'modified',
# PORT => undef,
REASON_ACCEPTED => 'reason_accepted',
# ROBOTS => undef,
# SAVE => undef,
TITLE => 'title',
@ -178,7 +184,7 @@ Options( \%options );
#
# Any arguments are taken to be URLs
#
@new_urls = @ARGV;
@urls = @ARGV;
#
# Default help
@ -199,11 +205,14 @@ my $DEBUG = ( $options{'debug'} ? $options{'debug'} : 0 );
my $cfgfile
= ( defined( $options{config} ) ? $options{config} : $configfile );
my $dry_run = ( defined( $options{'dry-run'} ) ? $options{'dry-run'} : 0 );
my $silent = ( defined( $options{silent} ) ? $options{silent} : 0 );
my $loadfile = $options{'load'};
my $deletefile = $options{'delete'};
my $scan = ( defined( $options{scan} ) ? $options{scan} : 0 );
my $html = ( defined( $options{html} ) ? $options{html} : 0 );
my $check = $options{check};
my $outfile = $options{out};
@ -213,32 +222,58 @@ my $json = $options{json};
my $opml = $options{opml};
my $template = $options{template};
#-------------------------------------------------------------------------------
# Validate and process options
#-------------------------------------------------------------------------------
#
# Sanity
#
die "Choose either -load or -delete, not both\n"
if (defined($loadfile) && defined($deletefile));
#
# Check the configuration file
#
die "Unable to find configuration file $cfgfile\n" unless ( -e $cfgfile );
#
# Check the load file
# Process the load option and the delete option, checking any files mentioned,
# and determining the primary action we're aiming for.
#
if ($loadfile) {
die "File $loadfile does not exist\n" unless -e $loadfile;
die "File $loadfile is not readable\n" unless -r $loadfile;
if (optionalFile('load', $loadfile)) {
$action_mode = 'load';
_debug(
$DEBUG > 0,
"Action mode: $action_mode",
( $loadfile eq ''
? "Load from arguments"
: "File to load: $loadfile"
)
);
}
elsif (optionalFile('delete', $deletefile)) {
$action_mode = 'delete';
_debug(
$DEBUG > 0,
"Action mode: $action_mode",
( $deletefile eq ''
? "Delete from arguments"
: "File to delete from $deletefile"
)
);
}
else {
$action_mode = 'none';
}
#
# Check the delete file
# The copyright checking mode defaults to 'auto' if the option has no value,
# or may be 'manual' or 'none'. If the option is not used at all it defaults
# to 'none'. It's only relevant to the 'load' action though.
#
if ($deletefile) {
die "File $deletefile does not exist\n" unless -e $deletefile;
die "File $deletefile is not readable\n" unless -r $deletefile;
}
#
# The checking mode defaults to 'auto' if the option has no value, or may be
# 'manual' or 'none'. If the option is not used at all it defaults to 'none'.
#
if ( defined($check) ) {
if ( $action_mode eq 'load' ) {
if ( $action_mode eq 'load' && defined($check) ) {
$check =~ s/(^\s+|\s+$)//g;
if ($check =~ /^$/) {
$check = "auto";
@ -249,14 +284,21 @@ if ( defined($check) ) {
"Values are <blank>, auto and manual\n"
unless ($check =~ /^(auto|manual|none)$/)
}
}
else {
}
else {
$check = 'none';
}
emit($silent,"Copyright check mode = $check\n");
}
emit($silent,"Dry run mode = " . ($dry_run ? "On" : "Off") . "\n");
emit($silent,"----\n");
# TODO: Does it make sense to have -load and -report, etc at the same time?
#
# We accept -report, meaning report everything or -report='title' to report
# just the feed with the given title.
# just the feed with the given title (actually, the title which contains the
# given string).
#
if ( defined($report) ) {
if ($report =~ /^$/) {
@ -266,6 +308,7 @@ if ( defined($report) ) {
$search_target = $report;
}
}
#
# We accept -json or -json=filename. In the former case we make a default
# name, otherwise we use the one provided.
@ -298,9 +341,9 @@ if ( defined($template) ) {
die "Error: Unable to find template $template\n" unless -r $template;
}
#
#-------------------------------------------------------------------------------
# Load configuration data
#
#-------------------------------------------------------------------------------
my $conf = new Config::General(
-ConfigFile => $cfgfile,
-InterPolateVars => 1,
@ -308,9 +351,9 @@ my $conf = new Config::General(
);
my %config = $conf->getall();
#
#-------------------------------------------------------------------------------
# Connect to the database
#
#-------------------------------------------------------------------------------
my $dbtype = $config{database}->{type};
my $dbfile = $config{database}->{file};
my $dbuser = $config{database}->{user};
@ -331,9 +374,7 @@ $dbh->do('PRAGMA foreign_keys = ON');
#
my $rows = countRows( $dbh, 'SELECT count(*) FROM urls' );
my $work = (
scalar(@new_urls) > 0
|| defined($loadfile)
|| defined($deletefile)
( scalar(@urls) > 0 && $action_mode =~ /load|delete/ )
|| ( defined($report)
|| defined($json)
|| defined($opml)
@ -341,7 +382,10 @@ my $work = (
|| $scan && $rows > 0 )
);
die "Nothing to do!\n" unless $work;
unless ($work) {
print STDERR "Nothing to do!\n";
exit;
}
#-------------------------------------------------------------------------------
# Set up logging keeping the default log layout except for the date
@ -391,9 +435,14 @@ if ($rejectfile) {
$rules = WWW::RobotRules->new($robot_name);
#-------------------------------------------------------------------------------
# Slurp the load file into @new_urls if the file is provided
# Check the mode we are in and prepare to load or delete according to the
# answer.
#-------------------------------------------------------------------------------
if ($loadfile) {
if ($action_mode eq 'load') {
#
# Slurp the load file into @urls if the file is provided
#
if ($loadfile) {
#
# Load the input file
#
@ -408,49 +457,49 @@ if ($loadfile) {
#
# Add the loaded URLs to the array
#
push( @new_urls, @loaded );
}
push( @urls, @loaded );
}
#
# Now, we either have URLs from the command line, or from the load file (or
# both), so we process these.
#
# It's a loop because 'loadUrls' might find some more URLs by scanning HTML
# URLs if given them. If it does we replace @new_urls with the found URLs and
# go again. When there's nothing returned the loop stops.
# ----
# NOTE: This seems dirty, but all the 'while' is testing is whether the array
# contains anything or not. It's not iterating over it or anything, which would
# be messy!
#
while (@new_urls) {
#
# Now, we either have URLs from the command line, or from the load file (or
# both), so we process these.
#
# It's a loop because 'loadUrls' might find some more URLs by scanning HTML
# URLs if given them. If it does we replace @urls with the found URLs and
# go again. When there's nothing returned the loop stops.
# ----
# NOTE: This seems dirty, but all the 'while' is testing is whether the array
# contains anything or not. It's not iterating over it or anything, which would
# be messy!
#
while (@urls) {
#
# Remove duplicates, finish if it deletes them all!
#
@new_urls = uniq(@new_urls);
last unless @new_urls;
@urls = uniq(@urls);
last unless @urls;
#
# Remove any commented out lines, finish if it deletes them all!
#
@new_urls = grep {!/^\s*#/} @new_urls;
last unless @new_urls;
@urls = grep {!/^\s*#/} @urls;
last unless @urls;
$LOG->info( "Received ", scalar(@new_urls),
" URLs to add to the database" );
$LOG->info( "Loading ", scalar(@urls), " URLs to the database" );
#
# Load these URLs as appropriate, returning any more that we find by
# following HTML urls. We overwrite the original list and start all over
# again.
#
@new_urls = loadUrls( $dbh, \@new_urls, $rules, \%keymap );
@urls = loadUrls( $dbh, \@urls, $rules, \%keymap, $dry_run );
}
}
#
# Process the delete file if there is one
#
if ($deletefile) {
elsif ($action_mode eq 'delete') {
#
# Process the delete file if there is one
#
if ($deletefile) {
#
# Load the delete file
#
@ -458,35 +507,48 @@ if ($deletefile) {
or die "$PROG : failed to open load file '$deletefile' : $!\n";
chomp( @deletions = <$del> );
close($del)
or warn "$PROG : failed to close load file '$deletefile' : $!\n";
or warn "$PROG : failed to close delete file '$deletefile' : $!\n";
#
# Add the loaded URLs to the array
#
push( @urls, @deletions );
#
# Remove duplicates
#
@deletions = uniq(@deletions);
$LOG->info( "Deleting ", scalar(@deletions), " URLs from the database" );
@urls = uniq(@urls);
}
if (@urls) {
#
# There are URLs to delete. Process them on by one.
# TODO: check that these URLs are actually in the database! Seems
# silly to report "Failed to delete" when it's not there anyway!
#
if (@deletions) {
# There are URLs to delete. Process them one by one.
#
if ($dry_run) {
emit( $silent,
"Would have deleted " . scalar(@urls) . " URLs\n" )
}
else {
$sth1 = $dbh->prepare(q{DELETE from urls WHERE url = ?});
foreach my $rec (@deletions) {
foreach my $rec (@urls) {
$rv = $sth1->execute($rec);
if ( $dbh->err ) {
warn $dbh->errstr;
}
if ( $rv != 0 ) {
emit ( $silent, "Deleted $rec ($rv rows)\n" );
$LOG->info( "Deleted URL '$rec' from the database" );
}
else {
emit ( $silent, "Failed to delete $rec\n" );
$LOG->warning( "Failed to delete '$rec' from the database" );
}
}
}
}
}
#-------------------------------------------------------------------------------
@ -494,9 +556,9 @@ if ($deletefile) {
# TODO: Needs to be developed; does nothing at the moment.
#-------------------------------------------------------------------------------
if ($scan) {
$LOG->info( "Scan is not fully implemented yet" );
$LOG->warning( "Scan is not fully implemented yet" );
# Testing. Processes the first two feeds
scanDB($dbh, \%keymap);
scanDB($dbh, \%keymap, $dry_run);
}
#-------------------------------------------------------------------------------
@ -625,7 +687,7 @@ if ($opml) {
#-------------------------------------------------------------------------------
# Fill and print the template if requested
# Fill and print the TT² template if requested
#-------------------------------------------------------------------------------
if ($template) {
my $tt = Template->new(
@ -664,12 +726,14 @@ exit;
#=== FUNCTION ================================================================
# NAME: loadUrls
# PURPOSE: To load URLs read from the input file into the database
# PURPOSE: To load URLs read from the input file (and the arguments) into
# the database
# PARAMETERS: $dbh database handle
# $new_urls arrayref containing URLs
# $rules WWW::RobotRules object
# $keymap hashref containing a map of key names to
# database field names
# $dry_run Boolean, set if in dry-run mode
# RETURNS: Any new URLs discovered by investigating non-feed URLs.
# DESCRIPTION:
# THROWS: No exceptions
@ -677,7 +741,7 @@ exit;
# SEE ALSO: N/A
#===============================================================================
sub loadUrls {
my ( $dbh, $new_urls, $rules, $keymap ) = @_;
my ( $dbh, $new_urls, $rules, $keymap, $dry_run ) = @_;
my ( $stream, $feed );
my ( %uridata, $roboturl, @found_urls );
@ -751,7 +815,7 @@ sub loadUrls {
emit( $silent, "Search for robots.txt file failed\n" );
}
else {
emit( $silent, "Check of robots.txt rules failed\n" );
emit( $silent, "Check of robots.txt rules blocks access\n" );
$uridata{SAVE} = 0;
next;
}
@ -790,13 +854,14 @@ sub loadUrls {
# weirdness of RSS and poor adherence to what standards there
# are).
#
print Dumper($feed), "\n" if ( $DEBUG > 2 );
_debug( $DEBUG > 2, Dumper($feed));
storeFeed($feed,\%uridata);
#
# Perform a check on the copyright. The routine sets
# $uridata{SAVE} = 0 if the copyright is not acceptable.
#
$uridata{CHECKMODE} = $check;
if ( $check ne 'none' ) {
unless (checkCopyright( $check, \%uridata )) {
#
@ -848,7 +913,7 @@ sub loadUrls {
# Decide whether to save what we have collected
#
if ( $uridata{SAVE} ) {
if ( addURI( $dbh, \%uridata, $keymap ) ) {
if ( addURI( $dbh, \%uridata, $keymap, $dry_run ) ) {
emit( $silent, "$uridata{URI} added to the database\n" );
$LOG->info("$uridata{TYPE} ",$uridata{URI},' added to the database');
@ -864,7 +929,7 @@ sub loadUrls {
if ( defined( $uridata{ENCLOSURE_COUNT} )
&& $uridata{ENCLOSURE_COUNT} > 0 )
{
if ( addEnclosures( $dbh, \%uridata ) ) {
if ( addEnclosures( $dbh, \%uridata, $dry_run ) ) {
emit( $silent, $uridata{ENCLOSURE_COUNT},
" enclosures for $uridata{URI} added to the database\n"
);
@ -882,7 +947,7 @@ sub loadUrls {
#
# Dump what we have if requested
#
print Dumper( \%uridata ), "\n" if ( $DEBUG > 1 );
_debug( $DEBUG > 1, Dumper( \%uridata ) );
emit( $silent, '-' x 80, "\n" );
}
@ -969,6 +1034,7 @@ sub searchTitle {
# PARAMETERS: $dbh database handle
# $keymap hashref containing a map of key names to
# database field names
# $dry_run Boolean, set if in dry-run mode
# RETURNS: Nothing
# DESCRIPTION:
# THROWS: No exceptions
@ -976,8 +1042,9 @@ sub searchTitle {
# SEE ALSO: N/A
#===============================================================================
sub scanDB {
my ($dbh, $keymap) = @_;
my ($dbh, $keymap, $dry_run) = @_;
# TODO: dry-run mode not implemented here yet
my ( $sql1, $sth1, $rv1, $h1 );
my ( $aref, @urls, $DT, $stream, $feed );
my ( %uridata, $urichanges, $enc_changes );
@ -1259,7 +1326,7 @@ sub scanFeed {
#
# Save the important feed components in the %uridata hash
#
print Dumper($feed), "\n" if ( $DEBUG > 2 );
_debug( $DEBUG > 2, Dumper( $feed ) );
storeFeed( $feed, $uridata );
}
else {
@ -1372,7 +1439,7 @@ sub findFeed {
sub reportFeed {
my ($feed, $fh) = @_;
my $lwidth = 12;
my $lwidth = 15;
#
# Hash for converting database keys to labels for the report and arrays
@ -1397,6 +1464,7 @@ sub reportFeed {
'ep_title' => 'Title',
'ep_urls_id' => 'URL key',
'urls_author' => 'Author',
'urls_check_type' => 'Check type',
'urls_content_type' => 'Content type',
'urls_copyright' => 'Copyright',
'urls_description' => 'Description',
@ -1411,6 +1479,7 @@ sub reportFeed {
'urls_last_update' => 'Last updated',
'urls_link' => 'Link',
'urls_modified' => 'Modified on',
'urls_reason_accepted' => 'Reason accepted',
'urls_title' => 'Title',
'urls_url' => 'Feed URL',
'urls_urltype' => 'URL type',
@ -1426,6 +1495,8 @@ sub reportFeed {
'urls_author',
'urls_content_type',
'urls_copyright',
'urls_check_type',
'urls_reason_accepted',
'urls_description',
'urls_dns',
'urls_generator',
@ -1577,6 +1648,7 @@ sub dbSearch {
# PURPOSE: To perform a non-SELECT query
# PARAMETERS: $dbh database handle
# $sql SQL expression to use
# $dry_run Boolean, set if in dry-run mode
# @args arguments for the 'execute'
# RETURNS: True (1) if the query succeeded, otherwise false (0).
# DESCRIPTION: Uses 'prepare_cached' to allow repeated calls with the same
@ -1587,10 +1659,15 @@ sub dbSearch {
# SEE ALSO: N/A
#===============================================================================
sub execSQL {
my ( $dbh, $sql, @args ) = @_;
my ( $dbh, $sql, $dry_run, @args ) = @_;
my ( $sth1, $rv );
if ($dry_run) {
emit($silent,"Dry-run: Would have run SQL '$sql'\n");
return;
}
$sth1 = $dbh->prepare_cached($sql);
try {
$rv = $sth1->execute(@args);
@ -1792,12 +1869,16 @@ sub checkContentType {
my ( $uri, $uridata, $headers, $children, $log ) = @_;
my @feeds;
my $content;
$uridata->{HTTP_STATUS} = 'Unknown';
my $browser = LWP::UserAgent->new or return 0;
$browser->timeout(10);
$browser->agent("$PROG/$VERSION");
my $response = $browser->head( $uri->as_string, %{$headers} )
#my $response = $browser->head( $uri->as_string, %{$headers} )
my $response = $browser->get( $uri->as_string, %{$headers} )
or return 0;
$uridata->{HTTP_STATUS} = $response->status_line;
@ -1830,15 +1911,25 @@ sub checkContentType {
}
else {
#
# This HTML and we found 'child' feeds of some kind
# This is HTML and we found 'child' feeds of some kind
#
emit( $silent, "Found ", scalar(@feeds),
" feeds within this HTML page\n" );
print Dumper( \@feeds ), "\n" if $DEBUG > 0;
_debug( $DEBUG > 0, Dumper( \@feeds ) );
push(@{$children}, @feeds);
}
}
#
# TODO: Get the title for an HTML page by parsing the source
#
if ( $uridata->{TYPE} eq 'HTML' ) {
unless ($uridata->{TITLE}) {
$content = $response->decoded_content;
$uridata->{TITLE} = getHTMLTitle($uri->as_string, $content);
}
}
$log->info( "URL content classified as: ", $uridata->{TYPE} );
emit( $silent, "URL content classified as: ", $uridata->{TYPE}, "\n" );
return 1;
@ -1849,6 +1940,41 @@ sub checkContentType {
}
}
#=== FUNCTION ================================================================
# NAME: getURL
# PURPOSE: Download the contents of an URL
# PARAMETERS: $url URL of the page to download
# RETURNS: String representation of the contents or undef if the
# download failed.
# DESCRIPTION: Issues a GET on the URL. If successful the contents are
# decoded and returned, otherwise undef is returned.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO:
#===============================================================================
sub getURL {
my ($url) = @_;
#
# Use LWP::UserAgent to get the feed and handle errors
#
my $ua = LWP::UserAgent->new;
$ua->timeout(10);
$ua->agent("$PROG/$VERSION");
my $response = $ua->get($url);
my $content;
if ( $response->is_success ) {
$content = $response->decoded_content;
return $content;
}
else {
warn "Failed to get $url\n";
warn $response->status_line, "\n";
return; # undef
}
}
#=== FUNCTION ================================================================
# NAME: getFeed
# PURPOSE: Download the contents of a feed URL
@ -1885,6 +2011,98 @@ sub getFeed {
}
}
#=== FUNCTION ================================================================
# NAME: start_handler
# PURPOSE: HTTP::Parser handler for <title> events
# PARAMETERS: <first> the name of the tag found
# <second> the object being processed
# RETURNS: Nothing (ignored anyway)
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
#sub start_handler {
#
# #
# # Ignore any tags which are not 'title'
# #
# return if shift ne "title";
#
# #
# # Define more handlers if we have a title. One to collect the title string
# # and the other to abort the parse on encountering the end of the title.
# #
# my $self = shift;
# $self->handler(text => sub { $main::html_title = shift }, "dtext");
# $self->handler(
# end => sub {
# shift->eof if shift eq "title";
# },
# "tagname,self"
# );
#}
#=== FUNCTION ================================================================
# NAME: getHTMLTitle
# PURPOSE: Parse an HTML page to get data. At the moment this is just the
# title in the header section.
# PARAMETERS: $url URL of the page; only used for messages and as
# a fallback title.
# $content Decoded content of the HTML page
# RETURNS: The title, if there is one, otherwise the URL
# DESCRIPTION:
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub getHTMLTitle {
my ($url, $content) = @_;
#
# Is this effectively a local subroutine?
#
my $start_handler = sub {
#
# Ignore any tags which are not 'title'
#
return if shift ne "title";
#
# Define more handlers if we have a title. One to collect the title string
# and the other to abort the parse on encountering the end of the title.
#
my $self = shift;
$self->handler(text => sub { $main::html_title = shift }, "dtext");
$self->handler(
end => sub {
shift->eof if shift eq "title";
},
"tagname,self"
);
};
#
# Create the HTML::Parser object
#
my $p = HTML::Parser->new(api_version => 3);
#
# Define a 'start' handler. When called with 'title' as the 'tagname' it
# creates a 'text' event handler to print the text and an 'end' handler to
# quit when the closing '</title' tag is found.
#
#$p->handler(start => \&start_handler, "tagname,self");
$p->handler(start => $start_handler, "tagname,self");
#
# Invoke the parser on the string containing the returned HTML.
#
$p->parse($content);
return $main::html_title // $url;
}
#=== FUNCTION ================================================================
# NAME: parseFeed
# PURPOSE: Parse a podcast feed that has already been downloaded
@ -2014,7 +2232,7 @@ sub storeFeed {
sub checkCopyright {
my ($checkmode, $uridata) = @_;
my ( $copyright, $re, $decision );
my ( $copyright, $re, $decision, $reason );
$LOG->info("Checking copyright of feed (mode: $checkmode)");
if ( $checkmode eq 'manual' ) {
@ -2040,14 +2258,36 @@ sub checkCopyright {
warn "Problem processing copyright decision: $_";
$decision = 0;
};
if ($decision) {
#
# If accepted we want a reason for this manual check
#
try {
$reason = prompt(
-in => *STDIN,
-out => *STDERR,
-prompt => 'Please give a reason for this decision:',
-style => 'bold red underlined',
-default => 'No reason given'
);
}
catch {
warn "Problem processing reason for copyright decision: $_";
$reason = 'Error while processing the reason';
};
$uridata->{REASON_ACCEPTED} = $reason;
}
}
else {
#
# Careful. Un-escaped spaces are ignored
# Automatic mode.
# Careful. Un-escaped spaces are ignored in the regex
#
$re = qr{(
CC|
Creative\ Commons|
\bCC\b|
\bCreative\ Commons\b|
creativecommons.org|
Attribution.NonCommercial.No.?Derivatives?
)}xmi;
@ -2106,9 +2346,10 @@ sub parseRSS {
# $uridata hashref containing data for the current URI
# $keymap hashref containing a map of key names to
# database field names
# $dry_run Boolean, set if in dry-run mode
# RETURNS: True (1) if the insert succeeded, false (0) otherwise
# DESCRIPTION: The hash keys are defined as an array to make it easy to slice
# the hash and the SQL is defined internally using the size of
# the hash, and the SQL is defined internally using the size of
# the key array as a guide to the number of '?' placeholders.
# These are passed to execSQL to do the work.
# THROWS: No exceptions
@ -2116,7 +2357,7 @@ sub parseRSS {
# SEE ALSO: N/A
#===============================================================================
sub addURI {
my ( $dbh, $uridata, $keymap ) = @_;
my ( $dbh, $uridata, $keymap, $dry_run ) = @_;
my @keys = (
'URI', 'DNS',
@ -2126,6 +2367,7 @@ sub addURI {
'DESCRIPTION', 'AUTHOR',
'MODIFIED', 'LINK',
'IMAGE', 'COPYRIGHT',
'CHECKTYPE', 'REASON_ACCEPTED',
'GENERATOR', 'LANGUAGE',
);
@ -2135,9 +2377,9 @@ sub addURI {
. 'VALUES('
. join( ',', ('?') x scalar(@keys) ) . ')';
print "addURI query: $sql\n" if $DEBUG > 0;
_debug( $DEBUG > 0, "addURI query: $sql");
return execSQL( $dbh, $sql, @{$uridata}{@keys} );
return execSQL( $dbh, $sql, $dry_run, @{$uridata}{@keys} );
}
@ -2206,6 +2448,7 @@ sub extractEnclosures {
# $uridata hashref containing data for the current URI
# including an arrayref of hashrefs of episode
# data
# $dry_run Boolean, set if in dry-run mode
# RETURNS: True (1) if all the inserts succeeded, false (0) otherwise
# DESCRIPTION: The SQL is defined internally and the hash keys are defined as
# an array to make it easy to slice the hash. The enclosures (or
@ -2219,7 +2462,7 @@ sub extractEnclosures {
# SEE ALSO: N/A
#===============================================================================
sub addEnclosures {
my ( $dbh, $uridata ) = @_;
my ( $dbh, $uridata, $dry_run ) = @_;
my $sql = q{INSERT INTO episodes
(urls_id, link, enclosure, title, author, category, source, ep_id,
@ -2233,7 +2476,7 @@ sub addEnclosures {
my $successes = 0;
foreach my $enc ( @{ $uridata->{ENCLOSURES} } ) {
if ( execSQL( $dbh, $sql, $uridata->{URI_ID}, @{$enc}{@keys} ) ) {
if ( execSQL( $dbh, $sql, $dry_run, $uridata->{URI_ID}, @{$enc}{@keys} ) ) {
$successes++;
}
else {
@ -2253,6 +2496,7 @@ sub addEnclosures {
# data
# $keymap hashref containing a map of key names to
# database field names
# $dry_run Boolean, set if in dry-run mode
# RETURNS: The number of changes made
# DESCRIPTION:
# THROWS: No exceptions
@ -2260,8 +2504,9 @@ sub addEnclosures {
# SEE ALSO: N/A
#===============================================================================
sub updateURI {
my ( $dbh, $uridata, $keymap ) = @_;
my ( $dbh, $uridata, $keymap, $dry_run ) = @_;
# TODO: dry-run mode not implemented here yet
my ( $sql1, $sth1, $rv1, $h1 );
my ( %fieldvals, %where );
my ( $diffs, $updates ) = ( 0, 0 );
@ -2353,6 +2598,7 @@ sub updateURI {
sub updateEnclosures {
my ( $dbh, $uridata ) = @_;
# TODO: This doesn't acttually update annything!
my ( $sql1, $sth1, $rv1, $h1 );
my ( %fieldvals, %where );
my ( $diffs, $updates ) = ( 0, 0 );
@ -2372,6 +2618,40 @@ sub updateEnclosures {
$h1 = $sth1->fetchrow_hashref;
}
#=== FUNCTION ================================================================
# NAME: optionalFile
# PURPOSE: Process an option of the form '-opt:s' where 's' is an
# optional filename.
# PARAMETERS: $optionName Name of option
# $optionValue Value of option (assumed to be blank of
# a filename)
# RETURNS: A boolean: 1 (true) if there is a filename, 0 (false) if the
# name has been omitted.
# DESCRIPTION: The $optionValue will be blank or a filename. If the latter
# then the existence of the file and its readbility are checked
# and the script dies if either test fails.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub optionalFile {
my ( $optionName, $optionValue ) = @_;
if (defined($optionValue)) {
if ( $optionValue =~ /^$/ ) {
return 1;
}
else {
die "File in '-$optionName=$optionValue' does not exist\n"
unless -e $optionValue;
die "File in '-$optionName=$optionValue' is not readable\n"
unless -r $optionValue;
return 1;
}
}
return 0;
}
#=== FUNCTION ================================================================
# NAME: equal
# PURPOSE: Compare two strings even if undefined
@ -2504,6 +2784,32 @@ sub emit {
}
}
#=== FUNCTION ================================================================
# NAME: _debug
# PURPOSE: Prints debug reports
# PARAMETERS: $active Boolean: 1 for print, 0 for no print
# $messages... Arbitrary list of messages to print
# RETURNS: Nothing
# DESCRIPTION: Outputs messages if $active is true. It removes any trailing
# newline from each one and then adds one in the 'print' to the
# caller doesn't have to bother. Prepends each message with 'D>'
# to show it's a debug message.
# THROWS: No exceptions
# COMMENTS: Differs from other functions of the same name
# SEE ALSO: N/A
#===============================================================================
sub _debug {
my $active = shift;
my $message;
return unless $active;
while ($message = shift) {
chomp($message);
print STDERR "D> $message\n";
}
}
#=== FUNCTION ================================================================
# NAME: Options
# PURPOSE: Processes command-line options
@ -2518,10 +2824,11 @@ sub Options {
my ($optref) = @_;
my @options = (
"help", "manpage", "debug=i", "silent!",
"load=s", "delete=s", "scan!", "report:s",
"check:s", "json:s", "opml:s", "config=s",
"out=s", "rejects:s", "template:s",
"help", "manpage", "debug=i", "dry-run!",
"silent!", "load:s", "delete:s", "scan!",
"report:s", "html!", "check:s", "json:s",
"opml:s", "config=s", "out=s", "rejects:s",
"template:s",
);
if ( !GetOptions( $optref, @options ) ) {

BIN
feedWatcher.db
View File

Binary file not shown.

24
feedWatcher.html
View File

@ -198,6 +198,14 @@
<dl>
<dt><a href="https://2.5admins.com">2.5 Admins</a> (<a href="https://2.5admins.com/feed/podcast">feed</a>)</dt>
<dd>2.5 Admins is a podcast featuring two sysadmins called Allan Jude and Jim Salter, and a producer/editor who can just about configure a Samba share called Joe Ressington. Every two weeks we get together, talk about recent tech news, and answer some of your admin-related questions.</dd>
<dt><a href="https://www.adminadminpodcast.co.uk">Admin Admin Podcast</a> (<a href="http://feeds.feedburner.com/TheAdminAdminPodcast">feed</a>)</dt>
@ -558,6 +566,14 @@
<dt><a href="https://www.linuxuserspace.show">Linux User Space</a> (<a href="https://www.linuxuserspace.show/rss">feed</a>)</dt>
<dd>How did your favorite Linux distribution get its start? Join us and find out! Linux User Space is hosted by Leo and Dan, and every two weeks we deep dive into the history of Linux distributions and the things that matter to us. Episodes drop every other Monday.</dd>
<dt><a href="https://www.linuxatwork.org">Linux at Work</a> (<a href="http://feeds.podtrac.com/mBdfP0QTX0iY">feed</a>)</dt>
@ -702,6 +718,14 @@
<dt><a href="http://www.scienceforthepeople.ca/">Science for the People</a> (<a href="http://feeds.feedburner.com/SkepticallySpeaking">feed</a>)</dt>
<dd>Science for the People is a long-format interview podcast that explores the connections between science, popular culture, history, and public policy, to help listeners understand the evidence and arguments behind what&#39;s in the news and on the shelves. Our hosts sit down with science researchers, writers, authors, journalists, and experts to discuss science from the past, the science that affects our lives today, and how science might change our future.</dd>
<dt><a href="https://twit.tv/shows/security-now">Security Now (Audio)</a> (<a href="http://feeds.twit.tv/sn.xml">feed</a>)</dt>

480
feedWatcher.json
View File

File diff suppressed because it is too large Load Diff

195
feedWatcher.mkd
View File

@ -2,459 +2,474 @@
### The finest selection of Free Culture Podcasts spanning the genres of Discussion, Drama, Education, Music, and beyond.
- **2.5 Admins**
- Website: https://2.5admins.com
- Feed: https://2.5admins.com/feed/podcast
- Copyright:
- **Admin Admin Podcast**
- Website: https://www.adminadminpodcast.co.uk
- Feed: http://feeds.feedburner.com/TheAdminAdminPodcast
- Licence:
- Copyright:
- **All About Android (Audio)**
- Website: https://twit.tv/shows/all-about-android
- Feed: http://feeds.twit.tv/aaa.xml
- Licence: This work is licensed under a Creative Commons License - Attribution-NonCommercial-NoDerivatives 4.0 International - http://creativecommons.org/licenses/by-nc-nd/4.0/
- Copyright: This work is licensed under a Creative Commons License - Attribution-NonCommercial-NoDerivatives 4.0 International - http://creativecommons.org/licenses/by-nc-nd/4.0/
- **All In IT Radio (ogg)**
- Website: http://aiit.se/radio/
- Feed: http://feeds.aiit.se/allinit-radio-ogg
- Licence:
- Copyright:
- **AmateurLogic.TV**
- Website: http://www.amateurlogic.tv
- Feed: http://amateurlogic.tv/transmitter/feeds/ipod.xml
- Licence: Creative Commons
- Copyright: Creative Commons
- **Ask Noah Show**
- Website: https://podcast.asknoahshow.com
- Feed: https://feeds.fireside.fm/asknoah/rss
- Licence: © 2023 CC-BY-ND
- Copyright: © 2023 CC-BY-ND
- **CCHits.net**
- Website: https://cchits.net/daily
- Feed: https://cchits.net/daily/rss
- Licence: The content created by this site is generated by a script which is licensed under the Affero General Public License version 3 (AGPL3). The generated content is released under a Creative Commons By-Attribution License.
- Copyright: The content created by this site is generated by a script which is licensed under the Affero General Public License version 3 (AGPL3). The generated content is released under a Creative Commons By-Attribution License.
- **CCJam**
- Website: https://ccjam.otherside.network/
- Feed: https://ccjam.otherside.network/feed/podcast/
- Licence:
- Copyright:
- **Destination Linux**
- Website: https://destinationlinux.org
- Feed: http://destinationlinux.org/feed/mp3/
- Licence:
- Copyright:
- **Edict Zero**
- Website: https://edictzero.com
- Feed: https://edictzero.wordpress.com/feed/
- Licence:
- Copyright:
- **English – Wikipediapodden**
- Website: http://wikipediapodden.se
- Feed: http://wikipediapodden.se/tag/english/feed/
- Licence: CC BY-SA 4.0 Wikipediapodden
- Copyright: CC BY-SA 4.0 Wikipediapodden
- **Escape Pod**
- Website: https://escapepod.org
- Feed: http://escapepod.org/feed/
- Licence:
- Copyright:
- **Expedition Sasquatch**
- Website: https://ExpeditionSasquatch.org
- Feed: https://expeditionsasquatch.org/episodes.mp3.rss
- Licence: CC BY SA 4.0
- Copyright: CC BY SA 4.0
- **FLOSS Weekly (Audio)**
- Website: https://twit.tv/shows/floss-weekly
- Feed: http://leoville.tv/podcasts/floss.xml
- Licence: This work is licensed under a Creative Commons License - Attribution-NonCommercial-NoDerivatives 4.0 International - http://creativecommons.org/licenses/by-nc-nd/4.0/
- Copyright: This work is licensed under a Creative Commons License - Attribution-NonCommercial-NoDerivatives 4.0 International - http://creativecommons.org/licenses/by-nc-nd/4.0/
- **FLOSS Weekly (Audio)**
- Website: https://twit.tv/shows/floss-weekly
- Feed: http://feeds.twit.tv/floss.xml
- Licence: This work is licensed under a Creative Commons License - Attribution-NonCommercial-NoDerivatives 4.0 International - http://creativecommons.org/licenses/by-nc-nd/4.0/
- Copyright: This work is licensed under a Creative Commons License - Attribution-NonCommercial-NoDerivatives 4.0 International - http://creativecommons.org/licenses/by-nc-nd/4.0/
- **FOSS and Crafts**
- Website: https://fossandcrafts.org
- Feed: https://fossandcrafts.org/rss-feed.rss
- Licence: Licensed under CC BY-SA 4.0 International
- Copyright: Licensed under CC BY-SA 4.0 International
- **FSFE Events**
- Website: https://fsfe.org/events/
- Feed: https://fsfe.org/events/events.en.rss
- Licence: Copyright (c) Free Software Foundation Europe. Verbatim copying and distribution
- Copyright: Copyright (c) Free Software Foundation Europe. Verbatim copying and distribution
of this entire article is permitted in any medium, provided this
notice is preserved.
- **FSFE News**
- Website: https://fsfe.org/news/
- Feed: https://fsfe.org/news/news.en.rss
- Licence: Copyright (c) Free Software Foundation Europe. Verbatim copying and distribution
- Copyright: Copyright (c) Free Software Foundation Europe. Verbatim copying and distribution
of this entire article is permitted in any medium, provided this
notice is preserved.
- **Free as in Freedom**
- Website: http://faif.us/cast/
- Feed: http://faif.us/feeds/cast-ogg/
- Licence: 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2018, 2019, 2020, 2021, Free as in Freedom. Licensed under a Creative Commons Attribution-Share Alike 3.0 USA License.
- Copyright: 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2018, 2019, 2020, 2021, Free as in Freedom. Licensed under a Creative Commons Attribution-Share Alike 3.0 USA License.
- **Free as in Freedom**
- Website: http://faif.us/cast/
- Feed: http://faif.us/feeds/cast-mp3/
- Licence: 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2018, 2019, 2020, 2021, Free as in Freedom. Licensed under a Creative Commons Attribution-Share Alike 3.0 USA License.
- Copyright: 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2018, 2019, 2020, 2021, Free as in Freedom. Licensed under a Creative Commons Attribution-Share Alike 3.0 USA License.
- **GAMERadio**
- Website: http://www.hwhq.com/
- Feed: http://hwhq.com/rss.xml
- Licence:
- Copyright:
- **GNU World Order Linux Cast**
- Website: http://www.gnuworldorder.info
- Feed: https://gnuworldorder.info/ogg.xml
- Licence:
- Copyright:
- **GNU/Linux RTM**
- Website: http://gnulinuxrtm.blogspot.com/
- Feed: http://feeds.feedburner.com/GNULinuxRTM
- Licence: This work is licensed under Creative Commons Attribution 4.0
- Copyright: This work is licensed under Creative Commons Attribution 4.0
- **Geek Speak with Lyle Troxell**
- Website: https://geekspeak.org/
- Feed: https://geekspeak.org/episodes/rss.xml
- Licence: Creative Commons Attribution 3.0 United States License
- Copyright: Creative Commons Attribution 3.0 United States License
- **GeekNights Mondays: Science Technology Computing**
- Website: http://frontrowcrew.com/geeknights/monday/
- Feed: http://feeds.feedburner.com/GNSciTech
- Licence: Creative Commons
- Copyright: Creative Commons
- **Going Linux**
- Website: https://goinglinux.com
- Feed: http://goinglinux.com/oggpodcast.xml
- Licence: Creative Commons Attribution 4.0 International License.
- Copyright: Creative Commons Attribution 4.0 International License.
- **Hacker Public Radio**
- Website: http://hackerpublicradio.org/about.php
- Feed: http://hackerpublicradio.org/hpr_ogg_rss.php
- Licence: Creative Commons Attribution-ShareAlike 4.0 International (CC BY-SA 4.0) License
- Copyright: Creative Commons Attribution-ShareAlike 4.0 International (CC BY-SA 4.0) License
- **How to Fix the Internet**
- Website: https://www.eff.org/how-to-fix-the-internet-podcast
- Feed: https://feeds.eff.org/howtofixtheinternet
- Licence:
- Copyright:
- **International Open Podcast**
- Website: http://internationalopenmagazine.org/category/podcast.html
- Feed: http://spielend-programmieren.at/intopenpodcast.xml
- Licence: Copyright 2015 CC-BY-SA
- Copyright: Copyright 2015 CC-BY-SA
- **Internet Archive**
- Website: https://archive.org/
- Feed: http://feeds.feedburner.com/archive/Lqwl
- Licence:
- Copyright:
- **Knightwise.com Audio Feed.**
- Website: https://knightwise.com
- Feed: http://feeds.feedburner.com/knightcastpodcast
- Licence: Creative commons apply ! Non commercial re-use is allowed.
- Copyright: Creative commons apply ! Non commercial re-use is allowed.
- **LQ Radio**
- Website: http://radio.linuxquestions.org
- Feed: http://feeds.feedburner.com/linuxquestions/LQRadioALL-ogg
- Licence:
- Copyright:
- **LaGER: GNU/Linux and Games/Entertainment Radio**
- Website: http://thelinuxlink.net/lager
- Feed: http://feeds.feedburner.com/thelinuxlink/Paek
- Licence: Creative Commons License Some Rights Reserved
- Copyright: Creative Commons License Some Rights Reserved
- **Late Night Linux (Ogg)**
- Website: https://latenightlinux.com
- Feed: http://latenightlinux.com/feed/ogg
- Licence:
- Copyright:
- **Libre Lounge**
- Website: https://librelounge.org
- Feed: https://librelounge.org/rss-feed.rss
- Licence: Licensed under CC BY-SA 4.0 International
- Copyright: Licensed under CC BY-SA 4.0 International
- **Line Noise Podcast**
- Website: https://www.eff.org/rss/podcast/mp3.xml
- Feed: http://www.eff.org/rss/podcast/mp3.xml
- Licence:
- Copyright:
- **Linux After Dark**
- Website: https://linuxafterdark.net
- Feed: https://linuxafterdark.net/feed/podcast
- Licence:
- Copyright:
- **Linux Downtime**
- Website: https://linuxdowntime.com
- Feed: https://latenightlinux.com/feed/extra
- Licence:
- Copyright:
- **Linux Game Cast**
- Website: https://linuxgamecast.com
- Feed: https://linuxgamecast.com/feed/
- Licence:
- Copyright:
- **Linux Game Cast**
- Website: https://linuxgamecast.com
- Feed: http://feeds.feedburner.com/LinuxgamecastWeeklyMp3
- Licence: CC BY-SA LinuxGameCast LLP 2022
- Copyright: CC BY-SA LinuxGameCast LLP 2022
- **Linux In Da House MP3 Feed**
- Website: http://linuxindahouse.com
- Feed: http://linuxindahouse.com/linuxindahouse_mp3.rss
- Licence: Creative Commons License Some Rights Reserved
- Copyright: Creative Commons License Some Rights Reserved
- **Linux Inlaws**
- Website: https://linuxinlaws.eu
- Feed: https://linuxinlaws.eu/inlaws_rss.xml
- Licence: Linux Inlaws (c) 2020 CC-BY-SA
- Copyright: Linux Inlaws (c) 2020 CC-BY-SA
- **Linux Lads**
- Website: https://linuxlads.com
- Feed: https://linuxlads.com/feed_ogg.rss
- Licence: Released under the Creative Commons Attribution-Share Alike 3.0 Unported Licence
- Copyright: Released under the Creative Commons Attribution-Share Alike 3.0 Unported Licence
- **Linux Outlaws**
- Website: http://sixgun.org
- Feed: http://feeds.feedburner.com/linuxoutlaws
- Licence: Copyright © 2007-2012 Sixgun Productions http://creativecommons.org/licenses/by-sa/3.0/
- Copyright: Copyright © 2007-2012 Sixgun Productions http://creativecommons.org/licenses/by-sa/3.0/
- **Linux Reality Podcast (MP3 Feed)**
- Website: http://www.linuxreality.com
- Feed: http://feeds.feedburner.com/linuxreality
- Licence: Linux Reality is released under a Attribution-Noncommercial-No Derivative Works 3.0 United States licence.
- Copyright: Linux Reality is released under a Attribution-Noncommercial-No Derivative Works 3.0 United States licence.
- **Linux Trivia Podcast**
- Website: http://setbit.org/lt.html
- Feed: http://setbit.org/lt-ogg.xml
- Licence: Creative Commons License Some Rights Reserved
- Copyright: Creative Commons License Some Rights Reserved
- **Linux User Space**
- Website: https://www.linuxuserspace.show
- Feed: https://www.linuxuserspace.show/rss
- Copyright: © 2023 Dan Simmons & Leo Chavez
- **Linux at Work**
- Website: https://www.linuxatwork.org
- Feed: http://feeds.podtrac.com/mBdfP0QTX0iY
- Licence: CC BY 4.0
- Copyright: CC BY 4.0
- **Linux in the Ham Shack**
- Website: https://lhspodcast.info/category/podcast-mp3/
- Feed: http://lhspodcast.info/category/podcast-mp3/feed/
- Licence: CC-BY-NC-4.0
- Copyright: CC-BY-NC-4.0
- **Linux in the Ham Shack (OGG Feed)**
- Website: https://lhspodcast.info/category/podcast-ogg/
- Feed: https://lhspodcast.info/category/podcast-ogg/feed/
- Licence: Attribution-NonCommercial-NoDerivatives 4.0 International
- Copyright: Attribution-NonCommercial-NoDerivatives 4.0 International
- **Linuxlugcast**
- Website: https://linuxlugcast.com
- Feed: http://feeds.feedburner.com/linuxlugcast-ogg
- Licence: Creative commons Attribution 4.0 International (CC BY 4.0)
- Copyright: Creative commons Attribution 4.0 International (CC BY 4.0)
- **Linuxlugcast**
- Website: https://linuxlugcast.com
- Feed: http://feeds.feedburner.com/linuxlugcast/dBDY
- Licence: Creative commons Attribution 4.0 International (CC BY 4.0)
- Copyright: Creative commons Attribution 4.0 International (CC BY 4.0)
- **LugRadio (high-quality ogg)**
- Website: http://www.lugradio.org/episodes.ogg.rss
- Feed: http://www.lugradio.org/episodes.ogg.rss
- Licence: LUGRadio is released under a Creative Commons Attribution
- Copyright: LUGRadio is released under a Creative Commons Attribution
NonCommercial NoDerivs licence.
- **Makers Corner, with Nate and Yannick**
- Website: https://makerscorner.tech
- Feed: https://makerscorner.tech/feed/podcast/
- Licence: Unless otherwise stated, this podcast is released under a Creative Commons, By Attribution, Share Alike license.
- Copyright: Unless otherwise stated, this podcast is released under a Creative Commons, By Attribution, Share Alike license.
- **OGG – mintCast**
- Website: https://mintcast.org
- Feed: https://mintcast.org/category/ogg/feed/
- Licence:
- Copyright:
- **OggcastSoftware Freedom Law Center**
- Website: http://www.softwarefreedom.org/podcast/http://www.softwarefreedom.org/
- Feed: http://www.softwarefreedom.org/feeds/podcast-mp3/
- Licence: 2008, 2009, 2010, 2011, Software Freedom Law Center. Licensed under a Creative Commons Attribution-No Derivative Works 3.0 United States License.
- Copyright: 2008, 2009, 2010, 2011, Software Freedom Law Center. Licensed under a Creative Commons Attribution-No Derivative Works 3.0 United States License.
- **Open Minds … from Creative Commons**
- Website: https://anchor.fm/creativecommons
- Feed: https://anchor.fm/s/4d70d828/podcast/rss
- Licence: Creative Commons
- Copyright: Creative Commons
- **Open Source Security Podcast**
- Website: http://opensourcesecuritypodcast.com
- Feed: https://opensourcesecuritypodcast.libsyn.com/rss
- Licence: This work is licensed under the Creative Commons Attribution 4.0 International License. To view a copy of this license, visit http://creativecommons.org/licenses/by/4.0/ or send a letter to Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
- Copyright: This work is licensed under the Creative Commons Attribution 4.0 International License. To view a copy of this license, visit http://creativecommons.org/licenses/by/4.0/ or send a letter to Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
- **PodCastle**
- Website: https://podcastle.org/
- Feed: http://podcastle.org/feed/
- Licence:
- Copyright:
- **Podcast Ubuntu Portugal**
- Website: https://podcastubuntuportugal.org/
- Feed: https://podcastubuntuportugal.org/feed/podcast/
- Licence: Este trabalho está licenciado com uma Licença Creative Commons - Atribuição 4.0 Internacional - http://creativecommons.org/licenses/by-nc-nd/4.0/
- Copyright: Este trabalho está licenciado com uma Licença Creative Commons - Atribuição 4.0 Internacional - http://creativecommons.org/licenses/by-nc-nd/4.0/
- **Podcast – Cory Doctorow's craphound.com**
- Website: https://craphound.com
- Feed: http://feeds.feedburner.com/doctorow_podcast
- Licence: Creative Commons by-nc-sa http://creativecommons.org/licenses/by-nc-sa/2.5/
- Copyright: Creative Commons by-nc-sa http://creativecommons.org/licenses/by-nc-sa/2.5/
- **Podcast – Launchpad blog**
- Website: https://blog.launchpad.net
- Feed: http://news.launchpad.net/category/podcast/feed
- Licence:
- Copyright:
- **PseudoPod**
- Website: https://pseudopod.org
- Feed: http://pseudopod.org/feed/
- Licence:
- Copyright:
- **RadioTux**
- Website: https://www.radiotux.de/
- Feed: http://radiotux.de/podcast/rss/radiotux-all.xml
- Licence: © Creative Commons - BY-SA 3.0 Unported
- Copyright: © Creative Commons - BY-SA 3.0 Unported
- **RatholeRadio.org (Ogg Version)**
- Website: https://ratholeradio.org
- Feed: http://feeds.feedburner.com/RatholeRadio-ogg
- Licence: Creative Commons BY-SA Licensed
- Copyright: Creative Commons BY-SA Licensed
- **Science for the People**
- Website: http://www.scienceforthepeople.ca/
- Feed: http://feeds.feedburner.com/SkepticallySpeaking
- Copyright: Copyright now Science for the People
- **Security Now (Audio)**
- Website: https://twit.tv/shows/security-now
- Feed: http://feeds.twit.tv/sn.xml
- Licence: This work is licensed under a Creative Commons License - Attribution-NonCommercial-NoDerivatives 4.0 International - http://creativecommons.org/licenses/by-nc-nd/4.0/
- Copyright: This work is licensed under a Creative Commons License - Attribution-NonCommercial-NoDerivatives 4.0 International - http://creativecommons.org/licenses/by-nc-nd/4.0/
- **Skepticule**
- Website: http://www.skepticule.co.uk/
- Feed: http://www.skepticule.co.uk/feeds/posts/default?alt=rss
- Licence: Creative Commons Attribution-NonCommercial-No Derivative Works 2.0 UK: England & Wales
- Copyright: Creative Commons Attribution-NonCommercial-No Derivative Works 2.0 UK: England & Wales
- **Software Freedom Podcast**
- Website: https://fsfe.org/news/podcast
- Feed: http://fsfe.org/news/podcast.en.rss
- Licence: Copyright (c) Free Software Foundation Europe. Creative Commons BY-SA 4.0
- Copyright: Copyright (c) Free Software Foundation Europe. Creative Commons BY-SA 4.0
- **Software Freedom Podcast**
- Website: https://fsfe.org/news/podcast
- Feed: http://fsfe.org/news/podcast-opus.en.rss
- Licence: Copyright (c) Free Software Foundation Europe. Creative Commons BY-SA 4.0
- Copyright: Copyright (c) Free Software Foundation Europe. Creative Commons BY-SA 4.0
- **Tea, Earl Grey, Hot !**
- Website: https://teaearlgreyhot.org/
- Feed: https://teaearlgreyhot.org/feed/podcast
- Licence: Unless otherwise stated, this podcast is released under a Creative Commons, By Attribution, Share Alike license.
- Copyright: Unless otherwise stated, this podcast is released under a Creative Commons, By Attribution, Share Alike license.
- **The Big Slurp**
- Website: https://thelovebug.org/series/slurp/
- Feed: https://thelovebug.org/feed/podcast/slurp
- Licence: CC BY-NC-SA 4.0 Attribution-NonCommercial-ShareAlike 4.0 International
- Copyright: CC BY-NC-SA 4.0 Attribution-NonCommercial-ShareAlike 4.0 International
- **The Binary Times Audiocast - ogg**
- Website: https://www.thebinarytimes.net
- Feed: https://www.thebinarytimes.net/rss-ogg.xml
- Licence: Unless otherwise stated, this podcast is released under a Creative Commons, By Attribution, Share Alike license.
- Copyright: Unless otherwise stated, this podcast is released under a Creative Commons, By Attribution, Share Alike license.
- **The Bugcast - Ogg Feed**
- Website: https://thebugcast.org/
- Feed: http://thebugcast.org/feed/ogg/
- Licence:
- Copyright:
- **The Dub Zone**
- Website: http://petecogle.co.uk/category/the-dub-zone/
- Feed: http://feeds.feedburner.com/TheDubZone
- Licence: Creative Commons
- Copyright: Creative Commons
- **The Duffercast in Ogg Vorbis**
- Website: https://duffercast.org/
- Feed: http://duffercast.org/feed/podcast/
- Licence:
- Copyright:
- **The Full Circle Weekly News**
- Website: https://fullcirclemagazine.org/
- Feed: https://fullcirclemagazine.org/feed/podcast
- Licence: CC-SA 2016-present, Full Circle Magazine
- Copyright: CC-SA 2016-present, Full Circle Magazine
- **The Jodcast (high bandwidth)**
- Website: http://www.jodcast.net/
- Feed: http://www.jodcast.net/rss-high.xml
- Licence: Licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 2.0 England & Wales Licence
- Copyright: Licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 2.0 England & Wales Licence
- **The Linux Action Show! OGG**
- Website: http://www.jupiterbroadcasting.com
- Feed: http://feeds2.feedburner.com/TheLinuxActionShowOGG
- Licence:
- Copyright:
- **The Linux Cast**
- Website: https://thelinuxcast.org/
- Feed: https://thelinuxcast.org/feed/feed.xml
- Licence:
- Copyright:
- **The Linux Link Tech Show Ogg-Vorbis Feed**
- Website: http://www.tllts.org
- Feed: http://feeds.feedburner.com/TheLinuxLinkTechShowOgg-vorbisFeed
- Licence: TLLTS is licensed under Creative Commons License for Non-Commercial Use
- Copyright: TLLTS is licensed under Creative Commons License for Non-Commercial Use
- **The Linux Link Tech Show Ogg-Vorbis Feed**
- Website: http://www.tllts.org
- Feed: http://www.thelinuxlink.net/tllts/tllts_ogg.rss
- Licence: TLLTS is licensed under Creative Commons License for Non-Commercial Use
- Copyright: TLLTS is licensed under Creative Commons License for Non-Commercial Use
- **This Week in Computer Hardware (Audio)**
- Website: https://twit.tv/shows/this-week-in-computer-hardware
- Feed: http://feeds.twit.tv/twich.xml
- Licence: This work is licensed under a Creative Commons License - Attribution-NonCommercial-NoDerivatives 4.0 International - http://creativecommons.org/licenses/by-nc-nd/4.0/
- Copyright: This work is licensed under a Creative Commons License - Attribution-NonCommercial-NoDerivatives 4.0 International - http://creativecommons.org/licenses/by-nc-nd/4.0/
- **This Week in Google (Audio)**
- Website: https://twit.tv/shows/this-week-in-google
- Feed: http://feeds.twit.tv/twig.xml
- Licence: This work is licensed under a Creative Commons License - Attribution-NonCommercial-NoDerivatives 4.0 International - http://creativecommons.org/licenses/by-nc-nd/4.0/
- Copyright: This work is licensed under a Creative Commons License - Attribution-NonCommercial-NoDerivatives 4.0 International - http://creativecommons.org/licenses/by-nc-nd/4.0/
- **This Week in Microbiology**
- Website: http://www.asm.org/twim/
- Feed: http://feeds.feedburner.com/twim
- Licence: Creative Commons Attribution - Noncommercial
- Copyright: Creative Commons Attribution - Noncommercial
- **This Week in Tech (Audio)**
- Website: https://twit.tv/shows/this-week-in-tech
- Feed: http://feeds.twit.tv/twit.xml
- Licence: This work is licensed under a Creative Commons License - Attribution-NonCommercial-NoDerivatives 4.0 International - http://creativecommons.org/licenses/by-nc-nd/4.0/
- Copyright: This work is licensed under a Creative Commons License - Attribution-NonCommercial-NoDerivatives 4.0 International - http://creativecommons.org/licenses/by-nc-nd/4.0/
- **TuxJam OGG**
- Website: https://tuxjam.otherside.network/
- Feed: https://tuxjam.otherside.network/?feed=podcast
- Licence: Creative Commons Attribution-ShareAlike 3.0 Unported (CC BY-SA 3.0)
- Copyright: Creative Commons Attribution-ShareAlike 3.0 Unported (CC BY-SA 3.0)
- **Ubuntu Podcast**
- Website: https://ubuntupodcast.org
- Feed: http://ubuntupodcast.org/feed/
- Licence:
- Copyright:
- **Ubuntu Podcast**
- Website: https://ubuntupodcast.org
- Feed: http://feed.ubuntupodcast.org/mp3
- Licence: Copyright Ubuntu Podcast Team. Some rights reserved. This audio is released under a Creative Commons Attribution-Share Alike license.
- Copyright: Copyright Ubuntu Podcast Team. Some rights reserved. This audio is released under a Creative Commons Attribution-Share Alike license.
- **Wikipediapodden**
- Website: http://wikipediapodden.se/prenumerera/
- Feed: http://wikipediapodden.se/feed/podcast/
- Licence: CC BY-SA 4.0 Wikipediapodden
- Copyright: CC BY-SA 4.0 Wikipediapodden
- **bsdtalk**
- Website: http://bsdtalk.blogspot.com/
- Feed: http://feeds.feedburner.com/Bsdtalk
- Licence:
- Copyright:
- **podcast – Full Circle Magazine**
- Website: https://fullcirclemagazine.org
- Feed: http://fullcirclemagazine.org/category/podcast/feed/
- Licence:
- Copyright:
- **urandom podcast**
- Website: https://urandom-podcast.info/
- Feed: http://feeds.feedburner.com/urandom-podcast/ogg
- Licence:
- Copyright:

7
feedWatcher.opml
View File

@ -2,8 +2,8 @@
<opml version="1.1">
<head>
<title>Free Culture Podcasts</title>
<dateCreated>2023-01-11 09:44:06</dateCreated>
<dateModified>2023-01-11 09:44:06</dateModified>
<dateCreated>2023-01-14 23:09:26</dateCreated>
<dateModified>2023-01-14 23:09:26</dateModified>
<ownerName></ownerName>
<ownerEmail></ownerEmail>
<expansionState></expansionState>
@ -14,6 +14,7 @@
<windowRight></windowRight>
</head>
<body>
<outline description="2.5 Admins is a podcast featuring two sysadmins called Allan Jude and Jim Salter, and a producer/editor who can just about configure a Samba share called Joe Ressington. Every two weeks we get together, talk about recent tech news, and answer some of your admin-related questions." htmlUrl="https://2.5admins.com" text="2.5 Admins" title="2.5 Admins" xmlUrl="https://2.5admins.com/feed/podcast" />
<outline description="A Podcast about servers and Networking" htmlUrl="https://www.adminadminpodcast.co.uk" text="Admin Admin Podcast" title="Admin Admin Podcast" xmlUrl="http://feeds.feedburner.com/TheAdminAdminPodcast" />
<outline description="All About Android delivers everything you want to know about Android each week--the biggest news, freshest hardware, best apps and geekiest how-tos--with Android enthusiasts Jason Howell, Ron Richards, Huyen Tue Dao, and a variety of special guests along the way. Records live every Tuesday at 8:00pm Eastern / 5:00pm Pacific / 01:00 (Wed) UTC." htmlUrl="https://twit.tv/shows/all-about-android" text="All About Android (Audio)" title="All About Android (Audio)" xmlUrl="http://feeds.twit.tv/aaa.xml" />
<outline description="Join us as we talk about everything related to Information Technology, and some other random stuff as well." htmlUrl="http://aiit.se/radio/" text="All In IT Radio (ogg)" title="All In IT Radio (ogg)" xmlUrl="http://feeds.aiit.se/allinit-radio-ogg" />
@ -59,6 +60,7 @@
<outline description="New media, new rules" htmlUrl="http://sixgun.org" text="Linux Outlaws" title="Linux Outlaws" xmlUrl="http://feeds.feedburner.com/linuxoutlaws" />
<outline description="A podcast for the new Linux user" htmlUrl="http://www.linuxreality.com" text="Linux Reality Podcast (MP3 Feed)" title="Linux Reality Podcast (MP3 Feed)" xmlUrl="http://feeds.feedburner.com/linuxreality" />
<outline description="Verbal's Linux Trivia Podcast" htmlUrl="http://setbit.org/lt.html" text="Linux Trivia Podcast" title="Linux Trivia Podcast" xmlUrl="http://setbit.org/lt-ogg.xml" />
<outline description="How did your favorite Linux distribution get its start? Join us and find out! Linux User Space is hosted by Leo and Dan, and every two weeks we deep dive into the history of Linux distributions and the things that matter to us. Episodes drop every other Monday." htmlUrl="https://www.linuxuserspace.show" text="Linux User Space" title="Linux User Space" xmlUrl="https://www.linuxuserspace.show/rss" />
<outline description="The Linux at Work podcast provides info and tips for people interested in using Linux in a professional environment. Featuring @chetwisniewski, @john_shier and @0xbennyv" htmlUrl="https://www.linuxatwork.org" text="Linux at Work" title="Linux at Work" xmlUrl="http://feeds.podtrac.com/mBdfP0QTX0iY" />
<outline description="Linux in the Ham Shack Podcast in MP3 Format" htmlUrl="https://lhspodcast.info/category/podcast-mp3/" text="Linux in the Ham Shack" title="Linux in the Ham Shack" xmlUrl="http://lhspodcast.info/category/podcast-mp3/feed/" />
<outline description="Linux in the Ham Shack Podcast in OGG Format" htmlUrl="https://lhspodcast.info/category/podcast-ogg/" text="Linux in the Ham Shack (OGG Feed)" title="Linux in the Ham Shack (OGG Feed)" xmlUrl="https://lhspodcast.info/category/podcast-ogg/feed/" />
@ -77,6 +79,7 @@
<outline description="The Sound of Horror" htmlUrl="https://pseudopod.org" text="PseudoPod" title="PseudoPod" xmlUrl="http://pseudopod.org/feed/" />
<outline description="Linux, Open Source und Netzkultur" htmlUrl="https://www.radiotux.de/" text="RadioTux" title="RadioTux" xmlUrl="http://radiotux.de/podcast/rss/radiotux-all.xml" />
<outline description="Music, Waffling, Live Performances &amp; Laughs" htmlUrl="https://ratholeradio.org" text="RatholeRadio.org (Ogg Version)" title="RatholeRadio.org (Ogg Version)" xmlUrl="http://feeds.feedburner.com/RatholeRadio-ogg" />
<outline description="Science for the People is a long-format interview podcast that explores the connections between science, popular culture, history, and public policy, to help listeners understand the evidence and arguments behind what's in the news and on the shelves. Our hosts sit down with science researchers, writers, authors, journalists, and experts to discuss science from the past, the science that affects our lives today, and how science might change our future." htmlUrl="http://www.scienceforthepeople.ca/" text="Science for the People" title="Science for the People" xmlUrl="http://feeds.feedburner.com/SkepticallySpeaking" />
<outline description="Steve Gibson, the man who coined the term spyware and created the first anti-spyware program, creator of SpinRite and ShieldsUP, discusses the hot topics in security today with Leo Laporte. Records live every Tuesday at 4:30pm Eastern / 1:30pm Pacific / 21:30 UTC." htmlUrl="https://twit.tv/shows/security-now" text="Security Now (Audio)" title="Security Now (Audio)" xmlUrl="http://feeds.twit.tv/sn.xml" />
<outline description="Spanking the bottom of ignorance since 2009" htmlUrl="http://www.skepticule.co.uk/" text="Skepticule" title="Skepticule" xmlUrl="http://www.skepticule.co.uk/feeds/posts/default?alt=rss" />
<outline description="The regular podcast about Free Software and ongoing activities hosted by the FSFE" htmlUrl="https://fsfe.org/news/podcast" text="Software Freedom Podcast" title="Software Freedom Podcast" xmlUrl="http://fsfe.org/news/podcast.en.rss" />

BIN
feedWatcher.pdf
View File

Binary file not shown.

2
feedWatcher_3.tpl
View File

@ -10,7 +10,7 @@
- **[% feeds.$i.urls_title %]**
- Website: [% feeds.$i.urls_link %]
- Feed: [% feeds.$i.urls_url %]
- Licence: [% feeds.$i.urls_copyright %]
- Copyright: [% feeds.$i.urls_copyright %]
[% i = i + 1 -%]
[% END -%]

1
feedWatcher_5.tpl
View File

@ -1,4 +1,5 @@
[%# feedWatcher_5.tpl 2022-11-21 -%]
[%# Lists all the URLs in the system, useful for dumping everything -%]
[% IF feeds.size > 0 -%]
[% i = 0 -%]
[% WHILE i < feeds.size -%]

4
feedWatcher_schema.sql
View File

@ -29,6 +29,8 @@ CREATE TABLE urls (
link varchar(1024),
image varchar(1024),
copyright varchar(80),
check_type varchar(10) DEFAULT 'none',
reason_accepted text,
generator varchar(80),
language varchar(40),
parent_id integer
@ -131,6 +133,8 @@ CREATE VIEW all_episodes AS
urls.link as urls_link,
urls.image as urls_image,
urls.copyright as urls_copyright,
urls.check_type as urls_check_type,
urls.reason_accepted as urls_reason_accepted,
urls.generator as urls_generator,
urls.language as urls_language,
urls.parent_id as urls_parent_id,