Do not count placeholders in comments

Database/query2csv, Database/query2json, Database/query2tt2:
    if the query contained comments with placeholders they would be
    added to the count and would usually cause the script to request
    '-dbarg=ARG' values be added. Now the SQL has all comments
    stripped to avoid this.
This commit is contained in:
Dave Morriss 2025-05-25 18:56:33 +01:00
parent b84ff7a4c8
commit b514cfa380
3 changed files with 221 additions and 83 deletions

View File

@ -19,9 +19,9 @@
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.4
# VERSION: 0.0.5
# CREATED: 2015-07-11 15:53:01
# REVISION: 2025-05-09 14:13:17
# REVISION: 2025-05-25 18:26:13
#
#===============================================================================
@ -46,7 +46,7 @@ use Data::Dumper;
#
# Version number (manually incremented)
#
our $VERSION = '0.0.4';
our $VERSION = '0.0.5';
#
# Script name
@ -135,11 +135,16 @@ else {
}
_debug( $DEBUG >= 3, '$query: ' . Dumper(\$query) );
#
# Strip SQL comments
#
$query = strip_sql_comments($query);
#
# Count placeholders in the query and the arguments provided. First remove all
# comments which may contain '?' characters, then count any that are left.
#
$query = join("\n", grep {!/^--/} split( "\n", $query ) );
#$query = join("\n", grep {!/^--/} split( "\n", $query ) );
$pcount = grep {/\?/} split( '', $query );
$acount = scalar(@dbargs);
@ -173,7 +178,7 @@ else {
# Load database configuration data; allow environment variables
#-------------------------------------------------------------------------------
my $conf = Config::General->new(
-ConfigFile => $configfile,
-ConfigFile => $cfgfile,
-InterPolateVars => 1,
-InterPolateEnv => 1,
-ExtendedAccess => 1
@ -181,7 +186,7 @@ my $conf = Config::General->new(
my %config = $conf->getall();
#
# Set defaults
# Set defaults in case values have been omitted
#
$config{database}->{dbtype} //= 'SQLite';
$config{database}->{host} //= '127.0.0.1';
@ -263,13 +268,13 @@ exit;
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub db_connect {
sub db_connect { #{{{
my ($cfg) = @_;
my ( $dbh, $dbtype, $dbname );
$dbtype = $config{database}->{dbtype};
$dbname = $config{database}->{name};
$dbtype = $cfg->{database}->{dbtype};
$dbname = $cfg->{database}->{name};
die "Database name is mandatory\n" unless $dbname;
#
@ -280,6 +285,7 @@ sub db_connect {
# The name for the SQLite driver is 'DBD:SQLite'
#
$dbtype = 'SQLite';
_debug( $DEBUG >= 3, '$dbtype: ' . $dbtype, '$dbname: ' . $dbname );
$dbh = DBI->connect( "DBI:$dbtype:dbname=$dbname",
"", "", { AutoCommit => 1, sqlite_unicode => 1, } )
@ -291,10 +297,10 @@ sub db_connect {
#
$dbtype = 'mysql';
my $dbhost = $config{database}->{host};
my $dbport = $config{database}->{port};
my $dbuser = $config{database}->{user};
my $dbpwd = $config{database}->{password};
my $dbhost = $cfg->{database}->{host};
my $dbport = $cfg->{database}->{port};
my $dbuser = $cfg->{database}->{user};
my $dbpwd = $cfg->{database}->{password};
$dbh = DBI->connect( "DBI:$dbtype:host=$dbhost;port=$dbport;database=$dbname",
$dbuser, $dbpwd, { AutoCommit => 1 } )
@ -320,28 +326,68 @@ sub db_connect {
}
return $dbh;
}
} #}}}
#=== FUNCTION ================================================================
# NAME: strip_sql_comments
# PURPOSE: Given a query as a scalar, strips all SQL comments
# PARAMETERS: $query string containing a query
# RETURNS: Stripped string
# DESCRIPTION: Two types of comments might exist in the query: the C-style
# and the SQL style. The string is treated as a single string
# even though it's multi-line, and any C-style comments are
# removed. Then the string is treated as multi-line and each
# line is scanned for SQL comments (which end at the end of the
# line), and these are stripped. Blank lines are skipped too to
# compress the output a little.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub strip_sql_comments { #{{{
my ($query) = @_;
my $result;
#
# Strip C-style comments
#
$query =~ s/\/\*.*?\*\///sg;
#
# Strip SQL line-oriented comments
#
foreach my $line (split(/\n/,$query)) {
next if $line =~ /^\s*$/;
$line =~ s/--.*$//;
$result .= "$line\n";
}
return $result;
} #}}}
#=== FUNCTION ================================================================
# NAME: _debug
# PURPOSE: Prints debug reports
# PARAMETERS: $active Boolean: 1 for print, 0 for no print
# $message Message to print
# @messages Messages to print
# RETURNS: Nothing
# DESCRIPTION: Outputs a message if $active is true. It removes any trailing
# newline and then adds one in the 'print' to the caller doesn't
# have to bother. Prepends the message with 'D> ' to show it's
# DESCRIPTION: Outputs messages if $active is true. It removes any trailing
# newlines and then adds one to each line so the caller doesn't
# have to bother. Prepends 'D> ' to each message to show it's
# a debug message.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub _debug {
my ( $active, $message ) = @_;
sub _debug { #{{{
my ( $active, @messages ) = @_;
chomp($message);
print STDERR "D> $message\n" if $active;
}
if ($active) {
chomp(@messages);
say STDERR "D> ", join( "\nD> ", @messages );
}
} #}}}
#=== FUNCTION ================================================================
# NAME: _dbargs
@ -354,7 +400,7 @@ sub _debug {
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub _dbargs {
sub _dbargs { #{{{
my ($opts) = @_;
my @args;
@ -364,7 +410,7 @@ sub _dbargs {
}
return (@args);
}
} #}}}
#=== FUNCTION ================================================================
# NAME: Options
@ -376,7 +422,7 @@ sub _dbargs {
# COMMENTS: none
# SEE ALSO: n/a
#===============================================================================
sub Options {
sub Options { #{{{
my ($optref) = @_;
my @options = (
@ -389,7 +435,7 @@ sub Options {
}
return;
}
} #}}}
__END__
@ -404,7 +450,7 @@ query2csv - A script for generating CSV from database query
=head1 VERSION
This documentation refers to query2csv version 0.0.4
This documentation refers to query2csv version 0.0.5
=head1 USAGE

View File

@ -18,9 +18,9 @@
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.3
# VERSION: 0.0.4
# CREATED: 2021-06-18 13:24:49
# REVISION: 2025-05-09 14:13:27
# REVISION: 2025-05-25 18:48:59
#
#===============================================================================
@ -45,7 +45,7 @@ use Data::Dumper;
#
# Version number (manually incremented)
#
our $VERSION = '0.0.3';
our $VERSION = '0.0.4';
#
# Script name
@ -131,11 +131,16 @@ else {
}
_debug( $DEBUG >= 3, '$query: ' . Dumper(\$query) );
#
# Strip SQL comments
#
$query = strip_sql_comments($query);
#
# Count placeholders in the query and the arguments provided. First remove all
# comments which may contain '?' characters, then count any that are left.
#
$query = join("\n", grep {!/^--/} split( "\n", $query ) );
#$query = join("\n", grep {!/^--/} split( "\n", $query ) );
$pcount = grep {/\?/} split( '', $query );
$acount = scalar(@dbargs);
@ -177,7 +182,7 @@ my $conf = Config::General->new(
my %config = $conf->getall();
#
# Set defaults
# Set defaults in case values have been omitted
#
$config{database}->{dbtype} //= 'SQLite';
$config{database}->{host} //= '127.0.0.1';
@ -247,13 +252,13 @@ exit;
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub db_connect {
sub db_connect { #{{{
my ($cfg) = @_;
my ( $dbh, $dbtype, $dbname );
$dbtype = $config{database}->{dbtype};
$dbname = $config{database}->{name};
$dbtype = $cfg->{database}->{dbtype};
$dbname = $cfg->{database}->{name};
die "Database name is mandatory\n" unless $dbname;
#
@ -264,6 +269,7 @@ sub db_connect {
# The name for the SQLite driver is 'DBD:SQLite'
#
$dbtype = 'SQLite';
_debug( $DEBUG >= 3, '$dbtype: ' . $dbtype, '$dbname: ' . $dbname );
$dbh = DBI->connect( "DBI:$dbtype:dbname=$dbname",
"", "", { AutoCommit => 1, sqlite_unicode => 1, } )
@ -275,10 +281,10 @@ sub db_connect {
#
$dbtype = 'mysql';
my $dbhost = $config{database}->{host};
my $dbport = $config{database}->{port};
my $dbuser = $config{database}->{user};
my $dbpwd = $config{database}->{password};
my $dbhost = $cfg->{database}->{host};
my $dbport = $cfg->{database}->{port};
my $dbuser = $cfg->{database}->{user};
my $dbpwd = $cfg->{database}->{password};
$dbh = DBI->connect( "DBI:$dbtype:host=$dbhost;port=$dbport;database=$dbname",
$dbuser, $dbpwd, { AutoCommit => 1 } )
@ -304,28 +310,68 @@ sub db_connect {
}
return $dbh;
}
} #}}}
#=== FUNCTION ================================================================
# NAME: strip_sql_comments
# PURPOSE: Given a query as a scalar, strips all SQL comments
# PARAMETERS: $query string containing a query
# RETURNS: Stripped string
# DESCRIPTION: Two types of comments might exist in the query: the C-style
# and the SQL style. The string is treated as a single string
# even though it's multi-line, and any C-style comments are
# removed. Then the string is treated as multi-line and each
# line is scanned for SQL comments (which end at the end of the
# line), and these are stripped. Blank lines are skipped too to
# compress the output a little.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub strip_sql_comments { #{{{
my ($query) = @_;
my $result;
#
# Strip C-style comments
#
$query =~ s/\/\*.*?\*\///sg;
#
# Strip SQL line-oriented comments
#
foreach my $line (split(/\n/,$query)) {
next if $line =~ /^\s*$/;
$line =~ s/--.*$//;
$result .= "$line\n";
}
return $result;
} #}}}
#=== FUNCTION ================================================================
# NAME: _debug
# PURPOSE: Prints debug reports
# PARAMETERS: $active Boolean: 1 for print, 0 for no print
# $message Message to print
# @messages Messages to print
# RETURNS: Nothing
# DESCRIPTION: Outputs a message if $active is true. It removes any trailing
# newline and then adds one in the 'print' to the caller doesn't
# have to bother. Prepends the message with 'D> ' to show it's
# DESCRIPTION: Outputs messages if $active is true. It removes any trailing
# newlines and then adds one to each line so the caller doesn't
# have to bother. Prepends 'D> ' to each message to show it's
# a debug message.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub _debug {
my ( $active, $message ) = @_;
sub _debug { #{{{
my ( $active, @messages ) = @_;
chomp($message);
print STDERR "D> $message\n" if $active;
}
if ($active) {
chomp(@messages);
say STDERR "D> ", join( "\nD> ", @messages );
}
} #}}}
#=== FUNCTION ================================================================
# NAME: _dbargs
@ -338,7 +384,7 @@ sub _debug {
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub _dbargs {
sub _dbargs { #{{{
my ($opts) = @_;
my @args;
@ -348,7 +394,7 @@ sub _dbargs {
}
return (@args);
}
} #}}}
#=== FUNCTION ================================================================
# NAME: Options
@ -360,7 +406,7 @@ sub _dbargs {
# COMMENTS: none
# SEE ALSO: n/a
#===============================================================================
sub Options {
sub Options { #{{{
my ($optref) = @_;
my @options = (
@ -373,7 +419,7 @@ sub Options {
}
return;
}
} #}}}
__END__
@ -388,7 +434,7 @@ query2json - A script for generating CSV from database query
=head1 VERSION
This documentation refers to query2json version 0.0.3
This documentation refers to query2json version 0.0.4
=head1 USAGE

View File

@ -26,9 +26,9 @@
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# VERSION: 0.0.8
# VERSION: 0.0.9
# CREATED: 2021-06-18 13:24:49
# REVISION: 2025-05-09 14:13:04
# REVISION: 2025-05-25 18:41:09
#
#===============================================================================
@ -54,7 +54,7 @@ use Data::Dumper;
#
# Version number (manually incremented)
#
our $VERSION = '0.0.8';
our $VERSION = '0.0.9';
#
# Script and directory names
@ -162,11 +162,16 @@ else {
}
_debug( $DEBUG >= 3, '$query: ' . Dumper(\$query) );
#
# Strip SQL comments
#
$query = strip_sql_comments($query);
#
# Count placeholders in the query and the arguments provided. First remove all
# comments which may contain '?' characters, then count any that are left.
#
$query = join("\n", grep {!/^--/} split( "\n", $query ) );
#$query = join("\n", grep {!/^--/} split( "\n", $query ) );
$pcount = grep {/\?/} split( '', $query );
$acount = scalar(@dbargs);
@ -323,13 +328,13 @@ exit;
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub db_connect {
sub db_connect { #{{{
my ($cfg) = @_;
my ( $dbh, $dbtype, $dbname );
$dbtype = $config{database}->{dbtype};
$dbname = $config{database}->{name};
$dbtype = $cfg->{database}->{dbtype};
$dbname = $cfg->{database}->{name};
die "Database name is mandatory\n" unless $dbname;
#
@ -340,6 +345,7 @@ sub db_connect {
# The name for the SQLite driver is 'DBD:SQLite'
#
$dbtype = 'SQLite';
_debug( $DEBUG >= 3, '$dbtype: ' . $dbtype, '$dbname: ' . $dbname );
$dbh = DBI->connect( "DBI:$dbtype:dbname=$dbname",
"", "", { AutoCommit => 1, sqlite_unicode => 1, } )
@ -351,10 +357,10 @@ sub db_connect {
#
$dbtype = 'mysql';
my $dbhost = $config{database}->{host};
my $dbport = $config{database}->{port};
my $dbuser = $config{database}->{user};
my $dbpwd = $config{database}->{password};
my $dbhost = $cfg->{database}->{host};
my $dbport = $cfg->{database}->{port};
my $dbuser = $cfg->{database}->{user};
my $dbpwd = $cfg->{database}->{password};
$dbh = DBI->connect( "DBI:$dbtype:host=$dbhost;port=$dbport;database=$dbname",
$dbuser, $dbpwd, { AutoCommit => 1 } )
@ -380,28 +386,68 @@ sub db_connect {
}
return $dbh;
}
} #}}}
#=== FUNCTION ================================================================
# NAME: strip_sql_comments
# PURPOSE: Given a query as a scalar, strips all SQL comments
# PARAMETERS: $query string containing a query
# RETURNS: Stripped string
# DESCRIPTION: Two types of comments might exist in the query: the C-style
# and the SQL style. The string is treated as a single string
# even though it's multi-line, and any C-style comments are
# removed. Then the string is treated as multi-line and each
# line is scanned for SQL comments (which end at the end of the
# line), and these are stripped. Blank lines are skipped too to
# compress the output a little.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub strip_sql_comments { #{{{
my ($query) = @_;
my $result;
#
# Strip C-style comments
#
$query =~ s/\/\*.*?\*\///sg;
#
# Strip SQL line-oriented comments
#
foreach my $line (split(/\n/,$query)) {
next if $line =~ /^\s*$/;
$line =~ s/--.*$//;
$result .= "$line\n";
}
return $result;
} #}}}
#=== FUNCTION ================================================================
# NAME: _debug
# PURPOSE: Prints debug reports
# PARAMETERS: $active Boolean: 1 for print, 0 for no print
# $message Message to print
# @messages Messages to print
# RETURNS: Nothing
# DESCRIPTION: Outputs a message if $active is true. It removes any trailing
# newline and then adds one in the 'print' to the caller doesn't
# have to bother. Prepends the message with 'D> ' to show it's
# DESCRIPTION: Outputs messages if $active is true. It removes any trailing
# newlines and then adds one to each line so the caller doesn't
# have to bother. Prepends 'D> ' to each message to show it's
# a debug message.
# THROWS: No exceptions
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub _debug {
my ( $active, $message ) = @_;
sub _debug { #{{{
my ( $active, @messages ) = @_;
chomp($message);
print STDERR "D> $message\n" if $active;
}
if ($active) {
chomp(@messages);
say STDERR "D> ", join( "\nD> ", @messages );
}
} #}}}
#=== FUNCTION ================================================================
# NAME: _dbargs
@ -414,7 +460,7 @@ sub _debug {
# COMMENTS: None
# SEE ALSO: N/A
#===============================================================================
sub _dbargs {
sub _dbargs { #{{{
my ($opts) = @_;
my @args;
@ -424,7 +470,7 @@ sub _dbargs {
}
return (@args);
}
} #}}}
#=== FUNCTION ================================================================
# NAME: _define
@ -439,7 +485,7 @@ sub _dbargs {
# COMMENTS: None
# SEE ALSO:
#===============================================================================
sub _define {
sub _define { #{{{
my ($opts) = @_;
my %defs;
@ -449,7 +495,7 @@ sub _define {
}
return (%defs);
}
} #}}}
#=== FUNCTION ================================================================
# NAME: Options
@ -461,7 +507,7 @@ sub _define {
# COMMENTS: none
# SEE ALSO: n/a
#===============================================================================
sub Options {
sub Options { #{{{
my ($optref) = @_;
my @options = (
@ -477,7 +523,7 @@ sub Options {
}
return;
}
} #}}}
__END__
@ -492,7 +538,7 @@ query2tt2 - A script for formatting a report from database query using a templat
=head1 VERSION
This documentation refers to query2tt2 version 0.0.8
This documentation refers to query2tt2 version 0.0.9
=head1 USAGE