From b514cfa380cf01305a813777b980737d84e2f992 Mon Sep 17 00:00:00 2001 From: Dave Morriss Date: Sun, 25 May 2025 18:56:33 +0100 Subject: [PATCH] Do not count placeholders in comments Database/query2csv, Database/query2json, Database/query2tt2: if the query contained comments with placeholders they would be added to the count and would usually cause the script to request '-dbarg=ARG' values be added. Now the SQL has all comments stripped to avoid this. --- Database/query2csv | 102 ++++++++++++++++++++++++++++++++------------ Database/query2json | 100 +++++++++++++++++++++++++++++++------------ Database/query2tt2 | 102 ++++++++++++++++++++++++++++++++------------ 3 files changed, 221 insertions(+), 83 deletions(-) diff --git a/Database/query2csv b/Database/query2csv index 0d3b19f..f05f31a 100755 --- a/Database/query2csv +++ b/Database/query2csv @@ -19,9 +19,9 @@ # BUGS: --- # NOTES: --- # AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com -# VERSION: 0.0.4 +# VERSION: 0.0.5 # CREATED: 2015-07-11 15:53:01 -# REVISION: 2025-05-09 14:13:17 +# REVISION: 2025-05-25 18:26:13 # #=============================================================================== @@ -46,7 +46,7 @@ use Data::Dumper; # # Version number (manually incremented) # -our $VERSION = '0.0.4'; +our $VERSION = '0.0.5'; # # Script name @@ -135,11 +135,16 @@ else { } _debug( $DEBUG >= 3, '$query: ' . Dumper(\$query) ); +# +# Strip SQL comments +# +$query = strip_sql_comments($query); + # # Count placeholders in the query and the arguments provided. First remove all # comments which may contain '?' characters, then count any that are left. # -$query = join("\n", grep {!/^--/} split( "\n", $query ) ); +#$query = join("\n", grep {!/^--/} split( "\n", $query ) ); $pcount = grep {/\?/} split( '', $query ); $acount = scalar(@dbargs); @@ -173,7 +178,7 @@ else { # Load database configuration data; allow environment variables #------------------------------------------------------------------------------- my $conf = Config::General->new( - -ConfigFile => $configfile, + -ConfigFile => $cfgfile, -InterPolateVars => 1, -InterPolateEnv => 1, -ExtendedAccess => 1 @@ -181,7 +186,7 @@ my $conf = Config::General->new( my %config = $conf->getall(); # -# Set defaults +# Set defaults in case values have been omitted # $config{database}->{dbtype} //= 'SQLite'; $config{database}->{host} //= '127.0.0.1'; @@ -263,13 +268,13 @@ exit; # COMMENTS: None # SEE ALSO: N/A #=============================================================================== -sub db_connect { +sub db_connect { #{{{ my ($cfg) = @_; my ( $dbh, $dbtype, $dbname ); - $dbtype = $config{database}->{dbtype}; - $dbname = $config{database}->{name}; + $dbtype = $cfg->{database}->{dbtype}; + $dbname = $cfg->{database}->{name}; die "Database name is mandatory\n" unless $dbname; # @@ -280,6 +285,7 @@ sub db_connect { # The name for the SQLite driver is 'DBD:SQLite' # $dbtype = 'SQLite'; + _debug( $DEBUG >= 3, '$dbtype: ' . $dbtype, '$dbname: ' . $dbname ); $dbh = DBI->connect( "DBI:$dbtype:dbname=$dbname", "", "", { AutoCommit => 1, sqlite_unicode => 1, } ) @@ -291,10 +297,10 @@ sub db_connect { # $dbtype = 'mysql'; - my $dbhost = $config{database}->{host}; - my $dbport = $config{database}->{port}; - my $dbuser = $config{database}->{user}; - my $dbpwd = $config{database}->{password}; + my $dbhost = $cfg->{database}->{host}; + my $dbport = $cfg->{database}->{port}; + my $dbuser = $cfg->{database}->{user}; + my $dbpwd = $cfg->{database}->{password}; $dbh = DBI->connect( "DBI:$dbtype:host=$dbhost;port=$dbport;database=$dbname", $dbuser, $dbpwd, { AutoCommit => 1 } ) @@ -320,28 +326,68 @@ sub db_connect { } return $dbh; -} +} #}}} + +#=== FUNCTION ================================================================ +# NAME: strip_sql_comments +# PURPOSE: Given a query as a scalar, strips all SQL comments +# PARAMETERS: $query string containing a query +# RETURNS: Stripped string +# DESCRIPTION: Two types of comments might exist in the query: the C-style +# and the SQL style. The string is treated as a single string +# even though it's multi-line, and any C-style comments are +# removed. Then the string is treated as multi-line and each +# line is scanned for SQL comments (which end at the end of the +# line), and these are stripped. Blank lines are skipped too to +# compress the output a little. +# THROWS: No exceptions +# COMMENTS: None +# SEE ALSO: N/A +#=============================================================================== +sub strip_sql_comments { #{{{ + my ($query) = @_; + + my $result; + + # + # Strip C-style comments + # + $query =~ s/\/\*.*?\*\///sg; + + # + # Strip SQL line-oriented comments + # + foreach my $line (split(/\n/,$query)) { + next if $line =~ /^\s*$/; + $line =~ s/--.*$//; + $result .= "$line\n"; + } + + return $result; +} #}}} #=== FUNCTION ================================================================ # NAME: _debug # PURPOSE: Prints debug reports # PARAMETERS: $active Boolean: 1 for print, 0 for no print -# $message Message to print +# @messages Messages to print # RETURNS: Nothing -# DESCRIPTION: Outputs a message if $active is true. It removes any trailing -# newline and then adds one in the 'print' to the caller doesn't -# have to bother. Prepends the message with 'D> ' to show it's +# DESCRIPTION: Outputs messages if $active is true. It removes any trailing +# newlines and then adds one to each line so the caller doesn't +# have to bother. Prepends 'D> ' to each message to show it's # a debug message. # THROWS: No exceptions # COMMENTS: None # SEE ALSO: N/A #=============================================================================== -sub _debug { - my ( $active, $message ) = @_; +sub _debug { #{{{ + my ( $active, @messages ) = @_; - chomp($message); - print STDERR "D> $message\n" if $active; -} + if ($active) { + chomp(@messages); + say STDERR "D> ", join( "\nD> ", @messages ); + } +} #}}} #=== FUNCTION ================================================================ # NAME: _dbargs @@ -354,7 +400,7 @@ sub _debug { # COMMENTS: None # SEE ALSO: N/A #=============================================================================== -sub _dbargs { +sub _dbargs { #{{{ my ($opts) = @_; my @args; @@ -364,7 +410,7 @@ sub _dbargs { } return (@args); -} +} #}}} #=== FUNCTION ================================================================ # NAME: Options @@ -376,7 +422,7 @@ sub _dbargs { # COMMENTS: none # SEE ALSO: n/a #=============================================================================== -sub Options { +sub Options { #{{{ my ($optref) = @_; my @options = ( @@ -389,7 +435,7 @@ sub Options { } return; -} +} #}}} __END__ @@ -404,7 +450,7 @@ query2csv - A script for generating CSV from database query =head1 VERSION -This documentation refers to query2csv version 0.0.4 +This documentation refers to query2csv version 0.0.5 =head1 USAGE diff --git a/Database/query2json b/Database/query2json index 3e6af1b..95b43d7 100755 --- a/Database/query2json +++ b/Database/query2json @@ -18,9 +18,9 @@ # BUGS: --- # NOTES: --- # AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com -# VERSION: 0.0.3 +# VERSION: 0.0.4 # CREATED: 2021-06-18 13:24:49 -# REVISION: 2025-05-09 14:13:27 +# REVISION: 2025-05-25 18:48:59 # #=============================================================================== @@ -45,7 +45,7 @@ use Data::Dumper; # # Version number (manually incremented) # -our $VERSION = '0.0.3'; +our $VERSION = '0.0.4'; # # Script name @@ -131,11 +131,16 @@ else { } _debug( $DEBUG >= 3, '$query: ' . Dumper(\$query) ); +# +# Strip SQL comments +# +$query = strip_sql_comments($query); + # # Count placeholders in the query and the arguments provided. First remove all # comments which may contain '?' characters, then count any that are left. # -$query = join("\n", grep {!/^--/} split( "\n", $query ) ); +#$query = join("\n", grep {!/^--/} split( "\n", $query ) ); $pcount = grep {/\?/} split( '', $query ); $acount = scalar(@dbargs); @@ -177,7 +182,7 @@ my $conf = Config::General->new( my %config = $conf->getall(); # -# Set defaults +# Set defaults in case values have been omitted # $config{database}->{dbtype} //= 'SQLite'; $config{database}->{host} //= '127.0.0.1'; @@ -247,13 +252,13 @@ exit; # COMMENTS: None # SEE ALSO: N/A #=============================================================================== -sub db_connect { +sub db_connect { #{{{ my ($cfg) = @_; my ( $dbh, $dbtype, $dbname ); - $dbtype = $config{database}->{dbtype}; - $dbname = $config{database}->{name}; + $dbtype = $cfg->{database}->{dbtype}; + $dbname = $cfg->{database}->{name}; die "Database name is mandatory\n" unless $dbname; # @@ -264,6 +269,7 @@ sub db_connect { # The name for the SQLite driver is 'DBD:SQLite' # $dbtype = 'SQLite'; + _debug( $DEBUG >= 3, '$dbtype: ' . $dbtype, '$dbname: ' . $dbname ); $dbh = DBI->connect( "DBI:$dbtype:dbname=$dbname", "", "", { AutoCommit => 1, sqlite_unicode => 1, } ) @@ -275,10 +281,10 @@ sub db_connect { # $dbtype = 'mysql'; - my $dbhost = $config{database}->{host}; - my $dbport = $config{database}->{port}; - my $dbuser = $config{database}->{user}; - my $dbpwd = $config{database}->{password}; + my $dbhost = $cfg->{database}->{host}; + my $dbport = $cfg->{database}->{port}; + my $dbuser = $cfg->{database}->{user}; + my $dbpwd = $cfg->{database}->{password}; $dbh = DBI->connect( "DBI:$dbtype:host=$dbhost;port=$dbport;database=$dbname", $dbuser, $dbpwd, { AutoCommit => 1 } ) @@ -304,28 +310,68 @@ sub db_connect { } return $dbh; -} +} #}}} + +#=== FUNCTION ================================================================ +# NAME: strip_sql_comments +# PURPOSE: Given a query as a scalar, strips all SQL comments +# PARAMETERS: $query string containing a query +# RETURNS: Stripped string +# DESCRIPTION: Two types of comments might exist in the query: the C-style +# and the SQL style. The string is treated as a single string +# even though it's multi-line, and any C-style comments are +# removed. Then the string is treated as multi-line and each +# line is scanned for SQL comments (which end at the end of the +# line), and these are stripped. Blank lines are skipped too to +# compress the output a little. +# THROWS: No exceptions +# COMMENTS: None +# SEE ALSO: N/A +#=============================================================================== +sub strip_sql_comments { #{{{ + my ($query) = @_; + + my $result; + + # + # Strip C-style comments + # + $query =~ s/\/\*.*?\*\///sg; + + # + # Strip SQL line-oriented comments + # + foreach my $line (split(/\n/,$query)) { + next if $line =~ /^\s*$/; + $line =~ s/--.*$//; + $result .= "$line\n"; + } + + return $result; +} #}}} #=== FUNCTION ================================================================ # NAME: _debug # PURPOSE: Prints debug reports # PARAMETERS: $active Boolean: 1 for print, 0 for no print -# $message Message to print +# @messages Messages to print # RETURNS: Nothing -# DESCRIPTION: Outputs a message if $active is true. It removes any trailing -# newline and then adds one in the 'print' to the caller doesn't -# have to bother. Prepends the message with 'D> ' to show it's +# DESCRIPTION: Outputs messages if $active is true. It removes any trailing +# newlines and then adds one to each line so the caller doesn't +# have to bother. Prepends 'D> ' to each message to show it's # a debug message. # THROWS: No exceptions # COMMENTS: None # SEE ALSO: N/A #=============================================================================== -sub _debug { - my ( $active, $message ) = @_; +sub _debug { #{{{ + my ( $active, @messages ) = @_; - chomp($message); - print STDERR "D> $message\n" if $active; -} + if ($active) { + chomp(@messages); + say STDERR "D> ", join( "\nD> ", @messages ); + } +} #}}} #=== FUNCTION ================================================================ # NAME: _dbargs @@ -338,7 +384,7 @@ sub _debug { # COMMENTS: None # SEE ALSO: N/A #=============================================================================== -sub _dbargs { +sub _dbargs { #{{{ my ($opts) = @_; my @args; @@ -348,7 +394,7 @@ sub _dbargs { } return (@args); -} +} #}}} #=== FUNCTION ================================================================ # NAME: Options @@ -360,7 +406,7 @@ sub _dbargs { # COMMENTS: none # SEE ALSO: n/a #=============================================================================== -sub Options { +sub Options { #{{{ my ($optref) = @_; my @options = ( @@ -373,7 +419,7 @@ sub Options { } return; -} +} #}}} __END__ @@ -388,7 +434,7 @@ query2json - A script for generating CSV from database query =head1 VERSION -This documentation refers to query2json version 0.0.3 +This documentation refers to query2json version 0.0.4 =head1 USAGE diff --git a/Database/query2tt2 b/Database/query2tt2 index 02b015c..5a0ae52 100755 --- a/Database/query2tt2 +++ b/Database/query2tt2 @@ -26,9 +26,9 @@ # BUGS: --- # NOTES: --- # AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com -# VERSION: 0.0.8 +# VERSION: 0.0.9 # CREATED: 2021-06-18 13:24:49 -# REVISION: 2025-05-09 14:13:04 +# REVISION: 2025-05-25 18:41:09 # #=============================================================================== @@ -54,7 +54,7 @@ use Data::Dumper; # # Version number (manually incremented) # -our $VERSION = '0.0.8'; +our $VERSION = '0.0.9'; # # Script and directory names @@ -162,11 +162,16 @@ else { } _debug( $DEBUG >= 3, '$query: ' . Dumper(\$query) ); +# +# Strip SQL comments +# +$query = strip_sql_comments($query); + # # Count placeholders in the query and the arguments provided. First remove all # comments which may contain '?' characters, then count any that are left. # -$query = join("\n", grep {!/^--/} split( "\n", $query ) ); +#$query = join("\n", grep {!/^--/} split( "\n", $query ) ); $pcount = grep {/\?/} split( '', $query ); $acount = scalar(@dbargs); @@ -323,13 +328,13 @@ exit; # COMMENTS: None # SEE ALSO: N/A #=============================================================================== -sub db_connect { +sub db_connect { #{{{ my ($cfg) = @_; my ( $dbh, $dbtype, $dbname ); - $dbtype = $config{database}->{dbtype}; - $dbname = $config{database}->{name}; + $dbtype = $cfg->{database}->{dbtype}; + $dbname = $cfg->{database}->{name}; die "Database name is mandatory\n" unless $dbname; # @@ -340,6 +345,7 @@ sub db_connect { # The name for the SQLite driver is 'DBD:SQLite' # $dbtype = 'SQLite'; + _debug( $DEBUG >= 3, '$dbtype: ' . $dbtype, '$dbname: ' . $dbname ); $dbh = DBI->connect( "DBI:$dbtype:dbname=$dbname", "", "", { AutoCommit => 1, sqlite_unicode => 1, } ) @@ -351,10 +357,10 @@ sub db_connect { # $dbtype = 'mysql'; - my $dbhost = $config{database}->{host}; - my $dbport = $config{database}->{port}; - my $dbuser = $config{database}->{user}; - my $dbpwd = $config{database}->{password}; + my $dbhost = $cfg->{database}->{host}; + my $dbport = $cfg->{database}->{port}; + my $dbuser = $cfg->{database}->{user}; + my $dbpwd = $cfg->{database}->{password}; $dbh = DBI->connect( "DBI:$dbtype:host=$dbhost;port=$dbport;database=$dbname", $dbuser, $dbpwd, { AutoCommit => 1 } ) @@ -380,28 +386,68 @@ sub db_connect { } return $dbh; -} +} #}}} + +#=== FUNCTION ================================================================ +# NAME: strip_sql_comments +# PURPOSE: Given a query as a scalar, strips all SQL comments +# PARAMETERS: $query string containing a query +# RETURNS: Stripped string +# DESCRIPTION: Two types of comments might exist in the query: the C-style +# and the SQL style. The string is treated as a single string +# even though it's multi-line, and any C-style comments are +# removed. Then the string is treated as multi-line and each +# line is scanned for SQL comments (which end at the end of the +# line), and these are stripped. Blank lines are skipped too to +# compress the output a little. +# THROWS: No exceptions +# COMMENTS: None +# SEE ALSO: N/A +#=============================================================================== +sub strip_sql_comments { #{{{ + my ($query) = @_; + + my $result; + + # + # Strip C-style comments + # + $query =~ s/\/\*.*?\*\///sg; + + # + # Strip SQL line-oriented comments + # + foreach my $line (split(/\n/,$query)) { + next if $line =~ /^\s*$/; + $line =~ s/--.*$//; + $result .= "$line\n"; + } + + return $result; +} #}}} #=== FUNCTION ================================================================ # NAME: _debug # PURPOSE: Prints debug reports # PARAMETERS: $active Boolean: 1 for print, 0 for no print -# $message Message to print +# @messages Messages to print # RETURNS: Nothing -# DESCRIPTION: Outputs a message if $active is true. It removes any trailing -# newline and then adds one in the 'print' to the caller doesn't -# have to bother. Prepends the message with 'D> ' to show it's +# DESCRIPTION: Outputs messages if $active is true. It removes any trailing +# newlines and then adds one to each line so the caller doesn't +# have to bother. Prepends 'D> ' to each message to show it's # a debug message. # THROWS: No exceptions # COMMENTS: None # SEE ALSO: N/A #=============================================================================== -sub _debug { - my ( $active, $message ) = @_; +sub _debug { #{{{ + my ( $active, @messages ) = @_; - chomp($message); - print STDERR "D> $message\n" if $active; -} + if ($active) { + chomp(@messages); + say STDERR "D> ", join( "\nD> ", @messages ); + } +} #}}} #=== FUNCTION ================================================================ # NAME: _dbargs @@ -414,7 +460,7 @@ sub _debug { # COMMENTS: None # SEE ALSO: N/A #=============================================================================== -sub _dbargs { +sub _dbargs { #{{{ my ($opts) = @_; my @args; @@ -424,7 +470,7 @@ sub _dbargs { } return (@args); -} +} #}}} #=== FUNCTION ================================================================ # NAME: _define @@ -439,7 +485,7 @@ sub _dbargs { # COMMENTS: None # SEE ALSO: #=============================================================================== -sub _define { +sub _define { #{{{ my ($opts) = @_; my %defs; @@ -449,7 +495,7 @@ sub _define { } return (%defs); -} +} #}}} #=== FUNCTION ================================================================ # NAME: Options @@ -461,7 +507,7 @@ sub _define { # COMMENTS: none # SEE ALSO: n/a #=============================================================================== -sub Options { +sub Options { #{{{ my ($optref) = @_; my @options = ( @@ -477,7 +523,7 @@ sub Options { } return; -} +} #}}} __END__ @@ -492,7 +538,7 @@ query2tt2 - A script for formatting a report from database query using a templat =head1 VERSION -This documentation refers to query2tt2 version 0.0.8 +This documentation refers to query2tt2 version 0.0.9 =head1 USAGE