possible speed improvement while inserting the rows

Signed-off-by: Banana <mail@bananas-playground.net>

diff --git a/parse-results.pl b/parse-results.pl
index b2be62dd8efc31b9b124072f76bbca9fc98c6013..8459ad7f5fea2b5f95f412351eb6f7fee0722e39 100644 (file)
--- a/parse-results.pl
+++ b/parse-results.pl
@@ -41,7 +41,8 @@ die "Could not read config! $ConfigReader::Simple::ERROR\n" unless ref $config;
 ## DB connection
 my %dbAttr = (
        PrintError=>0,# turn off error reporting via warn()
-    RaiseError=>1 # turn on error reporting via die()
+    RaiseError=>1, # turn on error reporting via die()
+       AutoCommit=>0 # manually use transactions
 );
 my $dbDsn = "DBI:mysql:database=".$config->get("DB_NAME").";host=".$config->get("DB_HOST").";port=".$config->get("DB_PORT");
 my $dbh = DBI->connect($dbDsn,$config->get("DB_USER"),$config->get("DB_PASS"), \%dbAttr);
@@ -68,7 +69,6 @@ $query->execute(@queryIds);
 while(my @row = $query->fetchrow_array) {
        $baseUrls{$row[0]} = $row[1];
 }
-#$query->finish();
 
 
 # get the string to ignore
@@ -80,7 +80,6 @@ $query->execute();
 while(my @row = $query->fetchrow) {
        push(@urlStringsToIgnore, $row[0])
 }
-#$query->finish();
 
 
 ## prepare linkExtor
@@ -166,6 +165,7 @@ sub insertIntoDb {
        sayLog $queryStr if $DEBUG;
        $query = $dbh->prepare($queryStr);
        my $md5 = Digest::MD5->new;
+       my $counter = 0;
        foreach my $link (@links) {
 
                sayLog $link if ($DEBUG);
@@ -186,6 +186,13 @@ sub insertIntoDb {
                $query->execute($digest, $link, $url->scheme."://".$url->host);
                $md5->reset;
 
+               $counter++;
+
+               if($counter >= 100) {
+                       $counter = 0;
+                       $dbh->commit();
+               }
+
                #sayLog $digest if ($DEBUG);
                #sayLog $url->scheme if ($DEBUG);
                #sayLog $url->host if ($DEBUG);
@@ -194,5 +201,5 @@ sub insertIntoDb {
 
                #sayLog "Inserted: $link" if($DEBUG);
        }
-       #$query->finish();
+       $dbh->commit();
 }