transactions

Signed-off-by: Banana <mail@bananas-playground.net>

CHANGELOG

@@ -1,6 +1,9 @@
 0.2
 + Cleanups
 + License change
++ Some db improvements
++ Default config file added
++ Updated requirements file
 
 0.1
 + initial release

cleanup.pl

@@ -46,7 +46,7 @@ die "failed to connect to MySQL database:DBI->errstr()" unless($dbh);
 
 
 
-# update the uniqe domains
+# update the unique domains
 my $queryStr = "INSERT IGNORE INTO unique_domain (url) select DISTINCT(baseurl) as url FROM url_to_fetch WHERE fetch_failed = 0";
 sayLog($queryStr) if $DEBUG;
 my $query = $dbh->prepare($queryStr);

documentation/install.md

@@ -10,6 +10,6 @@ Use setup.sql to create the `aranea` database and its tables. `mysql --user=user
 
 # Config
 
-Edit `config.txt`  at least to match the database server settings.
+Copy `config.default.txt` to `config.txt` and edit at least to match the database server settings.
 
 Make sure the directory `storage` can be written.

fetch.pl

@@ -40,7 +40,8 @@ die "Could not read config! $ConfigReader::Simple::ERROR\n" unless ref $config;
 ## DB connection
 my %dbAttr = (
 	PrintError=>0,# turn off error reporting via warn()
-    RaiseError=>1 # turn on error reporting via die()
+    RaiseError=>1, # turn on error reporting via die()
+	AutoCommit=>0 # manually use transactions
 );
 my $dbDsn = "DBI:mysql:database=".$config->get("DB_NAME").";host=".$config->get("DB_HOST").";port=".$config->get("DB_PORT");
 my $dbh = DBI->connect($dbDsn,$config->get("DB_USER"),$config->get("DB_PASS"), \%dbAttr);
@@ -130,7 +131,7 @@ sub updateFetched {
 		$query->bind_param(1,$idToUpdate);
 		$query->execute();
 	}
-	#$query->finish();
+	$dbh->commit();
 	sayGreen "Update fetch timestamps done";
 }
 
@@ -145,6 +146,6 @@ sub updateFailed {
 		$query->bind_param(1,$idToUpdate);
 		$query->execute();
 	}
-	#$query->finish();
+	$dbh->commit();
 	sayGreen "Update fetch failed done";
 }

parse-results.pl

@@ -129,9 +129,8 @@ foreach my $resultFile (@results) {
 @links = cleanLinks($dbh, \@links, \@urlStringsToIgnore);
 insertIntoDb($dbh, \@links);
 
-
 $dbh->disconnect();
-say CLEAR,GREEN, "Parse complete", RESET;
+sayGreen "Parse complete";
 
 
 ## cleanup the found links