From: Banana <mail@bananas-playground.net>
Date: Sun, 8 May 2022 07:43:00 +0000 (+0200)
Subject: fetch.pl
X-Git-Url: http://91.132.146.200/gitweb/?a=commitdiff_plain;h=24fb355861dbe98e90de09c9cf1e71c8e52a3d15;p=aranea.git

fetch.pl
updated the initial fetch sql
---

diff --git a/fetch.pl b/fetch.pl
index a94c4ad..04e7e57 100644
--- a/fetch.pl
+++ b/fetch.pl
@@ -47,7 +47,9 @@ my %urlsToFetch;
 my $query = $dbh->prepare("SELECT `id`, `url` 
 							FROM `url_to_fetch` 
 							WHERE `last_fetched` < NOW() - INTERVAL 1 WEEK
-								AND `fetch_failed` = 0");
+								OR `last_fetched` IS NULL
+								AND `fetch_failed` = 0
+							LIMIT 5000");
 $query->execute();
 while(my @row = $query->fetchrow_array) {
 	$urlsToFetch{$row[0]} = $row[1];
diff --git a/storage/.gitignore b/storage/.gitignore
new file mode 100644
index 0000000..d6b7ef3
--- /dev/null
+++ b/storage/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore