fetch.pl

updated the initial fetch sql

diff --git a/fetch.pl b/fetch.pl
index a94c4adb853fe8de9b62a3fe8288182cd368e5b4..04e7e576dda9cfa1149f77acac629366525cf866 100644 (file)
--- a/fetch.pl
+++ b/fetch.pl
@@ -47,7 +47,9 @@ my %urlsToFetch;
 my $query = $dbh->prepare("SELECT `id`, `url` 
                                                        FROM `url_to_fetch` 
                                                        WHERE `last_fetched` < NOW() - INTERVAL 1 WEEK
-                                                               AND `fetch_failed` = 0");
+                                                               OR `last_fetched` IS NULL
+                                                               AND `fetch_failed` = 0
+                                                       LIMIT 5000");
 $query->execute();
 while(my @row = $query->fetchrow_array) {
        $urlsToFetch{$row[0]} = $row[1];

diff --git a/storage/.gitignore b/storage/.gitignore
new file mode 100644 (file)
index 0000000..d6b7ef3
--- /dev/null
+++ b/storage/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore