From e9978a772218368f852d2e74741f0a1ae01c871a Mon Sep 17 00:00:00 2001 From: Banana Date: Tue, 8 Oct 2024 10:04:19 +0200 Subject: [PATCH] new directory structure to seperate the crawler and the upcomming webinterface Signed-off-by: Banana --- README.md | 5 +++++ cleanup.pl => crawler/cleanup.pl | 0 config.default.txt => crawler/config.default.txt | 0 crawler/documentation/install.md | 15 +++++++++++++++ .../documentation}/requirements.md | 0 .../documentation}/setup.sql | 0 crawler/documentation/upgrade.md | 6 ++++++ fetch.pl => crawler/fetch.pl | 0 {lib => crawler/lib}/Aranea/Common.pm | 0 parse-results.pl => crawler/parse-results.pl | 0 {storage => crawler/storage}/.gitignore | 0 documentation/install.md | 15 --------------- 12 files changed, 26 insertions(+), 15 deletions(-) rename cleanup.pl => crawler/cleanup.pl (100%) rename config.default.txt => crawler/config.default.txt (100%) create mode 100644 crawler/documentation/install.md rename {documentation => crawler/documentation}/requirements.md (100%) rename {documentation => crawler/documentation}/setup.sql (100%) create mode 100644 crawler/documentation/upgrade.md rename fetch.pl => crawler/fetch.pl (100%) rename {lib => crawler/lib}/Aranea/Common.pm (100%) rename parse-results.pl => crawler/parse-results.pl (100%) rename {storage => crawler/storage}/.gitignore (100%) delete mode 100644 documentation/install.md diff --git a/README.md b/README.md index 3ac7ebb..df524bd 100644 --- a/README.md +++ b/README.md @@ -27,3 +27,8 @@ The table `url_to_ignore` does have a small amount of domains and part of domains which will be ignored. Adding a global SPAM list would be overkill. A good idea is to run it with a DNS filter, which has a good blocklist. + +# Webinterface + +The folder `webroot` does contain a webinterface which displays the gathered data and status. +It does not provide a way to execute the crawler. diff --git a/cleanup.pl b/crawler/cleanup.pl similarity index 100% rename from cleanup.pl rename to crawler/cleanup.pl diff --git a/config.default.txt b/crawler/config.default.txt similarity index 100% rename from config.default.txt rename to crawler/config.default.txt diff --git a/crawler/documentation/install.md b/crawler/documentation/install.md new file mode 100644 index 0000000..c17acff --- /dev/null +++ b/crawler/documentation/install.md @@ -0,0 +1,15 @@ +# Requirements + +Please check the requirements file first. + +# Database + +You need a MySQL installation and an existing database. + +Use `setup.sql` to create the tables into your existing database: `mysql --user=user -p databasename < setup.sql` + +# Config + +Copy `config.default.txt` to `config.txt` and edit at least to match the database name and server settings. + +Make sure the directory `storage` can be written. diff --git a/documentation/requirements.md b/crawler/documentation/requirements.md similarity index 100% rename from documentation/requirements.md rename to crawler/documentation/requirements.md diff --git a/documentation/setup.sql b/crawler/documentation/setup.sql similarity index 100% rename from documentation/setup.sql rename to crawler/documentation/setup.sql diff --git a/crawler/documentation/upgrade.md b/crawler/documentation/upgrade.md new file mode 100644 index 0000000..b5780e6 --- /dev/null +++ b/crawler/documentation/upgrade.md @@ -0,0 +1,6 @@ +# Upgrade information + +Each release has its own upgrade file. +Follow the instructions in each file step by step, starting from your current one. + +Make sure paths and sql table names are modified to your settings. diff --git a/fetch.pl b/crawler/fetch.pl similarity index 100% rename from fetch.pl rename to crawler/fetch.pl diff --git a/lib/Aranea/Common.pm b/crawler/lib/Aranea/Common.pm similarity index 100% rename from lib/Aranea/Common.pm rename to crawler/lib/Aranea/Common.pm diff --git a/parse-results.pl b/crawler/parse-results.pl similarity index 100% rename from parse-results.pl rename to crawler/parse-results.pl diff --git a/storage/.gitignore b/crawler/storage/.gitignore similarity index 100% rename from storage/.gitignore rename to crawler/storage/.gitignore diff --git a/documentation/install.md b/documentation/install.md deleted file mode 100644 index b0a5198..0000000 --- a/documentation/install.md +++ /dev/null @@ -1,15 +0,0 @@ -# Requirements - -Please check the requirements file first. - -# Database - -You need a MySQL installation and a user which can create a database. - -Use `setup.sql` to create the `aranea` database and its tables. `mysql --user=user -p < setup.sql` - -# Config - -Copy `config.default.txt` to `config.txt` and edit at least to match the database server settings. - -Make sure the directory `storage` can be written. -- 2.39.5