From 0cafdf835515746bbccaca071c5770be70bac439 Mon Sep 17 00:00:00 2001 From: Banana Date: Sun, 28 Jan 2024 08:56:18 +0100 Subject: [PATCH] new search field. Work in progress --- CHANGELOG | 1 + TODO | 4 +- documentation/upgrade.txt | 13 ++- sources/updatecheck.txt | 2 +- upgrade/from-version-1.5.txt | 19 +++-- webclient/index.php | 2 +- .../lib/managecollectionfields.class.php | 13 ++- webclient/lib/managecollections.class.php | 83 ++++++++++++++++++- webclient/lib/possessed.class.php | 1 - webclient/lib/summoner.class.php | 13 +++ webclient/lib/trite.class.php | 2 + .../managecolletions/managecolletions.html | 27 +++++- .../managecolletions/managecolletions.php | 34 +++++--- .../default/managegroups/managegroups.html | 2 - 14 files changed, 178 insertions(+), 38 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index f543a84..4aaf9a8 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,7 @@ 1.x - Chizra * Config change. Added new entries. See upgrade/from-version-1.5.txt. It won't work if it is missing. * Config change: Added new theme config. See upgrade/from-version-1.5.txt. It won't work if it is missing. + * Added: A new entry field for better global search. Read upgrade/from-version-1.5.txt for more details. * Added: #6 Feature: Duplicate search This is a very limited like search based on the title field. Needs improvement and better search data. * Licence change to GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 diff --git a/TODO b/TODO index 9e84de3..9de748f 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,7 @@ * change the search data. Maybe an extra search field in which all the searchable information of an entry will be saved. +* query optimization and default indexes on columns +* make documentation markdown to be nicer * change multiple-attachment to a field which tells it is used for a image gallery * update JS and remove deprecations * complete profile view. Groups still missing. @@ -8,8 +10,6 @@ * create a real fallback theme, which does not depend on any styling/css ** change the css and js lookup too in main file * i18n support -* Automatic upgrades of DB -* drop in updates * Definition of fields in "card view" * User and groupmanagement: Check where a user or group is used! * Better error handling and display while adding / update and delete diff --git a/documentation/upgrade.txt b/documentation/upgrade.txt index 5270c89..26199c3 100644 --- a/documentation/upgrade.txt +++ b/documentation/upgrade.txt @@ -1,5 +1,6 @@ Each release has its own file in the upgrade folder. Basic flow would be to extract the new release and copy over the following files and directories: + ``` webclient/config/ webclient/lib/ @@ -9,8 +10,14 @@ webclient/api.php webclient/index.php ``` -In those files there are the steps needed to make an upgrade. +and follow the upgrade file(s). After the first launch, visit `/index.php?p=managesystem` and check if there any +further changes to be done. -If you upgrade multiple versions make sure to read all the files in the correct order. +In those files there are the steps needed to make an upgrade. If you upgrade multiple versions make sure to read +all the files in the correct order. Within the upgrade files itself there is an order. Make sure to follow. -Within the upgrade files itself there is an order. Make sure to follow. +# Why no automatic updates +Doing so, the process serving the application needs write access to the files. Which can be a security risk. +Providing write access only to storage or temporary files and not the application files itself, it is more +secure (there is no complete security). +The downside are manual updates. diff --git a/sources/updatecheck.txt b/sources/updatecheck.txt index 9c05c5a..88a97e0 100644 --- a/sources/updatecheck.txt +++ b/sources/updatecheck.txt @@ -11,7 +11,7 @@ upgrade file with correct placeholders correct placeholders in sql file for manual setup -correct version info in VERSION and index.php +correct version info in VERSION and index.php and cyberdemon, if needed correct tag diff --git a/upgrade/from-version-1.5.txt b/upgrade/from-version-1.5.txt index 85a0b80..4c1f716 100644 --- a/upgrade/from-version-1.5.txt +++ b/upgrade/from-version-1.5.txt @@ -1,18 +1,26 @@ -## Added new constants to config.php file. +# Added a new global search field +As of version 1.6, the field 'Combined Search' provides a much better default search base. +If you want to use it make sure you update its contents for all the existing entries in +your collection. After that and for all new entries the data will be created automatically. +The update and creation of the field is done by using the new option "Update combined search field data" +in collection management. It needs to be done for each if your collections but only once. + +# Added new constants to config.php file. Use config.php.default as a help. The new lines are: +``` const LOGFILE = PATH_SYSTEMOUT.'/bibliotheca.log'; - # CURL browser settings const BROWSER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0'; const BROWSER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'; const BROWSER_ACCEPT_LANG = 'en-US,en;q=0.5'; +``` -## Updated tools configs -Please compare the default config files for googlebooks, imdbweb and musicbrainz and make the required -changes. +# Updated tools configs +Please compare the default config files for googlebooks, imdbweb and musicbrainz and make the required changes. # Added new theme config to config.php file Use config.php.default as a help. The new setting is: +``` # additional config for each theme with fallback const UI_THEME_CONFIG = array( 'default' => array( @@ -22,3 +30,4 @@ const UI_THEME_CONFIG = array( 'coverImageMaxWidth' => 500 // in pixel. Supports image/jpeg, image/png, image/webp ) ); +``` diff --git a/webclient/index.php b/webclient/index.php index 8465a86..920190d 100644 --- a/webclient/index.php +++ b/webclient/index.php @@ -2,7 +2,7 @@ /** * Bibliotheca * - * Copyright 2018-2023 Johannes Keßler + * Copyright 2018-2024 Johannes Keßler * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/webclient/lib/managecollectionfields.class.php b/webclient/lib/managecollectionfields.class.php index c51db5f..a3d1916 100644 --- a/webclient/lib/managecollectionfields.class.php +++ b/webclient/lib/managecollectionfields.class.php @@ -260,8 +260,12 @@ class ManageCollectionFields { $this->_cacheExistingSysFields = array(); - $queryStr = "SELECT `cf`.`fk_field_id` AS id, `sf`.`type`, `sf`.`displayname`, `sf`.`identifier`, - `sf`.`createstring`, `sf`.`searchtype` + $queryStr = "SELECT `cf`.`fk_field_id` AS id, + `sf`.`type`, + `sf`.`displayname`, + `sf`.`identifier`, + `sf`.`createstring`, + `sf`.`searchtype` FROM `".DB_PREFIX."_collection_fields_".$this->_collectionId."` AS cf LEFT JOIN `".DB_PREFIX."_sys_fields` AS sf ON `cf`.`fk_field_id` = `sf`.`id`"; if($sortAZ === true) { @@ -307,11 +311,12 @@ class ManageCollectionFields { } } + // add systemfields $def['created'] = array('identifier' => 'created', 'displayname' => 'Created', 'type' => 'systemfield'); $def['modified'] = array('identifier' => 'modified', 'displayname' => 'Modified', 'type' => 'systemfield'); - $ret = $def + $ret; + $def['search'] = array('identifier' => 'search', 'displayname' => 'Combined Search', 'type' => 'systemfield'); - return $ret; + return $def + $ret; } /** diff --git a/webclient/lib/managecollections.class.php b/webclient/lib/managecollections.class.php index 0b630ee..ac52dfa 100644 --- a/webclient/lib/managecollections.class.php +++ b/webclient/lib/managecollections.class.php @@ -226,7 +226,9 @@ class ManageCollections { `owner` int NOT NULL, `group` int NOT NULL, `rights` char(9) COLLATE utf8mb4_bin NOT NULL, - PRIMARY KEY (`id`) + `search` text COLLATE utf8mb4_unicode_ci NOT NULL, + PRIMARY KEY (`id`), + FULLTEXT KEY `search` (`search`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"; if(QUERY_DEBUG) Summoner::sysLog("[QUERY] ".__METHOD__." query: ".Summoner::cleanForLog($queryCollectionEntry)); $this->_DB->query($queryCollectionEntry); @@ -338,7 +340,17 @@ class ManageCollections { // optimize does a recreation and the column collation // is considered $this->_DB->query("OPTIMIZE TABLE `".DB_PREFIX."_collection_entry_".$data['id']."`"); - } + } elseif($data['defaultSearchField'] === "search") { + // Special case. 1.6 adds the search field. Needs to be checked if there + // It is a new default column which is added at creation of a collection + // but needs to be added manually for existing ones. + // could be removed in future version... + $queryStr = "ALTER TABLE `".DB_PREFIX."_collection_entry_".$data['id']."` + ADD `search` TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci + NOT NULL AFTER `rights`, ADD FULLTEXT (`search`)"; + if(QUERY_DEBUG) Summoner::sysLog("[QUERY] ".__METHOD__." query: ".Summoner::cleanForLog($queryStr)); + $this->_DB->query($queryStr); + } } catch (Exception $e) { if($e->getCode() == "1061") { // duplicate key message if the index is already there. @@ -438,6 +450,72 @@ class ManageCollections { return $ret; } + /** + * Selects the text entry fields, gets their data, combines the words and writes it into the search field + * for every entry in the current loaded collection + * + * @param string $collectionId The id of the collection + * @param array $searchFields The available search fields of the given collection + * @return bool + */ + public function updateSearchData(string $collectionId, array $searchFields): bool { + $ret = false; + + // simple search fields for loaded collection + // Every field witch has a column in the entry table is a simple search field. + // Name starts with entry. Here we want only the text fields + // Those fields are the data for the combined search field + $dataFields = array(); + if(!empty($searchFields)) { + foreach($searchFields as $k=>$v) { + if(isset($v['searchtype']) && strpos($v['searchtype'],'Text') !== false) { + $dataFields[$k] = $v['identifier']; + } + } + } + + // get the search data for every entry in the collection + $entryData = array(); + if(!empty($dataFields)) { + $fieldStr = implode(",",$dataFields); + $queryStr = "SELECT id,".$fieldStr." FROM `".DB_PREFIX."_collection_entry_".$collectionId."`"; + if(QUERY_DEBUG) Summoner::sysLog("[QUERY] ".__METHOD__." query: ".Summoner::cleanForLog($queryStr)); + try { + $query = $this->_DB->query($queryStr); + if($query !== false && $query->num_rows > 0) { + $entryData = $query->fetch_all(MYSQLI_ASSOC); + } + } + catch (Exception $e) { + Summoner::sysLog("[ERROR] ".__METHOD__." mysql catch: ".$e->getMessage()); + } + } + + // build the search data and update the entries + if(!empty($entryData)) { + foreach($entryData as $d) { + $entryid = $d['id']; + unset($d['id']); + $searchData = implode(" ",$d); + $searchData = implode(" ", Summoner::words($searchData)); + + $queryStr = "UPDATE `".DB_PREFIX."_collection_entry_".$collectionId."` + SET `search` = '".$this->_DB->real_escape_string($searchData)."' + WHERE `id` = '".$entryid."'"; + if(QUERY_DEBUG) Summoner::sysLog("[QUERY] ".__METHOD__." query: ".Summoner::cleanForLog($queryStr)); + try { + $this->_DB->query($queryStr); + $ret = true; + } + catch (Exception $e) { + Summoner::sysLog("[ERROR] ".__METHOD__." mysql catch: ".$e->getMessage()); + } + } + } + + return $ret; + } + /** * Check if given name can be used as a new one * @@ -505,7 +583,6 @@ class ManageCollections { private function _updateToolRelation(string $id, array $tool): bool { $ret = false; - $queryStr = "DELETE FROM `".DB_PREFIX."_tool2collection` WHERE `fk_collection_id` = '".$this->_DB->real_escape_string($id)."'"; if(QUERY_DEBUG) Summoner::sysLog("[QUERY] ".__METHOD__." query: ".Summoner::cleanForLog($queryStr)); diff --git a/webclient/lib/possessed.class.php b/webclient/lib/possessed.class.php index 09184b2..a555096 100644 --- a/webclient/lib/possessed.class.php +++ b/webclient/lib/possessed.class.php @@ -24,7 +24,6 @@ * Some groups are protected and should not be removed. * * passwords used here: password_hash("somePassword", PASSWORD_DEFAULT); - * */ class Possessed { /** diff --git a/webclient/lib/summoner.class.php b/webclient/lib/summoner.class.php index 70b3a80..8ab6f3f 100644 --- a/webclient/lib/summoner.class.php +++ b/webclient/lib/summoner.class.php @@ -19,6 +19,7 @@ */ /** + * Class Summoner * a static helper class */ class Summoner { @@ -567,4 +568,16 @@ class Summoner { static function sysLog(string $msg): void { error_log(date("c")." ".$msg."\n", 3, LOGFILE); } + + /** + * Create unique words from the given data + * + * @param $data string + * @return array + * + */ + static function words(string $data): array { + preg_match_all('/\w{3,}+/u',$data,$matches); + return array_unique($matches[0]); + } } diff --git a/webclient/lib/trite.class.php b/webclient/lib/trite.class.php index 05295ff..53bcc1d 100644 --- a/webclient/lib/trite.class.php +++ b/webclient/lib/trite.class.php @@ -252,8 +252,10 @@ class Trite { } } + // add systemfields $def['created'] = array('identifier' => 'created', 'displayname' => 'Created', 'type' => 'systemfield'); $def['modified'] = array('identifier' => 'modified', 'displayname' => 'Modified', 'type' => 'systemfield'); + $def['search'] = array('identifier' => 'search', 'displayname' => 'Combined Search', 'type' => 'systemfield'); $ret = $def + $ret; return $ret; diff --git a/webclient/view/default/managecolletions/managecolletions.html b/webclient/view/default/managecolletions/managecolletions.html index e6bff5a..38a1c28 100644 --- a/webclient/view/default/managecolletions/managecolletions.html +++ b/webclient/view/default/managecolletions/managecolletions.html @@ -34,9 +34,30 @@ > () - The field is used in the global search. + + The field is used in the global search.
Altering the default search field results in - a DB reindex. This could take some time, depending on the amount of data. + a DB reindex. This could take some time to complete, depending on the amount of data. +
+ + As of version 1.6, the field 'Combined Search' + provides a much better default search base.
+ If you want to use it make sure you update its contents (following option) for all the existing entries in + your collection. After that and for all new entries the data will be created automatically. + + + +
+
Update combined search field data
+
+
@@ -62,7 +83,7 @@
- +