From b4290146b98bfb3feee28b0c113885e0005f7820 Mon Sep 17 00:00:00 2001 From: Jan Walther Date: Thu, 22 Aug 2024 17:36:53 +0200 Subject: [PATCH 1/5] relative term frequency --- src/Engines/MysqlEngine.php | 4 +++- src/Engines/RedisEngine.php | 3 ++- src/Engines/SqliteEngine.php | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Engines/MysqlEngine.php b/src/Engines/MysqlEngine.php index 8363fe8f..24189cbe 100644 --- a/src/Engines/MysqlEngine.php +++ b/src/Engines/MysqlEngine.php @@ -184,8 +184,10 @@ public function saveWordlist($stems) public function saveDoclist($terms, $docId) { $insertRows = []; + + $countTerms = count($terms); foreach ($terms as $key => $term) { - $insertRows[] = '('.$this->index->quote($term['id']).', '.$this->index->quote($docId).', '.$this->index->quote($term['hits']).')'; + $insertRows[] = '('.$this->index->quote($term['id']).', '.$this->index->quote($docId).', '.$this->index->quote($term['hits'] / $countTerms).')'; } $this->index->exec('INSERT INTO '.$this->indexName.'_doclist (term_id, doc_id, hit_count) VALUES '.implode(',', $insertRows).''); diff --git a/src/Engines/RedisEngine.php b/src/Engines/RedisEngine.php index 53ef65f1..1940c4aa 100644 --- a/src/Engines/RedisEngine.php +++ b/src/Engines/RedisEngine.php @@ -207,9 +207,10 @@ public function saveWordlist($stems) public function saveDoclist($terms, $docId) { + $countTerms = count($terms); foreach ($terms as $term => $docsHits) { $redisKey = $this->indexName . ':doclist:' . $term . ':' . $docId; - $this->redis->hset($redisKey, 'num_hits', $docsHits['num_hits']); + $this->redis->hset($redisKey, 'num_hits', $docsHits['num_hits'] / $countTerms); } } diff --git a/src/Engines/SqliteEngine.php b/src/Engines/SqliteEngine.php index 4df25f05..6c592701 100644 --- a/src/Engines/SqliteEngine.php +++ b/src/Engines/SqliteEngine.php @@ -300,11 +300,12 @@ public function saveDoclist($terms, $docId) $insert = "INSERT INTO doclist (term_id, doc_id, hit_count) VALUES (:id, :doc, :hits)"; $stmt = $this->index->prepare($insert); + $countTerms = count($terms); foreach ($terms as $key => $term) { $stmt->bindValue(':id', $term['id']); $stmt->bindValue(':doc', $docId); - $stmt->bindValue(':hits', $term['hits']); + $stmt->bindValue(':hits', $term['hits'] / $countTerms); try { $res = $stmt->execute(); } catch (\Exception $e) { From 9597e724a49f3c658f1c4cc708d43a8a0d081581 Mon Sep 17 00:00:00 2001 From: Jan Walther Date: Thu, 22 Aug 2024 18:18:28 +0200 Subject: [PATCH 2/5] relative term frequency --- src/TNTSearch.php | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/TNTSearch.php b/src/TNTSearch.php index 86f5e6e2..79202bc9 100644 --- a/src/TNTSearch.php +++ b/src/TNTSearch.php @@ -102,8 +102,7 @@ public function search($phrase, $numOfResults = 100) $keywords = $keywords->map(function ($keyword) { return $this->stemmer->stem($keyword); }); - $tfWeight = 1; - $dlWeight = 0.5; + $docScores = []; $count = $this->totalDocumentsInCollection(); $noLimit = $this->engine->fuzzy_no_limit; @@ -115,11 +114,7 @@ public function search($phrase, $numOfResults = 100) foreach ($this->getAllDocumentsForKeyword($term, $noLimit, $isLastKeyword) as $document) { $docID = $document['doc_id']; $tf = $document['hit_count']; - $num = ($tfWeight + 1) * $tf; - $denom = $tfWeight - * ((1 - $dlWeight) + $dlWeight) - + $tf; - $score = $idf * ($num / $denom); + $score = $idf * $tf; $docScores[$docID] = isset($docScores[$docID]) ? $docScores[$docID] + $score : $score; } From 555b15eebe6c928d2fe97b16f98903cdac034481 Mon Sep 17 00:00:00 2001 From: Jan Walther Date: Thu, 22 Aug 2024 18:25:24 +0200 Subject: [PATCH 3/5] relative term frequency --- src/Engines/MysqlEngine.php | 2 +- src/Engines/SqliteEngine.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Engines/MysqlEngine.php b/src/Engines/MysqlEngine.php index 24189cbe..df76c12b 100644 --- a/src/Engines/MysqlEngine.php +++ b/src/Engines/MysqlEngine.php @@ -41,7 +41,7 @@ public function createIndex($indexName) 'CREATE TABLE IF NOT EXISTS '.$this->indexName.'_doclist ( term_id INTEGER, doc_id VARCHAR(255), - hit_count INTEGER)' + hit_count FLOAT)' ); $this->index->exec( diff --git a/src/Engines/SqliteEngine.php b/src/Engines/SqliteEngine.php index 6c592701..7666fbb3 100644 --- a/src/Engines/SqliteEngine.php +++ b/src/Engines/SqliteEngine.php @@ -76,7 +76,7 @@ public function createIndex($indexName) $this->index->exec("CREATE TABLE IF NOT EXISTS doclist ( term_id INTEGER, doc_id INTEGER, - hit_count INTEGER)"); + hit_count FLOAT)"); $this->index->exec("CREATE TABLE IF NOT EXISTS fields ( id INTEGER PRIMARY KEY, From e80a2789b067c0cff2154ac0f059defff5061d93 Mon Sep 17 00:00:00 2001 From: Jan Walther Date: Wed, 3 Sep 2025 12:29:01 +0200 Subject: [PATCH 4/5] rebase to current master branch --- src/Engines/MysqlEngine.php | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Engines/MysqlEngine.php b/src/Engines/MysqlEngine.php index c5d08a70..171175c7 100644 --- a/src/Engines/MysqlEngine.php +++ b/src/Engines/MysqlEngine.php @@ -200,11 +200,15 @@ public function saveDoclist(array $terms, int $docId) { $insertRows = []; foreach ($terms as $term) { - $insertRows[] = '(' . $this->index->quote($term['id']) . ', ' . $this->index->quote($docId) . ', ' . $this->index->quote($term['hits']) . ')'; + $insertRows[] = '('.$this->index->quote($term['id']).', '.$this->index->quote($docId).', '.$this->index->quote($term['hits']).')'; } - $this->index->exec('INSERT INTO ' . $this->indexName . '_doclist (term_id, doc_id, hit_count) VALUES ' . implode(',', - $insertRows) . ''); + $this->index->exec( + 'INSERT INTO '.$this->indexName.'_doclist (term_id, doc_id, hit_count) VALUES '.implode( + ',', + $insertRows + ).'' + ); } public function saveHitList(array $stems, int $docId, array $termsList) From ef1a595422d7e7467d05523169e79d1334a1e510 Mon Sep 17 00:00:00 2001 From: Jan Walther Date: Wed, 3 Sep 2025 12:30:57 +0200 Subject: [PATCH 5/5] rebase to current master branch --- src/Engines/MysqlEngine.php | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Engines/MysqlEngine.php b/src/Engines/MysqlEngine.php index 171175c7..f87bf38f 100644 --- a/src/Engines/MysqlEngine.php +++ b/src/Engines/MysqlEngine.php @@ -198,16 +198,14 @@ public function saveWordlist(Collection $stems) public function saveDoclist(array $terms, int $docId) { + $countTerms = count($terms); $insertRows = []; foreach ($terms as $term) { - $insertRows[] = '('.$this->index->quote($term['id']).', '.$this->index->quote($docId).', '.$this->index->quote($term['hits']).')'; + $insertRows[] = '('.$this->index->quote($term['id']).', '.$this->index->quote($docId).', '.$this->index->quote($term['hits'] / $countTerms).')'; } $this->index->exec( - 'INSERT INTO '.$this->indexName.'_doclist (term_id, doc_id, hit_count) VALUES '.implode( - ',', - $insertRows - ).'' + 'INSERT INTO '.$this->indexName.'_doclist (term_id, doc_id, hit_count) VALUES '.implode(',', $insertRows) ); }