Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/stable 3 4 0/8710 address recommend by similarity performance #9258

Merged
merged 10 commits into from
Aug 25, 2023
Merged

Feature/stable 3 4 0/8710 address recommend by similarity performance #9258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
fb3935f
pkp/pkp-lib#8710 Added filter to exclude submission IDs
jonasraoni Apr 11, 2023
3415f3d
pkp/pkp-lib#8710 Updated assignedTo exception to ignore only the auth…
jonasraoni Apr 11, 2023
10c3aca
pkp/pkp-lib#8710 Added $allowShortWords to the SubmissionSearchIndex:…
jonasraoni Apr 12, 2023
f8d67f8
pkp/pkp-lib#8710 Updated searchPhrase filter to look for matches at t…
jonasraoni Apr 12, 2023
5a6b884
pkp/pkp-lib#8710 Added sort by search ranking
jonasraoni Apr 12, 2023
9b66011
pkp/pkp-lib#8710 Formatting
jonasraoni Apr 12, 2023
eb9b5ba
pkp/pkp-lib#8710 Added $allowNumericWords to the SubmissionSearchInde…
jonasraoni Apr 13, 2023
ee59ab4
pkp/pkp-lib#8710 Fixed sorting and numeric search (submission ID)
jonasraoni Aug 25, 2023
33d7922
pkp/pkp-lib#8710 Updated comments
jonasraoni Aug 25, 2023
f477ec8
pkp/pkp-lib#8710 Added optional maxSearchKeywords parameter
jonasraoni Aug 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 20 additions & 24 deletions classes/search/SubmissionSearchIndex.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,15 @@ abstract class SubmissionSearchIndex
/**
* Split a string into a clean array of keywords
*
* @param string $text
* @param string|array $text
* @param bool $allowWildcards
*
* @return array of keywords
* @return string[] of keywords
*/
public function filterKeywords($text, $allowWildcards = false)
public static function filterKeywords($text, $allowWildcards = false, bool $allowShortWords = false, bool $allowNumericWords = false): array
{
$minLength = Config::getVar('search', 'min_word_length');
$stopwords = $this->_loadStopwords();
$stopwords = static::loadStopwords();

// Join multiple lines into a single string
if (is_array($text)) {
Expand All @@ -56,11 +56,11 @@ public function filterKeywords($text, $allowWildcards = false)

// FIXME Do not perform further filtering for some fields, e.g., author names?

// Remove stopwords
$keywords = [];
foreach ($words as $k) {
if (!isset($stopwords[$k]) && PKPString::strlen($k) >= $minLength && !is_numeric($k)) {
$keywords[] = PKPString::substr($k, 0, self::SEARCH_KEYWORD_MAX_LENGTH);
foreach ($words as $word) {
// Ignores: stop words, short words (when $allowShortWords is false) and words composed solely of numbers (when $allowNumericWords is false)
if (empty($stopwords[$word]) && ($allowShortWords || PKPString::strlen($word) >= $minLength) && ($allowNumericWords || !is_numeric($word))) {
$keywords[] = PKPString::substr($word, 0, static::SEARCH_KEYWORD_MAX_LENGTH);
}
}
return $keywords;
Expand All @@ -70,26 +70,22 @@ public function filterKeywords($text, $allowWildcards = false)
* Return list of stopwords.
* FIXME: Should this be locale-specific?
*
* @return array with stopwords as keys
* @return array<string,int> Stop words (in lower case) as keys and 1 as value
*/
protected function _loadStopwords()
protected static function loadStopwords()
{
static $searchStopwords;

if (!isset($searchStopwords)) {
// Load stopwords only once per request
$searchStopwords = array_count_values(
array_filter(
array_map('trim', file(dirname(__FILE__, 5) . '/' . self::SEARCH_STOPWORDS_FILE)),
function ($a) {
return !empty($a) && $a[0] != '#';
}
)
);
$searchStopwords[''] = 1;
}

return $searchStopwords;
return $searchStopwords ??= array_fill_keys(
collect(file(base_path(static::SEARCH_STOPWORDS_FILE)))
->map(fn (string $word) => trim($word))
// Ignore comments/line-breaks
->filter(fn (string $word) => !empty($word) && $word[0] !== '#')
// Include a map for empty words
->push('')
->toArray(),
1
);
}

/**
Expand Down
Loading