Skip to content

Commit

Permalink
account for sqlite/pgsql expression indexes in meta value query scopes
Browse files Browse the repository at this point in the history
  • Loading branch information
@frasmage
frasmage committed Apr 28, 2024
1 parent 2005c8d commit 3862d24
Show file tree
Hide file tree
Showing 3 changed files with 163 additions and 4 deletions.
2 changes: 1 addition & 1 deletion migrations/2024_04_14_000000_add_meta_search_columns.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public function up()
);
} elseif (in_array($driver, ['pgsql', 'sqlite'])) {
$table->rawIndex(
"metable_type, key, substr(value, 1, $stringIndexLength)",
"metable_type, key, SUBSTR(value, 1, $stringIndexLength)",
'value_string_prefix_index'
);
}
Expand Down
103 changes: 100 additions & 3 deletions src/Metable.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,23 @@ public function scopeWhereMeta(
'meta',
function (Builder $q) use ($key, $operator, $stringValue, $value) {
$q->where('key', $key);
$q->where('value', $operator, $stringValue);
[
$needPartialMatch,
$needExactMatch
] = $this->determineQueryValueMatchTypes($q, [$stringValue]);

if ($needPartialMatch) {
$indexLength = (int)config('metable.stringValueIndexLength', 255);
$q->where(
$q->raw("SUBSTR(value, 1, $indexLength)"),
$operator,
substr($stringValue, 0, $indexLength)
);
}

if ($needExactMatch) {
$q->where('value', $operator, $stringValue);
}

// null and empty string look the same in the database,
// use the type column to differentiate.
Expand Down Expand Up @@ -433,7 +449,27 @@ public function scopeWhereMetaBetween(
'meta',
function (Builder $q) use ($key, $min, $max, $not) {
$q->where('key', $key);
$q->whereBetween('value', [$min, $max], 'and', $not);

[
$needPartialMatch,
$needExactMatch
] = $this->determineQueryValueMatchTypes($q, [$min, $max]);

if ($needPartialMatch) {
$indexLength = (int)config('metable.stringValueIndexLength', 255);
$q->whereBetween(
$q->raw("SUBSTR(value, 1, $indexLength)"),
[
substr($min, 0, $indexLength),
substr($max, 0, $indexLength)
],
'and',
$not
);
}
if ($needExactMatch) {
$q->whereBetween('value', [$min, $max], 'and', $not);
}
}
);
}
Expand Down Expand Up @@ -525,7 +561,27 @@ public function scopeWhereMetaIn(

$q->whereHas('meta', function (Builder $q) use ($key, $values, $not) {
$q->where('key', $key);
$q->whereIn('value', $values, 'and', $not);

[
$needPartialMatch,
$needExactMatch
] = $this->determineQueryValueMatchTypes($q, $values);
if ($needPartialMatch) {
$indexLength = (int)config('metable.stringValueIndexLength', 255);
$q->whereIn(
$q->raw("SUBSTR(value, 1, $indexLength)"),
array_map(
fn ($val) => substr($val, 0, $indexLength),
$values
),
'and',
$not
);
}

if ($needExactMatch) {
$q->whereIn('value', $values, 'and', $not);
}
});
}

Expand Down Expand Up @@ -578,6 +634,16 @@ public function scopeOrderByMeta(
bool $strict = false
): void {
$table = $this->joinMetaTable($q, $key, $strict ? 'inner' : 'left');

[$needPartialMatch] = $this->determineQueryValueMatchTypes($q, []);
if ($needPartialMatch) {
$indexLength = (int)config('metable.stringValueIndexLength', 255);
$q->orderBy(
$q->raw("SUBSTR({$table}.value, 1, $indexLength)"),
$direction
);
}

$q->orderBy("{$table}.value", $direction);
}

Expand Down Expand Up @@ -895,6 +961,37 @@ private function getHandlerForValue(mixed $value): HandlerInterface
return $registry->getHandlerForValue($value);
}

/**
* @param Builder $q
* @param string[] $stringValues
* @return array{bool, bool} [needPartialMatch, needExactMatch]
*/
protected function determineQueryValueMatchTypes(
Builder $q,
array $stringValues
): array {
$driver = $q->getConnection()->getDriverName();
$indexLength = (int)config('metable.stringValueIndexLength', 255);

// only sqlite and pgsql support expression indexes, which must be partially matched
// mysql and mariadb support prefix indexes, which works with the entire value
// sqlserv does not support any substring indexing mechanism
if (!in_array($driver, ['sqlite', 'pgsql'])) {
return [false, true];
}
// if any value is longer than the index length, we need to do both a
// substring match to leverage the index and an exact match to avoid false positives
foreach ($stringValues as $stringValue) {
if (strlen($stringValue) > $indexLength) {
return [true, true];
}
}

// if all values are shorter than the index length,
// we only need to do a substring match
return [true, false];
}

abstract public function getKey();

abstract public function getMorphClass();
Expand Down
62 changes: 62 additions & 0 deletions tests/Integration/MetableTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -666,6 +666,68 @@ public function test_it_can_order_query_by_numeric_meta_value_strict(): void
$this->assertEquals([1, 3], $results2->modelKeys());
}

public function test_it_can_query_long_strings(): void
{
config()->set('metable.stringValueIndexLength', 255);
$this->useDatabase();
$metable1 = $this->createMetable();
$metable1->setMeta('foo', $val1 = str_repeat('a', 255) . 'm');
$metable2 = $this->createMetable();
$metable2->setMeta('foo', $val2 = str_repeat('a', 255) . 'f');

$this->assertSame(
[$metable1->getKey()],
SampleMetable::whereMeta('foo', $val1)->get()->modelKeys()
);
$this->assertSame(
[$metable2->getKey()],
SampleMetable::whereMeta('foo', $val2)->get()->modelKeys()
);

$this->assertSame(
[$metable1->getKey()],
SampleMetable::whereMetaIn('foo', [$val1])->get()->modelKeys()
);

$this->assertSame(
[$metable2->getKey()],
SampleMetable::whereMetaIn('foo', [$val2])->get()->modelKeys()
);

$this->assertSame(
[$metable1->getKey(), $metable2->getKey()],
SampleMetable::whereMetaIn('foo', [$val1, $val2])->get()->modelKeys()
);

$this->assertSame(
[$metable2->getKey()],
SampleMetable::whereMetaBetween(
'foo',
str_repeat('a', 256),
str_repeat('a', 255) . 'l'
)->get()->modelKeys()
);

$this->assertSame(
[$metable1->getKey()],
SampleMetable::whereMetaBetween(
'foo',
str_repeat('a', 255) . 'm',
str_repeat('a', 255) . 'z'
)->get()->modelKeys()
);

$this->assertSame(
[$metable2->getKey(), $metable1->getKey()],
SampleMetable::orderByMeta('foo', 'asc')->get()->modelKeys()
);

$this->assertSame(
[$metable1->getKey(), $metable2->getKey()],
SampleMetable::orderByMeta('foo', 'desc')->get()->modelKeys()
);
}

public function test_set_relation_updates_index(): void
{
$metable = $this->makeMetable();
Expand Down

0 comments on commit 3862d24

Please sign in to comment.