[1.0.0] PDO Trigram Implementation Pt # 3

composer.json

@@ -81,6 +81,7 @@
       "LDAP_TESTS_ENABLED=1 php -d xdebug.mode=off vendor/bin/phpunit --testsuite integration"
     ],
     "test-load": "@php -d xdebug.mode=off -d opcache.enable_cli=1 -d opcache.jit=off tests/bin/ldap-load-test.php --backend=sqlite --runner=pcntl --duration=10 --warmup=2 --clients=8 --output=text",
+    "test-load-docker": "tests/profile/load-test.sh",
     "test-load-compare": "@php -d xdebug.mode=off -d opcache.enable_cli=1 -d opcache.jit_buffer_size=128M -d opcache.jit=tracing tests/bin/ldap-bench-compare.php",
     "profile": "tests/profile/profile.sh",
     "profile-up": "docker compose -f tests/profile/docker-compose.yml up -d --build --wait",

src/FreeDSx/Ldap/Server/Backend/Storage/Adapter/Pdo/PdoConfig.php

@@ -16,6 +16,7 @@
 use FreeDSx\Ldap\Server\Backend\Storage\Adapter\Dialect\MysqlDialect;
 use FreeDSx\Ldap\Server\Backend\Storage\Adapter\Dialect\PdoDialectInterface;
 use FreeDSx\Ldap\Server\Backend\Storage\Adapter\Dialect\SqliteDialect;
+use FreeDSx\Ldap\Server\Backend\Storage\Adapter\SubstringIndex\Fts5SubstringIndex;
 use FreeDSx\Ldap\Server\Backend\Storage\Adapter\SubstringIndex\SubstringIndexInterface;
 use FreeDSx\Ldap\Server\Backend\Storage\Adapter\SubstringIndex\TrigramSubstringIndex;
 use PDO;
@@ -63,6 +64,9 @@ public static function forSqlite(string $path): self
             ],
             serializeSwooleWrites: true,
             driverExtension: 'pdo_sqlite',
+            substringIndex: Fts5SubstringIndex::isSupported()
+                ? new Fts5SubstringIndex()
+                : new TrigramSubstringIndex(),
         );
     }
 

src/FreeDSx/Ldap/Server/Backend/Storage/Adapter/SubstringIndex/Fts5SubstringIndex.php

@@ -0,0 +1,179 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * This file is part of the FreeDSx LDAP package.
+ *
+ * (c) Chad Sikorra <Chad.Sikorra@gmail.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace FreeDSx\Ldap\Server\Backend\Storage\Adapter\SubstringIndex;
+
+use FreeDSx\Ldap\Entry\Entry;
+use FreeDSx\Ldap\Schema\Text;
+use FreeDSx\Ldap\Server\Backend\Storage\Adapter\Dialect\PdoDialectInterface;
+use FreeDSx\Ldap\Server\Backend\Storage\Adapter\SqlFilter\SqlFilterResult;
+use PDO;
+use Throwable;
+
+/**
+ * Native SQLite substring index using an FTS5 trigram virtual table synced from the value sidecar by triggers.
+ *
+ * @author Chad Sikorra <Chad.Sikorra@gmail.com>
+ */
+final class Fts5SubstringIndex implements SubstringIndexInterface
+{
+    /**
+     * @var list<string>
+     */
+    public const DEFAULT_ATTRIBUTES = [
+        'cn',
+        'sn',
+        'givenName',
+        'displayName',
+        'uid',
+        'mail',
+        'ou',
+    ];
+
+    private const MIN_TERM_LENGTH = 3;
+
+    private const VTABLE_SQL = <<<SQL
+        CREATE VIRTUAL TABLE IF NOT EXISTS entry_attribute_fts USING fts5(
+            value_original,
+            content = 'entry_attribute_values',
+            content_rowid = 'rowid',
+            tokenize = 'trigram'
+        )
+        SQL;
+
+    private const INSERT_TRIGGER_SQL = <<<SQL
+        CREATE TRIGGER IF NOT EXISTS entry_attribute_fts_ai AFTER INSERT ON entry_attribute_values
+            WHEN new.attr_name_lower IN (%s)
+        BEGIN
+            INSERT INTO entry_attribute_fts(rowid, value_original) VALUES (new.rowid, new.value_original);
+        END
+        SQL;
+
+    private const DELETE_TRIGGER_SQL = <<<SQL
+        CREATE TRIGGER IF NOT EXISTS entry_attribute_fts_ad AFTER DELETE ON entry_attribute_values
+            WHEN old.attr_name_lower IN (%s)
+        BEGIN
+            INSERT INTO entry_attribute_fts(entry_attribute_fts, rowid, value_original)
+                VALUES ('delete', old.rowid, old.value_original);
+        END
+        SQL;
+
+    private const MATCH_SQL = <<<SQL
+        lc_dn IN (
+            SELECT s.entry_lc_dn FROM entry_attribute_values s
+            WHERE s.attr_name_lower = ?
+              AND s.rowid IN (SELECT rowid FROM entry_attribute_fts WHERE entry_attribute_fts MATCH ?)
+        )
+        SQL;
+
+    private static ?bool $supported = null;
+
+    /**
+     * @var array<string, true> Indexed attribute names, lowercased.
+     */
+    private readonly array $attributes;
+
+    /**
+     * @param list<string> $attributes
+     */
+    public function __construct(array $attributes = self::DEFAULT_ATTRIBUTES)
+    {
+        $set = [];
+        foreach ($attributes as $attribute) {
+            $set[strtolower($attribute)] = true;
+        }
+
+        $this->attributes = $set;
+    }
+
+    /**
+     * Whether this SQLite build has FTS5 with the trigram tokenizer (compile-time, so a throwaway probe suffices).
+     */
+    public static function isSupported(): bool
+    {
+        if (self::$supported !== null) {
+            return self::$supported;
+        }
+
+        if (!extension_loaded('pdo_sqlite')) {
+            return self::$supported = false;
+        }
+
+        try {
+            $pdo = new PDO('sqlite::memory:');
+            $pdo->exec("CREATE VIRTUAL TABLE probe USING fts5(x, tokenize = 'trigram')");
+
+            return self::$supported = true;
+        } catch (Throwable) {
+            return self::$supported = false;
+        }
+    }
+
+    public function schemaStatements(PdoDialectInterface $dialect): array
+    {
+        if ($this->attributes === []) {
+            return [];
+        }
+
+        $scope = implode(
+            ', ',
+            array_map(
+                static fn(string $attribute): string => "'" . str_replace("'", "''", $attribute) . "'",
+                array_keys($this->attributes),
+            ),
+        );
+
+        return [
+            self::VTABLE_SQL,
+            sprintf(self::INSERT_TRIGGER_SQL, $scope),
+            sprintf(self::DELETE_TRIGGER_SQL, $scope),
+        ];
+    }
+
+    public function maintain(
+        string $lcDn,
+        Entry $entry,
+        callable $execute,
+    ): void {
+        // The sync triggers index/de-index off the sidecar rows PdoStorage already writes; nothing to do per entry.
+    }
+
+    public function buildSubstringPredicate(
+        string $attributeLower,
+        array $fragments,
+    ): ?SqlFilterResult {
+        if (!isset($this->attributes[$attributeLower])) {
+            return null;
+        }
+
+        $terms = [];
+        foreach ($fragments as $fragment) {
+            if (!Text::isUtf8($fragment) || mb_strlen($fragment, 'UTF-8') < self::MIN_TERM_LENGTH) {
+                continue;
+            }
+
+            $terms[] = '"' . str_replace('"', '""', $fragment) . '"';
+        }
+
+        if ($terms === []) {
+            return null;
+        }
+
+        return new SqlFilterResult(
+            self::MATCH_SQL,
+            [$attributeLower, implode(' AND ', $terms)],
+            isExact: false,
+            referencedAttributes: [$attributeLower],
+        );
+    }
+}

src/FreeDSx/Ldap/Server/Backend/Storage/Adapter/SubstringIndex/SubstringIndexReindexer.php

@@ -0,0 +1,73 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * This file is part of the FreeDSx LDAP package.
+ *
+ * (c) Chad Sikorra <Chad.Sikorra@gmail.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace FreeDSx\Ldap\Server\Backend\Storage\Adapter\SubstringIndex;
+
+use FreeDSx\Ldap\Entry\Dn;
+use FreeDSx\Ldap\Server\Backend\Storage\EntryStorageInterface;
+use FreeDSx\Ldap\Server\Backend\Storage\StorageListOptions;
+
+/**
+ * Backfills the substring index by re-storing every entry; run it after enabling indexing on an existing directory or changing the indexed attributes.
+ *
+ * @api
+ *
+ * @author Chad Sikorra <Chad.Sikorra@gmail.com>
+ */
+final readonly class SubstringIndexReindexer
+{
+    public function __construct(
+        private EntryStorageInterface $storage,
+    ) {}
+
+    /**
+     * Re-store every entry in one transaction via raw storage: the substring index is rebuilt while operational attributes are preserved verbatim and no change is journaled.
+     */
+    public function reindex(): void
+    {
+        $dns = $this->collectDns();
+
+        $this->storage->atomic(function () use ($dns): void {
+            foreach ($dns as $dn) {
+                $entry = $this->storage->find($dn);
+                if ($entry === null) {
+                    continue;
+                }
+
+                $this->storage->store($entry);
+            }
+        });
+    }
+
+    /**
+     * Drain the entry DNs up front so the entries table is not being read while it is re-stored.
+     *
+     * @return list<Dn>
+     */
+    private function collectDns(): array
+    {
+        $dns = [];
+        foreach ($this->storage->namingContexts() as $namingContext) {
+            $stream = $this->storage->list(StorageListOptions::matchAll(
+                $namingContext,
+                true,
+            ));
+
+            foreach ($stream->entries as $entry) {
+                $dns[] = $entry->getDn();
+            }
+        }
+
+        return $dns;
+    }
+}

tests/bin/internals/bench_bootstrap.php

@@ -32,6 +32,14 @@
     return;
 }
 
+/**
+ * The load-test driver forks one client process per connection. Tracing JIT corrupts forked
+ * workers (SIGILL under load), so the driver runs interpreted, mirroring the pcntl server default.
+ */
+if (basename($argv[0] ?? '') === 'ldap-load-test.php') {
+    return;
+}
+
 /**
  * In multi-driver mode the parent does no hot work — it only orchestrates child
  * workers (each spawned with JIT flags). Skipping the JIT re-exec here avoids a

tests/bin/ldap-load-test.php

@@ -6,8 +6,8 @@
  * Load-test driver for the FreeDSx LDAP storage backends.
  *
  * Spawns an LDAP server with the chosen backend+runner, fires concurrent LDAP operations from a
- * Swoole coroutine pool, then prints per-op latency/throughput/error stats. Run with --help to
- * see the full option list.
+ * forked client pool (one process per client), then prints per-op latency/throughput/error stats.
+ * Run with --help to see the full option list.
  */
 
 use Symfony\Component\Console\Application;