From 4a231d92df231569c582105df824d9935827fb78 Mon Sep 17 00:00:00 2001 From: Chad Sikorra Date: Fri, 29 May 2026 11:41:09 -0400 Subject: [PATCH] Support LDIF streaming when parsing. This allows large LDIF to be processed without issue (such as LDIF generated from the dump method, which could be from a large storage). --- docs/Server/General-Usage.md | 87 +++++++++++- src/FreeDSx/Ldap/LdapServer.php | 41 +++--- src/FreeDSx/Ldap/Ldif/LdifChanges.php | 25 ++++ src/FreeDSx/Ldap/Ldif/LdifParser.php | 38 +++-- .../Ldap/Ldif/Loader/FileLdifLoader.php | 37 ++++- .../Ldap/Ldif/Loader/LdifLoaderInterface.php | 8 +- .../Ldap/Ldif/Loader/StringLdifLoader.php | 24 +++- .../Ldif/Parser/LdifChangeRecordParser.php | 28 ++-- .../Ldap/Ldif/Parser/LdifDirective.php | 1 + .../Ldap/Ldif/Parser/LdifLineCursor.php | 131 +++++++++++++----- .../Server/Backend/Storage/LdapImporter.php | 31 +---- tests/integration/Ldif/LdapDumpServerTest.php | 8 +- tests/unit/Ldif/LdifParserTest.php | 37 ++--- tests/unit/Ldif/LdifWriterTest.php | 3 +- tests/unit/Ldif/Loader/FileLdifLoaderTest.php | 15 +- .../Parser/LdifChangeRecordParserTest.php | 41 +++--- .../Backend/Storage/LdapImporterTest.php | 17 ++- 17 files changed, 385 insertions(+), 187 deletions(-) diff --git a/docs/Server/General-Usage.md b/docs/Server/General-Usage.md index c76153a4..9ccd4f09 100644 --- a/docs/Server/General-Usage.md +++ b/docs/Server/General-Usage.md @@ -17,9 +17,11 @@ General LDAP Server Usage * [MysqlStorage](#mysqlstorage) * [Proxy Backend](#proxy-backend) * [Custom Filter Evaluation](#custom-filter-evaluation) -* [Loading LDIF Data](#loading-ldif-data) +* [LDIF Data](#ldif-data) * [Seeding Initial Entries](#seeding-initial-entries) * [Replaying LDIF Changelogs](#replaying-ldif-changelogs) + * [Dumping the Directory](#dumping-the-directory) + * [Inspecting Parsed LDIF](#inspecting-parsed-ldif) * [Authentication](#authentication) * [Default Authentication](#default-authentication) * [Custom Bind Name Resolution](#custom-bind-name-resolution) @@ -677,7 +679,12 @@ class MySqlFilterEvaluator implements FilterEvaluatorInterface } ``` -## Loading LDIF Data +## LDIF Data + +`seed()`, `applyChanges()`, and `dump()` all stream. LDIF input is always taken through `LdifLoaderInterface` +such as `FileLdifLoader` for a path, `StringLdifLoader` for an in-memory string, or your own implementation for any +other source (database, remote URL, gzip stream, etc.). LDIF output uses the parallel `LdifOutputInterface` such as +`FileLdifOutput` and `StringLdifOutput`. ### Seeding Initial Entries @@ -704,9 +711,8 @@ $server->run(); The optional second argument is the creator DN, stamped as `creatorsName`/`modifiersName` on each imported entry — defaults to the empty (anonymous) DN. -`seed()` accepts only content records (entries without `changetype:`). LDIF sources are pluggable via `LdifLoaderInterface`: -use `FileLdifLoader`, `StringLdifLoader`, or implement your own (e.g. fetching from a database or remote URL). The -operation itself is an upsert that overwrites. +`seed()` accepts only content records (entries without `changetype:`) and requires depth-first input (parents first, +then children entries). LDIF produced by `dump()` is already in this order. The operation itself is an upsert that overwrites. For Swoole factories (`::forSwoole()`), call `seed()` inside `Swoole\Coroutine\run()` so the adapter's coroutine-scoped connection is available during import. @@ -749,6 +755,77 @@ Unlike `seed()`, `applyChanges()` dispatches each request through the same write requests. Supported changetypes: `add`, `delete`, `modify` (`add:`/`delete:`/`replace:` mod-specs), and `modrdn`/`moddn` (rename or move; supports optional `newsuperior:` for moving across subtrees). +### Dumping the Directory + +`LdapServer::dump()` streams the configured storage backend's entries to an LDIF output as RFC 2849 content records. +Operational attributes (`entryUUID`, `createTimestamp`, etc.) are preserved. So `dump()` then `seed()` restores the +entries exactly as they were. + +```php +use FreeDSx\Ldap\LdapServer; +use FreeDSx\Ldap\Ldif\Output\FileLdifOutput; +use FreeDSx\Ldap\Server\Backend\Storage\Adapter\SqliteStorage; + +(new LdapServer()) + ->useStorage(SqliteStorage::forPcntl('/var/lib/myapp/ldap.sqlite')) + ->dump(new FileLdifOutput('/var/backups/ldap-snapshot.ldif')); +``` + +For in-memory use (logging, tests, piping over the network) use `StringLdifOutput`, which collects the chunks and is +both `Stringable` and exposes `getLdif()`: + +```php +use FreeDSx\Ldap\Ldif\Output\StringLdifOutput; + +$output = new StringLdifOutput(); +$server->dump($output); + +echo $output; // or $output->getLdif() +``` + +Use `DumpOptions` to filter the dump by any filter you want. Useful for partial backups or extracting a single OU: + +```php +use FreeDSx\Ldap\Entry\Dn; +use FreeDSx\Ldap\Search\Filters; +use FreeDSx\Ldap\Server\Backend\Storage\Export\DumpOptions; + +$options = (new DumpOptions()) + ->setBaseDn(new Dn('ou=people,dc=example,dc=com')) + ->setFilter(Filters::equal('objectClass', 'inetOrgPerson')); + +$server->dump( + new FileLdifOutput('/tmp/people.ldif'), + $options, +); +``` + +### Inspecting Parsed LDIF + +For one-off tooling that needs to inspect a parsed LDIF before applying it (counting records, filtering by changetype, +etc) `LdifChanges` is a buffered collection with type filters: + +```php +use FreeDSx\Ldap\Ldif\LdifChanges; +use FreeDSx\Ldap\Ldif\Loader\FileLdifLoader; + +$changes = LdifChanges::fromLoader(new FileLdifLoader('/path/to/changes.ldif')); + +foreach ($changes->entries() as $entry) { + // each AddRequest's Entry +} + +$changes->count(); // total changes +$changes->adds(); // AddRequest[] +$changes->modifies(); // ModifyRequest[] +$changes->deletes(); // DeleteRequest[] +$changes->modifyDns(); // ModifyDnRequest[] +``` + +`LdifChanges::fromString($ldif)` is the same flow for an in-memory string. The collection materializes every request, +so prefer the streaming `seed()`/`applyChanges()`/`dump()` methods normal data paths; `LdifChanges` is best suited to +small change sets. + ## Authentication The `PasswordAuthenticatableInterface` covers all bind types through two methods: diff --git a/src/FreeDSx/Ldap/LdapServer.php b/src/FreeDSx/Ldap/LdapServer.php index 30cfe35f..122cfa94 100644 --- a/src/FreeDSx/Ldap/LdapServer.php +++ b/src/FreeDSx/Ldap/LdapServer.php @@ -14,6 +14,7 @@ namespace FreeDSx\Ldap; use FreeDSx\Ldap\Entry\Dn; +use FreeDSx\Ldap\Entry\Entry; use FreeDSx\Ldap\Exception\InvalidArgumentException; use FreeDSx\Ldap\Exception\LdifParseException; use FreeDSx\Ldap\Exception\OperationException; @@ -21,6 +22,7 @@ use FreeDSx\Ldap\Ldif\LdifParser; use FreeDSx\Ldap\Ldif\Loader\LdifLoaderInterface; use FreeDSx\Ldap\Ldif\Output\LdifOutputInterface; +use FreeDSx\Ldap\Operation\Request\AddRequest; use FreeDSx\Ldap\Schema\SchemaValidationMode; use FreeDSx\Ldap\Schema\Validation\SchemaValidator; use FreeDSx\Ldap\Server\AccessControl\AccessControlInterface; @@ -42,6 +44,7 @@ use FreeDSx\Ldap\Server\ServerRunner\ServerRunnerInterface; use FreeDSx\Ldap\Server\Backend\Storage\FilterEvaluatorInterface; use FreeDSx\Socket\Exception\ConnectionException; +use Generator; use Psr\Log\LoggerInterface; /** @@ -172,25 +175,34 @@ public function seed( throw new RuntimeException('seed() requires a storage backend configured via useStorage().'); } - $changes = (new LdifParser()) - ->parse($loader->load()); - - if (!$changes->isAddOnly()) { - throw new RuntimeException( - 'seed() only accepts content records (adds). Use applyChanges() for modify/delete/rename.', - ); - } - (new LdapImporter( $backend->getStorage(), $backend->getOperationalAttributeGenerator(), $backend->getSchemaValidator(), $creatorDn, - ))->importEntries($changes->entries()); + ))->importEntries($this->streamSeedEntries($loader)); return $this; } + /** + * @return Generator + * @throws RuntimeException when the LDIF contains a non-add change record + * @throws LdifParseException + */ + private function streamSeedEntries(LdifLoaderInterface $loader): Generator + { + foreach ((new LdifParser())->parse($loader) as $request) { + if (!$request instanceof AddRequest) { + throw new RuntimeException( + 'seed() only accepts content records (adds). Use applyChanges() for modify/delete/rename.', + ); + } + + yield $request->getEntry(); + } + } + /** * Replays an LDIF changelog against the configured backend via the live write path. * @@ -208,17 +220,10 @@ public function applyChanges(LdifLoaderInterface $loader): self throw new RuntimeException('applyChanges() requires a writable backend.'); } - $changes = (new LdifParser()) - ->parse($loader->load()); - - if (count($changes) === 0) { - return $this; - } - (new WriteRequestReplayer( $backend, $this->options->getWriteHandlers(), - ))->apply($changes); + ))->apply((new LdifParser())->parse($loader)); return $this; } diff --git a/src/FreeDSx/Ldap/Ldif/LdifChanges.php b/src/FreeDSx/Ldap/Ldif/LdifChanges.php index c2277429..6316e6be 100644 --- a/src/FreeDSx/Ldap/Ldif/LdifChanges.php +++ b/src/FreeDSx/Ldap/Ldif/LdifChanges.php @@ -16,6 +16,8 @@ use ArrayIterator; use Countable; use FreeDSx\Ldap\Entry\Entry; +use FreeDSx\Ldap\Ldif\Loader\LdifLoaderInterface; +use FreeDSx\Ldap\Ldif\Loader\StringLdifLoader; use FreeDSx\Ldap\Operation\Request\AddRequest; use FreeDSx\Ldap\Operation\Request\DeleteRequest; use FreeDSx\Ldap\Operation\Request\ModifyDnRequest; @@ -48,6 +50,29 @@ public function __construct(RequestInterface ...$requests) $this->requests = $requests; } + /** + * Buffers a loader's parsed records into a collection. For streaming, iterate {@see LdifParser::parse()} directly. + */ + public static function fromLoader( + LdifLoaderInterface $loader, + LdifParser $parser = new LdifParser(), + ): self { + return new self(...$parser->parse($loader)); + } + + /** + * Convenience for buffering an in-memory LDIF string via {@see StringLdifLoader}. + */ + public static function fromString( + string $ldif, + LdifParser $parser = new LdifParser(), + ): self { + return self::fromLoader( + new StringLdifLoader($ldif), + $parser, + ); + } + /** * @return RequestInterface[] */ diff --git a/src/FreeDSx/Ldap/Ldif/LdifParser.php b/src/FreeDSx/Ldap/Ldif/LdifParser.php index 7574c6a3..65287b56 100644 --- a/src/FreeDSx/Ldap/Ldif/LdifParser.php +++ b/src/FreeDSx/Ldap/Ldif/LdifParser.php @@ -15,13 +15,15 @@ use FreeDSx\Ldap\Entry\Entry; use FreeDSx\Ldap\Exception\LdifParseException; +use FreeDSx\Ldap\Ldif\Loader\LdifLoaderInterface; +use FreeDSx\Ldap\Ldif\Loader\StringLdifLoader; use FreeDSx\Ldap\Ldif\Parser\LdifChangeRecordParser; use FreeDSx\Ldap\Ldif\Parser\LdifLineCursor; use FreeDSx\Ldap\Operation\Request\AddRequest; use FreeDSx\Ldap\Operation\Request\RequestInterface; use FreeDSx\Ldap\Operations; +use Generator; -use function count; use function sprintf; /** @@ -42,12 +44,15 @@ public function __construct( ) {} /** + * Streams parsed write requests from an LDIF source. + * + * @return Generator * @throws LdifParseException */ - public function parse(string $ldif): LdifChanges + public function parse(LdifLoaderInterface $loader): Generator { - $cursor = LdifLineCursor::forInput($ldif); - $requests = []; + $cursor = LdifLineCursor::forLoader($loader); + $recordsSeen = 0; while (!$cursor->atEnd()) { $line = $cursor->current(); @@ -64,15 +69,27 @@ public function parse(string $ldif): LdifChanges $key = $cursor->keyOf($line); if ($key === self::DN) { - $requests[] = $this->parseRecord($cursor); + yield $this->parseRecord($cursor); + + $recordsSeen++; } elseif ($key === self::VERSION) { - $this->assertVersion($cursor, count($requests)); + $this->assertVersion($cursor, $recordsSeen); } else { $cursor->error('Expected a "dn:" line to begin a record'); } } + } - return new LdifChanges(...$requests); + /** + * Convenience for parsing an in-memory LDIF string. + * + * @return Generator + * @throws LdifParseException + */ + public static function parseString(string $ldif): Generator + { + return (new self()) + ->parse(new StringLdifLoader($ldif)); } /** @@ -114,8 +131,8 @@ private function parseContentRecord( $directive = $cursor->readDirective(); if ($directive->is(self::CHANGETYPE)) { - $cursor->errorAt( - $directive->position, + $cursor->errorFor( + $directive, '"changetype:" must be the first directive after dn', ); } @@ -153,10 +170,12 @@ private function assertVersion( int $recordsSeen, ): void { $at = $cursor->position(); + $sourceLine = $cursor->current(); if ($recordsSeen !== 0) { $cursor->errorAt( $at, + $sourceLine, 'The version directive must appear before any records', ); } @@ -166,6 +185,7 @@ private function assertVersion( if ($version !== '1') { $cursor->errorAt( $at, + $sourceLine, sprintf( 'Unsupported LDIF version "%s"', $version, diff --git a/src/FreeDSx/Ldap/Ldif/Loader/FileLdifLoader.php b/src/FreeDSx/Ldap/Ldif/Loader/FileLdifLoader.php index 48206afc..8e8d80f9 100644 --- a/src/FreeDSx/Ldap/Ldif/Loader/FileLdifLoader.php +++ b/src/FreeDSx/Ldap/Ldif/Loader/FileLdifLoader.php @@ -14,13 +14,19 @@ namespace FreeDSx\Ldap\Ldif\Loader; use FreeDSx\Ldap\Exception\RuntimeException; +use Generator; -use function file_get_contents; +use function fclose; +use function feof; +use function fgets; +use function fopen; use function is_file; use function is_readable; +use function rtrim; +use function sprintf; /** - * Loads LDIF text from a file. + * Streams LDIF lines from a file. * * @author Chad Sikorra */ @@ -29,9 +35,10 @@ public function __construct(private string $file) {} /** + * @return Generator * @throws RuntimeException when the file is missing or unreadable */ - public function load(): string + public function load(): Generator { if (!is_file($this->file) || !is_readable($this->file)) { throw new RuntimeException(sprintf( @@ -40,15 +47,33 @@ public function load(): string )); } - $ldif = file_get_contents($this->file); + $handle = fopen( + $this->file, + 'r', + ); - if ($ldif === false) { + if ($handle === false) { throw new RuntimeException(sprintf( 'Unable to read the LDIF file "%s".', $this->file, )); } - return $ldif; + try { + while (!feof($handle)) { + $line = fgets($handle); + + if ($line === false) { + break; + } + + yield rtrim( + $line, + "\r\n", + ); + } + } finally { + fclose($handle); + } } } diff --git a/src/FreeDSx/Ldap/Ldif/Loader/LdifLoaderInterface.php b/src/FreeDSx/Ldap/Ldif/Loader/LdifLoaderInterface.php index ac28ea8a..dddccb88 100644 --- a/src/FreeDSx/Ldap/Ldif/Loader/LdifLoaderInterface.php +++ b/src/FreeDSx/Ldap/Ldif/Loader/LdifLoaderInterface.php @@ -13,15 +13,17 @@ namespace FreeDSx\Ldap\Ldif\Loader; +use Generator; + /** - * Yields LDIF text from some source (file, string, database, etc.) for parsing. + * Yields LDIF lines from some source (file, string, database, etc.) for streaming parse. * * @author Chad Sikorra */ interface LdifLoaderInterface { /** - * Return the LDIF text to be parsed. + * @return Generator LDIF lines without trailing newlines */ - public function load(): string; + public function load(): Generator; } diff --git a/src/FreeDSx/Ldap/Ldif/Loader/StringLdifLoader.php b/src/FreeDSx/Ldap/Ldif/Loader/StringLdifLoader.php index 95349709..449f171b 100644 --- a/src/FreeDSx/Ldap/Ldif/Loader/StringLdifLoader.php +++ b/src/FreeDSx/Ldap/Ldif/Loader/StringLdifLoader.php @@ -13,8 +13,12 @@ namespace FreeDSx\Ldap\Ldif\Loader; +use Generator; + +use function preg_split; + /** - * Loads LDIF text held in memory as a string. + * Yields LDIF lines from a string held in memory. * * @author Chad Sikorra */ @@ -22,8 +26,22 @@ { public function __construct(private string $ldif) {} - public function load(): string + /** + * @return Generator + */ + public function load(): Generator { - return $this->ldif; + $lines = preg_split( + "/\r\n|\r|\n/", + $this->ldif, + ); + + if ($lines === false) { + return; + } + + foreach ($lines as $line) { + yield $line; + } } } diff --git a/src/FreeDSx/Ldap/Ldif/Parser/LdifChangeRecordParser.php b/src/FreeDSx/Ldap/Ldif/Parser/LdifChangeRecordParser.php index 62f90a30..a22d4400 100644 --- a/src/FreeDSx/Ldap/Ldif/Parser/LdifChangeRecordParser.php +++ b/src/FreeDSx/Ldap/Ldif/Parser/LdifChangeRecordParser.php @@ -77,8 +77,8 @@ private function readChangetype(LdifLineCursor $cursor): string $directive = $cursor->readDirective(); if (!$directive->is(self::CHANGETYPE)) { - $cursor->errorAt( - $directive->position, + $cursor->errorFor( + $directive, 'Expected "changetype:" directive after DN', ); } @@ -146,8 +146,8 @@ private function parseModSpec( $attr = $directive->value; if ($op === null) { - $cursor->errorAt( - $directive->position, + $cursor->errorFor( + $directive, sprintf( 'Expected an add:, delete:, or replace: mod-spec, got "%s:"', $directive->name, @@ -196,8 +196,8 @@ private function readModSpecValues( $directive = $cursor->readDirective(); if (!$directive->is($attr)) { - $cursor->errorAt( - $directive->position, + $cursor->errorFor( + $directive, sprintf( 'Mod-spec attribute "%s" does not match values for "%s"', $directive->name, @@ -245,8 +245,8 @@ private function parseModRdnRecord( $directive, $cursor, ), - null => $cursor->errorAt( - $directive->position, + null => $cursor->errorFor( + $directive, sprintf( 'Unexpected directive "%s:" in modrdn record', $directive->name, @@ -279,8 +279,8 @@ private function assignFieldOnce( LdifLineCursor $cursor, ): string { if ($current !== null) { - $cursor->errorAt( - $directive->position, + $cursor->errorFor( + $directive, sprintf( 'Duplicate "%s:" in modrdn record', strtolower($directive->name), @@ -300,8 +300,8 @@ private function assignDeleteOldRdnOnce( LdifLineCursor $cursor, ): bool { if ($current !== null) { - $cursor->errorAt( - $directive->position, + $cursor->errorFor( + $directive, 'Duplicate "deleteoldrdn:" in modrdn record', ); } @@ -312,8 +312,8 @@ private function assignDeleteOldRdnOnce( return true; } - $cursor->errorAt( - $directive->position, + $cursor->errorFor( + $directive, sprintf( '"deleteoldrdn" must be 0 or 1, got "%s"', $directive->value, diff --git a/src/FreeDSx/Ldap/Ldif/Parser/LdifDirective.php b/src/FreeDSx/Ldap/Ldif/Parser/LdifDirective.php index 346d4b71..0ef77938 100644 --- a/src/FreeDSx/Ldap/Ldif/Parser/LdifDirective.php +++ b/src/FreeDSx/Ldap/Ldif/Parser/LdifDirective.php @@ -26,6 +26,7 @@ public function __construct( public string $name, public string $value, public int $position, + public ?string $sourceLine = null, ) {} /** diff --git a/src/FreeDSx/Ldap/Ldif/Parser/LdifLineCursor.php b/src/FreeDSx/Ldap/Ldif/Parser/LdifLineCursor.php index 1321ea6a..fbcdc199 100644 --- a/src/FreeDSx/Ldap/Ldif/Parser/LdifLineCursor.php +++ b/src/FreeDSx/Ldap/Ldif/Parser/LdifLineCursor.php @@ -14,18 +14,18 @@ namespace FreeDSx\Ldap\Ldif\Parser; use FreeDSx\Ldap\Exception\LdifParseException; +use FreeDSx\Ldap\Ldif\Loader\LdifLoaderInterface; +use Generator; use function base64_decode; -use function count; -use function explode; use function ltrim; -use function str_replace; +use function preg_split; use function strpos; use function strtolower; use function substr; /** - * Cursor over LDIF lines exposing the shared low-level reading primitives. + * Streaming cursor over LDIF lines exposing the shared low-level reading primitives. * * @author Chad Sikorra */ @@ -37,44 +37,76 @@ final class LdifLineCursor private const URL_MARKER = '<'; + private ?string $current = null; + + private int $lineNumber = 0; + + /** + * @param Generator $source + */ + private function __construct(private readonly Generator $source) + { + $this->source->rewind(); + $this->advance(); + } + /** - * @param string[] $lines + * @param iterable $lines */ - private function __construct( - private array $lines, - private int $pos = 0, - ) {} + public static function forIterable(iterable $lines): self + { + return new self(self::toGenerator($lines)); + } public static function forInput(string $ldif): self { - return new self(explode( - "\n", - str_replace( - ["\r\n", "\r"], - "\n", - $ldif, - ), - )); + $lines = preg_split("/\r\n|\r|\n/", $ldif); + + return self::forIterable($lines === false ? [] : $lines); + } + + public static function forLoader(LdifLoaderInterface $loader): self + { + return self::forIterable($loader->load()); } public function atEnd(): bool { - return $this->pos >= count($this->lines); + return $this->current === null; } public function current(): string { - return $this->lines[$this->pos]; + return $this->current ?? ''; } public function position(): int { - return $this->pos; + return $this->lineNumber; } public function advance(): void { - $this->pos++; + if (!$this->source->valid()) { + $this->current = null; + + return; + } + + $this->current = $this->source->current(); + $this->lineNumber++; + $this->source->next(); + } + + /** + * @param iterable $lines + * @return Generator + */ + private static function toGenerator(iterable $lines): Generator + { + foreach ($lines as $line) { + yield $line; + } } /** @@ -84,20 +116,22 @@ public function advance(): void */ public function readDirective(): LdifDirective { - $at = $this->pos; - $line = $this->lines[$at]; + $startLine = $this->lineNumber; + $startSource = $this->current; + $line = $this->current ?? ''; $colon = strpos($line, self::SEPARATOR); if ($colon === false || $colon === 0) { $this->errorAt( - $at, + $startLine, + $startSource, 'Expected a "name: value" directive', ); } $name = substr($line, 0, $colon); $marker = $line[$colon + 1] ?? ''; - $this->pos++; + $this->advance(); if ($marker === self::SEPARATOR) { $value = $this->decodeBase64( @@ -105,11 +139,13 @@ public function readDirective(): LdifDirective substr($line, $colon + 2), ' ', )), - $at, + $startLine, + $startSource, ); } elseif ($marker === self::URL_MARKER) { $this->errorAt( - $at, + $startLine, + $startSource, 'URL-referenced values ("name:< url") are not yet supported', ); } else { @@ -122,7 +158,8 @@ public function readDirective(): LdifDirective return new LdifDirective( $name, $value, - $at, + $startLine, + $startSource, ); } @@ -133,7 +170,7 @@ public function readFolded(string $value): string { while ($this->isAtContinuation()) { $value .= substr($this->current(), 1); - $this->pos++; + $this->advance(); } return $value; @@ -141,10 +178,10 @@ public function readFolded(string $value): string public function skipComment(): void { - $this->pos++; + $this->advance(); while ($this->isAtContinuation()) { - $this->pos++; + $this->advance(); } } @@ -175,15 +212,17 @@ public function keyOf(string $line): string /** * @throws LdifParseException */ - public function decodeBase64( + private function decodeBase64( string $raw, - int $at, + int $line, + ?string $sourceLine, ): string { $decoded = base64_decode($raw, true); if ($decoded === false) { $this->errorAt( - $at, + $line, + $sourceLine, 'A base64-encoded value is not valid', ); } @@ -197,7 +236,8 @@ public function decodeBase64( public function error(string $message): never { $this->errorAt( - $this->pos, + $this->lineNumber, + $this->current, $message, ); } @@ -206,13 +246,28 @@ public function error(string $message): never * @throws LdifParseException */ public function errorAt( - int $at, + int $line, + ?string $sourceLine, string $message, ): never { throw new LdifParseException( $message, - $at + 1, - $this->lines[$at] ?? null, + $line, + $sourceLine, + ); + } + + /** + * @throws LdifParseException + */ + public function errorFor( + LdifDirective $directive, + string $message, + ): never { + $this->errorAt( + $directive->position, + $directive->sourceLine, + $message, ); } } diff --git a/src/FreeDSx/Ldap/Server/Backend/Storage/LdapImporter.php b/src/FreeDSx/Ldap/Server/Backend/Storage/LdapImporter.php index 0e49179e..6ad91dd2 100644 --- a/src/FreeDSx/Ldap/Server/Backend/Storage/LdapImporter.php +++ b/src/FreeDSx/Ldap/Server/Backend/Storage/LdapImporter.php @@ -37,25 +37,20 @@ public function __construct( } /** - * Persist all entries in one atomic batch; no-op when the list is empty. + * Stream entries into storage under a single atomic write. * - * @param Entry[] $entries + * Input must be in depth-first order (each entry's parent appears before its children). Unsorted input will fail + * at the parent check. + * + * @param iterable $entries * @param bool $ignoreValidation when true, skips parent and schema checks. only do this if you know what you're doing. * @throws InvalidArgumentException when a non-top-level entry's parent is not present in storage yet * @throws OperationException when an entry violates the schema and validation mode is strict */ public function importEntries( - array $entries, + iterable $entries, bool $ignoreValidation = false, ): void { - if ($entries === []) { - return; - } - - if (!$ignoreValidation) { - $entries = $this->sortByDepth($entries); - } - $this->storage->atomic(function (EntryStorageInterface $storage) use ($entries, $ignoreValidation): void { foreach ($entries as $entry) { if (!$ignoreValidation) { @@ -78,20 +73,6 @@ public function importEntries( }); } - /** - * @param Entry[] $entries - * @return Entry[] - */ - private function sortByDepth(array $entries): array - { - usort( - $entries, - static fn(Entry $a, Entry $b): int => $a->getDn()->count() <=> $b->getDn()->count(), - ); - - return $entries; - } - /** * @throws InvalidArgumentException */ diff --git a/tests/integration/Ldif/LdapDumpServerTest.php b/tests/integration/Ldif/LdapDumpServerTest.php index 8402a7a3..b3096a03 100644 --- a/tests/integration/Ldif/LdapDumpServerTest.php +++ b/tests/integration/Ldif/LdapDumpServerTest.php @@ -15,9 +15,9 @@ use FreeDSx\Ldap\Entry\Dn; use FreeDSx\Ldap\LdapServer; +use FreeDSx\Ldap\Ldif\LdifChanges; use FreeDSx\Ldap\Ldif\Loader\FileLdifLoader; use FreeDSx\Ldap\Ldif\Loader\StringLdifLoader; -use FreeDSx\Ldap\Ldif\LdifParser; use FreeDSx\Ldap\Server\Backend\Storage\Adapter\InMemoryStorage; use FreeDSx\Ldap\ServerOptions; use Tests\Integration\FreeDSx\Ldap\ServerTestCase; @@ -82,8 +82,7 @@ public function test_the_dump_file_was_written_and_starts_with_the_version_heade public function test_the_dump_file_parses_into_the_seeded_entries(): void { - $loader = new FileLdifLoader(self::$dumpPath); - $parsed = (new LdifParser())->parse($loader->load()); + $parsed = LdifChanges::fromLoader(new FileLdifLoader(self::$dumpPath)); $dns = []; foreach ($parsed->entries() as $entry) { @@ -132,8 +131,7 @@ public function test_a_fresh_server_seeded_from_the_dump_reconstructs_the_direct public function test_the_dump_preserves_operational_attributes_for_round_trip(): void { - $loader = new FileLdifLoader(self::$dumpPath); - $parsed = (new LdifParser())->parse($loader->load()); + $parsed = LdifChanges::fromLoader(new FileLdifLoader(self::$dumpPath)); $alice = null; foreach ($parsed->entries() as $entry) { diff --git a/tests/unit/Ldif/LdifParserTest.php b/tests/unit/Ldif/LdifParserTest.php index 3809451f..065bd520 100644 --- a/tests/unit/Ldif/LdifParserTest.php +++ b/tests/unit/Ldif/LdifParserTest.php @@ -14,23 +14,16 @@ namespace Tests\Unit\FreeDSx\Ldap\Ldif; use FreeDSx\Ldap\Exception\LdifParseException; -use FreeDSx\Ldap\Ldif\LdifParser; +use FreeDSx\Ldap\Ldif\LdifChanges; use FreeDSx\Ldap\Operation\Request\AddRequest; use FreeDSx\Ldap\Operation\Request\ModifyRequest; use PHPUnit\Framework\TestCase; final class LdifParserTest extends TestCase { - private LdifParser $subject; - - protected function setUp(): void - { - $this->subject = new LdifParser(); - } - public function test_it_parses_a_single_content_record_with_multi_valued_attributes(): void { - $result = $this->subject->parse( + $result = LdifChanges::fromString( "dn: cn=foo,dc=example,dc=com\nobjectClass: top\nobjectClass: person\ncn: foo\nsn: Bar\n", ); @@ -52,7 +45,7 @@ public function test_it_parses_a_single_content_record_with_multi_valued_attribu public function test_it_parses_multiple_records_separated_by_blank_lines(): void { - $result = $this->subject->parse( + $result = LdifChanges::fromString( "dn: cn=a,dc=x\ncn: a\n\ndn: cn=b,dc=x\ncn: b\n", ); @@ -64,7 +57,7 @@ public function test_it_parses_multiple_records_separated_by_blank_lines(): void public function test_it_unfolds_continued_lines(): void { - $entry = $this->subject->parse( + $entry = LdifChanges::fromString( "dn: cn=foo,dc=x\ndescription: this is a long\n description value\n", )->entries()[0]; @@ -76,7 +69,7 @@ public function test_it_unfolds_continued_lines(): void public function test_it_decodes_a_base64_value(): void { - $entry = $this->subject->parse( + $entry = LdifChanges::fromString( "dn: cn=foo,dc=x\ncn:: " . base64_encode('Bär') . "\n", )->entries()[0]; @@ -88,7 +81,7 @@ public function test_it_decodes_a_base64_value(): void public function test_it_decodes_a_base64_dn(): void { - $entry = $this->subject->parse( + $entry = LdifChanges::fromString( "dn:: " . base64_encode('cn=Bär,dc=x') . "\ncn: x\n", )->entries()[0]; @@ -100,7 +93,7 @@ public function test_it_decodes_a_base64_dn(): void public function test_it_skips_comments_including_folded_ones(): void { - $result = $this->subject->parse( + $result = LdifChanges::fromString( "# a top comment\ndn: cn=foo,dc=x\n# inline comment\n# folded\n more comment\ncn: foo\n", ); @@ -115,7 +108,7 @@ public function test_it_accepts_a_version_one_header(): void { self::assertCount( 1, - $this->subject->parse("version: 1\ndn: cn=foo,dc=x\ncn: foo\n"), + LdifChanges::fromString("version: 1\ndn: cn=foo,dc=x\ncn: foo\n"), ); } @@ -123,19 +116,19 @@ public function test_it_rejects_an_unsupported_version(): void { $this->expectException(LdifParseException::class); - $this->subject->parse("version: 2\ndn: cn=foo,dc=x\ncn: foo\n"); + LdifChanges::fromString("version: 2\ndn: cn=foo,dc=x\ncn: foo\n"); } public function test_it_rejects_a_version_after_a_record(): void { $this->expectException(LdifParseException::class); - $this->subject->parse("dn: cn=foo,dc=x\ncn: foo\n\nversion: 1\n"); + LdifChanges::fromString("dn: cn=foo,dc=x\ncn: foo\n\nversion: 1\n"); } public function test_it_parses_a_mixed_file_with_content_and_change_records(): void { - $result = $this->subject->parse( + $result = LdifChanges::fromString( "dn: cn=foo,dc=x\ncn: foo\nsn: Bar\n" . "\n" . "dn: cn=baz,dc=x\nchangetype: modify\nreplace: sn\nsn: Quux\n-\n", @@ -157,13 +150,13 @@ public function test_it_rejects_url_referenced_values(): void $this->expectException(LdifParseException::class); $this->expectExceptionMessage('URL-referenced'); - $this->subject->parse("dn: cn=foo,dc=x\njpegPhoto:< file:///tmp/x.jpg\n"); + LdifChanges::fromString("dn: cn=foo,dc=x\njpegPhoto:< file:///tmp/x.jpg\n"); } public function test_it_reports_the_line_number_of_a_malformed_line(): void { try { - $this->subject->parse("dn: cn=foo,dc=x\ncn: foo\nthis-has-no-colon\n"); + LdifChanges::fromString("dn: cn=foo,dc=x\ncn: foo\nthis-has-no-colon\n"); self::fail('Expected an LdifParseException.'); } catch (LdifParseException $e) { self::assertSame(3, $e->getLineNumber()); @@ -176,7 +169,7 @@ public function test_it_reports_the_line_number_of_a_malformed_line(): void public function test_it_parses_an_empty_value(): void { - $entry = $this->subject->parse("dn: cn=foo,dc=x\ndescription:\n")->entries()[0]; + $entry = LdifChanges::fromString("dn: cn=foo,dc=x\ndescription:\n")->entries()[0]; self::assertSame( [''], @@ -186,6 +179,6 @@ public function test_it_parses_an_empty_value(): void public function test_it_returns_no_records_for_empty_input(): void { - self::assertCount(0, $this->subject->parse('')); + self::assertCount(0, LdifChanges::fromString('')); } } diff --git a/tests/unit/Ldif/LdifWriterTest.php b/tests/unit/Ldif/LdifWriterTest.php index 6548767f..733d6b7a 100644 --- a/tests/unit/Ldif/LdifWriterTest.php +++ b/tests/unit/Ldif/LdifWriterTest.php @@ -18,7 +18,6 @@ use FreeDSx\Ldap\Exception\InvalidArgumentException; use FreeDSx\Ldap\Ldif\LdifChanges; use FreeDSx\Ldap\Ldif\LdifOutputOptions; -use FreeDSx\Ldap\Ldif\LdifParser; use FreeDSx\Ldap\Ldif\LdifWriter; use FreeDSx\Ldap\Operation\Request\AddRequest; use FreeDSx\Ldap\Operation\Request\DeleteRequest; @@ -349,7 +348,7 @@ public function test_it_round_trips_through_the_parser_for_all_changetypes(): vo ); $ldif = (new LdifWriter($options))->write($changes); - $parsed = (new LdifParser())->parse($ldif); + $parsed = LdifChanges::fromString($ldif); self::assertSame( $this->normalize($changes), diff --git a/tests/unit/Ldif/Loader/FileLdifLoaderTest.php b/tests/unit/Ldif/Loader/FileLdifLoaderTest.php index d4a5df20..35ffad13 100644 --- a/tests/unit/Ldif/Loader/FileLdifLoaderTest.php +++ b/tests/unit/Ldif/Loader/FileLdifLoaderTest.php @@ -19,7 +19,7 @@ final class FileLdifLoaderTest extends TestCase { - public function test_it_loads_the_file_contents(): void + public function test_it_yields_lines_without_trailing_newlines(): void { $path = tempnam( sys_get_temp_dir(), @@ -32,9 +32,13 @@ public function test_it_loads_the_file_contents(): void ); try { - self::assertSame( - "dn: dc=x\ndc: x\n", + $lines = iterator_to_array( (new FileLdifLoader($path))->load(), + false, + ); + self::assertSame( + ['dn: dc=x', 'dc: x'], + $lines, ); } finally { unlink($path); @@ -45,6 +49,9 @@ public function test_it_throws_for_a_missing_file(): void { $this->expectException(RuntimeException::class); - (new FileLdifLoader('/does/not/exist/seed.ldif'))->load(); + iterator_to_array( + (new FileLdifLoader('/does/not/exist/seed.ldif'))->load(), + false, + ); } } diff --git a/tests/unit/Ldif/Parser/LdifChangeRecordParserTest.php b/tests/unit/Ldif/Parser/LdifChangeRecordParserTest.php index 76213f15..59280f3c 100644 --- a/tests/unit/Ldif/Parser/LdifChangeRecordParserTest.php +++ b/tests/unit/Ldif/Parser/LdifChangeRecordParserTest.php @@ -15,7 +15,7 @@ use FreeDSx\Ldap\Entry\Change; use FreeDSx\Ldap\Exception\LdifParseException; -use FreeDSx\Ldap\Ldif\LdifParser; +use FreeDSx\Ldap\Ldif\LdifChanges; use FreeDSx\Ldap\Operation\Request\AddRequest; use FreeDSx\Ldap\Operation\Request\DeleteRequest; use FreeDSx\Ldap\Operation\Request\ModifyDnRequest; @@ -24,16 +24,9 @@ final class LdifChangeRecordParserTest extends TestCase { - private LdifParser $subject; - - protected function setUp(): void - { - $this->subject = new LdifParser(); - } - public function test_it_parses_a_changetype_add_record_into_an_add_request(): void { - $result = $this->subject->parse( + $result = LdifChanges::fromString( "dn: cn=alice,dc=x\nchangetype: add\nobjectClass: top\nobjectClass: person\ncn: alice\nsn: A\n", ); @@ -58,7 +51,7 @@ public function test_it_parses_a_changetype_add_record_into_an_add_request(): vo public function test_it_parses_a_changetype_delete_record_into_a_delete_request(): void { - $result = $this->subject->parse("dn: cn=bob,dc=x\nchangetype: delete\n"); + $result = LdifChanges::fromString("dn: cn=bob,dc=x\nchangetype: delete\n"); self::assertCount( 1, @@ -80,12 +73,12 @@ public function test_it_rejects_content_after_a_delete_changetype(): void $this->expectException(LdifParseException::class); $this->expectExceptionMessage('Unexpected directive after "changetype: delete"'); - $this->subject->parse("dn: cn=bob,dc=x\nchangetype: delete\ncn: trailing\n"); + LdifChanges::fromString("dn: cn=bob,dc=x\nchangetype: delete\ncn: trailing\n"); } public function test_it_parses_a_modify_record_with_a_single_replace_modspec(): void { - $result = $this->subject->parse( + $result = LdifChanges::fromString( "dn: cn=alice,dc=x\nchangetype: modify\nreplace: sn\nsn: Anderson\n-\n", ); @@ -115,7 +108,7 @@ public function test_it_parses_a_modify_record_with_a_single_replace_modspec(): public function test_it_parses_a_modify_record_with_multiple_modspecs_terminated_by_dash(): void { - $result = $this->subject->parse( + $result = LdifChanges::fromString( "dn: cn=alice,dc=x\nchangetype: modify\n" . "add: telephoneNumber\ntelephoneNumber: 555-0100\n-\n" . "delete: description\n-\n" @@ -160,7 +153,7 @@ public function test_it_parses_a_modify_record_with_multiple_modspecs_terminated public function test_it_parses_a_modify_modspec_deleting_a_specific_value(): void { - $result = $this->subject->parse( + $result = LdifChanges::fromString( "dn: cn=alice,dc=x\nchangetype: modify\ndelete: telephoneNumber\ntelephoneNumber: 555-0100\n-\n", ); @@ -185,7 +178,7 @@ public function test_it_rejects_a_modspec_without_a_dash_terminator(): void $this->expectException(LdifParseException::class); $this->expectExceptionMessage('missing "-" terminator'); - $this->subject->parse("dn: cn=alice,dc=x\nchangetype: modify\nreplace: sn\nsn: Anderson\n"); + LdifChanges::fromString("dn: cn=alice,dc=x\nchangetype: modify\nreplace: sn\nsn: Anderson\n"); } public function test_it_rejects_a_modspec_value_with_a_mismatched_attribute(): void @@ -193,7 +186,7 @@ public function test_it_rejects_a_modspec_value_with_a_mismatched_attribute(): v $this->expectException(LdifParseException::class); $this->expectExceptionMessage('does not match values for'); - $this->subject->parse( + LdifChanges::fromString( "dn: cn=alice,dc=x\nchangetype: modify\nreplace: sn\ncn: not-sn\n-\n", ); } @@ -203,14 +196,14 @@ public function test_it_rejects_an_unknown_modspec_op(): void $this->expectException(LdifParseException::class); $this->expectExceptionMessage('Expected an add:, delete:, or replace:'); - $this->subject->parse( + LdifChanges::fromString( "dn: cn=alice,dc=x\nchangetype: modify\nbogus: sn\nsn: x\n-\n", ); } public function test_it_parses_a_modrdn_record_without_newsuperior(): void { - $result = $this->subject->parse( + $result = LdifChanges::fromString( "dn: cn=alice,dc=x\nchangetype: modrdn\nnewrdn: cn=alicia\ndeleteoldrdn: 1\n", ); @@ -233,7 +226,7 @@ public function test_it_parses_a_modrdn_record_without_newsuperior(): void public function test_it_parses_a_modrdn_record_with_newsuperior_and_deleteoldrdn_zero(): void { - $result = $this->subject->parse( + $result = LdifChanges::fromString( "dn: cn=alice,ou=old,dc=x\nchangetype: modrdn\nnewrdn: cn=alicia\ndeleteoldrdn: 0\nnewsuperior: ou=new,dc=x\n", ); @@ -253,7 +246,7 @@ public function test_it_decodes_a_base64_newrdn(): void { $ldif = "dn: cn=foo,dc=x\nchangetype: modrdn\nnewrdn:: " . base64_encode('cn=Bär') . "\ndeleteoldrdn: 1\n"; - $request = $this->subject->parse($ldif)->toArray()[0]; + $request = LdifChanges::fromString($ldif)->toArray()[0]; self::assertInstanceOf( ModifyDnRequest::class, $request, @@ -269,7 +262,7 @@ public function test_it_rejects_a_modrdn_record_missing_newrdn(): void $this->expectException(LdifParseException::class); $this->expectExceptionMessage('Missing "newrdn:"'); - $this->subject->parse("dn: cn=alice,dc=x\nchangetype: modrdn\ndeleteoldrdn: 1\n"); + LdifChanges::fromString("dn: cn=alice,dc=x\nchangetype: modrdn\ndeleteoldrdn: 1\n"); } public function test_it_rejects_a_modrdn_record_missing_deleteoldrdn(): void @@ -277,7 +270,7 @@ public function test_it_rejects_a_modrdn_record_missing_deleteoldrdn(): void $this->expectException(LdifParseException::class); $this->expectExceptionMessage('Missing "deleteoldrdn:"'); - $this->subject->parse("dn: cn=alice,dc=x\nchangetype: modrdn\nnewrdn: cn=alicia\n"); + LdifChanges::fromString("dn: cn=alice,dc=x\nchangetype: modrdn\nnewrdn: cn=alicia\n"); } public function test_it_rejects_deleteoldrdn_that_is_not_zero_or_one(): void @@ -285,7 +278,7 @@ public function test_it_rejects_deleteoldrdn_that_is_not_zero_or_one(): void $this->expectException(LdifParseException::class); $this->expectExceptionMessage('must be 0 or 1'); - $this->subject->parse( + LdifChanges::fromString( "dn: cn=alice,dc=x\nchangetype: modrdn\nnewrdn: cn=alicia\ndeleteoldrdn: 2\n", ); } @@ -295,6 +288,6 @@ public function test_it_rejects_an_unknown_changetype(): void $this->expectException(LdifParseException::class); $this->expectExceptionMessage('Unsupported changetype "bogus"'); - $this->subject->parse("dn: cn=alice,dc=x\nchangetype: bogus\n"); + LdifChanges::fromString("dn: cn=alice,dc=x\nchangetype: bogus\n"); } } diff --git a/tests/unit/Server/Backend/Storage/LdapImporterTest.php b/tests/unit/Server/Backend/Storage/LdapImporterTest.php index a6dd85d7..3e0a2af1 100644 --- a/tests/unit/Server/Backend/Storage/LdapImporterTest.php +++ b/tests/unit/Server/Backend/Storage/LdapImporterTest.php @@ -48,14 +48,13 @@ public function test_importEntries_persists_all_entries(): void self::assertNotNull($storage->find(new Dn('cn=alice,dc=example,dc=com'))); } - public function test_importEntries_is_noop_when_empty(): void + public function test_importEntries_handles_empty_input(): void { - $storage = $this->createMock(EntryStorageInterface::class); - $storage - ->expects(self::never()) - ->method('atomic'); + $storage = new InMemoryStorage(); (new LdapImporter($storage))->importEntries([]); + + self::assertNull($storage->find(new Dn('dc=example,dc=com'))); } public function test_importEntries_runs_in_single_atomic_call(): void @@ -71,17 +70,17 @@ public function test_importEntries_runs_in_single_atomic_call(): void ]); } - public function test_importEntries_sorts_by_depth_so_input_order_does_not_matter(): void + public function test_importEntries_requires_input_in_depth_first_order(): void { $storage = new InMemoryStorage(); + self::expectException(InvalidArgumentException::class); + self::expectExceptionMessage('Parent entry "dc=example,dc=com" does not exist for "cn=Alice,dc=example,dc=com".'); + (new LdapImporter($storage))->importEntries([ new Entry(new Dn('cn=Alice,dc=example,dc=com'), new Attribute('cn', 'Alice')), new Entry(new Dn('dc=example,dc=com'), new Attribute('dc', 'example')), ]); - - self::assertNotNull($storage->find(new Dn('dc=example,dc=com'))); - self::assertNotNull($storage->find(new Dn('cn=alice,dc=example,dc=com'))); } public function test_importEntries_throws_when_parent_is_missing(): void