diff --git a/lib/Builder.php b/lib/Builder.php index c03ca035..e53379ed 100644 --- a/lib/Builder.php +++ b/lib/Builder.php @@ -13,6 +13,7 @@ use Doctrine\RST\Event\PreBuildParseEvent; use Doctrine\RST\Event\PreBuildRenderEvent; use Doctrine\RST\Event\PreBuildScanEvent; +use Doctrine\RST\Meta\CachedMetasLoader; use Doctrine\RST\Meta\Metas; use Symfony\Component\Filesystem\Filesystem; use function is_dir; @@ -34,15 +35,12 @@ class Builder /** @var Metas */ private $metas; + /** @var CachedMetasLoader */ + private $cachedMetasLoader; + /** @var Documents */ private $documents; - /** @var ParseQueue */ - private $parseQueue; - - /** @var Scanner */ - private $scanner; - /** @var Copier */ private $copier; @@ -61,15 +59,13 @@ public function __construct(?Kernel $kernel = null) $this->metas = new Metas(); + $this->cachedMetasLoader = new CachedMetasLoader(); + $this->documents = new Builder\Documents( $this->filesystem, $this->metas ); - $this->parseQueue = new Builder\ParseQueue($this->documents); - - $this->scanner = new Builder\Scanner($this->parseQueue, $this->metas); - $this->copier = new Builder\Copier($this->filesystem); $this->kernel->initBuilder($this); @@ -95,11 +91,6 @@ public function getDocuments() : Documents return $this->documents; } - public function getParseQueue() : Builder\ParseQueue - { - return $this->parseQueue; - } - public function getErrorManager() : ErrorManager { return $this->errorManager; @@ -126,11 +117,17 @@ public function build( $this->filesystem->mkdir($targetDirectory, 0755); } - $this->scan($directory, $targetDirectory); + if ($this->configuration->getUseCachedMetas()) { + $this->cachedMetasLoader->loadCachedMetaEntries($targetDirectory, $this->metas); + } + + $parseQueue = $this->scan($directory, $targetDirectory); - $this->parse($directory, $targetDirectory); + $this->parse($directory, $targetDirectory, $parseQueue); $this->render($directory, $targetDirectory); + + $this->cachedMetasLoader->cacheMetaEntries($targetDirectory, $this->metas); } public function copy(string $source, ?string $destination = null) : self @@ -147,19 +144,23 @@ public function mkdir(string $directory) : self return $this; } - private function scan(string $directory, string $targetDirectory) : void + private function scan(string $directory, string $targetDirectory) : ParseQueue { $this->configuration->dispatchEvent( PreBuildScanEvent::PRE_BUILD_SCAN, new PreBuildScanEvent($this, $directory, $targetDirectory) ); - $this->scanner->scan($directory, $this->getIndexName()); + $scanner = new Scanner( + $this->configuration->getSourceFileExtension(), + $directory, + $this->metas + ); - $this->scanner->scanMetas($directory); + return $scanner->scan(); } - private function parse(string $directory, string $targetDirectory) : void + private function parse(string $directory, string $targetDirectory, ParseQueue $parseQueue) : void { $this->configuration->dispatchEvent( PreBuildParseEvent::PRE_BUILD_PARSE, @@ -169,16 +170,14 @@ private function parse(string $directory, string $targetDirectory) : void $parseQueueProcessor = new ParseQueueProcessor( $this->kernel, $this->errorManager, - $this->parseQueue, $this->metas, $this->documents, - $this->scanner, $directory, $targetDirectory, $this->configuration->getFileExtension() ); - $parseQueueProcessor->process(); + $parseQueueProcessor->process($parseQueue); } private function render(string $directory, string $targetDirectory) : void diff --git a/lib/Builder/ParseQueue.php b/lib/Builder/ParseQueue.php index ff7f39d7..a72d788c 100644 --- a/lib/Builder/ParseQueue.php +++ b/lib/Builder/ParseQueue.php @@ -4,51 +4,52 @@ namespace Doctrine\RST\Builder; -use function array_shift; +use InvalidArgumentException; +use function array_filter; +use function array_key_exists; +use function array_keys; +use function sprintf; -class ParseQueue +final class ParseQueue { - /** @var Documents */ - private $documents; - - /** @var string[] */ - private $parseQueue = []; - - /** @var int[] */ - private $states = []; - - public function __construct(Documents $documents) + /** + * An array where each key is the filename and the value is a + * boolean indicating if the file needs to be parsed or not. + * + * @var bool[] + */ + private $fileStatuses = []; + + public function addFile(string $filename, bool $parseNeeded) : void { - $this->documents = $documents; - } + if (isset($this->fileStatuses[$filename])) { + throw new InvalidArgumentException(sprintf('File "%s" is already in the parse queue', $filename)); + } - public function getState(string $file) : ?int - { - return $this->states[$file] ?? null; + $this->fileStatuses[$filename] = $parseNeeded; } - public function setState(string $file, int $state) : void + public function isFileKnownToParseQueue(string $filename) : bool { - $this->states[$file] = $state; + return array_key_exists($filename, $this->fileStatuses); } - public function getFileToParse() : ?string + public function doesFileRequireParsing(string $filename) : bool { - if ($this->parseQueue !== []) { - return array_shift($this->parseQueue); + if (! $this->isFileKnownToParseQueue($filename)) { + throw new InvalidArgumentException(sprintf('File "%s" is not known to the parse queue', $filename)); } - return null; + return $this->fileStatuses[$filename]; } - public function addToParseQueue(string $file) : void + /** + * @return string[] + */ + public function getAllFilesThatRequireParsing() : array { - $this->states[$file] = State::PARSE; - - if ($this->documents->hasDocument($file)) { - return; - } - - $this->parseQueue[$file] = $file; + return array_keys(array_filter($this->fileStatuses, static function (bool $parseNeeded) { + return $parseNeeded; + })); } } diff --git a/lib/Builder/ParseQueueProcessor.php b/lib/Builder/ParseQueueProcessor.php index 2ea35efc..cb5adab3 100644 --- a/lib/Builder/ParseQueueProcessor.php +++ b/lib/Builder/ParseQueueProcessor.php @@ -9,8 +9,6 @@ use Doctrine\RST\Meta\Metas; use Doctrine\RST\Nodes\DocumentNode; use Doctrine\RST\Parser; -use function array_filter; -use function file_exists; use function filectime; class ParseQueueProcessor @@ -21,18 +19,12 @@ class ParseQueueProcessor /** @var ErrorManager */ private $errorManager; - /** @var ParseQueue */ - private $parseQueue; - /** @var Metas */ private $metas; /** @var Documents */ private $documents; - /** @var Scanner */ - private $scanner; - /** @var string */ private $directory; @@ -45,28 +37,24 @@ class ParseQueueProcessor public function __construct( Kernel $kernel, ErrorManager $errorManager, - ParseQueue $parseQueue, Metas $metas, Documents $documents, - Scanner $scanner, string $directory, string $targetDirectory, string $fileExtension ) { $this->kernel = $kernel; $this->errorManager = $errorManager; - $this->parseQueue = $parseQueue; $this->metas = $metas; $this->documents = $documents; - $this->scanner = $scanner; $this->directory = $directory; $this->targetDirectory = $targetDirectory; $this->fileExtension = $fileExtension; } - public function process() : void + public function process(ParseQueue $parseQueue) : void { - while ($file = $this->parseQueue->getFileToParse()) { + foreach ($parseQueue->getAllFilesThatRequireParsing() as $file) { $this->processFile($file); } } @@ -85,12 +73,6 @@ private function processFile(string $file) : void $this->kernel->postParse($document); - $dependencies = $environment->getDependencies(); - - foreach ($this->buildDependenciesToScan($dependencies) as $dependency) { - $this->scanner->scan($this->directory, $dependency); - } - $this->metas->set( $file, $this->buildDocumentUrl($document), @@ -98,7 +80,7 @@ private function processFile(string $file) : void $document->getTitles(), $document->getTocs(), (int) filectime($fileAbsolutePath), - $dependencies, + $environment->getDependencies(), $environment->getLinks() ); } @@ -117,18 +99,6 @@ private function createFileParser(string $file) : Parser return $parser; } - /** - * @param string[] $dependencies - * - * @return string[] - */ - private function buildDependenciesToScan(array $dependencies) : array - { - return array_filter($dependencies, function (string $dependency) : bool { - return file_exists($this->buildFileAbsolutePath($dependency)); - }); - } - private function buildFileAbsolutePath(string $file) : string { return $this->directory . '/' . $file . '.rst'; diff --git a/lib/Builder/Scanner.php b/lib/Builder/Scanner.php index 205cdf1e..df477686 100644 --- a/lib/Builder/Scanner.php +++ b/lib/Builder/Scanner.php @@ -4,76 +4,143 @@ namespace Doctrine\RST\Builder; -use Doctrine\RST\Meta\MetaEntry; use Doctrine\RST\Meta\Metas; -use function file_exists; -use function filectime; +use InvalidArgumentException; +use Symfony\Component\Finder\Finder; +use Symfony\Component\Finder\SplFileInfo; +use function sprintf; +use function strlen; +use function substr; class Scanner { - /** @var ParseQueue */ - private $parseQueue; + /** @var string */ + private $fileExtension; + + /** @var string */ + private $directory; /** @var Metas */ private $metas; - public function __construct(ParseQueue $parseQueue, Metas $metas) + /** @var Finder */ + private $finder; + + /** @var SplFileInfo[] */ + private $fileInfos = []; + + public function __construct(string $fileExtension, string $directory, Metas $metas, ?Finder $finder = null) { - $this->parseQueue = $parseQueue; - $this->metas = $metas; + $this->fileExtension = $fileExtension; + $this->directory = $directory; + $this->metas = $metas; + + $this->finder = $finder ?: new Finder(); + $this->finder->in($this->directory) + ->files() + ->name('*.' . $this->fileExtension); } - public function scan(string $directory, string $file) : void + /** + * Scans a directory recursively looking for all files to parse. + * + * This takes into account the presence of cached & fresh MetaEntry + * objects, and avoids adding files to the parse queue that have + * not changed and whose direct dependencies have not changed. + */ + public function scan() : ParseQueue { - if ($this->parseQueue->getState($file) !== null) { - return; + // completely populate the splFileInfos property + $this->fileInfos = []; + foreach ($this->finder as $fileInfo) { + $relativeFilename = $fileInfo->getRelativePathname(); + // strip off the extension + $documentPath = substr($relativeFilename, 0, -(strlen($this->fileExtension) + 1)); + + $this->fileInfos[$documentPath] = $fileInfo; } - $this->parseQueue->setState($file, State::NO_PARSE); - - $entry = $this->metas->get($file); - - $rst = $directory . '/' . $file; - - if ($entry === null || ! file_exists($rst) || $entry->getCtime() < filectime($rst)) { - // File was never seen or changed and thus need to be parsed - $this->parseQueue->addToParseQueue($file); - } else { - $this->scanMetaEntry($entry, $directory, $file); + $parseQueue = new ParseQueue(); + foreach ($this->fileInfos as $filename => $fileInfo) { + $parseQueue->addFile( + $filename, + $this->doesFileRequireParsing($filename, $parseQueue) + ); } + + return $parseQueue; } - public function scanMetaEntry(MetaEntry $entry, string $directory, string $file) : void + private function doesFileRequireParsing(string $filename, ParseQueue $parseQueue) : bool { - // Have a look to the file dependencies to know if you need to parse - // it or not - $depends = $entry->getDepends(); + if (! isset($this->fileInfos[$filename])) { + throw new InvalidArgumentException(sprintf('No file info found for "%s" - file does not exist.', $filename)); + } + + $file = $this->fileInfos[$filename]; - $parent = $entry->getParent(); + $documentFilename = $this->getFilenameFromFile($file); + $entry = $this->metas->get($documentFilename); - if ($parent !== null) { - $depends[] = $parent; + if ($this->hasFileBeenUpdated($filename)) { + // File is new or changed and thus need to be parsed + return true; } - foreach ($depends as $dependency) { - $this->scan($directory, $dependency); + // Look to the file's dependencies to know if you need to parse it or not + $dependencies = $entry !== null ? $entry->getDepends() : []; - // If any dependency needs to be parsed, this file needs also to be - // parsed - if ($this->parseQueue->getState($dependency) !== State::PARSE) { - continue; + if ($entry !== null && $entry->getParent() !== null) { + $dependencies[] = $entry->getParent(); + } + + foreach ($dependencies as $dependency) { + /* + * The dependency check is NOT recursive on purpose. + * If fileA has a link to fileB that uses its "headline", + * for example, then fileA is "dependent" on fileB. If + * fileB changes, it means that its MetaEntry needs to + * be updated. And because fileA gets the headline from + * the MetaEntry, it means that fileA must also be re-parsed. + * However, if fileB depends on fileC and file C only is + * updated, fileB *does* need to be re-parsed, but fileA + * does not, because the MetaEntry for fileB IS still + * "fresh" - fileB did not actually change, so any metadata + * about headlines, etc, is still fresh. Therefore, fileA + * does not need to be parsed. + */ + + // dependency no longer exists? We should re-parse this file + if (! isset($this->fileInfos[$dependency])) { + return true; } - $this->parseQueue->addToParseQueue($file); + // finally, we need to recursively ask if this file needs parsing + if ($this->hasFileBeenUpdated($dependency)) { + return true; + } } + + // Meta is fresh and no dependencies need parsing + return false; } - public function scanMetas(string $directory) : void + private function hasFileBeenUpdated(string $filename) : bool { - $entries = $this->metas->getAll(); + $file = $this->fileInfos[$filename]; - foreach ($entries as $file => $infos) { - $this->scan($directory, $file); - } + $documentFilename = $this->getFilenameFromFile($file); + $entry = $this->metas->get($documentFilename); + + // File is new or changed + return $entry === null || $entry->getCtime() < $file->getCTime(); + } + + /** + * Converts foo/bar.rst to foo/bar (the document filename) + */ + private function getFilenameFromFile(SplFileInfo $file) : string + { + return substr($file->getRelativePathname(), 0, -(strlen($this->fileExtension) + 1)); } } diff --git a/lib/Builder/State.php b/lib/Builder/State.php deleted file mode 100644 index 3020c5e6..00000000 --- a/lib/Builder/State.php +++ /dev/null @@ -1,15 +0,0 @@ -indentHTML; } + public function setUseCachedMetas(bool $useCachedMetas) : void + { + $this->useCachedMetas = $useCachedMetas; + } + + public function getUseCachedMetas() : bool + { + return $this->useCachedMetas; + } + public function getFileExtension() : string { return $this->fileExtension; @@ -277,6 +293,11 @@ public function getFormat() : Format return $this->formats[$this->fileExtension]; } + public function getSourceFileExtension() : string + { + return $this->sourceFileExtension; + } + private function createNodeInstantiator(Environment $environment, string $type, string $nodeClassName) : NodeInstantiator { return new NodeInstantiator( diff --git a/lib/Environment.php b/lib/Environment.php index 23ab13cd..937297f5 100644 --- a/lib/Environment.php +++ b/lib/Environment.php @@ -15,6 +15,7 @@ use function dirname; use function iconv; use function implode; +use function in_array; use function preg_replace; use function sprintf; use function strtolower; @@ -58,6 +59,9 @@ class Environment /** @var string[] */ private $dependencies = []; + /** @var string[] */ + private $unresolvedDependencies = []; + /** @var string[] */ private $variables = []; @@ -148,6 +152,23 @@ public function resolve(string $section, string $data) : ?ResolvedReference if ($resolvedReference === null) { $this->addInvalidLink(new InvalidLink($data)); + + if ($this->getMetaEntry() !== null) { + $this->getMetaEntry()->removeDependency( + // use the unique, unresolved name + $this->unresolvedDependencies[$data] ?? $data + ); + } + + return null; + } + + if (isset($this->unresolvedDependencies[$data]) && $this->getMetaEntry() !== null) { + $this->getMetaEntry()->resolveDependency( + // use the unique, unresolved name + $this->unresolvedDependencies[$data], + $resolvedReference->getFile() + ); } return $resolvedReference; @@ -279,9 +300,13 @@ public function getLink(string $name, bool $relative = true) : string return ''; } - public function addDependency(string $dependency) : void + public function addDependency(string $dependency, bool $requiresResolving = false) : void { - $dependency = $this->canonicalUrl($dependency); + if (! $requiresResolving) { + // the dependency is already a filename, probably a :doc: + // or from a toc-tree - change it to the canonical URL + $dependency = $this->canonicalUrl($dependency); + } if ($dependency === null) { throw new InvalidArgumentException(sprintf( @@ -290,7 +315,19 @@ public function addDependency(string $dependency) : void )); } - $this->dependencies[] = $dependency; + if ($requiresResolving) { + // a hack to avoid collisions between resolved and unresolved dependencies + $dependencyName = 'UNRESOLVED__' . $dependency; + $this->unresolvedDependencies[$dependency] = $dependencyName; + } else { + $dependencyName = $dependency; + } + + if (in_array($dependencyName, $this->dependencies, true)) { + return; + } + + $this->dependencies[] = $dependencyName; } /** diff --git a/lib/Kernel.php b/lib/Kernel.php index 6a18f0d8..38fde6fb 100644 --- a/lib/Kernel.php +++ b/lib/Kernel.php @@ -42,7 +42,7 @@ public function __construct( $this->references = array_merge([ new References\Doc(), - new References\Doc('ref'), + new References\Doc('ref', true), ], $this->createReferences(), $references); } diff --git a/lib/Meta/CachedMetasLoader.php b/lib/Meta/CachedMetasLoader.php new file mode 100644 index 00000000..6c157d6f --- /dev/null +++ b/lib/Meta/CachedMetasLoader.php @@ -0,0 +1,42 @@ +getMetaCachePath($targetDirectory); + if (! file_exists($metaCachePath)) { + return; + } + + $contents = file_get_contents($metaCachePath); + + if ($contents === false) { + throw new LogicException(sprintf('Could not load file "%s"', $contents)); + } + + $metas->setMetaEntries(unserialize($contents)); + } + + public function cacheMetaEntries(string $targetDirectory, Metas $metas) : void + { + file_put_contents($this->getMetaCachePath($targetDirectory), serialize($metas->getAll())); + } + + private function getMetaCachePath(string $targetDirectory) : string + { + return $targetDirectory . '/metas.php'; + } +} diff --git a/lib/Meta/MetaEntry.php b/lib/Meta/MetaEntry.php index fc12be5f..807acd8d 100644 --- a/lib/Meta/MetaEntry.php +++ b/lib/Meta/MetaEntry.php @@ -5,9 +5,13 @@ namespace Doctrine\RST\Meta; use Doctrine\RST\Environment; +use LogicException; use function array_merge; +use function array_search; +use function in_array; use function is_array; use function is_string; +use function sprintf; class MetaEntry { @@ -32,6 +36,9 @@ class MetaEntry /** @var string[] */ private $depends; + /** @var string[] */ + private $resolvedDependencies = []; + /** @var string[] */ private $links; @@ -118,6 +125,41 @@ public function getDepends() : array return $this->depends; } + /** + * Call to replace a dependency with the resolved, real filename. + */ + public function resolveDependency(string $originalDependency, ?string $newDependency) : void + { + if ($newDependency === null) { + return; + } + + // we only need to resolve a dependency one time + if (in_array($originalDependency, $this->resolvedDependencies, true)) { + return; + } + + $key = array_search($originalDependency, $this->depends, true); + + if ($key === false) { + throw new LogicException(sprintf('Could not find dependency "%s" in MetaEntry for "%s"', $originalDependency, $this->file)); + } + + $this->depends[$key] = $newDependency; + $this->resolvedDependencies[] = $originalDependency; + } + + public function removeDependency(string $dependency) : void + { + $key = array_search($dependency, $this->depends, true); + + if ($key === false) { + throw new LogicException(sprintf('Could not find dependency "%s" in MetaEntry for "%s"', $dependency, $this->file)); + } + + unset($this->depends[$key]); + } + /** * @return string[] */ diff --git a/lib/Meta/Metas.php b/lib/Meta/Metas.php index 3b618128..0626c464 100644 --- a/lib/Meta/Metas.php +++ b/lib/Meta/Metas.php @@ -97,6 +97,14 @@ public function get(string $url) : ?MetaEntry return null; } + /** + * @param MetaEntry[] $metaEntries + */ + public function setMetaEntries(array $metaEntries) : void + { + $this->entries = $metaEntries; + } + /** * @param string[] $links */ diff --git a/lib/References/Doc.php b/lib/References/Doc.php index e75f4454..05f3bbaf 100644 --- a/lib/References/Doc.php +++ b/lib/References/Doc.php @@ -14,10 +14,20 @@ class Doc extends Reference /** @var Resolver */ private $resolver; - public function __construct(string $name = 'doc') + /** + * Used with "ref" - it means the dependencies added in found() + * must be resolved to their final path later (they are not + * already document names). + * + * @var bool + */ + private $dependenciesMustBeResolved; + + public function __construct(string $name = 'doc', bool $dependenciesMustBeResolved = false) { - $this->name = $name; - $this->resolver = new Resolver(); + $this->name = $name; + $this->resolver = new Resolver(); + $this->dependenciesMustBeResolved = $dependenciesMustBeResolved; } public function getName() : string @@ -32,6 +42,6 @@ public function resolve(Environment $environment, string $data) : ?ResolvedRefer public function found(Environment $environment, string $data) : void { - $environment->addDependency($data); + $environment->addDependency($data, $this->dependenciesMustBeResolved); } } diff --git a/lib/References/ResolvedReference.php b/lib/References/ResolvedReference.php index 90f7e951..09feafc9 100644 --- a/lib/References/ResolvedReference.php +++ b/lib/References/ResolvedReference.php @@ -11,6 +11,9 @@ class ResolvedReference { + /** @var ?string */ + private $file; + /** @var string|null */ private $title; @@ -27,8 +30,9 @@ class ResolvedReference * @param string[][]|string[][][] $titles * @param string[] $attributes */ - public function __construct(?string $title, ?string $url, array $titles = [], array $attributes = []) + public function __construct(?string $file, ?string $title, ?string $url, array $titles = [], array $attributes = []) { + $this->file = $file; $this->title = $title; $this->url = $url; $this->titles = $titles; @@ -37,6 +41,11 @@ public function __construct(?string $title, ?string $url, array $titles = [], ar $this->attributes = $attributes; } + public function getFile() : ?string + { + return $this->file; + } + public function getTitle() : ?string { return $this->title; diff --git a/lib/References/Resolver.php b/lib/References/Resolver.php index e790769a..b568a39c 100644 --- a/lib/References/Resolver.php +++ b/lib/References/Resolver.php @@ -52,7 +52,7 @@ private function resolveFileReference( return null; } - return $this->createResolvedReference($environment, $entry, $attributes); + return $this->createResolvedReference($file, $environment, $entry, $attributes); } /** @@ -66,7 +66,7 @@ private function resolveAnchorReference( $entry = $environment->getMetas()->findLinkMetaEntry($data); if ($entry !== null) { - return $this->createResolvedReference($environment, $entry, $attributes, $data); + return $this->createResolvedReference($entry->getFile(), $environment, $entry, $attributes, $data); } return null; @@ -76,6 +76,7 @@ private function resolveAnchorReference( * @param string[] $attributes */ private function createResolvedReference( + ?string $file, Environment $environment, MetaEntry $entry, array $attributes = [], @@ -88,6 +89,7 @@ private function createResolvedReference( } return new ResolvedReference( + $file, $entry->getTitle(), $url, $entry->getTitles(), diff --git a/tests/BaseBuilderTest.php b/tests/BaseBuilderTest.php index ea2f7368..2d0860b1 100644 --- a/tests/BaseBuilderTest.php +++ b/tests/BaseBuilderTest.php @@ -22,6 +22,7 @@ protected function setUp() : void shell_exec('rm -rf ' . $this->targetFile()); $this->builder = new Builder(); + $this->builder->getConfiguration()->setUseCachedMetas(false); $this->builder->build($this->sourceFile(), $this->targetFile()); } diff --git a/tests/Builder/BuilderTest.php b/tests/Builder/BuilderTest.php index f1a0c6cb..9e59f4d5 100644 --- a/tests/Builder/BuilderTest.php +++ b/tests/Builder/BuilderTest.php @@ -4,12 +4,21 @@ namespace Doctrine\Tests\RST\Builder; +use Doctrine\RST\Builder; +use Doctrine\RST\Meta\MetaEntry; use Doctrine\Tests\RST\BaseBuilderTest; +use function array_unique; +use function array_values; use function file_exists; +use function file_get_contents; +use function file_put_contents; use function is_dir; use function range; +use function sleep; use function sprintf; +use function str_replace; use function substr_count; +use function unserialize; /** * Unit testing for RST @@ -38,6 +47,62 @@ public function testBuild() : void self::assertTrue(file_exists($this->targetFile('subdir/file.html'))); } + public function testCachedMetas() : void + { + // check that metas were cached + self::assertTrue(file_exists($this->targetFile('metas.php'))); + $cachedContents = (string) file_get_contents($this->targetFile('metas.php')); + /** @var MetaEntry[] $metaEntries */ + $metaEntries = unserialize($cachedContents); + self::assertArrayHasKey('index', $metaEntries); + self::assertSame('Summary', $metaEntries['index']->getTitle()); + + // look at all the other documents this document depends + // on, like :doc: and :ref: + self::assertSame([ + 'index', + 'toc-glob', + 'subdir/index', + ], array_values(array_unique($metaEntries['introduction']->getDepends()))); + + // assert the self-refs don't mess up dependencies + self::assertSame([ + 'subdir/index', + 'index', + 'subdir/file', + ], array_values(array_unique($metaEntries['subdir/index']->getDepends()))); + + // update meta cache to see that it was used + // Summary is the main header in "index.rst" + // we reference it in link-to-index.rst + // it should cause link-to-index.rst to re-render with the new + // title as the link + file_put_contents( + $this->targetFile('metas.php'), + str_replace('Summary', 'Sumario', $cachedContents) + ); + + // also we need to trigger the link-to-index.rst as looking updated + sleep(1); + $contents = file_get_contents(__DIR__ . '/input/link-to-index.rst'); + file_put_contents( + __DIR__ . '/input/link-to-index.rst', + $contents . ' ' + ); + // change it back + file_put_contents( + __DIR__ . '/input/link-to-index.rst', + $contents + ); + + // new builder, which will use cached metas + $builder = new Builder(); + $builder->build($this->sourceFile(), $this->targetFile()); + + $contents = $this->getFileContents($this->targetFile('link-to-index.html')); + self::assertContains('Sumario', $contents); + } + /** * Tests the ..url :: directive */ diff --git a/tests/Builder/ParseQueueProcessorTest.php b/tests/Builder/ParseQueueProcessorTest.php index eed54c2c..f8a79013 100644 --- a/tests/Builder/ParseQueueProcessorTest.php +++ b/tests/Builder/ParseQueueProcessorTest.php @@ -7,7 +7,6 @@ use Doctrine\RST\Builder\Documents; use Doctrine\RST\Builder\ParseQueue; use Doctrine\RST\Builder\ParseQueueProcessor; -use Doctrine\RST\Builder\Scanner; use Doctrine\RST\ErrorManager; use Doctrine\RST\Kernel; use Doctrine\RST\Meta\Metas; @@ -24,18 +23,12 @@ class ParseQueueProcessorTest extends TestCase /** @var ErrorManager|MockObject */ private $errorManager; - /** @var ParseQueue|MockObject */ - private $parseQueue; - /** @var Metas|MockObject */ private $metas; /** @var Documents|MockObject */ private $documents; - /** @var Scanner|MockObject */ - private $scanner; - /** @var string */ private $directory; @@ -52,13 +45,8 @@ public function testProcess() : void { touch($this->directory . '/file.rst'); - $this->parseQueue->expects(self::at(0)) - ->method('getFileToParse') - ->willReturn('file'); - - $this->parseQueue->expects(self::at(1)) - ->method('getFileToParse') - ->willReturn(null); + $parseQueue = new ParseQueue(); + $parseQueue->addFile('file', true); $this->documents->expects(self::once()) ->method('addDocument') @@ -70,17 +58,15 @@ public function testProcess() : void $this->metas->expects(self::once()) ->method('set'); - $this->parseQueueProcessor->process(); + $this->parseQueueProcessor->process($parseQueue); } protected function setUp() : void { $this->kernel = $this->createMock(Kernel::class); $this->errorManager = $this->createMock(ErrorManager::class); - $this->parseQueue = $this->createMock(ParseQueue::class); $this->metas = $this->createMock(Metas::class); $this->documents = $this->createMock(Documents::class); - $this->scanner = $this->createMock(Scanner::class); $this->directory = sys_get_temp_dir(); $this->targetDirectory = '/target'; $this->fileExtension = 'rst'; @@ -88,10 +74,8 @@ protected function setUp() : void $this->parseQueueProcessor = new ParseQueueProcessor( $this->kernel, $this->errorManager, - $this->parseQueue, $this->metas, $this->documents, - $this->scanner, $this->directory, $this->targetDirectory, $this->fileExtension diff --git a/tests/Builder/ParseQueueTest.php b/tests/Builder/ParseQueueTest.php index eb107d3e..3b3263a6 100644 --- a/tests/Builder/ParseQueueTest.php +++ b/tests/Builder/ParseQueueTest.php @@ -4,57 +4,24 @@ namespace Doctrine\Tests\RST\Builder; -use Doctrine\RST\Builder\Documents; use Doctrine\RST\Builder\ParseQueue; -use Doctrine\RST\Builder\State; -use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; class ParseQueueTest extends TestCase { - /** @var Documents|MockObject */ - private $documents; - - /** @var ParseQueue */ - private $parseQueue; - - public function testGetSetState() : void - { - self::assertNull($this->parseQueue->getState('file')); - - $this->parseQueue->setState('file', State::NO_PARSE); - - self::assertSame(State::NO_PARSE, $this->parseQueue->getState('file')); - } - - public function testGetFileToParse() : void + public function testAddingFiles() : void { - self::assertNull($this->parseQueue->getFileToParse()); + $parseQueue = new ParseQueue(); + $parseQueue->addFile('file_needs_parsing1', true); + $parseQueue->addFile('file_no_parsing1', false); - $this->parseQueue->addToParseQueue('file1'); - $this->parseQueue->addToParseQueue('file2'); + self::assertTrue($parseQueue->isFileKnownToParseQueue('file_needs_parsing1')); + self::assertTrue($parseQueue->isFileKnownToParseQueue('file_no_parsing1')); + self::assertFalse($parseQueue->isFileKnownToParseQueue('other_file')); - self::assertSame('file1', $this->parseQueue->getFileToParse()); - self::assertSame('file2', $this->parseQueue->getFileToParse()); - self::assertNull($this->parseQueue->getFileToParse()); - } - - public function testAddToParseQueue() : void - { - $this->documents->expects(self::once()) - ->method('hasDocument') - ->with('file') - ->willReturn(true); - - $this->parseQueue->addToParseQueue('file'); - - self::assertNull($this->parseQueue->getFileToParse()); - } - - protected function setUp() : void - { - $this->documents = $this->createMock(Documents::class); + self::assertTrue($parseQueue->doesFileRequireParsing('file_needs_parsing1')); + self::assertFalse($parseQueue->doesFileRequireParsing('file_no_parsing1')); - $this->parseQueue = new ParseQueue($this->documents); + self::assertSame(['file_needs_parsing1'], $parseQueue->getAllFilesThatRequireParsing()); } } diff --git a/tests/Builder/ScannerTest.php b/tests/Builder/ScannerTest.php index 6002b0bf..e5d88a3e 100644 --- a/tests/Builder/ScannerTest.php +++ b/tests/Builder/ScannerTest.php @@ -4,18 +4,20 @@ namespace Doctrine\Tests\RST\Builder; -use Doctrine\RST\Builder\ParseQueue; +use ArrayIterator; use Doctrine\RST\Builder\Scanner; -use Doctrine\RST\Builder\State; use Doctrine\RST\Meta\MetaEntry; use Doctrine\RST\Meta\Metas; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; +use Symfony\Component\Finder\Finder; +use Symfony\Component\Finder\SplFileInfo; +use function time; class ScannerTest extends TestCase { - /** @var ParseQueue|MockObject */ - private $parseQueue; + /** @var Finder|MockObject */ + private $finder; /** @var Metas|MockObject */ private $metas; @@ -23,96 +25,220 @@ class ScannerTest extends TestCase /** @var Scanner */ private $scanner; - public function testScan() : void - { - $this->parseQueue->expects(self::once()) - ->method('getState') - ->with('file') - ->willReturn(null); - - $this->parseQueue->expects(self::once()) - ->method('setState') - ->with('file', State::NO_PARSE); + /** @var SplFileInfo[]|MockObject[]|ArrayIterator */ + private $fileMocks; - $metaEntry = $this->createMock(MetaEntry::class); + /** @var MetaEntry[]|MockObject[] */ + private $metaEntryMocks = []; - $this->metas->expects(self::once()) + public function testScanWithNoMetas() : void + { + $this->metas->expects(self::any()) ->method('get') - ->with('file') - ->willReturn($metaEntry); - - $this->parseQueue->expects(self::once()) - ->method('addToParseQueue') - ->with('file'); + ->willReturn(null); - $this->scanner->scan('/directory', 'file'); + $this->addFileMockToFinder('file1.rst'); + $this->addFileMockToFinder('file2.rst'); + $this->addFileMockToFinder('file3.rst'); + $this->addFileMockToFinder('subdir/file4.rst'); + $this->addFileMockToFinder('subdir/file5.rst'); + + $parseQueue = $this->scanner->scan(); + self::assertSame([ + 'file1', + 'file2', + 'file3', + 'subdir/file4', + 'subdir/file5', + ], $parseQueue->getAllFilesThatRequireParsing()); } - public function testScanMetaEntry() : void + public function testScanWithNonFreshMetas() : void { - $metaEntry = $this->createMock(MetaEntry::class); - - $metaEntry->expects(self::once()) + $file1InfoMock = $this->addFileMockToFinder('file1.rst'); + $file1MetaMock = $this->createMetaEntryMock('file1'); + + // file1.rst was modified 50 seconds ago + $file1InfoMock->method('getCTime')->willReturn(time() - 50); + // but file1 MetaEntry was modified 100 seconds ago (is out of date) + $file1MetaMock->method('getCTime')->willReturn(time() - 100); + // should never be called because the meta is definitely not fresh + $file1MetaMock->expects(self::never())->method('getDepends'); + + $file2InfoMock = $this->addFileMockToFinder('file2.rst'); + $file2MetaMock = $this->createMetaEntryMock('file2'); + + // file2.rst was modified 50 seconds ago + $lastModifiedTime = time() - 50; + $file2InfoMock->method('getCTime')->willReturn($lastModifiedTime); + // and file2 MetaEntry was also 50 seconds ago, fresh + $file2MetaMock->method('getCTime')->willReturn($lastModifiedTime); + // ignore dependencies for this test + $file2MetaMock->expects(self::once()) ->method('getDepends') - ->willReturn(['dependency']); - - $metaEntry->expects(self::once()) - ->method('getParent') - ->willReturn('parent'); - - $this->parseQueue->expects(self::at(0)) - ->method('getState') - ->with('dependency') - ->willReturn(State::PARSE); + ->willReturn([]); - $this->parseQueue->expects(self::at(1)) - ->method('getState') - ->with('dependency') - ->willReturn(State::PARSE); - - $this->parseQueue->expects(self::at(2)) - ->method('addToParseQueue') - ->with('file'); - - $this->scanner->scanMetaEntry($metaEntry, '/directory', 'file'); + $parseQueue = $this->scanner->scan(); + self::assertSame(['file1'], $parseQueue->getAllFilesThatRequireParsing()); + self::assertFalse($parseQueue->doesFileRequireParsing('file2')); } - public function testScanMetas() : void + public function testScanWithDependencies() : void { - $metaEntry = $this->createMock(MetaEntry::class); - - $metaEntries = ['file' => $metaEntry]; + /* + * Here is the dependency tree and results: + * * file1 (unmodified) + * depends on: file2 + * * file2 (unmodified) + * depends on: file3, file1 + * * file3 (unmodified) + * depends on: file4, file5, file3, file2 + * * file4 (unmodified) + * depends on: nothing + * * file5 (MODIFIED) + * depends on: file3, file6 + * * file6 (unmodified) + * depends on: file4 + * + * Result is that the following files are fresh: + * * file1 + * * file2 + * * file4 + * * file6 + */ + + $metaCTime = time() - 50; + + $file1InfoMock = $this->addFileMockToFinder('file1.rst'); + $file1InfoMock->method('getCTime')->willReturn($metaCTime); + $file1MetaMock = $this->createMetaEntryMock('file1'); + $file1MetaMock->method('getDepends') + ->willReturn(['file2']); + $file1MetaMock->method('getCTime')->willReturn($metaCTime); + + $file2InfoMock = $this->addFileMockToFinder('file2.rst'); + $file2InfoMock->method('getCTime')->willReturn($metaCTime); + $file2MetaMock = $this->createMetaEntryMock('file2'); + $file2MetaMock->method('getDepends') + ->willReturn(['file2', 'file3']); + $file2MetaMock->method('getCTime')->willReturn($metaCTime); + + $file3InfoMock = $this->addFileMockToFinder('file3.rst'); + $file3InfoMock->method('getCTime')->willReturn($metaCTime); + $file3MetaMock = $this->createMetaEntryMock('file3'); + $file3MetaMock->method('getDepends') + ->willReturn(['file4', 'file5', 'file3', 'file2']); + $file3MetaMock->method('getCTime')->willReturn($metaCTime); + + $file4InfoMock = $this->addFileMockToFinder('file4.rst'); + $file4InfoMock->method('getCTime')->willReturn($metaCTime); + $file4MetaMock = $this->createMetaEntryMock('file4'); + $file4MetaMock->method('getDepends') + ->willReturn([]); + $file4MetaMock->method('getCTime')->willReturn($metaCTime); + + $file5InfoMock = $this->addFileMockToFinder('file5.rst'); + // THIS file is the one file that's modified + $file5InfoMock->method('getCTime')->willReturn(time() - 10); + $file5MetaMock = $this->createMetaEntryMock('file5'); + $file5MetaMock->method('getDepends') + ->willReturn(['file3', 'file6']); + $file5MetaMock->method('getCTime')->willReturn($metaCTime); + + $file6InfoMock = $this->addFileMockToFinder('file6.rst'); + $file6InfoMock->method('getCTime')->willReturn($metaCTime); + $file6MetaMock = $this->createMetaEntryMock('file6'); + $file6MetaMock->method('getDepends') + ->willReturn(['file4']); + $file6MetaMock->method('getCTime')->willReturn($metaCTime); + + $parseQueue = $this->scanner->scan(); + self::assertSame([ + 'file3', + 'file5', + ], $parseQueue->getAllFilesThatRequireParsing()); + } - $this->metas->expects(self::once()) - ->method('getAll') - ->willReturn($metaEntries); + public function testScanWithNonExistentDependency() : void + { + /* + * * file1 (unmodified) + * depends on: file2 + * * file2 (does not exist) + * depends on: file3, file1 + * + * Result is that file 1 DOES need to be parsed + */ + + $metaCTime = time() - 50; + + $file1InfoMock = $this->addFileMockToFinder('file1.rst'); + $file1InfoMock->method('getCTime')->willReturn($metaCTime); + $file1MetaMock = $this->createMetaEntryMock('file1'); + $file1MetaMock->method('getDepends') + ->willReturn(['file2']); + $file1MetaMock->method('getCTime')->willReturn($metaCTime); + + // no file info made for file2 + + $parseQueue = $this->scanner->scan(); + self::assertSame(['file1'], $parseQueue->getAllFilesThatRequireParsing()); + } - $this->parseQueue->expects(self::once()) - ->method('getState') - ->with('file') - ->willReturn(null); + protected function setUp() : void + { + $this->fileMocks = new ArrayIterator(); + $this->finder = $this->createMock(Finder::class); + $this->finder->expects(self::any()) + ->method('getIterator') + ->willReturn($this->fileMocks); + $this->finder->expects(self::once()) + ->method('in') + ->with('/directory') + ->willReturnSelf(); + $this->finder->expects(self::once()) + ->method('files') + ->with() + ->willReturnSelf(); + $this->finder->expects(self::once()) + ->method('name') + ->with('*.rst') + ->willReturnSelf(); + + $this->metas = $this->createMock(Metas::class); + $this->metas->expects(self::any()) + ->method('get') + ->willReturnCallback(function ($filename) { + return $this->metaEntryMocks[$filename] ?? null; + }); - $this->parseQueue->expects(self::once()) - ->method('setState') - ->with('file', State::NO_PARSE); + $this->scanner = new Scanner('rst', '/directory', $this->metas, $this->finder); + } - $this->metas->expects(self::once()) - ->method('get') - ->with('file') - ->willReturn($metaEntry); + /** + * @return MockObject|SplFileInfo + */ + private function addFileMockToFinder(string $relativePath) + { + $fileInfo = $this->createMock(SplFileInfo::class); + $fileInfo->expects(self::any()) + ->method('getRelativePathname') + ->willReturn($relativePath); - $this->parseQueue->expects(self::once()) - ->method('addToParseQueue') - ->with('file'); + $this->fileMocks[$relativePath] = $fileInfo; - $this->scanner->scanMetas('/directory'); + return $fileInfo; } - protected function setUp() : void + /** + * @return MockObject|MetaEntry + */ + private function createMetaEntryMock(string $filename) { - $this->parseQueue = $this->createMock(ParseQueue::class); - $this->metas = $this->createMock(Metas::class); + $meta = $this->createMock(MetaEntry::class); + + $this->metaEntryMocks[$filename] = $meta; - $this->scanner = new Scanner($this->parseQueue, $this->metas); + return $meta; } } diff --git a/tests/Builder/input/introduction.rst b/tests/Builder/input/introduction.rst index 662806e3..64106b3f 100644 --- a/tests/Builder/input/introduction.rst +++ b/tests/Builder/input/introduction.rst @@ -13,8 +13,8 @@ Reference to the :doc:`Index, paragraph toc ` Reference to the :doc:`Glob TOC ` -Reference to the :doc:`Summary Reference ` +Reference to the :ref:`Summary Reference ` -Reference to the :doc:`Test Reference ` +Reference to the :ref:`Test Reference ` -Reference to the :doc:`Camel Case Reference ` +Reference to the :ref:`Camel Case Reference ` diff --git a/tests/Builder/input/link-to-index.rst b/tests/Builder/input/link-to-index.rst new file mode 100644 index 00000000..9a17aa76 --- /dev/null +++ b/tests/Builder/input/link-to-index.rst @@ -0,0 +1,5 @@ +Document with a Link to Index +----------------------------- + +I have a reference to :doc:`index` and this link +is dependent on the title of that page. diff --git a/tests/BuilderInvalidReferences/BuilderInvalidReferencesTest.php b/tests/BuilderInvalidReferences/BuilderInvalidReferencesTest.php index cb1d23f0..269b10c5 100644 --- a/tests/BuilderInvalidReferences/BuilderInvalidReferencesTest.php +++ b/tests/BuilderInvalidReferences/BuilderInvalidReferencesTest.php @@ -18,6 +18,7 @@ class BuilderInvalidReferencesTest extends BaseBuilderTest protected function setUp() : void { $this->configuration = new Configuration(); + $this->configuration->setUseCachedMetas(false); $this->builder = new Builder(new Kernel($this->configuration)); } diff --git a/tests/BuilderToctree/BuilderTocTreeTest.php b/tests/BuilderToctree/BuilderTocTreeTest.php index a07d3abd..ec556b25 100644 --- a/tests/BuilderToctree/BuilderTocTreeTest.php +++ b/tests/BuilderToctree/BuilderTocTreeTest.php @@ -12,7 +12,7 @@ class BuilderTocTreeTest extends BaseBuilderTest public function testTocTreeGlob() : void { self::assertTrue(file_exists($this->targetFile('subdir/toctree.html'))); - self::assertFalse(file_exists($this->targetFile('not-parsed/file.html'))); + self::assertTrue(file_exists($this->targetFile('orphaned/file.html'))); } public function testMaxDepth() : void diff --git a/tests/BuilderToctree/input/not-parsed/file.rst b/tests/BuilderToctree/input/orphaned/file.rst similarity index 100% rename from tests/BuilderToctree/input/not-parsed/file.rst rename to tests/BuilderToctree/input/orphaned/file.rst diff --git a/tests/BuilderUrl/BuilderUrlTest.php b/tests/BuilderUrl/BuilderUrlTest.php index 5713346a..1fed78d6 100644 --- a/tests/BuilderUrl/BuilderUrlTest.php +++ b/tests/BuilderUrl/BuilderUrlTest.php @@ -127,7 +127,8 @@ public function testRelativeUrl() : void protected function setUp() : void { $this->configuration = new Configuration(); - $this->builder = new Builder(new Kernel($this->configuration)); + $this->configuration->setUseCachedMetas(false); + $this->builder = new Builder(new Kernel($this->configuration)); } protected function getFixturesDirectory() : string diff --git a/tests/GlobSearcherTest.php b/tests/GlobSearcherTest.php index 6887abe3..9194bbdf 100644 --- a/tests/GlobSearcherTest.php +++ b/tests/GlobSearcherTest.php @@ -34,7 +34,7 @@ public function testGlobSearch() : void self::assertCount(3, $files); $expected = [ - '/not-parsed/file', + '/orphaned/file', '/index', '/subdir/toctree', ]; diff --git a/tests/LiteralNestedInDirective/BuilderTest.php b/tests/LiteralNestedInDirective/BuilderTest.php index f3c7a169..e4a27c5f 100644 --- a/tests/LiteralNestedInDirective/BuilderTest.php +++ b/tests/LiteralNestedInDirective/BuilderTest.php @@ -25,6 +25,7 @@ protected function setUp() : void ); $this->builder = new Builder($kernel); + $this->builder->getConfiguration()->setUseCachedMetas(false); $this->builder->build($this->sourceFile(), $this->targetFile()); } diff --git a/tests/Meta/CachedMetasLoaderTest.php b/tests/Meta/CachedMetasLoaderTest.php new file mode 100644 index 00000000..3e126b0a --- /dev/null +++ b/tests/Meta/CachedMetasLoaderTest.php @@ -0,0 +1,29 @@ +cacheMetaEntries($targetDir, $metasBefore); + $loader->loadCachedMetaEntries($targetDir, $metasAfter); + self::assertEquals([$meta1, $meta2], $metasAfter->getAll()); + } +} diff --git a/tests/RefInsideDirective/BuilderTest.php b/tests/RefInsideDirective/BuilderTest.php index ada48982..600441ce 100644 --- a/tests/RefInsideDirective/BuilderTest.php +++ b/tests/RefInsideDirective/BuilderTest.php @@ -15,6 +15,7 @@ public function testRefInsideDirective() : void { $kernel = new Kernel(null, [new VersionAddedDirective()]); $builder = new Builder($kernel); + $builder->getConfiguration()->setUseCachedMetas(false); $builder->build( __DIR__ . '/input', diff --git a/tests/References/ResolvedReferenceTest.php b/tests/References/ResolvedReferenceTest.php index 2cccb7cb..5486dcc0 100644 --- a/tests/References/ResolvedReferenceTest.php +++ b/tests/References/ResolvedReferenceTest.php @@ -19,7 +19,7 @@ class ResolvedReferenceTest extends TestCase */ public function testCreateResolvedReference(array $attributes) : void { - $resolvedReference = new ResolvedReference('title', 'url', [['title' => 'title']], $attributes); + $resolvedReference = new ResolvedReference('file', 'title', 'url', [['title' => 'title']], $attributes); self::assertSame('title', $resolvedReference->getTitle()); self::assertSame('url', $resolvedReference->getUrl()); @@ -67,7 +67,7 @@ public function testCreateResolvedReferenceWithAttributesInvalid(array $attribut self::expectException(RuntimeException::class); self::expectExceptionMessage(sprintf('Attribute with name "%s" is not allowed', key($attributes))); - new ResolvedReference('title', 'url', [], $attributes); + new ResolvedReference('file', 'title', 'url', [], $attributes); } /** diff --git a/tests/References/ResolverTest.php b/tests/References/ResolverTest.php index b7472a13..03824d32 100644 --- a/tests/References/ResolverTest.php +++ b/tests/References/ResolverTest.php @@ -66,7 +66,7 @@ public function testResolveFileReference() : void ->willReturn('/url'); self::assertEquals( - new ResolvedReference('title', '/url', [], ['attr' => 'value']), + new ResolvedReference('file', 'title', '/url', [], ['attr' => 'value']), $this->resolver->resolve($this->environment, 'url', ['attr' => 'value']) ); } @@ -86,7 +86,7 @@ public function testResolveAnchorReference() : void ->willReturn('/url'); self::assertEquals( - new ResolvedReference('title', '/url#anchor', [], ['attr' => 'value']), + new ResolvedReference('', 'title', '/url#anchor', [], ['attr' => 'value']), $this->resolver->resolve($this->environment, 'anchor', ['attr' => 'value']) ); }