Merge pull request #91 from weaverryan/re-add-meta-caching

Re-adding caching of Metas & fixing how files are scanned and ref dependencies
doctrine · Feb 2, 2019 · e94cf58 · e94cf58
2 parents 908578a + a128be2
commit e94cf58
Show file tree

Hide file tree

Showing 31 changed files with 674 additions and 300 deletions.
diff --git a/lib/Builder.php b/lib/Builder.php
@@ -13,6 +13,7 @@
 use Doctrine\RST\Event\PreBuildParseEvent;
 use Doctrine\RST\Event\PreBuildRenderEvent;
 use Doctrine\RST\Event\PreBuildScanEvent;
+use Doctrine\RST\Meta\CachedMetasLoader;
 use Doctrine\RST\Meta\Metas;
 use Symfony\Component\Filesystem\Filesystem;
 use function is_dir;
@@ -34,15 +35,12 @@ class Builder
     /** @var Metas */
     private $metas;
 
+    /** @var CachedMetasLoader */
+    private $cachedMetasLoader;
+
     /** @var Documents */
     private $documents;
 
-    /** @var ParseQueue */
-    private $parseQueue;
-
-    /** @var Scanner */
-    private $scanner;
-
     /** @var Copier */
     private $copier;
 
@@ -61,15 +59,13 @@ public function __construct(?Kernel $kernel = null)
 
         $this->metas = new Metas();
 
+        $this->cachedMetasLoader = new CachedMetasLoader();
+
         $this->documents = new Builder\Documents(
             $this->filesystem,
             $this->metas
         );
 
-        $this->parseQueue = new Builder\ParseQueue($this->documents);
-
-        $this->scanner = new Builder\Scanner($this->parseQueue, $this->metas);
-
         $this->copier = new Builder\Copier($this->filesystem);
 
         $this->kernel->initBuilder($this);
@@ -95,11 +91,6 @@ public function getDocuments() : Documents
         return $this->documents;
     }
 
-    public function getParseQueue() : Builder\ParseQueue
-    {
-        return $this->parseQueue;
-    }
-
     public function getErrorManager() : ErrorManager
     {
         return $this->errorManager;
@@ -126,11 +117,17 @@ public function build(
             $this->filesystem->mkdir($targetDirectory, 0755);
         }
 
-        $this->scan($directory, $targetDirectory);
+        if ($this->configuration->getUseCachedMetas()) {
+            $this->cachedMetasLoader->loadCachedMetaEntries($targetDirectory, $this->metas);
+        }
+
+        $parseQueue = $this->scan($directory, $targetDirectory);
 
-        $this->parse($directory, $targetDirectory);
+        $this->parse($directory, $targetDirectory, $parseQueue);
 
         $this->render($directory, $targetDirectory);
+
+        $this->cachedMetasLoader->cacheMetaEntries($targetDirectory, $this->metas);
     }
 
     public function copy(string $source, ?string $destination = null) : self
@@ -147,19 +144,23 @@ public function mkdir(string $directory) : self
         return $this;
     }
 
-    private function scan(string $directory, string $targetDirectory) : void
+    private function scan(string $directory, string $targetDirectory) : ParseQueue
     {
         $this->configuration->dispatchEvent(
             PreBuildScanEvent::PRE_BUILD_SCAN,
             new PreBuildScanEvent($this, $directory, $targetDirectory)
         );
 
-        $this->scanner->scan($directory, $this->getIndexName());
+        $scanner = new Scanner(
+            $this->configuration->getSourceFileExtension(),
+            $directory,
+            $this->metas
+        );
 
-        $this->scanner->scanMetas($directory);
+        return $scanner->scan();
     }
 
-    private function parse(string $directory, string $targetDirectory) : void
+    private function parse(string $directory, string $targetDirectory, ParseQueue $parseQueue) : void
     {
         $this->configuration->dispatchEvent(
             PreBuildParseEvent::PRE_BUILD_PARSE,
@@ -169,16 +170,14 @@ private function parse(string $directory, string $targetDirectory) : void
         $parseQueueProcessor = new ParseQueueProcessor(
             $this->kernel,
             $this->errorManager,
-            $this->parseQueue,
             $this->metas,
             $this->documents,
-            $this->scanner,
             $directory,
             $targetDirectory,
             $this->configuration->getFileExtension()
         );
 
-        $parseQueueProcessor->process();
+        $parseQueueProcessor->process($parseQueue);
     }
 
     private function render(string $directory, string $targetDirectory) : void

diff --git a/lib/Builder/ParseQueue.php b/lib/Builder/ParseQueue.php
@@ -4,51 +4,52 @@
 
 namespace Doctrine\RST\Builder;
 
-use function array_shift;
+use InvalidArgumentException;
+use function array_filter;
+use function array_key_exists;
+use function array_keys;
+use function sprintf;
 
-class ParseQueue
+final class ParseQueue
 {
-    /** @var Documents */
-    private $documents;
-
-    /** @var string[] */
-    private $parseQueue = [];
-
-    /** @var int[] */
-    private $states = [];
-
-    public function __construct(Documents $documents)
+    /**
+     * An array where each key is the filename and the value is a
+     * boolean indicating if the file needs to be parsed or not.
+     *
+     * @var bool[]
+     */
+    private $fileStatuses = [];
+
+    public function addFile(string $filename, bool $parseNeeded) : void
     {
-        $this->documents = $documents;
-    }
+        if (isset($this->fileStatuses[$filename])) {
+            throw new InvalidArgumentException(sprintf('File "%s" is already in the parse queue', $filename));
+        }
 
-    public function getState(string $file) : ?int
-    {
-        return $this->states[$file] ?? null;
+        $this->fileStatuses[$filename] = $parseNeeded;
     }
 
-    public function setState(string $file, int $state) : void
+    public function isFileKnownToParseQueue(string $filename) : bool
     {
-        $this->states[$file] = $state;
+        return array_key_exists($filename, $this->fileStatuses);
     }
 
-    public function getFileToParse() : ?string
+    public function doesFileRequireParsing(string $filename) : bool
     {
-        if ($this->parseQueue !== []) {
-            return array_shift($this->parseQueue);
+        if (! $this->isFileKnownToParseQueue($filename)) {
+            throw new InvalidArgumentException(sprintf('File "%s" is not known to the parse queue', $filename));
         }
 
-        return null;
+        return $this->fileStatuses[$filename];
     }
 
-    public function addToParseQueue(string $file) : void
+    /**
+     * @return string[]
+     */
+    public function getAllFilesThatRequireParsing() : array
     {
-        $this->states[$file] = State::PARSE;
-
-        if ($this->documents->hasDocument($file)) {
-            return;
-        }
-
-        $this->parseQueue[$file] = $file;
+        return array_keys(array_filter($this->fileStatuses, static function (bool $parseNeeded) {
+            return $parseNeeded;
+        }));
     }
 }
diff --git a/lib/Builder/ParseQueueProcessor.php b/lib/Builder/ParseQueueProcessor.php
@@ -9,8 +9,6 @@
 use Doctrine\RST\Meta\Metas;
 use Doctrine\RST\Nodes\DocumentNode;
 use Doctrine\RST\Parser;
-use function array_filter;
-use function file_exists;
 use function filectime;
 
 class ParseQueueProcessor
@@ -21,18 +19,12 @@ class ParseQueueProcessor
     /** @var ErrorManager */
     private $errorManager;
 
-    /** @var ParseQueue */
-    private $parseQueue;
-
     /** @var Metas */
     private $metas;
 
     /** @var Documents */
     private $documents;
 
-    /** @var Scanner */
-    private $scanner;
-
     /** @var string */
     private $directory;
 
@@ -45,28 +37,24 @@ class ParseQueueProcessor
     public function __construct(
         Kernel $kernel,
         ErrorManager $errorManager,
-        ParseQueue $parseQueue,
         Metas $metas,
         Documents $documents,
-        Scanner $scanner,
         string $directory,
         string $targetDirectory,
         string $fileExtension
     ) {
         $this->kernel          = $kernel;
         $this->errorManager    = $errorManager;
-        $this->parseQueue      = $parseQueue;
         $this->metas           = $metas;
         $this->documents       = $documents;
-        $this->scanner         = $scanner;
         $this->directory       = $directory;
         $this->targetDirectory = $targetDirectory;
         $this->fileExtension   = $fileExtension;
     }
 
-    public function process() : void
+    public function process(ParseQueue $parseQueue) : void
     {
-        while ($file = $this->parseQueue->getFileToParse()) {
+        foreach ($parseQueue->getAllFilesThatRequireParsing() as $file) {
             $this->processFile($file);
         }
     }
@@ -85,20 +73,14 @@ private function processFile(string $file) : void
 
         $this->kernel->postParse($document);
 
-        $dependencies = $environment->getDependencies();
-
-        foreach ($this->buildDependenciesToScan($dependencies) as $dependency) {
-            $this->scanner->scan($this->directory, $dependency);
-        }
-
         $this->metas->set(
             $file,
             $this->buildDocumentUrl($document),
             (string) $document->getTitle(),
             $document->getTitles(),
             $document->getTocs(),
             (int) filectime($fileAbsolutePath),
-            $dependencies,
+            $environment->getDependencies(),
             $environment->getLinks()
         );
     }
@@ -117,18 +99,6 @@ private function createFileParser(string $file) : Parser
         return $parser;
     }
 
-    /**
-     * @param string[] $dependencies
-     *
-     * @return string[]
-     */
-    private function buildDependenciesToScan(array $dependencies) : array
-    {
-        return array_filter($dependencies, function (string $dependency) : bool {
-            return file_exists($this->buildFileAbsolutePath($dependency));
-        });
-    }
-
     private function buildFileAbsolutePath(string $file) : string
     {
         return $this->directory . '/' . $file . '.rst';