Refactoring how the Scanner system works

It's much simpler: Sphinx finds all files in the given directory,
recursively, that have the .rst file extension. And it parses all
of them. There is no need to follow "dependencies" to find more things
to parse: just parse all files you find. Sure, then later you might
be able to give a warning if some documents you rendered were not
included in any toctree's - but you will still render those orphan
documents.

lib/Builder.php

@@ -13,7 +13,6 @@
 use Doctrine\RST\Event\PreBuildParseEvent;
 use Doctrine\RST\Event\PreBuildRenderEvent;
 use Doctrine\RST\Event\PreBuildScanEvent;
-use Doctrine\RST\Meta\MetaEntry;
 use Doctrine\RST\Meta\Metas;
 use Symfony\Component\Filesystem\Filesystem;
 use function is_dir;
@@ -38,12 +37,6 @@ class Builder
     /** @var Documents */
     private $documents;
 
-    /** @var ParseQueue */
-    private $parseQueue;
-
-    /** @var Scanner */
-    private $scanner;
-
     /** @var Copier */
     private $copier;
 
@@ -67,10 +60,6 @@ public function __construct(?Kernel $kernel = null)
             $this->metas
         );
 
-        $this->parseQueue = new Builder\ParseQueue($this->documents);
-
-        $this->scanner = new Builder\Scanner($this->parseQueue, $this->metas);
-
         $this->copier = new Builder\Copier($this->filesystem);
 
         $this->kernel->initBuilder($this);
@@ -124,9 +113,9 @@ public function build(
 
         $this->loadCachedMetas($targetDirectory);
 
-        $this->scan($directory, $targetDirectory);
+        $parseQueue = $this->scan($directory, $targetDirectory);
 
-        $this->parse($directory, $targetDirectory);
+        $this->parse($directory, $targetDirectory, $parseQueue);
 
         $this->render($directory, $targetDirectory);
 
@@ -147,19 +136,23 @@ public function mkdir(string $directory) : self
         return $this;
     }
 
-    private function scan(string $directory, string $targetDirectory) : void
+    private function scan(string $directory, string $targetDirectory) : ParseQueue
     {
         $this->configuration->dispatchEvent(
             PreBuildScanEvent::PRE_BUILD_SCAN,
             new PreBuildScanEvent($this, $directory, $targetDirectory)
         );
 
-        $this->scanner->scan($directory, $this->getIndexName());
+        $scanner = new Scanner(
+            $this->configuration->getSourceFileExtension(),
+            $directory,
+            $this->metas
+        );
 
-        $this->scanner->scanMetas($directory);
+        return $scanner->scan($directory, $this->metas);
     }
 
-    private function parse(string $directory, string $targetDirectory) : void
+    private function parse(string $directory, string $targetDirectory, ParseQueue $parseQueue) : void
     {
         $this->configuration->dispatchEvent(
             PreBuildParseEvent::PRE_BUILD_PARSE,
@@ -169,16 +162,14 @@ private function parse(string $directory, string $targetDirectory) : void
         $parseQueueProcessor = new ParseQueueProcessor(
             $this->kernel,
             $this->errorManager,
-            $this->parseQueue,
             $this->metas,
             $this->documents,
-            $this->scanner,
             $directory,
             $targetDirectory,
             $this->configuration->getFileExtension()
         );
 
-        $parseQueueProcessor->process();
+        $parseQueueProcessor->process($parseQueue);
     }
 
     private function render(string $directory, string $targetDirectory) : void

lib/Builder/ParseQueue.php

@@ -4,51 +4,46 @@
 
 namespace Doctrine\RST\Builder;
 
-use function array_shift;
-
-class ParseQueue
+final class ParseQueue
 {
-    /** @var Documents */
-    private $documents;
-
-    /** @var string[] */
-    private $parseQueue = [];
-
-    /** @var int[] */
-    private $states = [];
-
-    public function __construct(Documents $documents)
+    /**
+     * An array where each key is the filename and the value is a
+     * boolean indicating if the file needs to be parsed or not.
+     *
+     * @var bool[]
+     */
+    private $fileStatuses = [];
+
+    public function addFile(string $filename, bool $parseNeeded) : void
     {
-        $this->documents = $documents;
-    }
-
-    public function getState(string $file) : ?int
-    {
-        return $this->states[$file] ?? null;
+        if (isset($this->fileStatuses[$filename])) {
+            throw new \InvalidArgumentException(sprintf('File "%s" is already in the parse queue', $filename));
+        }
+
+        $this->fileStatuses[$filename] = $parseNeeded;
     }
 
-    public function setState(string $file, int $state) : void
+    public function isFileKnownToParseQueue(string $filename) : bool
     {
-        $this->states[$file] = $state;
+        return array_key_exists($filename, $this->fileStatuses);
     }
 
-    public function getFileToParse() : ?string
+    public function doesFileRequireParsing(string $filename) : bool
     {
-        if ($this->parseQueue !== []) {
-            return array_shift($this->parseQueue);
+        if (!$this->isFileKnownToParseQueue($filename)) {
+            throw new \InvalidArgumentException(sprintf('File "%s" is not known to the parse queue', $filename));
         }
 
-        return null;
+        return $this->fileStatuses[$filename];
     }
 
-    public function addToParseQueue(string $file) : void
+    /**
+     * @return string[]
+     */
+    public function getAllFilesThatRequireParsing() : array
     {
-        $this->states[$file] = State::PARSE;
-
-        if ($this->documents->hasDocument($file)) {
-            return;
-        }
-
-        $this->parseQueue[$file] = $file;
+        return array_keys(array_filter($this->fileStatuses, function(bool $parseNeeded) {
+            return $parseNeeded;
+        }));
     }
 }

lib/Builder/ParseQueueProcessor.php

@@ -7,6 +7,7 @@
 use Doctrine\RST\Environment;
 use Doctrine\RST\ErrorManager;
 use Doctrine\RST\Kernel;
+use Doctrine\RST\Meta\DocumentDependency;
 use Doctrine\RST\Meta\Metas;
 use Doctrine\RST\Nodes\DocumentNode;
 use Doctrine\RST\Parser;
@@ -22,18 +23,12 @@ class ParseQueueProcessor
     /** @var ErrorManager */
     private $errorManager;
 
-    /** @var ParseQueue */
-    private $parseQueue;
-
     /** @var Metas */
     private $metas;
 
     /** @var Documents */
     private $documents;
 
-    /** @var Scanner */
-    private $scanner;
-
     /** @var string */
     private $directory;
 
@@ -46,28 +41,24 @@ class ParseQueueProcessor
     public function __construct(
         Kernel $kernel,
         ErrorManager $errorManager,
-        ParseQueue $parseQueue,
         Metas $metas,
         Documents $documents,
-        Scanner $scanner,
         string $directory,
         string $targetDirectory,
         string $fileExtension
     ) {
         $this->kernel          = $kernel;
         $this->errorManager    = $errorManager;
-        $this->parseQueue      = $parseQueue;
         $this->metas           = $metas;
         $this->documents       = $documents;
-        $this->scanner         = $scanner;
         $this->directory       = $directory;
         $this->targetDirectory = $targetDirectory;
         $this->fileExtension   = $fileExtension;
     }
 
-    public function process() : void
+    public function process(ParseQueue $parseQueue) : void
     {
-        while ($file = $this->parseQueue->getFileToParse()) {
+        foreach ($parseQueue->getAllFilesThatRequireParsing() as $file) {
             $this->processFile($file);
         }
     }
@@ -86,20 +77,14 @@ private function processFile(string $file) : void
 
         $this->kernel->postParse($document);
 
-        $dependencies = $environment->getDependencies();
-
-        foreach ($this->buildDependenciesToScan($dependencies) as $dependency) {
-            $this->scanner->scan($this->directory, $dependency);
-        }
-
         $this->metas->set(
             $file,
             $this->buildDocumentUrl($document),
             (string) $document->getTitle(),
             $document->getTitles(),
             $document->getTocs(),
             (int) filectime($fileAbsolutePath),
-            $dependencies,
+            $environment->getDependencies(),
             $environment->getLinks()
         );
     }
@@ -119,18 +104,6 @@ private function createFileParser(string $file) : Parser
         return $parser;
     }
 
-    /**
-     * @param string[] $dependencies
-     *
-     * @return string[]
-     */
-    private function buildDependenciesToScan(array $dependencies) : array
-    {
-        return array_filter($dependencies, function (string $dependency) : bool {
-            return file_exists($this->buildFileAbsolutePath($dependency));
-        });
-    }
-
     private function buildFileAbsolutePath(string $file) : string
     {
         return $this->directory . '/' . $file . '.rst';