Expand command to compile Elm Interactive environment

+ Add an option on the command to filter the Elm modules from the sources for a given set of root modules.
+ Add an option on the command to disable the lowering step explicitly.
+ Add an option on the command to compress the compacted representation further using gzip.
+ Expand logging to report statistics.

implement/Pine.Core/CommandLineInterface.cs

@@ -0,0 +1,21 @@
+using System.Globalization;
+
+namespace Pine.Core;
+
+public class CommandLineInterface
+{
+    public static string FormatIntegerForDisplay(long integer) =>
+        FormatIntegerForDisplay(integer, '_');
+
+    public static string FormatIntegerForDisplay(long integer, char thousandsSeparator) =>
+        integer.ToString("N",
+            BuildNumberFormatInfo(thousandsSeparator));
+
+    private static NumberFormatInfo BuildNumberFormatInfo(char thousandsSeparator) =>
+        new()
+        {
+            NumberGroupSizes = [3, 3, 3, 3, 3, 3],
+            NumberGroupSeparator = thousandsSeparator.ToString(),
+            NumberDecimalDigits = 0
+        };
+}

implement/Pine.Core/Elm/BundledElmEnvironments.cs

@@ -101,7 +101,15 @@ public static Result<string, IReadOnlyDictionary<string, PineValue>> LoadBundled
     public static void CompressAndWriteBundleFile(
         IReadOnlyDictionary<TreeNodeWithStringPath, PineValue> compiledEnvironments)
     {
-        var fileContent = CompressResourceFile(BuildBundleResourceFileJsonUtf8(compiledEnvironments));
+        var (allListEntries, uncompressed) =
+            BuildBundleResourceFileJsonUtf8(compiledEnvironments);
+
+        System.Console.WriteLine(
+            "Built " + CommandLineInterface.FormatIntegerForDisplay(allListEntries.Count) + " list entries for " +
+            compiledEnvironments.Count + " compiled environments with an uncompressed size of " +
+            CommandLineInterface.FormatIntegerForDisplay(uncompressed.Length) + " bytes.");
+
+        var fileContent = CompressResourceFile(uncompressed);
 
         CompressAndWriteBundleFile(fileContent);
     }
@@ -110,7 +118,7 @@ public static void CompressAndWriteBundleFile(
         System.ReadOnlyMemory<byte> fileContent)
     {
         System.Console.WriteLine(
-        "Current working directory: " + System.Environment.CurrentDirectory);
+            "Current working directory: " + System.Environment.CurrentDirectory);
 
         var absolutePath = System.IO.Path.GetFullPath(ResourceFilePath);
 
@@ -143,7 +151,16 @@ public static System.ReadOnlyMemory<byte> CompressResourceFile(
         return memoryStream.ToArray();
     }
 
-    public static System.ReadOnlyMemory<byte> BuildBundleResourceFileJsonUtf8(
+    public static (IReadOnlyList<PineValueCompactBuild.ListEntry>, System.ReadOnlyMemory<byte>) BuildBundleResourceFileJsonUtf8(
+        IReadOnlyDictionary<TreeNodeWithStringPath, PineValue> compiledEnvironments)
+    {
+        IReadOnlyList<PineValueCompactBuild.ListEntry> allEntries =
+            BuildBundleResourceFileListItems(compiledEnvironments);
+
+        return (allEntries, System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(allEntries));
+    }
+
+    public static IReadOnlyList<PineValueCompactBuild.ListEntry> BuildBundleResourceFileListItems(
         IReadOnlyDictionary<TreeNodeWithStringPath, PineValue> compiledEnvironments)
     {
         var (listValues, blobValues) =
@@ -174,11 +191,9 @@ public static System.ReadOnlyMemory<byte> BuildBundleResourceFileJsonUtf8(
             })
             .ToImmutableArray();
 
-        IReadOnlyList<PineValueCompactBuild.ListEntry> allEntries =
+        return
             [..sharedValuesDict.listEntries
             , ..environmentEntries
             ];
-
-        return System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(allEntries);
     }
 }

implement/pine/Elm/ElmCompiler.cs

@@ -95,7 +95,10 @@ public static Result<string, ElmCompiler> BuildElmCompiler(
             ElmCompilerFileTreeFromBundledFileTree(compilerSourceFiles);
 
         return
-            LoadOrCompileInteractiveEnvironment(compilerWithPackagesTree)
+            LoadOrCompileInteractiveEnvironment(
+                compilerWithPackagesTree,
+                rootFilePaths: [],
+                skipLowering: true)
             .AndThen(compiledEnv =>
             {
                 return ElmCompilerFromEnvValue(
@@ -207,9 +210,12 @@ public static TreeNodeWithStringPath MergeElmCoreModules(
     }
 
     public static Result<string, PineValue> LoadOrCompileInteractiveEnvironment(
-        TreeNodeWithStringPath compilerSourceFiles)
+        TreeNodeWithStringPath compilerSourceFiles,
+        IReadOnlyList<IReadOnlyList<string>> rootFilePaths,
+        bool skipLowering)
     {
-        if (BundledElmEnvironments.BundledElmEnvironmentFromFileTree(compilerSourceFiles) is { } fromBundle)
+        if (rootFilePaths.Count is 0 &&
+            BundledElmEnvironments.BundledElmEnvironmentFromFileTree(compilerSourceFiles) is { } fromBundle)
         {
             return Result<string, PineValue>.ok(fromBundle);
         }
@@ -218,16 +224,23 @@ public static Result<string, PineValue> LoadOrCompileInteractiveEnvironment(
          * TODO: Load from cache
          * */
 
-        return CompileInteractiveEnvironment(compilerSourceFiles);
+        return
+            CompileInteractiveEnvironment(
+                compilerSourceFiles,
+                rootFilePaths: rootFilePaths,
+                skipLowering: skipLowering);
     }
 
     public static Result<string, PineValue> CompileInteractiveEnvironment(
-        TreeNodeWithStringPath compilerSourceFiles)
+        TreeNodeWithStringPath compilerSourceFiles,
+        IReadOnlyList<IReadOnlyList<string>> rootFilePaths,
+        bool skipLowering)
     {
         return
             ElmTime.ElmInteractive.ElmInteractive.CompileInteractiveEnvironment(
-                initialState: null,
                 appCodeTree: compilerSourceFiles,
+                rootFilePaths: rootFilePaths,
+                skipLowering: skipLowering,
                 compilationCacheBefore: ElmTime.ElmInteractive.ElmInteractive.CompilationCache.Empty)
             .Map(result => result.compileResult);
     }

implement/pine/ElmInteractive/ElmInteractive.cs

@@ -41,7 +41,15 @@ public static Result<string, EvaluatedStruct> EvaluateSubmissionAndGetResultingV
         string submission,
         IReadOnlyList<string>? previousLocalSubmissions = null)
     {
-        var modulesTexts = ModulesTextsFromAppCodeTree(appCodeTree);
+        var modulesTexts =
+            appCodeTree is null
+            ?
+            []
+            :
+            ModulesTextsFromAppCodeTree(
+                appCodeTree,
+                skipLowering: false,
+                rootFilePaths: []);
 
         var argumentsJson = System.Text.Json.JsonSerializer.Serialize(
             new
@@ -79,22 +87,29 @@ public static IReadOnlyList<string> GetDefaultElmCoreModulesTexts(
     }
 
     public static Result<string, (PineValue compileResult, CompilationCache compilationCache)> CompileInteractiveEnvironment(
-        PineValue? initialState,
         TreeNodeWithStringPath? appCodeTree,
+        IReadOnlyList<IReadOnlyList<string>> rootFilePaths,
+        bool skipLowering,
         CompilationCache compilationCacheBefore)
     {
-        var allModulesTexts =
+        var includedModulesTexts =
             /*
             GetDefaultElmCoreModulesTexts(evalElmPreparedJavaScriptEngine)
             .Concat(ModulesTextsFromAppCodeTree(appCodeTree) ?? [])
             .ToImmutableList();
             */
-            ModulesTextsFromAppCodeTree(appCodeTree) ?? [];
+            appCodeTree is null
+            ?
+            []
+            :
+            ModulesTextsFromAppCodeTree(
+                appCodeTree,
+                skipLowering,
+                rootFilePaths: rootFilePaths);
 
         return
             CompileInteractiveEnvironmentForModulesCachingIncrements(
-                initialState: initialState,
-                elmModulesTextsBeforeSort: allModulesTexts,
+                elmModulesTextsBeforeSort: includedModulesTexts,
                 compilationCacheBefore)
             .Map(compileResultAndCache =>
             (compileResult: compileResultAndCache.compileResult.environmentPineValue,
@@ -103,13 +118,9 @@ public static IReadOnlyList<string> GetDefaultElmCoreModulesTexts(
 
     private static Result<string, (CompileInteractiveEnvironmentResult compileResult, CompilationCache compilationCache)>
         CompileInteractiveEnvironmentForModulesCachingIncrements(
-        PineValue? initialState,
         IReadOnlyList<string> elmModulesTextsBeforeSort,
         CompilationCache compilationCacheBefore)
     {
-        if (initialState is not null)
-            throw new NotImplementedException("initialState is not implemented");
-
         var elmModulesTexts =
             ElmSyntax.ElmModule.ModulesTextOrderedForCompilationByDependencies(elmModulesTextsBeforeSort)
             .ToImmutableList();
@@ -820,42 +831,77 @@ public static PineValue ParsePineValueFromJson(
         return dictionary;
     }
 
-    public static IReadOnlyList<string>? ModulesTextsFromAppCodeTree(TreeNodeWithStringPath? appCodeTree) =>
-        ModulesFilePathsAndTextsFromAppCodeTree(appCodeTree)
-        ?.Select(pathAndText => pathAndText.moduleText)
-        .ToImmutableArray();
+    public static IReadOnlyList<string> ModulesTextsFromAppCodeTree(
+        TreeNodeWithStringPath appCodeTree,
+        bool skipLowering,
+        IReadOnlyList<IReadOnlyList<string>> rootFilePaths)
+    {
+        var allModules =
+            ModulesFilePathsAndTextsFromAppCodeTree(appCodeTree, skipLowering)
+            .Where(file => !ShouldIgnoreSourceFile(file.filePath, file.fileContent))
+            .ToImmutableArray();
+
+        var rootModules =
+            allModules
+            .Where(c => rootFilePaths.Count is 0 || rootFilePaths.Any(rootFilePath => c.filePath.SequenceEqual(rootFilePath)))
+            .ToImmutableArray();
+
+        return
+            ElmSyntax.ElmModule.ModulesTextOrderedForCompilationByDependencies(
+                rootModulesTexts:
+                [.. rootModules.Select(m => m.moduleText)],
+                [.. allModules.Select(m => m.moduleText)]);
+    }
+
+    static bool ShouldIgnoreSourceFile(IReadOnlyList<string> filePath, ReadOnlyMemory<byte> fileContent)
+    {
+        /*
+         * Adapt to observation 2024-10-25:
+         * Unhandled exception. System.Exception: Failed compilation: Failed to prepare the initial context: Failed to compile elm module 'Reporter': Failed to compile declaration: Failed to compile function 'main': Failed to compile Elm function syntax: Did not find module 'ElmTestRunner.Reporter'. There are 18 declarations in this scope: Basics, Bitwise, Bytes, Bytes.Decode, Bytes.Encode, Char, Dict, Elm.Kernel.Parser, Hex, Json.Decode, Json.Encode, Kernel.Json.Decode, Kernel.Json.Encode, List, Maybe, Result, String, Tuple
+         * 
+         * It turns out a tool had created a "port module Reporter" and "port module Runner"
+         * in "elm-compiler\elm-stuff\tests-0.19.1\src\Reporter.elm"
+         * */
+
+        if (filePath.Contains("elm-stuff"))
+            return true;
+
+        return false;
+    }
 
-    public static IReadOnlyList<(IReadOnlyList<string> filePath, ReadOnlyMemory<byte> fileContent, string moduleText)>?
+    public static IReadOnlyList<(IReadOnlyList<string> filePath, ReadOnlyMemory<byte> fileContent, string moduleText)>
         ModulesFilePathsAndTextsFromAppCodeTree(
-        TreeNodeWithStringPath? appCodeTree) =>
-        appCodeTree == null ?
-        null
-        :
-        CompileTree(appCodeTree) is { } compiledTree ?
-        TreeToFlatDictionaryWithPathComparer(compiledTree)
-        .Where(sourceFile => sourceFile.Key.Last().EndsWith(".elm"))
-        .Select(appCodeFile => (appCodeFile.Key, appCodeFile.Value, Encoding.UTF8.GetString(appCodeFile.Value.Span)))
-        .ToImmutableList()
-        :
-        null;
-
-    private static TreeNodeWithStringPath? CompileTree(TreeNodeWithStringPath? sourceTree)
+        TreeNodeWithStringPath appCodeTree,
+        bool skipLowering)
     {
-        if (sourceTree is null)
-            return null;
+        var loweredTree =
+            skipLowering
+            ?
+            appCodeTree
+            :
+            CompileTree(appCodeTree);
+
+        return
+            [.. TreeToFlatDictionaryWithPathComparer(loweredTree)
+            .Where(sourceFile => sourceFile.Key.Last().EndsWith(".elm"))
+            .Select(appCodeFile => (appCodeFile.Key, appCodeFile.Value, Encoding.UTF8.GetString(appCodeFile.Value.Span)))
+            ];
+    }
 
+    private static TreeNodeWithStringPath CompileTree(TreeNodeWithStringPath sourceTree)
+    {
         if (sourceTree.GetNodeAtPath(["elm.json"]) is null)
             return sourceTree;
 
         var sourceFiles = TreeToFlatDictionaryWithPathComparer(sourceTree);
 
-        if (sourceFiles.Count == 0)
-            return null;
+        if (sourceFiles.Count is 0)
+            return sourceTree;
 
         var compilationRootFilePath = sourceFiles.FirstOrDefault(c => c.Key[c.Key.Count - 1].EndsWith(".elm")).Key;
 
-        if (compilationRootFilePath.Count == 0)
-            return null;
+        if (compilationRootFilePath.Count is 0)
+            return sourceTree;
 
         var compilationResult = ElmAppCompilation.AsCompletelyLoweredElmApp(
             sourceFiles: TreeToFlatDictionaryWithPathComparer(sourceTree),
@@ -909,10 +955,10 @@ public static string PrepareJavaScriptToEvaluateElm(
         IImmutableDictionary<IReadOnlyList<string>, ReadOnlyMemory<byte>> compileElmProgramCodeFiles)
     {
         var elmMakeResult =
-        Elm019Binaries.ElmMakeToJavascript(
-            compileElmProgramCodeFiles,
-            workingDirectoryRelative: null,
-            ["src", "ElmInteractiveMain.elm"]);
+            Elm019Binaries.ElmMakeToJavascript(
+                compileElmProgramCodeFiles,
+                workingDirectoryRelative: null,
+                ["src", "ElmInteractiveMain.elm"]);
 
         var javascriptFromElmMake =
             Encoding.UTF8.GetString(

implement/pine/ElmInteractive/InteractiveSessionPine.cs

@@ -347,7 +347,7 @@ public record ParsedModule(
         TreeNodeWithStringPath appSourceFiles)
     {
         var compileableSourceModules =
-            ElmInteractive.ModulesFilePathsAndTextsFromAppCodeTree(appSourceFiles) ?? [];
+            ElmInteractive.ModulesFilePathsAndTextsFromAppCodeTree(appSourceFiles, skipLowering: false) ?? [];
 
         var baseTree =
             compileableSourceModules

implement/pine/ElmSyntax/ElmModule.cs

@@ -56,6 +56,20 @@ bool IsRootModule(string moduleText) =>
                 )))
             .ToImmutableList();
 
+        foreach (var module in parsedModules)
+        {
+            var modulesWithSameName =
+                parsedModules
+                .Where(parsedModule => parsedModule.parsedModule.ModuleName.SequenceEqual(module.parsedModule.ModuleName))
+                .ToImmutableList();
+
+            if (1 < modulesWithSameName.Count)
+            {
+                throw new Exception(
+                    "Multiple modules with the same name: " + string.Join(".", module.parsedModule.ModuleName));
+            }
+        }
+
         var parsedModulesByName =
             parsedModules
             .ToImmutableDictionary(
@@ -97,6 +111,7 @@ IReadOnlySet<IReadOnlyList<string>> ListImportsOfModuleTransitive(IReadOnlyList<
             .SelectMany(rootModule =>
             ListImportsOfModuleTransitive(rootModule.parsedModule.ModuleName)
             .Prepend(rootModule.parsedModule.ModuleName))
+            .Concat(ElmCoreAutoImportedModulesNames)
             .Intersect(
                 parsedModules.Select(pm => pm.parsedModule.ModuleName),
                 EnumerableExtension.EqualityComparer<IReadOnlyList<string>>())