make self-hosted the default compiler

stage1 is available behind the -fstage1 flag.

closes #89

CMakeLists.txt

@@ -12,7 +12,7 @@ if(NOT CMAKE_BUILD_TYPE)
 endif()
 
 if(NOT CMAKE_INSTALL_PREFIX)
-    set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/stage1" CACHE STRING
+    set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/stage2" CACHE STRING
       "Directory to install zig to" FORCE)
 endif()
 

build.zig

@@ -64,9 +64,9 @@ pub fn build(b: *Builder) !void {
 
     const only_install_lib_files = b.option(bool, "lib-files-only", "Only install library files") orelse false;
 
-    const is_stage1 = b.option(bool, "stage1", "Build the stage1 compiler, put stage2 behind a feature flag") orelse false;
+    const have_stage1 = b.option(bool, "enable-stage1", "Include the stage1 compiler behind a feature flag") orelse false;
     const static_llvm = b.option(bool, "static-llvm", "Disable integration with system-installed LLVM, Clang, LLD, and libc++") orelse false;
-    const enable_llvm = b.option(bool, "enable-llvm", "Build self-hosted compiler with LLVM backend enabled") orelse (is_stage1 or static_llvm);
+    const enable_llvm = b.option(bool, "enable-llvm", "Build self-hosted compiler with LLVM backend enabled") orelse (have_stage1 or static_llvm);
     const llvm_has_m68k = b.option(
         bool,
         "llvm-has-m68k",
@@ -136,7 +136,7 @@ pub fn build(b: *Builder) !void {
     };
 
     const main_file: ?[]const u8 = mf: {
-        if (!is_stage1) break :mf "src/main.zig";
+        if (!have_stage1) break :mf "src/main.zig";
         if (use_zig0) break :mf null;
         break :mf "src/stage1.zig";
     };
@@ -247,7 +247,7 @@ pub fn build(b: *Builder) !void {
             }
         };
 
-        if (is_stage1) {
+        if (have_stage1) {
             const softfloat = b.addStaticLibrary("softfloat", null);
             softfloat.setBuildMode(.ReleaseFast);
             softfloat.setTarget(target);
@@ -359,7 +359,7 @@ pub fn build(b: *Builder) !void {
     exe_options.addOption(bool, "enable_tracy_callstack", tracy_callstack);
     exe_options.addOption(bool, "enable_tracy_allocation", tracy_allocation);
     exe_options.addOption(bool, "value_tracing", value_tracing);
-    exe_options.addOption(bool, "is_stage1", is_stage1);
+    exe_options.addOption(bool, "have_stage1", have_stage1);
     if (tracy) |tracy_path| {
         const client_cpp = fs.path.join(
             b.allocator,
@@ -394,7 +394,7 @@ pub fn build(b: *Builder) !void {
     test_cases_options.addOption(bool, "enable_link_snapshots", enable_link_snapshots);
     test_cases_options.addOption(bool, "skip_non_native", skip_non_native);
     test_cases_options.addOption(bool, "skip_stage1", skip_stage1);
-    test_cases_options.addOption(bool, "is_stage1", is_stage1);
+    test_cases_options.addOption(bool, "have_stage1", have_stage1);
     test_cases_options.addOption(bool, "have_llvm", enable_llvm);
     test_cases_options.addOption(bool, "llvm_has_m68k", llvm_has_m68k);
     test_cases_options.addOption(bool, "llvm_has_csky", llvm_has_csky);
@@ -455,7 +455,6 @@ pub fn build(b: *Builder) !void {
         skip_libc,
         skip_stage1,
         false,
-        is_stage1,
     ));
 
     toolchain_step.dependOn(tests.addPkgTests(
@@ -470,7 +469,6 @@ pub fn build(b: *Builder) !void {
         true, // skip_libc
         skip_stage1,
         true, // TODO get these all passing
-        is_stage1,
     ));
 
     toolchain_step.dependOn(tests.addPkgTests(
@@ -485,7 +483,6 @@ pub fn build(b: *Builder) !void {
         true, // skip_libc
         skip_stage1,
         true, // TODO get these all passing
-        is_stage1,
     ));
 
     toolchain_step.dependOn(tests.addCompareOutputTests(b, test_filter, modes));
@@ -525,7 +522,6 @@ pub fn build(b: *Builder) !void {
         skip_libc,
         skip_stage1,
         true, // TODO get these all passing
-        is_stage1,
     );
 
     const test_step = b.step("test", "Run all the tests");

ci/azure/build.zig

@@ -37,9 +37,9 @@ pub fn build(b: *Builder) !void {
     const docs_step = b.step("docs", "Build documentation");
     docs_step.dependOn(&docgen_cmd.step);
 
-    const is_stage1 = b.option(bool, "stage1", "Build the stage1 compiler, put stage2 behind a feature flag") orelse false;
+    const have_stage1 = b.option(bool, "enable-stage1", "Include the stage1 compiler behind a feature flag") orelse false;
     const static_llvm = b.option(bool, "static-llvm", "Disable integration with system-installed LLVM, Clang, LLD, and libc++") orelse false;
-    const enable_llvm = b.option(bool, "enable-llvm", "Build self-hosted compiler with LLVM backend enabled") orelse (is_stage1 or static_llvm);
+    const enable_llvm = b.option(bool, "enable-llvm", "Build self-hosted compiler with LLVM backend enabled") orelse (have_stage1 or static_llvm);
     const llvm_has_m68k = b.option(
         bool,
         "llvm-has-m68k",
@@ -101,7 +101,7 @@ pub fn build(b: *Builder) !void {
         break :blk 4;
     };
 
-    const main_file: ?[]const u8 = if (is_stage1) null else "src/main.zig";
+    const main_file: ?[]const u8 = if (have_stage1) null else "src/main.zig";
 
     const exe = b.addExecutable("zig", main_file);
     exe.strip = strip;
@@ -190,7 +190,7 @@ pub fn build(b: *Builder) !void {
     if (enable_llvm) {
         const cmake_cfg = if (static_llvm) null else findAndParseConfigH(b, config_h_path_option);
 
-        if (is_stage1) {
+        if (have_stage1) {
             const softfloat = b.addStaticLibrary("softfloat", null);
             softfloat.setBuildMode(.ReleaseFast);
             softfloat.setTarget(target);
@@ -298,7 +298,7 @@ pub fn build(b: *Builder) !void {
     exe_options.addOption(bool, "enable_tracy_callstack", tracy_callstack);
     exe_options.addOption(bool, "enable_tracy_allocation", tracy_allocation);
     exe_options.addOption(bool, "value_tracing", value_tracing);
-    exe_options.addOption(bool, "is_stage1", is_stage1);
+    exe_options.addOption(bool, "have_stage1", have_stage1);
     if (tracy) |tracy_path| {
         const client_cpp = fs.path.join(
             b.allocator,

ci/azure/macos_script

@@ -34,7 +34,7 @@ git fetch --tags
 mkdir build
 cd build
 cmake .. \
-  -DCMAKE_INSTALL_PREFIX="$(pwd)/release" \
+  -DCMAKE_INSTALL_PREFIX="$(pwd)/stage2" \
   -DCMAKE_PREFIX_PATH="$PREFIX" \
   -DCMAKE_BUILD_TYPE=Release \
   -DZIG_TARGET_TRIPLE="$TARGET" \
@@ -52,31 +52,26 @@ make $JOBS install
 # Here we rebuild zig but this time using the Zig binary we just now produced to
 # build zig1.o rather than relying on the one built with stage0. See
 # https://github.com/ziglang/zig/issues/6830 for more details.
-cmake .. -DZIG_EXECUTABLE="$(pwd)/release/bin/zig"
+cmake .. -DZIG_EXECUTABLE="$(pwd)/stage2/bin/zig"
 make $JOBS install
 
-# Build stage2 standalone so that we can test stage2 against stage2 compiler-rt.
-release/bin/zig build -p stage2 -Denable-llvm
+stage2/bin/zig build -p release -Denable-llvm -Denable-stage1
 
-stage2/bin/zig build test-behavior
-
-# TODO: upgrade these to test stage2 instead of stage1
-# TODO: upgrade these to test stage3 instead of stage2
-release/bin/zig build test-behavior         -Denable-macos-sdk -Domit-stage2
 release/bin/zig build test-compiler-rt      -Denable-macos-sdk
+release/bin/zig build test-behavior         -Denable-macos-sdk
 release/bin/zig build test-std              -Denable-macos-sdk
 release/bin/zig build test-universal-libc   -Denable-macos-sdk
 release/bin/zig build test-compare-output   -Denable-macos-sdk
 release/bin/zig build test-standalone       -Denable-macos-sdk
-release/bin/zig build test-stack-traces     -Denable-macos-sdk
+release/bin/zig build test-stack-traces     -Denable-macos-sdk -fstage1
 release/bin/zig build test-cli              -Denable-macos-sdk
 release/bin/zig build test-asm-link         -Denable-macos-sdk
 release/bin/zig build test-translate-c      -Denable-macos-sdk
 release/bin/zig build test-run-translated-c -Denable-macos-sdk
 release/bin/zig build docs                  -Denable-macos-sdk
 release/bin/zig build test-fmt              -Denable-macos-sdk
 release/bin/zig build test-cases            -Denable-macos-sdk -Dsingle-threaded
-release/bin/zig build test-link             -Denable-macos-sdk -Domit-stage2
+release/bin/zig build test-link             -Denable-macos-sdk
 
 if [ "${BUILD_REASON}" != "PullRequest" ]; then
   mv ../LICENSE release/

ci/azure/pipelines.yml

@@ -68,9 +68,7 @@ jobs:
       & "${ZIGPREFIXPATH}/bin/zig.exe" build `
         --prefix "$ZIGINSTALLDIR" `
         --search-prefix "$ZIGPREFIXPATH" `
-        -Dstage1 `
-        <# stage2 is omitted until we resolve https://github.com/ziglang/zig/issues/6485 #> `
-        -Domit-stage2 `
+        -Denable-stage1 `
         -Dstatic-llvm `
         -Drelease `
         -Dstrip `

ci/srht/freebsd_script

@@ -38,10 +38,11 @@ cmake .. \
     -GNinja
 samu install
 
-# TODO ld.lld: error: undefined symbol: main
-# >>> referenced by crt1_c.c:75 (/usr/src/lib/csu/amd64/crt1_c.c:75)
-# >>>               /usr/lib/crt1.o:(_start)
-#release/bin/zig test ../test/behavior.zig -fno-stage1 -fLLVM -I ../test
+# Here we rebuild zig but this time using the Zig binary we just now produced to
+# build zig1.o rather than relying on the one built with stage0. This makes it
+# a stage3 build rather than a stage2 build.
+cmake .. -DZIG_EXECUTABLE="$PREFIX/bin/zig"
+samu install
 
 # Here we skip some tests to save time.
 release/bin/zig build test -Dskip-stage1 -Dskip-non-native

ci/zinc/linux_test.sh

@@ -33,41 +33,39 @@ unset CXX
 
 ninja install
 
-STAGE1_ZIG="$DEBUG_STAGING/bin/zig"
-
 # Here we rebuild zig but this time using the Zig binary we just now produced to
 # build zig1.o rather than relying on the one built with stage0. See
 # https://github.com/ziglang/zig/issues/6830 for more details.
-cmake .. -DZIG_EXECUTABLE="$STAGE1_ZIG"
+cmake .. -DZIG_EXECUTABLE="$DEBUG_STAGING/bin/zig"
 ninja install
 
 cd $WORKSPACE
 
+"$DEBUG_STAGING/bin/zig" build -p stage3 -Denable-stage1 -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
+
+# simultaneously test building self-hosted without LLVM and with 32-bit arm
+stage3/bin/zig build -Dtarget=arm-linux-musleabihf
+
 echo "Looking for non-conforming code formatting..."
 echo "Formatting errors can be fixed by running 'zig fmt' on the files printed here."
-$STAGE1_ZIG fmt --check . --exclude test/cases/
-
-$STAGE1_ZIG    build -p stage2 -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
-stage2/bin/zig build -p stage3 -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
-stage3/bin/zig build # test building self-hosted without LLVM
-stage3/bin/zig build -Dtarget=arm-linux-musleabihf # test building self-hosted for 32-bit arm
-
-stage3/bin/zig build test-compiler-rt      -fqemu -fwasmtime -Denable-llvm
-stage3/bin/zig build test-behavior         -fqemu -fwasmtime -Denable-llvm
-stage3/bin/zig build test-std              -fqemu -fwasmtime -Denable-llvm
-stage3/bin/zig build test-universal-libc   -fqemu -fwasmtime -Denable-llvm
-stage3/bin/zig build test-compare-output   -fqemu -fwasmtime -Denable-llvm
-stage3/bin/zig build test-asm-link         -fqemu -fwasmtime -Denable-llvm
-stage3/bin/zig build test-fmt              -fqemu -fwasmtime -Denable-llvm
-stage3/bin/zig build test-translate-c      -fqemu -fwasmtime -Denable-llvm
-stage3/bin/zig build test-run-translated-c -fqemu -fwasmtime -Denable-llvm
-stage3/bin/zig build test-standalone       -fqemu -fwasmtime -Denable-llvm
-stage3/bin/zig build test-cli              -fqemu -fwasmtime -Denable-llvm
+stage3/bin/zig fmt --check . --exclude test/cases/
+
+stage3/bin/zig build test-compiler-rt      -fqemu -fwasmtime -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
+stage3/bin/zig build test-behavior         -fqemu -fwasmtime -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
+stage3/bin/zig build test-std              -fqemu -fwasmtime -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
+stage3/bin/zig build test-universal-libc   -fqemu -fwasmtime -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
+stage3/bin/zig build test-compare-output   -fqemu -fwasmtime -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
+stage3/bin/zig build test-asm-link         -fqemu -fwasmtime -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
+stage3/bin/zig build test-fmt              -fqemu -fwasmtime -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
+stage3/bin/zig build test-translate-c      -fqemu -fwasmtime -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
+stage3/bin/zig build test-run-translated-c -fqemu -fwasmtime -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
+stage3/bin/zig build test-standalone       -fqemu -fwasmtime -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
+stage3/bin/zig build test-cli              -fqemu -fwasmtime -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
 stage3/bin/zig build test-cases            -fqemu -fwasmtime -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
-stage3/bin/zig build test-link             -fqemu -fwasmtime -Denable-llvm
+stage3/bin/zig build test-link             -fqemu -fwasmtime -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
+stage3/bin/zig build docs                  -fqemu -fwasmtime -Dstatic-llvm -Dtarget=native-native-musl --search-prefix "$DEPS_LOCAL"
 
-$STAGE1_ZIG build test-stack-traces     -fqemu -fwasmtime
-$STAGE1_ZIG build docs                  -fqemu -fwasmtime
+stage3/bin/zig build test-stack-traces -fqemu -fwasmtime -fstage1
 
 # Produce the experimental std lib documentation.
 mkdir -p "$RELEASE_STAGING/docs/std"
@@ -87,7 +85,7 @@ stage3/bin/zig build \
   -Drelease \
   -Dstrip \
   -Dtarget="$TARGET" \
-  -Dstage1
+  -Denable-stage1
 
 # Explicit exit helps show last command duration.
 exit

doc/docgen.zig

@@ -285,6 +285,7 @@ const Code = struct {
     link_objects: []const []const u8,
     target_str: ?[]const u8,
     link_libc: bool,
+    backend_stage1: bool,
     link_mode: ?std.builtin.LinkMode,
     disable_cache: bool,
     verbose_cimport: bool,
@@ -554,6 +555,7 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc {
                     var link_mode: ?std.builtin.LinkMode = null;
                     var disable_cache = false;
                     var verbose_cimport = false;
+                    var backend_stage1 = false;
 
                     const source_token = while (true) {
                         const content_tok = try eatToken(tokenizer, Token.Id.Content);
@@ -586,6 +588,8 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc {
                             link_libc = true;
                         } else if (mem.eql(u8, end_tag_name, "link_mode_dynamic")) {
                             link_mode = .Dynamic;
+                        } else if (mem.eql(u8, end_tag_name, "backend_stage1")) {
+                            backend_stage1 = true;
                         } else if (mem.eql(u8, end_tag_name, "code_end")) {
                             _ = try eatToken(tokenizer, Token.Id.BracketClose);
                             break content_tok;
@@ -609,6 +613,7 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc {
                             .link_objects = link_objects.toOwnedSlice(),
                             .target_str = target_str,
                             .link_libc = link_libc,
+                            .backend_stage1 = backend_stage1,
                             .link_mode = link_mode,
                             .disable_cache = disable_cache,
                             .verbose_cimport = verbose_cimport,
@@ -1187,6 +1192,9 @@ fn printShell(out: anytype, shell_content: []const u8) !void {
     try out.writeAll("</samp></pre></figure>");
 }
 
+// Override this to skip to later tests
+const debug_start_line = 0;
+
 fn genHtml(
     allocator: Allocator,
     tokenizer: *Tokenizer,
@@ -1266,6 +1274,13 @@ fn genHtml(
                     continue;
                 }
 
+                if (debug_start_line > 0) {
+                    const loc = tokenizer.getTokenLocation(code.source_token);
+                    if (debug_start_line > loc.line) {
+                        continue;
+                    }
+                }
+
                 const raw_source = tokenizer.buffer[code.source_token.start..code.source_token.end];
                 const trimmed_raw_source = mem.trim(u8, raw_source, " \n");
                 const tmp_source_file_name = try fs.path.join(
@@ -1311,6 +1326,10 @@ fn genHtml(
                             try build_args.append("-lc");
                             try shell_out.print("-lc ", .{});
                         }
+                        if (code.backend_stage1) {
+                            try build_args.append("-fstage1");
+                            try shell_out.print("-fstage1", .{});
+                        }
                         const target = try std.zig.CrossTarget.parse(.{
                             .arch_os_abi = code.target_str orelse "native",
                         });
@@ -1443,6 +1462,10 @@ fn genHtml(
                             try test_args.append("-lc");
                             try shell_out.print("-lc ", .{});
                         }
+                        if (code.backend_stage1) {
+                            try test_args.append("-fstage1");
+                            try shell_out.print("-fstage1", .{});
+                        }
                         if (code.target_str) |triple| {
                             try test_args.appendSlice(&[_][]const u8{ "-target", triple });
                             try shell_out.print("-target {s} ", .{triple});
@@ -1490,6 +1513,14 @@ fn genHtml(
                                 try shell_out.print("-O {s} ", .{@tagName(code.mode)});
                             },
                         }
+                        if (code.link_libc) {
+                            try test_args.append("-lc");
+                            try shell_out.print("-lc ", .{});
+                        }
+                        if (code.backend_stage1) {
+                            try test_args.append("-fstage1");
+                            try shell_out.print("-fstage1", .{});
+                        }
                         const result = try ChildProcess.exec(.{
                             .allocator = allocator,
                             .argv = test_args.items,