Improve JS Sourcemaps (#674)

* feat: better sourcemaps

* feat(js): improve JS sourcemaps

* test: fixup tests

* test: add JS sourcemap tests

* chore: add changeset

* Add internal/js_scanner/js_scanner_test.go#FuzzHoistImport  (#675)

* Add internal/js_scanner/js_scanner_test.go#FuzzHoistImport

Runnable via `go test ./internal/js_scanner -fuzz=FuzzHoistImports`

* Update internal/js_scanner/js_scanner_test.go

Co-authored-by: Nate Moore <natemoo-re@users.noreply.github.com>

* fix(js_scanner): assert that i < len(source) when scanning

Co-authored-by: Nate Moore <natemoo-re@users.noreply.github.com>
Co-authored-by: Nate Moore <nate@astro.build>

Co-authored-by: Nate Moore <nate@astro.build>
Co-authored-by: Caleb Jasik <calebjasik@jasik.xyz>

.changeset/stupid-papayas-accept.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/compiler': patch
+---
+
+Improve fidelity of sourcemaps for frontmatter

.gitpod.yml

@@ -0,0 +1,9 @@
+# This configuration file was automatically generated by Gitpod.
+# Please adjust to your needs (see https://www.gitpod.io/docs/config-gitpod-file)
+# and commit this file to your remote git repository to share the goodness with others.
+
+tasks:
+  - init: pnpm install && pnpm run build && go get && go build ./... && go test ./... && make
+    command: go run .
+
+

internal/js_scanner/js_scanner.go

@@ -13,24 +13,33 @@ import (
 )
 
 type HoistedScripts struct {
-	Hoisted [][]byte
-	Body    []byte
+	Hoisted     [][]byte
+	HoistedLocs []loc.Loc
+	Body        [][]byte
+	BodyLocs    []loc.Loc
 }
 
 func HoistExports(source []byte) HoistedScripts {
 	shouldHoist := bytes.Contains(source, []byte("export"))
 	if !shouldHoist {
+		body := make([][]byte, 0)
+		body = append(body, source)
+		bodyLocs := make([]loc.Loc, 0)
+		bodyLocs = append(bodyLocs, loc.Loc{Start: 0})
 		return HoistedScripts{
-			Body: source,
+			Body:     body,
+			BodyLocs: bodyLocs,
 		}
 	}
 
 	l := js.NewLexer(parse.NewInputBytes(source))
 	i := 0
 	end := 0
 
-	hoisted := make([][]byte, 1)
-	body := make([]byte, 0)
+	hoisted := make([][]byte, 0)
+	hoistedLocs := make([]loc.Loc, 0)
+	body := make([][]byte, 0)
+	bodyLocs := make([]loc.Loc, 0)
 	pairs := make(map[byte]int)
 
 	// Let's lex the script until we find what we need!
@@ -47,8 +56,15 @@ outer:
 
 		if token == js.ErrorToken {
 			if l.Err() != io.EOF {
+				body := make([][]byte, 0)
+				body = append(body, source)
+				bodyLocs := make([]loc.Loc, 0)
+				bodyLocs = append(bodyLocs, loc.Loc{Start: 0})
 				return HoistedScripts{
-					Body: source,
+					Hoisted:     hoisted,
+					HoistedLocs: hoistedLocs,
+					Body:        body,
+					BodyLocs:    bodyLocs,
 				}
 			}
 			break
@@ -127,17 +143,26 @@ outer:
 
 				if foundIdent && foundSemicolonOrLineTerminator && pairs['{'] == 0 && pairs['('] == 0 && pairs['['] == 0 {
 					hoisted = append(hoisted, source[start:i])
+					hoistedLocs = append(hoistedLocs, loc.Loc{Start: start})
 					if end < start {
-						body = append(body, source[end:start]...)
+						body = append(body, source[end:start])
+						bodyLocs = append(bodyLocs, loc.Loc{Start: end})
 					}
 					end = i
 					continue outer
 				}
 
 				if next == js.ErrorToken {
 					if l.Err() != io.EOF {
+						body := make([][]byte, 0)
+						body = append(body, source)
+						bodyLocs := make([]loc.Loc, 0)
+						bodyLocs = append(bodyLocs, loc.Loc{Start: 0})
 						return HoistedScripts{
-							Body: source,
+							Hoisted:     hoisted,
+							HoistedLocs: hoistedLocs,
+							Body:        body,
+							BodyLocs:    bodyLocs,
 						}
 					}
 					break outer
@@ -164,11 +189,14 @@ outer:
 		i += len(value)
 	}
 
-	body = append(body, source[end:]...)
+	body = append(body, source[end:])
+	bodyLocs = append(bodyLocs, loc.Loc{Start: end})
 
 	return HoistedScripts{
-		Hoisted: hoisted,
-		Body:    body,
+		Hoisted:     hoisted,
+		HoistedLocs: hoistedLocs,
+		Body:        body,
+		BodyLocs:    bodyLocs,
 	}
 }
 
@@ -178,18 +206,25 @@ func isKeyword(value []byte) bool {
 
 func HoistImports(source []byte) HoistedScripts {
 	imports := make([][]byte, 0)
-	body := make([]byte, 0)
+	importLocs := make([]loc.Loc, 0)
+	body := make([][]byte, 0)
+	bodyLocs := make([]loc.Loc, 0)
 	prev := 0
-	for i, statement := NextImportStatement(source, 0); i > -1; i, statement = NextImportStatement(source, i) {
-		body = append(body, source[prev:statement.Span.Start]...)
+	for i, statement := NextImportStatement(source, 0); i > -1 && i < len(source)+1; i, statement = NextImportStatement(source, i) {
+		bodyLocs = append(bodyLocs, loc.Loc{Start: prev})
+		body = append(body, source[prev:statement.Span.Start])
 		imports = append(imports, statement.Value)
+		importLocs = append(importLocs, loc.Loc{Start: statement.Span.Start})
 		prev = i
 	}
 	if prev == 0 {
-		return HoistedScripts{Body: source}
+		bodyLocs = append(bodyLocs, loc.Loc{Start: 0})
+		body = append(body, source)
+		return HoistedScripts{Body: body, BodyLocs: bodyLocs}
 	}
-	body = append(body, source[prev:]...)
-	return HoistedScripts{Hoisted: imports, Body: body}
+	bodyLocs = append(bodyLocs, loc.Loc{Start: prev})
+	body = append(body, source[prev:])
+	return HoistedScripts{Hoisted: imports, HoistedLocs: importLocs, Body: body, BodyLocs: bodyLocs}
 }
 
 type Props struct {
@@ -467,7 +502,7 @@ func NextImportStatement(source []byte, pos int) (int, ImportStatement) {
 	for {
 		token, value := l.Next()
 
-		if token == js.DivToken || token == js.DivEqToken {
+		if len(source) > i && token == js.DivToken || token == js.DivEqToken {
 			lns := bytes.Split(source[i+1:], []byte{'\n'})
 			if bytes.Contains(lns[0], []byte{'/'}) {
 				token, value = l.RegExp()
@@ -494,7 +529,7 @@ func NextImportStatement(source []byte, pos int) (int, ImportStatement) {
 			pairs := make(map[byte]int)
 			for {
 				next, nextValue := l.Next()
-				if next == js.DivToken || next == js.DivEqToken {
+				if len(source) > i && (next == js.DivToken || next == js.DivEqToken) {
 					lns := bytes.Split(source[i+1:], []byte{'\n'})
 					if bytes.Contains(lns[0], []byte{'/'}) {
 						next, nextValue = l.RegExp()
@@ -524,8 +559,10 @@ func NextImportStatement(source []byte, pos int) (int, ImportStatement) {
 				}
 
 				if !foundSpecifier && next == js.StringToken {
-					specifier = string(nextValue[1 : len(nextValue)-1])
-					foundSpecifier = true
+					if len(nextValue) > 1 {
+						specifier = string(nextValue[1 : len(nextValue)-1])
+						foundSpecifier = true
+					}
 					continue
 				}
 

internal/js_scanner/js_scanner_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"strings"
 	"testing"
+	"unicode/utf8"
 
 	"github.com/withastro/compiler/internal/test_utils"
 )
@@ -16,8 +17,8 @@ type testcase struct {
 	only   bool
 }
 
-func TestHoistImport(t *testing.T) {
-	tests := []testcase{
+func fixturesHoistImport() []testcase {
+	return []testcase{
 		{
 			name:   "basic",
 			source: `const value = "test"`,
@@ -40,18 +41,18 @@ const article2 = await import('../markdown/article2.md')
 		{
 			name: "big import",
 			source: `import {
-  a,
-  b,
-  c,
-  d,
+a,
+b,
+c,
+d,
 } from "package"
 
 const b = await fetch();`,
 			want: `import {
-  a,
-  b,
-  c,
-  d,
+a,
+b,
+c,
+d,
 } from "package"
 `,
 		},
@@ -73,18 +74,18 @@ const b = await fetch();`,
 			name: "import assertion 2",
 			source: `// comment
 import {
-  fn
+fn
 } from
-  "package" assert {
-    it: 'works'
-  };
+"package" assert {
+it: 'works'
+};
 const b = await fetch();`,
 			want: `import {
-  fn
+fn
 } from
-  "package" assert {
-    it: 'works'
-  };
+"package" assert {
+it: 'works'
+};
 `,
 		},
 		{
@@ -96,10 +97,10 @@ import Test from "../components/Test.astro";`,
 		{
 			name: "import.meta.env II",
 			source: `console.log(
-	import
-		.meta
-		.env
-		.FOO
+import
+	.meta
+	.env
+	.FOO
 );
 import Test from "../components/Test.astro";`,
 			want: `import Test from "../components/Test.astro";`,
@@ -115,7 +116,7 @@ const b = await fetch()`,
 			name: "getStaticPaths",
 			source: `import { fn } from "package";
 export async function getStaticPaths() {
-	const content = Astro.fetchContent('**/*.md');
+const content = Astro.fetchContent('**/*.md');
 }
 const b = await fetch()`,
 			want: `import { fn } from "package";`,
@@ -124,7 +125,7 @@ const b = await fetch()`,
 			name: "getStaticPaths with comments",
 			source: `import { fn } from "package";
 export async function getStaticPaths() {
-  const content = Astro.fetchContent('**/*.md');
+const content = Astro.fetchContent('**/*.md');
 }
 const b = await fetch()`,
 			want: `import { fn } from "package";`,
@@ -133,29 +134,29 @@ const b = await fetch()`,
 			name: "getStaticPaths with semicolon",
 			source: `import { fn } from "package";
 export async function getStaticPaths() {
-  const content = Astro.fetchContent('**/*.md');
+const content = Astro.fetchContent('**/*.md');
 }; const b = await fetch()`,
 			want: `import { fn } from "package";`,
 		},
 		{
 			name: "getStaticPaths with RegExp escape",
 			source: `export async function getStaticPaths() {
-  const pattern = /\.md$/g.test('value');
+const pattern = /\.md$/g.test('value');
 }
 import a from "a";`,
 			want: `import a from "a";`,
 		},
 		{
 			name: "getStaticPaths with divider",
 			source: `export async function getStaticPaths() {
-  const pattern = a / b;
+const pattern = a / b;
 }`,
 			want: ``,
 		},
 		{
 			name: "getStaticPaths with divider and following content",
 			source: `export async function getStaticPaths() {
-  const value = 1 / 2;
+const value = 1 / 2;
 }
 // comment
 import { b } from "b";
@@ -165,7 +166,7 @@ const { a } = Astro.props;`,
 		{
 			name: "getStaticPaths with regex and following content",
 			source: `export async function getStaticPaths() {
-  const value = /2/g;
+const value = /2/g;
 }
 // comment
 import { b } from "b";
@@ -203,6 +204,10 @@ import { c } from "c";
 `,
 		},
 	}
+}
+
+func TestHoistImport(t *testing.T) {
+	tests := fixturesHoistImport()
 	for _, tt := range tests {
 		if tt.only {
 			tests = make([]testcase, 0)
@@ -226,6 +231,24 @@ import { c } from "c";
 	}
 }
 
+func FuzzHoistImport(f *testing.F) {
+	tests := fixturesHoistImport()
+	for _, tt := range tests {
+		f.Add(tt.source) // Use f.Add to provide a seed corpus
+	}
+	f.Fuzz(func(t *testing.T, source string) {
+		result := HoistImports([]byte(source))
+		got := []byte{}
+		for _, imp := range result.Hoisted {
+			got = append(got, bytes.TrimSpace(imp)...)
+			got = append(got, '\n')
+		}
+		if utf8.ValidString(source) && !utf8.ValidString(string(got)) {
+			t.Errorf("Import hoisting produced an invalid string: %q", got)
+		}
+	})
+}
+
 func TestHoistExport(t *testing.T) {
 	tests := []testcase{
 		{

internal/js_scanner/testdata/fuzz/FuzzHoistImport/ec55358ab2929fbf4deab52587664e42682f0a6ea201a325c5c33f9d18c50456

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("import\"\nimport \"\";")