Switch Unicode Escaping to a VSCode-like system

modules/charset/breakwriter.go

@@ -0,0 +1,44 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package charset
+
+import (
+	"bytes"
+	"io"
+)
+
+// BreakWriter wraps an io.Writer to always write '\n' as '<br>'
+type BreakWriter struct {
+	io.Writer
+}
+
+// Write writes the provided bs transparently replacing '\n' with '<br>'
+func (b *BreakWriter) Write(bs []byte) (n int, err error) {
+	pos := 0
+	for pos < len(bs) {
+		idx := bytes.IndexByte(bs[pos:], '\n')
+		if idx < 0 {
+			wn, err := b.Writer.Write(bs[pos:])
+			return n + wn, err
+		}
+
+		if idx > 0 {
+			wn, err := b.Writer.Write(bs[pos : pos+idx])
+			n += wn
+			if err != nil {
+				return n, err
+			}
+		}
+
+		if _, err = b.Writer.Write([]byte("<br>")); err != nil {
+			return n, err
+		}
+		pos += idx + 1
+
+		n++
+	}
+
+	return n, err
+}

modules/charset/breakwriter_test.go

@@ -0,0 +1,69 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package charset
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestBreakWriter_Write(t *testing.T) {
+	tests := []struct {
+		name    string
+		kase    string
+		expect  string
+		wantErr bool
+	}{
+		{
+			name:   "noline",
+			kase:   "abcdefghijklmnopqrstuvwxyz",
+			expect: "abcdefghijklmnopqrstuvwxyz",
+		},
+		{
+			name:   "endline",
+			kase:   "abcdefghijklmnopqrstuvwxyz\n",
+			expect: "abcdefghijklmnopqrstuvwxyz<br>",
+		},
+		{
+			name:   "startline",
+			kase:   "\nabcdefghijklmnopqrstuvwxyz",
+			expect: "<br>abcdefghijklmnopqrstuvwxyz",
+		},
+		{
+			name:   "onlyline",
+			kase:   "\n\n\n",
+			expect: "<br><br><br>",
+		},
+		{
+			name:   "empty",
+			kase:   "",
+			expect: "",
+		},
+		{
+			name:   "midline",
+			kase:   "\nabc\ndefghijkl\nmnopqrstuvwxy\nz",
+			expect: "<br>abc<br>defghijkl<br>mnopqrstuvwxy<br>z",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			buf := &strings.Builder{}
+			b := &BreakWriter{
+				Writer: buf,
+			}
+			n, err := b.Write([]byte(tt.kase))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("BreakWriter.Write() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if n != len(tt.kase) {
+				t.Errorf("BreakWriter.Write() = %v, want %v", n, len(tt.kase))
+			}
+			if buf.String() != tt.expect {
+				t.Errorf("BreakWriter.Write() wrote %q, want %v", buf.String(), tt.expect)
+			}
+		})
+	}
+}

modules/charset/escape.go

@@ -16,8 +16,11 @@ import (
 	"code.gitea.io/gitea/modules/translation"
 )
 
+// RuneNBSP is the codepoint for NBSP
+const RuneNBSP = 0xa0
+
 // EscapeControlHTML escapes the unicode control sequences in a provided html document
-func EscapeControlHTML(text string, locale translation.Locale, allowed ...rune) (escaped EscapeStatus, output string) {
+func EscapeControlHTML(text string, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, output string) {
 	sb := &strings.Builder{}
 	outputStream := &HTMLStreamerWriter{Writer: sb}
 	streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer)
@@ -30,7 +33,7 @@ func EscapeControlHTML(text string, locale translation.Locale, allowed ...rune)
 }
 
 // EscapeControlReaders escapes the unicode control sequences in a provider reader and writer in a locale and returns the findings as an EscapeStatus and the escaped []byte
-func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped EscapeStatus, err error) {
+func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) {
 	outputStream := &HTMLStreamerWriter{Writer: writer}
 	streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer)
 
@@ -42,7 +45,7 @@ func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.
 }
 
 // EscapeControlString escapes the unicode control sequences in a provided string and returns the findings as an EscapeStatus and the escaped string
-func EscapeControlString(text string, locale translation.Locale, allowed ...rune) (escaped EscapeStatus, output string) {
+func EscapeControlString(text string, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, output string) {
 	sb := &strings.Builder{}
 	outputStream := &HTMLStreamerWriter{Writer: sb}
 	streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer)

modules/charset/escape_status.go

@@ -14,7 +14,7 @@ type EscapeStatus struct {
 }
 
 // Or combines two EscapeStatus structs into one representing the conjunction of the two
-func (status EscapeStatus) Or(other EscapeStatus) EscapeStatus {
+func (status *EscapeStatus) Or(other *EscapeStatus) *EscapeStatus {
 	st := status
 	st.Escaped = st.Escaped || other.Escaped
 	st.HasError = st.HasError || other.HasError

modules/charset/escape_stream.go

@@ -22,6 +22,7 @@ var defaultWordRegexp = regexp.MustCompile(`(-?\d*\.\d\w*)|([^\` + "`" + `\~\!\@
 
 func NewEscapeStreamer(locale translation.Locale, next HTMLStreamer, allowed ...rune) HTMLStreamer {
 	return &escapeStreamer{
+		escaped:                 &EscapeStatus{},
 		PassthroughHTMLStreamer: *NewPassthroughStreamer(next),
 		locale:                  locale,
 		ambiguousTables:         AmbiguousTablesForLocale(locale),
@@ -31,13 +32,13 @@ func NewEscapeStreamer(locale translation.Locale, next HTMLStreamer, allowed ...
 
 type escapeStreamer struct {
 	PassthroughHTMLStreamer
-	escaped         EscapeStatus
+	escaped         *EscapeStatus
 	locale          translation.Locale
 	ambiguousTables []*AmbiguousTable
 	allowed         []rune
 }
 
-func (e *escapeStreamer) EscapeStatus() EscapeStatus {
+func (e *escapeStreamer) EscapeStatus() *EscapeStatus {
 	return e.escaped
 }
 
@@ -177,7 +178,7 @@ func (e *escapeStreamer) ambiguousRune(r, c rune) error {
 	}); err != nil {
 		return err
 	}
-	if err := e.PassthroughHTMLStreamer.Text(fmt.Sprintf("%c", r)); err != nil {
+	if err := e.PassthroughHTMLStreamer.Text(string(r)); err != nil {
 		return err
 	}
 	if err := e.PassthroughHTMLStreamer.EndTag("span"); err != nil {
@@ -206,7 +207,7 @@ func (e *escapeStreamer) invisibleRune(r rune) error {
 	}); err != nil {
 		return err
 	}
-	if err := e.PassthroughHTMLStreamer.Text(fmt.Sprintf("%c", r)); err != nil {
+	if err := e.PassthroughHTMLStreamer.Text(string(r)); err != nil {
 		return err
 	}
 	if err := e.PassthroughHTMLStreamer.EndTag("span"); err != nil {
@@ -286,11 +287,11 @@ func (e *escapeStreamer) isAllowed(r rune) bool {
 	if len(e.allowed) == 0 {
 		return false
 	}
-	if len(e.allowed) == 1 && e.allowed[0] == r {
-		return true
+	if len(e.allowed) == 1 {
+		return e.allowed[0] == r
 	}
 
 	return sort.Search(len(e.allowed), func(i int) bool {
-		return e.allowed[i] <= r
+		return e.allowed[i] >= r
 	}) >= 0
 }

routers/web/repo/blame.go

@@ -40,7 +40,7 @@ type blameRow struct {
 	CommitMessage  string
 	CommitSince    gotemplate.HTML
 	Code           gotemplate.HTML
-	EscapeStatus   charset.EscapeStatus
+	EscapeStatus   *charset.EscapeStatus
 }
 
 // RefBlame render blame page
@@ -235,7 +235,7 @@ func renderBlame(ctx *context.Context, blameParts []git.BlamePart, commitNames m
 	}
 	lines := make([]string, 0)
 	rows := make([]*blameRow, 0)
-	escapeStatus := charset.EscapeStatus{}
+	escapeStatus := &charset.EscapeStatus{}
 
 	i := 0
 	commitCnt := 0

routers/web/repo/view.go

@@ -328,33 +328,31 @@ func renderReadmeFile(ctx *context.Context, readmeFile *namedBlob, readmeTreelin
 	if markupType := markup.Type(readmeFile.name); markupType != "" {
 		ctx.Data["IsMarkup"] = true
 		ctx.Data["MarkupType"] = markupType
-		var result strings.Builder
-		err := markup.Render(&markup.RenderContext{
+
+		ctx.Data["EscapeStatus"], ctx.Data["FileContent"], err = markupRender(ctx, &markup.RenderContext{
 			Ctx:          ctx,
 			RelativePath: path.Join(ctx.Repo.TreePath, readmeFile.name), // ctx.Repo.TreePath is the directory not the Readme so we must append the Readme filename (and path).
 			URLPrefix:    readmeTreelink,
 			Metas:        ctx.Repo.Repository.ComposeDocumentMetas(),
 			GitRepo:      ctx.Repo.GitRepo,
-		}, rd, &result)
+		}, rd)
 		if err != nil {
-			log.Error("Render failed: %v then fallback", err)
+			log.Error("Render failed for %s in %-v: %v Falling back to rendering source", readmeFile.name, ctx.Repo.Repository, err)
 			buf := &bytes.Buffer{}
 			ctx.Data["EscapeStatus"], _ = charset.EscapeControlReader(rd, buf, ctx.Locale)
 			ctx.Data["FileContent"] = strings.ReplaceAll(
 				gotemplate.HTMLEscapeString(buf.String()), "\n", `<br>`,
 			)
-		} else {
-			ctx.Data["EscapeStatus"], ctx.Data["FileContent"] = charset.EscapeControlHTML(result.String(), ctx.Locale, 0xa0)
 		}
 	} else {
 		ctx.Data["IsRenderedHTML"] = true
 		buf := &bytes.Buffer{}
-		ctx.Data["EscapeStatus"], err = charset.EscapeControlReader(rd, buf, ctx.Locale)
+		ctx.Data["EscapeStatus"], err = charset.EscapeControlReader(rd, &charset.BreakWriter{Writer: buf}, ctx.Locale, charset.RuneNBSP)
 		if err != nil {
 			log.Error("Read failed: %v", err)
 		}
 
-		ctx.Data["FileContent"] = strings.ReplaceAll(buf.String(), "\n", `<br>`)
+		ctx.Data["FileContent"] = buf.String()
 	}
 }
 
@@ -496,27 +494,25 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
 		if markupType != "" && !shouldRenderSource {
 			ctx.Data["IsMarkup"] = true
 			ctx.Data["MarkupType"] = markupType
-			var result strings.Builder
 			if !detected {
 				markupType = ""
 			}
 			metas := ctx.Repo.Repository.ComposeDocumentMetas()
 			metas["BranchNameSubURL"] = ctx.Repo.BranchNameSubURL()
-			err := markup.Render(&markup.RenderContext{
+			ctx.Data["EscapeStatus"], ctx.Data["FileContent"], err = markupRender(ctx, &markup.RenderContext{
 				Ctx:          ctx,
 				Type:         markupType,
 				RelativePath: ctx.Repo.TreePath,
 				URLPrefix:    path.Dir(treeLink),
 				Metas:        metas,
 				GitRepo:      ctx.Repo.GitRepo,
-			}, rd, &result)
+			}, rd)
 			if err != nil {
 				ctx.ServerError("Render", err)
 				return
 			}
 			// to prevent iframe load third-party url
 			ctx.Resp.Header().Add("Content-Security-Policy", "frame-src 'self'")
-			ctx.Data["EscapeStatus"], ctx.Data["FileContent"] = charset.EscapeControlHTML(result.String(), ctx.Locale, 0xa0)
 		} else if readmeExist && !shouldRenderSource {
 			buf := &bytes.Buffer{}
 			ctx.Data["IsRenderedHTML"] = true
@@ -568,8 +564,8 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
 				log.Error("highlight.File failed, fallback to plain text: %v", err)
 				fileContent = highlight.PlainText(buf)
 			}
-			status := charset.EscapeStatus{}
-			statuses := make([]charset.EscapeStatus, len(fileContent))
+			status := &charset.EscapeStatus{}
+			statuses := make([]*charset.EscapeStatus, len(fileContent))
 			for i, line := range fileContent {
 				statuses[i], fileContent[i] = charset.EscapeControlHTML(line, ctx.Locale)
 				status = status.Or(statuses[i])
@@ -612,20 +608,17 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
 			rd := io.MultiReader(bytes.NewReader(buf), dataRc)
 			ctx.Data["IsMarkup"] = true
 			ctx.Data["MarkupType"] = markupType
-			var result strings.Builder
-			err := markup.Render(&markup.RenderContext{
+			ctx.Data["EscapeStatus"], ctx.Data["FileContent"], err = markupRender(ctx, &markup.RenderContext{
 				Ctx:          ctx,
 				RelativePath: ctx.Repo.TreePath,
 				URLPrefix:    path.Dir(treeLink),
 				Metas:        ctx.Repo.Repository.ComposeDocumentMetas(),
 				GitRepo:      ctx.Repo.GitRepo,
-			}, rd, &result)
+			}, rd)
 			if err != nil {
 				ctx.ServerError("Render", err)
 				return
 			}
-
-			ctx.Data["EscapeStatus"], ctx.Data["FileContent"] = charset.EscapeControlHTML(result.String(), ctx.Locale, 0xa0)
 		}
 	}
 
@@ -644,6 +637,23 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
 	}
 }
 
+func markupRender(ctx *context.Context, renderCtx *markup.RenderContext, input io.Reader) (escaped *charset.EscapeStatus, output string, err error) {
+	markupRd, markupWr := io.Pipe()
+	defer markupWr.Close()
+	done := make(chan struct{})
+	go func() {
+		sb := &strings.Builder{}
+		// We allow NBSP here this is rendered
+		escaped, _ = charset.EscapeControlReader(markupRd, sb, ctx.Locale, charset.RuneNBSP)
+		output = sb.String()
+		close(done)
+	}()
+	err = markup.Render(renderCtx, input, markupWr)
+	_ = markupWr.CloseWithError(err)
+	<-done
+	return
+}
+
 func safeURL(address string) string {
 	u, err := url.Parse(address)
 	if err != nil {