From d6c7eccc7cbc6ac7e08baaee68124bcf695e8025 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sun, 25 Sep 2022 16:11:58 +0200 Subject: [PATCH 1/8] Upgrade chroma to v2.3.0 --- go.mod | 6 +++--- go.sum | 16 +++++++++------- modules/highlight/highlight.go | 20 +++++++++++++++----- modules/markup/markdown/markdown.go | 4 ++-- modules/markup/orgmode/orgmode.go | 4 ++-- 5 files changed, 31 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 8976900d92be..97fa29a8fc2a 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/42wim/sshsig v0.0.0-20211121163825-841cf5bbc121 github.com/NYTimes/gziphandler v1.1.1 github.com/PuerkitoBio/goquery v1.8.0 - github.com/alecthomas/chroma v0.10.0 + github.com/alecthomas/chroma/v2 v2.3.0 github.com/blevesearch/bleve/v2 v2.3.2 github.com/buildkite/terminal-to-html/v3 v3.7.0 github.com/caddyserver/certmagic v0.17.0 @@ -89,8 +89,8 @@ require ( github.com/urfave/cli v1.22.9 github.com/xanzy/go-gitlab v0.73.1 github.com/yohcop/openid-go v1.0.0 - github.com/yuin/goldmark v1.4.13 - github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594 + github.com/yuin/goldmark v1.4.15 + github.com/yuin/goldmark-highlighting/v2 v2.0.0-20220924101305-151362477c87 github.com/yuin/goldmark-meta v1.1.0 go.jolheiser.com/hcaptcha v0.0.4 go.jolheiser.com/pwn v0.0.3 diff --git a/go.sum b/go.sum index 78b1df2c2a6c..aca2c4b991c0 100644 --- a/go.sum +++ b/go.sum @@ -158,9 +158,12 @@ github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= -github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= -github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= +github.com/alecthomas/chroma/v2 v2.2.0/go.mod h1:vf4zrexSH54oEjJ7EdB65tGNHmH3pGZmVkgTP5RHvAs= +github.com/alecthomas/chroma/v2 v2.3.0 h1:83xfxrnjv8eK+Cf8qZDzNo3PPF9IbTWHs7z28GY6D0U= +github.com/alecthomas/chroma/v2 v2.3.0/go.mod h1:mZxeWZlxP2Dy+/8cBob2PYd8O2DwNAzave5AY7A2eQw= github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= +github.com/alecthomas/repr v0.0.0-20220113201626-b1b626ac65ae/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= +github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -1467,11 +1470,10 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.5/go.mod h1:rmuwmfZ0+bvzB24eSC//bk1R1Zp3hM0OXYv/G2LIilg= -github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594 h1:yHfZyN55+5dp1wG7wDKv8HQ044moxkyGq12KFFMFDxg= -github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594/go.mod h1:U9ihbh+1ZN7fR5Se3daSPoz1CGF9IYtSvWwVQtnzGHU= +github.com/yuin/goldmark v1.4.15 h1:CFa84T0goNn/UIXYS+dmjjVxMyTAvpOmzld40N/nfK0= +github.com/yuin/goldmark v1.4.15/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark-highlighting/v2 v2.0.0-20220924101305-151362477c87 h1:Py16JEzkSdKAtEFJjiaYLYBOWGXc1r/xHj/Q/5lA37k= +github.com/yuin/goldmark-highlighting/v2 v2.0.0-20220924101305-151362477c87/go.mod h1:ovIvrum6DQJA4QsJSovrkC4saKHQVs7TvcaeO8AIl5I= github.com/yuin/goldmark-meta v1.1.0 h1:pWw+JLHGZe8Rk0EGsMVssiNb/AaPMHfSRszZeUeiOUc= github.com/yuin/goldmark-meta v1.1.0/go.mod h1:U4spWENafuA7Zyg+Lj5RqK/MF+ovMYtBvXi1lBb2VP0= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= diff --git a/modules/highlight/highlight.go b/modules/highlight/highlight.go index af3376e8d712..abfc909f3606 100644 --- a/modules/highlight/highlight.go +++ b/modules/highlight/highlight.go @@ -19,10 +19,10 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "github.com/alecthomas/chroma" - "github.com/alecthomas/chroma/formatters/html" - "github.com/alecthomas/chroma/lexers" - "github.com/alecthomas/chroma/styles" + "github.com/alecthomas/chroma/v2" + "github.com/alecthomas/chroma/v2/formatters/html" + "github.com/alecthomas/chroma/v2/lexers" + "github.com/alecthomas/chroma/v2/styles" lru "github.com/hashicorp/golang-lru" ) @@ -134,6 +134,16 @@ func CodeFromLexer(lexer chroma.Lexer, code string) string { return strings.TrimSuffix(htmlbuf.String(), "\n") } +// use a nop wrapper because PreventSurroundingPre(true) causes chroma to not emit +// line wrappers which are necessary to split the html string into lines +type nopPreWrapper struct{} + +// Start is called to write a start 
 element.
+func (nopPreWrapper) Start(code bool, styleAttr string) string { return "" }
+
+// End is called to write the end
element. +func (nopPreWrapper) End(code bool) string { return "" } + // File returns a slice of chroma syntax highlighted HTML lines of code func File(fileName, language string, code []byte) ([]string, error) { NewContext() @@ -144,7 +154,7 @@ func File(fileName, language string, code []byte) ([]string, error) { formatter := html.New(html.WithClasses(true), html.WithLineNumbers(false), - html.PreventSurroundingPre(true), + html.WithPreWrapper(nopPreWrapper{}), ) htmlBuf := bytes.Buffer{} diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index c0e72fd6ce51..fa289986ccec 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -18,9 +18,9 @@ import ( "code.gitea.io/gitea/modules/setting" giteautil "code.gitea.io/gitea/modules/util" - chromahtml "github.com/alecthomas/chroma/formatters/html" + chromahtml "github.com/alecthomas/chroma/v2/formatters/html" "github.com/yuin/goldmark" - highlighting "github.com/yuin/goldmark-highlighting" + highlighting "github.com/yuin/goldmark-highlighting/v2" meta "github.com/yuin/goldmark-meta" "github.com/yuin/goldmark/extension" "github.com/yuin/goldmark/parser" diff --git a/modules/markup/orgmode/orgmode.go b/modules/markup/orgmode/orgmode.go index a78531720dc8..1c02f274ba62 100644 --- a/modules/markup/orgmode/orgmode.go +++ b/modules/markup/orgmode/orgmode.go @@ -17,8 +17,8 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - "github.com/alecthomas/chroma" - "github.com/alecthomas/chroma/lexers" + "github.com/alecthomas/chroma/v2" + "github.com/alecthomas/chroma/v2/lexers" "github.com/niklasfasching/go-org/org" ) From befc28331974cdf7223dd1792076971930e355d6 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sun, 25 Sep 2022 17:37:54 +0200 Subject: [PATCH 2/8] use SplitTokensIntoLines to split into lines --- modules/highlight/highlight.go | 47 +++++++++++++--------------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/modules/highlight/highlight.go b/modules/highlight/highlight.go index abfc909f3606..33c0a84d700a 100644 --- a/modules/highlight/highlight.go +++ b/modules/highlight/highlight.go @@ -134,16 +134,6 @@ func CodeFromLexer(lexer chroma.Lexer, code string) string { return strings.TrimSuffix(htmlbuf.String(), "\n") } -// use a nop wrapper because PreventSurroundingPre(true) causes chroma to not emit -// line wrappers which are necessary to split the html string into lines -type nopPreWrapper struct{} - -// Start is called to write a start 
 element.
-func (nopPreWrapper) Start(code bool, styleAttr string) string { return "" }
-
-// End is called to write the end
element. -func (nopPreWrapper) End(code bool) string { return "" } - // File returns a slice of chroma syntax highlighted HTML lines of code func File(fileName, language string, code []byte) ([]string, error) { NewContext() @@ -154,12 +144,9 @@ func File(fileName, language string, code []byte) ([]string, error) { formatter := html.New(html.WithClasses(true), html.WithLineNumbers(false), - html.WithPreWrapper(nopPreWrapper{}), + html.PreventSurroundingPre(true), ) - htmlBuf := bytes.Buffer{} - htmlWriter := bufio.NewWriter(&htmlBuf) - var lexer chroma.Lexer // provided language overrides everything @@ -185,28 +172,30 @@ func File(fileName, language string, code []byte) ([]string, error) { } } + iterator, err := lexer.Tokenise(nil, string(code)) if err != nil { return nil, fmt.Errorf("can't tokenize code: %w", err) } - err = formatter.Format(htmlWriter, styles.GitHub, iterator) - if err != nil { - return nil, fmt.Errorf("can't format code: %w", err) - } - - _ = htmlWriter.Flush() + tokensArray := chroma.SplitTokensIntoLines(iterator.Tokens()) + htmlBuf := bytes.Buffer{} + htmlWriter := bufio.NewWriter(&htmlBuf) - // at the moment, Chroma generates stable output `...\n` for each line - htmlStr := htmlBuf.String() - lines := strings.Split(htmlStr, ``) - m := make([]string, 0, len(lines)) - for i := 1; i < len(lines); i++ { - line := lines[i] - line = strings.TrimSuffix(line, "") - m = append(m, line) + lines := []string{} + for _, tokens := range(tokensArray) { + iterator := chroma.Literator(tokens...) + err = formatter.Format(htmlWriter, styles.GitHub, iterator) + if err != nil { + return nil, fmt.Errorf("can't format code: %w", err) + } + _ = htmlWriter.Flush() + str := htmlBuf.String() + htmlBuf.Reset() + lines = append(lines, str) } - return m, nil + + return lines, nil } // PlainText returns non-highlighted HTML for code From 9da024544aa064afb72d248ad8f8c9a874372164 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sun, 25 Sep 2022 17:46:26 +0200 Subject: [PATCH 3/8] fmt --- modules/highlight/highlight.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/highlight/highlight.go b/modules/highlight/highlight.go index 33c0a84d700a..aa899d02c405 100644 --- a/modules/highlight/highlight.go +++ b/modules/highlight/highlight.go @@ -172,7 +172,6 @@ func File(fileName, language string, code []byte) ([]string, error) { } } - iterator, err := lexer.Tokenise(nil, string(code)) if err != nil { return nil, fmt.Errorf("can't tokenize code: %w", err) @@ -183,7 +182,7 @@ func File(fileName, language string, code []byte) ([]string, error) { htmlWriter := bufio.NewWriter(&htmlBuf) lines := []string{} - for _, tokens := range(tokensArray) { + for _, tokens := range tokensArray { iterator := chroma.Literator(tokens...) err = formatter.Format(htmlWriter, styles.GitHub, iterator) if err != nil { From 4aae6ea3c28fec4e44038afecf9d3201347263c1 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sun, 25 Sep 2022 17:48:26 +0200 Subject: [PATCH 4/8] remove useless variable --- modules/highlight/highlight.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/highlight/highlight.go b/modules/highlight/highlight.go index aa899d02c405..d29e83842644 100644 --- a/modules/highlight/highlight.go +++ b/modules/highlight/highlight.go @@ -189,9 +189,8 @@ func File(fileName, language string, code []byte) ([]string, error) { return nil, fmt.Errorf("can't format code: %w", err) } _ = htmlWriter.Flush() - str := htmlBuf.String() + lines = append(lines, htmlBuf.String()) htmlBuf.Reset() - lines = append(lines, str) } return lines, nil From 0cfc7caa630362a10ebe6f6290dcf2f2ddd59313 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sun, 25 Sep 2022 18:31:21 +0200 Subject: [PATCH 5/8] fix tests in gitdiff/highlightdiff_test.go --- services/gitdiff/highlightdiff_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/gitdiff/highlightdiff_test.go b/services/gitdiff/highlightdiff_test.go index 1cd78bc94272..205f9a0773a9 100644 --- a/services/gitdiff/highlightdiff_test.go +++ b/services/gitdiff/highlightdiff_test.go @@ -21,11 +21,11 @@ func TestDiffWithHighlight(t *testing.T) { " run(db)\n", ) - expected := ` run('<>')` + "\n" + expected := ` run('<>')` output := diffToHTML(nil, diffs, DiffLineDel) assert.Equal(t, expected, output) - expected = ` run(db)` + "\n" + expected = ` run(db)` output = diffToHTML(nil, diffs, DiffLineAdd) assert.Equal(t, expected, output) @@ -57,7 +57,7 @@ func TestDiffWithHighlightPlaceholder(t *testing.T) { assert.Equal(t, "", hcd.placeholderTokenMap[0x00100000]) assert.Equal(t, "", hcd.placeholderTokenMap[0x0010FFFD]) - expected := fmt.Sprintf(`a='%s'`, "\U00100000") + expected := fmt.Sprintf(`a='%s'`, "\U00100000") output := diffToHTML(hcd.lineWrapperTags, diffs, DiffLineDel) assert.Equal(t, expected, output) From 2df06c18ecbde09f82edd8ac33fa75ad7f846425 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sun, 25 Sep 2022 18:32:59 +0200 Subject: [PATCH 6/8] fix orgmode_test.go --- modules/markup/orgmode/orgmode_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/markup/orgmode/orgmode_test.go b/modules/markup/orgmode/orgmode_test.go index 4fc0a20db2b2..cd12cd8c171a 100644 --- a/modules/markup/orgmode/orgmode_test.go +++ b/modules/markup/orgmode/orgmode_test.go @@ -79,9 +79,9 @@ func HelloWorld() { } #+end_src `, `
-
// HelloWorld prints "Hello World"
-func HelloWorld() {
-	fmt.Println("Hello World")
-}
+
// HelloWorld prints "Hello World"
+func HelloWorld() {
+	fmt.Println("Hello World")
+}
`) } From 4f517164493b35192034be6aae1233373380b35b Mon Sep 17 00:00:00 2001 From: silverwind Date: Sun, 25 Sep 2022 20:23:34 +0200 Subject: [PATCH 7/8] restart ci From 5569198dfc7b1858d9de23376c2d7a1f0a6637a0 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 26 Sep 2022 10:42:27 +0800 Subject: [PATCH 8/8] remove htmlWriter, rename tokensArray to tokensLines to match the function chroma.SplitTokensIntoLines, pre-allocate memory for lines --- modules/highlight/highlight.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/modules/highlight/highlight.go b/modules/highlight/highlight.go index d29e83842644..65ed74b01995 100644 --- a/modules/highlight/highlight.go +++ b/modules/highlight/highlight.go @@ -177,18 +177,16 @@ func File(fileName, language string, code []byte) ([]string, error) { return nil, fmt.Errorf("can't tokenize code: %w", err) } - tokensArray := chroma.SplitTokensIntoLines(iterator.Tokens()) - htmlBuf := bytes.Buffer{} - htmlWriter := bufio.NewWriter(&htmlBuf) - - lines := []string{} - for _, tokens := range tokensArray { - iterator := chroma.Literator(tokens...) - err = formatter.Format(htmlWriter, styles.GitHub, iterator) + tokensLines := chroma.SplitTokensIntoLines(iterator.Tokens()) + htmlBuf := &bytes.Buffer{} + + lines := make([]string, 0, len(tokensLines)) + for _, tokens := range tokensLines { + iterator = chroma.Literator(tokens...) + err = formatter.Format(htmlBuf, styles.GitHub, iterator) if err != nil { return nil, fmt.Errorf("can't format code: %w", err) } - _ = htmlWriter.Flush() lines = append(lines, htmlBuf.String()) htmlBuf.Reset() }