WIP: wiki - page toc and directory support

custom/conf/app.ini.sample

@@ -145,6 +145,17 @@ CUSTOM_URL_SCHEMES =
 ; List of file extensions that should be rendered/edited as Markdown
 ; Separate the extensions with a comma. To render files without any extension as markdown, just put a comma
 FILE_EXTENSIONS = .md,.markdown,.mdown,.mkd
+; Create 'Table of Pages' on internal wiki pages
+DEFAULT_TOC_WIKI_TREE = true
+; Create Table of Contents based on headlines on wiki page (does not alter file)
+DEFAULT_TOC_WIKI_FILE = true
+; Create 'Table of Contents' on all rendered markup files. This does not alter files. Enabled: overwrites DEFAULT_TOC_MARKUP_BY_FLAG
+DEFAULT_TOC_MARKUP_ALWAYS = false
+; Create 'Table of Contents' on rendered markup files if line '%%TOC%%' is present.
+DEFAULT_TOC_MARKUP_BY_FLAG = true
+; List of markup file extensions that TOC should be created on, set to not interfere with external markup renderer.
+; Separate the extensions with a comma. To ignore file extension check, just put a comma.
+TOC_MARKUP_FILE_EXTENSIONS = .md,.markdown,.mdown,.mkd,.org
 
 [server]
 ; The protocol the server listens on. One of 'http', 'https', 'unix' or 'fcgi'.

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -107,6 +107,13 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 ## Markdown (`markdown`)
 
 - `ENABLE_HARD_LINE_BREAK`: **false**: Enable Markdown's hard line break extension.
+- `DEFAULT_TOC_WIKI_TREE`: **true**: Create 'Table of Pages' on internal wiki pages.
+- `DEFAULT_TOC_WIKI_FILE`: **true**: Create Table of Contents based on headlines on wiki page (does
+   not alter file)
+- `DEFAULT_TOC_MARKUP_ALWAYS`: **false** : Create 'Table of Contents' on all rendered markup files. This does not alter files. Enabled: overwrites `DEFAULT_TOC_MARKUP_BY_FLAG`
+- `DEFAULT_TOC_MARKUP_BY_FLAG`: **true** : Create 'Table of Contents' on rendered markup files if line `%%TOC%%` is present.
+- `TOC_MARKUP_FILE_EXTENSIONS`: **.md,.markdown,.mdown,.mkd,.org** : List of markup file extensions that TOC should be created on, set to not interfere with external markup renderer.
+   Separate the extensions with a comma. To ignore file extension check, just put a comma.
 
 ## Server (`server`)
 

models/migrations/migrations.go

@@ -232,6 +232,10 @@ var migrations = []Migration{
 	NewMigration("add avatar field to repository", addAvatarFieldToRepository),
 	// v88 -> v89
 	NewMigration("add commit status context field to commit_status", addCommitStatusContext),
+	// v89 -> v90
+	NewMigration("add toc on wiki and markedown", addCanTocOnWikiAndMarkdown),
+	// v90 -> v91
+	NewMigration("add pagetoc to wiki", addCanWikiPageToc),
 }
 
 // Migrate database to current version

models/migrations/v89.go

@@ -0,0 +1,40 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/go-xorm/xorm"
+)
+
+func addCanTocOnWikiAndMarkdown(x *xorm.Engine) error {
+
+	type Repository struct {
+		TocWikiFile     bool `xorm:"NOT NULL DEFAULT true"`
+		TocMarkupAlways bool `xorm:"NOT NULL DEFAULT false"`
+		TocMarkupByFlag bool `xorm:"NOT NULL DEFAULT true"`
+	}
+
+	if err := x.Sync2(new(Repository)); err != nil {
+		return err
+	}
+
+	if _, err := x.Exec("UPDATE repository SET toc_wiki_file = ?",
+		setting.Markdown.DefaultTocWikiFile); err != nil {
+		return err
+	}
+
+	if _, err := x.Exec("UPDATE repository SET toc_markup_always = ?",
+		setting.Markdown.DefaultTocMarkupAlways); err != nil {
+		return err
+	}
+
+	if _, err := x.Exec("UPDATE repository SET toc_markup_by_flag = ?",
+		setting.Markdown.DefaultTocMarkupByFlag); err != nil {
+		return err
+	}
+	return nil
+}

models/migrations/v90.go

@@ -0,0 +1,28 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/go-xorm/xorm"
+)
+
+func addCanWikiPageToc(x *xorm.Engine) error {
+
+	type Repository struct {
+		TocWikiTree bool `xorm:"NOT NULL DEFAULT true"`
+	}
+
+	if err := x.Sync2(new(Repository)); err != nil {
+		return err
+	}
+
+	if _, err := x.Exec("UPDATE repository SET toc_wiki_tree = ?",
+		setting.Markdown.DefaultTocWikiTree); err != nil {
+		return err
+	}
+	return nil
+}

models/repo.go

@@ -169,6 +169,10 @@ type Repository struct {
 	IndexerStatus                   *RepoIndexerStatus `xorm:"-"`
 	IsFsckEnabled                   bool               `xorm:"NOT NULL DEFAULT true"`
 	CloseIssuesViaCommitInAnyBranch bool               `xorm:"NOT NULL DEFAULT false"`
+	TocWikiTree                     bool               `xorm:"NOT NULL DEFAULT true"`
+	TocWikiFile                     bool               `xorm:"NOT NULL DEFAULT true"`
+	TocMarkupAlways                 bool               `xorm:"NOT NULL DEFAULT false"`
+	TocMarkupByFlag                 bool               `xorm:"NOT NULL DEFAULT true"`
 	Topics                          []string           `xorm:"TEXT JSON"`
 
 	// Avatar: ID(10-20)-md5(32) - must fit into 64 symbols
@@ -1361,6 +1365,10 @@ func CreateRepository(doer, u *User, opts CreateRepoOptions) (_ *Repository, err
 		IsPrivate:                       opts.IsPrivate,
 		IsFsckEnabled:                   !opts.IsMirror,
 		CloseIssuesViaCommitInAnyBranch: setting.Repository.DefaultCloseIssuesViaCommitsInAnyBranch,
+		TocWikiTree:                     setting.Markdown.DefaultTocWikiTree,
+		TocWikiFile:                     setting.Markdown.DefaultTocWikiFile,
+		TocMarkupAlways:                 setting.Markdown.DefaultTocMarkupAlways,
+		TocMarkupByFlag:                 setting.Markdown.DefaultTocMarkupByFlag,
 	}
 
 	sess := x.NewSession()

models/wiki.go

@@ -9,17 +9,19 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/sync"
+	"code.gitea.io/gitea/modules/util"
 
 	"github.com/Unknwon/com"
 )
 
 var (
-	reservedWikiNames = []string{"_pages", "_new", "_edit", "raw"}
+	reservedWikiNames = []string{"_pages", "_new", "_edit", "_delete", "raw"}
 	wikiWorkingPool   = sync.NewExclusivePool()
 )
 
@@ -28,9 +30,17 @@ func NormalizeWikiName(name string) string {
 	return strings.Replace(name, "-", " ", -1)
 }
 
-// WikiNameToSubURL converts a wiki name to its corresponding sub-URL.
+// WikiNameToSubURL converts a wiki name to its corresponding sub-URL. This will escape dangerous letters.
 func WikiNameToSubURL(name string) string {
-	return url.QueryEscape(strings.Replace(name, " ", "-", -1))
+	// remove path up
+	re1 := regexp.MustCompile(`(\.\.\/)`)
+	name = re1.ReplaceAllString(name, "")
+	// trim whitespace and /
+	name = strings.Trim(name, "\n\r\t /")
+	name = url.QueryEscape(name)
+	//restore spaces
+	re3 := regexp.MustCompile(`(?m)(%20|\+)`)
+	return re3.ReplaceAllString(name, "%20")
 }
 
 // WikiNameToFilename converts a wiki name to its corresponding filename.
@@ -39,17 +49,53 @@ func WikiNameToFilename(name string) string {
 	return url.QueryEscape(name) + ".md"
 }
 
+// WikiNameToPathFilename converts a wiki name to its corresponding filename, keep directory paths.
+func WikiNameToPathFilename(name string) string {
+	var restore = [1][2]string{
+		{`(\.\.\/)`, ""}, // remove path up
+	}
+	for _, kv := range restore {
+		loopRe := regexp.MustCompile(kv[0])
+		name = loopRe.ReplaceAllString(name, kv[1])
+	}
+	name = strings.Trim(name, "\n\r\t ./") // trim whitespace and / .
+	return name + ".md"
+}
+
+// FilenameToPathFilename converts a wiki filename to filename with filepath.
+func FilenameToPathFilename(name string) string {
+	// restore spaces and slashes
+	var restore = [4][2]string{
+		{`(?m)%2F`, "/"},      //recover slashes /
+		{`(?m)(%20|\+)`, " "}, //restore spaces
+		{`(?m)(%25)`, "%"},    //restore %
+		{`(?m)(%26)`, "&"},    //restore &
+	}
+	for _, kv := range restore {
+		loopRe := regexp.MustCompile(kv[0])
+		name = loopRe.ReplaceAllString(name, kv[1])
+	}
+	return name
+}
+
+// WikiNameToRawPrefix Get raw file path inside wiki, removes last path element and returns
+func WikiNameToRawPrefix(repositoryName string, wikiPage string) string {
+	a := strings.Split(wikiPage, "/")
+	a = a[:len(a)-1]
+	return util.URLJoin(repositoryName, "wiki", "raw", strings.Join(a, "/"))
+}
+
 // WikiFilenameToName converts a wiki filename to its corresponding page name.
-func WikiFilenameToName(filename string) (string, error) {
+func WikiFilenameToName(filename string) (string, string, error) {
 	if !strings.HasSuffix(filename, ".md") {
-		return "", ErrWikiInvalidFileName{filename}
+		return "", "", ErrWikiInvalidFileName{filename}
 	}
 	basename := filename[:len(filename)-3]
 	unescaped, err := url.QueryUnescape(basename)
 	if err != nil {
-		return "", err
+		return basename, basename, err
 	}
-	return NormalizeWikiName(unescaped), nil
+	return unescaped, basename, nil
 }
 
 // WikiCloneLink returns clone URLs of repository wiki.
@@ -149,6 +195,8 @@ func (repo *Repository) updateWikiPage(doer *User, oldWikiName, newWikiName, con
 	}
 
 	newWikiPath := WikiNameToFilename(newWikiName)
+	newWikiDirPath := WikiNameToPathFilename(newWikiName)
+
 	if isNew {
 		filesInIndex, err := gitRepo.LsFiles(newWikiPath)
 		if err != nil {
@@ -160,6 +208,26 @@ func (repo *Repository) updateWikiPage(doer *User, oldWikiName, newWikiName, con
 				return ErrWikiAlreadyExist{newWikiPath}
 			}
 		}
+		filesInIndex, err = gitRepo.LsFiles(newWikiDirPath)
+		if err != nil {
+			log.Error("%v", err)
+			return err
+		}
+		for _, file := range filesInIndex {
+			if file == newWikiDirPath {
+				return ErrWikiAlreadyExist{newWikiDirPath}
+			}
+		}
+		filesInIndex, err = gitRepo.LsFiles(FilenameToPathFilename(newWikiDirPath))
+		if err != nil {
+			log.Error("%v", err)
+			return err
+		}
+		for _, file := range filesInIndex {
+			if file == newWikiDirPath {
+				return ErrWikiAlreadyExist{newWikiDirPath}
+			}
+		}
 	} else {
 		oldWikiPath := WikiNameToFilename(oldWikiName)
 		filesInIndex, err := gitRepo.LsFiles(oldWikiPath)
@@ -181,8 +249,30 @@ func (repo *Repository) updateWikiPage(doer *User, oldWikiName, newWikiName, con
 				return err
 			}
 		}
+		oldWikiDirPath := WikiNameToPathFilename(oldWikiName)
+		filesInIndex, err = gitRepo.LsFiles(oldWikiDirPath)
+		if err != nil {
+			log.Error("%v", err)
+			return err
+		}
+		found = false
+		for _, file := range filesInIndex {
+			if file == oldWikiDirPath {
+				found = true
+				break
+			}
+		}
+		if found {
+			err := gitRepo.RemoveFilesFromIndex(oldWikiDirPath)
+			if err != nil {
+				log.Error("%v", err)
+				return err
+			}
+		}
 	}
 
+	newWikiDirPath = FilenameToPathFilename(newWikiDirPath)
+
 	// FIXME: The wiki doesn't have lfs support at present - if this changes need to check attributes here
 
 	objectHash, err := gitRepo.HashObject(strings.NewReader(content))
@@ -191,7 +281,7 @@ func (repo *Repository) updateWikiPage(doer *User, oldWikiName, newWikiName, con
 		return err
 	}
 
-	if err := gitRepo.AddObjectToIndex("100644", objectHash, newWikiPath); err != nil {
+	if err := gitRepo.AddObjectToIndex("100644", objectHash, newWikiDirPath); err != nil {
 		log.Error("%v", err)
 		return err
 	}
@@ -291,7 +381,23 @@ func (repo *Repository) DeleteWikiPage(doer *User, wikiName string) (err error)
 			return err
 		}
 	} else {
-		return os.ErrNotExist
+		wikiDirPath := WikiNameToPathFilename(wikiName)
+		filesInIndex, err = gitRepo.LsFiles(wikiDirPath)
+		found = false
+		for _, file := range filesInIndex {
+			if file == wikiDirPath {
+				found = true
+				break
+			}
+		}
+		if found {
+			err := gitRepo.RemoveFilesFromIndex(wikiDirPath)
+			if err != nil {
+				return err
+			}
+		} else {
+			return os.ErrNotExist
+		}
 	}
 
 	// FIXME: The wiki doesn't have lfs support at present - if this changes need to check attributes here