From e9618db6352e0c4a285e8aeb6c5ad3e40a7bef79 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 29 Oct 2021 16:45:50 +0800 Subject: [PATCH 1/8] Only allow webhook to send requests to allowed hosts --- cmd/web.go | 1 + custom/conf/app.example.ini | 3 + .../doc/advanced/config-cheat-sheet.en-us.md | 1 + modules/migrations/migrate.go | 12 +-- modules/setting/webhook.go | 38 +++++++-- modules/util/util.go | 11 +++ services/webhook/deliver.go | 74 ++++++++++++++++- services/webhook/deliver_test.go | 82 +++++++++++++++++++ 8 files changed, 199 insertions(+), 23 deletions(-) diff --git a/cmd/web.go b/cmd/web.go index 963c816207475..6d8808824d0a9 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -194,6 +194,7 @@ func listen(m http.Handler, handleRedirector bool) error { listenAddr = net.JoinHostPort(listenAddr, setting.HTTPPort) } log.Info("Listen: %v://%s%s", setting.Protocol, listenAddr, setting.AppSubURL) + log.Info("AppURL: %s", setting.AppURL) if setting.LFS.StartServer { log.Info("LFS server enabled") diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 1753ed2330706..b601499480cb8 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1396,6 +1396,9 @@ PATH = ;; Deliver timeout in seconds ;DELIVER_TIMEOUT = 5 ;; +;; Webhook can only call allowed hosts for security reasons. Comma separated list: loopback, private, global, or all, or CIDR list (1.2.3.0/8), or wildcard hosts (*.mydomain.com) +; ALLOWED_HOST_LIST = global +;; ;; Allow insecure certification ;SKIP_TLS_VERIFY = false ;; diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 91c62dbec34ae..0a47635482422 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -581,6 +581,7 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type - `QUEUE_LENGTH`: **1000**: Hook task queue length. Use caution when editing this value. - `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks. +- `ALLOWED_HOST_LIST`: **global**: Webhook can only call allowed hosts for security reasons. Comma separated list: `loopback`, `private`, `global`, or `all`, or CIDR list (1.2.3.0/8), or wildcard hosts (*.mydomain.com) - `SKIP_TLS_VERIFY`: **false**: Allow insecure certification. - `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page. - `PROXY_URL`: **\**: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy. If not given, will use global proxy setting. diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index c5d78fba735af..dbe69259f4ecd 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -89,7 +89,7 @@ func IsMigrateURLAllowed(remoteURL string, doer *models.User) error { return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true} } for _, addr := range addrList { - if isIPPrivate(addr) || !addr.IsGlobalUnicast() { + if util.IsIPPrivate(addr) || !addr.IsGlobalUnicast() { return &models.ErrInvalidCloneAddr{Host: u.Host, PrivateNet: addr.String(), IsPermissionDenied: true} } } @@ -474,13 +474,3 @@ func Init() error { return nil } - -// TODO: replace with `ip.IsPrivate()` if min go version is bumped to 1.17 -func isIPPrivate(ip net.IP) bool { - if ip4 := ip.To4(); ip4 != nil { - return ip4[0] == 10 || - (ip4[0] == 172 && ip4[1]&0xf0 == 16) || - (ip4[0] == 192 && ip4[1] == 168) - } - return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc -} diff --git a/modules/setting/webhook.go b/modules/setting/webhook.go index 8ef54f5cbe2ca..dd3f90c8b2012 100644 --- a/modules/setting/webhook.go +++ b/modules/setting/webhook.go @@ -5,7 +5,9 @@ package setting import ( + "net" "net/url" + "strings" "code.gitea.io/gitea/modules/log" ) @@ -13,14 +15,16 @@ import ( var ( // Webhook settings Webhook = struct { - QueueLength int - DeliverTimeout int - SkipTLSVerify bool - Types []string - PagingNum int - ProxyURL string - ProxyURLFixed *url.URL - ProxyHosts []string + QueueLength int + DeliverTimeout int + SkipTLSVerify bool + AllowedHostList []string // loopback,private,global, or all, or CIDR list, or wildcard hosts + AllowedHostIPNets []*net.IPNet + Types []string + PagingNum int + ProxyURL string + ProxyURLFixed *url.URL + ProxyHosts []string }{ QueueLength: 1000, DeliverTimeout: 5, @@ -31,11 +35,29 @@ var ( } ) +// ParseWebhookAllowedHostList parses the ALLOWED_HOST_LIST value +func ParseWebhookAllowedHostList(allowedHostListStr string) (allowedHostList []string, allowedHostIPNets []*net.IPNet) { + for _, s := range strings.Split(allowedHostListStr, ",") { + s = strings.TrimSpace(s) + if s == "" { + continue + } + _, ipNet, err := net.ParseCIDR(s) + if err == nil { + allowedHostIPNets = append(allowedHostIPNets, ipNet) + } else { + allowedHostList = append(allowedHostList, s) + } + } + return +} + func newWebhookService() { sec := Cfg.Section("webhook") Webhook.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000) Webhook.DeliverTimeout = sec.Key("DELIVER_TIMEOUT").MustInt(5) Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool() + Webhook.AllowedHostList, Webhook.AllowedHostIPNets = ParseWebhookAllowedHostList(sec.Key("ALLOWED_HOST_LIST").MustString("global")) Webhook.Types = []string{"gitea", "gogs", "slack", "discord", "dingtalk", "telegram", "msteams", "feishu", "matrix", "wechatwork"} Webhook.PagingNum = sec.Key("PAGING_NUM").MustInt(10) Webhook.ProxyURL = sec.Key("PROXY_URL").MustString("") diff --git a/modules/util/util.go b/modules/util/util.go index cbc6eb4f8a01c..4f48b14f82363 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -9,6 +9,7 @@ import ( "crypto/rand" "errors" "math/big" + "net" "strconv" "strings" ) @@ -161,3 +162,13 @@ func RandomString(length int64) (string, error) { } return string(bytes), nil } + +// IsIPPrivate for net.IP.IsPrivate. TODO: replace with `ip.IsPrivate()` if min go version is bumped to 1.17 +func IsIPPrivate(ip net.IP) bool { + if ip4 := ip.To4(); ip4 != nil { + return ip4[0] == 10 || + (ip4[0] == 172 && ip4[1]&0xf0 == 16) || + (ip4[0] == 192 && ip4[1] == 168) + } + return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc +} diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index 28c3b23b2f896..b18ede21013f2 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -16,9 +16,11 @@ import ( "net" "net/http" "net/url" + "path/filepath" "strconv" "strings" "sync" + "syscall" "time" "code.gitea.io/gitea/models" @@ -26,9 +28,13 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/proxy" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + "github.com/gobwas/glob" ) +var contextKeyWebhookRequest interface{} = "contextKeyWebhookRequest" + // Deliver deliver hook task func Deliver(t *models.HookTask) error { w, err := models.GetWebhookByID(t.HookID) @@ -171,7 +177,7 @@ func Deliver(t *models.HookTask) error { return fmt.Errorf("Webhook task skipped (webhooks disabled): [%d]", t.ID) } - resp, err := webhookHTTPClient.Do(req) + resp, err := webhookHTTPClient.Do(req.WithContext(context.WithValue(req.Context(), contextKeyWebhookRequest, req))) if err != nil { t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err) return err @@ -288,19 +294,79 @@ func webhookProxy() func(req *http.Request) (*url.URL, error) { } } +func isWebhookRequestAllowed(allowedHostList []string, allowedHostIPNets []*net.IPNet, host string, ip net.IP) bool { + var allowed bool + ipStr := ip.String() +loop: + for _, allowedHost := range allowedHostList { + switch allowedHost { + case "": + continue + case "all": + allowed = true + break loop + case "global": + if allowed = ip.IsGlobalUnicast() && !util.IsIPPrivate(ip); allowed { + break loop + } + case "private": + if allowed = util.IsIPPrivate(ip); allowed { + break loop + } + case "loopback": + if allowed = ip.IsLoopback(); allowed { + break loop + } + default: + if ok, _ := filepath.Match(allowedHost, host); ok { + allowed = true + break loop + } + if ok, _ := filepath.Match(allowedHost, ipStr); ok { + allowed = true + break loop + } + } + } + if !allowed { + for _, allowIPNet := range allowedHostIPNets { + if allowIPNet.Contains(ip) { + allowed = true + break + } + } + } + return allowed +} + // InitDeliverHooks starts the hooks delivery thread func InitDeliverHooks() { timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second webhookHTTPClient = &http.Client{ + Timeout: timeout, Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify}, Proxy: webhookProxy(), - Dial: func(netw, addr string) (net.Conn, error) { - return net.DialTimeout(netw, addr, timeout) // dial timeout + DialContext: func(ctx context.Context, network, addrOrHost string) (net.Conn, error) { + dialer := net.Dialer{ + Timeout: timeout, + Control: func(network, ipAddr string, c syscall.RawConn) error { + // in Control func, the addr was already resolved to IP:PORT format, there is no cost to do ResolveTCPAddr here + tcpAddr, err := net.ResolveTCPAddr(network, ipAddr) + req := ctx.Value(contextKeyWebhookRequest).(*http.Request) + if err != nil { + return fmt.Errorf("webhook can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%v", req.Host, network, ipAddr, err) + } + if !isWebhookRequestAllowed(setting.Webhook.AllowedHostList, setting.Webhook.AllowedHostIPNets, req.Host, tcpAddr.IP) { + return fmt.Errorf("webhook can only call allowed HTTP servers (check your webhook.ALLOWED_HOST_LIST setting), deny '%s(%s)'", req.Host, ipAddr) + } + return nil + }, + } + return dialer.DialContext(ctx, network, addrOrHost) }, }, - Timeout: timeout, // request timeout } go graceful.GetManager().RunWithShutdownContext(DeliverHooks) diff --git a/services/webhook/deliver_test.go b/services/webhook/deliver_test.go index cfc99d796a536..52efc04ee511a 100644 --- a/services/webhook/deliver_test.go +++ b/services/webhook/deliver_test.go @@ -5,6 +5,7 @@ package webhook import ( + "net" "net/http" "net/url" "testing" @@ -37,3 +38,84 @@ func TestWebhookProxy(t *testing.T) { } } } + +func TestIsWebhookRequestAllowed(t *testing.T) { + type tc struct { + host string + ip net.IP + expected bool + } + + ah, an := setting.ParseWebhookAllowedHostList("private, global, *.google.com, 169.254.1.0/24") + cases := []tc{ + {"", net.IPv4zero, false}, + + {"", net.ParseIP("127.0.0.1"), false}, + + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + + {"", net.ParseIP("8.8.8.8"), true}, + + {"google.com", net.IPv4zero, false}, + {"sub.google.com", net.IPv4zero, true}, + + {"", net.ParseIP("169.254.1.1"), true}, + {"", net.ParseIP("169.254.2.2"), false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, isWebhookRequestAllowed(ah, an, c.host, c.ip), "case %s(%v)", c.host, c.ip) + } + + ah, an = setting.ParseWebhookAllowedHostList("loopback") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), true}, + {"", net.ParseIP("10.0.1.1"), false}, + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("8.8.8.8"), false}, + {"google.com", net.IPv4zero, false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, isWebhookRequestAllowed(ah, an, c.host, c.ip), "case %s(%v)", c.host, c.ip) + } + + ah, an = setting.ParseWebhookAllowedHostList("private") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), false}, + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"", net.ParseIP("8.8.8.8"), false}, + {"google.com", net.IPv4zero, false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, isWebhookRequestAllowed(ah, an, c.host, c.ip), "case %s(%v)", c.host, c.ip) + } + + ah, an = setting.ParseWebhookAllowedHostList("global") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), false}, + {"", net.ParseIP("10.0.1.1"), false}, + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("8.8.8.8"), true}, + {"google.com", net.IPv4zero, false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, isWebhookRequestAllowed(ah, an, c.host, c.ip), "case %s(%v)", c.host, c.ip) + } + + ah, an = setting.ParseWebhookAllowedHostList("all") + cases = []tc{ + {"", net.IPv4zero, true}, + {"", net.ParseIP("127.0.0.1"), true}, + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"", net.ParseIP("8.8.8.8"), true}, + {"google.com", net.IPv4zero, true}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, isWebhookRequestAllowed(ah, an, c.host, c.ip), "case %s(%v)", c.host, c.ip) + } +} From d007a940d338700fd7b6edaa0854dfead9f99872 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sat, 30 Oct 2021 17:15:42 +0800 Subject: [PATCH 2/8] Improve documents, unit tests and comments. --- cmd/web.go | 5 +- custom/conf/app.example.ini | 7 +- .../doc/advanced/config-cheat-sheet.en-us.md | 9 +- modules/setting/webhook.go | 23 +--- modules/util/net.go | 93 ++++++++++++++ modules/util/net_test.go | 119 ++++++++++++++++++ modules/util/util.go | 11 -- services/webhook/deliver.go | 48 +------ services/webhook/deliver_test.go | 82 ------------ 9 files changed, 233 insertions(+), 164 deletions(-) create mode 100644 modules/util/net.go create mode 100644 modules/util/net_test.go diff --git a/cmd/web.go b/cmd/web.go index 6d8808824d0a9..8d9387e06f7ab 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -194,7 +194,10 @@ func listen(m http.Handler, handleRedirector bool) error { listenAddr = net.JoinHostPort(listenAddr, setting.HTTPPort) } log.Info("Listen: %v://%s%s", setting.Protocol, listenAddr, setting.AppSubURL) - log.Info("AppURL: %s", setting.AppURL) + // This can be useful for users, many users do wrong to their config and get strange behaviors behind a reverse-proxy. + // A user may fix the configuration mistake when he sees this log. + // And this is also very helpful to maintainers to provide help to users to resolve their configuration problems. + log.Info("AppURL(ROOT_URL): %s", setting.AppURL) if setting.LFS.StartServer { log.Info("LFS server enabled") diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index b601499480cb8..b3afead2cbcc9 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1396,8 +1396,11 @@ PATH = ;; Deliver timeout in seconds ;DELIVER_TIMEOUT = 5 ;; -;; Webhook can only call allowed hosts for security reasons. Comma separated list: loopback, private, global, or all, or CIDR list (1.2.3.0/8), or wildcard hosts (*.mydomain.com) -; ALLOWED_HOST_LIST = global +;; Webhook can only call allowed hosts for security reasons. Comma separated list, eg: external, 192.168.1.0/24, *.mydomain.com +;; Built-in: loopback (for localhost), private (for LAN/intranet), external (for public hosts on internet), all (for all hosts) +;; CIDR list: 1.2.3.0/8, 2001:db8::/32 +;; Wildcard hosts: *.mydomain.com, 192.168.100.* +; ALLOWED_HOST_LIST = external ;; ;; Allow insecure certification ;SKIP_TLS_VERIFY = false diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 0a47635482422..ca35074d23f94 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -581,7 +581,14 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type - `QUEUE_LENGTH`: **1000**: Hook task queue length. Use caution when editing this value. - `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks. -- `ALLOWED_HOST_LIST`: **global**: Webhook can only call allowed hosts for security reasons. Comma separated list: `loopback`, `private`, `global`, or `all`, or CIDR list (1.2.3.0/8), or wildcard hosts (*.mydomain.com) +- `ALLOWED_HOST_LIST`: **external**: Webhook can only call allowed hosts for security reasons. Comma separated list. + - Built-in networks: + - `loopback`: 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. + - `private`: RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet. + - `external`: A valid non-private unicast IP, you can access all hosts on public internet. + - `all`: All hosts are allowed. + - CIDR list: `1.2.3.0/8` for IPv4 and `2001:db8::/32` for IPv6 + - Wildcard hosts: `*.mydomain.com`, `192.168.100.*` - `SKIP_TLS_VERIFY`: **false**: Allow insecure certification. - `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page. - `PROXY_URL`: **\**: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy. If not given, will use global proxy setting. diff --git a/modules/setting/webhook.go b/modules/setting/webhook.go index dd3f90c8b2012..1daa258232f34 100644 --- a/modules/setting/webhook.go +++ b/modules/setting/webhook.go @@ -7,9 +7,9 @@ package setting import ( "net" "net/url" - "strings" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" ) var ( @@ -18,7 +18,7 @@ var ( QueueLength int DeliverTimeout int SkipTLSVerify bool - AllowedHostList []string // loopback,private,global, or all, or CIDR list, or wildcard hosts + AllowedHostList []string AllowedHostIPNets []*net.IPNet Types []string PagingNum int @@ -35,29 +35,12 @@ var ( } ) -// ParseWebhookAllowedHostList parses the ALLOWED_HOST_LIST value -func ParseWebhookAllowedHostList(allowedHostListStr string) (allowedHostList []string, allowedHostIPNets []*net.IPNet) { - for _, s := range strings.Split(allowedHostListStr, ",") { - s = strings.TrimSpace(s) - if s == "" { - continue - } - _, ipNet, err := net.ParseCIDR(s) - if err == nil { - allowedHostIPNets = append(allowedHostIPNets, ipNet) - } else { - allowedHostList = append(allowedHostList, s) - } - } - return -} - func newWebhookService() { sec := Cfg.Section("webhook") Webhook.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000) Webhook.DeliverTimeout = sec.Key("DELIVER_TIMEOUT").MustInt(5) Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool() - Webhook.AllowedHostList, Webhook.AllowedHostIPNets = ParseWebhookAllowedHostList(sec.Key("ALLOWED_HOST_LIST").MustString("global")) + Webhook.AllowedHostList, Webhook.AllowedHostIPNets = util.ParseHostMatchList(sec.Key("ALLOWED_HOST_LIST").MustString(util.HostListBuiltinExternal)) Webhook.Types = []string{"gitea", "gogs", "slack", "discord", "dingtalk", "telegram", "msteams", "feishu", "matrix", "wechatwork"} Webhook.PagingNum = sec.Key("PAGING_NUM").MustInt(10) Webhook.ProxyURL = sec.Key("PROXY_URL").MustString("") diff --git a/modules/util/net.go b/modules/util/net.go new file mode 100644 index 0000000000000..c8fe24c095d24 --- /dev/null +++ b/modules/util/net.go @@ -0,0 +1,93 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package util + +import ( + "net" + "path/filepath" + "strings" +) + +//HostListBuiltinAll all hosts are matched +const HostListBuiltinAll = "all" + +//HostListBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched +const HostListBuiltinExternal = "external" + +//HostListBuiltinPrivate RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet. +const HostListBuiltinPrivate = "private" + +//HostListBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. +const HostListBuiltinLoopback = "loopback" + +// IsIPPrivate for net.IP.IsPrivate. TODO: replace with `ip.IsPrivate()` if min go version is bumped to 1.17 +func IsIPPrivate(ip net.IP) bool { + if ip4 := ip.To4(); ip4 != nil { + return ip4[0] == 10 || + (ip4[0] == 172 && ip4[1]&0xf0 == 16) || + (ip4[0] == 192 && ip4[1] == 168) + } + return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc +} + +// ParseHostMatchList parses the host list for HostOrIPMatchesList +func ParseHostMatchList(hostListStr string) (hostList []string, ipNets []*net.IPNet) { + for _, s := range strings.Split(hostListStr, ",") { + s = strings.TrimSpace(s) + if s == "" { + continue + } + _, ipNet, err := net.ParseCIDR(s) + if err == nil { + ipNets = append(ipNets, ipNet) + } else { + hostList = append(hostList, s) + } + } + return +} + +// HostOrIPMatchesList checks if the host or IP matches an allow/deny(block) list +func HostOrIPMatchesList(host string, ip net.IP, hostList []string, ipNets []*net.IPNet) bool { + var matched bool + ipStr := ip.String() +loop: + for _, hostInList := range hostList { + switch hostInList { + case "": + continue + case HostListBuiltinAll: + matched = true + break loop + case HostListBuiltinExternal: + if matched = ip.IsGlobalUnicast() && !IsIPPrivate(ip); matched { + break loop + } + case HostListBuiltinPrivate: + if matched = IsIPPrivate(ip); matched { + break loop + } + case HostListBuiltinLoopback: + if matched = ip.IsLoopback(); matched { + break loop + } + default: + if matched, _ = filepath.Match(hostInList, host); matched { + break loop + } + if matched, _ = filepath.Match(hostInList, ipStr); matched { + break loop + } + } + } + if !matched { + for _, ipNet := range ipNets { + if matched = ipNet.Contains(ip); matched { + break + } + } + } + return matched +} diff --git a/modules/util/net_test.go b/modules/util/net_test.go new file mode 100644 index 0000000000000..f3bc59db966a5 --- /dev/null +++ b/modules/util/net_test.go @@ -0,0 +1,119 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package util + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHostOrIPMatchesList(t *testing.T) { + type tc struct { + host string + ip net.IP + expected bool + } + + // for IPv6: "::1" is loopback, "fd00::/8" is private + + ah, an := ParseHostMatchList("private, external, *.mydomain.com, 169.254.1.0/24") + cases := []tc{ + {"", net.IPv4zero, false}, + {"", net.IPv6zero, false}, + + {"", net.ParseIP("127.0.0.1"), false}, + {"", net.ParseIP("::1"), false}, + + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"", net.ParseIP("fd00::1"), true}, + + {"", net.ParseIP("8.8.8.8"), true}, + {"", net.ParseIP("1001::1"), true}, + + {"mydomain.com", net.IPv4zero, false}, + {"sub.mydomain.com", net.IPv4zero, true}, + + {"", net.ParseIP("169.254.1.1"), true}, + {"", net.ParseIP("169.254.2.2"), false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, HostOrIPMatchesList(c.host, c.ip, ah, an), "case %s(%v)", c.host, c.ip) + } + + ah, an = ParseHostMatchList("loopback") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), true}, + {"", net.ParseIP("10.0.1.1"), false}, + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("8.8.8.8"), false}, + + {"", net.ParseIP("::1"), true}, + {"", net.ParseIP("fd00::1"), false}, + {"", net.ParseIP("1000::1"), false}, + + {"mydomain.com", net.IPv4zero, false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, HostOrIPMatchesList(c.host, c.ip, ah, an), "case %s(%v)", c.host, c.ip) + } + + ah, an = ParseHostMatchList("private") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), false}, + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"", net.ParseIP("8.8.8.8"), false}, + + {"", net.ParseIP("::1"), false}, + {"", net.ParseIP("fd00::1"), true}, + {"", net.ParseIP("1000::1"), false}, + + {"mydomain.com", net.IPv4zero, false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, HostOrIPMatchesList(c.host, c.ip, ah, an), "case %s(%v)", c.host, c.ip) + } + + ah, an = ParseHostMatchList("external") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), false}, + {"", net.ParseIP("10.0.1.1"), false}, + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("8.8.8.8"), true}, + + {"", net.ParseIP("::1"), false}, + {"", net.ParseIP("fd00::1"), false}, + {"", net.ParseIP("1000::1"), true}, + + {"mydomain.com", net.IPv4zero, false}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, HostOrIPMatchesList(c.host, c.ip, ah, an), "case %s(%v)", c.host, c.ip) + } + + ah, an = ParseHostMatchList("all") + cases = []tc{ + {"", net.IPv4zero, true}, + {"", net.ParseIP("127.0.0.1"), true}, + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"", net.ParseIP("8.8.8.8"), true}, + + {"", net.ParseIP("::1"), true}, + {"", net.ParseIP("fd00::1"), true}, + {"", net.ParseIP("1000::1"), true}, + + {"mydomain.com", net.IPv4zero, true}, + } + for _, c := range cases { + assert.Equalf(t, c.expected, HostOrIPMatchesList(c.host, c.ip, ah, an), "case %s(%v)", c.host, c.ip) + } +} diff --git a/modules/util/util.go b/modules/util/util.go index 4f48b14f82363..cbc6eb4f8a01c 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -9,7 +9,6 @@ import ( "crypto/rand" "errors" "math/big" - "net" "strconv" "strings" ) @@ -162,13 +161,3 @@ func RandomString(length int64) (string, error) { } return string(bytes), nil } - -// IsIPPrivate for net.IP.IsPrivate. TODO: replace with `ip.IsPrivate()` if min go version is bumped to 1.17 -func IsIPPrivate(ip net.IP) bool { - if ip4 := ip.To4(); ip4 != nil { - return ip4[0] == 10 || - (ip4[0] == 172 && ip4[1]&0xf0 == 16) || - (ip4[0] == 192 && ip4[1] == 168) - } - return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc -} diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index b18ede21013f2..a0df7a67d9ee4 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -16,7 +16,6 @@ import ( "net" "net/http" "net/url" - "path/filepath" "strconv" "strings" "sync" @@ -294,51 +293,6 @@ func webhookProxy() func(req *http.Request) (*url.URL, error) { } } -func isWebhookRequestAllowed(allowedHostList []string, allowedHostIPNets []*net.IPNet, host string, ip net.IP) bool { - var allowed bool - ipStr := ip.String() -loop: - for _, allowedHost := range allowedHostList { - switch allowedHost { - case "": - continue - case "all": - allowed = true - break loop - case "global": - if allowed = ip.IsGlobalUnicast() && !util.IsIPPrivate(ip); allowed { - break loop - } - case "private": - if allowed = util.IsIPPrivate(ip); allowed { - break loop - } - case "loopback": - if allowed = ip.IsLoopback(); allowed { - break loop - } - default: - if ok, _ := filepath.Match(allowedHost, host); ok { - allowed = true - break loop - } - if ok, _ := filepath.Match(allowedHost, ipStr); ok { - allowed = true - break loop - } - } - } - if !allowed { - for _, allowIPNet := range allowedHostIPNets { - if allowIPNet.Contains(ip) { - allowed = true - break - } - } - } - return allowed -} - // InitDeliverHooks starts the hooks delivery thread func InitDeliverHooks() { timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second @@ -358,7 +312,7 @@ func InitDeliverHooks() { if err != nil { return fmt.Errorf("webhook can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%v", req.Host, network, ipAddr, err) } - if !isWebhookRequestAllowed(setting.Webhook.AllowedHostList, setting.Webhook.AllowedHostIPNets, req.Host, tcpAddr.IP) { + if !util.HostOrIPMatchesList(req.Host, tcpAddr.IP, setting.Webhook.AllowedHostList, setting.Webhook.AllowedHostIPNets) { return fmt.Errorf("webhook can only call allowed HTTP servers (check your webhook.ALLOWED_HOST_LIST setting), deny '%s(%s)'", req.Host, ipAddr) } return nil diff --git a/services/webhook/deliver_test.go b/services/webhook/deliver_test.go index 52efc04ee511a..cfc99d796a536 100644 --- a/services/webhook/deliver_test.go +++ b/services/webhook/deliver_test.go @@ -5,7 +5,6 @@ package webhook import ( - "net" "net/http" "net/url" "testing" @@ -38,84 +37,3 @@ func TestWebhookProxy(t *testing.T) { } } } - -func TestIsWebhookRequestAllowed(t *testing.T) { - type tc struct { - host string - ip net.IP - expected bool - } - - ah, an := setting.ParseWebhookAllowedHostList("private, global, *.google.com, 169.254.1.0/24") - cases := []tc{ - {"", net.IPv4zero, false}, - - {"", net.ParseIP("127.0.0.1"), false}, - - {"", net.ParseIP("10.0.1.1"), true}, - {"", net.ParseIP("192.168.1.1"), true}, - - {"", net.ParseIP("8.8.8.8"), true}, - - {"google.com", net.IPv4zero, false}, - {"sub.google.com", net.IPv4zero, true}, - - {"", net.ParseIP("169.254.1.1"), true}, - {"", net.ParseIP("169.254.2.2"), false}, - } - for _, c := range cases { - assert.Equalf(t, c.expected, isWebhookRequestAllowed(ah, an, c.host, c.ip), "case %s(%v)", c.host, c.ip) - } - - ah, an = setting.ParseWebhookAllowedHostList("loopback") - cases = []tc{ - {"", net.IPv4zero, false}, - {"", net.ParseIP("127.0.0.1"), true}, - {"", net.ParseIP("10.0.1.1"), false}, - {"", net.ParseIP("192.168.1.1"), false}, - {"", net.ParseIP("8.8.8.8"), false}, - {"google.com", net.IPv4zero, false}, - } - for _, c := range cases { - assert.Equalf(t, c.expected, isWebhookRequestAllowed(ah, an, c.host, c.ip), "case %s(%v)", c.host, c.ip) - } - - ah, an = setting.ParseWebhookAllowedHostList("private") - cases = []tc{ - {"", net.IPv4zero, false}, - {"", net.ParseIP("127.0.0.1"), false}, - {"", net.ParseIP("10.0.1.1"), true}, - {"", net.ParseIP("192.168.1.1"), true}, - {"", net.ParseIP("8.8.8.8"), false}, - {"google.com", net.IPv4zero, false}, - } - for _, c := range cases { - assert.Equalf(t, c.expected, isWebhookRequestAllowed(ah, an, c.host, c.ip), "case %s(%v)", c.host, c.ip) - } - - ah, an = setting.ParseWebhookAllowedHostList("global") - cases = []tc{ - {"", net.IPv4zero, false}, - {"", net.ParseIP("127.0.0.1"), false}, - {"", net.ParseIP("10.0.1.1"), false}, - {"", net.ParseIP("192.168.1.1"), false}, - {"", net.ParseIP("8.8.8.8"), true}, - {"google.com", net.IPv4zero, false}, - } - for _, c := range cases { - assert.Equalf(t, c.expected, isWebhookRequestAllowed(ah, an, c.host, c.ip), "case %s(%v)", c.host, c.ip) - } - - ah, an = setting.ParseWebhookAllowedHostList("all") - cases = []tc{ - {"", net.IPv4zero, true}, - {"", net.ParseIP("127.0.0.1"), true}, - {"", net.ParseIP("10.0.1.1"), true}, - {"", net.ParseIP("192.168.1.1"), true}, - {"", net.ParseIP("8.8.8.8"), true}, - {"google.com", net.IPv4zero, true}, - } - for _, c := range cases { - assert.Equalf(t, c.expected, isWebhookRequestAllowed(ah, an, c.host, c.ip), "case %s(%v)", c.host, c.ip) - } -} From b5d0fc673d57c2efe32579ff24071b8b77cb2147 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sat, 30 Oct 2021 19:37:54 +0800 Subject: [PATCH 3/8] fix spaces --- modules/util/net.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/util/net.go b/modules/util/net.go index c8fe24c095d24..d051f874c5dd9 100644 --- a/modules/util/net.go +++ b/modules/util/net.go @@ -10,16 +10,16 @@ import ( "strings" ) -//HostListBuiltinAll all hosts are matched +// HostListBuiltinAll all hosts are matched const HostListBuiltinAll = "all" -//HostListBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched +// HostListBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched const HostListBuiltinExternal = "external" -//HostListBuiltinPrivate RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet. +// HostListBuiltinPrivate RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet. const HostListBuiltinPrivate = "private" -//HostListBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. +// HostListBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. const HostListBuiltinLoopback = "loopback" // IsIPPrivate for net.IP.IsPrivate. TODO: replace with `ip.IsPrivate()` if min go version is bumped to 1.17 From d897184a22c4da12727e29ce8187f0491355d6d8 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 31 Oct 2021 17:38:35 +0800 Subject: [PATCH 4/8] refactor to modules/hostmatcher --- modules/hostmatcher/hostmatcher.go | 92 +++++++++++++++++++ .../hostmatcher_test.go} | 22 ++--- modules/setting/webhook.go | 8 +- modules/util/net.go | 74 --------------- services/webhook/deliver.go | 4 +- 5 files changed, 107 insertions(+), 93 deletions(-) create mode 100644 modules/hostmatcher/hostmatcher.go rename modules/{util/net_test.go => hostmatcher/hostmatcher_test.go} (76%) diff --git a/modules/hostmatcher/hostmatcher.go b/modules/hostmatcher/hostmatcher.go new file mode 100644 index 0000000000000..dcb48f027aab3 --- /dev/null +++ b/modules/hostmatcher/hostmatcher.go @@ -0,0 +1,92 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package hostmatcher + +import ( + "net" + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/util" +) + +// HostMatchList is used to check if a host or ip is in a list +type HostMatchList struct { + hosts []string + ipNets []*net.IPNet +} + +// MatchBuiltinAll all hosts are matched +const MatchBuiltinAll = "all" + +// MatchBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched +const MatchBuiltinExternal = "external" + +// MatchBuiltinPrivate RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet. +const MatchBuiltinPrivate = "private" + +// MatchBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. +const MatchBuiltinLoopback = "loopback" + +// ParseHostMatchList parses the host list HostMatchList +func ParseHostMatchList(hostList string) *HostMatchList { + hl := &HostMatchList{} + for _, s := range strings.Split(hostList, ",") { + s = strings.TrimSpace(s) + if s == "" { + continue + } + _, ipNet, err := net.ParseCIDR(s) + if err == nil { + hl.ipNets = append(hl.ipNets, ipNet) + } else { + hl.hosts = append(hl.hosts, s) + } + } + return hl +} + +// MatchesHostOrIP checks if the host or IP matches an allow/deny(block) list +func (hl *HostMatchList) MatchesHostOrIP(host string, ip net.IP) bool { + var matched bool + ipStr := ip.String() +loop: + for _, hostInList := range hl.hosts { + switch hostInList { + case "": + continue + case MatchBuiltinAll: + matched = true + break loop + case MatchBuiltinExternal: + if matched = ip.IsGlobalUnicast() && !util.IsIPPrivate(ip); matched { + break loop + } + case MatchBuiltinPrivate: + if matched = util.IsIPPrivate(ip); matched { + break loop + } + case MatchBuiltinLoopback: + if matched = ip.IsLoopback(); matched { + break loop + } + default: + if matched, _ = filepath.Match(hostInList, host); matched { + break loop + } + if matched, _ = filepath.Match(hostInList, ipStr); matched { + break loop + } + } + } + if !matched { + for _, ipNet := range hl.ipNets { + if matched = ipNet.Contains(ip); matched { + break + } + } + } + return matched +} diff --git a/modules/util/net_test.go b/modules/hostmatcher/hostmatcher_test.go similarity index 76% rename from modules/util/net_test.go rename to modules/hostmatcher/hostmatcher_test.go index f3bc59db966a5..206ed8b22cbcd 100644 --- a/modules/util/net_test.go +++ b/modules/hostmatcher/hostmatcher_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package util +package hostmatcher import ( "net" @@ -20,7 +20,7 @@ func TestHostOrIPMatchesList(t *testing.T) { // for IPv6: "::1" is loopback, "fd00::/8" is private - ah, an := ParseHostMatchList("private, external, *.mydomain.com, 169.254.1.0/24") + hl := ParseHostMatchList("private, external, *.mydomain.com, 169.254.1.0/24") cases := []tc{ {"", net.IPv4zero, false}, {"", net.IPv6zero, false}, @@ -42,10 +42,10 @@ func TestHostOrIPMatchesList(t *testing.T) { {"", net.ParseIP("169.254.2.2"), false}, } for _, c := range cases { - assert.Equalf(t, c.expected, HostOrIPMatchesList(c.host, c.ip, ah, an), "case %s(%v)", c.host, c.ip) + assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) } - ah, an = ParseHostMatchList("loopback") + hl = ParseHostMatchList("loopback") cases = []tc{ {"", net.IPv4zero, false}, {"", net.ParseIP("127.0.0.1"), true}, @@ -60,10 +60,10 @@ func TestHostOrIPMatchesList(t *testing.T) { {"mydomain.com", net.IPv4zero, false}, } for _, c := range cases { - assert.Equalf(t, c.expected, HostOrIPMatchesList(c.host, c.ip, ah, an), "case %s(%v)", c.host, c.ip) + assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) } - ah, an = ParseHostMatchList("private") + hl = ParseHostMatchList("private") cases = []tc{ {"", net.IPv4zero, false}, {"", net.ParseIP("127.0.0.1"), false}, @@ -78,10 +78,10 @@ func TestHostOrIPMatchesList(t *testing.T) { {"mydomain.com", net.IPv4zero, false}, } for _, c := range cases { - assert.Equalf(t, c.expected, HostOrIPMatchesList(c.host, c.ip, ah, an), "case %s(%v)", c.host, c.ip) + assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) } - ah, an = ParseHostMatchList("external") + hl = ParseHostMatchList("external") cases = []tc{ {"", net.IPv4zero, false}, {"", net.ParseIP("127.0.0.1"), false}, @@ -96,10 +96,10 @@ func TestHostOrIPMatchesList(t *testing.T) { {"mydomain.com", net.IPv4zero, false}, } for _, c := range cases { - assert.Equalf(t, c.expected, HostOrIPMatchesList(c.host, c.ip, ah, an), "case %s(%v)", c.host, c.ip) + assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) } - ah, an = ParseHostMatchList("all") + hl = ParseHostMatchList("all") cases = []tc{ {"", net.IPv4zero, true}, {"", net.ParseIP("127.0.0.1"), true}, @@ -114,6 +114,6 @@ func TestHostOrIPMatchesList(t *testing.T) { {"mydomain.com", net.IPv4zero, true}, } for _, c := range cases { - assert.Equalf(t, c.expected, HostOrIPMatchesList(c.host, c.ip, ah, an), "case %s(%v)", c.host, c.ip) + assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) } } diff --git a/modules/setting/webhook.go b/modules/setting/webhook.go index 1daa258232f34..51938ec699a88 100644 --- a/modules/setting/webhook.go +++ b/modules/setting/webhook.go @@ -5,11 +5,10 @@ package setting import ( - "net" "net/url" + "code.gitea.io/gitea/modules/hostmatcher" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" ) var ( @@ -18,8 +17,7 @@ var ( QueueLength int DeliverTimeout int SkipTLSVerify bool - AllowedHostList []string - AllowedHostIPNets []*net.IPNet + AllowedHostList *hostmatcher.HostMatchList Types []string PagingNum int ProxyURL string @@ -40,7 +38,7 @@ func newWebhookService() { Webhook.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000) Webhook.DeliverTimeout = sec.Key("DELIVER_TIMEOUT").MustInt(5) Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool() - Webhook.AllowedHostList, Webhook.AllowedHostIPNets = util.ParseHostMatchList(sec.Key("ALLOWED_HOST_LIST").MustString(util.HostListBuiltinExternal)) + Webhook.AllowedHostList = hostmatcher.ParseHostMatchList(sec.Key("ALLOWED_HOST_LIST").MustString(hostmatcher.MatchBuiltinExternal)) Webhook.Types = []string{"gitea", "gogs", "slack", "discord", "dingtalk", "telegram", "msteams", "feishu", "matrix", "wechatwork"} Webhook.PagingNum = sec.Key("PAGING_NUM").MustInt(10) Webhook.ProxyURL = sec.Key("PROXY_URL").MustString("") diff --git a/modules/util/net.go b/modules/util/net.go index d051f874c5dd9..54c0a2ca395f0 100644 --- a/modules/util/net.go +++ b/modules/util/net.go @@ -6,22 +6,8 @@ package util import ( "net" - "path/filepath" - "strings" ) -// HostListBuiltinAll all hosts are matched -const HostListBuiltinAll = "all" - -// HostListBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched -const HostListBuiltinExternal = "external" - -// HostListBuiltinPrivate RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet. -const HostListBuiltinPrivate = "private" - -// HostListBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. -const HostListBuiltinLoopback = "loopback" - // IsIPPrivate for net.IP.IsPrivate. TODO: replace with `ip.IsPrivate()` if min go version is bumped to 1.17 func IsIPPrivate(ip net.IP) bool { if ip4 := ip.To4(); ip4 != nil { @@ -31,63 +17,3 @@ func IsIPPrivate(ip net.IP) bool { } return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc } - -// ParseHostMatchList parses the host list for HostOrIPMatchesList -func ParseHostMatchList(hostListStr string) (hostList []string, ipNets []*net.IPNet) { - for _, s := range strings.Split(hostListStr, ",") { - s = strings.TrimSpace(s) - if s == "" { - continue - } - _, ipNet, err := net.ParseCIDR(s) - if err == nil { - ipNets = append(ipNets, ipNet) - } else { - hostList = append(hostList, s) - } - } - return -} - -// HostOrIPMatchesList checks if the host or IP matches an allow/deny(block) list -func HostOrIPMatchesList(host string, ip net.IP, hostList []string, ipNets []*net.IPNet) bool { - var matched bool - ipStr := ip.String() -loop: - for _, hostInList := range hostList { - switch hostInList { - case "": - continue - case HostListBuiltinAll: - matched = true - break loop - case HostListBuiltinExternal: - if matched = ip.IsGlobalUnicast() && !IsIPPrivate(ip); matched { - break loop - } - case HostListBuiltinPrivate: - if matched = IsIPPrivate(ip); matched { - break loop - } - case HostListBuiltinLoopback: - if matched = ip.IsLoopback(); matched { - break loop - } - default: - if matched, _ = filepath.Match(hostInList, host); matched { - break loop - } - if matched, _ = filepath.Match(hostInList, ipStr); matched { - break loop - } - } - } - if !matched { - for _, ipNet := range ipNets { - if matched = ipNet.Contains(ip); matched { - break - } - } - } - return matched -} diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index a0df7a67d9ee4..04cec4c1c4ec9 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -27,8 +27,6 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/proxy" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" - "github.com/gobwas/glob" ) @@ -312,7 +310,7 @@ func InitDeliverHooks() { if err != nil { return fmt.Errorf("webhook can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%v", req.Host, network, ipAddr, err) } - if !util.HostOrIPMatchesList(req.Host, tcpAddr.IP, setting.Webhook.AllowedHostList, setting.Webhook.AllowedHostIPNets) { + if !setting.Webhook.AllowedHostList.MatchesHostOrIP(req.Host, tcpAddr.IP) { return fmt.Errorf("webhook can only call allowed HTTP servers (check your webhook.ALLOWED_HOST_LIST setting), deny '%s(%s)'", req.Host, ipAddr) } return nil From 4156a749dd10a300ec01c74f960418ca643f91ba Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 31 Oct 2021 17:40:14 +0800 Subject: [PATCH 5/8] fix spaces --- modules/hostmatcher/hostmatcher.go | 2 +- modules/setting/webhook.go | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/hostmatcher/hostmatcher.go b/modules/hostmatcher/hostmatcher.go index dcb48f027aab3..cf26f7f904587 100644 --- a/modules/hostmatcher/hostmatcher.go +++ b/modules/hostmatcher/hostmatcher.go @@ -14,7 +14,7 @@ import ( // HostMatchList is used to check if a host or ip is in a list type HostMatchList struct { - hosts []string + hosts []string ipNets []*net.IPNet } diff --git a/modules/setting/webhook.go b/modules/setting/webhook.go index 51938ec699a88..acd5bd0455052 100644 --- a/modules/setting/webhook.go +++ b/modules/setting/webhook.go @@ -14,15 +14,15 @@ import ( var ( // Webhook settings Webhook = struct { - QueueLength int - DeliverTimeout int - SkipTLSVerify bool - AllowedHostList *hostmatcher.HostMatchList - Types []string - PagingNum int - ProxyURL string - ProxyURLFixed *url.URL - ProxyHosts []string + QueueLength int + DeliverTimeout int + SkipTLSVerify bool + AllowedHostList *hostmatcher.HostMatchList + Types []string + PagingNum int + ProxyURL string + ProxyURLFixed *url.URL + ProxyHosts []string }{ QueueLength: 1000, DeliverTimeout: 5, From c6f560de552a75ba4e91890172e6af47e0a5e176 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 31 Oct 2021 22:32:29 +0800 Subject: [PATCH 6/8] use "*" instead of "all", improve comments --- custom/conf/app.example.ini | 2 +- docs/content/doc/advanced/config-cheat-sheet.en-us.md | 2 +- modules/hostmatcher/hostmatcher.go | 5 +++-- modules/hostmatcher/hostmatcher_test.go | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index b3afead2cbcc9..005de067eb8ba 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1397,7 +1397,7 @@ PATH = ;DELIVER_TIMEOUT = 5 ;; ;; Webhook can only call allowed hosts for security reasons. Comma separated list, eg: external, 192.168.1.0/24, *.mydomain.com -;; Built-in: loopback (for localhost), private (for LAN/intranet), external (for public hosts on internet), all (for all hosts) +;; Built-in: loopback (for localhost), private (for LAN/intranet), external (for public hosts on internet), * (for all hosts) ;; CIDR list: 1.2.3.0/8, 2001:db8::/32 ;; Wildcard hosts: *.mydomain.com, 192.168.100.* ; ALLOWED_HOST_LIST = external diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index ca35074d23f94..6cc6043cae7d0 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -586,7 +586,7 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type - `loopback`: 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. - `private`: RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet. - `external`: A valid non-private unicast IP, you can access all hosts on public internet. - - `all`: All hosts are allowed. + - `*`: All hosts are allowed. - CIDR list: `1.2.3.0/8` for IPv4 and `2001:db8::/32` for IPv6 - Wildcard hosts: `*.mydomain.com`, `192.168.100.*` - `SKIP_TLS_VERIFY`: **false**: Allow insecure certification. diff --git a/modules/hostmatcher/hostmatcher.go b/modules/hostmatcher/hostmatcher.go index cf26f7f904587..a95557def1a40 100644 --- a/modules/hostmatcher/hostmatcher.go +++ b/modules/hostmatcher/hostmatcher.go @@ -12,14 +12,15 @@ import ( "code.gitea.io/gitea/modules/util" ) -// HostMatchList is used to check if a host or ip is in a list +// HostMatchList is used to check if a host or IP is in a list. +// If you only need to do wildcard matching, consider to use modules/matchlist type HostMatchList struct { hosts []string ipNets []*net.IPNet } // MatchBuiltinAll all hosts are matched -const MatchBuiltinAll = "all" +const MatchBuiltinAll = "*" // MatchBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched const MatchBuiltinExternal = "external" diff --git a/modules/hostmatcher/hostmatcher_test.go b/modules/hostmatcher/hostmatcher_test.go index 206ed8b22cbcd..e9c8f68dc6f6b 100644 --- a/modules/hostmatcher/hostmatcher_test.go +++ b/modules/hostmatcher/hostmatcher_test.go @@ -99,7 +99,7 @@ func TestHostOrIPMatchesList(t *testing.T) { assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) } - hl = ParseHostMatchList("all") + hl = ParseHostMatchList("*") cases = []tc{ {"", net.IPv4zero, true}, {"", net.ParseIP("127.0.0.1"), true}, From d1d5ff0182f48a2f148624b931a669bc8c0ced66 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 1 Nov 2021 11:46:11 +0800 Subject: [PATCH 7/8] fix space --- custom/conf/app.example.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 005de067eb8ba..eadc1c0d96256 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1400,7 +1400,7 @@ PATH = ;; Built-in: loopback (for localhost), private (for LAN/intranet), external (for public hosts on internet), * (for all hosts) ;; CIDR list: 1.2.3.0/8, 2001:db8::/32 ;; Wildcard hosts: *.mydomain.com, 192.168.100.* -; ALLOWED_HOST_LIST = external +;ALLOWED_HOST_LIST = external ;; ;; Allow insecure certification ;SKIP_TLS_VERIFY = false From e134bf20d5186aff166ae042783bd47e83debaf0 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 1 Nov 2021 12:00:11 +0800 Subject: [PATCH 8/8] use lowercase --- modules/hostmatcher/hostmatcher.go | 3 ++- modules/hostmatcher/hostmatcher_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/hostmatcher/hostmatcher.go b/modules/hostmatcher/hostmatcher.go index a95557def1a40..f8a787c575e3d 100644 --- a/modules/hostmatcher/hostmatcher.go +++ b/modules/hostmatcher/hostmatcher.go @@ -35,7 +35,7 @@ const MatchBuiltinLoopback = "loopback" func ParseHostMatchList(hostList string) *HostMatchList { hl := &HostMatchList{} for _, s := range strings.Split(hostList, ",") { - s = strings.TrimSpace(s) + s = strings.ToLower(strings.TrimSpace(s)) if s == "" { continue } @@ -52,6 +52,7 @@ func ParseHostMatchList(hostList string) *HostMatchList { // MatchesHostOrIP checks if the host or IP matches an allow/deny(block) list func (hl *HostMatchList) MatchesHostOrIP(host string, ip net.IP) bool { var matched bool + host = strings.ToLower(host) ipStr := ip.String() loop: for _, hostInList := range hl.hosts { diff --git a/modules/hostmatcher/hostmatcher_test.go b/modules/hostmatcher/hostmatcher_test.go index e9c8f68dc6f6b..8eaafbdbc809b 100644 --- a/modules/hostmatcher/hostmatcher_test.go +++ b/modules/hostmatcher/hostmatcher_test.go @@ -20,7 +20,7 @@ func TestHostOrIPMatchesList(t *testing.T) { // for IPv6: "::1" is loopback, "fd00::/8" is private - hl := ParseHostMatchList("private, external, *.mydomain.com, 169.254.1.0/24") + hl := ParseHostMatchList("private, External, *.myDomain.com, 169.254.1.0/24") cases := []tc{ {"", net.IPv4zero, false}, {"", net.IPv6zero, false},