diff --git a/cmd/server/docs/docs.go b/cmd/server/docs/docs.go index 8e00e7afcd..353d8ee526 100644 --- a/cmd/server/docs/docs.go +++ b/cmd/server/docs/docs.go @@ -5049,6 +5049,9 @@ const docTemplate = `{ "clone_url_ssh": { "type": "string" }, + "config_extension_endpoint": { + "type": "string" + }, "config_file": { "type": "string" }, @@ -5095,6 +5098,9 @@ const docTemplate = `{ "scm": { "$ref": "#/definitions/SCMKind" }, + "secret_extension_endpoint": { + "type": "string" + }, "timeout": { "type": "integer" }, @@ -5121,6 +5127,9 @@ const docTemplate = `{ "$ref": "#/definitions/WebhookEvent" } }, + "config_extension_endpoint": { + "type": "string" + }, "config_file": { "type": "string" }, @@ -5130,6 +5139,9 @@ const docTemplate = `{ "netrc_only_trusted": { "type": "boolean" }, + "secret_extension_endpoint": { + "type": "string" + }, "timeout": { "type": "integer" }, diff --git a/cmd/server/flags.go b/cmd/server/flags.go index 4b5e5a8117..ea650c971a 100644 --- a/cmd/server/flags.go +++ b/cmd/server/flags.go @@ -21,6 +21,7 @@ import ( "github.com/urfave/cli/v3" + "go.woodpecker-ci.org/woodpecker/v2/server/services/utils/hostmatcher" "go.woodpecker-ci.org/woodpecker/v2/shared/constant" "go.woodpecker-ci.org/woodpecker/v2/shared/logger" ) @@ -203,6 +204,12 @@ var flags = append([]cli.Flag{ Name: "config-service-endpoint", Usage: "url used for calling configuration service endpoint", }, + &cli.StringFlag{ + Sources: cli.EnvVars("WOODPECKER_ALLOWED_EXTENSIONS_HOSTS"), + Name: "allowed-extensions-hosts", + Usage: "Hosts that are allowed to be used by extensions", + Value: hostmatcher.MatchBuiltinExternal, + }, &cli.StringFlag{ Sources: cli.EnvVars("WOODPECKER_DATABASE_DRIVER"), Name: "driver", diff --git a/docs/docs/20-usage/72-extensions/40-configuration-extension.md b/docs/docs/20-usage/72-extensions/40-configuration-extension.md new file mode 100644 index 0000000000..e2ab1d3f43 --- /dev/null +++ b/docs/docs/20-usage/72-extensions/40-configuration-extension.md @@ -0,0 +1,164 @@ +# Configuration extension + +The configuration extension can be used to modify or generate Woodpeckers pipeline configurations. You can configure a HTTP +endpoint in the repository settings in the extensions tab. + +Using such an extension can be useful if you want to: + + +- Preprocess the original configuration file with something like go templating +- Convert custom attributes to Woodpecker attributes +- Add defaults to the configuration like default steps +- Convert configuration files from a totally different format like Gitlab CI config, Starlark, Jsonnet, ... +- Centralize configuration for multiple repositories in one place + +## Security + +:::warning +As Woodpecker will pass private information like tokens and will execute the returned configuration, it is extremely important to secure the external extension. Therefore Woodpecker signs every request. Read more about it in the [security section](./10-extensions.md#security). +::: + +## Global configuration + +In addition to the ability to configure the extension per repository, you can also configure a global endpoint in the Woodpecker server configuration. This can be useful if you want to use the extension for all repositories. Be careful if +you share your Woodpecker server with others as they will also use your configuration extension. + +The global configuration will be called before the repository specific configuration extension if both are configured. + +```ini title="Server" +WOODPECKER_CONFIG_SERVICE_ENDPOINT=https://example.com/ciconfig +``` + +## How it works + +When a pipeline is triggered Woodpecker will fetch the pipeline configuration from the repository, then make a HTTP POST request to the configured extension with a JSON payload containing some data like the repository, pipeline information and the current config files retrieved from the repository. The extension can then send back modified or even new pipeline configurations following Woodpeckers official yaml format that should be used. + +:::tip +The netrc data is pretty powerful as it contains credentials to access the repository. You can use this to clone the repository or even use the forge (Github or Gitlab, ...) api to get more information about the repository. +::: + +### Request + +The extension receives an HTTP POST request with the following JSON payload: + +```ts +class Request { + repo: Repo; + pipeline: Pipeline; + netrc: Netrc; + configuration: { + name: string; // filename of the configuration file + data: string; // content of the configuration file + }[]; +} +``` + +Checkout the following models for more information: + +- [repo model](https://github.com/woodpecker-ci/woodpecker/blob/main/server/model/repo.go) +- [pipeline model](https://github.com/woodpecker-ci/woodpecker/blob/main/server/model/pipeline.go) +- [netrc model](https://github.com/woodpecker-ci/woodpecker/blob/main/server/model/netrc.go) + +:::tip +The `netrc` data is pretty powerful as it contains credentials to access the repository. You can use this to clone the repository or even use the forge (Github or Gitlab, ...) api to get more information about the repository. +::: + +Example request: + +```json +{ + "repo": { + "id": 100, + "uid": "", + "user_id": 0, + "namespace": "", + "name": "woodpecker-test-pipeline", + "slug": "", + "scm": "git", + "git_http_url": "", + "git_ssh_url": "", + "link": "", + "default_branch": "", + "private": true, + "visibility": "private", + "active": true, + "config": "", + "trusted": false, + "protected": false, + "ignore_forks": false, + "ignore_pulls": false, + "cancel_pulls": false, + "timeout": 60, + "counter": 0, + "synced": 0, + "created": 0, + "updated": 0, + "version": 0 + }, + "pipeline": { + "author": "myUser", + "author_avatar": "https://myforge.com/avatars/d6b3f7787a685fcdf2a44e2c685c7e03", + "author_email": "my@email.com", + "branch": "main", + "changed_files": ["some-filename.txt"], + "commit": "2fff90f8d288a4640e90f05049fe30e61a14fd50", + "created_at": 0, + "deploy_to": "", + "enqueued_at": 0, + "error": "", + "event": "push", + "finished_at": 0, + "id": 0, + "link_url": "https://myforge.com/myUser/woodpecker-testpipe/commit/2fff90f8d288a4640e90f05049fe30e61a14fd50", + "message": "test old config\n", + "number": 0, + "parent": 0, + "ref": "refs/heads/main", + "refspec": "", + "clone_url": "", + "reviewed_at": 0, + "reviewed_by": "", + "sender": "myUser", + "signed": false, + "started_at": 0, + "status": "", + "timestamp": 1645962783, + "title": "", + "updated_at": 0, + "verified": false + }, + "configs": [ + { + "name": ".woodpecker.yaml", + "data": "steps:\n - name: backend\n image: alpine\n commands:\n - echo \"Hello there from Repo (.woodpecker.yaml)\"\n" + } + ] +} +``` + +### Response + +The extension should respond with a JSON payload containing the new configuration files in Woodpeckers official yaml format. +If the extension wants to keep the existing configuration files, it can respond with **HTTP 204**. + +```ts +class Response { + configs: { + name: string; // filename of the configuration file + data: string; // content of the configuration file + }[]; +} +``` + +Example response: + +```json +{ + "configs": [ + { + "name": "central-override", + "data": "steps:\n - name: backend\n image: alpine\n commands:\n - echo \"Hello there from ConfigAPI\"\n" + } + ] +} +``` diff --git a/docs/docs/20-usage/72-extensions/72-extensions.md b/docs/docs/20-usage/72-extensions/72-extensions.md new file mode 100644 index 0000000000..54ce33a98f --- /dev/null +++ b/docs/docs/20-usage/72-extensions/72-extensions.md @@ -0,0 +1,34 @@ +# Extensions + +Woodpecker allows you to replace internal logic with external extensions by using pre-defined http endpoints. + +There are currently two types of extensions available: + +- [Configuration extension](./40-configuration-extension.md) to modify or generate Woodpeckers pipeline configurations. + +## Security + +:::warning +You need to trust the extensions as they are receiving private information like secrets and tokens and might return harmful +data like malicious pipeline configurations that could be executed. +::: + +To prevent your extensions from such attacks, Woodpecker is signing all http-requests using [http signatures](https://tools.ietf.org/html/draft-cavage-http-signatures). Woodpecker therefore uses a public-private ed25519 key pair. +To verify the requests your extension has to verify the signature of all request using the public key with some library like [httpsign](https://github.com/yaronf/httpsign). +You can get the public Woodpecker key by opening `http://my-woodpecker.tld/api/signature/public-key` or by visiting the Woodpecker UI, going to you repo settings and opening the extensions page. + +## Example extensions + +A simplistic service providing endpoints for a config and secrets extension can be found here: [https://github.com/woodpecker-ci/example-extensions](https://github.com/woodpecker-ci/example-extensions) + +## Configuration + +To prevent extensions from calling local services by default only external hosts / ip-addresses are allowed. You can change this behavior by setting the `WOODPECKER_ALLOWED_EXTENSIONS_HOSTS` environment variable. You can use a comma separated list of: + +- Built-in networks: + - `loopback`: 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. + - `private`: RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet. + - `external`: A valid non-private unicast IP, you can access all hosts on public internet. + - `*`: All hosts are allowed. +- CIDR list: `1.2.3.0/8` for IPv4 and `2001:db8::/32` for IPv6 +- (Wildcard) hosts: `example.com`, `*.example.com`, `192.168.100.*` diff --git a/docs/docs/20-usage/72-extensions/_category_.yml b/docs/docs/20-usage/72-extensions/_category_.yml new file mode 100644 index 0000000000..6819ad9c0a --- /dev/null +++ b/docs/docs/20-usage/72-extensions/_category_.yml @@ -0,0 +1,7 @@ +label: 'Extensions' +# position: 3 +collapsible: true +collapsed: true +link: + type: 'doc' + id: 'extensions' diff --git a/docs/docs/30-administration/10-server-config.md b/docs/docs/30-administration/10-server-config.md index 4b3c7c3cba..82df226178 100644 --- a/docs/docs/30-administration/10-server-config.md +++ b/docs/docs/30-administration/10-server-config.md @@ -482,6 +482,12 @@ Supported variables: Specify a configuration service endpoint, see [Configuration Extension](./40-advanced/100-external-configuration-api.md) +### `WOODPECKER_ALLOWED_EXTENSIONS_HOSTS` + +> Default: `external` + +Comma-separated list of allowed hosts for extensions. Possible values are `loopback`, `private`, `external`, `*` or CIDR list. + ### `WOODPECKER_FORGE_TIMEOUT` > Default: 3s diff --git a/server/api/repo.go b/server/api/repo.go index b96eeaae68..f2267582ac 100644 --- a/server/api/repo.go +++ b/server/api/repo.go @@ -263,6 +263,9 @@ func PatchRepo(c *gin.Context) { return } } + if in.ConfigExtensionEndpoint != nil { + repo.ConfigExtensionEndpoint = *in.ConfigExtensionEndpoint + } err := _store.UpdateRepo(repo) if err != nil { diff --git a/server/model/pagination.go b/server/model/pagination.go index 7d60bf5199..51848d12ce 100644 --- a/server/model/pagination.go +++ b/server/model/pagination.go @@ -14,6 +14,11 @@ package model +import ( + "fmt" + "strings" +) + type ListOptions struct { All bool Page int @@ -32,3 +37,21 @@ func ApplyPagination[T any](d *ListOptions, slice []T) []T { } return slice[d.PerPage*(d.Page-1) : d.PerPage*(d.Page)] } + +func (d *ListOptions) Encode() string { + var query []string + + if d.Page != 0 { + query = append(query, fmt.Sprintf("page=%d", d.Page)) + } + + if d.PerPage != 0 { + query = append(query, fmt.Sprintf("per_page=%d", d.PerPage)) + } + + if d.All { + query = append(query, "all=true") + } + + return strings.Join(query, "&") +} diff --git a/server/model/repo.go b/server/model/repo.go index 36080f93e3..71ee2af201 100644 --- a/server/model/repo.go +++ b/server/model/repo.go @@ -50,6 +50,7 @@ type Repo struct { Hash string `json:"-" xorm:"varchar(500) 'hash'"` Perm *Perm `json:"-" xorm:"-"` CancelPreviousPipelineEvents []WebhookEvent `json:"cancel_previous_pipeline_events" xorm:"json 'cancel_previous_pipeline_events'"` + ConfigExtensionEndpoint string `json:"config_extension_endpoint" xorm:"varchar(500) 'config_extension_endpoint'"` NetrcOnlyTrusted bool `json:"netrc_only_trusted" xorm:"NOT NULL DEFAULT true 'netrc_only_trusted'"` } // @name Repo @@ -116,6 +117,7 @@ type RepoPatch struct { AllowPull *bool `json:"allow_pr,omitempty"` AllowDeploy *bool `json:"allow_deploy,omitempty"` CancelPreviousPipelineEvents *[]WebhookEvent `json:"cancel_previous_pipeline_events"` + ConfigExtensionEndpoint *string `json:"config_extension_endpoint,omitempty"` NetrcOnlyTrusted *bool `json:"netrc_only_trusted"` } // @name RepoPatch diff --git a/server/services/config/combined_test.go b/server/services/config/combined_test.go index 70c2f407ee..5d671a185c 100644 --- a/server/services/config/combined_test.go +++ b/server/services/config/combined_test.go @@ -35,6 +35,7 @@ import ( forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/server/services/config" + "go.woodpecker-ci.org/woodpecker/v2/server/services/utils" ) func TestFetchFromConfigService(t *testing.T) { @@ -186,7 +187,13 @@ func TestFetchFromConfigService(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(fixtureHandler)) defer ts.Close() - httpFetcher := config.NewHTTP(ts.URL+"/", privEd25519Key) + + client, err := utils.NewHTTPClient(privEd25519Key, "loopback") + if !assert.NoError(t, err) { + return + } + + httpFetcher := config.NewHTTP(ts.URL+"/", client) for _, tt := range testTable { t.Run(tt.name, func(t *testing.T) { diff --git a/server/services/config/http.go b/server/services/config/http.go index 33dcfc7569..381314d950 100644 --- a/server/services/config/http.go +++ b/server/services/config/http.go @@ -16,7 +16,6 @@ package config import ( "context" - "crypto/ed25519" "fmt" net_http "net/http" @@ -27,8 +26,8 @@ import ( ) type http struct { - endpoint string - privateKey ed25519.PrivateKey + endpoint string + client *utils.Client } // configData same as forge.FileMeta but with json tags and string data. @@ -47,8 +46,8 @@ type responseStructure struct { Configs []*configData `json:"configs"` } -func NewHTTP(endpoint string, privateKey ed25519.PrivateKey) Service { - return &http{endpoint, privateKey} +func NewHTTP(endpoint string, client *utils.Client) Service { + return &http{endpoint, client} } func (h *http) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline, oldConfigData []*types.FileMeta, _ bool) ([]*types.FileMeta, error) { @@ -64,7 +63,7 @@ func (h *http) Fetch(ctx context.Context, forge forge.Forge, user *model.User, r Netrc: netrc, } - status, err := utils.Send(ctx, net_http.MethodPost, h.endpoint, h.privateKey, body, response) + status, err := h.client.Send(ctx, net_http.MethodPost, h.endpoint, body, response) if err != nil && status != 204 { return nil, fmt.Errorf("failed to fetch config via http (%d) %w", status, err) } diff --git a/server/services/manager.go b/server/services/manager.go index e38b140e9f..d4c5751c69 100644 --- a/server/services/manager.go +++ b/server/services/manager.go @@ -16,6 +16,7 @@ package services import ( "crypto" + "strings" "time" "github.com/jellydator/ttlcache/v3" @@ -27,6 +28,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/services/environment" "go.woodpecker-ci.org/woodpecker/v2/server/services/registry" "go.woodpecker-ci.org/woodpecker/v2/server/services/secret" + "go.woodpecker-ci.org/woodpecker/v2/server/services/utils" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) @@ -59,6 +61,7 @@ type manager struct { environment environment.Service forgeCache *ttlcache.Cache[int64, forge.Forge] setupForge SetupForge + client *utils.Client } func NewManager(c *cli.Command, store store.Store, setupForge SetupForge) (Manager, error) { @@ -72,7 +75,12 @@ func NewManager(c *cli.Command, store store.Store, setupForge SetupForge) (Manag return nil, err } - configService, err := setupConfigService(c, signaturePrivateKey) + client, err := utils.NewHTTPClient(signaturePrivateKey, c.String("allowed-extensions-hosts")) + if err != nil { + return nil, err + } + + configService, err := setupConfigService(c, client) if err != nil { return nil, err } @@ -87,6 +95,7 @@ func NewManager(c *cli.Command, store store.Store, setupForge SetupForge) (Manag environment: environment.Parse(c.StringSlice("environment")), forgeCache: ttlcache.New(ttlcache.WithDisableTouchOnHit[int64, forge.Forge]()), setupForge: setupForge, + client: client, }, nil } @@ -110,8 +119,11 @@ func (m *manager) RegistryService() registry.Service { return m.registry } -func (m *manager) ConfigServiceFromRepo(_ *model.Repo) config.Service { - // TODO: decide based on repo property which config service to use +func (m *manager) ConfigServiceFromRepo(repo *model.Repo) config.Service { + if repo.ConfigExtensionEndpoint != "" { + return config.NewCombined(m.config, config.NewHTTP(strings.TrimRight(repo.ConfigExtensionEndpoint, "/"), m.client)) + } + return m.config } diff --git a/server/services/setup.go b/server/services/setup.go index 751a30ddd9..4f3e3f63c8 100644 --- a/server/services/setup.go +++ b/server/services/setup.go @@ -30,6 +30,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/services/config" "go.woodpecker-ci.org/woodpecker/v2/server/services/registry" "go.woodpecker-ci.org/woodpecker/v2/server/services/secret" + "go.woodpecker-ci.org/woodpecker/v2/server/services/utils" "go.woodpecker-ci.org/woodpecker/v2/server/store" "go.woodpecker-ci.org/woodpecker/v2/server/store/types" ) @@ -57,7 +58,7 @@ func setupSecretService(store store.Store) secret.Service { return secret.NewDB(store) } -func setupConfigService(c *cli.Command, privateSignatureKey ed25519.PrivateKey) (config.Service, error) { +func setupConfigService(c *cli.Command, client *utils.Client) (config.Service, error) { timeout := c.Duration("forge-timeout") retries := c.Uint("forge-retry") if retries == 0 { @@ -66,7 +67,7 @@ func setupConfigService(c *cli.Command, privateSignatureKey ed25519.PrivateKey) configFetcher := config.NewForge(timeout, uint(retries)) if endpoint := c.String("config-service-endpoint"); endpoint != "" { - httpFetcher := config.NewHTTP(endpoint, privateSignatureKey) + httpFetcher := config.NewHTTP(endpoint, client) return config.NewCombined(configFetcher, httpFetcher), nil } diff --git a/server/services/utils/hostmatcher/hostmatcher.go b/server/services/utils/hostmatcher/hostmatcher.go new file mode 100644 index 0000000000..cc902cd267 --- /dev/null +++ b/server/services/utils/hostmatcher/hostmatcher.go @@ -0,0 +1,163 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT. + +// cSpell:words hostmatcher +package hostmatcher + +import ( + "net" + "path/filepath" + "strings" +) + +// HostMatchList is used to check if a host or IP is in a list. +type HostMatchList struct { + SettingKeyHint string + SettingValue string + + // builtins networks + builtins []string + // patterns for host names (with wildcard support) + patterns []string + // ipNets is the CIDR network list + ipNets []*net.IPNet +} + +// MatchBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched. +const MatchBuiltinExternal = "external" + +// MatchBuiltinPrivate RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet. +const MatchBuiltinPrivate = "private" + +// MatchBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. +const MatchBuiltinLoopback = "loopback" + +func isBuiltin(s string) bool { + return s == MatchBuiltinExternal || s == MatchBuiltinPrivate || s == MatchBuiltinLoopback +} + +// ParseHostMatchList parses the host list HostMatchList. +func ParseHostMatchList(settingKeyHint, hostList string) *HostMatchList { + hl := &HostMatchList{SettingKeyHint: settingKeyHint, SettingValue: hostList} + for _, s := range strings.Split(hostList, ",") { + s = strings.ToLower(strings.TrimSpace(s)) + if s == "" { + continue + } + _, ipNet, err := net.ParseCIDR(s) + switch { + case err == nil: + hl.ipNets = append(hl.ipNets, ipNet) + case isBuiltin(s): + hl.builtins = append(hl.builtins, s) + default: + hl.patterns = append(hl.patterns, s) + } + } + return hl +} + +// ParseSimpleMatchList parse a simple match-list (no built-in networks, no CIDR support, only wildcard pattern match). +func ParseSimpleMatchList(settingKeyHint, matchList string) *HostMatchList { + hl := &HostMatchList{ + SettingKeyHint: settingKeyHint, + SettingValue: matchList, + } + for _, s := range strings.Split(matchList, ",") { + s = strings.ToLower(strings.TrimSpace(s)) + if s == "" { + continue + } + // we keep the same result as old `match-list`, so no builtin/CIDR support here, we only match wildcard patterns + hl.patterns = append(hl.patterns, s) + } + return hl +} + +// AppendBuiltin appends more builtins to match. +func (hl *HostMatchList) AppendBuiltin(builtin string) { + hl.builtins = append(hl.builtins, builtin) +} + +// AppendPattern appends more pattern to match. +func (hl *HostMatchList) AppendPattern(pattern string) { + hl.patterns = append(hl.patterns, pattern) +} + +// IsEmpty checks if the checklist is empty. +func (hl *HostMatchList) IsEmpty() bool { + return hl == nil || (len(hl.builtins) == 0 && len(hl.patterns) == 0 && len(hl.ipNets) == 0) +} + +func (hl *HostMatchList) checkPattern(host string) bool { + host = strings.ToLower(strings.TrimSpace(host)) + for _, pattern := range hl.patterns { + if matched, _ := filepath.Match(pattern, host); matched { + return true + } + } + return false +} + +func (hl *HostMatchList) checkIP(ip net.IP) bool { + for _, pattern := range hl.patterns { + if pattern == "*" { + return true + } + } + for _, builtin := range hl.builtins { + switch builtin { + case MatchBuiltinExternal: + if ip.IsGlobalUnicast() && !ip.IsPrivate() { + return true + } + case MatchBuiltinPrivate: + if ip.IsPrivate() { + return true + } + case MatchBuiltinLoopback: + if ip.IsLoopback() { + return true + } + } + } + for _, ipNet := range hl.ipNets { + if ipNet.Contains(ip) { + return true + } + } + return false +} + +// MatchHostName checks if the host matches an allow/deny(block) list. +func (hl *HostMatchList) MatchHostName(host string) bool { + if hl == nil { + return false + } + + hostname, _, err := net.SplitHostPort(host) + if err != nil { + hostname = host + } + if hl.checkPattern(hostname) { + return true + } + if ip := net.ParseIP(hostname); ip != nil { + return hl.checkIP(ip) + } + return false +} + +// MatchIPAddr checks if the IP matches an allow/deny(block) list, it's safe to pass `nil` to `ip`. +func (hl *HostMatchList) MatchIPAddr(ip net.IP) bool { + if hl == nil { + return false + } + host := ip.String() // nil-safe, we will get "" if ip is nil + return hl.checkPattern(host) || hl.checkIP(ip) +} + +// MatchHostOrIP checks if the host or IP matches an allow/deny(block) list. +func (hl *HostMatchList) MatchHostOrIP(host string, ip net.IP) bool { + return hl.MatchHostName(host) || hl.MatchIPAddr(ip) +} diff --git a/server/services/utils/hostmatcher/hostmatcher_test.go b/server/services/utils/hostmatcher/hostmatcher_test.go new file mode 100644 index 0000000000..e8ef9ddaec --- /dev/null +++ b/server/services/utils/hostmatcher/hostmatcher_test.go @@ -0,0 +1,161 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT. + +package hostmatcher + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHostOrIPMatchesList(t *testing.T) { + type tc struct { + host string + ip net.IP + expected bool + } + + // for IPv6: "::1" is loopback, "fd00::/8" is private + + hl := ParseHostMatchList("", "private, External, *.myDomain.com, 169.254.1.0/24") + + test := func(cases []tc) { + for _, c := range cases { + assert.Equalf(t, c.expected, hl.MatchHostOrIP(c.host, c.ip), "case domain=%s, ip=%v, expected=%v", c.host, c.ip, c.expected) + } + } + + cases := []tc{ + {"", net.IPv4zero, false}, + {"", net.IPv6zero, false}, + + {"", net.ParseIP("127.0.0.1"), false}, + {"127.0.0.1", nil, false}, + {"", net.ParseIP("::1"), false}, + + {"", net.ParseIP("10.0.1.1"), true}, + {"10.0.1.1", nil, true}, + {"10.0.1.1:8080", nil, true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"192.168.1.1", nil, true}, + {"", net.ParseIP("fd00::1"), true}, + {"fd00::1", nil, true}, + + {"", net.ParseIP("8.8.8.8"), true}, + {"", net.ParseIP("1001::1"), true}, + + {"mydomain.com", net.IPv4zero, false}, + {"sub.mydomain.com", net.IPv4zero, true}, + {"sub.mydomain.com:8080", net.IPv4zero, true}, + + {"", net.ParseIP("169.254.1.1"), true}, + {"169.254.1.1", nil, true}, + {"", net.ParseIP("169.254.2.2"), false}, + {"169.254.2.2", nil, false}, + } + test(cases) + + hl = ParseHostMatchList("", "loopback") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), true}, + {"", net.ParseIP("10.0.1.1"), false}, + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("8.8.8.8"), false}, + + {"", net.ParseIP("::1"), true}, + {"", net.ParseIP("fd00::1"), false}, + {"", net.ParseIP("1000::1"), false}, + + {"mydomain.com", net.IPv4zero, false}, + } + test(cases) + + hl = ParseHostMatchList("", "private") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), false}, + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"", net.ParseIP("8.8.8.8"), false}, + + {"", net.ParseIP("::1"), false}, + {"", net.ParseIP("fd00::1"), true}, + {"", net.ParseIP("1000::1"), false}, + + {"mydomain.com", net.IPv4zero, false}, + } + test(cases) + + hl = ParseHostMatchList("", "external") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), false}, + {"", net.ParseIP("10.0.1.1"), false}, + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("8.8.8.8"), true}, + + {"", net.ParseIP("::1"), false}, + {"", net.ParseIP("fd00::1"), false}, + {"", net.ParseIP("1000::1"), true}, + + {"mydomain.com", net.IPv4zero, false}, + } + test(cases) + + hl = ParseHostMatchList("", "*") + cases = []tc{ + {"", net.IPv4zero, true}, + {"", net.ParseIP("127.0.0.1"), true}, + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"", net.ParseIP("8.8.8.8"), true}, + + {"", net.ParseIP("::1"), true}, + {"", net.ParseIP("fd00::1"), true}, + {"", net.ParseIP("1000::1"), true}, + + {"mydomain.com", net.IPv4zero, true}, + } + test(cases) + + // built-in network names can be escaped (warping the first char with `[]`) to be used as a real host name + // this mechanism is reversed for internal usage only (maybe for some rare cases), it's not supposed to be used by end users + // a real user should never use loopback/private/external as their host names + hl = ParseHostMatchList("", "loopback, [p]rivate") + cases = []tc{ + {"loopback", nil, false}, + {"", net.ParseIP("127.0.0.1"), true}, + {"private", nil, true}, + {"", net.ParseIP("192.168.1.1"), false}, + } + test(cases) + + hl = ParseSimpleMatchList("", "loopback, *.domain.com") + cases = []tc{ + {"loopback", nil, true}, + {"", net.ParseIP("127.0.0.1"), false}, + {"sub.domain.com", nil, true}, + {"other.com", nil, false}, + {"", net.ParseIP("1.1.1.1"), false}, + } + test(cases) + + hl = ParseSimpleMatchList("", "external") + cases = []tc{ + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("1.1.1.1"), false}, + {"external", nil, true}, + } + test(cases) + + hl = ParseSimpleMatchList("", "") + cases = []tc{ + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("1.1.1.1"), false}, + {"external", nil, false}, + } + test(cases) +} diff --git a/server/services/utils/hostmatcher/http.go b/server/services/utils/hostmatcher/http.go new file mode 100644 index 0000000000..11063b2dbd --- /dev/null +++ b/server/services/utils/hostmatcher/http.go @@ -0,0 +1,73 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT. + +// cSpell:words hostmatcher +package hostmatcher + +import ( + "context" + "fmt" + "net" + "net/url" + "syscall" + "time" +) + +// NewDialContext returns a DialContext for Transport, the DialContext will do allow/block list check. +func NewDialContext(usage string, allowList, blockList *HostMatchList) func(ctx context.Context, network, addr string) (net.Conn, error) { + return NewDialContextWithProxy(usage, allowList, blockList, nil) +} + +func NewDialContextWithProxy(usage string, allowList, blockList *HostMatchList, proxy *url.URL) func(ctx context.Context, network, addr string) (net.Conn, error) { + // How Go HTTP Client works with redirection: + // transport.RoundTrip URL=http://domain.com, Host=domain.com + // transport.DialContext addrOrHost=domain.com:80 + // dialer.Control tcp4:11.22.33.44:80 + // transport.RoundTrip URL=http://www.domain.com/, Host=(empty here, in the direction, HTTP client doesn't fill the Host field) + // transport.DialContext addrOrHost=domain.com:80 + // dialer.Control tcp4:11.22.33.44:80 + return func(ctx context.Context, network, addrOrHost string) (net.Conn, error) { + // default values are from http.DefaultTransport + const dialTimeout = 30 * time.Second + const dialKeepAlive = 30 * time.Second + + dialer := net.Dialer{ + Timeout: dialTimeout, + KeepAlive: dialKeepAlive, + + Control: func(network, ipAddr string, _ syscall.RawConn) error { + host, port, err := net.SplitHostPort(addrOrHost) + if err != nil { + return err + } + if proxy != nil { + // Always allow the host of the proxy, but only on the specified port. + if host == proxy.Hostname() && port == proxy.Port() { + return nil + } + } + + // in Control func, the addr was already resolved to IP:PORT format, there is no cost to do ResolveTCPAddr here + tcpAddr, err := net.ResolveTCPAddr(network, ipAddr) + if err != nil { + return fmt.Errorf("%s can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%w", usage, host, network, ipAddr, err) + } + + var blockedError error + if blockList.MatchHostOrIP(host, tcpAddr.IP) { + blockedError = fmt.Errorf("%s can not call blocked HTTP servers (check your %s setting), deny '%s(%s)'", usage, blockList.SettingKeyHint, host, ipAddr) + } + + // if we have an allow-list, check the allow-list first + if !allowList.IsEmpty() { + if !allowList.MatchHostOrIP(host, tcpAddr.IP) { + return fmt.Errorf("%s can only call allowed HTTP servers (check your %s setting), deny '%s(%s)'", usage, allowList.SettingKeyHint, host, ipAddr) + } + } + // otherwise, we always follow the blocked list + return blockedError + }, + } + return dialer.DialContext(ctx, network, addrOrHost) + } +} diff --git a/server/services/utils/http.go b/server/services/utils/http.go index 40a50269c7..46b9e0b7bd 100644 --- a/server/services/utils/http.go +++ b/server/services/utils/http.go @@ -17,19 +17,74 @@ package utils import ( "bytes" "context" + "crypto" "crypto/ed25519" + "crypto/tls" "encoding/json" "fmt" "io" "net/http" "net/url" + "time" "github.com/yaronf/httpsign" + + host_matcher "go.woodpecker-ci.org/woodpecker/v2/server/services/utils/hostmatcher" ) +type Client struct { + *httpsign.Client +} + +func getHTTPClient(privateKey crypto.PrivateKey, allowedHostListValue string) (*httpsign.Client, error) { + timeout := time.Duration(10) + + if allowedHostListValue == "" { + allowedHostListValue = host_matcher.MatchBuiltinExternal + } + allowedHostMatcher := host_matcher.ParseHostMatchList("WOODPECKER_ALLOWED_EXTENSIONS_HOSTS", allowedHostListValue) + + pubKeyID := "woodpecker-ci-extensions" + + ed25519Key, ok := privateKey.(ed25519.PrivateKey) + if !ok { + return nil, fmt.Errorf("invalid private key type") + } + + signer, err := httpsign.NewEd25519Signer(ed25519Key, + httpsign.NewSignConfig(), + httpsign.Headers("@request-target", "content-digest")) // The Content-Digest header will be auto-generated + if err != nil { + return nil, err + } + + client := http.Client{ + Timeout: timeout, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: false}, + DialContext: host_matcher.NewDialContext("extensions", allowedHostMatcher, nil), + }, + } + + config := httpsign.NewClientConfig().SetSignatureName(pubKeyID).SetSigner(signer) + + return httpsign.NewClient(client, config), nil +} + +func NewHTTPClient(privateKey crypto.PrivateKey, allowedHostList string) (*Client, error) { + client, err := getHTTPClient(privateKey, allowedHostList) + if err != nil { + return nil, err + } + + return &Client{ + Client: client, + }, nil +} + // Send makes an http request to the given endpoint, writing the input // to the request body and un-marshaling the output from the response body. -func Send(ctx context.Context, method, path string, privateKey ed25519.PrivateKey, in, out any) (int, error) { +func (e *Client) Send(ctx context.Context, method, path string, in, out any) (int, error) { uri, err := url.Parse(path) if err != nil { return 0, err @@ -54,18 +109,13 @@ func Send(ctx context.Context, method, path string, privateKey ed25519.PrivateKe req.Header.Set("Content-Type", "application/json") } - client, err := signClient(privateKey) - if err != nil { - return 0, err - } - - resp, err := client.Do(req) + resp, err := e.Do(req) if err != nil { return 0, err } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { body, err := io.ReadAll(resp.Body) if err != nil { return resp.StatusCode, err @@ -78,15 +128,3 @@ func Send(ctx context.Context, method, path string, privateKey ed25519.PrivateKe err = json.NewDecoder(resp.Body).Decode(out) return resp.StatusCode, err } - -func signClient(privateKey ed25519.PrivateKey) (*httpsign.Client, error) { - pubKeyID := "woodpecker-ci-extensions" - - signer, err := httpsign.NewEd25519Signer(privateKey, - httpsign.NewSignConfig(), - httpsign.Headers("@request-target", "content-digest")) // The Content-Digest header will be auto-generated - if err != nil { - return nil, err - } - return httpsign.NewDefaultClient(httpsign.NewClientConfig().SetSignatureName(pubKeyID).SetSigner(signer)), nil // sign requests, don't verify responses -} diff --git a/server/services/utils/http_test.go b/server/services/utils/http_test.go index 57d7eece29..495be6a884 100644 --- a/server/services/utils/http_test.go +++ b/server/services/utils/http_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package utils +package utils_test import ( "bytes" @@ -25,6 +25,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/yaronf/httpsign" + + "go.woodpecker-ci.org/woodpecker/v2/server/services/utils" ) func TestSignClient(t *testing.T) { @@ -59,7 +61,7 @@ func TestSignClient(t *testing.T) { req.Header.Set("Date", time.Now().Format(time.RFC3339)) req.Header.Set("Content-Type", "application/json") - client, err := signClient(privEd25519Key) + client, err := utils.NewHTTPClient(privEd25519Key, "loopback") if !assert.NoError(t, err) { return } diff --git a/web/components.d.ts b/web/components.d.ts index deb27eae3c..3a26df1c41 100644 --- a/web/components.d.ts +++ b/web/components.d.ts @@ -23,11 +23,13 @@ declare module 'vue' { Button: typeof import('./src/components/atomic/Button.vue')['default'] Checkbox: typeof import('./src/components/form/Checkbox.vue')['default'] CheckboxesField: typeof import('./src/components/form/CheckboxesField.vue')['default'] + CodeBox: typeof import('./src/components/layout/CodeBox.vue')['default'] Container: typeof import('./src/components/layout/Container.vue')['default'] CronTab: typeof import('./src/components/repo/settings/CronTab.vue')['default'] DeployPipelinePopup: typeof import('./src/components/layout/popups/DeployPipelinePopup.vue')['default'] DocsLink: typeof import('./src/components/atomic/DocsLink.vue')['default'] Error: typeof import('./src/components/atomic/Error.vue')['default'] + ExtensionsTab: typeof import('./src/components/repo/settings/ExtensionsTab.vue')['default'] GeneralTab: typeof import('./src/components/repo/settings/GeneralTab.vue')['default'] Header: typeof import('./src/components/layout/scaffold/Header.vue')['default'] IBiCheckCircleFill: typeof import('~icons/bi/check-circle-fill')['default'] diff --git a/web/src/assets/locales/en.json b/web/src/assets/locales/en.json index 3a1ec3313f..2c0f216c3b 100644 --- a/web/src/assets/locales/en.json +++ b/web/src/assets/locales/en.json @@ -488,10 +488,20 @@ "cli_login_failed": "Login to CLI failed", "cli_login_denied": "Login to CLI denied", "return_to_cli": "You can now close this tab and return to the CLI.", + "save": "Save", "settings": "Settings", "oauth_error": "Error while authenticating against OAuth provider", "internal_error": "Some internal error occurred", "registration_closed": "The registration is closed", "access_denied": "You are not allowed to access this instance", - "invalid_state": "The OAuth state is invalid" + "invalid_state": "The OAuth state is invalid", + "extensions": "Extensions", + "extensions_description": "Extensions are HTTP services that can be called by Woodpecker instead of using the builtin ones.", + "secrets_extension_endpoint": "Secrets extension endpoint", + "secrets_extension_alpha_state": "The secret extension is in alpha state and might change in the future.", + "extension_endpoint_placeholder": "e.g. https://example.com/api", + "config_extension_endpoint": "Config extension endpoint", + "extensions_signatures_public_key": "Public key for signatures", + "extensions_signatures_public_key_description": "This public key should be used by your extensions to verify webhook calls from Woodpecker.", + "extensions_configuration_saved": "Extensions configuration saved" } diff --git a/web/src/components/layout/CodeBox.vue b/web/src/components/layout/CodeBox.vue new file mode 100644 index 0000000000..0f820a4690 --- /dev/null +++ b/web/src/components/layout/CodeBox.vue @@ -0,0 +1,10 @@ + + + diff --git a/web/src/components/repo/settings/ExtensionsTab.vue b/web/src/components/repo/settings/ExtensionsTab.vue new file mode 100644 index 0000000000..b7671dd102 --- /dev/null +++ b/web/src/components/repo/settings/ExtensionsTab.vue @@ -0,0 +1,67 @@ + + + diff --git a/web/src/lib/api/index.ts b/web/src/lib/api/index.ts index a707419018..412f13e4c5 100644 --- a/web/src/lib/api/index.ts +++ b/web/src/lib/api/index.ts @@ -2,6 +2,7 @@ import ApiClient, { encodeQueryString } from './client'; import type { Agent, Cron, + ExtensionSettings, Forge, Org, OrgPermissions, @@ -73,7 +74,7 @@ export default class WoodpeckerClient extends ApiClient { return this._post(`/api/repos?forge_remote_id=${forgeRemoteId}`) as Promise; } - async updateRepo(repoId: number, repoSettings: RepoSettings): Promise { + async updateRepo(repoId: number, repoSettings: Partial): Promise { return this._patch(`/api/repos/${repoId}`, repoSettings); } @@ -305,6 +306,10 @@ export default class WoodpeckerClient extends ApiClient { return this._post('/api/user/token') as Promise; } + async getSignaturePublicKey(): Promise { + return this._get('/api/signature/public-key') as Promise; + } + async getAgents(opts?: PaginationOptions): Promise { const query = encodeQueryString(opts); return this._get(`/api/agents?${query}`) as Promise; diff --git a/web/src/lib/api/types/repo.ts b/web/src/lib/api/types/repo.ts index a6b73576fa..b407ffa414 100644 --- a/web/src/lib/api/types/repo.ts +++ b/web/src/lib/api/types/repo.ts @@ -73,6 +73,9 @@ export interface Repo { cancel_previous_pipeline_events: string[]; netrc_only_trusted: boolean; + + // Endpoint for config extensions + config_extension_endpoint: string; } /* eslint-disable no-unused-vars */ @@ -96,6 +99,8 @@ export type RepoSettings = Pick< | 'netrc_only_trusted' >; +export type ExtensionSettings = Pick; + export interface RepoPermissions { pull: boolean; push: boolean; diff --git a/web/src/views/repo/RepoSettings.vue b/web/src/views/repo/RepoSettings.vue index 8079019760..0610859568 100644 --- a/web/src/views/repo/RepoSettings.vue +++ b/web/src/views/repo/RepoSettings.vue @@ -31,6 +31,9 @@ + + + @@ -47,6 +50,7 @@ import Tab from '~/components/layout/scaffold/Tab.vue'; import ActionsTab from '~/components/repo/settings/ActionsTab.vue'; import BadgeTab from '~/components/repo/settings/BadgeTab.vue'; import CronTab from '~/components/repo/settings/CronTab.vue'; +import ExtensionsTab from '~/components/repo/settings/ExtensionsTab.vue'; import GeneralTab from '~/components/repo/settings/GeneralTab.vue'; import RegistriesTab from '~/components/repo/settings/RegistriesTab.vue'; import SecretsTab from '~/components/repo/settings/SecretsTab.vue';