From 4a31c5e69432bf8273534acb35f5bc0e05036800 Mon Sep 17 00:00:00 2001 From: Chlins Zhang Date: Mon, 9 Dec 2024 10:43:21 +0800 Subject: [PATCH] feat: add prefetch rate limit configuration (#336) Signed-off-by: chlins --- charts/dragonfly/Chart.yaml | 16 ++++++++-------- charts/dragonfly/README.md | 12 +++++++----- charts/dragonfly/values.yaml | 16 +++++++++++----- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/charts/dragonfly/Chart.yaml b/charts/dragonfly/Chart.yaml index 6998ab36..b8e2414e 100644 --- a/charts/dragonfly/Chart.yaml +++ b/charts/dragonfly/Chart.yaml @@ -3,8 +3,8 @@ name: dragonfly description: Dragonfly is an intelligent P2P based image and file distribution system icon: https://raw.githubusercontent.com/dragonflyoss/Dragonfly2/main/docs/images/logo/dragonfly.svg type: application -version: 1.2.24 -appVersion: 2.1.64 +version: 1.2.25 +appVersion: 2.1.65 keywords: - dragonfly - d7y @@ -27,7 +27,7 @@ sources: annotations: artifacthub.io/changes: | - - dfinit support docker runtime. + - Support to confiure the rate limit for prefetch. artifacthub.io/links: | - name: Chart Source @@ -38,15 +38,15 @@ annotations: url: https://github.com/dragonflyoss/client artifacthub.io/images: | - name: manager - image: dragonflyoss/manager:v2.1.64 + image: dragonflyoss/manager:v2.1.65 - name: scheduler - image: dragonflyoss/scheduler:v2.1.64 + image: dragonflyoss/scheduler:v2.1.65 - name: client - image: dragonflyoss/client:v0.1.118 + image: dragonflyoss/client:v0.1.124 - name: seed-client - image: dragonflyoss/client:v0.1.118 + image: dragonflyoss/client:v0.1.124 - name: dfinit - image: dragonflyoss/dfinit:v0.1.118 + image: dragonflyoss/dfinit:v0.1.124 dependencies: - name: mysql diff --git a/charts/dragonfly/README.md b/charts/dragonfly/README.md index ed4cdec6..7e4e5a6f 100644 --- a/charts/dragonfly/README.md +++ b/charts/dragonfly/README.md @@ -144,6 +144,7 @@ helm delete dragonfly --namespace dragonfly-system | client.config.metrics.server.port | int | `4002` | port is the port to the metrics server. | | client.config.proxy.disableBackToSource | bool | `false` | disableBackToSource indicates whether disable to download back-to-source when download failed. | | client.config.proxy.prefetch | bool | `false` | prefetch pre-downloads full of the task when download with range request. `X-Dragonfly-Prefetch` header's priority is higher than prefetch in config. If the value is "true", the range request will prefetch the entire file. If the value is "false", the range request will fetch the range content. | +| client.config.proxy.prefetchRateLimit | string | `"2GiB"` | prefetchRateLimit is the rate limit of prefetching in GiB/Mib/Kib per second, default is 2GiB/s. The prefetch request has lower priority so limit the rate to avoid occupying the bandwidth impact other download tasks. | | client.config.proxy.readBufferSize | int | `32768` | readBufferSize is the buffer size for reading piece from disk, default is 32KB. | | client.config.proxy.registryMirror.addr | string | `"https://index.docker.io"` | addr is the default address of the registry mirror. Proxy will start a registry mirror service for the client to pull the image. The client can use the default address of the registry mirror in configuration to pull the image. The `X-Dragonfly-Registry` header can instead of the default address of registry mirror. | | client.config.proxy.rules | list | `[{"regex":"blobs/sha256.*"}]` | rules is the list of rules for the proxy server. regex is the regex of the request url. useTLS indicates whether use tls for the proxy backend. redirect is the redirect url. filteredQueryParams is the filtered query params to generate the task id. When filter is ["Signature", "Expires", "ns"], for example: http://example.com/xyz?Expires=e1&Signature=s1&ns=docker.io and http://example.com/xyz?Expires=e2&Signature=s2&ns=docker.io will generate the same task id. Default value includes the filtered query params of s3, gcs, oss, obs, cos. `X-Dragonfly-Use-P2P` header can instead of the regular expression of the rule. If the value is "true", the request will use P2P technology to distribute the content. If the value is "false", but url matches the regular expression in rules. The request will also use P2P technology to distribute the content. | @@ -173,7 +174,7 @@ helm delete dragonfly --namespace dragonfly-system | client.dfinit.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. | | client.dfinit.image.registry | string | `"docker.io"` | Image registry. | | client.dfinit.image.repository | string | `"dragonflyoss/dfinit"` | Image repository. | -| client.dfinit.image.tag | string | `"v0.1.118"` | Image tag. | +| client.dfinit.image.tag | string | `"v0.1.124"` | Image tag. | | client.enable | bool | `true` | Enable client. | | client.extraVolumeMounts | list | `[{"mountPath":"/var/lib/dragonfly/","name":"storage"},{"mountPath":"/var/log/dragonfly/dfdaemon/","name":"logs"}]` | Extra volumeMounts for dfdaemon. | | client.extraVolumes | list | `[{"hostPath":{"path":"/var/lib/dragonfly/","type":"DirectoryOrCreate"},"name":"storage"},{"emptyDir":{},"name":"logs"}]` | Extra volumes for dfdaemon. | @@ -187,7 +188,7 @@ helm delete dragonfly --namespace dragonfly-system | client.image.pullSecrets | list | `[]` (defaults to global.imagePullSecrets). | Image pull secrets. | | client.image.registry | string | `"docker.io"` | Image registry. | | client.image.repository | string | `"dragonflyoss/client"` | Image repository. | -| client.image.tag | string | `"v0.1.118"` | Image tag. | +| client.image.tag | string | `"v0.1.124"` | Image tag. | | client.initContainer.image.digest | string | `""` | Image digest. | | client.initContainer.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. | | client.initContainer.image.registry | string | `"docker.io"` | Image registry. | @@ -290,7 +291,7 @@ helm delete dragonfly --namespace dragonfly-system | manager.image.pullSecrets | list | `[]` (defaults to global.imagePullSecrets). | Image pull secrets. | | manager.image.registry | string | `"docker.io"` | Image registry. | | manager.image.repository | string | `"dragonflyoss/manager"` | Image repository. | -| manager.image.tag | string | `"v2.1.64"` | Image tag. | +| manager.image.tag | string | `"v2.1.65"` | Image tag. | | manager.ingress.annotations | object | `{}` | Ingress annotations. | | manager.ingress.className | string | `""` | Ingress class name. Requirement: kubernetes >=1.18. | | manager.ingress.enable | bool | `false` | Enable ingress. | @@ -394,7 +395,7 @@ helm delete dragonfly --namespace dragonfly-system | scheduler.image.pullSecrets | list | `[]` (defaults to global.imagePullSecrets). | Image pull secrets. | | scheduler.image.registry | string | `"docker.io"` | Image registry. | | scheduler.image.repository | string | `"dragonflyoss/scheduler"` | Image repository. | -| scheduler.image.tag | string | `"v2.1.64"` | Image tag. | +| scheduler.image.tag | string | `"v2.1.65"` | Image tag. | | scheduler.initContainer.image.digest | string | `""` | Image digest. | | scheduler.initContainer.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. | | scheduler.initContainer.image.registry | string | `"docker.io"` | Image registry. | @@ -446,6 +447,7 @@ helm delete dragonfly --namespace dragonfly-system | seedClient.config.metrics.server.port | int | `4002` | port is the port to the metrics server. | | seedClient.config.proxy.disableBackToSource | bool | `false` | disableBackToSource indicates whether disable to download back-to-source when download failed. | | seedClient.config.proxy.prefetch | bool | `false` | prefetch pre-downloads full of the task when download with range request. `X-Dragonfly-Prefetch` header's priority is higher than prefetch in config. If the value is "true", the range request will prefetch the entire file. If the value is "false", the range request will fetch the range content. | +| seedClient.config.proxy.prefetchRateLimit | string | `"2GiB"` | prefetchRateLimit is the rate limit of prefetching in GiB/Mib/Kib per second, default is 2GiB/s. The prefetch request has lower priority so limit the rate to avoid occupying the bandwidth impact other download tasks. | | seedClient.config.proxy.readBufferSize | int | `32768` | readBufferSize is the buffer size for reading piece from disk, default is 32KB. | | seedClient.config.proxy.registryMirror.addr | string | `"https://index.docker.io"` | addr is the default address of the registry mirror. Proxy will start a registry mirror service for the client to pull the image. The client can use the default address of the registry mirror in configuration to pull the image. The `X-Dragonfly-Registry` header can instead of the default address of registry mirror. | | seedClient.config.proxy.rules | list | `[{"regex":"blobs/sha256.*"}]` | rules is the list of rules for the proxy server. regex is the regex of the request url. useTLS indicates whether use tls for the proxy backend. redirect is the redirect url. filteredQueryParams is the filtered query params to generate the task id. When filter is ["Signature", "Expires", "ns"], for example: http://example.com/xyz?Expires=e1&Signature=s1&ns=docker.io and http://example.com/xyz?Expires=e2&Signature=s2&ns=docker.io will generate the same task id. Default value includes the filtered query params of s3, gcs, oss, obs, cos. `X-Dragonfly-Use-P2P` header can instead of the regular expression of the rule. If the value is "true", the request will use P2P technology to distribute the content. If the value is "false", but url matches the regular expression in rules. The request will also use P2P technology to distribute the content. | @@ -477,7 +479,7 @@ helm delete dragonfly --namespace dragonfly-system | seedClient.image.pullSecrets | list | `[]` (defaults to global.imagePullSecrets). | Image pull secrets. | | seedClient.image.registry | string | `"docker.io"` | Image registry. | | seedClient.image.repository | string | `"dragonflyoss/client"` | Image repository. | -| seedClient.image.tag | string | `"v0.1.118"` | Image tag. | +| seedClient.image.tag | string | `"v0.1.124"` | Image tag. | | seedClient.initContainer.image.digest | string | `""` | Image digest. | | seedClient.initContainer.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. | | seedClient.initContainer.image.registry | string | `"docker.io"` | Image registry. | diff --git a/charts/dragonfly/values.yaml b/charts/dragonfly/values.yaml index 0751f170..c5fed48b 100644 --- a/charts/dragonfly/values.yaml +++ b/charts/dragonfly/values.yaml @@ -39,7 +39,7 @@ manager: # -- Image repository. repository: dragonflyoss/manager # -- Image tag. - tag: v2.1.64 + tag: v2.1.65 # -- Image digest. digest: "" # -- Image pull policy. @@ -333,7 +333,7 @@ scheduler: # -- Image repository. repository: dragonflyoss/scheduler # -- Image tag. - tag: v2.1.64 + tag: v2.1.65 # -- Image digest. digest: "" # -- Image pull policy. @@ -712,7 +712,7 @@ seedClient: # -- Image repository. repository: dragonflyoss/client # -- Image tag. - tag: v0.1.118 + tag: v0.1.124 # -- Image digest. digest: "" # -- Image pull policy. @@ -985,6 +985,9 @@ seedClient: # If the value is "true", the range request will prefetch the entire file. # If the value is "false", the range request will fetch the range content. prefetch: false + # -- prefetchRateLimit is the rate limit of prefetching in GiB/Mib/Kib per second, default is 2GiB/s. + # The prefetch request has lower priority so limit the rate to avoid occupying the bandwidth impact other download tasks. + prefetchRateLimit: 2GiB # -- readBufferSize is the buffer size for reading piece from disk, default is 32KB. readBufferSize: 32768 health: @@ -1080,7 +1083,7 @@ client: # -- Image repository. repository: dragonflyoss/client # -- Image tag. - tag: v0.1.118 + tag: v0.1.124 # -- Image digest. digest: "" # -- Image pull policy. @@ -1163,7 +1166,7 @@ client: # -- Image repository. repository: dragonflyoss/dfinit # -- Image tag. - tag: v0.1.118 + tag: v0.1.124 # -- Image digest. digest: "" # -- Image pull policy. @@ -1416,6 +1419,9 @@ client: # If the value is "true", the range request will prefetch the entire file. # If the value is "false", the range request will fetch the range content. prefetch: false + # -- prefetchRateLimit is the rate limit of prefetching in GiB/Mib/Kib per second, default is 2GiB/s. + # The prefetch request has lower priority so limit the rate to avoid occupying the bandwidth impact other download tasks. + prefetchRateLimit: 2GiB # -- readBufferSize is the buffer size for reading piece from disk, default is 32KB. readBufferSize: 32768 health: