diff --git a/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/OCI_TERMINOLOGY.md b/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/OCI_TERMINOLOGY.md index 78f22021e9c..b264000cb49 100644 --- a/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/OCI_TERMINOLOGY.md +++ b/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/OCI_TERMINOLOGY.md @@ -9,7 +9,10 @@ Given an OCI HelmRepository CRD with URL like `"oci://ghcr.io/stefanprodan/chart - `oci://` - URL scheme, indicating this is an *OCI chart repository*, as opposed to an *HTTP chart repository* - `ghcr.io` - OCI registry host - `/stefanprodan/charts` - registry path -- That repository may contain multiple charts, e.g. `"podinfo"` and `"nginx"`. Chart names are not part of the URL and must be specified by user when creating HelmChart or HelmRelease CRD +- That OCI registry may contain multiple helm chart repositories, such as `"podinfo"` and `"nginx"`. The associated OCI references would be: + - `oci://ghcr.io/stefanprodan/charts/podinfo` + - `oci://ghcr.io/stefanprodan/charts/nginx` +- Each of the repositories may only contain a single chart, whose name matches that of the repository basename. For example, repository with the basename (the last segment of the URL path) `"podinfo"` may only contain a single chart also called `"podinfo"`. Also see helm section below. - Each of the charts may have multiple versions a.k.a. tags, e.g. "`6.1.5"`, `"6.1.4"`, etc. References: @@ -18,7 +21,7 @@ References: - https://github.com/fluxcd/source-controller/blob/main/controllers/helmrepository_controller_oci.go ## ORAS v2 go libraries -Given a remote OCI registry, such as `ghcr.io`, will list all repository names hosted in the format `"{REGISTRY_PATH}/{NAME}"`. Unlike the flux section, REGISTRY_PATH does not begin with a slash. For example, assuming the remote registry with the URL `"oci://ghcr.io/stefanprodan/charts"` contains 2 repositories, `"podinfo"` and `"podinfo-2"`, then the following list is returned: +Given a remote OCI registry, such as `ghcr.io`, will list all repository names hosted in the format `"{REGISTRY_PATH}/{NAME}"`. Unlike the flux section, REGISTRY_PATH does not begin with a slash. For example, assuming the remote registry with the URL `"oci://ghcr.io/stefanprodan/charts"` contains 2 repositories, `"podinfo"` and `"podinfo-2"`, then the following list is returned from ORAS `Registry.Repositories()` API: 1. `"stefanprodan/charts/podinfo"` 2. `"stefanprodan/charts/podinfo-2"` @@ -27,8 +30,6 @@ References: - https://oras.land/ - https://github.com/oras-project/oras-go/blob/14422086e41897a44cb706726e687d39dc728805/registry/remote/url.go#L43 - - ## Helm One can login to or logout from a registry host, such as @@ -51,48 +52,45 @@ The registry reference basename is inferred from the chart's name, and the tag i Certain registries require the repository and/or namespace (if specified) to be created beforehand ``` -For example, once logged in, you may use the command ```"helm push"```, with repository URL like this: +From [helm HIPS spec](https://github.com/helm/community/blob/main/hips/hip-0006.md#4-chart-names--oci-reference-basenames): ``` -$ helm push podinfo-6.1.5.tgz oci://ghcr.io/gfichtenholt -Pushed: ghcr.io/gfichtenholt/podinfo:6.1.5 -Digest: sha256:80e6d2e7f6d21800621530fc4c5b70d0e458f11b2c05386ea5d058c4e86d6e93 +To keep things simple, the basename (the last segment of the URL path) on a registry reference should be equivalent to the chart name. + +For example, given a chart with the name pepper and the version 1.2.3, users may run a command such as the following: + +$ helm push pepper-1.2.3.tgz oci://r.myreg.io/mycharts + +which would result in the following reference: + +oci://r.myreg.io/mycharts/pepper:1.2.3 + +By placing such restrictions on registry URLs Helm users are less likely to do "strange things" with charts in registries ``` + In this case: - - a single repository named `"gfichtenholt/podinfo"` will be created if one does not exist - - the repository contains a chart named `"podinfo"` - - the chart `"podinfo"` has a version `"6.1.5"` + - a single repository named `"mycharts/pepper"` will be created if one does not exist + - the repository contains a chart named `"pepper"` + - the chart `"pepper"` has a version `"1.2.3"` -Or you may use ```"helm push"```, with repository URL like this: -``` -$ helm push podinfo-6.1.5.tgz oci://ghcr.io/gfichtenholt/charts -Pushed: ghcr.io/gfichtenholt/charts/podinfo:6.1.5 -Digest: sha256:80e6d2e7f6d21800621530fc4c5b70d0e458f11b2c05386ea5d058c4e86d6e93 +You can use the command ```helm show all``` to see (some) information about the `"pepper"` chart: ``` -In this case the repository name is `"gfichtenholt/charts/podinfo"` while the rest is as above. You can use the command ```helm show all``` to see (some) information about the `"podinfo"` chart: -``` -$ helm show all oci://ghcr.io/gfichtenholt/charts/podinfo | head -9 +$ helm show all oci://r.myreg.io/mycharts/pepper | head -9 apiVersion: v1 -appVersion: 6.1.5 -description: Podinfo Helm chart for Kubernetes -home: https://github.com/stefanprodan/podinfo +appVersion: 1.2.3 +description: ... +home: ... kubeVersion: '>=1.19.0-0' maintainers: - email: stefanprodan@users.noreply.github.com name: stefanprodan -name: podinfo +name: pepper ... ``` -Or you may use -``` -helm push podinfo-6.1.5.tgz oci://ghcr.io/gfichtenholt/charts/podinfo -Pushed: ghcr.io/gfichtenholt/charts/podinfo/podinfo:6.1.5 -Digest: sha256:80e6d2e7f6d21800621530fc4c5b70d0e458f11b2c05386ea5d058c4e86d6e93 -``` -In this case, the repository name is `"gfichtenholt/charts/podinfo/podinfo"`, while the rest is the same. And so on and so forth. References: - https://helm.sh/blog/storing-charts-in-oci/ - https://helm.sh/docs/topics/registries/ + - https://github.com/helm/community/blob/main/hips/hip-0006.md#specification ## GitHub Container Registry `ghcr.io` Take an OCI registry URL like `"oci://ghcr.io/gfichtenholt/charts/podinfo:6.1.5"` @@ -120,5 +118,5 @@ Here is probably the most confusing part of the whole document: 2. Assume the remote OCI registry contains a single chart `"podinfo"` with version `"6.1.5"` 3. ORAS go library will return repository list `["gfichtenholt/helm-charts/podinfo"]` 4. kubeapps flux plugin will call `RegistryClient.Tags()` with respect to OCI reference `"ghcr.io/gfichtenholt/helm-charts/podinfo"` which will return `["6.1.5"]` - 5. kubeapps flux plugin will call `RegistryClient.DownloadChart()` with respect to a chart with version `"6.1.5"` a URL `"ghcr.io/gfichtenholt/helm-charts/podinfo:6.1.5"`. Here, the identifier `"podinfo"` refers BOTH to repository name AND the chart name! + 5. kubeapps flux plugin will call `RegistryClient.DownloadChart()` with respect to a chart with version `"6.1.5"` a URL `"ghcr.io/gfichtenholt/helm-charts/podinfo:6.1.5"`. Here, the identifier `"podinfo"` refers **BOTH to repository basename AND the chart name!** --- diff --git a/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/global_vars_test.go b/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/global_vars_test.go index da3002e9ef5..fcc7142ec5c 100644 --- a/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/global_vars_test.go +++ b/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/global_vars_test.go @@ -3507,14 +3507,11 @@ var ( repositories: []fakeRepo{ { name: "podinfo", - charts: []fakeChart{ - { - name: "podinfo", - versions: []fakeChartVersion{ - { - version: "6.1.5", - tgzBytes: chartBytes, - }, + chart: fakeChart{ + versions: []fakeChartVersion{ + { + version: "6.1.5", + tgzBytes: chartBytes, }, }, }, @@ -3536,6 +3533,30 @@ var ( }, } + oci_repo_available_package_summaries_2 = []*corev1.AvailablePackageSummary{ + { + Name: "podinfo", + DisplayName: "podinfo", + LatestVersion: &corev1.PackageAppVersion{ + PkgVersion: "6.1.5", + }, + AvailablePackageRef: availableRef("repo-1/podinfo", "namespace-1"), + Categories: []string{""}, + ShortDescription: "Podinfo Helm chart for Kubernetes", + }, + { + Name: "airflow", + DisplayName: "airflow", + LatestVersion: &corev1.PackageAppVersion{ + PkgVersion: "6.7.1", + }, + IconUrl: "https://bitnami.com/assets/stacks/airflow/img/airflow-stack-110x117.png", + AvailablePackageRef: availableRef("repo-1/airflow", "namespace-1"), + Categories: []string{"WorkFlow"}, + ShortDescription: "Apache Airflow is a platform to programmatically author, schedule and monitor workflows.", + }, + } + oci_podinfo_charts_spec = []testSpecChartWithUrl{ { chartID: "repo-1/podinfo", @@ -3605,22 +3626,56 @@ var ( repositories: []fakeRepo{ { name: "podinfo", - charts: []fakeChart{ - { - name: "podinfo", - versions: []fakeChartVersion{ - { - version: "6.1.5", - tgzBytes: chartBytes1, - }, - { - version: "6.0.0", - tgzBytes: chartBytes2, - }, - { - version: "6.0.3", - tgzBytes: chartBytes3, - }, + chart: fakeChart{ + versions: []fakeChartVersion{ + { + version: "6.1.5", + tgzBytes: chartBytes1, + }, + { + version: "6.0.0", + tgzBytes: chartBytes2, + }, + { + version: "6.0.3", + tgzBytes: chartBytes3, + }, + }, + }, + }, + }, + }, nil + } + + newFakeRemoteOciRegistryData_3 = func() (*fakeRemoteOciRegistryData, error) { + chartBytes1, err := ioutil.ReadFile(testTgz("podinfo-6.1.5.tgz")) + if err != nil { + return nil, err + } + chartBytes2, err := ioutil.ReadFile(testTgz("airflow-6.7.1.tgz")) + if err != nil { + return nil, err + } + return &fakeRemoteOciRegistryData{ + repositories: []fakeRepo{ + { + name: "podinfo", + chart: fakeChart{ + versions: []fakeChartVersion{ + { + version: "6.1.5", + tgzBytes: chartBytes1, + }, + }, + }, + }, + { + name: "airflow", + chart: fakeChart{ + versions: []fakeChartVersion{ + { + version: "6.7.1", + tgzBytes: chartBytes2, }, }, }, diff --git a/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/oci_repo.go b/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/oci_repo.go index 0b5bf4540e1..4d14b29f577 100644 --- a/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/oci_repo.go +++ b/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/oci_repo.go @@ -397,7 +397,11 @@ func (s *repoEventSink) onAddOciRepo(repo sourcev1.HelmRepository) ([]byte, bool Type: repo.Spec.Type, } - // repository names a.k.a. application names, e.g. "stefanprodan/charts/podinfo" + // repository names, e.g. "stefanprodan/charts/podinfo" + // asset-syncer calls them appNames + // see func (r *OCIRegistry) Charts(fetchLatestOnly bool) ([]models.Chart, error) { + // also per https://github.com/helm/community/blob/main/hips/hip-0006.md#4-chart-names--oci-reference-basenames + // appName == chartName == the basename (the last segment of the URL path) on a registry reference appNames, err := ociChartRepo.listRepositoryNames() if err != nil { return nil, false, err diff --git a/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/oci_repo_test.go b/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/oci_repo_test.go index 9813d1142e5..80ac4e8286b 100644 --- a/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/oci_repo_test.go +++ b/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/oci_repo_test.go @@ -80,19 +80,13 @@ func (fake *fakeRegistryClientType) Tags(ref string) ([]string, error) { return nil, fmt.Errorf("ref [%s] is not in expected format", ref) } refRepository := parts[2] - refChart := refRepository // just for now for _, r := range fake.repositories { if refRepository == r.name { - for _, c := range r.charts { - if refChart == c.name { - tags := []string{} - for _, cv := range c.versions { - tags = append(tags, cv.version) - } - return tags, nil - } + tags := []string{} + for _, cv := range r.chart.versions { + tags = append(tags, cv.version) } - return nil, fmt.Errorf("no chart named [%s] found for repository [%s] in the registry", refChart, refRepository) + return tags, nil } } return nil, fmt.Errorf("no repositories found for ref [%s] in the registry", ref) @@ -104,18 +98,13 @@ func (fake *fakeRegistryClientType) DownloadChart(chartVersion *repo.ChartVersio // see OCI_TERMINOLOGY.md repoName := chartVersion.Name for _, r := range fake.repositories { - if repoName == r.name { - for _, c := range r.charts { - if chartVersion.Name == c.name { - for _, v := range c.versions { - if chartVersion.Version == v.version { - return bytes.NewBuffer(v.tgzBytes), nil - } - } - return nil, fmt.Errorf("no version [%s] found for chart [%s]", chartVersion.Version, chartVersion.Name) + if repoName == r.name && chartVersion.Name == r.name { + for _, v := range r.chart.versions { + if chartVersion.Version == v.version { + return bytes.NewBuffer(v.tgzBytes), nil } } - return nil, fmt.Errorf("no charts named [%s] found in the repository [%s]", chartVersion.Name, repoName) + return nil, fmt.Errorf("no version [%s] found for chart [%s]", chartVersion.Version, chartVersion.Name) } } return nil, fmt.Errorf("no repositories named [%s] found in the registry", repoName) @@ -134,13 +123,13 @@ type fakeChartVersion struct { } type fakeChart struct { - name string versions []fakeChartVersion } type fakeRepo struct { - name string - charts []fakeChart + name string + // see OCI_TERMINOLOGY.md. Only a single chart is allowed + chart fakeChart } type fakeRemoteOciRegistryData struct { diff --git a/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/repo_test.go b/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/repo_test.go index 0640c8042e7..eea018c6404 100644 --- a/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/repo_test.go +++ b/cmd/kubeapps-apis/plugins/fluxv2/packages/v1alpha1/repo_test.go @@ -2273,17 +2273,21 @@ func TestDeletePackageRepository(t *testing.T) { } func TestGetOciAvailablePackageSummariesWithoutPagination(t *testing.T) { - type testSpecGetOciAvailablePackageSummaries struct { - repoName string - repoNamespace string - repoUrl string + seed_data_1, err := newFakeRemoteOciRegistryData_1() + if err != nil { + t.Fatal(err) } - data, err := newFakeRemoteOciRegistryData_1() + seed_data_3, err := newFakeRemoteOciRegistryData_3() if err != nil { t.Fatal(err) } - initOciFakeClientBuilder(t, *data) + + type testSpecGetOciAvailablePackageSummaries struct { + repoName string + repoNamespace string + repoUrl string + } testCases := []struct { name string @@ -2291,6 +2295,7 @@ func TestGetOciAvailablePackageSummariesWithoutPagination(t *testing.T) { repos []testSpecGetOciAvailablePackageSummaries expectedResponse *corev1.GetAvailablePackageSummariesResponse expectedErrorCode codes.Code + seedData *fakeRemoteOciRegistryData }{ { name: "returns a single available package", @@ -2306,11 +2311,30 @@ func TestGetOciAvailablePackageSummariesWithoutPagination(t *testing.T) { AvailablePackageSummaries: oci_repo_available_package_summaries, }, expectedErrorCode: codes.OK, + seedData: seed_data_1, + }, + { + name: "returns available packages from multiple repos", + repos: []testSpecGetOciAvailablePackageSummaries{ + { + repoName: "repo-1", + repoNamespace: "namespace-1", + repoUrl: "oci://localhost:54321/userX/charts", + }, + }, + request: &corev1.GetAvailablePackageSummariesRequest{Context: &corev1.Context{}}, + expectedResponse: &corev1.GetAvailablePackageSummariesResponse{ + AvailablePackageSummaries: oci_repo_available_package_summaries_2, + }, + expectedErrorCode: codes.OK, + seedData: seed_data_3, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + initOciFakeClientBuilder(t, *tc.seedData) + repos := []sourcev1.HelmRepository{} for _, rs := range tc.repos {