Skip to content

Commit

Permalink
Modify clickhouse statistic related func for theia-manager (#132)
Browse files Browse the repository at this point in the history
1. Define REST request and return payload struct for related API endpoints and add code generation under pkg/apis/stats
    a. REST endpoint apis/stats.theia.antrea.io/v1alpha1/clickhouse
2. Adapt CLI to use Theia Manager
3. Modify e2e test
  • Loading branch information
yuntanghsu authored Dec 1, 2022
1 parent 5a15bb8 commit b6c1c1f
Show file tree
Hide file tree
Showing 29 changed files with 1,505 additions and 692 deletions.
6 changes: 6 additions & 0 deletions build/charts/theia/templates/theia-cli/clusterrole.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,10 @@ rules:
- list
- create
- delete
- apiGroups:
- stats.theia.antrea.io
resources:
- clickhouse
verbs:
- get
{{- end }}
2 changes: 1 addition & 1 deletion ci/jenkins/test-vmc.sh
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ function deliver_antrea {

${GIT_CHECKOUT_DIR}/hack/generate-manifest.sh --ch-size 100Mi --ch-monitor-threshold 0.1 > ${GIT_CHECKOUT_DIR}/build/yamls/flow-visibility.yml
${GIT_CHECKOUT_DIR}/hack/generate-manifest.sh --no-grafana --spark-operator --theia-manager > ${GIT_CHECKOUT_DIR}/build/yamls/flow-visibility-with-spark.yml
${GIT_CHECKOUT_DIR}/hack/generate-manifest.sh --no-grafana > ${GIT_CHECKOUT_DIR}/build/yamls/flow-visibility-ch-only.yml
${GIT_CHECKOUT_DIR}/hack/generate-manifest.sh --no-grafana --theia-manager > ${GIT_CHECKOUT_DIR}/build/yamls/flow-visibility-ch-only.yml

# policy/v1beta1 is deprecated in v1.21+, unavailable in v1.25+, while policy/v1 is available in v1.21+
sed -i -e "s|apiVersion: policy/v1|apiVersion: policy/v1beta1|g" ${GIT_CHECKOUT_DIR}/build/yamls/flow-visibility.yml
Expand Down
2 changes: 1 addition & 1 deletion ci/kind/test-e2e-kind.sh
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ TESTBED_CMD=$(dirname $0)"/kind-setup.sh"
YML_DIR=$(dirname $0)"/../../build/yamls"
FLOW_VISIBILITY_CMD=$(dirname $0)"/../../hack/generate-manifest.sh --ch-size 100Mi --ch-monitor-threshold 0.1"
FLOW_VISIBILITY_WITH_SPARK_CMD=$(dirname $0)"/../../hack/generate-manifest.sh --no-grafana --spark-operator --theia-manager"
FLOW_VISIBILITY_CH_ONLY_CMD=$(dirname $0)"/../../hack/generate-manifest.sh --no-grafana"
FLOW_VISIBILITY_CH_ONLY_CMD=$(dirname $0)"/../../hack/generate-manifest.sh --no-grafana --theia-manager"
CH_OPERATOR_YML=$(dirname $0)"/../../build/charts/theia/crds/clickhouse-operator-install-bundle.yaml"

make theia-linux
Expand Down
12 changes: 9 additions & 3 deletions cmd/theia-manager/theia-manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (

"antrea.io/theia/pkg/apiserver"
"antrea.io/theia/pkg/apiserver/certificate"
"antrea.io/theia/pkg/apiserver/utils/stats"
crdclientset "antrea.io/theia/pkg/client/clientset/versioned"
crdinformers "antrea.io/theia/pkg/client/informers/externalversions"
"antrea.io/theia/pkg/controller/networkpolicyrecommendation"
Expand All @@ -50,7 +51,9 @@ func createAPIServerConfig(
bindPort int,
cipherSuites []uint16,
tlsMinVersion uint16,
nprq querier.NPRecommendationQuerier) (*apiserver.Config, error) {
nprq querier.NPRecommendationQuerier,
chq querier.ClickHouseStatQuerier,
) (*apiserver.Config, error) {
secureServing := genericoptions.NewSecureServingOptions().WithLoopback()
authentication := genericoptions.NewDelegatingAuthenticationOptions()
authorization := genericoptions.NewDelegatingAuthorizationOptions()
Expand Down Expand Up @@ -90,7 +93,8 @@ func createAPIServerConfig(
serverConfig,
client,
caCertController,
nprq), nil
nprq,
chq), nil
}

func run(o *Options) error {
Expand Down Expand Up @@ -123,6 +127,7 @@ func run(o *Options) error {
npRecommendationInformer := crdInformerFactory.Crd().V1alpha1().NetworkPolicyRecommendations()
recommendedNPInformer := crdInformerFactory.Crd().V1alpha1().RecommendedNetworkPolicies()
npRecoController := networkpolicyrecommendation.NewNPRecommendationController(crdClient, kubeClient, npRecommendationInformer, recommendedNPInformer)
clickHouseStatQuerierImpl := stats.NewClickHouseStatQuerierImpl(kubeClient)

cipherSuites, err := cipher.GenerateCipherSuitesList(o.config.APIServer.TLSCipherSuites)
if err != nil {
Expand All @@ -135,7 +140,8 @@ func run(o *Options) error {
o.config.APIServer.APIPort,
cipherSuites,
cipher.TLSVersionMap[o.config.APIServer.TLSMinVersion],
npRecoController)
npRecoController,
clickHouseStatQuerierImpl)
if err != nil {
return fmt.Errorf("error creating API server config: %v", err)
}
Expand Down
1 change: 1 addition & 0 deletions hack/update-codegen-dockerized.sh
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ $GOPATH/bin/informer-gen \
$GOPATH/bin/deepcopy-gen \
--input-dirs "${THEIA_PKG}/pkg/apis/intelligence/v1alpha1" \
--input-dirs "${THEIA_PKG}/pkg/apis/crd/v1alpha1" \
--input-dirs "${THEIA_PKG}/pkg/apis/stats/v1alpha1" \
-O zz_generated.deepcopy \
--go-header-file hack/boilerplate/license_header.go.txt

Expand Down
Original file line number Diff line number Diff line change
@@ -1,35 +1,28 @@
// Copyright 2022 Antrea Authors.
// Copyright 2022 Antrea Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package policyrecommendation
package install

import (
"fmt"
"strings"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"

"github.com/google/uuid"
"antrea.io/theia/pkg/apis/stats/v1alpha1"
)

func ParseRecommendationName(npName string) error {
if !strings.HasPrefix(npName, "pr-") {
return fmt.Errorf("input name %s is not a valid policy recommendation job name", npName)

}
id := npName[3:]
_, err := uuid.Parse(id)
if err != nil {
return fmt.Errorf("input name %s does not contain a valid UUID, parsing error: %v", npName, err)
}
return nil
// Install registers the API group and adds types to a scheme
func Install(scheme *runtime.Scheme) {
utilruntime.Must(v1alpha1.AddToScheme(scheme))
utilruntime.Must(scheme.SetVersionPriority(v1alpha1.SchemeGroupVersion))
}
20 changes: 20 additions & 0 deletions pkg/apis/stats/v1alpha1/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Copyright 2022 Antrea Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +k8s:openapi-gen=true
// +k8s:deepcopy-gen=package
// +k8s:defaulter-gen=TypeMeta
// +groupName=intelligence.theia.antrea.io

package v1alpha1
58 changes: 58 additions & 0 deletions pkg/apis/stats/v1alpha1/register.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// Copyright 2022 Antrea Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1alpha1

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
)

const GroupName = "stats.theia.antrea.io"

var (
SchemeGroupVersion = schema.GroupVersion{
Group: GroupName,
Version: "v1alpha1"}

StatusResource = schema.GroupVersionResource{
Group: SchemeGroupVersion.Group,
Version: SchemeGroupVersion.Version,
Resource: "clickhouse"}
)

var (
SchemeBuilder runtime.SchemeBuilder
localSchemeBuilder = &SchemeBuilder
AddToScheme = localSchemeBuilder.AddToScheme
)

func init() {
localSchemeBuilder.Register(addKnownTypes)
}

func Resource(resource string) schema.GroupResource {
return SchemeGroupVersion.WithResource(resource).GroupResource()
}

func addKnownTypes(scheme *runtime.Scheme) error {
scheme.AddKnownTypes(
SchemeGroupVersion,
&ClickHouseStats{},
)

metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
return nil
}
64 changes: 64 additions & 0 deletions pkg/apis/stats/v1alpha1/types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// Copyright 2022 Antrea Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1alpha1

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// +genclient
// +genclient:nonNamespaced
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

type ClickHouseStats struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

DiskInfos []DiskInfo `json:"diskInfos,omitempty"`
TableInfos []TableInfo `json:"tableInfos,omitempty"`
InsertRates []InsertRate `json:"insertRates,omitempty"`
StackTraces []StackTrace `json:"stackTraces,omitempty"`
ErrorMsg []string `json:"errorMsg,omitempty"`
}

type DiskInfo struct {
Shard string `json:"shard,omitempty"`
Database string `json:"name,omitempty"`
Path string `json:"path,omitempty"`
FreeSpace string `json:"freeSpace,omitempty"`
TotalSpace string `json:"totalSpace,omitempty"`
UsedPercentage string `json:"usedPercentage,omitempty"`
}

type TableInfo struct {
Shard string `json:"shard,omitempty"`
Database string `json:"database,omitempty"`
TableName string `json:"tableName,omitempty"`
TotalRows string `json:"totalRows,omitempty"`
TotalBytes string `json:"totalBytes,omitempty"`
TotalCols string `json:"totalCols,omitempty"`
}

type InsertRate struct {
Shard string `json:"shard,omitempty"`
RowsPerSec string `json:"rowsPerSec,omitempty"`
BytesPerSec string `json:"bytesPerSec,omitempty"`
}

type StackTrace struct {
Shard string `json:"shard,omitempty"`
TraceFunctions string `json:"traceFunctions,omitempty"`
Count string `json:"count,omitempty"`
}
Loading

0 comments on commit b6c1c1f

Please sign in to comment.