From 7ff5896c88881a579197d56525535b5772d1bd11 Mon Sep 17 00:00:00 2001 From: raghupathi Date: Tue, 20 Feb 2024 08:51:27 +0530 Subject: [PATCH 01/21] helm chart for ml-analytics --- .../templates/ml-analytics-service.conf | 265 ++++++++++++++++++ .../core/ml-analytics-service/.helmignore | 23 ++ .../core/ml-analytics-service/Chart.yaml | 24 ++ .../ml-analytics-service/templates/NOTES.txt | 22 ++ .../templates/_helpers.tpl | 62 ++++ .../templates/configmap.yaml | 11 + .../templates/deployment.yaml | 47 ++++ .../templates/service.yaml | 14 + .../core/ml-analytics-service/values.yaml | 82 ++++++ 9 files changed, 550 insertions(+) create mode 100644 ansible/roles/stack-sunbird/templates/ml-analytics-service.conf create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/.helmignore create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/Chart.yaml create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/templates/NOTES.txt create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/templates/_helpers.tpl create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/templates/configmap.yaml create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/templates/service.yaml create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/values.yaml diff --git a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf new file mode 100644 index 0000000000..935085d36a --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf @@ -0,0 +1,265 @@ +[ML_APP_NAME] + +survey_app = {{ ml_analytics_survey_app_name }} + +integrated_app = {{ ml_analytics_integrated_app_name }} + +integrated_portal = {{ ml_analytics_integrated_portal }} + + +[API_ENDPOINTS] + +base_url = {{ ml_analytics_api_base_url }} + + +[API_HEADERS] + +content_type = application/json + +authorization = {{ ml_analytics_api_authorization_key }} + +internal_access_token = {{ ml_analytics_api_access_token }} + +authorization_access_token = {{ ml_analytics_authorization_access_token }} + +[ML_SURVEY_SERVICE_URL] + +url = {{ ml_analytics_survey_service }} + +entity_related_end_point = assessment/api/v1/entities/relatedEntities/ + +user_profile_end_point = assessment/api/v1/userExtension/getProfile/ + +evidence_base_url = {{ ml_analytics_evidence_base_url }} + + +[MONGO] + +url = mongodb://{{ ml_analytics_mongodb_url }} + +database_name = {{ ml_analytics_mongo_db_name }} + + +# ------ Mongo Collections ------- # +observation_sub_collection = {{ ml_analytics_mongo_observation_submission_collection }} + +solutions_collection = {{ ml_analytics_mongo_solution_collection }} + +observations_collection = {{ ml_analytics_mongo_observation_collection }} + +entity_type_collection = {{ ml_analytics_mongo_entity_type_collection }} + +questions_collection = {{ ml_analytics_mongo_question_collection }} + +criteria_collection = {{ ml_analytics_mongo_criteria_collection }} + +entities_collection = {{ ml_analytics_mongo_entities_collection }} + +programs_collection = {{ ml_analytics_mongo_program_collection }} + +user_roles_collection = {{ ml_analytics_mongo_userroles_collection }} + +criteria_questions_collection = {{ ml_analytics_mongo_criteria_questions_collection }} + +projects_collection = {{ ml_analytics_mongo_projects_collection }} + +survey_submissions_collection = {{ ml_analytics_mongo_survey_submissions_collection }} + +survey_collection = {{ ml_analytics_mongo_survey_collection }} + +[DRUID] + +metadata_url = http://{{ ml_analytics_druid_url }}/druid/coordinator/v1/datasources/ + +batch_url = http://{{ ml_analytics_druid_url }}/druid/indexer/v1/task + +metadata_rollup_url = http://{{ ml_analytics_druid_rollup_url }}/druid/coordinator/v1/datasources/ + +batch_rollup_url = http://{{ ml_analytics_druid_rollup_url }}/druid/indexer/v1/task + +urlQuery = http://{{ ml_analytics_druid_query_url }}/druid/v2?pretty + +observation_status_injestion_spec = {{ ml_analytics_druid_observation_status_injestion_spec }} + +project_injestion_spec = {{ ml_analytics_druid_project_injestion_spec }} + +ml_distinctCnt_obs_status_spec = {{ ml_analytics_druid_distinctCnt_obs_injestion_spec }} + +ml_distinctCnt_obs_domain_spec = {{ ml_analytics_druid_distinctCnt_obs_domain_injestion_spec }} + +ml_distinctCnt_obs_domain_criteria_spec = {{ ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec }} + +ml_distinctCnt_projects_status_spec = {{ ml_analytics_druid_distinctCnt_projects_status_injestion_spec }} + +ml_distinctCnt_prglevel_projects_status_spec = {{ ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec }} + +ml_distinctCnt_survey_status_spec ={{ ml_analytics_druid_distinctCnt_survey_injestion_spec }} + +observation_status_rollup_injestion_spec = {{ ml_analytics_druid_observation_status_rollup_injestion_spec }} + +project_rollup_injestion_spec = {{ ml_analytics_druid_project_rollup_injestion_spec }} + +ml_survey_rollup_spec = {{ml_analytics_druid_survey_rollup_injestion_spec}} + +survey_status_injestion_spec = {{ ml_analytics_druid_survey_status_injestion_spec }} + +observation_query_spec = {{ ml_analytics_druid_observation_query_spec }} + +observation_injestion_spec = {{ml_analytics_druid_observation_batch_ingestion_spec}} + +survey_query_spec = {{ml_analytics_druid_survey_query_spec}} + +survey_injestion_spec = {{ml_analytics_druid_survey_batch_ingestion_spec}} + +intervals = {{ml_analytics_druid_interval_list}} + + +[KAFKA] + +url = {{ ml_analytics_kafka_url }} + +observation_raw_topic = {{ ml_analytics_kafka_observation_topic_name }} + +observation_druid_topic = {{ ml_analytics_kafka_observation_druid_topic_name }} + +observation_evidence_druid_topic = {{ ml_analytics_kafka_observation_evidence_topic_name }} + +survey_evidence_druid_topic = {{ ml_analytics_kafka_survey_evidence_topic_name }} + +survey_raw_topic = {{ ml_analytics_kafka_survey_topic_name }} + +survey_druid_topic = {{ ml_analytics_kafka_survey_druid_topic_name }} + + +[LOGS] + +observation_streaming_success_error = {{ ml_analytics_observation_log_folder_path }}/ + +observation_streaming_evidence_success_error = {{ ml_analytics_observation_log_folder_path }}/evidence/ + +observation_status_success_error = {{ ml_analytics_observation_log_folder_path }}/status/ + +project_success_error = {{ ml_analytics_project_log_folder_path }}/ + +survey_evidence_streaming_success_error = {{ ml_analytics_survey_log_folder_path }}/evidence/ + +survey_streaming_success_error = {{ ml_analytics_survey_log_folder_path }}/ + +survey_status_success_error = {{ ml_analytics_survey_log_folder_path }}/status/ + + +[CLOUD] + +container_name = {{ cloud_storage_telemetry_bucketname }} + +[OUTPUT_DIR] + +project = {{ ml_analytics_project_output_dir }} + +observation_status = {{ ml_analytics_observation_status_output_dir }} + +observation_distinctCount_status = {{ ml_analytics_obs_distinctCnt_output_dir }} + +observation_distinctCount_domain = {{ ml_analytics_obs_distinctCnt_domain_output_dir }} + +observation_distinctCount_domain_criteria = {{ ml_analytics_obs_distinctCnt_domain_criteria_output_dir }} + +projects_distinctCount = {{ ml_analytics_projects_distinctCnt_output_dir }} + +projects_distinctCount_prgmlevel = {{ ml_analytics_projects_distinctCnt_prglevel_output_dir }} + +survey_distinctCount_status = {{ ml_analytics_survey_distinctCnt_output_dir }} + +project_rollup = {{ ml_analytics_project_rollup_output_dir }} + +observation_status_rollup = {{ ml_analytics_observation_status_rollup_output_dir }} + +survey_rollup = {{ ml_analytics_survey_rollup_output_dir }} + +survey_status = {{ ml_analytics_survey_status_output_dir }} + +observation_sub_ids = {{ ml_analytics_observation_submission_id_filepath }} + +observation_druid_data = {{ ml_analytics_observation_batchupdate_output_dir }} + +survey_sub_ids = {{ml_analytics_survey_submission_id_filepath}} + +survey_druid_data = {{ml_analytics_survey_batchupdate_output_dir}} + +program_text_file = {{ml_analytics_project_program}} + +[SLACK] + +token = {{ml_slack_token}} + +channel = {{ml_slack_channel}} + + +[VAM] + +druid_query_url = {{ ml_druid_query_data }} + +program_dashboard_data = {{ ml_program_dashboard_data }} + + +[COMMON] + +cloud_module_path = {{ ml_analytics_cloud_package_path }} + +observation_blob_path = {{ ml_analytics_observation_cloud_blob_path }} + +projects_blob_path = {{ ml_analytics_project_cloud_blob_path }} + +observation_distinctCount_blob_path = {{ ml_analytics_obs_distinctCnt_cloud_blob_path }} + +observation_distinctCount_domain_blob_path = {{ ml_analytics_obs_distinctCnt_domain_cloud_blob_path }} + +observation_distinctCount_domain_criteria_blob_path = {{ ml_analytics_obs_distinctCnt_domain_criteria_cloud_blob_path }} + +projects_distinctCnt_blob_path = {{ ml_analytics_projects_distinctCnt_cloud_blob_path }} + +projects_distinctCnt_prgmlevel_blob_path = {{ ml_analytics_projects_distinctCnt_prglevel_cloud_blob_path }} + +survey_distinctCount_blob_path = {{ ml_analytics_survey_distinctCnt_cloud_blob_path }} + +projects_rollup_blob_path = {{ ml_analytics_project_rollup_cloud_blob_path }} + +observation_rollup_blob_path = {{ ml_analytics_observation_rollup_cloud_blob_path }} + +survey_rollup_blob_path = {{ ml_analytics_survey_rollup_cloud_blob_path }} + +survey_blob_path = {{ ml_analytics_survey_cloud_blob_path }} + +projects_program_csv = {{ ml_analytics_program_dashboard_cloud_blob_path }} + +observation_batch_ingestion_data_del = {{ ml_analytics_observation_batchupdate_cloud_blob_path }} + +survey_batch_ingestion_data_del = {{ ml_analytics_survey_batchupdate_cloud_blob_path}} + +cname_url = {{ ml_analytics_cname_url }} + +nvsk_imp_projects_data_local_path = {{ ml_analytics_nvsk_imp_projects_data_local_path }} + +nvsk_imp_projects_data_blob_path = {{ ml_analytics_nvsk_imp_projects_data_blob_path }} + +[API_CREDENTIALS] + +client_id = {{ ml_analytics_client_id }} + +client_secret = {{ ml_analytics_client_secret }} + +username = {{ ml_analytics_username }} + +password = {{ ml_analytics_password }} + +[JSON_VARIABLE] + +createdBy = {{ ml_analytics_createdBy }} + +container = {{ ml_analytics_reports_container }} + +store = {{ ml_analytics_reports_store }} + +[ML_CORE_SERVICE_URL] + +url = {{ ml_analytics_core_service }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/.helmignore b/kubernetes/helm_charts/core/ml-analytics-service/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/core/ml-analytics-service/Chart.yaml b/kubernetes/helm_charts/core/ml-analytics-service/Chart.yaml new file mode 100644 index 0000000000..eb2395299a --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: ml-analytics-service +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/NOTES.txt b/kubernetes/helm_charts/core/ml-analytics-service/templates/NOTES.txt new file mode 100644 index 0000000000..4088de05ac --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "ml-analytics-service.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "ml-analytics-service.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "ml-analytics-service.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "ml-analytics-service.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/_helpers.tpl b/kubernetes/helm_charts/core/ml-analytics-service/templates/_helpers.tpl new file mode 100644 index 0000000000..3f238f7ff7 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "ml-analytics-service.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ml-analytics-service.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ml-analytics-service.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "ml-analytics-service.labels" -}} +helm.sh/chart: {{ include "ml-analytics-service.chart" . }} +{{ include "ml-analytics-service.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "ml-analytics-service.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ml-analytics-service.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ml-analytics-service.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "ml-analytics-service.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/configmap.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/configmap.yaml new file mode 100644 index 0000000000..5f03f92832 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/configmap.yaml @@ -0,0 +1,11 @@ +--- +{{- $keys := .Files.Glob "keys/*" }} +{{ if $keys }} +apiVersion: v1 +kind: Secret +metadata: + name: ml-analytics-access-keys +type: Opaque +data: +{{ (.Files.Glob "keys/*").AsSecrets | indent 2 }} +{{ end }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml new file mode 100644 index 0000000000..5b700eaf70 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + volumes: + - name: {{ .Chart.Name }}-config + configMap: + name: {{ .Chart.Name }}-config + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + volumeMounts: + - name: {{ .Chart.Name }}-config + mountPath: /opt/sparkjobs/ml-analytics-service + subPath: config.ini diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/service.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/service.yaml new file mode 100644 index 0000000000..4b47ed5332 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/values.yaml b/kubernetes/helm_charts/core/ml-analytics-service/values.yaml new file mode 100644 index 0000000000..d9b4066a6a --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/values.yaml @@ -0,0 +1,82 @@ +# Default values for ml-analytics-service. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: nginx + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} From 5a6c0ae7a7a1483e92ce5ce5ec78f81392e73f76 Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Tue, 20 Feb 2024 09:03:52 +0530 Subject: [PATCH 02/21] Update and rename values.yaml to values.j2 --- .../core/ml-analytics-service/values.j2 | 21 +++++ .../core/ml-analytics-service/values.yaml | 82 ------------------- 2 files changed, 21 insertions(+), 82 deletions(-) create mode 100644 kubernetes/helm_charts/core/ml-analytics-service/values.j2 delete mode 100644 kubernetes/helm_charts/core/ml-analytics-service/values.yaml diff --git a/kubernetes/helm_charts/core/ml-analytics-service/values.j2 b/kubernetes/helm_charts/core/ml-analytics-service/values.j2 new file mode 100644 index 0000000000..d0e76829f5 --- /dev/null +++ b/kubernetes/helm_charts/core/ml-analytics-service/values.j2 @@ -0,0 +1,21 @@ +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{ml_analytics_replicacount|default(1)}} +repository: {{ml_analytics_repository|default('ml-analytics-service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{ml_analytics_cpu_req|default('1000m')}} + memory: {{ml_analytics_mem_req|default('1000Mi')}} + limits: + cpu: {{ml_analytics_cpu_limit|default('1.5')}} + memory: {{ml_analytics_mem_limit|default('2048Mi')}} +network: + port: 7000 + targetport: 7000 +strategy: + type: RollingUpdate + maxsurge: {{ ml_core_maxsurge|default('25%') }} + maxunavailable: {{ ml_core_maxunavailable|default('25%') }} diff --git a/kubernetes/helm_charts/core/ml-analytics-service/values.yaml b/kubernetes/helm_charts/core/ml-analytics-service/values.yaml deleted file mode 100644 index d9b4066a6a..0000000000 --- a/kubernetes/helm_charts/core/ml-analytics-service/values.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# Default values for ml-analytics-service. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: nginx - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} From 1f2eb116cafe35383d03fe021e2ec4520e2d2090 Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Tue, 20 Feb 2024 09:11:43 +0530 Subject: [PATCH 03/21] Update main.yml --- ansible/roles/stack-sunbird/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/roles/stack-sunbird/defaults/main.yml b/ansible/roles/stack-sunbird/defaults/main.yml index 581f28458b..0c0a678be5 100644 --- a/ansible/roles/stack-sunbird/defaults/main.yml +++ b/ansible/roles/stack-sunbird/defaults/main.yml @@ -268,6 +268,7 @@ service_env: ml-reports-service: ../../../../ansible/roles/stack-sunbird/templates/ml-reports-service.env ml-core-service: ../../../../ansible/roles/stack-sunbird/templates/ml-core-service.env ml-projects-service: ../../../../ansible/roles/stack-sunbird/templates/ml-projects-service.env + ml-analytics-service: ../../../../ansible/roles/stack-sunbird/templates/ml-analytics-service.conf inbound: ../../../../ansible/roles/stack-sunbird/templates/inbound.env outbound: - ../../../../ansible/roles/stack-sunbird/templates/outbound.env From 16a90557e4c28e3192425408b4a6432701294acf Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Tue, 20 Feb 2024 10:06:08 +0530 Subject: [PATCH 04/21] Update ml-analytics-service.conf --- .../templates/ml-analytics-service.conf | 218 +++++++++--------- 1 file changed, 109 insertions(+), 109 deletions(-) diff --git a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf index 935085d36a..9787c1efe8 100644 --- a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf +++ b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf @@ -1,265 +1,265 @@ [ML_APP_NAME] -survey_app = {{ ml_analytics_survey_app_name }} +survey_app = "{{ ml_survey_app_name | default('sunbirdsurvey') }}" -integrated_app = {{ ml_analytics_integrated_app_name }} +integrated_app = "{{ ml_integrated_app_name | default('sunbird') }}" -integrated_portal = {{ ml_analytics_integrated_portal }} +integrated_portal = "{{ ml_integrated_portal | default('dev.sunbird.portal') }}" [API_ENDPOINTS] -base_url = {{ ml_analytics_api_base_url }} +base_url = "https://{{ domain_name }}/" [API_HEADERS] content_type = application/json -authorization = {{ ml_analytics_api_authorization_key }} +authorization = "{{ml_api_auth_token | default('sunbird_api_auth_token')}}" -internal_access_token = {{ ml_analytics_api_access_token }} +internal_access_token = "{{ml_api_access_token | default('ml_core_internal_access_token')}}" -authorization_access_token = {{ ml_analytics_authorization_access_token }} +authorization_access_token = "{{ ml_analytics_authorization_access_token }}" [ML_SURVEY_SERVICE_URL] -url = {{ ml_analytics_survey_service }} +url = "http://{{private_ingressgateway_ip}}/ml-survey" entity_related_end_point = assessment/api/v1/entities/relatedEntities/ user_profile_end_point = assessment/api/v1/userExtension/getProfile/ -evidence_base_url = {{ ml_analytics_evidence_base_url }} +evidence_base_url = "{{ cloud_storage_url }}/{{ ml_analytics_public_storage }}/" [MONGO] -url = mongodb://{{ ml_analytics_mongodb_url }} +url = "mongodb://{{ml_mongodb_host | default(groups['mongo_master'][0]+':27017')}}" -database_name = {{ ml_analytics_mongo_db_name }} +database_name = "{{ml_mongodb | default('ml-survey')}}" # ------ Mongo Collections ------- # -observation_sub_collection = {{ ml_analytics_mongo_observation_submission_collection }} +observation_sub_collection = observationSubmissions -solutions_collection = {{ ml_analytics_mongo_solution_collection }} +solutions_collection = solutions -observations_collection = {{ ml_analytics_mongo_observation_collection }} +observations_collection = observations -entity_type_collection = {{ ml_analytics_mongo_entity_type_collection }} +entity_type_collection = entityTypes -questions_collection = {{ ml_analytics_mongo_question_collection }} +questions_collection = questions -criteria_collection = {{ ml_analytics_mongo_criteria_collection }} +criteria_collection = criteria -entities_collection = {{ ml_analytics_mongo_entities_collection }} +entities_collection = entities -programs_collection = {{ ml_analytics_mongo_program_collection }} +programs_collection = programs -user_roles_collection = {{ ml_analytics_mongo_userroles_collection }} +user_roles_collection = userRoles -criteria_questions_collection = {{ ml_analytics_mongo_criteria_questions_collection }} +criteria_questions_collection = criteriaQuestions -projects_collection = {{ ml_analytics_mongo_projects_collection }} +projects_collection = projects -survey_submissions_collection = {{ ml_analytics_mongo_survey_submissions_collection }} +survey_submissions_collection = surveySubmissions -survey_collection = {{ ml_analytics_mongo_survey_collection }} +survey_collection = surveys [DRUID] -metadata_url = http://{{ ml_analytics_druid_url }}/druid/coordinator/v1/datasources/ +metadata_url = "http://{{groups['druid'][0]}}:8081/druid/coordinator/v1/datasources/" -batch_url = http://{{ ml_analytics_druid_url }}/druid/indexer/v1/task +batch_url = "http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task" -metadata_rollup_url = http://{{ ml_analytics_druid_rollup_url }}/druid/coordinator/v1/datasources/ +metadata_rollup_url = "http://{{groups['druid'][0]}}:8081/druid/coordinator/v1/datasources/" -batch_rollup_url = http://{{ ml_analytics_druid_rollup_url }}/druid/indexer/v1/task +batch_rollup_url = "http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task" -urlQuery = http://{{ ml_analytics_druid_query_url }}/druid/v2?pretty +urlQuery = "http://{{groups['druid'][0]}}:8082/druid/v2?pretty" -observation_status_injestion_spec = {{ ml_analytics_druid_observation_status_injestion_spec }} +observation_status_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_code","school_externalId","state_name","state_code","state_externalId","district_name","district_code","district_externalId","block_name","block_code","block_externalId","cluster_name","cluster_code","cluster_externalId","completedDate","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name","ecm_marked_na","board_name","updatedAt","organisation_id","user_type","observed_school_name","observed_school_id","observed_school_code","observed_state_name","observed_state_id","observed_state_code","observed_district_name","observed_district_id","observed_district_code","observed_block_name","observed_block_id","observed_block_code","observed_cluster_name","observed_cluster_id","observed_cluster_code","isRubricDriven","criteriaLevelReport"]},"metricsSpec":[]}}}' -project_injestion_spec = {{ ml_analytics_druid_project_injestion_spec }} +project_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-project","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"auto"},"dimensionsSpec":{"dimensions":[]},"metricsSpec":[]}}}' -ml_distinctCnt_obs_status_spec = {{ ml_analytics_druid_distinctCnt_obs_injestion_spec }} +ml_distinctCnt_obs_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' -ml_distinctCnt_obs_domain_spec = {{ ml_analytics_druid_distinctCnt_obs_domain_injestion_spec }} +ml_distinctCnt_obs_domain_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' -ml_distinctCnt_obs_domain_criteria_spec = {{ ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec }} +ml_distinctCnt_obs_domain_criteria_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain-criteria","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"string","name":"criteria_name"},{"type":"string","name":"criteria_score"},{"type":"string","name":"criteria_id"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' -ml_distinctCnt_projects_status_spec = {{ ml_analytics_druid_distinctCnt_projects_status_injestion_spec }} +ml_distinctCnt_projects_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"project_title"},{"type":"string","name":"solution_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_solution"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"long","name":"no_of_imp_with_evidence_inprogress"},{"type":"string","name":"time_stamp"},{"type":"long","name":"no_of_certificate_issued"}]},"metricsSpec":[]}}}' -ml_distinctCnt_prglevel_projects_status_spec = {{ ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec }} +ml_distinctCnt_prglevel_projects_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-programLevel-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"},{"type":"long","name":"no_of_certificate_issued"}]},"metricsSpec":[]}}}' -ml_distinctCnt_survey_status_spec ={{ ml_analytics_druid_distinctCnt_survey_injestion_spec }} +ml_distinctCnt_survey_status_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-surveydistinctCount-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"survey_name"},{"type":"string","name":"survey_id"},{"type":"string","name":"submission_status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_users"},{"type":"long","name":"unique_submissions"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' -observation_status_rollup_injestion_spec = {{ ml_analytics_druid_observation_status_rollup_injestion_spec }} +observation_status_rollup_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-observation-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["status","user_id","solution_id","submission_id","entity_name","completedDate","program_id","private_program","solution_type","updatedAt","role_title","solution_name","program_name","channel","parent_channel","block_name","district_name","school_name","cluster_name","state_name","organisation_name","board_name","district_externalId","state_externalId","block_externalId","cluster_externalId","school_externalId","organisation_id",{"type":"long","name":"status_code"}]},"metricsSpec":[{"type":"count","name":"count"},{"type":"longSum","name":"sum___v","fieldName":"__v","expression":null},{"type":"HLLSketchBuild","name":"count_distinct_solution","fieldName":"solution_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_submission_id","fieldName":"submission_id","lgK":12,"tgtHllType":"HLL_4","round":false},{"type":"HLLSketchBuild","name":"count_distinct_user_id","fieldName":"user_id","lgK":12,"tgtHllType":"HLL_4","round":false}]}}}' -project_rollup_injestion_spec = {{ ml_analytics_druid_project_rollup_injestion_spec }} +project_rollup_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"iso"},"dimensionsSpec":{"dimensions":["project_title","project_goal","area_of_improvement","status_of_project","tasks_name","tasks_status","designation","task_evidence_status","project_id","task_id","project_created_type","parent_channel","program_id","program_name","project_updated_date","createdBy","program_externalId","private_program","task_deleted_flag","project_terms_and_condition","state_externalId","block_externalId","district_externalId","cluster_externalId","school_externalId","state_name","block_name","district_name","cluster_name","school_name","board_name","organisation_name","solution_id","organisation_id",{"name":"status_code","type":"long"}]},"metricsSpec":[{"name":"count","type":"count"},{"name":"sum___v","type":"longSum","fieldName":"__v"},{"name":"sum_status_code","type":"longMax","fieldName":"status_code"},{"type":"HLLSketchBuild","name":"count_of_createBy","fieldName":"createdBy"},{"type":"HLLSketchBuild","name":"count_of_project_id","fieldName":"project_id"},{"type":"HLLSketchBuild","name":"count_of_solution_id","fieldName":"solution_id"},{"type":"HLLSketchBuild","name":"count_of_program_id","fieldName":"program_id"}]}}}' -ml_survey_rollup_spec = {{ml_analytics_druid_survey_rollup_injestion_spec}} +ml_survey_rollup_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status-rollup","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel",{"type":"long","name":"status_code"}, "solution_name", "solution_id"]},"metricsSpec":[{"name":"count","type":"count"},{"name":"sum___v","type":"longSum","fieldName":"__v"},{"name":"sum_status_code","type":"longMax","fieldName":"status_code"},{"type":"HLLSketchBuild","name":"count_of_user_id","fieldName":"user_id"},{"type":"HLLSketchBuild","name":"count_of_survey_submission_id","fieldName":"survey_submission_id"},{"type":"HLLSketchBuild","name":"count_of_solution_id","fieldName":"solution_id"},{"type":"HLLSketchBuild","name":"count_of_program_id","fieldName":"program_id"}]}}}' -survey_status_injestion_spec = {{ ml_analytics_druid_survey_status_injestion_spec }} +survey_status_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris":["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-survey-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":true,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"iso"},"dimensionsSpec":{"dimensions":["survey_submission_id", "submission_status", "user_id", "user_sub_type", "user_type", "state_externalId", "block_externalId", "district_externalId", "cluster_externalId", "school_externalId", "state_name", "block_name", "district_name", "cluster_name", "school_name", "board_name", "organisation_id", "organisation_name", "program_externalId", "program_id", "program_name", "survey_name", "survey_id", "survey_externalId", "created_date", "submission_date", "updatedAt", "parent_channel", "solution_name", "solution_id","private_program","state_code","school_code","district_code","block_code","cluster_code"]},"metricsSpec":[]}}}' -observation_query_spec = {{ ml_analytics_druid_observation_query_spec }} +observation_query_spec = '{"queryType":"scan","dataSource":"sl-observation","resultFormat":"list","columns":["completedDate","createdAt","createdBy","criteriaExternalId","criteriaId","criteriaName","entityType","entityTypeId","observationId","observationName","observationSubmissionId","questionAnswer","questionECM","questionExternalId","questionId","questionName","questionResponseLabel","questionResponseType","solutionExternalId","solutionId","solutionName","updatedAt","instanceParentId","instanceId","instanceParentResponsetype","instanceParentQuestion","questionSequenceByEcm","maxScore","minScore","percentageScore","pointsBasedScoreInParent","totalScore","scoreAchieved","totalpercentage","instanceParentExternalId","instanceParentEcmSequence","remarks","total_evidences","evidence_count","school","block","district","cluster","state","schoolName","blockName","districtName","clusterName","stateName","schoolExternalId","blockExternalId","districtExternalId","clusterExternalId","stateExternalId","schoolTypes","administrationTypes","instanceParentCriteriaId","instanceParentCriteriaExternalId","instanceParentCriteriaName","role_title","location_validated_with_geotag","distance_in_meters","entity","entityExternalId","entityName","isAPrivateProgram","programId","programName","programExternalId","questionResponseLabel_number","criteriaLevel","criteriaScore","submissionNumber","submissionTitle","channel","parent_channel","user_districtName","user_blockName","user_clusterName","appName","evidences","user_stateName","domainName","domainExternalId","childName","childType","childExternalid","level","criteriaDescription","programDescription","solutionDescription","label","imp_project_id","imp_project_title","imp_project_goal","imp_project_externalId","ancestorName","scoringSystem","domainLevel","domainScore","criteriaLevelReport","user_schoolName","user_schoolId","user_schoolUDISE_code","solution_type","organisation_name","user_boardName","district_externalId","state_externalId","block_externalId","cluster_externalId","organisation_id","user_type"],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}' -observation_injestion_spec = {{ml_analytics_druid_observation_batch_ingestion_spec}} +observation_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"entityType"},{"type":"string","name":"entityTypeId"},{"type":"string","name":"observationId"},{"type":"string","name":"observationName"},{"type":"string","name":"observationSubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"school"},{"type":"string","name":"block"},{"type":"string","name":"district"},{"type":"string","name":"cluster"},{"type":"string","name":"state"},{"type":"string","name":"schoolName"},{"type":"string","name":"blockName"},{"type":"string","name":"districtName"},{"type":"string","name":"clusterName"},{"type":"string","name":"stateName"},{"type":"string","name":"schoolExternalId"},{"type":"string","name":"blockExternalId"},{"type":"string","name":"districtExternalId"},{"type":"string","name":"clusterExternalId"},{"type":"string","name":"stateExternalId"},{"type":"string","name":"schoolTypes"},{"type":"string","name":"administrationTypes"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"role_title"},{"type":"string","name":"location_validated_with_geotag"},{"type":"string","name":"distance_in_meters"},{"type":"string","name":"entity"},{"type":"string","name":"entityExternalId"},{"type":"string","name":"entityName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"criteriaLevel"},{"type":"string","name":"criteriaScore"},{"type":"string","name":"submissionNumber"},{"type":"string","name":"submissionTitle"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"user_districtName"},{"type":"string","name":"user_blockName"},{"type":"string","name":"user_clusterName"},{"type":"string","name":"appName"},{"type":"string","name":"evidences"},{"type":"string","name":"user_stateName"},{"type":"string","name":"domainName"},{"type":"string","name":"domainExternalId"},{"type":"string","name":"childName"},{"type":"string","name":"childType"},{"type":"string","name":"childExternalid"},{"type":"string","name":"level"},{"type":"string","name":"criteriaDescription"},{"type":"string","name":"programDescription"},{"type":"string","name":"solutionDescription"},{"type":"string","name":"label"},{"type":"string","name":"imp_project_id"},{"type":"string","name":"imp_project_title"},{"type":"string","name":"imp_project_goal"},{"type":"string","name":"imp_project_externalId"},{"type":"string","name":"ancestorName"},{"type":"string","name":"scoringSystem"},{"type":"string","name":"domainLevel"},{"type":"string","name":"domainScore"},{"name":"criteriaLevelReport","type":"boolean"},{"type":"string","name":"user_schoolName"},{"type":"string","name":"user_schoolId"},{"type":"string","name":"user_schoolUDISE_code"},{"type":"string","name":"solution_type"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_boardName"},{"type":"string","name":"district_externalId"},{"type":"string","name":"state_externalId"},{"type":"string","name":"block_externalId"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"user_type"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}' -survey_query_spec = {{ml_analytics_druid_survey_query_spec}} +survey_query_spec = '{"queryType":"scan","dataSource":"sl-survey","resultFormat":"list","columns":["completedDate","createdAt","createdBy","criteriaExternalId","criteriaId","criteriaName","surveyId","surveyName","surveySubmissionId","questionAnswer","questionECM","questionExternalId","questionId","questionName","questionResponseLabel","questionResponseType","solutionExternalId","solutionId","solutionName","updatedAt","instanceParentId","instanceId","instanceParentResponsetype","instanceParentQuestion","questionSequenceByEcm","maxScore","minScore","percentageScore","pointsBasedScoreInParent","totalScore","scoreAchieved","totalpercentage","instanceParentExternalId","instanceParentEcmSequence","remarks","total_evidences","evidence_count","instanceParentCriteriaId","instanceParentCriteriaExternalId","instanceParentCriteriaName","isAPrivateProgram","programId","programName","programExternalId","questionResponseLabel_number","channel","parent_channel","appName","organisation_name","user_subtype","user_type","board_name","district_code","district_name","district_externalId","block_code","block_name","block_externalId","school_code","school_name","school_externalId","cluster_code","cluster_name","cluster_externalId","state_code","state_name","state_externalId","organisation_id","evidences"],"intervals":["1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00"]}' -survey_injestion_spec = {{ml_analytics_druid_survey_batch_ingestion_spec}} +survey_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-survey","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"completedDate","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"completedDate"},{"type":"string","name":"createdAt"},{"type":"string","name":"createdBy"},{"type":"string","name":"criteriaExternalId"},{"type":"string","name":"criteriaId"},{"type":"string","name":"criteriaName"},{"type":"string","name":"surveyId"},{"type":"string","name":"surveyName"},{"type":"string","name":"surveySubmissionId"},{"type":"string","name":"questionAnswer"},{"type":"string","name":"questionECM"},{"type":"string","name":"questionExternalId"},{"type":"string","name":"questionId"},{"type":"string","name":"questionName"},{"type":"string","name":"questionResponseLabel"},{"type":"string","name":"questionResponseType"},{"type":"string","name":"solutionExternalId"},{"type":"string","name":"solutionId"},{"type":"string","name":"solutionName"},{"type":"string","name":"updatedAt"},{"type":"string","name":"instanceParentId"},{"type":"string","name":"instanceId"},{"type":"string","name":"instanceParentResponsetype"},{"type":"string","name":"instanceParentQuestion"},{"type":"string","name":"questionSequenceByEcm"},{"type":"string","name":"maxScore"},{"type":"string","name":"minScore"},{"type":"string","name":"percentageScore"},{"type":"string","name":"pointsBasedScoreInParent"},{"type":"string","name":"totalScore"},{"type":"string","name":"scoreAchieved"},{"type":"string","name":"totalpercentage"},{"type":"string","name":"instanceParentExternalId"},{"type":"string","name":"instanceParentEcmSequence"},{"type":"string","name":"remarks"},{"type":"string","name":"total_evidences"},{"type":"string","name":"evidence_count"},{"type":"string","name":"evidences"},{"type":"string","name":"instanceParentCriteriaId"},{"type":"string","name":"instanceParentCriteriaExternalId"},{"type":"string","name":"instanceParentCriteriaName"},{"type":"string","name":"isAPrivateProgram"},{"type":"string","name":"programId"},{"type":"string","name":"programName"},{"type":"string","name":"programExternalId"},{"name":"questionResponseLabel_number","type":"float"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"appName"},{"type":"string","name":"organisation_name"},{"type":"string","name":"user_subtype"},{"type":"string","name":"user_type"},{"type":"string","name":"board_name"},{"type":"string","name":"district_code"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_code"},{"type":"string","name":"block_name"},{"type":"string","name":"block_externalId"},{"type":"string","name":"school_code"},{"type":"string","name":"school_name"},{"type":"string","name":"school_externalId"},{"type":"string","name":"cluster_code"},{"type":"string","name":"cluster_name"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"state_code"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"organisation_id"},{"type":"string","name":"isSubmissionDeleted"}]},"metricsSpec":[{"type":"floatSum","name":"question_response_number","fieldName":"questionResponseLabel_number"}]}}}' -intervals = {{ml_analytics_druid_interval_list}} +intervals = '["1901-01-01T00:00:00+00:00/2020-01-01T00:00:00+00:00","2020-01-01T00:00:00+00:00/2020-06-01T00:00:00+00:00","2020-06-01T00:00:00+00:00/2021-01-01T00:00:00+00:00","2021-01-01T00:00:00+00:00/2021-06-01T00:00:00+00:00","2021-06-01T00:00:00+00:00/2022-01-01T00:00:00+00:00","2022-01-01T00:00:00+00:00/2022-03-01T00:00:00+00:00","2022-03-01T00:00:00+00:00/2022-06-01T00:00:00+00:00","2022-06-01T00:00:00+00:00/2022-09-01T00:00:00+00:00","2022-09-01T00:00:00+00:00/2023-01-01T00:00:00+00:00"]' [KAFKA] -url = {{ ml_analytics_kafka_url }} +url = "{{ml_kafka_host | default(groups['processing-cluster-kafka'][0]+':9092')}}" -observation_raw_topic = {{ ml_analytics_kafka_observation_topic_name }} +observation_raw_topic = "{{ env_name }}.ml.observation.raw" -observation_druid_topic = {{ ml_analytics_kafka_observation_druid_topic_name }} +observation_druid_topic = "{{ env_name }}.ml.observation.druid" -observation_evidence_druid_topic = {{ ml_analytics_kafka_observation_evidence_topic_name }} +observation_evidence_druid_topic = "{{ env_name }}.ml.observation.evidence.druid" -survey_evidence_druid_topic = {{ ml_analytics_kafka_survey_evidence_topic_name }} +survey_evidence_druid_topic = "{{ env_name }}.ml.survey.evidence.druid" -survey_raw_topic = {{ ml_analytics_kafka_survey_topic_name }} +survey_raw_topic = "{{ env_name }}.ml.survey.raw" -survey_druid_topic = {{ ml_analytics_kafka_survey_druid_topic_name }} +survey_druid_topic = "{{ env_name }}.ml.survey.druid" [LOGS] -observation_streaming_success_error = {{ ml_analytics_observation_log_folder_path }}/ +observation_streaming_success_error = /opt/sparkjobs/logs/observation/ -observation_streaming_evidence_success_error = {{ ml_analytics_observation_log_folder_path }}/evidence/ +observation_streaming_evidence_success_error = /opt/sparkjobs/logs/observation/evidence/ -observation_status_success_error = {{ ml_analytics_observation_log_folder_path }}/status/ +observation_status_success_error = /opt/sparkjobs/logs/observation/status/ -project_success_error = {{ ml_analytics_project_log_folder_path }}/ +project_success_error = /opt/sparkjobs/logs/project/ -survey_evidence_streaming_success_error = {{ ml_analytics_survey_log_folder_path }}/evidence/ +survey_evidence_streaming_success_error = /opt/sparkjobs/logs/project/evidence/ -survey_streaming_success_error = {{ ml_analytics_survey_log_folder_path }}/ +survey_streaming_success_error = /opt/sparkjobs/logs/survey -survey_status_success_error = {{ ml_analytics_survey_log_folder_path }}/status/ +survey_status_success_error = /opt/sparkjobs/logs/survey/status/ [CLOUD] -container_name = {{ cloud_storage_telemetry_bucketname }} +container_name = "{{ cloud_storage_telemetry_bucketname }}" [OUTPUT_DIR] -project = {{ ml_analytics_project_output_dir }} +project = "/opt/sparkjobs/source/projects/output" -observation_status = {{ ml_analytics_observation_status_output_dir }} +observation_status = "/opt/sparkjobs/source/observations/status/output" -observation_distinctCount_status = {{ ml_analytics_obs_distinctCnt_output_dir }} +observation_distinctCount_status = /opt/sparkjobs/source/observations/distinctCount/output -observation_distinctCount_domain = {{ ml_analytics_obs_distinctCnt_domain_output_dir }} +observation_distinctCount_domain = /opt/sparkjobs/source/observations/distinctCount_domain/output -observation_distinctCount_domain_criteria = {{ ml_analytics_obs_distinctCnt_domain_criteria_output_dir }} +observation_distinctCount_domain_criteria = /opt/sparkjobs/source/observations/distinctCount_domain_criteria/output -projects_distinctCount = {{ ml_analytics_projects_distinctCnt_output_dir }} +projects_distinctCount = /opt/sparkjobs/source/projects/distinctCount/output -projects_distinctCount_prgmlevel = {{ ml_analytics_projects_distinctCnt_prglevel_output_dir }} +projects_distinctCount_prgmlevel = /opt/sparkjobs/source/projects/distinctCountPrglevel/output -survey_distinctCount_status = {{ ml_analytics_survey_distinctCnt_output_dir }} +survey_distinctCount_status = /opt/sparkjobs/source/survey/distinctCount/output -project_rollup = {{ ml_analytics_project_rollup_output_dir }} +project_rollup = /opt/sparkjobs/source/projects/output_rollup -observation_status_rollup = {{ ml_analytics_observation_status_rollup_output_dir }} +observation_status_rollup = /opt/sparkjobs/source/observations/output_rollup -survey_rollup = {{ ml_analytics_survey_rollup_output_dir }} +survey_rollup = /opt/sparkjobs/source/survey/output -survey_status = {{ ml_analytics_survey_status_output_dir }} +survey_status = /opt/sparkjobs/source/survey/status/output -observation_sub_ids = {{ ml_analytics_observation_submission_id_filepath }} +observation_sub_ids = /opt/sparkjobs/ml-analytics-service/observations/submissions.csv -observation_druid_data = {{ ml_analytics_observation_batchupdate_output_dir }} +observation_druid_data = /opt/sparkjobs/source/observations/ -survey_sub_ids = {{ml_analytics_survey_submission_id_filepath}} +survey_sub_ids = /opt/sparkjobs/ml-analytics-service/survey/submissions.csv -survey_druid_data = {{ml_analytics_survey_batchupdate_output_dir}} +survey_druid_data = /opt/sparkjobs/source/survey/ -program_text_file = {{ml_analytics_project_program}} +program_text_file = /opt/sparkjobs/ml-analytics-service/projects/program_ids.txt [SLACK] -token = {{ml_slack_token}} +token = "{{ ml_analytics_slack_token | default('') }}" -channel = {{ml_slack_channel}} +channel = "{{ ml_analytics_slack_channel | default('') }}" [VAM] -druid_query_url = {{ ml_druid_query_data }} +druid_query_url = "{{ ml_druid_query | default('') }}" -program_dashboard_data = {{ ml_program_dashboard_data }} +program_dashboard_data = "{{ ml_program_data | default('') }}" [COMMON] -cloud_module_path = {{ ml_analytics_cloud_package_path }} +cloud_module_path = "/opt/sparkjobs/ml-analytics-service/cloud_storage" -observation_blob_path = {{ ml_analytics_observation_cloud_blob_path }} +observation_blob_path = "observation/status/" -projects_blob_path = {{ ml_analytics_project_cloud_blob_path }} +projects_blob_path = "projects/" -observation_distinctCount_blob_path = {{ ml_analytics_obs_distinctCnt_cloud_blob_path }} +observation_distinctCount_blob_path = "observation/distinctCount/" -observation_distinctCount_domain_blob_path = {{ ml_analytics_obs_distinctCnt_domain_cloud_blob_path }} +observation_distinctCount_domain_blob_path = "observation/distinctCount_domain/" -observation_distinctCount_domain_criteria_blob_path = {{ ml_analytics_obs_distinctCnt_domain_criteria_cloud_blob_path }} +observation_distinctCount_domain_criteria_blob_path = "observation/distinctCount_domain_criteria/" -projects_distinctCnt_blob_path = {{ ml_analytics_projects_distinctCnt_cloud_blob_path }} +projects_distinctCnt_blob_path = "projects/distinctCount/" -projects_distinctCnt_prgmlevel_blob_path = {{ ml_analytics_projects_distinctCnt_prglevel_cloud_blob_path }} +projects_distinctCnt_prgmlevel_blob_path = "projects/distinctCountPrglevel/" -survey_distinctCount_blob_path = {{ ml_analytics_survey_distinctCnt_cloud_blob_path }} +survey_distinctCount_blob_path = "survey/distinctCount/" -projects_rollup_blob_path = {{ ml_analytics_project_rollup_cloud_blob_path }} +projects_rollup_blob_path = "projects/rollup" -observation_rollup_blob_path = {{ ml_analytics_observation_rollup_cloud_blob_path }} +observation_rollup_blob_path = "observation/rollup" -survey_rollup_blob_path = {{ ml_analytics_survey_rollup_cloud_blob_path }} +survey_rollup_blob_path = "survey/rollup/" -survey_blob_path = {{ ml_analytics_survey_cloud_blob_path }} +survey_blob_path = "survey/status/" -projects_program_csv = {{ ml_analytics_program_dashboard_cloud_blob_path }} +projects_program_csv = "{{ ml_program_blob_path | default('') }}" -observation_batch_ingestion_data_del = {{ ml_analytics_observation_batchupdate_cloud_blob_path }} +observation_batch_ingestion_data_del = "observation/batchDeletion" -survey_batch_ingestion_data_del = {{ ml_analytics_survey_batchupdate_cloud_blob_path}} +survey_batch_ingestion_data_del = "survey/batchDeletion" -cname_url = {{ ml_analytics_cname_url }} +cname_url = "{{ cloud_storage_url }}/{{ cloud_storage_samiksha_bucketname }}" -nvsk_imp_projects_data_local_path = {{ ml_analytics_nvsk_imp_projects_data_local_path }} +nvsk_imp_projects_data_local_path = "{{ config_path }}/urgent_data_metrics/output/" -nvsk_imp_projects_data_blob_path = {{ ml_analytics_nvsk_imp_projects_data_blob_path }} +nvsk_imp_projects_data_blob_path = "Manage_Learn_Data/micro_improvement/" [API_CREDENTIALS] -client_id = {{ ml_analytics_client_id }} +client_id = "{{ ml_analytics_client_id }}" -client_secret = {{ ml_analytics_client_secret }} +client_secret = "{{ ml_analytics_client_secret }}" -username = {{ ml_analytics_username }} +username = "{{ ml_analytics_username }}" -password = {{ ml_analytics_password }} +password = "{{ ml_analytics_password }}" [JSON_VARIABLE] -createdBy = {{ ml_analytics_createdBy }} +createdBy = "{{ ml_analytics_createdBy }}" -container = {{ ml_analytics_reports_container }} +container = "{{ ml_analytics_reports_container }}" -store = {{ ml_analytics_reports_store }} +store = "{{ ml_analytics_reports_store }}" [ML_CORE_SERVICE_URL] -url = {{ ml_analytics_core_service }} +url = "http://{{private_ingressgateway_ip}}/ml-core" From 0386abd2b42df35723ca7f2c7a748b4d6af1e8bf Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Wed, 21 Feb 2024 15:25:23 +0530 Subject: [PATCH 05/21] Update ml-analytics-service.conf --- ansible/roles/stack-sunbird/templates/ml-analytics-service.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf index 9787c1efe8..24db65482e 100644 --- a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf +++ b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf @@ -238,7 +238,7 @@ survey_batch_ingestion_data_del = "survey/batchDeletion" cname_url = "{{ cloud_storage_url }}/{{ cloud_storage_samiksha_bucketname }}" -nvsk_imp_projects_data_local_path = "{{ config_path }}/urgent_data_metrics/output/" +nvsk_imp_projects_data_local_path = "/opt/sparkjobs/ml-analytics-service/urgent_data_metrics/output/" nvsk_imp_projects_data_blob_path = "Manage_Learn_Data/micro_improvement/" From 7d72f4f74d130779f7b049e2464bab9cdae05d68 Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Wed, 21 Feb 2024 15:29:06 +0530 Subject: [PATCH 06/21] Delete kubernetes/helm_charts/core/ml-analytics-service/templates/NOTES.txt --- .../ml-analytics-service/templates/NOTES.txt | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 kubernetes/helm_charts/core/ml-analytics-service/templates/NOTES.txt diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/NOTES.txt b/kubernetes/helm_charts/core/ml-analytics-service/templates/NOTES.txt deleted file mode 100644 index 4088de05ac..0000000000 --- a/kubernetes/helm_charts/core/ml-analytics-service/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "ml-analytics-service.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "ml-analytics-service.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "ml-analytics-service.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "ml-analytics-service.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} From fe60c5e9d68f2e00fdc139f5366fbc47f6b805bc Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Wed, 21 Feb 2024 15:30:27 +0530 Subject: [PATCH 07/21] Update deployment.yaml --- .../core/ml-analytics-service/templates/deployment.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml index 5b700eaf70..dcb8ec34b7 100644 --- a/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml @@ -35,12 +35,6 @@ spec: {{ toYaml .Values.resources | indent 10 }} ports: - containerPort: {{ .Values.network.port }} - {{- if .Values.healthcheck }} - livenessProbe: -{{ toYaml .Values.livenessProbe | indent 10 }} - readinessProbe: -{{ toYaml .Values.readinessProbe | indent 10 }} - {{- end }} volumeMounts: - name: {{ .Chart.Name }}-config mountPath: /opt/sparkjobs/ml-analytics-service From 6a3ad550c0d75eba606a8e5167cbe4147237dcd3 Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Thu, 22 Feb 2024 13:46:01 +0530 Subject: [PATCH 08/21] Update deployment.yaml --- .../core/ml-analytics-service/templates/deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml index dcb8ec34b7..ce73e25ecb 100644 --- a/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml @@ -33,9 +33,9 @@ spec: imagePullPolicy: Always resources: {{ toYaml .Values.resources | indent 10 }} - ports: - - containerPort: {{ .Values.network.port }} - volumeMounts: + ports: + - containerPort: {{ .Values.network.port }} + volumeMounts: - name: {{ .Chart.Name }}-config mountPath: /opt/sparkjobs/ml-analytics-service subPath: config.ini From b2fa461d2f30d35af65d60c20263c30e69b426be Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Thu, 22 Feb 2024 13:51:13 +0530 Subject: [PATCH 09/21] Update deployment.yaml From 7b5646ddc1822c00ac4e89622083292f047e4d4d Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Thu, 22 Feb 2024 13:53:27 +0530 Subject: [PATCH 10/21] Update values.j2 --- kubernetes/helm_charts/core/ml-analytics-service/values.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/core/ml-analytics-service/values.j2 b/kubernetes/helm_charts/core/ml-analytics-service/values.j2 index d0e76829f5..2a2a1c8638 100644 --- a/kubernetes/helm_charts/core/ml-analytics-service/values.j2 +++ b/kubernetes/helm_charts/core/ml-analytics-service/values.j2 @@ -10,7 +10,7 @@ resources: cpu: {{ml_analytics_cpu_req|default('1000m')}} memory: {{ml_analytics_mem_req|default('1000Mi')}} limits: - cpu: {{ml_analytics_cpu_limit|default('1.5')}} + cpu: {{ml_analytics_cpu_limit|default('2')}} memory: {{ml_analytics_mem_limit|default('2048Mi')}} network: port: 7000 From f7fb364eaded97852cbef39786d0ff719a7138da Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Thu, 22 Feb 2024 13:57:49 +0530 Subject: [PATCH 11/21] Update deployment.yaml --- .../templates/deployment.yaml | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml index ce73e25ecb..c716b9d73d 100644 --- a/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml @@ -24,18 +24,18 @@ spec: - name: {{ .Values.imagepullsecrets }} {{- end }} volumes: - - name: {{ .Chart.Name }}-config - configMap: - name: {{ .Chart.Name }}-config + - name: {{ .Chart.Name }}-config + configMap: + name: {{ .Chart.Name }}-config containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" - imagePullPolicy: Always - resources: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + resources: {{ toYaml .Values.resources | indent 10 }} - ports: - - containerPort: {{ .Values.network.port }} - volumeMounts: - - name: {{ .Chart.Name }}-config - mountPath: /opt/sparkjobs/ml-analytics-service - subPath: config.ini + ports: + - containerPort: {{ .Values.network.port }} + volumeMounts: + - name: {{ .Chart.Name }}-config + mountPath: /opt/sparkjobs/ml-analytics-service + subPath: config.ini From 261f0096c79775ccaa4e4bdbbc1b11004dcafd69 Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Thu, 22 Feb 2024 15:39:02 +0530 Subject: [PATCH 12/21] Update deployment.yaml --- .../core/ml-analytics-service/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml index c716b9d73d..13970a1462 100644 --- a/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml @@ -37,5 +37,5 @@ spec: - containerPort: {{ .Values.network.port }} volumeMounts: - name: {{ .Chart.Name }}-config - mountPath: /opt/sparkjobs/ml-analytics-service + mountPath: /opt/sparkjobs/ml-analytics-service/config.ini subPath: config.ini From 7e7c0b0cbb8a035815eb3b357537dc2e39bf1b87 Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Thu, 22 Feb 2024 16:55:38 +0530 Subject: [PATCH 13/21] Update ml-analytics-service.conf --- .../templates/ml-analytics-service.conf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf index 24db65482e..c7963155eb 100644 --- a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf +++ b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf @@ -133,19 +133,19 @@ survey_druid_topic = "{{ env_name }}.ml.survey.druid" [LOGS] -observation_streaming_success_error = /opt/sparkjobs/logs/observation/ +observation_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/observation/ -observation_streaming_evidence_success_error = /opt/sparkjobs/logs/observation/evidence/ +observation_streaming_evidence_success_error = /opt/sparkjobs/ml-analytics-service/logs/observation/evidence/ -observation_status_success_error = /opt/sparkjobs/logs/observation/status/ +observation_status_success_error = /opt/sparkjobs/ml-analytics-service/logs/observation/status/ -project_success_error = /opt/sparkjobs/logs/project/ +project_success_error = /opt/sparkjobs/ml-analytics-service/logs/project/ -survey_evidence_streaming_success_error = /opt/sparkjobs/logs/project/evidence/ +survey_evidence_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/project/evidence/ -survey_streaming_success_error = /opt/sparkjobs/logs/survey +survey_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/survey -survey_status_success_error = /opt/sparkjobs/logs/survey/status/ +survey_status_success_error = /opt/sparkjobs/ml-analytics-service/logs/survey/status/ [CLOUD] From 387d943fb47296600f3e6230fe9838bee407eb37 Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Thu, 22 Feb 2024 16:56:49 +0530 Subject: [PATCH 14/21] Update deployment.yaml --- .../core/ml-analytics-service/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml index 13970a1462..d28628b7e5 100644 --- a/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml +++ b/kubernetes/helm_charts/core/ml-analytics-service/templates/deployment.yaml @@ -38,4 +38,4 @@ spec: volumeMounts: - name: {{ .Chart.Name }}-config mountPath: /opt/sparkjobs/ml-analytics-service/config.ini - subPath: config.ini + subPath: ml-analytics-service.conf From e1aa31732b7086e968ef8ec496a65ba002da71e6 Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Thu, 22 Feb 2024 19:56:57 +0530 Subject: [PATCH 15/21] Update ml-analytics-service.conf --- .../templates/ml-analytics-service.conf | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf index c7963155eb..6a60ceba58 100644 --- a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf +++ b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf @@ -9,35 +9,35 @@ integrated_portal = "{{ ml_integrated_portal | default('dev.sunbird.portal') }}" [API_ENDPOINTS] -base_url = "https://{{ domain_name }}/" +base_url = https://{{ domain_name }}/ [API_HEADERS] content_type = application/json -authorization = "{{ml_api_auth_token | default('sunbird_api_auth_token')}}" +authorization = {{ml_api_auth_token | default('sunbird_api_auth_token')}} -internal_access_token = "{{ml_api_access_token | default('ml_core_internal_access_token')}}" +internal_access_token = {{ml_api_access_token | default('ml_core_internal_access_token')}} -authorization_access_token = "{{ ml_analytics_authorization_access_token }}" +authorization_access_token = {{ ml_analytics_authorization_access_token }} [ML_SURVEY_SERVICE_URL] -url = "http://{{private_ingressgateway_ip}}/ml-survey" +url = http://{{private_ingressgateway_ip}}/ml-survey entity_related_end_point = assessment/api/v1/entities/relatedEntities/ user_profile_end_point = assessment/api/v1/userExtension/getProfile/ -evidence_base_url = "{{ cloud_storage_url }}/{{ ml_analytics_public_storage }}/" +evidence_base_url = {{ cloud_storage_url }}/{{ ml_analytics_public_storage }}/ [MONGO] -url = "mongodb://{{ml_mongodb_host | default(groups['mongo_master'][0]+':27017')}}" +url = mongodb://{{ml_mongodb_host | default(groups['mongo_master'][0]+':27017')}} -database_name = "{{ml_mongodb | default('ml-survey')}}" +database_name = {{ml_mongodb | default('ml-survey')}} # ------ Mongo Collections ------- # @@ -69,15 +69,15 @@ survey_collection = surveys [DRUID] -metadata_url = "http://{{groups['druid'][0]}}:8081/druid/coordinator/v1/datasources/" +metadata_url = http://{{groups['druid'][0]}}:8081/druid/coordinator/v1/datasources/ -batch_url = "http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task" +batch_url = http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task -metadata_rollup_url = "http://{{groups['druid'][0]}}:8081/druid/coordinator/v1/datasources/" +metadata_rollup_url = http://{{groups['druid'][0]}}:8081/druid/coordinator/v1/datasources/ -batch_rollup_url = "http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task" +batch_rollup_url = http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task -urlQuery = "http://{{groups['druid'][0]}}:8082/druid/v2?pretty" +urlQuery = http://{{groups['druid'][0]}}:8082/druid/v2?pretty observation_status_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_code","school_externalId","state_name","state_code","state_externalId","district_name","district_code","district_externalId","block_name","block_code","block_externalId","cluster_name","cluster_code","cluster_externalId","completedDate","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name","ecm_marked_na","board_name","updatedAt","organisation_id","user_type","observed_school_name","observed_school_id","observed_school_code","observed_state_name","observed_state_id","observed_state_code","observed_district_name","observed_district_id","observed_district_code","observed_block_name","observed_block_id","observed_block_code","observed_cluster_name","observed_cluster_id","observed_cluster_code","isRubricDriven","criteriaLevelReport"]},"metricsSpec":[]}}}' @@ -116,19 +116,19 @@ intervals = '["1901-01-01T00:00:00+00:00/2020-01-01T00:00:00+00:00","2020-01-01T [KAFKA] -url = "{{ml_kafka_host | default(groups['processing-cluster-kafka'][0]+':9092')}}" +url = {{ml_kafka_host | default(groups['processing-cluster-kafka'][0]+':9092')}} -observation_raw_topic = "{{ env_name }}.ml.observation.raw" +observation_raw_topic = {{ env_name }}.ml.observation.raw -observation_druid_topic = "{{ env_name }}.ml.observation.druid" +observation_druid_topic = {{ env_name }}.ml.observation.druid -observation_evidence_druid_topic = "{{ env_name }}.ml.observation.evidence.druid" +observation_evidence_druid_topic = {{ env_name }}.ml.observation.evidence.druid -survey_evidence_druid_topic = "{{ env_name }}.ml.survey.evidence.druid" +survey_evidence_druid_topic = {{ env_name }}.ml.survey.evidence.druid -survey_raw_topic = "{{ env_name }}.ml.survey.raw" +survey_raw_topic = {{ env_name }}.ml.survey.raw -survey_druid_topic = "{{ env_name }}.ml.survey.druid" +survey_druid_topic = {{ env_name }}.ml.survey.druid [LOGS] @@ -150,13 +150,13 @@ survey_status_success_error = /opt/sparkjobs/ml-analytics-service/logs/survey/st [CLOUD] -container_name = "{{ cloud_storage_telemetry_bucketname }}" +container_name = {{ cloud_storage_telemetry_bucketname }} [OUTPUT_DIR] -project = "/opt/sparkjobs/source/projects/output" +project = /opt/sparkjobs/source/projects/output -observation_status = "/opt/sparkjobs/source/observations/status/output" +observation_status = /opt/sparkjobs/source/observations/status/output observation_distinctCount_status = /opt/sparkjobs/source/observations/distinctCount/output @@ -190,76 +190,76 @@ program_text_file = /opt/sparkjobs/ml-analytics-service/projects/program_ids.txt [SLACK] -token = "{{ ml_analytics_slack_token | default('') }}" +token = {{ ml_analytics_slack_token | default('') }} -channel = "{{ ml_analytics_slack_channel | default('') }}" +channel = {{ ml_analytics_slack_channel | default('') }} [VAM] -druid_query_url = "{{ ml_druid_query | default('') }}" +druid_query_url = {{ ml_druid_query | default('') }} -program_dashboard_data = "{{ ml_program_data | default('') }}" +program_dashboard_data = {{ ml_program_data | default('') }} [COMMON] -cloud_module_path = "/opt/sparkjobs/ml-analytics-service/cloud_storage" +cloud_module_path = /opt/sparkjobs/ml-analytics-service/cloud_storage -observation_blob_path = "observation/status/" +observation_blob_path = observation/status/ -projects_blob_path = "projects/" +projects_blob_path = projects/ -observation_distinctCount_blob_path = "observation/distinctCount/" +observation_distinctCount_blob_path = observation/distinctCount/ -observation_distinctCount_domain_blob_path = "observation/distinctCount_domain/" +observation_distinctCount_domain_blob_path = observation/distinctCount_domain/ -observation_distinctCount_domain_criteria_blob_path = "observation/distinctCount_domain_criteria/" +observation_distinctCount_domain_criteria_blob_path = observation/distinctCount_domain_criteria/ -projects_distinctCnt_blob_path = "projects/distinctCount/" +projects_distinctCnt_blob_path = projects/distinctCount/ -projects_distinctCnt_prgmlevel_blob_path = "projects/distinctCountPrglevel/" +projects_distinctCnt_prgmlevel_blob_path = projects/distinctCountPrglevel/ -survey_distinctCount_blob_path = "survey/distinctCount/" +survey_distinctCount_blob_path = survey/distinctCount/ -projects_rollup_blob_path = "projects/rollup" +projects_rollup_blob_path = projects/rollup -observation_rollup_blob_path = "observation/rollup" +observation_rollup_blob_path = observation/rollup -survey_rollup_blob_path = "survey/rollup/" +survey_rollup_blob_path = survey/rollup/ -survey_blob_path = "survey/status/" +survey_blob_path = survey/status/ -projects_program_csv = "{{ ml_program_blob_path | default('') }}" +projects_program_csv = {{ ml_program_blob_path | default('') }} -observation_batch_ingestion_data_del = "observation/batchDeletion" +observation_batch_ingestion_data_del = observation/batchDeletion -survey_batch_ingestion_data_del = "survey/batchDeletion" +survey_batch_ingestion_data_del = survey/batchDeletion -cname_url = "{{ cloud_storage_url }}/{{ cloud_storage_samiksha_bucketname }}" +cname_url = {{ cloud_storage_url }}/{{ cloud_storage_samiksha_bucketname }} -nvsk_imp_projects_data_local_path = "/opt/sparkjobs/ml-analytics-service/urgent_data_metrics/output/" +nvsk_imp_projects_data_local_path = /opt/sparkjobs/ml-analytics-service/urgent_data_metrics/output/ -nvsk_imp_projects_data_blob_path = "Manage_Learn_Data/micro_improvement/" +nvsk_imp_projects_data_blob_path = Manage_Learn_Data/micro_improvement/ [API_CREDENTIALS] -client_id = "{{ ml_analytics_client_id }}" +client_id = {{ ml_analytics_client_id }} -client_secret = "{{ ml_analytics_client_secret }}" +client_secret = {{ ml_analytics_client_secret }} -username = "{{ ml_analytics_username }}" +username = {{ ml_analytics_username }} -password = "{{ ml_analytics_password }}" +password = {{ ml_analytics_password }} [JSON_VARIABLE] -createdBy = "{{ ml_analytics_createdBy }}" +createdBy = {{ ml_analytics_createdBy }} -container = "{{ ml_analytics_reports_container }}" +container = {{ ml_analytics_reports_container }} -store = "{{ ml_analytics_reports_store }}" +store = {{ ml_analytics_reports_store }} [ML_CORE_SERVICE_URL] -url = "http://{{private_ingressgateway_ip}}/ml-core" +url = http://{{private_ingressgateway_ip}}/ml-core From e837492f1dba20b4c94fe5f2f364cb637e87162f Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Thu, 22 Feb 2024 20:09:26 +0530 Subject: [PATCH 16/21] Update ml-analytics-service.conf --- .../stack-sunbird/templates/ml-analytics-service.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf index 6a60ceba58..e56eb046e2 100644 --- a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf +++ b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf @@ -130,6 +130,13 @@ survey_raw_topic = {{ env_name }}.ml.survey.raw survey_druid_topic = {{ env_name }}.ml.survey.druid +survey_meta_druid_topic = ml.survey.meta + +survey_completed_druid_topic = ml.survey.status.completed + +survey_started_druid_topic = ml.survey.status.started + +survey_inprogress_druid_topic = ml.survey.status.inprogress [LOGS] From b72173b7cb5769c839fbe0ae14b4a3aacc39e214 Mon Sep 17 00:00:00 2001 From: Raghupathi Guduri Date: Thu, 22 Feb 2024 20:10:13 +0530 Subject: [PATCH 17/21] Update ml-analytics-service.conf --- .../roles/stack-sunbird/templates/ml-analytics-service.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf index e56eb046e2..ae94018c2f 100644 --- a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf +++ b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf @@ -1,10 +1,10 @@ [ML_APP_NAME] -survey_app = "{{ ml_survey_app_name | default('sunbirdsurvey') }}" +survey_app = {{ ml_survey_app_name | default('sunbirdsurvey') }} -integrated_app = "{{ ml_integrated_app_name | default('sunbird') }}" +integrated_app = {{ ml_integrated_app_name | default('sunbird') }} -integrated_portal = "{{ ml_integrated_portal | default('dev.sunbird.portal') }}" +integrated_portal = {{ ml_integrated_portal | default('dev.sunbird.portal') }} [API_ENDPOINTS] From e9911a22fa6b26e22030fff6a713a987956750cb Mon Sep 17 00:00:00 2001 From: Vivek M <125434153+Vivek-M-08@users.noreply.github.com> Date: Fri, 8 Mar 2024 15:54:34 +0530 Subject: [PATCH 18/21] Update ml-analytics-service.conf --- .../stack-sunbird/templates/ml-analytics-service.conf | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf index ae94018c2f..a0b86b1ad9 100644 --- a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf +++ b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf @@ -79,6 +79,8 @@ batch_rollup_url = http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task urlQuery = http://{{groups['druid'][0]}}:8082/druid/v2?pretty +sql_url = http://{{groups['druid'][0]}}:8082/druid/v2/sql/ + observation_status_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_code","school_externalId","state_name","state_code","state_externalId","district_name","district_code","district_externalId","block_name","block_code","block_externalId","cluster_name","cluster_code","cluster_externalId","completedDate","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name","ecm_marked_na","board_name","updatedAt","organisation_id","user_type","observed_school_name","observed_school_id","observed_school_code","observed_state_name","observed_state_id","observed_state_code","observed_district_name","observed_district_id","observed_district_code","observed_block_name","observed_block_id","observed_block_code","observed_cluster_name","observed_cluster_id","observed_cluster_code","isRubricDriven","criteriaLevelReport"]},"metricsSpec":[]}}}' project_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-project","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"auto"},"dimensionsSpec":{"dimensions":[]},"metricsSpec":[]}}}' @@ -130,13 +132,13 @@ survey_raw_topic = {{ env_name }}.ml.survey.raw survey_druid_topic = {{ env_name }}.ml.survey.druid -survey_meta_druid_topic = ml.survey.meta +survey_meta_druid_topic = {{ env_name }}.ml.survey.meta -survey_completed_druid_topic = ml.survey.status.completed +survey_completed_druid_topic = {{ env_name }}.ml.survey.status.completed -survey_started_druid_topic = ml.survey.status.started +survey_started_druid_topic = {{ env_name }}.ml.survey.status.started -survey_inprogress_druid_topic = ml.survey.status.inprogress +survey_inprogress_druid_topic = {{ env_name }}.ml.survey.status.inprogress [LOGS] From e11141d1266406a6eba0e58d1c24b2589df1fa78 Mon Sep 17 00:00:00 2001 From: Vivek M <125434153+Vivek-M-08@users.noreply.github.com> Date: Tue, 12 Mar 2024 15:16:34 +0530 Subject: [PATCH 19/21] Updated ml-analytics-service.conf --- .../roles/stack-sunbird/templates/ml-analytics-service.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf index a0b86b1ad9..13514a1e38 100644 --- a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf +++ b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf @@ -71,6 +71,8 @@ survey_collection = surveys metadata_url = http://{{groups['druid'][0]}}:8081/druid/coordinator/v1/datasources/ +datasource_url = http://{{groups['druid'][0]}}:8082/druid/v2/datasources + batch_url = http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task metadata_rollup_url = http://{{groups['druid'][0]}}:8081/druid/coordinator/v1/datasources/ @@ -152,7 +154,7 @@ project_success_error = /opt/sparkjobs/ml-analytics-service/logs/project/ survey_evidence_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/project/evidence/ -survey_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/survey +survey_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/survey/ survey_status_success_error = /opt/sparkjobs/ml-analytics-service/logs/survey/status/ From d5456d085580cfa5e6df67654170b3b7fb63c64f Mon Sep 17 00:00:00 2001 From: Vivek M <125434153+Vivek-M-08@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:27:55 +0530 Subject: [PATCH 20/21] added new kafka topics for observations --- .../stack-sunbird/templates/ml-analytics-service.conf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf index 13514a1e38..f88342cc77 100644 --- a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf +++ b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf @@ -142,6 +142,14 @@ survey_started_druid_topic = {{ env_name }}.ml.survey.status.started survey_inprogress_druid_topic = {{ env_name }}.ml.survey.status.inprogress +observation_meta_druid_topic = {{ env_name }}.ml.observation.meta + +observation_started_druid_topic = {{ env_name }}.ml.observation.status.started + +observation_inprogress_druid_topic = {{ env_name }}.ml.observation.status.inprogress + +observation_completed_druid_topic = {{ env_name }}.ml.observation.status.completed + [LOGS] observation_streaming_success_error = /opt/sparkjobs/ml-analytics-service/logs/observation/ From 6a853065723a1802078a55a72b1c40394c261476 Mon Sep 17 00:00:00 2001 From: Vivek M <125434153+Vivek-M-08@users.noreply.github.com> Date: Mon, 1 Apr 2024 18:15:26 +0530 Subject: [PATCH 21/21] Update ml-analytics-service.conf --- .../roles/stack-sunbird/templates/ml-analytics-service.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf index f88342cc77..74799fdf4c 100644 --- a/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf +++ b/ansible/roles/stack-sunbird/templates/ml-analytics-service.conf @@ -71,7 +71,7 @@ survey_collection = surveys metadata_url = http://{{groups['druid'][0]}}:8081/druid/coordinator/v1/datasources/ -datasource_url = http://{{groups['druid'][0]}}:8082/druid/v2/datasources +datasource_url = http://{{groups['ml-raw-broker'][0]}}:8082/druid/v2/datasources batch_url = http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task @@ -81,7 +81,7 @@ batch_rollup_url = http://{{groups['druid'][0]}}:8081/druid/indexer/v1/task urlQuery = http://{{groups['druid'][0]}}:8082/druid/v2?pretty -sql_url = http://{{groups['druid'][0]}}:8082/druid/v2/sql/ +sql_url = http://{{groups['ml-raw-broker'][0]}}:8082/druid/v2/sql/ observation_status_injestion_spec = '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "CloudType","uris": ["CloudURI"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_code","school_externalId","state_name","state_code","state_externalId","district_name","district_code","district_externalId","block_name","block_code","block_externalId","cluster_name","cluster_code","cluster_externalId","completedDate","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name","ecm_marked_na","board_name","updatedAt","organisation_id","user_type","observed_school_name","observed_school_id","observed_school_code","observed_state_name","observed_state_id","observed_state_code","observed_district_name","observed_district_id","observed_district_code","observed_block_name","observed_block_id","observed_block_code","observed_cluster_name","observed_cluster_id","observed_cluster_code","isRubricDriven","criteriaLevelReport"]},"metricsSpec":[]}}}'