Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

merge release-6.0.0 to 7.0.0 #3908

Merged
merged 6 commits into from
Oct 5, 2023
Merged

merge release-6.0.0 to 7.0.0 #3908

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
408c8c4
Fix syntax (#3894)
santhosh-tg Sep 14, 2023
d6c6875
Updated vars for cron schedule (#3895)
raghupathiguduri Sep 19, 2023
8dc81f6
[ED-2793] Fix: release-6.0.0 installation issues (#3900)
santhosh-tg Sep 25, 2023
6320664
Update main.yml with cassandra (#3845) (#3901)
santhosh-tg Sep 25, 2023
6ce00cf
ED-2838: ##Release-6.0.0 new variables (#3899)
tsprasath Sep 26, 2023
3924727
[ED-2793] Fix installation issues (#3905)
santhosh-tg Sep 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ansible/roles/cassandra/defaults/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
user_home: "/home/{{ ansible_ssh_user }}/"
cassandra_repo: 'deb http://www.apache.org/dist/cassandra/debian 311x main'
cassandra_repo_key: 'https://www.apache.org/dist/cassandra/KEYS'
cassandra_repo: 'deb https://debian.cassandra.apache.org 311x main'
cassandra_repo_key: 'https://downloads.apache.org/cassandra/KEYS'
resource_crunch: "yes" # make this value yes if heap size should be quarter of the Server Memory when you are deploying multiple databases, else make this value no to have 50% of the value as per the best practice of cassandra
cassandra_home_directory: "/var/lib/cassandra"
5 changes: 5 additions & 0 deletions ansible/roles/ml-analytics-service/defaults/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,3 +133,8 @@ ml_analytics_reports_store: "{{ cloud_service_provider }}"
ml_analytics_reports_container: "{{ cloud_storage_privatereports_bucketname }}"
ml_analytics_driver_memory: "{{ ml_analytics_default_driver_memory | default('5g') }}"
ml_analytics_executor_memory: "{{ ml_analytics_default_executor_memory | default('5g') }}"
ml_analytics_batch_cron_minute: "{{ ml_batch_cron_minute | default('30') }}"
ml_analytics_batch_cron_hour: "{{ ml_batch_cron_hour | default('18') }}"
ml_analytics_nvsk_cron_minute: "{{ ml_nvsk_cron_minute | default('30') }}"
ml_analytics_nvsk_cron_hour: "{{ ml_nvsk_cron_hour | default('7') }}"
ml_analytics_nvsk_cron_weekday: "{{ ml_nvsk_cron_weekday | default('4') }}"
11 changes: 6 additions & 5 deletions ansible/roles/ml-analytics-service/tasks/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
- python3-virtualenv
- zip
- unzip
- acl
state: present

- name: CHANGE THE OWNERSHIP FOR THIS {{ BASEPATH }} DIRECTORY
Expand Down Expand Up @@ -135,15 +136,15 @@
cron:
name: "Run Batch Ingestion Job"
user: "{{ USER }}"
minute: "30"
hour: "18"
minute: "{{ ml_analytics_batch_cron_minute }}"
hour: "{{ ml_analytics_batch_cron_hour }}"
job: "{{ BASEPATH }}/ml-analytics-service/run.sh > {{ BASEPATH }}/ml-analytics-service/crontab_job.log"

- name: CREATE THE CRON NVSK Data Upload
cron:
name: "NVSK Data Upload JOB"
user: "{{ USER }}"
minute: "30"
hour: "7"
weekday: "4"
minute: "{{ ml_analytics_nvsk_cron_minute }}"
hour: "{{ ml_analytics_nvsk_cron_hour }}"
weekday: "{{ ml_analytics_nvsk_cron_weekday }}"
job: "{{ BASEPATH }}/ml-analytics-service/run_weekly.sh > {{ BASEPATH }}/ml-analytics-service/nvsk_data_weekly.logs"
7 changes: 4 additions & 3 deletions deploy/jenkins/jenkins-server-setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ echo -e "\n\e[0;32m${bold}Installating JDK8${normal}\n"
apt-get install -y openjdk-8-jdk

echo -e "\n\e[0;32m${bold}Installating Jenkins${normal}"
wget -q -O - https://pkg.jenkins.io/debian-stable/jenkins.io.key | apt-key add -
wget -q -O - https://pkg.jenkins.io/debian-stable/jenkins.io-2023.key | apt-key add -
apt-add-repository "deb https://pkg.jenkins.io/debian-stable binary/"
apt-get update
apt-get install -y jenkins=2.319.3
apt-get install -y jenkins=2.346.3

echo -e "\n\e[0;32m${bold}Installating PIP${normal}"
apt-get install -y python-pip
Expand Down Expand Up @@ -115,6 +115,7 @@ su jenkins bash -c "curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.

echo -e "\n\e[0;32m${bold}Installing jmespath${normal}"
sudo apt install -y python3-jmespath
sudo apt install python-jmespath #python2

#### Kubernetes Tools ####

Expand Down Expand Up @@ -155,7 +156,7 @@ rm openjdk-11.0.2_linux-x64_bin.tar.gz

#Install maven 3.6.3
echo -e "\n\e[0;32m${bold}Installating maven 3.6.3${normal}"
wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
wget https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
tar -xf apache-maven-3.6.3-bin.tar.gz
mv apache-maven-3.6.3 /opt/
mv /opt/apache-maven-3.6.3/bin/mvn /opt/apache-maven-3.6.3/bin/mvn3.6
Expand Down
108 changes: 108 additions & 0 deletions ...nkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
<?xml version="1.1" encoding="UTF-8" standalone="no"?><flow-definition plugin="workflow-job@2.40">
<actions>
<org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction plugin="workflow-multibranch@2.23">
<jobPropertyDescriptors>
<string>hudson.model.ParametersDefinitionProperty</string>
<string>com.sonyericsson.rebuild.RebuildSettings</string>
</jobPropertyDescriptors>
</org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction>
</actions>
<description/>
<keepDependencies>false</keepDependencies>
<properties>
<jenkins.model.BuildDiscarderProperty>
<strategy class="hudson.tasks.LogRotator">
<daysToKeep>-1</daysToKeep>
<numToKeep>10</numToKeep>
<artifactDaysToKeep>-1</artifactDaysToKeep>
<artifactNumToKeep>1</artifactNumToKeep>
</strategy>
</jenkins.model.BuildDiscarderProperty>
<org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
<com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.32">
<autoRebuild>false</autoRebuild>
<rebuildDisabled>false</rebuildDisabled>
</com.sonyericsson.rebuild.RebuildSettings>
<hudson.model.ParametersDefinitionProperty>
<parameterDefinitions>
<hudson.model.StringParameterDefinition>
<name>absolute_job_path</name>
<description>&lt;font color=dimgray size=2&gt;&lt;b&gt;Do not change this value! The metadata.json will be copied from this job.&lt;/b&gt;&lt;/font&gt;</description>
<defaultValue>Build/KnowledgePlatform/InquiryFlinkJob</defaultValue>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>image_tag</name>
<description>&lt;font color=darkgreen size=2&gt;&lt;b&gt;OPTIONAL: Specify the tag to upload a specific image version to the container registry.&lt;/b&gt;&lt;/font&gt;</description>
<defaultValue/>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
<hudson.model.ChoiceParameterDefinition>
<name>artifact_source</name>
<description>&lt;font color=dimgray size=2&gt;&lt;b&gt;
ArtifactRepo - Push the docker image to container registry.
&lt;/b&gt;&lt;/font&gt;</description>
<choices class="java.util.Arrays$ArrayList">
<a class="string-array">
<string>ArtifactRepo</string>
</a>
</choices>
</hudson.model.ChoiceParameterDefinition>
</parameterDefinitions>
</hudson.model.ParametersDefinitionProperty>
<hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.2">
<maxConcurrentPerNode>0</maxConcurrentPerNode>
<maxConcurrentTotal>0</maxConcurrentTotal>
<categories class="java.util.concurrent.CopyOnWriteArrayList"/>
<throttleEnabled>false</throttleEnabled>
<throttleOption>project</throttleOption>
<limitOneJobWithMatchingParams>false</limitOneJobWithMatchingParams>
<paramsToUseForLimit/>
</hudson.plugins.throttleconcurrents.ThrottleJobProperty>

<org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
<triggers>
<jenkins.triggers.ReverseBuildTrigger>
<spec/>
<upstreamProjects>Build/KnowledgePlatform/InquiryFlinkJob</upstreamProjects>
<threshold>
<name>SUCCESS</name>
<ordinal>0</ordinal>
<color>BLUE</color>
<completeBuild>true</completeBuild>
</threshold>
</jenkins.triggers.ReverseBuildTrigger>
</triggers>
</org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
</properties>
<definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.90">
<scm class="hudson.plugins.git.GitSCM" plugin="git@4.7.1">
<configVersion>2</configVersion>
<userRemoteConfigs>
<hudson.plugins.git.UserRemoteConfig>
<url>https://github.com/Sunbird-inQuiry/data-pipeline.git</url>
</hudson.plugins.git.UserRemoteConfig>
</userRemoteConfigs>
<branches>
<hudson.plugins.git.BranchSpec>
<name>${inquiry_pipeline_branch_or_tag}</name>
</hudson.plugins.git.BranchSpec>
</branches>
<doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
<submoduleCfg class="empty-list"/>
<extensions>
<hudson.plugins.git.extensions.impl.CloneOption>
<shallow>true</shallow>
<noTags>false</noTags>
<reference/>
<depth>0</depth>
<honorRefspec>false</honorRefspec>
</hudson.plugins.git.extensions.impl.CloneOption>
</extensions>
</scm>
<scriptPath>kubernetes/pipelines/upload/Jenkinsfile</scriptPath>
<lightweight>false</lightweight>
</definition>
<triggers/>
<disabled>false</disabled>
</flow-definition>
31 changes: 25 additions & 6 deletions deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,30 @@
<defaultValue>refs/heads/${public_repo_branch}</defaultValue>
<trim>true</trim>
</hudson.model.StringParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>CLOUD_STORE_GROUP_ID</name>
<description>&lt;font style="color:dimgray;font-size:14px;"&gt;&lt;b&gt;
&lt;li&gt;Mention the cloud storage sdk group id&lt;/li&gt;
&lt;/b&gt;&lt;/font&gt;</description>
<defaultValue>org.sunbird</defaultValue>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>CLOUD_STORE_ARTIFACT_ID</name>
<description>&lt;font style="color:dimgray;font-size:14px;"&gt;&lt;b&gt;
&lt;li&gt;Mention the cloud storage sdk artifact id&lt;/li&gt;
&lt;/b&gt;&lt;/font&gt;</description>
<defaultValue>cloud-store-sdk_2.12</defaultValue>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>CLOUD_STORE_VERSION</name>
<description>&lt;font style="color:dimgray;font-size:14px;"&gt;&lt;b&gt;
&lt;li&gt;Mention the cloud storage sdk version&lt;/li&gt;
&lt;/b&gt;&lt;/font&gt;</description>
<defaultValue>1.4.0</defaultValue>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
</parameterDefinitions>
</hudson.model.ParametersDefinitionProperty>
<hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.0.3">
Expand All @@ -48,12 +72,7 @@
</hudson.plugins.throttleconcurrents.ThrottleJobProperty>

<org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
<triggers>
<hudson.triggers.SCMTrigger>
<spec>H/15 * * * *</spec>
<ignorePostCommitHooks>false</ignorePostCommitHooks>
</hudson.triggers.SCMTrigger>
</triggers>
<triggers/>
</org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
</properties>
<definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.86">
Expand Down
55 changes: 37 additions & 18 deletions deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<?xml version="1.1" encoding="UTF-8" standalone="no"?><flow-definition plugin="workflow-job@2.40">
<actions>
<org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction plugin="workflow-multibranch@2.22">
<org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction plugin="workflow-multibranch@2.23">
<jobPropertyDescriptors>
<string>hudson.model.ParametersDefinitionProperty</string>
<string>com.sonyericsson.rebuild.RebuildSettings</string>
Expand All @@ -19,26 +19,47 @@
</strategy>
</jenkins.model.BuildDiscarderProperty>
<org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
<com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.31">
<com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.32">
<autoRebuild>false</autoRebuild>
<rebuildDisabled>false</rebuildDisabled>
</com.sonyericsson.rebuild.RebuildSettings>
<hudson.model.ParametersDefinitionProperty>
<parameterDefinitions>
<hudson.model.StringParameterDefinition>
<name>github_release_tag</name>
<name>inquiry_release_tag</name>
<description>&lt;font style="color:dimgray;font-size:14px;"&gt;&lt;b&gt;
&lt;li&gt;To build from a tag, use refs/tags/github_tag&lt;/li&gt;
&lt;li&gt;To build from a branch, use refs/heads/github_branch&lt;/li&gt;
&lt;li&gt;The default value of ${public_repo_branch} will be the release / tag version set in global configuration&lt;/li&gt;
&lt;li&gt;To build from a differnt branch, replace the ${public_repo_branch} with your branch&lt;/li&gt;
&lt;li&gt;The default value of ${inquiry_service_build_branch_or_tag} will be the release / tag version set in global configuration&lt;/li&gt;
&lt;li&gt;To build from a differnt branch, replace the ${inquiry_service_build_branch_or_tag} with your branch&lt;/li&gt;
&lt;/b&gt;&lt;/font&gt;</description>
<defaultValue>refs/heads/${public_repo_branch}</defaultValue>
<defaultValue>refs/heads/${inquiry_service_build_branch_or_tag}</defaultValue>
<trim>true</trim>
</hudson.model.StringParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>core_release_tag</name>
<description>&lt;font style="color:dimgray;font-size:14px;"&gt;&lt;b&gt;
&lt;li&gt;Provide this value to build core components from Knowlg BB&lt;/li&gt;
&lt;li&gt;To build from a tag, use refs/tags/github_tag&lt;/li&gt;
&lt;li&gt;To build from a branch, use refs/heads/github_branch&lt;/li&gt;
&lt;li&gt;The default value of ${inquiry_core_build_branch_or_tag} will be the release / tag version set in global configuration&lt;/li&gt;
&lt;li&gt;To build from a differnt branch, replace the ${inquiry_core_build_branch_or_tag} with your branch&lt;/li&gt;
&lt;/b&gt;&lt;/font&gt;</description>
<defaultValue>refs/heads/${inquiry_core_build_branch_or_tag}</defaultValue>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
<hudson.model.ChoiceParameterDefinition>
<name>core_repo_link</name>
<description>Knowlg Core Repo</description>
<choices class="java.util.Arrays$ArrayList">
<a class="string-array">
<string>https://github.com/project-sunbird/knowledge-platform.git</string>
</a>
</choices>
</hudson.model.ChoiceParameterDefinition>
</parameterDefinitions>
</hudson.model.ParametersDefinitionProperty>
<hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.0.3">
<hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.2">
<maxConcurrentPerNode>0</maxConcurrentPerNode>
<maxConcurrentTotal>0</maxConcurrentTotal>
<categories class="java.util.concurrent.CopyOnWriteArrayList"/>
Expand All @@ -49,31 +70,29 @@
</hudson.plugins.throttleconcurrents.ThrottleJobProperty>

<org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
<triggers>
<hudson.triggers.SCMTrigger>
<spec>H/15 * * * *</spec>
<ignorePostCommitHooks>false</ignorePostCommitHooks>
</hudson.triggers.SCMTrigger>
</triggers>
<triggers/>
</org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
</properties>
<definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.87">
<scm class="hudson.plugins.git.GitSCM" plugin="git@4.5.0">
<definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.90">
<scm class="hudson.plugins.git.GitSCM" plugin="git@4.7.1">
<configVersion>2</configVersion>
<userRemoteConfigs>
<hudson.plugins.git.UserRemoteConfig>
<url>https://github.com/project-sunbird/knowledge-platform.git</url>
<url>https://github.com/Sunbird-inQuiry/inquiry-api-service.git</url>
</hudson.plugins.git.UserRemoteConfig>
</userRemoteConfigs>
<branches>
<hudson.plugins.git.BranchSpec>
<name>${github_release_tag}</name>
<name>${inquiry_release_tag}</name>
</hudson.plugins.git.BranchSpec>
</branches>
<doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
<submoduleCfg class="empty-list"/>
<extensions/>
</scm>
<scriptPath>build/assessment-service/Jenkinsfile</scriptPath>
<lightweight>false</lightweight>
</definition>
<triggers/>
<disabled>false</disabled>
</flow-definition>
</flow-definition>
18 changes: 18 additions & 0 deletions deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,24 @@
<defaultValue>refs/heads/${public_repo_branch}</defaultValue>
<trim>true</trim>
</hudson.model.StringParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>CLOUD_STORE_GROUP_ID</name>
<description>Set the Cloud store sdk group id. e.g. org.sunbird</description>
<defaultValue>${cloud_store_group_id}</defaultValue>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>CLOUD_STORE_ARTIFACT_ID</name>
<description>Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12</description>
<defaultValue>${cloud_store_artifact_id}</defaultValue>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>CLOUD_STORE_VERSION</name>
<description>Set the Cloud store sdk version. e.g. 1.4.6</description>
<defaultValue>${cloud_store_version}</defaultValue>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
</parameterDefinitions>
</hudson.model.ParametersDefinitionProperty>
<hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.1">
Expand Down
20 changes: 19 additions & 1 deletion deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,24 @@
<defaultValue>refs/heads/${public_repo_branch}</defaultValue>
<trim>true</trim>
</hudson.model.StringParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>cloud_store_group_id</name>
<description>Set the Cloud store sdk group id. e.g. org.sunbird</description>
<defaultValue>${cloud_store_group_id}</defaultValue>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>cloud_store_artifact_id</name>
<description>Set the Cloud store sdk artifact id. e.g. cloud-store-sdk</description>
<defaultValue>${cloud_store_artifact_id}</defaultValue>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>cloud_store_version</name>
<description>Set the Cloud store sdk version. e.g. 1.4.6</description>
<defaultValue>${cloud_store_version}</defaultValue>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
</parameterDefinitions>
</hudson.model.ParametersDefinitionProperty>
<hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.0.3">
Expand All @@ -51,7 +69,7 @@
<org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
<triggers>
<hudson.triggers.SCMTrigger>
<spec>H/15 * * * *</spec>
<spec/>
<ignorePostCommitHooks>false</ignorePostCommitHooks>
</hudson.triggers.SCMTrigger>
</triggers>
Expand Down
Loading
Loading