Skip to content
This repository has been archived by the owner on May 3, 2024. It is now read-only.

Commit

Permalink
Merge branch 'main' into fis-release
Browse files Browse the repository at this point in the history
  • Loading branch information
@madhavemuri
madhavemuri authored Aug 25, 2022
2 parents 8e322c4 + b23be2c commit 5c27c8e
Show file tree
Hide file tree
Showing 5 changed files with 98 additions and 57 deletions.
14 changes: 7 additions & 7 deletions be/ut/extmap.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
#include "be/ut/helper.h"
#include "be/extmap.h"

#define EXTMAP_UT_UNIT_SIZE 10
#define EXTMAP_UT_UNIT_SIZE 16
#define EXTMAP_UT_CS_SIZE 16

static struct m0_be_ut_backend be_ut_emap_backend;
Expand Down Expand Up @@ -550,8 +550,8 @@ static void test_paste_checksum_validation(void)
M0_UT_ASSERT(seg->ee_ext.e_end == M0_BINDEX_MAX + 1);

idx = 0;
e.e_start = 50;
e.e_end = 100;
e.e_start = m0_round_up(50, EXTMAP_UT_UNIT_SIZE);
e.e_end = m0_round_up(100, EXTMAP_UT_UNIT_SIZE);
es[idx] = e_temp[idx] = e;
e_val[idx] = 12;

Expand Down Expand Up @@ -601,8 +601,8 @@ static void test_paste_checksum_validation(void)
* New segment paste operation 1
*/
idx = 1;
e.e_start = 100;
e.e_end = 150;
e.e_start = m0_round_up(100, EXTMAP_UT_UNIT_SIZE);
e.e_end = m0_round_up(150, EXTMAP_UT_UNIT_SIZE);
es[idx] = e_temp[idx] = e;
e_val[idx] = 11;

Expand Down Expand Up @@ -660,8 +660,8 @@ static void test_paste_checksum_validation(void)
* New segment overwrite paste operation
*/
idx = 2;
e.e_start = 80;
e.e_end = 130;
e.e_start = m0_round_up(80, EXTMAP_UT_UNIT_SIZE);
e.e_end = m0_round_up(130, EXTMAP_UT_UNIT_SIZE);
es[idx] = e_temp[idx] = e;
e_val[idx] = 13;

Expand Down
56 changes: 43 additions & 13 deletions motr/io_nw_xfer.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -528,15 +528,29 @@ static void target_ioreq_seg_add(struct target_ioreq *ti,

/* For checksum of Parity Unit the global object offset will be
* assumed to be aligned with PG start offset, so removing the
* additional value NxUS w.r.t PG start, which is added by the
* additional value NxUS w.r.t PG start, which is added by the
* nw_xfer_io_distribute() function. src.sa_unit = layout_n(play) + unit
* Removing this offset N will help to compute PG unit idx as 0,1..,k-1
* Removing this offset N will help to compute PG unit idx as 0,1..,k-1
* which is the index of pi_paritybufs
*/
goff_cksum = unit_type == M0_PUT_DATA ? gob_offset :
*/
goff_cksum = unit_type == M0_PUT_DATA ? gob_offset :
(gob_offset + (src->sa_unit - layout_n(play)) *
layout_unit_size(play));


/**
* There are scenarios where K > N in such case when the
* unit type is M0_PUT_PARITY, adding [(K-N) x gob offset] into
* global object offset so that right PG and Unit index will get
* computed.
*/
if ((unit_type == M0_PUT_PARITY) &&
(layout_k(play) > layout_n(play))) {
m0_bcount_t goff_delta = (layout_k(play) -
layout_n(play)) *
gob_offset;
goff_cksum += goff_delta;
}

M0_LOG(M0_DEBUG,
"[gpos %" PRIu64 ", count %" PRIu64 "] [%" PRIu64 ", %" PRIu64 "]"
"->[%" PRIu64 ",%" PRIu64 "] %c", gob_offset, count, src->sa_group,
Expand Down Expand Up @@ -642,7 +656,7 @@ static void target_ioreq_seg_add(struct target_ioreq *ti,
* goff_ivec according to target offset. This creates a
* mapping between target offset and cheksum offset.
*
* This mapping will be used to compute PG Index and
* This mapping will be used to compute PG Index and
* Unit Index for each target when FOP is being prepared.
*/
INDEX(goff_ivec, seg) = goff_cksum;
Expand Down Expand Up @@ -900,7 +914,7 @@ static int target_ioreq_prepare_checksum(struct m0_op_io *ioo,

/* Number of units will not be zero as its already checked */
num_units = irfop->irf_cksum_data.cd_num_units;

/**
* Note: No need to free this as RPC layer will free this
* Allocate cksum buffer for number of units added to target_ioreq ti
Expand Down Expand Up @@ -963,6 +977,8 @@ static void target_ioreq_calc_idx(struct m0_op_io *ioo,
struct fop_cksum_data *fop_cs_data = &irfop->irf_cksum_data;
uint32_t seg;
m0_bindex_t goff;
struct m0_pdclust_layout *play = pdlayout_get(ioo);
m0_bcount_t grp0_idx;

/**
* Loop through all the segments added and check & add
Expand All @@ -975,10 +991,24 @@ static void target_ioreq_calc_idx(struct m0_op_io *ioo,
&fop_cs_data->cd_idx[fop_cs_data->cd_num_units];
M0_ASSERT(cs_idx->ci_pg_idx == UINT32_MAX &&
cs_idx->ci_unit_idx == UINT32_MAX);
grp0_idx = pgdata->fg_pgrp0_index;
/**
* There are scenarios where K > N in such case when the
* unit type is parity, shifting grp0 index by
* [(K-N) x pgrp0] for parity so that right PG and Unit
* index will get computed.
*/
if((irfop->irf_pattr == PA_PARITY) &&
(layout_k(play) > layout_n(play))) {
grp0_idx += (layout_k(play) -
layout_n(play)) *
pgdata->fg_pgrp0_index;
}

/* Compute PG Index & remaining exta wrt PG boundary */
cs_idx->ci_pg_idx = (goff - pgdata->fg_pgrp0_index) /
cs_idx->ci_pg_idx = (goff - grp0_idx) /
pgdata->fg_pgrp_sz;
rem_pg_sz = (goff - pgdata->fg_pgrp0_index) %
rem_pg_sz = (goff - grp0_idx) %
pgdata->fg_pgrp_sz;
cs_idx->ci_unit_idx = rem_pg_sz/pgdata->fg_unit_sz;
fop_cs_data->cd_num_units++;
Expand Down Expand Up @@ -1094,13 +1124,13 @@ static int target_ioreq_iofops_prepare(struct target_ioreq *ti,

ndom_max_segs = m0_net_domain_get_max_buffer_segments(ndom);

di_enabled = m0__obj_is_di_enabled(ioo) &&
di_enabled = m0__obj_is_di_enabled(ioo) &&
ti->ti_goff_ivec.iv_vec.v_nr;

if (di_enabled) {
struct m0_pdclust_layout *play = pdlayout_get(ioo);
/* Init object global offset currsor for a given target */

/* Init object global offset currsor for a given target */
m0_ivec_cursor_init(&goff_curr, &ti->ti_goff_ivec);
seg_sz = m0__page_size(ioo);
pgdata.fg_seg_sz = m0__page_size(ioo);
Expand All @@ -1109,7 +1139,7 @@ static int target_ioreq_iofops_prepare(struct target_ioreq *ti,
pgdata.fg_unit_sz = layout_unit_size(pdlayout_get(ioo));

/**
* There are scenarios where K > N in such case when the
* There are scenarios where K > N in such case when the
* filter is PA_PARITY, increase the size of PG so that
* right PG and Unit index will get computed based on goff
*/
Expand Down
5 changes: 1 addition & 4 deletions motr/st/utils/motr_sync_replication_st.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,7 @@ m0t1fs_dir="$motr_st_util_dir/../../../m0t1fs/linux_kernel/st"
. $motr_st_util_dir/motr_local_conf.sh
. $motr_st_util_dir/motr_st_inc.sh

# TODO : N will be reverted to 1 after addressing the code fix in DI checksum
# computation in read path, for the specific case of N=1, K=2, 2 disc failure
# degraded read.
N=2
N=1
K=2
S=2
P=15
Expand Down
3 changes: 3 additions & 0 deletions motr/ut/io_nw_xfer.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,8 @@ static void ut_test_target_ioreq_seg_add(void)
m0_bufvec_alloc(&ti->ti_auxbufvec, 1, unit_size);
M0_SET0(&ioo->ioo_attr);
M0_ALLOC_ARR(ti->ti_pageattrs, 1);
m0_indexvec_alloc(&ti->ti_goff_ivec, 1);
ti->ti_goff_ivec.iv_vec.v_nr = 0;

target_ioreq_seg_add(ti, src, tgt, 111, 1, map);
M0_UT_ASSERT(ti->ti_ivec.iv_vec.v_nr == 1);
Expand All @@ -287,6 +289,7 @@ static void ut_test_target_ioreq_seg_add(void)
m0_bufvec_free(&ti->ti_bufvec);
m0_bufvec_free(&ti->ti_auxbufvec);
m0_indexvec_free(&ti->ti_ivec);
m0_indexvec_free(&ti->ti_goff_ivec);

ut_dummy_pargrp_iomap_delete(map, instance);
m0_free(tgt);
Expand Down
77 changes: 44 additions & 33 deletions scripts/jenkins/code-coverage.groovy
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
pipeline {
pipeline {

agent {
label "jenkins-client-for-cortx-motr"
label "code-coverage"
}

parameters {
string(name: 'NODE_HOST', defaultValue: '', description: 'Node 1 Host FQDN', trim: true)
string(name: 'NODE_USER', defaultValue: '', description: 'Host machine root user', trim: true)
string(name: 'NODE_PASS', defaultValue: '', description: 'Host machine root user password', trim: true)
string(name: 'BRANCH', defaultValue: '', description: 'Branch name', trim: true)
string(name: 'OPTIONS', defaultValue: '', description: 'Build options', trim: true)
booleanParam(name: 'UT', defaultValue: true, description: 'Run UT')
Expand All @@ -17,58 +14,72 @@ pipeline {
options {
timeout(time: 180, unit: 'MINUTES')
timestamps()
ansiColor('xterm')
ansiColor('xterm')
buildDiscarder(logRotator(numToKeepStr: "30"))
}

environment {
NODE_PASS = "${NODE_PASS.isEmpty() ? NODE_DEFAULT_SSH_CRED_PSW : NODE_PASS}"
TestList = "/tmp/list-of-unit-tests.txt"
}

stages {
stage('Checkout') {
steps {
checkout([$class: 'GitSCM', branches: [[name: "main"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PathRestriction', excludedRegions: '', includedRegions: 'scripts/third-party-rpm/.*']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'cortx-admin-github', url: "https://github.com/Seagate/cortx-motr"]]])
}
}
stage ('Compile, execute test and report code coverage') {
steps {
script { build_stage = env.STAGE_NAME }
sh label: 'Clean up and build motr and run ut for code coverage', script: '''
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} hostname
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "yum install -y httpd lcov"
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "systemctl start httpd.service"
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "rm -rf /root/cortx-motr"
sh label: 'Clean up, build motr, run ut/st and generate code coverage report', script: '''
hostname
pwd
yum install -y httpd lcov
systemctl start httpd.service
make uninstall || true
make clean || true
rm -rf cortx-motr
OLD_BUILD_NUMBER=$(echo "$BUILD_NUMBER - 10" | bc)
echo "Old build number is : $OLD_BUILD_NUMBER"
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "rm -rf /var/www/html/$OLD_BUILD_NUMBER || true"
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "cd /root/ ; git clone --recursive -b ${BRANCH} https://github.com/Seagate/cortx-motr"
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "cd /root/cortx-motr ; sudo ./scripts/install-build-deps"
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "cd /root/cortx-motr ; ./autogen.sh"
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "cd /root/cortx-motr ; ./configure --enable-coverage --enable-debug ${OPTIONS}"
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "cd /root/cortx-motr ; make -j6"
rm -rf /var/www/html/$OLD_BUILD_NUMBER || true
git clone --recursive -b ${BRANCH} https://github.com/Seagate/cortx-motr cortx-motr
cd cortx-motr
sudo ./scripts/install-build-deps
./autogen.sh
./configure --enable-coverage --enable-debug ${OPTIONS}
make -j6
if [ ${UT} == true ]
then
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "cd /root/cortx-motr ; sudo ./scripts/m0 run-ut"
./scripts/m0 run-ut -l > "$TestList"
set +o errexit
while read Test
do
echo "Executing: $Test"
timeout --preserve-status -k 600s 600s ./scripts/m0 run-ut -t $Test
echo "Test $Test Executed and exited with return code $?"
done < "$TestList"
fi
if [ ${ST} == true ]
then
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "cd /root/cortx-motr ; sudo ./scripts/m0 run-st"
./scripts/m0 run-ut -l > "$TestList"
set +o errexit
while read Test
do
echo "Executing: $Test"
timeout --preserve-status -k 600s 600s ./scripts/m0 run-st -t $Test
echo "Test $Test Executed and exited with return code $?"
done < "$TestList"
fi
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "cd /root/cortx-motr ; ./scripts/coverage/gcov-gen-html user ./ ./"
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "mkdir -p /var/www/html/$BUILD_NUMBER"
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "cp -r /root/cortx-motr/* /var/www/html/$BUILD_NUMBER/"
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "systemctl start firewalld.service"
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "firewall-cmd --permanent --add-port=80/tcp"
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "firewall-cmd --permanent --add-port=443/tcp"
sshpass -p ${NODE_PASS} ssh -o StrictHostKeyChecking=no ${NODE_USER}@${NODE_HOST} "firewall-cmd --reload"
echo "http://${NODE_HOST}/${BUILD_NUMBER}/"
./scripts/coverage/gcov-gen-html user ./ ./
mkdir -p /var/www/html/$BUILD_NUMBER
cp -r * /var/www/html/$BUILD_NUMBER/
systemctl start firewalld.service
firewall-cmd --permanent --add-port=80/tcp
firewall-cmd --permanent --add-port=443/tcp
firewall-cmd --reload
echo "http://$(hostname)/${BUILD_NUMBER}/"
'''
}
}
}
}

0 comments on commit 5c27c8e

Please sign in to comment.