diff --git a/be/btree.c b/be/btree.c index 646bcff7819..ec31a8d3bb9 100644 --- a/be/btree.c +++ b/be/btree.c @@ -4113,6 +4113,7 @@ static bool fkvv_invariant(const struct nd *node) M0_BT_COB_OBJECT_INDEX, M0_BT_COB_FILEATTR_BASIC, M0_BT_CONFDB, + M0_BT_DTM0_LOG, M0_BT_UT_KV_OPS))) && _0C(h->fkvv_ksize != 0); } diff --git a/be/btree.h b/be/btree.h index 3afab759d40..3eb55dd456a 100644 --- a/be/btree.h +++ b/be/btree.h @@ -61,6 +61,7 @@ enum m0_btree_types { M0_BT_COB_FILEATTR_OMG, M0_BT_COB_BYTECOUNT, M0_BT_CONFDB, + M0_BT_DTM0_LOG, M0_BT_UT_KV_OPS, M0_BT_NR }; diff --git a/be/linux_kernel/stubs.c b/be/linux_kernel/stubs.c index 1f3db35aa8e..068f09e52c3 100644 --- a/be/linux_kernel/stubs.c +++ b/be/linux_kernel/stubs.c @@ -111,6 +111,13 @@ M0_INTERNAL bool m0_be_op_is_done(struct m0_be_op *op) return true; } +M0_INTERNAL int m0_be_op_tick_ret(struct m0_be_op *op, + struct m0_fom *fom, + int next_state) +{ + return 0; +} + M0_INTERNAL struct m0_be_allocator *m0_be_seg_allocator(struct m0_be_seg *seg) { return NULL; @@ -187,6 +194,11 @@ M0_INTERNAL void m0_be_tx_open(struct m0_be_tx *tx) tx->t_sm.sm_state = M0_BTS_ACTIVE; } +M0_INTERNAL int m0_be_tx_exclusive_open_sync(struct m0_be_tx *tx) +{ + return -ENOSYS; +} + M0_INTERNAL void m0_be_tx_capture(struct m0_be_tx *tx, const struct m0_be_reg *reg) { @@ -293,6 +305,12 @@ M0_INTERNAL struct m0_be_seg *m0_be_domain_seg0_get(struct m0_be_domain *dom) return NULL; } +M0_INTERNAL struct m0_be_seg * +m0_be_domain_seg_first(const struct m0_be_domain *dom) +{ + return NULL; +} + int m0_be_tx_fol_add(struct m0_be_tx *tx, struct m0_fol_rec *rec) { return M0_ERR(-EINVAL); @@ -327,5 +345,22 @@ M0_INTERNAL void m0_be_queue_unlock(struct m0_be_queue *bq) { } +void m0_be_0type_add_credit(struct m0_be_domain *dom, + const struct m0_be_0type *zt, + const char *suffix, + const struct m0_buf *data, + struct m0_be_tx_credit *credit) +{ +} + +int m0_be_0type_add(struct m0_be_0type *zt, + struct m0_be_domain *dom, + struct m0_be_tx *tx, + const char *suffix, + const struct m0_buf *data) +{ + return -ENOSYS; +} + #undef M0_TRACE_SUBSYSTEM diff --git a/be/seg0.c b/be/seg0.c index cd988e5ee8a..d5653ffcd85 100644 --- a/be/seg0.c +++ b/be/seg0.c @@ -115,7 +115,7 @@ int m0_be_0type_del(struct m0_be_0type *zt, const char *suffix) { struct m0_be_seg *seg; - struct m0_buf *opt; + struct m0_buf *opt = NULL; char keyname[256] = {}; int rc; diff --git a/be/seg0.h b/be/seg0.h index feee03cf5c8..6cea86708d8 100644 --- a/be/seg0.h +++ b/be/seg0.h @@ -114,6 +114,15 @@ int m0_be_0type_del(struct m0_be_0type *zt, struct m0_be_tx *tx, const char *suffix); + +/** + * Returns a record with a given m0_be_0type and a given suffix. + * @return -ENOENT no such record exists. + */ +int m0_be_0type_get(struct m0_be_0type *zt, + struct m0_be_domain *dom, + const char *suffix, + struct m0_buf *data); /** *
* @section seg0-metadata Meta-segment (seg0), systematic BE storage startup. diff --git a/be/ut/helper.c b/be/ut/helper.c index eaac6a1a3aa..3af7eb31360 100644 --- a/be/ut/helper.c +++ b/be/ut/helper.c @@ -262,6 +262,7 @@ void m0_be_ut_backend_cfg_default(struct m0_be_domain_cfg *cfg) extern struct m0_be_0type m0_be_cob0; extern struct m0_be_0type m0_be_active_record0; extern struct m0_be_0type m0_be_dtm0; + extern struct m0_be_0type m0_dtm0_log0; static struct m0_atomic64 dom_key = { .a_value = 0xbef11e }; static const struct m0_be_0type *zts[] = { @@ -269,6 +270,7 @@ void m0_be_ut_backend_cfg_default(struct m0_be_domain_cfg *cfg) &m0_be_cob0, &m0_be_active_record0, &m0_be_dtm0, + &m0_dtm0_log0, }; struct m0_reqh *reqh = cfg->bc_engine.bec_reqh; diff --git a/cas/cas.c b/cas/cas.c index e8e37995d80..5ca2749bc9e 100644 --- a/cas/cas.c +++ b/cas/cas.c @@ -34,6 +34,7 @@ #include "mdservice/fsync_fops.h" /* m0_fsync_fom_ops */ #include "mdservice/fsync_fops_xc.h" /* m0_fop_fsync_xc */ #include "cas/client.h" /* m0_cas_sm_conf_init */ +#include "lib/memory.h" /* M0_ALLOC_PTR */ struct m0_fom_type_ops; struct m0_sm_conf; @@ -285,6 +286,62 @@ M0_INTERNAL bool m0_crv_is_none(const struct m0_crv *crv) return memcmp(crv, &M0_CRV_INIT_NONE, sizeof(*crv)) == 0; } +M0_INTERNAL bool m0_cas_fop_is_redoable(struct m0_fop *fop) +{ + struct m0_fop_type *cas_fopt = &cas_put_fopt; + return fop->f_type == cas_fopt; +} + +M0_INTERNAL int m0_cas_fop2redo(const struct m0_fop *fop, + struct m0_dtm0_redo *redo) +{ + struct m0_cas_op *op = m0_fop_data(fop); + struct m0_buf payload = {}; + int rc; + + /* TODO: encode fop opcode and fop reply as well. */ + + rc = m0_xcode_obj_enc_to_buf(&M0_XCODE_OBJ(m0_cas_op_xc, op), + &payload.b_addr, &payload.b_nob); + if (rc != 0) + return M0_ERR(rc); + + rc = m0_dtm0_redo_init(redo, &op->cg_descriptor, + &payload, M0_DTX0_PAYLOAD_CAS); + m0_buf_free(&payload); + + return M0_RC(rc); +} + +M0_INTERNAL int m0_cas_redo2fop(struct m0_fop *fop, + const struct m0_dtm0_redo *redo) +{ + struct m0_buf *payload = redo->dtr_payload.dtp_data.ab_elems; + struct m0_cas_op *op; + struct m0_fop_type *cas_fopt; + int rc; + + M0_PRE(redo->dtr_payload.dtp_type == M0_DTX0_PAYLOAD_CAS); + M0_PRE(redo->dtr_payload.dtp_data.ab_count == 1); + + /* TODO: Select the right fop type based on encoded req type. */ + cas_fopt = &cas_put_fopt; + + M0_ALLOC_PTR(op); + if (op == NULL) + return M0_ERR(-ENOMEM); + + rc = m0_xcode_obj_dec_from_buf(&M0_XCODE_OBJ(m0_cas_op_xc, op), + payload->b_addr, payload->b_nob); + if (rc == 0) + m0_fop_init(fop, cas_fopt, op, &m0_fop_release); + else + m0_free(op); + + return M0_RC(rc); +} + + #undef M0_TRACE_SUBSYSTEM /** @} end of cas group */ diff --git a/cas/cas.h b/cas/cas.h index f1079619924..a2cec4a2ec8 100644 --- a/cas/cas.h +++ b/cas/cas.h @@ -42,6 +42,8 @@ #include "dix/layout_xc.h" #include "dtm0/tx_desc.h" /* tx_desc */ #include "dtm0/tx_desc_xc.h" /* xc for tx_desc */ +#include "dtm0/dtm0.h" /* m0_dtx0_descriptor */ +#include "dtm0/dtm0_xc.h" /* xc for m0_dtx0_descriptor */ /** * @page cas-fspec The catalogue service (CAS) @@ -398,6 +400,12 @@ struct m0_cas_op { * Transaction descriptor associated with CAS operation. */ struct m0_dtm0_tx_desc cg_txd; + + /** + * Transaction descriptor (new DTM0) associated with this CAS + * operation. + */ + struct m0_dtx0_descriptor cg_descriptor; } M0_XCA_RECORD M0_XCA_DOMAIN(rpc); /** @@ -475,6 +483,11 @@ M0_INTERNAL void m0_cas__ut_svc_be_set(struct m0_reqh_service *svc, struct m0_be_domain *dom); M0_INTERNAL struct m0_be_domain * m0_cas__ut_svc_be_get(struct m0_reqh_service *svc); +struct m0_dtm0_domain; +M0_INTERNAL void m0_cas__ut_svc_dtm0_domain_set(struct m0_reqh_service *svc, + struct m0_dtm0_domain *dod); +M0_INTERNAL struct m0_be_domain * +m0_cas__ut_svc_be_get(struct m0_reqh_service *svc); M0_INTERNAL int m0_cas_fom_spawn( struct m0_fom *lead, struct m0_fom_thralldom *thrall, @@ -542,6 +555,15 @@ M0_INTERNAL struct m0_dtm0_ts m0_crv_ts(const struct m0_crv *crv); M0_INTERNAL void m0_crv_ts_set(struct m0_crv *crv, const struct m0_dtm0_ts *ts); +struct m0_fop; +struct m0_dtm0_redo; + +M0_INTERNAL bool m0_cas_fop_is_redoable(struct m0_fop *fop); +M0_INTERNAL int m0_cas_fop2redo(const struct m0_fop *fop, + struct m0_dtm0_redo *redo); +M0_INTERNAL int m0_cas_redo2fop(struct m0_fop *fop, + const struct m0_dtm0_redo *redo); + /** @} end of cas_dfspec */ #endif /* __MOTR_CAS_CAS_H__ */ diff --git a/cas/client.c b/cas/client.c index 1b5f80b1a01..2db96124eb5 100644 --- a/cas/client.c +++ b/cas/client.c @@ -1649,6 +1649,47 @@ static int cas_req_prep(struct m0_cas_req *req, return M0_RC(rc); } +static int +m0_dtm0_tx_desc2dtx0_descriptor_prep(const struct m0_dtm0_tx_desc *txd, + struct m0_dtx0_descriptor *descriptor) +{ + static int i = 0; + + /* + * TODO: Check if txd is valid or not. If txd is not valid then + * just fill the descriptor with some random data. + * (txd is not valid == it was zeroed). + */ + descriptor->dtd_id.dti_originator_sdev_fid.f_key = i++; + descriptor->dtd_id.dti_originator_sdev_fid.f_container = 1; + descriptor->dtd_id.dti_timestamp = m0_time_now(); + M0_LOG(M0_DEBUG, "DTX id: " DTID1_F, DTID1_P(&descriptor->dtd_id)); + + /** + * @todo update all participants of this request in the descriptor, + * until then originator only is added as a participant. + */ + if (txd->dtd_ps.dtp_nr == 0) { + descriptor->dtd_participants.dtpa_participants_nr = 1; + M0_ALLOC_ARR(descriptor->dtd_participants.dtpa_participants, 1); + if (descriptor->dtd_participants.dtpa_participants == NULL) + return -ENOMEM; + descriptor->dtd_participants.dtpa_participants[0] = + descriptor->dtd_id.dti_originator_sdev_fid; + return 0; + } + + descriptor->dtd_participants.dtpa_participants_nr = txd->dtd_ps.dtp_nr; + M0_ALLOC_ARR(descriptor->dtd_participants.dtpa_participants, + txd->dtd_ps.dtp_nr); + if (descriptor->dtd_participants.dtpa_participants == NULL) + return -ENOMEM; + for (i = 0; i < txd->dtd_ps.dtp_nr; i++) + descriptor->dtd_participants.dtpa_participants[i] = + txd->dtd_ps.dtp_pa->p_fid; + return 0; +} + M0_INTERNAL int m0_cas_put(struct m0_cas_req *req, struct m0_cas_id *index, const struct m0_bufvec *keys, @@ -1683,6 +1724,12 @@ M0_INTERNAL int m0_cas_put(struct m0_cas_req *req, rc = m0_dtx0_txd_copy(dtx, &op->cg_txd); if (rc != 0) return M0_ERR(rc); + + rc = m0_dtm0_tx_desc2dtx0_descriptor_prep(&op->cg_txd, + &op->cg_descriptor); + if (rc != 0) + return M0_ERR(rc); + rc = creq_fop_create_and_prepare(req, &cas_put_fopt, op, &next_state); if (rc == 0) { cas_fop_send(req); diff --git a/cas/index_gc.c b/cas/index_gc.c index 33d2cbfe9b1..2ed530ba046 100644 --- a/cas/index_gc.c +++ b/cas/index_gc.c @@ -490,6 +490,7 @@ static void cgc_start_fom(struct m0_fom *fom0, struct m0_fop *fop) m0_fom_init(fom0, &fop->f_type->ft_fom_type, &cgc_fom_ops, fop, NULL, fom->cg_reqh); fom0->fo_local = true; + fom0->fo_local_update = true; fom->cg_ctg_op_initialized = false; m0_long_lock_link_init(&fom->cg_dead_index, fom0, &fom->cg_dead_index_addb2); diff --git a/cas/service.c b/cas/service.c index a8b1059af26..6558af0ef4d 100644 --- a/cas/service.c +++ b/cas/service.c @@ -26,6 +26,7 @@ #include "be/dtm0_log.h" /* m0_be_dtm0_log API */ #include "dtm0/fop.h" /* DTM0 msg and tx_desc */ #include "dtm0/service.h" /* m0_dtm0_service API */ +#include "dtm0/dtx0.h" /* m0_dtx0_redo_add */ #include "lib/trace.h" #include "lib/memory.h" #include "lib/finject.h" @@ -323,6 +324,7 @@ struct cas_service { * In this case a special invalid value is used. */ uint32_t c_sdev_id; + struct m0_dtm0_domain *c_dtm0_domain; }; struct cas_kv { @@ -383,6 +385,7 @@ struct cas_fom { uint64_t cf_kv_stats[STATS_KV_NR] [STATS_KV_IO_NR] [STATS_NR]; + struct m0_dtm0_redo *cf_redo; }; enum cas_fom_phase { @@ -458,6 +461,7 @@ static void cas_prep (struct cas_fom *fom, struct m0_cas_ctg *ctg, uint64_t rec_pos, struct m0_be_tx_credit *accum); +static int cas_dtm0_prep(struct cas_fom *fom); static int cas_exec (struct cas_fom *fom, enum m0_cas_opcode opc, enum m0_cas_type ct, @@ -517,6 +521,8 @@ static int cas_ctidx_lookup(struct cas_fom *fom, const struct m0_cas_id *in_cid, int next); static int cas_ctidx_delete(struct cas_fom *fom, const struct m0_cas_id *in_cid, int next); +static int cas_redo_alloc(struct m0_fom *fom0, struct m0_dtm0_redo **out); +static void cas_redo_free0(struct m0_dtm0_redo **redo); static const struct m0_reqh_service_ops cas_service_ops; static const struct m0_reqh_service_type_ops cas_service_type_ops; @@ -562,6 +568,13 @@ M0_INTERNAL void m0_cas__ut_svc_be_set(struct m0_reqh_service *svc, service->c_be_domain = dom; } +M0_INTERNAL void m0_cas__ut_svc_dtm0_domain_set(struct m0_reqh_service *svc, + struct m0_dtm0_domain *dod) +{ + struct cas_service *service = M0_AMB(service, svc, c_service); + service->c_dtm0_domain = dod; +} + M0_INTERNAL struct m0_be_domain * m0_cas__ut_svc_be_get(struct m0_reqh_service *svc) { @@ -606,6 +619,7 @@ static int cas_service_sdev_id_set(struct cas_service *cas_svc) sdev = M0_CONF_CAST(obj, m0_conf_sdev); if (!found) { cas_svc->c_sdev_id = sdev->sd_dev_idx; + M0_LOG(M0_DEBUG,"dev id:%d", (int) cas_svc->c_sdev_id); found = true; } else { /* @@ -624,17 +638,28 @@ static int cas_service_sdev_id_set(struct cas_service *cas_svc) return M0_RC(rc); } +M0_INTERNAL struct m0_dtm0_domain * +m0_cas__ut_svc_dtm0_domain_get(struct m0_reqh_service *svc) +{ + struct cas_service *service = M0_AMB(service, svc, c_service); + return service->c_dtm0_domain; +} + static int cas_service_start(struct m0_reqh_service *svc) { - int rc; - struct cas_service *service = M0_AMB(service, svc, c_service); - struct m0_be_domain *ut_dom; + int rc; + struct cas_service *service = M0_AMB(service, svc, c_service); + struct m0_be_domain *ut_be_dom; + struct m0_dtm0_domain *dod; M0_PRE(m0_reqh_service_state_get(svc) == M0_RST_STARTING); - ut_dom = m0_cas__ut_svc_be_get(svc); + ut_be_dom = m0_cas__ut_svc_be_get(svc); + dod = m0_cas__ut_svc_dtm0_domain_get(svc); /* XXX It's a workaround. It's needed until we have a better way. */ - service->c_be_domain = ut_dom != NULL ? - ut_dom : svc->rs_reqh_ctx->rc_beseg->bs_domain; + service->c_be_domain = ut_be_dom != NULL ? ut_be_dom : + svc->rs_reqh_ctx->rc_beseg->bs_domain; + service->c_dtm0_domain = dod != NULL ? dod : + &svc->rs_reqh_ctx->rc_dtm0_domain; service->c_sdev_id = INVALID_CAS_SDEV_ID; rc = m0_ctg_store_init(service->c_be_domain); if (rc == 0) { @@ -1146,12 +1171,70 @@ static int cas_dtm0_logrec_add(struct m0_fom *fom0, m0_dtm0_logrec_update(dtms->dos_log, &fom0->fo_tx.tx_betx, msg, &buf); m0_buf_free(&buf); - return rc; } +static inline const struct m0_fid *cas_fom_sdev_fid(struct cas_fom *fom) +{ + uint32_t sdev_idx; + struct m0_fom *fom0 = &fom->cf_fom; + struct m0_reqh *reqh = m0_fom2reqh(fom0); + struct m0_pool_version *pver; + struct m0_poolmach *pm; + struct m0_pooldev *sdev; + struct m0_cas_op *op = cas_op(fom0); + struct m0_cas_id *cid = &op->cg_id; + + M0_ASSERT(cas_fid_is_cctg(&cid->ci_fid)); + sdev_idx = m0_dix_fid_cctg_device_id(&cid->ci_fid); + M0_LOG(M0_DEBUG,"dev id:%d", (int) sdev_idx); + pver = m0_pool_version_find(reqh->rh_pools, + &cid->ci_layout.u.dl_desc.ld_pver); + if (pver != NULL) { + int i; + pm = &pver->pv_mach; + for (i = 0; i < pm->pm_state->pst_nr_devices; i++) { + sdev = &pm->pm_state->pst_devices_array[i]; + + if (sdev->pd_sdev_idx == sdev_idx) + return &sdev->pd_id; + } + } + return NULL; +} + +static void cas_fom_executed(struct cas_fom *fom) +{ + int rc; + struct m0_dtm0_domain *dod = + m0_cas__ut_svc_dtm0_domain_get(fom->cf_fom.fo_service); + struct m0_be_tx *tx = &fom->cf_fom.fo_tx.tx_betx; + struct m0_dtm0_redo *redo = fom->cf_redo; + const struct m0_fid *sdev_fid; + struct m0_fom *fom0 = &fom->cf_fom; + enum m0_cas_opcode opc = m0_cas_opcode(fom0->fo_fop); + + if (M0_IN(opc, (CO_PUT, CO_DEL)) && + cas_fid_is_cctg(&cas_op(fom0)->cg_id.ci_fid) && + redo != NULL) { + M0_LOG(M0_DEBUG, "Got CAS with txid: " DTID0_F, + DTID0_P(&cas_op(fom0)->cg_txd.dtd_id)); + M0_LOG(M0_DEBUG, "Got CAS with new txid: " DTID1_F, + DTID1_P(&cas_op(fom0)->cg_descriptor.dtd_id)); + sdev_fid = cas_fom_sdev_fid(fom); + if (sdev_fid == NULL) + return; + rc = m0_dtx0_redo_add(dod, tx, redo, sdev_fid); + M0_ASSERT_INFO(rc == 0, "Failed to update DTM0 log (%d)", rc); + cas_redo_free0(&fom->cf_redo); + //m0_dtm0_redo_fini(redo); + fom->cf_redo = NULL; + } +} + static void cas_fom_success(struct cas_fom *fom, enum m0_cas_opcode opc) { + cas_fom_executed(fom); cas_fom_cleanup(fom, opc == CO_CUR); m0_fom_phase_set(&fom->cf_fom, M0_FOPH_SUCCESS); } @@ -1275,9 +1358,12 @@ static int cas_fom_tick(struct m0_fom *fom0) M0_PRE(ergo(op->cg_flags & COF_NO_DTM, m0_dtm0_tx_desc_is_none(&op->cg_txd))); - if (!M0_IS0(&op->cg_txd) && phase == M0_FOPH_INIT) + if (!M0_IS0(&op->cg_txd) && phase == M0_FOPH_INIT) { M0_LOG(M0_DEBUG, "Got CAS with txid: " DTID0_F, DTID0_P(&op->cg_txd.dtd_id)); + M0_LOG(M0_DEBUG, "Got CAS new with txid: " DTID1_F, + DTID1_P(&op->cg_descriptor.dtd_id)); + } is_index_drop = op_is_index_drop(opc, ct); @@ -1504,19 +1590,12 @@ static int cas_fom_tick(struct m0_fom *fom0) cas_op(fom0)->cg_flags); } - /* - * If dtm0 is used we need to calculate credits for creating - * a dtm0 log record. - */ - if (is_dtm0_used) { - rc = cas_dtm0_logrec_credit_add(fom0); - if (rc != 0) { - cas_fom_failure(fom, M0_ERR(rc), false); - break; - } - } + rc = cas_dtm0_prep(fom); + if (rc != 0) + cas_fom_failure(fom, M0_ERR(rc), false); + else + m0_fom_phase_set(fom0, M0_FOPH_TXN_OPEN); - m0_fom_phase_set(fom0, M0_FOPH_TXN_OPEN); /* * @todo waiting for transaction open with btree (which can be * the meta-catalogue) locked, because tree height has to be @@ -1805,6 +1884,8 @@ static void cas_fom_fini(struct m0_fom *fom0) struct cas_fom *fom = M0_AMB(fom, fom0, cf_fom); uint64_t i; + cas_redo_free0(&fom->cf_redo); + for (i = 0; i < op->cg_rec.cr_nr; i++) { rec = cas_at(op, i); @@ -2220,6 +2301,39 @@ static void cas_prep(struct cas_fom *fom, enum m0_cas_opcode opc, } } +static int cas_dtm0_prep(struct cas_fom *fom) +{ + int rc; + struct m0_fom *fom0 = &fom->cf_fom; + struct m0_cas_op *op = cas_op(fom0); + struct m0_dtm0_domain *dod = + m0_cas__ut_svc_dtm0_domain_get(fom->cf_fom.fo_service); + /* Try to update "old" dtm0 log. */ + if (!m0_dtm0_tx_desc_is_none(&op->cg_txd)) { + rc = cas_dtm0_logrec_credit_add(fom0); + if (rc != 0) + goto out; + } else + rc = 0; + + M0_ASSERT(fom->cf_redo == NULL); + + /* Try to update "new" dtm0 log. */ + if (m0_cas_fop_is_redoable(fom0->fo_fop) && + cas_type(fom0) == CT_BTREE) { + /* See cas_redo_free0. */ + rc = cas_redo_alloc(fom0, &fom->cf_redo); + if (rc != 0) + goto out; + m0_dtx0_redo_add_credit(dod, fom->cf_redo, + &fom0->fo_tx.tx_betx_cred); + } else + rc = 0; + +out: + return rc; +} + static struct m0_cas_rec *cas_at(struct m0_cas_op *op, int idx) { M0_PRE(0 <= idx && idx < op->cg_rec.cr_nr); @@ -2688,6 +2802,7 @@ M0_INTERNAL int m0_cas_fom_spawn( rc = cas_fom_create(cas_fop, &new_fom0, reqh); if (rc == 0) { new_fom0->fo_local = true; + new_fom0->fo_local_update = true; m0_fom_enthrall(lead, new_fom0, thrall, @@ -2768,6 +2883,32 @@ static void cas_fom_addb2_descr(struct m0_fom *fom) } } +static int cas_redo_alloc(struct m0_fom *fom0, struct m0_dtm0_redo **out) +{ + struct m0_dtm0_redo *redo; + struct m0_fop *fop = fom0->fo_fop; + int rc; + + M0_ENTRY("fom0=%p", fom0); + M0_ALLOC_PTR(redo); + if (redo == NULL) + return M0_ERR(-ENOMEM); + rc = m0_cas_fop2redo(fop, redo); + if (rc == 0) + *out = redo; + else + m0_free(redo); + return M0_RC_INFO(rc, "redo=%p", redo); +} + +static void cas_redo_free0(struct m0_dtm0_redo **redo) +{ + if (*redo != NULL) { + m0_dtm0_redo_fini(*redo); + m0_free0(redo); + } +} + static const struct m0_fom_ops cas_fom_ops = { .fo_tick = &cas_fom_tick, .fo_home_locality = &cas_fom_home_locality, diff --git a/cas/ut/service_ut.c b/cas/ut/service_ut.c index af6ceddda74..66c08723595 100644 --- a/cas/ut/service_ut.c +++ b/cas/ut/service_ut.c @@ -45,6 +45,9 @@ #include "fdmi/fdmi.h" #include "rpc/rpc_machine.h" +#include "dtm0/cfg_default.h" /* m0_dtm0_domain_cfg_default_dup */ +#include "dtm0/domain.h" /* m0_dtm0_domain */ + #define IFID(x, y) M0_FID_TINIT('i', (x), (y)) #define TFID(x, y) M0_FID_TINIT('T', (x), (y)) @@ -57,7 +60,9 @@ struct meta_rec { static struct m0_reqh reqh; static struct m0_be_ut_backend be; +static struct m0_dtm0_domain dtm0_domain; static struct m0_be_seg *seg0; +//struct m0_be_ut_seg ut_seg; static struct m0_reqh_service *cas; static struct m0_reqh_service *fdmi; static struct m0_rpc_machine rpc_machine; @@ -136,9 +141,37 @@ static void reqh_init(bool mkfs, bool use_small_credits) cfg.bc_seg0_cfg.bsc_addr = m0_be_ut_seg_allocate_addr(segment_size); result = m0_be_ut_backend_init_cfg(&be, &cfg, mkfs); +// m0_be_ut_backend_init(&be); + /* m0_be_ut_seg_init(&ut_seg, &be, + M0_DTM0_UT_DOMAIN_SEG_SIZE); +*/ M0_ASSERT(result == 0); } +static void ut_dod_init(struct m0_dtm0_domain *dod, struct m0_reqh *reqh, + struct m0_be_domain *be_domain) +{ + int rc; + struct m0_dtm0_domain_cfg cfg = {}; + + rc = m0_dtm0_domain_cfg_default_dup(&cfg, true); + M0_UT_ASSERT(rc == 0); + + cfg.dodc_log.dlc_be_domain = be_domain; + cfg.dodc_log.dlc_seg = seg0; //m0_be_domain_seg_first(be_domain); +// M0_UT_ASSERT(cfg.dodc_log.dlc_seg != NULL); + cfg.dod_reqh = reqh; + + rc = m0_dtm0_domain_init(&dtm0_domain, &cfg); + M0_UT_ASSERT(rc == 0); +} + +static void ut_dod_fini(struct m0_dtm0_domain *dod) +{ + m0_dtm0_domain_fini(&dtm0_domain); + /* TODO */ +} + static void _init(bool mkfs, bool use_small_credits) { int result; @@ -160,6 +193,8 @@ static void _init(bool mkfs, bool use_small_credits) M0_UT_ASSERT(result == 0); m0_reqh_service_init(cas, &reqh, NULL); m0_cas__ut_svc_be_set(cas, &be.but_dom); + ut_dod_init(&dtm0_domain, &reqh, &be.but_dom); + m0_cas__ut_svc_dtm0_domain_set(cas, &dtm0_domain); m0_reqh_service_start(cas); m0_reqh_start(&reqh); cas__ut_cb_done = &cb_done; @@ -188,6 +223,7 @@ static void service_stop(void) static void fini(void) { service_stop(); + ut_dod_fini(&dtm0_domain); m0_be_ut_backend_fini(&be); m0_fi_disable("cas_in_ut", "ut"); rep_clear(); @@ -219,6 +255,8 @@ static void init_fail(void) reqh_init(true, false); + ut_dod_init(&dtm0_domain, &reqh, &be.but_dom); + /* Failure to add meta-index to segment dictionary. */ rc = m0_reqh_service_allocate(&cas, &m0_cas_service_type, NULL); M0_UT_ASSERT(rc == 0); @@ -978,7 +1016,7 @@ static void insert(void) init(); meta_fid_submit(&cas_put_fopt, &ifid); M0_UT_ASSERT(rep_check(0, 0, BUNSET, BUNSET)); - index_op(&cas_put_fopt, &ifid, 1, 2); + index_op(&cas_put_fopt, &ifid, 1, 2); M0_UT_ASSERT(rep.cgr_rc == 0); M0_UT_ASSERT(rep.cgr_rep.cr_nr == 1); M0_UT_ASSERT(rep_check(0, 0, BUNSET, BUNSET)); @@ -1854,7 +1892,9 @@ enum { * 2000 is enough to test multiple transactions if decrease transaction * size limit by using -c switch. */ - BIG_ROWS_NUMBER = 2000, + /* TODO: return back to 2000 when dtm0 log is on be_seg1 + * (currently it's on seg0) */ + BIG_ROWS_NUMBER = 500, /* * Number of rows for 2-level btree. */ @@ -1964,6 +2004,56 @@ static void init_cgc_fail_fini(void) m0_fi_disable("cgc_fom_tick", "fail_after_index_found"); } +static void dtm0_encdec(void) +{ + int ref_key = 0xABCD; + int ref_value = 0xCDBA; + struct m0_fid ref_index = m0_cas_meta_fid; + struct m0_cas_rec ref_rec[1] = { + { + .cr_key = (struct m0_rpc_at_buf) { + .ab_type = 1, + .u.ab_buf = M0_BUF_INIT_PTR(&ref_key), + }, + .cr_val = (struct m0_rpc_at_buf) { + .ab_type = 1, + .u.ab_buf = M0_BUF_INIT_PTR(&ref_value), + }, + .cr_rc = 0, + }, + }; + struct m0_cas_op ref_op[1] = {{ + .cg_id = { .ci_fid = ref_index }, + .cg_rec = { .cr_rec = ref_rec, .cr_nr = 1 } + }}; + struct m0_cas_op *redo_op; + struct m0_fop put_fop[1]; + struct m0_fop get_fop[1]; + struct m0_dtm0_redo put_redo[1]; + struct m0_fop put_redo_fop[1]; + int rc; + + m0_fop_init(put_fop, &cas_put_fopt, ref_op, &fop_release); + m0_fop_init(get_fop, &cas_get_fopt, ref_op, &fop_release); + + M0_UT_ASSERT(m0_cas_fop_is_redoable(put_fop)); + M0_UT_ASSERT(!m0_cas_fop_is_redoable(get_fop)); + + rc = m0_cas_fop2redo(put_fop, put_redo); + M0_UT_ASSERT(rc == 0); + + rc = m0_cas_redo2fop(put_redo_fop, put_redo); + M0_UT_ASSERT(rc == 0); + + redo_op = m0_fop_data(put_redo_fop); + M0_UT_ASSERT(m0_fid_eq(&redo_op->cg_id.ci_fid, + &ref_op->cg_id.ci_fid)); + M0_UT_ASSERT(m0_buf_eq(&redo_op->cg_rec.cr_rec->cr_key.u.ab_buf, + &ref_op->cg_rec.cr_rec->cr_key.u.ab_buf)); + M0_UT_ASSERT(m0_buf_eq(&redo_op->cg_rec.cr_rec->cr_val.u.ab_buf, + &ref_op->cg_rec.cr_rec->cr_val.u.ab_buf)); +} + struct m0_ut_suite cas_service_ut = { .ts_name = "cas-service", .ts_owners = "Nikita", @@ -2022,6 +2112,7 @@ struct m0_ut_suite cas_service_ut = { { "cctg-create-lookup", &cctg_create_lookup, "Sergey" }, { "cctg-create-delete", &cctg_create_delete, "Sergey" }, { "server-restart-nomkfs", &server_restart_nomkfs, "Egor" }, + { "dtm0-encdec", &dtm0_encdec, "Ivan" }, { NULL, NULL } } }; diff --git a/dtm0/Kbuild.sub b/dtm0/Kbuild.sub index 13cd6ea059d..1095c7f1ffd 100644 --- a/dtm0/Kbuild.sub +++ b/dtm0/Kbuild.sub @@ -7,6 +7,8 @@ m0tr_objects += \ dtm0/dtx0.o \ dtm0/fop.o \ dtm0/fop_xc.o \ + dtm0/dtm0.o \ + dtm0/dtm0_xc.o \ dtm0/linux_kernel/stubs.o \ dtm0/log.o \ dtm0/net.o \ diff --git a/dtm0/Makefile.sub b/dtm0/Makefile.sub index 3d9a3ffee64..76c94fcd75e 100644 --- a/dtm0/Makefile.sub +++ b/dtm0/Makefile.sub @@ -4,6 +4,7 @@ nobase_motr_include_HEADERS += \ dtm0/clk_src.h \ dtm0/domain.h \ dtm0/drlink.h \ + dtm0/dtm0.h \ dtm0/dtx.h \ dtm0/dtx0.h \ dtm0/fop.h \ @@ -24,6 +25,7 @@ motr_libmotr_la_SOURCES += \ dtm0/clk_src.c \ dtm0/domain.c \ dtm0/drlink.c \ + dtm0/dtm0.c \ dtm0/dtx.c \ dtm0/dtx0.c \ dtm0/fop.c \ @@ -40,12 +42,14 @@ motr_libmotr_la_SOURCES += \ nodist_motr_libmotr_la_SOURCES += \ dtm0/clk_src_xc.c \ + dtm0/dtm0_xc.c \ dtm0/fop_xc.c \ dtm0/tx_desc_xc.c XC_FILES += \ dtm0/clk_src_xc.h \ + dtm0/dtm0_xc.h \ dtm0/fop_xc.h \ dtm0/tx_desc_xc.h diff --git a/dtm0/cfg_default.c b/dtm0/cfg_default.c index 0562911ba28..d1b565eb2bd 100644 --- a/dtm0/cfg_default.c +++ b/dtm0/cfg_default.c @@ -32,12 +32,17 @@ #include "dtm0/domain.h" /* struct m0_dtm0_domain_cfg */ +#include "fid/fid.h" /* M0_FID_TINIT */ + M0_INTERNAL int -m0_dtm0_domain_cfg_default_dup(struct m0_dtm0_domain_cfg *dod_cfg) +m0_dtm0_domain_cfg_default_dup(struct m0_dtm0_domain_cfg *dod_cfg, bool mkfs) { *dod_cfg = (struct m0_dtm0_domain_cfg){ .dodc_log = { + .dlc_seg0_suffix = "default", + .dlc_be_domain = NULL, + .dlc_btree_fid = M0_FID_TINIT('b', 1 /* XXX */, 2), }, .dodc_pruner = { }, @@ -47,6 +52,8 @@ m0_dtm0_domain_cfg_default_dup(struct m0_dtm0_domain_cfg *dod_cfg) }, .dodc_net = { }, + .dod_create = mkfs, + .dod_destroy = false, }; return M0_RC(0); } diff --git a/dtm0/cfg_default.h b/dtm0/cfg_default.h index 38d8698a26d..1bae6df42ce 100644 --- a/dtm0/cfg_default.h +++ b/dtm0/cfg_default.h @@ -33,7 +33,7 @@ struct m0_dtm0_domain_cfg; M0_INTERNAL int -m0_dtm0_domain_cfg_default_dup(struct m0_dtm0_domain_cfg *dod_cfg); +m0_dtm0_domain_cfg_default_dup(struct m0_dtm0_domain_cfg *dod_cfg, bool mkfs); M0_INTERNAL struct m0_dtm0_domain_cfg * m0_dtm0_domain_cfg_dup(struct m0_dtm0_domain_cfg *dod_cfg); M0_INTERNAL void m0_dtm0_domain_cfg_free(struct m0_dtm0_domain_cfg *dod_cfg); diff --git a/dtm0/domain.c b/dtm0/domain.c index 555d986948d..b6022022d43 100644 --- a/dtm0/domain.c +++ b/dtm0/domain.c @@ -33,7 +33,8 @@ #include "lib/bob.h" /* M0_BOB_DEFINE */ #include "module/instance.h" /* m0_get */ #include "conf/helpers.h" /* m0_conf_process2service_get */ -#include "reqh/reqh.h" /* m0_reqh2confc */ +#include "reqh/reqh.h" /* m0_reqh fields */ + static const struct m0_bob_type dtm0_domain_bob_type = { .bt_name = "m0_dtm0_domain", @@ -60,8 +61,10 @@ static void dtm0_domain_level_leave(struct m0_module *module); enum dtm0_domain_level { M0_DTM0_DOMAIN_LEVEL_INIT, - M0_DTM0_DOMAIN_LEVEL_LOG_MKFS, - M0_DTM0_DOMAIN_LEVEL_LOG_INIT, + M0_DTM0_DOMAIN_LEVEL_LOG_CREATE, + M0_DTM0_DOMAIN_LEVEL_LOG_OPEN, + + M0_DTM0_DOMAIN_LEVEL_CFS_INIT, M0_DTM0_DOMAIN_LEVEL_REMACH_INIT, M0_DTM0_DOMAIN_LEVEL_PMACH_INIT, @@ -71,6 +74,7 @@ enum dtm0_domain_level { M0_DTM0_DOMAIN_LEVEL_REMACH_START, M0_DTM0_DOMAIN_LEVEL_PMACH_START, M0_DTM0_DOMAIN_LEVEL_PRUNER_INIT, + M0_DTM0_DOMAIN_LEVEL_PRUNER_START, M0_DTM0_DOMAIN_LEVEL_READY, }; @@ -78,8 +82,10 @@ enum dtm0_domain_level { static const struct m0_modlev levels_dtm0_domain[] = { DTM0_DOMAIN_LEVEL(M0_DTM0_DOMAIN_LEVEL_INIT), - DTM0_DOMAIN_LEVEL(M0_DTM0_DOMAIN_LEVEL_LOG_MKFS), - DTM0_DOMAIN_LEVEL(M0_DTM0_DOMAIN_LEVEL_LOG_INIT), + DTM0_DOMAIN_LEVEL(M0_DTM0_DOMAIN_LEVEL_LOG_CREATE), + DTM0_DOMAIN_LEVEL(M0_DTM0_DOMAIN_LEVEL_LOG_OPEN), + + DTM0_DOMAIN_LEVEL(M0_DTM0_DOMAIN_LEVEL_CFS_INIT), DTM0_DOMAIN_LEVEL(M0_DTM0_DOMAIN_LEVEL_REMACH_INIT), DTM0_DOMAIN_LEVEL(M0_DTM0_DOMAIN_LEVEL_PMACH_INIT), @@ -89,6 +95,7 @@ static const struct m0_modlev levels_dtm0_domain[] = { DTM0_DOMAIN_LEVEL(M0_DTM0_DOMAIN_LEVEL_REMACH_START), DTM0_DOMAIN_LEVEL(M0_DTM0_DOMAIN_LEVEL_PMACH_START), DTM0_DOMAIN_LEVEL(M0_DTM0_DOMAIN_LEVEL_PRUNER_INIT), + DTM0_DOMAIN_LEVEL(M0_DTM0_DOMAIN_LEVEL_PRUNER_START), DTM0_DOMAIN_LEVEL(M0_DTM0_DOMAIN_LEVEL_READY), }; @@ -99,18 +106,30 @@ static int dtm0_domain_level_enter(struct m0_module *module) enum dtm0_domain_level level = module->m_cur + 1; struct m0_dtm0_domain *dod = dtm0_module2domain(module); struct m0_dtm0_domain_cfg *cfg = &dod->dod_cfg; + struct m0_reqh *reqh = cfg->dod_reqh; M0_ENTRY("dod=%p level=%d level_name=%s", dod, level, levels_dtm0_domain[level].ml_name); switch (level) { case M0_DTM0_DOMAIN_LEVEL_INIT: + if (reqh != NULL && reqh->rh_beseg != (void *)1 && + reqh->rh_beseg != NULL) + cfg->dodc_log.dlc_be_domain = reqh->rh_beseg->bs_domain; + cfg->dodc_pruner.dpc_cfs = &dod->dod_cfs; + cfg->dodc_pruner.dpc_dol = &dod->dod_log; return M0_RC(0); - case M0_DTM0_DOMAIN_LEVEL_LOG_MKFS: + case M0_DTM0_DOMAIN_LEVEL_LOG_CREATE: + if (cfg->dod_create) + return M0_RC(m0_dtm0_log_create(&dod->dod_log, + &cfg->dodc_log)); return M0_RC(0); - case M0_DTM0_DOMAIN_LEVEL_LOG_INIT: - return M0_RC(m0_dtm0_log_init(&dod->dod_log, + case M0_DTM0_DOMAIN_LEVEL_LOG_OPEN: + return M0_RC(m0_dtm0_log_open(&dod->dod_log, &cfg->dodc_log)); + case M0_DTM0_DOMAIN_LEVEL_CFS_INIT: + return M0_RC(m0_co_fom_service_init(&dod->dod_cfs, reqh)); + case M0_DTM0_DOMAIN_LEVEL_REMACH_INIT: return M0_RC(m0_dtm0_remach_init(&dod->dod_remach, &cfg->dodc_remach)); @@ -130,6 +149,9 @@ static int dtm0_domain_level_enter(struct m0_module *module) case M0_DTM0_DOMAIN_LEVEL_PRUNER_INIT: return M0_RC(m0_dtm0_pruner_init(&dod->dod_pruner, &cfg->dodc_pruner)); + case M0_DTM0_DOMAIN_LEVEL_PRUNER_START: + m0_dtm0_pruner_start(&dod->dod_pruner); + return M0_RC(0); case M0_DTM0_DOMAIN_LEVEL_READY: return M0_RC(0); default: @@ -139,19 +161,27 @@ static int dtm0_domain_level_enter(struct m0_module *module) static void dtm0_domain_level_leave(struct m0_module *module) { - enum dtm0_domain_level level = module->m_cur; - struct m0_dtm0_domain *dod = dtm0_module2domain(module); + enum dtm0_domain_level level = module->m_cur; + struct m0_dtm0_domain *dod = dtm0_module2domain(module); + struct m0_dtm0_domain_cfg *cfg = &dod->dod_cfg; M0_ENTRY("dod=%p level=%d level_name=%s", dod, level, levels_dtm0_domain[level].ml_name); switch (level) { case M0_DTM0_DOMAIN_LEVEL_INIT: break; - case M0_DTM0_DOMAIN_LEVEL_LOG_MKFS: + case M0_DTM0_DOMAIN_LEVEL_LOG_CREATE: + if (cfg->dod_destroy) + m0_dtm0_log_destroy(&dod->dod_log); + break; + case M0_DTM0_DOMAIN_LEVEL_LOG_OPEN: + m0_dtm0_log_close(&dod->dod_log); break; - case M0_DTM0_DOMAIN_LEVEL_LOG_INIT: - m0_dtm0_log_fini(&dod->dod_log); + + case M0_DTM0_DOMAIN_LEVEL_CFS_INIT: + m0_co_fom_service_fini(&dod->dod_cfs); break; + case M0_DTM0_DOMAIN_LEVEL_REMACH_INIT: m0_dtm0_remach_fini(&dod->dod_remach); break; @@ -173,6 +203,10 @@ static void dtm0_domain_level_leave(struct m0_module *module) case M0_DTM0_DOMAIN_LEVEL_PRUNER_INIT: m0_dtm0_pruner_fini(&dod->dod_pruner); break; + case M0_DTM0_DOMAIN_LEVEL_PRUNER_START: + m0_dtm0_log_end(&dod->dod_log); + m0_dtm0_pruner_stop(&dod->dod_pruner); + break; case M0_DTM0_DOMAIN_LEVEL_READY: break; @@ -184,20 +218,32 @@ static void dtm0_domain_level_leave(struct m0_module *module) M0_INTERNAL int m0_dtm0_domain_init(struct m0_dtm0_domain *dod, struct m0_dtm0_domain_cfg *dod_cfg) { - int rc; + int rc; + bool has_be_domain = dod_cfg->dod_reqh != NULL && + dod_cfg->dod_reqh->rh_beseg != (void *)1 && + dod_cfg->dod_reqh->rh_beseg != NULL && + dod_cfg->dod_reqh->rh_beseg->bs_domain != NULL; M0_ENTRY("dod=%p dod_cfg=%p", dod, dod_cfg); m0_module_setup(&dod->dod_module, "m0_dtm0_domain module", levels_dtm0_domain, ARRAY_SIZE(levels_dtm0_domain), m0_get()); + m0_dtm0_domain_bob_init(dod); /* * TODO use m0_dtm0_domain_cfg_dup to copy the configuration * into dod::dod_cfg. */ - m0_dtm0_domain_bob_init(dod); - rc = m0_module_init(&dod->dod_module, M0_DTM0_DOMAIN_LEVEL_READY); + dod->dod_cfg = *dod_cfg; + /* + * TODO: volatile log is not implemented yet, so skip init for the + * case where BE is not present (client side). + */ + rc = m0_module_init(&dod->dod_module, + has_be_domain ? M0_DTM0_DOMAIN_LEVEL_READY : + M0_DTM0_DOMAIN_LEVEL_INIT); if (rc != 0) m0_module_fini(&dod->dod_module, M0_MODLEV_NONE); + return M0_RC(rc); } diff --git a/dtm0/domain.h b/dtm0/domain.h index 82a40160489..f95cd62d9e2 100644 --- a/dtm0/domain.h +++ b/dtm0/domain.h @@ -29,6 +29,7 @@ #include "dtm0/pmach.h" /* m0_dtm0_pmach */ #include "dtm0/remach.h" /* m0_dtm0_remach */ #include "dtm0/net.h" /* m0_dtm0_net */ +#include "dtm0/service.h" /* m0_co_fom_service */ #include "module/module.h" /* m0_module */ struct m0_reqh; @@ -54,10 +55,15 @@ struct m0_dtm0_domain_cfg { struct m0_dtm0_remach_cfg dodc_remach; struct m0_dtm0_pmach_cfg dodc_pmach; struct m0_dtm0_net_cfg dodc_net; + + /* TODO: s/dod/dodc/ */ + bool dod_create; + bool dod_destroy; + struct m0_reqh *dod_reqh; }; struct m0_dtm0_domain_create_cfg { - struct m0_dtm0_log_create_cfg dcc_log; + int unused; }; struct m0_dtm0_domain { @@ -66,6 +72,7 @@ struct m0_dtm0_domain { struct m0_dtm0_remach dod_remach; struct m0_dtm0_pmach dod_pmach; struct m0_dtm0_net dod_net; + struct m0_co_fom_service dod_cfs; struct m0_dtm0_domain_cfg dod_cfg; struct m0_module dod_module; uint64_t dod_magix; diff --git a/dtm0/dtm0.c b/dtm0/dtm0.c new file mode 100644 index 00000000000..d530986c10b --- /dev/null +++ b/dtm0/dtm0.c @@ -0,0 +1,145 @@ +/* -*- C -*- */ +/* + * Copyright (c) 2013-2020 Seagate Technology LLC and/or its Affiliates + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For any questions about this software or licensing, + * please email opensource@seagate.com or cortx-questions@seagate.com. + * + */ + +/** + * @addtogroup dtm0 + * + * @{ + */ + +#define M0_TRACE_SUBSYSTEM M0_TRACE_SUBSYS_DTM0 +#include "lib/trace.h" + +#include "dtm0/dtm0.h" +#include "dtm0/clk_src.h" +#include "dtm0/pruner.h" +#include "lib/memory.h" /* M0_ALLOC_ARR */ +#include "lib/errno.h" /* ENOMEM */ + +M0_INTERNAL bool m0_dtx0_id_eq(const struct m0_dtx0_id *left, + const struct m0_dtx0_id *right) +{ + return m0_dtx0_id_cmp(left, right) == 0; +} + +M0_INTERNAL int m0_dtx0_id_cmp(const struct m0_dtx0_id *left, + const struct m0_dtx0_id *right) +{ + return M0_3WAY(left->dti_timestamp, right->dti_timestamp) ?: + m0_fid_cmp(&left->dti_originator_sdev_fid, + &right->dti_originator_sdev_fid); +} + +static int descriptor_copy(struct m0_dtx0_descriptor *dst, + const struct m0_dtx0_descriptor *src) +{ + struct m0_fid *dst_arr; + struct m0_fid *src_arr = src->dtd_participants.dtpa_participants; + uint64_t nr = src->dtd_participants.dtpa_participants_nr; + + if (nr == 0) { + M0_SET0(dst); + return 0; + } + + M0_ALLOC_ARR(dst_arr, nr); + if (dst_arr == NULL) + return M0_ERR(-ENOMEM); + + memcpy(dst_arr, src_arr, sizeof(*dst_arr) * nr); + + dst->dtd_participants.dtpa_participants = dst_arr; + dst->dtd_participants.dtpa_participants_nr = nr; + dst->dtd_id = src->dtd_id; + return 0; +} + +static int buf2bufs_copy(struct m0_bufs *dst, const struct m0_buf *src) +{ + struct m0_buf *dst_buf; + int rc; + + M0_ALLOC_PTR(dst_buf); + if (dst_buf == NULL) + return M0_ERR(-ENOMEM); + rc = m0_buf_copy(dst_buf, src); + if (rc != 0) { + m0_free(dst_buf); + return M0_ERR(rc); + } + dst->ab_count = 1; + dst->ab_elems = dst_buf; + return 0; +} + +M0_INTERNAL int +m0_dtm0_redo_init(struct m0_dtm0_redo *redo, + const struct m0_dtx0_descriptor *descriptor, + const struct m0_buf *payload, + enum m0_dtx0_payload_type type) +{ + int rc; + + redo->dtr_payload.dtp_type = type; + M0_LOG(M0_DEBUG, "dtx id: " DTID1_F "participants:%d", + DTID1_P(&descriptor->dtd_id), + (int)descriptor->dtd_participants.dtpa_participants_nr); + rc = descriptor_copy(&redo->dtr_descriptor, descriptor) ?: + buf2bufs_copy(&redo->dtr_payload.dtp_data, payload); + if (rc != 0) + m0_dtm0_redo_fini(redo); + return rc; +} + +M0_INTERNAL void m0_dtm0_redo_fini(struct m0_dtm0_redo *redo) +{ + m0_free(redo->dtr_descriptor.dtd_participants.dtpa_participants); + m0_bufs_free(&redo->dtr_payload.dtp_data); +} + +M0_INTERNAL int m0_dtm0_mod_init(void) +{ + /* TODO: Register stype here (and other types). */ + m0_dtm0_pruner_mod_init(); + return 0; +} + +M0_INTERNAL void m0_dtm0_mod_fini(void) +{ + m0_dtm0_pruner_mod_fini(); +} + +#undef M0_TRACE_SUBSYSTEM + +/** @} end of dtm0 group */ + +/* + * Local variables: + * c-indentation-style: "K&R" + * c-basic-offset: 8 + * tab-width: 8 + * fill-column: 80 + * scroll-step: 1 + * End: + */ +/* + * vim: tabstop=8 shiftwidth=8 noexpandtab textwidth=80 nowrap + */ diff --git a/dtm0/dtm0.h b/dtm0/dtm0.h index b89f6c69624..77693d42e8a 100644 --- a/dtm0/dtm0.h +++ b/dtm0/dtm0.h @@ -91,6 +91,72 @@ * @{ */ +#include "fid/fid.h" /* m0_fid */ +#include "fid/fid_xc.h" /* m0_fid_xc */ +#include "lib/buf.h" /* m0_bufs */ +#include "lib/buf_xc.h" /* m0_bufs_xc */ +#include "lib/types.h" /* uint64_t */ +#include "xcode/xcode.h" /* M0_XCA_RECORD */ + + +struct m0_dtx0_id { + struct m0_fid dti_originator_sdev_fid; + uint64_t dti_timestamp; /* XXX fix the type */ +} M0_XCA_RECORD M0_XCA_DOMAIN(rpc|be); + + +struct m0_dtx0_participants { + uint64_t dtpa_participants_nr; + struct m0_fid *dtpa_participants; +} M0_XCA_SEQUENCE M0_XCA_DOMAIN(rpc|be); + +struct m0_dtx0_descriptor { + struct m0_dtx0_id dtd_id; + struct m0_dtx0_participants dtd_participants; +} M0_XCA_RECORD M0_XCA_DOMAIN(rpc|be); + +#define DTID1_F "{" DTS0_F "," FID_F "}" +#define DTID1_P(__tid) (__tid)->dti_timestamp, FID_P(&(__tid)->dti_originator_sdev_fid) +enum m0_dtx0_payload_type { + M0_DTX0_PAYLOAD_CAS, /** it's supposed to be handled by CAS */ + M0_DTX0_PAYLOAD_BLOB, /** + * configurable handler. + * @see m0_dtm0_remach_cfg::dtrc_blob_handler() + */ +} M0_XCA_ENUM M0_XCA_DOMAIN(rpc|be); + +struct m0_dtx0_payload { + uint32_t dtp_type M0_XCA_FENUM(m0_dtx0_payload_type); + struct m0_bufs dtp_data; +} M0_XCA_RECORD M0_XCA_DOMAIN(rpc|be); + +struct m0_dtm0_redo { + struct m0_dtx0_descriptor dtr_descriptor; + struct m0_dtx0_payload dtr_payload; +}; + +struct m0_dtm0_p { + struct m0_dtx0_id dtmp_id; + struct m0_fid dtmp_sdev_fid; +}; + +M0_INTERNAL int +m0_dtm0_redo_init(struct m0_dtm0_redo *redo, + const struct m0_dtx0_descriptor *descriptor, + const struct m0_buf *payload, + enum m0_dtx0_payload_type type); + +M0_INTERNAL void m0_dtm0_redo_fini(struct m0_dtm0_redo *redo); + +M0_INTERNAL bool m0_dtx0_id_eq(const struct m0_dtx0_id *left, + const struct m0_dtx0_id *right); + +M0_INTERNAL int m0_dtx0_id_cmp(const struct m0_dtx0_id *left, + const struct m0_dtx0_id *right); + +M0_INTERNAL int m0_dtm0_mod_init(void); +M0_INTERNAL void m0_dtm0_mod_fini(void); + #endif /* __MOTR_DTM0_DTM0_H__ */ /* diff --git a/dtm0/dtx0.c b/dtm0/dtx0.c index 4252dc25292..b4beb90b191 100644 --- a/dtm0/dtx0.c +++ b/dtm0/dtx0.c @@ -29,15 +29,21 @@ #include "lib/trace.h" #include "dtm0/dtx0.h" +#include "dtm0/domain.h" /* m0_dtm0_domain */ -M0_INTERNAL int m0_dtx0_init(struct m0_dtx0 *dtx0, - struct m0_dtm0_domain *dod) +M0_INTERNAL void m0_dtx0_redo_add_credit(struct m0_dtm0_domain *dod, + struct m0_dtm0_redo *redo, + struct m0_be_tx_credit *accum) { - return 0; + m0_dtm0_log_redo_add_credit(&dod->dod_log, redo, accum); } -M0_INTERNAL void m0_dtx0_fini(struct m0_dtx0 *dtx0) +M0_INTERNAL int m0_dtx0_redo_add(struct m0_dtm0_domain *dod, + struct m0_be_tx *tx, + struct m0_dtm0_redo *redo, + const struct m0_fid *sdev) { + return m0_dtm0_log_redo_add(&dod->dod_log, tx, redo, sdev); } #undef M0_TRACE_SUBSYSTEM diff --git a/dtm0/dtx0.h b/dtm0/dtx0.h index 49046fc95ec..45eda0b9aa7 100644 --- a/dtm0/dtx0.h +++ b/dtm0/dtx0.h @@ -24,8 +24,6 @@ #ifndef __MOTR___DTM0_DTX0_H__ #define __MOTR___DTM0_DTX0_H__ -#include "sm/sm.h" /* m0_sm */ - /** * @defgroup dtm0 * @@ -82,53 +80,20 @@ * */ - struct m0_dtm0_domain; -struct m0_dtm0_tx_desc; -struct m0_fid; -struct m0_buf; -struct m0_be_tx_credit; +struct m0_dtm0_redo; struct m0_be_tx; -struct m0_be_op; - -struct m0_dtx0 { - /** See m0_dtx0_state */ - struct m0_sm dtx0_sm; -}; - -enum m0_dtx0_state { - M0_DTX0_STATE_INIT, - M0_DTX0_STATE_EXECUTED, - M0_DTX0_STATE_STABLE, -}; - -M0_INTERNAL int m0_dtx0_init(struct m0_dtx0 *dtx0, - struct m0_dtm0_domain *dod); -M0_INTERNAL void m0_dtx0_fini(struct m0_dtx0 *dtx0); - -M0_INTERNAL int m0_dtx0_set(struct m0_dtx0 *dtx0, - const struct m0_dtm0_tx_desc *txd, - const struct m0_buf *buf); - -M0_INTERNAL void m0_dtx0_timestamp_set(struct m0_dtx0 *dtx0); - -M0_INTERNAL int m0_dtx0_participants_set(struct m0_dtx0 *dtx0, - const struct m0_fid *rdtm_svcs, - int rdtm_svcs_nr); - -M0_INTERNAL int m0_dtx0_buf_set(struct m0_dtx0 *dtx0, - const struct m0_buf *buf); - -M0_INTERNAL void m0_dtx0_credit(struct m0_dtx0 *dtx0, - struct m0_be_tx_credit *accum); +struct m0_be_tx_credit; +struct m0_fid; -M0_INTERNAL int m0_dtx0_log_update(struct m0_dtx0 *dtx0, - struct m0_be_tx *tx, - struct m0_be_op *op, - bool is_redo); +M0_INTERNAL void m0_dtx0_redo_add_credit(struct m0_dtm0_domain *dod, + struct m0_dtm0_redo *redo, + struct m0_be_tx_credit *accum); -/* TODO */ -M0_INTERNAL void m0_dtx0_cancel(struct m0_dtx0 *dtx0); +M0_INTERNAL int m0_dtx0_redo_add(struct m0_dtm0_domain *dod, + struct m0_be_tx *tx, + struct m0_dtm0_redo *redo, + const struct m0_fid *sdev); /** @} end of dtm0 group */ #endif /* __MOTR___DTM0_DTX0_H__ */ diff --git a/dtm0/helper.c b/dtm0/helper.c index 9a23d2fa731..2d6d8de7fbd 100644 --- a/dtm0/helper.c +++ b/dtm0/helper.c @@ -57,7 +57,7 @@ struct m0_be_0type m0_be_dtm0 = { .b0_fini = dtm0_log_fini, }; -M0_INTERNAL int m0_dtm0_log_create(struct m0_sm_group *grp, +M0_INTERNAL int m0_dtm0_old_log_create(struct m0_sm_group *grp, struct m0_be_domain *bedom, struct m0_be_seg *seg) { diff --git a/dtm0/helper.h b/dtm0/helper.h index 36da0dc4095..c62b7469eef 100644 --- a/dtm0/helper.h +++ b/dtm0/helper.h @@ -29,7 +29,7 @@ struct m0_reqh_service; struct m0_reqh; struct m0_fid; -M0_INTERNAL int m0_dtm0_log_create(struct m0_sm_group *grp, +M0_INTERNAL int m0_dtm0_old_log_create(struct m0_sm_group *grp, struct m0_be_domain *bedom, struct m0_be_seg *seg); diff --git a/dtm0/log.c b/dtm0/log.c index 4511f0cb16a..de177e7b684 100644 --- a/dtm0/log.c +++ b/dtm0/log.c @@ -30,24 +30,497 @@ #include "dtm0/log.h" -M0_INTERNAL int m0_dtm0_log_init(struct m0_dtm0_log *dol, +#include "lib/buf.h" /* m0_buf */ +#include "lib/vec.h" /* m0_vec_count */ +#include "lib/errno.h" /* EINVAL */ +#include "lib/assert.h" /* M0_ASSERT */ +#include "lib/string.h" /* strlen */ +#include "lib/locality.h" /* m0_locality0_get */ + +#include "be/domain.h" /* m0_be_domain_seg0_get */ +#include "be/btree.h" /* m0_btree */ +#include "be/list.h" /* m0_be_list */ +#include "be/seg0.h" /* m0_be_0type */ +#include "be/seg.h" /* m0_be_seg_allocator */ +#include "be/tx.h" /* m0_be_tx */ +#include "be/op.h" /* M0_BE_OP_SYNC */ + +#include "dtm0/dtm0.h" /* m0_dtm0_redo */ + +struct m0_sm_group; +struct m0_be_domain; + +enum { + M0_DTM0_LOG_ROOT_NODE_SIZE = (8 * 1024), + M0_DTM0_LOG_SHIFT = 12, +}; + +struct dtm0_log_data { + uint8_t dtld_node[M0_DTM0_LOG_ROOT_NODE_SIZE]; + struct m0_btree dtld_transactions; + struct m0_be_list dtld_all_p; +} M0_XCA_RECORD M0_XCA_DOMAIN(be); + +struct dtm0_log_record { + struct m0_dtx0_descriptor lr_descriptor; + uint32_t lr_payload_type + M0_XCA_FENUM(m0_dtx0_payload_type); + struct m0_buf lr_payload_data; + struct m0_be_list_link lr_link_all_p; + uint64_t lr_magic; +} M0_XCA_RECORD M0_XCA_DOMAIN(be); + +static int dtm0_log0_init(struct m0_be_domain *dom, const char *suffix, + const struct m0_buf *data); +static void dtm0_log0_fini(struct m0_be_domain *dom, const char *suffix, + const struct m0_buf *data); + +static bool dtm0_log_invariant(const struct m0_dtm0_log *dol) +{ + return _0C(m0_mutex_is_locked(&dol->dtl_lock)) && + _0C(ergo(dol->dtl_the_end, dol->dtl_allp_op == NULL)) && + ergo(dol->dtl_allp_op != NULL, + _0C(dol->dtl_allp != NULL) && + _0C(dol->dtl_successful != NULL)); +} + +struct m0_be_0type m0_dtm0_log0 = { + .b0_name = "M0_DTM0:LOG", + .b0_init = &dtm0_log0_init, + .b0_fini = &dtm0_log0_fini, +}; + +static m0_bcount_t dtm0_log_transactions_ksize(const void *key); +static m0_bcount_t dtm0_log_transactions_vsize(const void *key); +static int dtm0_log_transactions_compare(const void *key0, const void *key1); + +M0_BE_LIST_DESCR_DEFINE(dtm0_log_all_p, "dtm0_log_data::dtld_all_p", + static, struct dtm0_log_record, + lr_link_all_p, lr_magic, 1 /* XXX */, 2 /* XXX */); +M0_BE_LIST_DEFINE(dtm0_log_all_p, static, struct dtm0_log_record); + + +M0_INTERNAL int m0_dtm0_log_open(struct m0_dtm0_log *dol, struct m0_dtm0_log_cfg *dol_cfg) { + struct dtm0_log_data *log_data; + struct m0_be_domain *dom = dol_cfg->dlc_be_domain; + struct m0_be_seg *seg0 = m0_be_domain_seg0_get(dom); + struct m0_be_seg *seg = dol_cfg->dlc_seg != NULL ? dol_cfg->dlc_seg : + m0_be_domain_seg_first(dom); + struct m0_buf *log_data_buf = NULL; + size_t size_prefix; + size_t size_suffix; + char name[0x100]; + int rc; + struct m0_btree_op b_op = {}; + struct m0_btree_rec_key_op keycmp = { + keycmp.rko_keycmp = dtm0_log_transactions_compare }; + + M0_PRE(dom != NULL); + M0_PRE(dol->dtl_data == NULL); + + dol->dtl_cfg = *dol_cfg; + dol->dtl_cfg.dlc_seg = seg; + size_prefix = strlen(m0_dtm0_log0.b0_name); + /* XXX check that there is at least one \0 in the array */ + size_suffix = strlen((const char *)&dol_cfg->dlc_seg0_suffix); + if (size_prefix + size_suffix + 1 > ARRAY_SIZE(name)) + return M0_ERR(-EINVAL); + memcpy(&name, m0_dtm0_log0.b0_name, size_prefix); + memcpy(&name[size_prefix], &dol_cfg->dlc_seg0_suffix, size_suffix); + name[size_prefix + size_suffix] = '\0'; + rc = m0_be_seg_dict_lookup(seg0, (const char *)&name, + (void **)&log_data_buf); + if (rc != 0) + return M0_ERR(rc); + log_data = *(struct dtm0_log_data **)log_data_buf->b_addr; + rc = M0_BTREE_OP_SYNC_WITH_RC(&b_op, + m0_btree_open(&log_data->dtld_node, + sizeof log_data->dtld_node, + &log_data->dtld_transactions, + seg, &b_op, + &keycmp)); + M0_ASSERT(rc == 0); + + dol->dtl_data = log_data; + m0_mutex_init(&dol->dtl_lock); + dol->dtl_allp_op = NULL; + dol->dtl_allp = NULL; + dol->dtl_the_end = false; + return M0_RC(0); +} + +M0_INTERNAL void m0_dtm0_log_close(struct m0_dtm0_log *dol) +{ + struct m0_btree_op b_op = {}; + + M0_PRE(dol->dtl_data != NULL); + M0_PRE(dol->dtl_allp_op == NULL); + m0_mutex_fini(&dol->dtl_lock); + M0_BTREE_OP_SYNC_WITH_RC(&b_op, + m0_btree_close(&dol->dtl_data->dtld_transactions, + &b_op)); + dol->dtl_data = NULL; +} + +M0_INTERNAL int m0_dtm0_log_create(struct m0_dtm0_log *dol, + struct m0_dtm0_log_cfg *dol_cfg) +{ + struct m0_be_tx_credit cred = {}; + struct dtm0_log_data *log_data; + struct m0_be_domain *dom = dol_cfg->dlc_be_domain; + struct m0_sm_group *grp = m0_locality0_get()->lo_grp; + struct m0_be_seg *seg = dol_cfg->dlc_seg != NULL ? dol_cfg->dlc_seg : + m0_be_domain_seg_first(dom); + struct m0_be_tx tx = {}; + int rc; + struct m0_btree_type bt; + struct m0_btree_op b_op = {}; + struct m0_btree_rec_key_op keycmp = { + keycmp.rko_keycmp = dtm0_log_transactions_compare }; + + M0_ENTRY(); + + + rc = m0_dtm0_log_open(dol, dol_cfg); + if (rc == 0) + m0_dtm0_log_close(dol); + + if (!M0_IN(rc, (0, -ENOENT))) + return M0_ERR(rc); + + if (rc == 0) { + M0_LOG(M0_DEBUG, "DTM0 log already exists."); + return M0_RC(0); + } + + /* TODO check that there is \0 in dol_cfg->dlc_seg0_suffix */ + m0_sm_group_lock(grp); + m0_be_tx_init(&tx, 0, dom, grp, NULL, NULL, NULL, NULL); + m0_be_allocator_credit(m0_be_seg_allocator(seg), M0_BAO_ALLOC, + sizeof(*log_data), 0, &cred); + m0_be_0type_add_credit(dom, &m0_dtm0_log0, + (const char *)&dol_cfg->dlc_seg0_suffix, + &M0_BUF_INIT(sizeof log_data, &log_data), &cred); + + dtm0_log_all_p_be_list_credit(M0_BLO_CREATE, 1, &cred); + + bt = (struct m0_btree_type) { + .tt_id = M0_BT_DTM0_LOG, + .ksize = sizeof(struct m0_dtx0_id), + .vsize = -1, + }; + m0_btree_create_credit(&bt, &cred, 1); + + m0_be_tx_prep(&tx, &cred); + rc = m0_be_tx_exclusive_open_sync(&tx); + M0_ASSERT(rc == 0); /* XXX */ + M0_BE_ALLOC_ALIGN_PTR_SYNC(log_data, M0_DTM0_LOG_SHIFT, + seg, &tx); + M0_ASSERT(log_data != NULL); /* XXX */ + rc = m0_be_0type_add(&m0_dtm0_log0, dom, &tx, + (const char *)&dol_cfg->dlc_seg0_suffix, + &M0_BUF_INIT(sizeof log_data, &log_data)); + + M0_ASSERT(rc == 0); /* XXX */ + rc = M0_BTREE_OP_SYNC_WITH_RC(&b_op, + m0_btree_create(&log_data->dtld_node, + sizeof log_data->dtld_node, + &bt, M0_BCT_NO_CRC, &b_op, + &log_data->dtld_transactions, seg, + &dol_cfg->dlc_btree_fid, &tx, &keycmp)); + M0_ASSERT(rc == 0); + + + dtm0_log_all_p_be_list_create(&log_data->dtld_all_p, &tx); + + /* TODO capture log_data? */ + + m0_be_tx_close_sync(&tx); + m0_be_tx_fini(&tx); + m0_sm_group_unlock(grp); + return M0_RC(0); +} + +M0_INTERNAL void m0_dtm0_log_destroy(struct m0_dtm0_log *dol) +{ + /* XXX No-op for now. TODO implement */ +} + +static int dtm0_log0_init(struct m0_be_domain *dom, const char *suffix, + const struct m0_buf *data) +{ + return 0; +} + +static void dtm0_log0_fini(struct m0_be_domain *dom, const char *suffix, + const struct m0_buf *data) +{ +} + +static inline m0_bcount_t dtm0_log_transactions_ksize(const void *key) +{ + return sizeof ((struct dtm0_log_record *)NULL)->lr_descriptor.dtd_id; +} + +static inline m0_bcount_t dtm0_log_transactions_vsize(const void *value) +{ + return sizeof(struct dtm0_log_record *); +} + +static int dtm0_log_transactions_compare(const void *key0, const void *key1) +{ + return m0_dtx0_id_cmp(key0, key1); +} + +static int redo_insert_callback(struct m0_btree_cb *cb, + struct m0_btree_rec *rec) +{ + struct m0_btree_rec *datum = cb->c_datum; + + /** Write the Key and Value to the location indicated in rec. */ + m0_bufvec_copy(&rec->r_key.k_data, &datum->r_key.k_data, + m0_vec_count(&datum->r_key.k_data.ov_vec)); + m0_bufvec_copy(&rec->r_val, &datum->r_val, + m0_vec_count(&rec->r_val.ov_vec)); + return 0; +} + +static int redo_insert(struct m0_btree *tree, struct m0_be_tx *tx, + struct m0_buf *key, struct m0_buf *val) +{ + struct m0_btree_op kv_op = {}; + void *k_ptr = key->b_addr; + void *v_ptr = val->b_addr; + m0_bcount_t ksize = key->b_nob; + m0_bcount_t vsize = val->b_nob; + struct m0_btree_rec rec = { + .r_key.k_data = M0_BUFVEC_INIT_BUF(&k_ptr, &ksize), + .r_val = M0_BUFVEC_INIT_BUF(&v_ptr, &vsize), + .r_crc_type = M0_BCT_NO_CRC, + }; + struct m0_btree_cb redo_insert_cb = {.c_act = redo_insert_callback, + .c_datum = &rec, + }; + return M0_BTREE_OP_SYNC_WITH_RC(&kv_op, + m0_btree_put(tree, &rec, &redo_insert_cb, + &kv_op, tx)); +} + +M0_INTERNAL int m0_dtm0_log_redo_add(struct m0_dtm0_log *dol, + struct m0_be_tx *tx, + const struct m0_dtm0_redo *redo, + const struct m0_fid *p_sdev_fid) +{ + struct dtm0_log_record *rec; + struct m0_be_seg *seg = dol->dtl_cfg.dlc_seg; + + M0_LOG(M0_DEBUG, "fid:"FID_F, FID_P(p_sdev_fid)); + m0_mutex_lock(&dol->dtl_lock); + M0_PRE(!dol->dtl_the_end); + M0_PRE(dtm0_log_invariant(dol)); + m0_mutex_unlock(&dol->dtl_lock); + + /* TODO lookup before insert */ + /* TODO check memory allocation errors */ + M0_BE_ALLOC_PTR_SYNC(rec, seg, tx); + M0_ASSERT(redo->dtr_payload.dtp_data.ab_count == 1); + rec->lr_payload_data.b_nob = + redo->dtr_payload.dtp_data.ab_elems[0].b_nob; + M0_BE_ALLOC_BUF_SYNC(&rec->lr_payload_data, seg, tx); + /* TODO check if memcpy can be avoided */ + m0_buf_memcpy(&rec->lr_payload_data, + &redo->dtr_payload.dtp_data.ab_elems[0]); + rec->lr_descriptor = redo->dtr_descriptor; + M0_LOG(M0_DEBUG, "redo add txid: " DTID1_F, + DTID1_P(&rec->lr_descriptor.dtd_id)); + redo_insert(&dol->dtl_data->dtld_transactions, tx, + &M0_BUF_INIT_PTR(&rec->lr_descriptor.dtd_id), + &M0_BUF_INIT(sizeof rec, &rec)); + dtm0_log_all_p_be_tlink_create(rec, tx); + + m0_mutex_lock(&dol->dtl_lock); + M0_ASSERT(!dol->dtl_the_end); + if (dtm0_log_all_p_be_list_head(&dol->dtl_data->dtld_all_p) == NULL && + dol->dtl_allp_op != NULL) { + M0_ASSERT(dol->dtl_allp != NULL); + *dol->dtl_allp = rec->lr_descriptor.dtd_id; + *dol->dtl_successful = true; + m0_be_op_done(dol->dtl_allp_op); + dol->dtl_allp_op = NULL; + } + dtm0_log_all_p_be_list_add_tail(&dol->dtl_data->dtld_all_p, tx, rec); + m0_mutex_unlock(&dol->dtl_lock); + /* TODO capture rec->lr_payload_data */ + /* TODO capture rec */ return 0; } -M0_INTERNAL void m0_dtm0_log_fini(struct m0_dtm0_log *dol) +M0_INTERNAL void m0_dtm0_log_redo_add_credit(struct m0_dtm0_log *dol, + const struct m0_dtm0_redo *redo, + struct m0_be_tx_credit *accum) { + struct dtm0_log_record *rec = NULL; /* XXX */ + struct m0_be_seg *seg = dol->dtl_cfg.dlc_seg; + struct m0_be_allocator *a = m0_be_seg_allocator(seg); + + M0_PRE(redo->dtr_payload.dtp_data.ab_count == 1); + + m0_be_allocator_credit(a, M0_BAO_ALLOC, sizeof *rec, 0, accum); + m0_be_allocator_credit(a, M0_BAO_ALLOC, + redo->dtr_payload.dtp_data.ab_elems[0].b_nob, + 0, accum); + m0_btree_put_credit(&dol->dtl_data->dtld_transactions, 1, + sizeof rec->lr_descriptor.dtd_id, + sizeof(void *), accum); + dtm0_log_all_p_be_list_credit(M0_BLO_TLINK_CREATE, 1, accum); + dtm0_log_all_p_be_list_credit(M0_BLO_ADD, 1, accum); } -M0_INTERNAL int m0_dtm0_log__create(struct m0_dtm0_log *dol, - struct m0_dtm0_log_create_cfg *dlc_cfg) +static int redo_log_lookup_callback(struct m0_btree_cb *cb, + struct m0_btree_rec *rec) { + struct m0_btree_rec *datum = cb->c_datum; + + /** Only copy the Value for the caller. */ + m0_bufvec_copy(&datum->r_val, &rec->r_val, + m0_vec_count(&rec->r_val.ov_vec)); return 0; } -M0_INTERNAL void m0_dtm0_log_destroy(struct m0_dtm0_log *dol) +static int redo_log_lookup(struct m0_btree *tree, struct m0_buf *key, + struct m0_buf *out) +{ + struct m0_btree_op kv_op = {}; + void *k_ptr = key->b_addr; + void *v_ptr = out->b_addr; + m0_bcount_t ksize = key->b_nob; + m0_bcount_t vsize = out->b_nob; + struct m0_btree_rec rec = { + .r_key.k_data = M0_BUFVEC_INIT_BUF(&k_ptr, &ksize), + .r_val = M0_BUFVEC_INIT_BUF(&v_ptr, &vsize), + }; + struct m0_btree_cb lookup_cb = {.c_act = redo_log_lookup_callback, + .c_datum = &rec, + }; + + return M0_BTREE_OP_SYNC_WITH_RC(&kv_op, + m0_btree_get(tree, &rec.r_key, + &lookup_cb, + BOF_EQUAL, &kv_op)); +} + +static void redo_log_delete(struct m0_btree *tree, struct m0_be_tx *tx, + struct m0_buf *key) +{ + struct m0_btree_op kv_op = {}; + void *k_ptr = key->b_addr; + m0_bcount_t ksize = key->b_nob; + struct m0_btree_key r_key = { + .k_data = M0_BUFVEC_INIT_BUF(&k_ptr, &ksize), + }; + + M0_BTREE_OP_SYNC_WITH_RC(&kv_op, + m0_btree_del(tree, &r_key, NULL, + &kv_op, tx)); +} + +M0_INTERNAL void m0_dtm0_log_prune(struct m0_dtm0_log *dol, + struct m0_be_tx *tx, + struct m0_dtx0_id *dtx0_id) +{ + struct dtm0_log_record *rec = NULL; + struct m0_buf rec_buf = M0_BUF_INIT(sizeof rec, &rec); + int rc; + + M0_LOG(M0_DEBUG, "DTX id: " DTID1_F, DTID1_P(dtx0_id)); + /* TODO handle lookup errors */ + rc = redo_log_lookup(&dol->dtl_data->dtld_transactions, + &M0_BUF_INIT_PTR(dtx0_id), &rec_buf); + M0_ASSERT(ergo(rc != 0, rec == NULL)); + M0_ASSERT(M0_IN(rc, (0, -ENOENT))); + m0_mutex_lock(&dol->dtl_lock); + M0_ASSERT(dtm0_log_invariant(dol)); + dtm0_log_all_p_be_list_del(&dol->dtl_data->dtld_all_p, tx, rec); + m0_mutex_unlock(&dol->dtl_lock); + dtm0_log_all_p_be_tlink_destroy(rec, tx); + /* TODO free rec */ + /* TODO check delete result, i.e. t_rc */ + redo_log_delete(&dol->dtl_data->dtld_transactions, tx, + &M0_BUF_INIT_PTR(dtx0_id)); +} + +M0_INTERNAL void m0_dtm0_log_prune_credit(struct m0_dtm0_log *dol, + struct m0_be_tx_credit *accum) +{ + struct dtm0_log_record *rec = NULL; /* XXX */ + + m0_btree_del_credit(&dol->dtl_data->dtld_transactions, 1, + sizeof rec->lr_descriptor.dtd_id, + sizeof(void *), accum); + dtm0_log_all_p_be_list_credit(M0_BLO_DEL, 1, accum); + dtm0_log_all_p_be_list_credit(M0_BLO_TLINK_DESTROY, 1, accum); +} + + +M0_INTERNAL void m0_dtm0_log_p_get_none_left(struct m0_dtm0_log *dol, + struct m0_be_op *op, + struct m0_dtx0_id *dtx0_id, + bool *successful) { + struct dtm0_log_record *rec; + + m0_be_op_active(op); + m0_mutex_lock(&dol->dtl_lock); + M0_PRE(dtm0_log_invariant(dol)); + M0_PRE(dol->dtl_allp_op == NULL); + + if (dol->dtl_the_end) { + *successful = false; + m0_be_op_done(op); + m0_mutex_unlock(&dol->dtl_lock); + return; + } + + rec = dtm0_log_all_p_be_list_head(&dol->dtl_data->dtld_all_p); + if (rec != NULL) { + *dtx0_id = rec->lr_descriptor.dtd_id; + *successful = true; + m0_be_op_done(op); + } else { + dol->dtl_allp_op = op; + dol->dtl_allp = dtx0_id; + dol->dtl_successful = successful; + } + m0_mutex_unlock(&dol->dtl_lock); +} + +M0_INTERNAL void m0_dtm0_log_end(struct m0_dtm0_log *dol) +{ + m0_mutex_lock(&dol->dtl_lock); + M0_PRE(dtm0_log_invariant(dol)); + M0_PRE(!dol->dtl_the_end); + dol->dtl_the_end = true; + if (dol->dtl_allp_op != NULL) { + *dol->dtl_successful = false; + m0_be_op_done(dol->dtl_allp_op); + dol->dtl_allp_op = NULL; + } + m0_mutex_unlock(&dol->dtl_lock); +} + +M0_INTERNAL bool m0_dtm0_log_is_empty(struct m0_dtm0_log *dol) +{ + bool result; + + M0_PRE(dol != NULL); + m0_mutex_lock(&dol->dtl_lock); + M0_PRE(dol->dtl_data != NULL); + result = m0_btree_is_empty(&dol->dtl_data->dtld_transactions) && + dtm0_log_all_p_be_list_head(&dol->dtl_data->dtld_all_p) == NULL; + m0_mutex_unlock(&dol->dtl_lock); + return M0_RC(!!result); } #undef M0_TRACE_SUBSYSTEM diff --git a/dtm0/log.h b/dtm0/log.h index 603d842d43c..2efa5935f57 100644 --- a/dtm0/log.h +++ b/dtm0/log.h @@ -24,10 +24,12 @@ #ifndef __MOTR___DTM0_LOG_H__ #define __MOTR___DTM0_LOG_H__ -#include "dtm0/tx_desc.h" /* m0_dtm0_tx_desc */ -#include "lib/buf.h" /* m0_buf */ +#include "dtm0/tx_desc.h" /* m0_dtm0_tx_desc */ +#include "fid/fid.h" /* m0_fid */ +#include "lib/mutex.h" /* m0_mutex */ + /** * @defgroup dtm0 * @@ -55,47 +57,117 @@ * DTM0 log has no FOMs. It reacts to the incoming events re-using * their sm groups. The log has one single lock that protects container * from modification. Individual log records has no lock protection. + * + * @section m0_dtm0_remach interface + * + * - m0_dtm0_log_iter_init() - initializes log record iterator for a + * sdev participant. It iterates over all records that were in the log during + * last local process restart or during last remote process restart for the + * process that handles that sdev. + * - m0_dtm0_log_iter_next() - gives next log record for the sdev participant. + * - m0_dtm0_log_iter_fini() - finalises the iterator. It MUST be done for every + * call of m0_dtm0_log_iter_init(). + * - m0_dtm0_log_participant_restarted() - notifies the log that the participant + * has restarted. All iterators for the participant MUST be finalized at the + * time of the call. Any record that doesn't have P from the participant at + * the time of the call will be returned during the next iteration for the + * participant. + * + * @section pmach interface + * + * - m0_dtm0_log_p_get_local() - returns the next P message that becomes local. + * Returns M0_FID0 during m0_dtm0_log_stop() call. After M0_FID0 is returned + * new calls to the log MUST NOT be made. + * - m0_dtm0_log_p_put() - records that P message was received for the sdev + * participant. + * + * @section pruner interface + * + * - m0_dtm0_log_p_get_none_left() - returns dtx0 id for the dtx which has all + * participants (except originator) reported P for the dtx0. Also returns all + * dtx0 which were cancelled. + * - m0_dtm0_log_prune() - remove the REDO message about dtx0 from the log + * + * dtx0 interface, client & server + * + * - m0_dtm0_log_redo_add() - adds a REDO message and, optionally, P message, to + * the log. + * + * @section dtx0 interface, client only + * + * - m0_dtm0_log_redo_p_wait() - returns the number of P messages for the dtx + * and waits until either the number increases or m0_dtm0_log_redo_cancel() is + * called. + * - m0_dtm0_log_redo_cancel() - notification that the client doesn't need the + * dtx anymore. Before the function returns the op + * - m0_dtm0_log_redo_end() - notifies dtx0 that the operation dtx0 is a part of + * is complete. This function MUST be called for every m0_dtm0_log_redo_add(). */ struct m0_be_op; struct m0_be_tx; struct m0_be_tx_credit; +struct m0_be_domain; -struct m0_dtm0_log_cfg { -}; +struct m0_dtm0_redo; +struct m0_dtx0_id; +struct dtm0_log_data; -struct m0_dtm0_log_create_cfg { -}; -struct m0_dtm0_log { +/* TODO s/dlc_/dtlc_/g */ +struct m0_dtm0_log_cfg { + char dlc_seg0_suffix[0x100]; + struct m0_be_domain *dlc_be_domain; + struct m0_be_seg *dlc_seg; + struct m0_fid dlc_btree_fid; }; -struct m0_dtm0_log_record { - struct m0_dtm0_tx_desc lr_desc; - struct m0_buf lr_data; +struct m0_dtm0_log { + struct m0_dtm0_log_cfg dtl_cfg; + struct dtm0_log_data *dtl_data; + struct m0_mutex dtl_lock; + + struct m0_be_op *dtl_allp_op; + struct m0_dtx0_id *dtl_allp; + bool *dtl_successful; + bool dtl_the_end; }; - -M0_INTERNAL int m0_dtm0_log_init(struct m0_dtm0_log *dol, +M0_INTERNAL int m0_dtm0_log_open(struct m0_dtm0_log *dol, struct m0_dtm0_log_cfg *dol_cfg); -M0_INTERNAL void m0_dtm0_log_fini(struct m0_dtm0_log *dol); +M0_INTERNAL void m0_dtm0_log_close(struct m0_dtm0_log *dol); -M0_INTERNAL int m0_dtm0_log__create(struct m0_dtm0_log *dol, - struct m0_dtm0_log_create_cfg *dlc_cfg); +M0_INTERNAL int m0_dtm0_log_create(struct m0_dtm0_log *dol, + struct m0_dtm0_log_cfg *dol_cfg); M0_INTERNAL void m0_dtm0_log_destroy(struct m0_dtm0_log *dol); -M0_INTERNAL void m0_dtm0_log_update_credit(struct m0_dtm0_log *dol, - struct m0_dtm0_log_record *rec, - struct m0_be_tx_credit *accum); -M0_INTERNAL int m0_dtm0_log_update(struct m0_dtm0_log *dol, - struct m0_be_tx *tx, - struct m0_be_op *op, - const struct m0_dtm0_log_record *rec, - bool is_redo); +M0_INTERNAL int m0_dtm0_log_redo_add(struct m0_dtm0_log *dol, + struct m0_be_tx *tx, + const struct m0_dtm0_redo *redo, + const struct m0_fid *p_sdev_fid); +M0_INTERNAL void m0_dtm0_log_redo_add_credit(struct m0_dtm0_log *dol, + const struct m0_dtm0_redo *redo, + struct m0_be_tx_credit *accum); + + +M0_INTERNAL void m0_dtm0_log_prune(struct m0_dtm0_log *dol, + struct m0_be_tx *tx, + struct m0_dtx0_id *dtx0_id); +M0_INTERNAL void m0_dtm0_log_prune_credit(struct m0_dtm0_log *dol, + struct m0_be_tx_credit *accum); + +M0_INTERNAL void m0_dtm0_log_p_get_none_left(struct m0_dtm0_log *dol, + struct m0_be_op *op, + struct m0_dtx0_id *dtx0_id, + bool *successful); + +M0_INTERNAL void m0_dtm0_log_end(struct m0_dtm0_log *dol); + +M0_INTERNAL bool m0_dtm0_log_is_empty(struct m0_dtm0_log *dol); /** @} end of dtm0 group */ #endif /* __MOTR___DTM0_LOG_H__ */ diff --git a/dtm0/pruner.c b/dtm0/pruner.c index 9804d30d59d..13786ece7ba 100644 --- a/dtm0/pruner.c +++ b/dtm0/pruner.c @@ -20,36 +20,309 @@ */ /** - * @addtogroup XXX + * @addtogroup dtm0 * * @{ */ -#define M0_TRACE_SUBSYSTEM M0_TRACE_SUBSYS_XXX +#define M0_TRACE_SUBSYSTEM M0_TRACE_SUBSYS_DTM0 #include "lib/trace.h" #include "dtm0/pruner.h" +#include "dtm0/service.h" /* m0_co_fom_service */ +#include "lib/memory.h" /* M0_ALLOC_PTR */ +#include "lib/errno.h" /* ENOMEM */ +#include "rpc/rpc_opcodes.h" /* M0_DTM0_PRUNER_OPCODE */ +#include "dtm0/dtm0.h" /* m0_dtx0_id */ +#include "fop/fom_generic.h" /* m0_generic_phases_trans */ +#include "dtm0/log.h" /* m0_dtm0_log_p_get_none_left */ -M0_INTERNAL int m0_dtm0_pruner_init(struct m0_dtm0_pruner *dpn, - struct m0_dtm0_pruner_cfg *dpn_cfg) +/* + * INIT -> GET_NEXT -> WAIT_NEXT + * + * + * INIT -> WAIT -> TXN_INIT -> TXN_OPEN -> ... -> SPECIFIC -> REMOVE + * + * REMOVE -> SUCCESS -> FOL_REC_ADD -> ... -> TXN_COMMIT -> ... + * + * ... -> TXN_DONE_WAIT -> WAIT + * + * + * +------------------------------------------+ + * | | + * v | + * INIT -> WAIT -> TX_OPEN -> REMOVE -> TX_CLOSE -> ... + + * | + * v + * FINAL + */ + +struct m0_dtm0_pruner_fom { + struct m0_fom dpf_gen; + + struct m0_dtm0_pruner *dpf_pruner; + struct m0_semaphore dpf_start_sem; + struct m0_semaphore dpf_stop_sem; + + bool dpf_successful; + struct m0_dtx0_id dpf_dtx0_id; + struct m0_be_op dpf_op; +}; + +enum pruner_fom_state { + PFS_INIT = M0_FOM_PHASE_INIT, + PFS_FINISH = M0_FOM_PHASE_FINISH, + /* XXX: Use aliases for every gen fom state used here? */ + /*PFS_TXN_DONE = M0_FOPH_TXN_DONE_WAIT,*/ + PFS_TXN_OPENED = M0_FOPH_TYPE_SPECIFIC, + PFS_GET_NEXT, + PFS_WAIT_NEXT, + PFS_REMOVE, + PFS_NR, +}; + +static struct m0_sm_state_descr pruner_fom_states[] = { +#define __S(name, flags, allowed) \ + [name] = { \ + .sd_flags = flags, \ + .sd_name = #name, \ + .sd_allowed = allowed \ + } + __S(PFS_GET_NEXT, 0, M0_BITS(PFS_WAIT_NEXT)), + __S(PFS_WAIT_NEXT,0, M0_BITS(PFS_FINISH, M0_FOPH_TXN_INIT)), + __S(PFS_REMOVE, 0, M0_BITS(M0_FOPH_SUCCESS)), +#undef __S +}; + +struct m0_sm_trans_descr pruner_fom_trans[] = { + [ARRAY_SIZE(m0_generic_phases_trans)] = + { "todo1", PFS_INIT, PFS_GET_NEXT }, + { "todo2", PFS_GET_NEXT, PFS_WAIT_NEXT }, + { "todo3", PFS_WAIT_NEXT, PFS_FINISH }, + { "todo4", M0_FOPH_TXN_DONE_WAIT, PFS_GET_NEXT }, + { "todo5", PFS_WAIT_NEXT, M0_FOPH_TXN_INIT }, + { "todo6", PFS_TXN_OPENED, PFS_REMOVE }, + { "todo7", PFS_REMOVE, M0_FOPH_SUCCESS }, + +}; + +static struct m0_dtm0_pruner_fom *fom2pruner_fom(const struct m0_fom *fom) +{ + /* XXX TODO bob_of() */ + return container_of(fom, struct m0_dtm0_pruner_fom, dpf_gen); +} + +static struct m0_fom *pruner_fom2fom(struct m0_dtm0_pruner_fom *dpf) +{ + return &dpf->dpf_gen; +} + +static struct m0_sm_conf pruner_fom_sm_conf = { + .scf_name = "m0_dtm0_pruner", + .scf_nr_states = ARRAY_SIZE(pruner_fom_states), + .scf_state = pruner_fom_states, + .scf_trans_nr = ARRAY_SIZE(pruner_fom_trans), + .scf_trans = pruner_fom_trans +}; + +static struct m0_fom_type pruner_fom_type; + +static const struct m0_fom_type_ops pruner_fom_type_ops = { + .fto_create = NULL +}; + +static void pruner_fom_fini(struct m0_fom *fom) +{ + struct m0_dtm0_pruner_fom *dpf = fom2pruner_fom(fom); + + m0_fom_fini(fom); + m0_semaphore_up(&dpf->dpf_stop_sem); +} + +static size_t pruner_fom_locality(const struct m0_fom *fom) +{ + (void)fom; + return 0; +} + +/* + * TODO: This function does not handle scenario where an attempt to open be_tx + * fails. In case of E2BIG, it is a bug (add assert). If ENOMEM happened, + * then we should wait a bit (it is fine to skip pruning), but if happens + * frequently then we should write a warning into the trace; or even + * notify the HA. + */ +static int pruner_fom_tick(struct m0_fom *fom) +{ + struct m0_dtm0_pruner_fom *dpf = fom2pruner_fom(fom); + struct m0_dtm0_pruner *dp = dpf->dpf_pruner; + struct m0_dtm0_log *dol = dp->dp_cfg.dpc_dol; + enum m0_fom_phase_outcome result = M0_FSO_NR; + + M0_ENTRY("pruner_fom=%p pruner=%p phase=%s", dpf, dp, + m0_fom_phase_name(fom, m0_fom_phase(fom))); + + switch (m0_fom_phase(fom)) { + case M0_FOPH_TXN_OPEN: + m0_dtm0_log_prune_credit(dol, &fom->fo_tx.tx_betx_cred); + break; + + case M0_FOPH_TXN_DONE_WAIT: + if (m0_be_tx_state(m0_fom_tx(fom)) == M0_BTS_DONE) { + m0_dtx_fini(&fom->fo_tx); + m0_fom_phase_set(fom, PFS_GET_NEXT); + M0_SET0(&fom->fo_tx); + } + break; + + default: + break; + } + + if (m0_fom_phase(fom) > M0_FOPH_INIT && + m0_fom_phase(fom) < M0_FOPH_NR) { + result = m0_fom_tick_generic(fom); + } + + switch (m0_fom_phase(fom)) { + case PFS_INIT: + m0_fom_phase_set(fom, PFS_GET_NEXT); + m0_semaphore_up(&dpf->dpf_start_sem); + result = M0_FSO_AGAIN; + break; + case PFS_GET_NEXT: + m0_be_op_init(&dpf->dpf_op); + m0_dtm0_log_p_get_none_left(dol, &dpf->dpf_op, + &dpf->dpf_dtx0_id, + &dpf->dpf_successful); + result = m0_be_op_tick_ret(&dpf->dpf_op, fom, PFS_WAIT_NEXT); + break; + case PFS_WAIT_NEXT: + /* TODO: reset? */ + m0_be_op_fini(&dpf->dpf_op); + M0_SET0(&dpf->dpf_op); + if (dpf->dpf_successful) { + m0_fom_phase_set(fom, M0_FOPH_TXN_INIT); + result = M0_FSO_AGAIN; + } else { /* log_end() was called */ + m0_fom_phase_set(fom, PFS_FINISH); + result = M0_FSO_WAIT; + } + break; + case PFS_TXN_OPENED: + m0_fom_phase_set(fom, PFS_REMOVE); + result = M0_FSO_AGAIN; + break; + case PFS_REMOVE: + m0_dtm0_log_prune(dol, &fom->fo_tx.tx_betx, &dpf->dpf_dtx0_id); + /* TODO: Skip M0_FOPH_FOL_REC_ADD */ + m0_fom_phase_set(fom, M0_FOPH_SUCCESS); + result = M0_FSO_AGAIN; + break; + default: + break; + } + + M0_LEAVE("result=%d", result); + return result; +} + +static const struct m0_fom_ops pruner_fom_ops = { + .fo_fini = pruner_fom_fini, + .fo_tick = pruner_fom_tick, + .fo_home_locality = pruner_fom_locality +}; + +M0_INTERNAL int m0_dtm0_pruner_init(struct m0_dtm0_pruner *dp, + struct m0_dtm0_pruner_cfg *dp_cfg) { + struct m0_dtm0_pruner_fom *dpf; + + M0_PRE(dp_cfg->dpc_cfs->cfs_reqh_service->rs_type == &m0_cfs_stype); + + M0_ALLOC_PTR(dpf); + if (dpf == NULL) + return M0_ERR(-ENOMEM); + + m0_semaphore_init(&dpf->dpf_start_sem, 0); + m0_semaphore_init(&dpf->dpf_stop_sem, 0); + + dp->dp_cfg = *dp_cfg; + dp->dp_pruner_fom = dpf; + dpf->dpf_pruner = dp; return 0; } -M0_INTERNAL void m0_dtm0_pruner_fini(struct m0_dtm0_pruner *dpn) +M0_INTERNAL void m0_dtm0_pruner_fini(struct m0_dtm0_pruner *dp) +{ + struct m0_dtm0_pruner_fom *dpf = dp->dp_pruner_fom; + + m0_semaphore_fini(&dpf->dpf_start_sem); + m0_semaphore_fini(&dpf->dpf_stop_sem); + m0_free(dpf); + dp->dp_pruner_fom = NULL; +} + +M0_INTERNAL void m0_dtm0_pruner_start(struct m0_dtm0_pruner *dp) { + struct m0_dtm0_pruner_fom *dpf = dp->dp_pruner_fom; + struct m0_reqh *reqh = + dp->dp_cfg.dpc_cfs->cfs_reqh_service->rs_reqh; + struct m0_fom *fom = pruner_fom2fom(dpf); + + m0_fom_init(fom, &pruner_fom_type, + &pruner_fom_ops, NULL, NULL, reqh); + fom->fo_local = true; + fom->fo_local_update = true; + m0_fom_queue(fom); + m0_semaphore_down(&dpf->dpf_start_sem); } -M0_INTERNAL void m0_dtm0_pruner_start(struct m0_dtm0_pruner *dpn) +M0_INTERNAL void m0_dtm0_pruner_stop(struct m0_dtm0_pruner *dp) { + struct m0_dtm0_pruner_fom *dpf = dp->dp_pruner_fom; + + m0_semaphore_down(&dpf->dpf_stop_sem); +} + +M0_INTERNAL void m0_dtm0_pruner_mod_init(void) +{ + m0_sm_conf_extend(m0_generic_conf.scf_state, pruner_fom_states, + min_check(m0_generic_conf.scf_nr_states, + pruner_fom_sm_conf.scf_nr_states)); + m0_sm_conf_trans_extend(&m0_generic_conf, &pruner_fom_sm_conf); + /* + * TODO: turn it into an array and move the array closer to the + * definition of pruner_fom_states. + */ + pruner_fom_states[M0_FOPH_INIT].sd_allowed |= M0_BITS(PFS_GET_NEXT); + pruner_fom_states[M0_FOPH_TXN_DONE_WAIT].sd_allowed |= + M0_BITS(PFS_GET_NEXT); + + /* + * TODO: Generic FOM sm does not have transitions from TYPE_SPECIFIC to + * FINISH, SUCCESS, FAILED. Because of that, we either should explicitly + * add them in our sm or remove them. Here we are removing them. + * Consider adding the missing transitions right into the generic FOM + * sm. + */ + pruner_fom_states[M0_FOPH_TYPE_SPECIFIC].sd_allowed = + M0_BITS(PFS_REMOVE); + + m0_sm_conf_init(&pruner_fom_sm_conf); + + m0_fom_type_init(&pruner_fom_type, M0_DTM0_PRUNER_OPCODE, + &pruner_fom_type_ops, &m0_cfs_stype, + &pruner_fom_sm_conf); } -M0_INTERNAL void m0_dtm0_pruner_stop(struct m0_dtm0_pruner *dpn) +M0_INTERNAL void m0_dtm0_pruner_mod_fini(void) { + m0_sm_conf_fini(&pruner_fom_sm_conf); } #undef M0_TRACE_SUBSYSTEM -/** @} end of XXX group */ +/** @} end of dtm0 group */ /* * Local variables: diff --git a/dtm0/pruner.h b/dtm0/pruner.h index 4129cbfc81e..a6820b0feea 100644 --- a/dtm0/pruner.h +++ b/dtm0/pruner.h @@ -24,6 +24,8 @@ #ifndef __MOTR___DTM0_PRUNER_H__ #define __MOTR___DTM0_PRUNER_H__ +#include "fop/fom.h" + /** * @defgroup dtm0 * @@ -54,10 +56,16 @@ * +----+ */ -struct m0_dtm0_pruner { -}; +struct m0_dtm0_pruner_fom; struct m0_dtm0_pruner_cfg { + struct m0_co_fom_service *dpc_cfs; + struct m0_dtm0_log *dpc_dol; +}; + +struct m0_dtm0_pruner { + struct m0_dtm0_pruner_fom *dp_pruner_fom; + struct m0_dtm0_pruner_cfg dp_cfg; }; M0_INTERNAL int m0_dtm0_pruner_init(struct m0_dtm0_pruner *dpn, @@ -66,6 +74,8 @@ M0_INTERNAL void m0_dtm0_pruner_fini(struct m0_dtm0_pruner *dpn); M0_INTERNAL void m0_dtm0_pruner_start(struct m0_dtm0_pruner *dpn); M0_INTERNAL void m0_dtm0_pruner_stop(struct m0_dtm0_pruner *dpn); +M0_INTERNAL void m0_dtm0_pruner_mod_init(void); +M0_INTERNAL void m0_dtm0_pruner_mod_fini(void); /** @} end of dtm0 group */ #endif /* __MOTR___DTM0_PRUNER_H__ */ diff --git a/dtm0/remach.h b/dtm0/remach.h index 1f7eeddaaff..63c75da5484 100644 --- a/dtm0/remach.h +++ b/dtm0/remach.h @@ -67,10 +67,15 @@ * +--------+ +----+ */ +struct m0_dtx0_payload; + struct m0_dtm0_remach { }; struct m0_dtm0_remach_cfg { + void (*dtrc_blob_handler)(struct m0_dtx0_payload *payload, + void *datum); + void *dtrc_blob_handler_datum; }; M0_INTERNAL int m0_dtm0_remach_init(struct m0_dtm0_remach *drm, diff --git a/dtm0/service.c b/dtm0/service.c index a046a48cf28..d25181f5944 100644 --- a/dtm0/service.c +++ b/dtm0/service.c @@ -524,6 +524,100 @@ M0_INTERNAL void dtm0_service_conns_term(struct m0_dtm0_service *service) M0_LEAVE(); } +/* -----8<------------------------------------------------------------------- */ +/* CO_FOM service */ + +struct cfs_service { + struct m0_reqh_service cfs_base; +}; + +static int cfs_allocate(struct m0_reqh_service **out, + const struct m0_reqh_service_type *stype); + +static const struct m0_reqh_service_type_ops cfs_stype_ops = { + .rsto_service_allocate = cfs_allocate +}; + +struct m0_reqh_service_type m0_cfs_stype = { + .rst_name = "co-fom-service", + .rst_ops = &cfs_stype_ops, + /* Level justification: co_foms may be needed before NORMAL level. */ + .rst_level = M0_BE_TX_SVC_LEVEL, +}; + +M0_INTERNAL int m0_cfs_register(void) +{ + return m0_reqh_service_type_register(&m0_cfs_stype); +} + +M0_INTERNAL void m0_cfs_unregister(void) +{ + m0_reqh_service_type_unregister(&m0_cfs_stype); +} + +static int cfs_start(struct m0_reqh_service *service); +static void cfs_stop(struct m0_reqh_service *service); +static void cfs_fini(struct m0_reqh_service *service); + +static const struct m0_reqh_service_ops cfs_ops = { + .rso_start = cfs_start, + .rso_stop = cfs_stop, + .rso_fini = cfs_fini +}; + +static int cfs_allocate(struct m0_reqh_service **service, + const struct m0_reqh_service_type *stype) +{ + struct cfs_service *s; + + M0_ENTRY(); + M0_PRE(stype == &m0_cfs_stype); + + M0_ALLOC_PTR(s); + if (s == NULL) + return M0_ERR(-ENOMEM); + + s->cfs_base.rs_ops = &cfs_ops; + *service = &s->cfs_base; + + return M0_RC(0); +} + +static void cfs_fini(struct m0_reqh_service *service) +{ + M0_ENTRY(); + m0_free(container_of(service, struct cfs_service, cfs_base)); + M0_LEAVE(); +} + +static int cfs_start(struct m0_reqh_service *service) +{ + M0_ENTRY(); + return M0_RC(0); +} + +static void cfs_stop(struct m0_reqh_service *service) +{ + M0_ENTRY(); + M0_LEAVE(); +} + +M0_INTERNAL int m0_co_fom_service_init(struct m0_co_fom_service *cfs, + struct m0_reqh *reqh) +{ + M0_PRE(cfs->cfs_reqh_service == NULL); + return m0_reqh_service_setup(&cfs->cfs_reqh_service, &m0_cfs_stype, + reqh, NULL, NULL); +} + +M0_INTERNAL void m0_co_fom_service_fini(struct m0_co_fom_service *cfs) +{ + m0_reqh_service_quit(cfs->cfs_reqh_service); + cfs->cfs_reqh_service = NULL; +} + +/* -----8<------------------------------------------------------------------- */ + /* * Local variables: * c-indentation-style: "K&R" diff --git a/dtm0/service.h b/dtm0/service.h index 6801bd21e21..376439f4fd3 100644 --- a/dtm0/service.h +++ b/dtm0/service.h @@ -93,6 +93,23 @@ M0_INTERNAL struct m0_dtm0_service *m0_dtm0_fom2service(struct m0_fom *fom); M0_INTERNAL bool m0_dtm0_in_ut(void); M0_INTERNAL bool m0_dtm0_is_expecting_redo_from_client(void); +/* -----8<------------------------------------------------------------------- */ +struct m0_reqh_service; + +struct m0_co_fom_service { + struct m0_reqh_service *cfs_reqh_service; +}; + +M0_INTERNAL int m0_co_fom_service_init(struct m0_co_fom_service *cfs, + struct m0_reqh *reqh); +M0_INTERNAL void m0_co_fom_service_fini(struct m0_co_fom_service *cfs); +M0_INTERNAL int m0_cfs_register(void); +M0_INTERNAL void m0_cfs_unregister(void); + +extern struct m0_reqh_service_type m0_cfs_stype; + +/* -----8<------------------------------------------------------------------- */ + #endif /* __MOTR_DTM0_SERVICE_H__ */ /* diff --git a/dtm0/tx_desc.c b/dtm0/tx_desc.c index 9c1657ddf3d..33971d3aa7a 100644 --- a/dtm0/tx_desc.c +++ b/dtm0/tx_desc.c @@ -80,7 +80,7 @@ M0_INTERNAL int m0_dtm0_tx_desc_copy(const struct m0_dtm0_tx_desc *src, M0_ENTRY(); M0_PRE(m0_dtm0_tx_desc__invariant(src)); - + M0_LOG(M0_DEBUG, "participants nr:%d", src->dtd_ps.dtp_nr); rc = m0_dtm0_tx_desc_init(dst, src->dtd_ps.dtp_nr); if (rc == 0) { dst->dtd_id = src->dtd_id; diff --git a/dtm0/ut/domain.c b/dtm0/ut/domain.c index d8bd483e4d5..8c429ae4ba7 100644 --- a/dtm0/ut/domain.c +++ b/dtm0/ut/domain.c @@ -32,9 +32,61 @@ #include "lib/memory.h" /* M0_ALLOC_PTR */ #include "dtm0/cfg_default.h" /* m0_dtm0_domain_cfg_default_dup */ #include "ut/ut.h" /* M0_UT_ASSERT */ +#include "be/ut/helper.h" /* m0_be_ut_backend_init */ +#include "reqh/reqh.h" /* m0_reqh */ +enum { + M0_DTM0_UT_DOMAIN_SEG_SIZE = 0x2000000, +}; -void m0_dtm0_ut_domain_init_fini(void) +struct dtm0_ut_domain_ctx { + struct m0_be_ut_backend ut_be; + struct m0_be_ut_seg ut_seg; + struct m0_reqh reqh; + struct m0_dtm0_domain_cfg dod_cfg; + struct m0_dtm0_domain dod; +}; + +static void dtm0_ut_domain_init(struct dtm0_ut_domain_ctx *dctx) +{ + int rc; + + rc = m0_dtm0_domain_cfg_default_dup(&dctx->dod_cfg, true); + M0_UT_ASSERT(rc == 0); + + m0_be_ut_backend_init(&dctx->ut_be); + m0_be_ut_seg_init(&dctx->ut_seg, &dctx->ut_be, + M0_DTM0_UT_DOMAIN_SEG_SIZE); + + /* TODO: Move it to log_init ? */ + dctx->dod_cfg.dodc_log.dlc_be_domain = &dctx->ut_be.but_dom; + dctx->dod_cfg.dodc_log.dlc_seg = + m0_be_domain_seg_first(dctx->dod_cfg.dodc_log.dlc_be_domain); + + rc = M0_REQH_INIT(&dctx->reqh, + .rhia_dtm = (void*)1, + .rhia_mdstore = (void*)1, + .rhia_db = dctx->dod_cfg.dodc_log.dlc_seg, + .rhia_fid = &g_process_fid); + M0_UT_ASSERT(rc == 0); + m0_reqh_start(&dctx->reqh); + + dctx->dod_cfg.dod_reqh = &dctx->reqh; + rc = m0_dtm0_domain_init(&dctx->dod, &dctx->dod_cfg); + M0_UT_ASSERT(rc == 0); +} + +static void dtm0_ut_domain_fini(struct dtm0_ut_domain_ctx *dctx) +{ + m0_dtm0_domain_fini(&dctx->dod); + m0_reqh_services_terminate(&dctx->reqh); + m0_reqh_fini(&dctx->reqh); + m0_be_ut_seg_fini(&dctx->ut_seg); + m0_be_ut_backend_fini(&dctx->ut_be); + m0_dtm0_domain_cfg_free(&dctx->dod_cfg); +} + +void m0_dtm0_ut_volatile_domain_init_fini(void) { struct m0_dtm0_domain *dod; struct m0_dtm0_domain_cfg *dod_cfg; @@ -45,10 +97,12 @@ void m0_dtm0_ut_domain_init_fini(void) M0_ALLOC_PTR(dod_cfg); M0_UT_ASSERT(dod_cfg != NULL); - rc = m0_dtm0_domain_cfg_default_dup(dod_cfg); + rc = m0_dtm0_domain_cfg_default_dup(dod_cfg, true); + M0_UT_ASSERT(rc == 0); + + rc = m0_dtm0_domain_init(dod, dod_cfg); M0_UT_ASSERT(rc == 0); - m0_dtm0_domain_init(dod, dod_cfg); m0_dtm0_domain_fini(dod); m0_dtm0_domain_cfg_free(dod_cfg); @@ -57,6 +111,14 @@ void m0_dtm0_ut_domain_init_fini(void) m0_free(dod); } +void m0_dtm0_ut_persistent_domain_init_fini(void) +{ + struct dtm0_ut_domain_ctx dctx = {}; + + dtm0_ut_domain_init(&dctx); + dtm0_ut_domain_fini(&dctx); +} + #undef M0_TRACE_SUBSYSTEM /** @} end of dtm0 group */ diff --git a/dtm0/ut/helper.c b/dtm0/ut/helper.c index 32a4891413c..d26f8287181 100644 --- a/dtm0/ut/helper.c +++ b/dtm0/ut/helper.c @@ -35,11 +35,14 @@ #include "lib/misc.h" /* M0_IS0 */ #include "net/net.h" /* m0_net_all_xprt_get */ #include "dtm0/service.h" /* m0_dtm0_service */ - - +#include "dtm0/cfg_default.h" /* m0_dtm0_domain_cfg_default_dup */ +#include "dtm0/dtm0.h" /* m0_dtm0_redo */ +#include "conf/objs/common.h" /* M0_CONF__SDEV_FT_ID */ +#include "be/tx_bulk.h" /* m0_be_tx_bulk */ struct m0_reqh_service; enum { + M0_DTM0_UT_LOG_SIMPLE_SEG_SIZE = 0x2000000, MAX_RPCS_IN_FLIGHT = 10, }; @@ -100,12 +103,14 @@ M0_INTERNAL void m0_ut_dtm0_helper_init(struct m0_ut_dtm0_helper *udh) &udh->udh_client_dtm0_fid, &svc); M0_UT_ASSERT(rc == 0); + M0_UT_ASSERT(svc != NULL); udh->udh_client_dtm0_service = container_of(svc, struct m0_dtm0_service, dos_generic); svc = m0_reqh_service_lookup(udh->udh_server_reqh, &udh->udh_server_dtm0_fid); + M0_UT_ASSERT(svc != NULL); /* TODO export the function which does bob_of() */ udh->udh_server_dtm0_service = container_of(svc, struct m0_dtm0_service, dos_generic); @@ -122,6 +127,207 @@ M0_INTERNAL void m0_ut_dtm0_helper_fini(struct m0_ut_dtm0_helper *udh) m0_rpc_server_stop(&udh->udh_sctx); } +M0_INTERNAL struct dtm0_ut_log_ctx *dtm0_ut_log_init(void) +{ + struct dtm0_ut_log_ctx *lctx; + int rc; + + M0_ALLOC_PTR(lctx); + M0_UT_ASSERT(lctx != NULL); + + m0_be_ut_backend_init(&lctx->ut_be); + m0_be_ut_seg_init(&lctx->ut_seg, &lctx->ut_be, + M0_DTM0_UT_LOG_SIMPLE_SEG_SIZE); + rc = m0_dtm0_domain_cfg_default_dup(&lctx->dod_cfg, true); + M0_UT_ASSERT(rc == 0); + lctx->dod_cfg.dodc_log.dlc_be_domain = &lctx->ut_be.but_dom; + lctx->dod_cfg.dodc_log.dlc_seg = + m0_be_domain_seg_first(lctx->dod_cfg.dodc_log.dlc_be_domain); + + rc = m0_dtm0_log_create(&lctx->dol, &lctx->dod_cfg.dodc_log); + M0_UT_ASSERT(rc == 0); + + return lctx; +} + +M0_INTERNAL void dtm0_ut_log_fini(struct dtm0_ut_log_ctx *lctx) +{ + m0_dtm0_log_destroy(&lctx->dol); + m0_dtm0_domain_cfg_free(&lctx->dod_cfg); + m0_be_ut_seg_fini(&lctx->ut_seg); + m0_be_ut_backend_fini(&lctx->ut_be); + m0_free(lctx); +} + +enum { + M0_DTM0_UT_LOG_SIMPLE_REDO_SIZE = 0x100, +}; + +M0_INTERNAL struct m0_dtm0_redo *dtm0_ut_redo_get(int timestamp) +{ + int rc; + struct m0_buf *redo_buf; + struct m0_dtm0_redo *redo; + static struct m0_fid p_sdev_fid; + uint64_t seed = 42; + int i; + + M0_ALLOC_PTR(redo); + M0_UT_ASSERT(redo != NULL); + + M0_ALLOC_PTR(redo_buf); + M0_UT_ASSERT(redo_buf != NULL); + + rc = m0_buf_alloc(redo_buf, M0_DTM0_UT_LOG_SIMPLE_REDO_SIZE); + M0_UT_ASSERT(rc == 0); + for (i = 0; i < redo_buf->b_nob; ++i) + ((char *)redo_buf->b_addr)[i] = m0_rnd64(&seed) & 0xff; + p_sdev_fid = M0_FID_TINIT(M0_CONF__SDEV_FT_ID, 1, 2); + + *redo = (struct m0_dtm0_redo){ + .dtr_descriptor = { + .dtd_id = { + .dti_timestamp = timestamp, + .dti_originator_sdev_fid = p_sdev_fid, + }, + .dtd_participants = { + .dtpa_participants_nr = 1, + .dtpa_participants = &p_sdev_fid, + }, + }, + .dtr_payload = { + .dtp_type = M0_DTX0_PAYLOAD_BLOB, + .dtp_data = { + .ab_count = 1, + .ab_elems = redo_buf, + }, + }, + }; + + return redo; +} + +M0_INTERNAL void dtm0_ut_redo_put(struct m0_dtm0_redo *redo) +{ + /* TODO */ +} + + +static void dtm0_ut_log_produce_redo_add(struct m0_be_tx_bulk *tb, + struct dtm0_ut_log_ctx *lctx) +{ + int i; + struct m0_dtm0_redo *redo; + struct m0_be_tx_credit credit; + + for (i = 0; i < MPSC_NR_REC_TOTAL; ++i) { + redo = dtm0_ut_redo_get(i); + credit = M0_BE_TX_CREDIT(0, 0); + m0_dtm0_log_redo_add_credit(&lctx->dol, redo, &credit); + M0_BE_OP_SYNC(op, + m0_be_tx_bulk_put(tb, &op, &credit, 0, 0, redo)); + } + + m0_be_tx_bulk_end(tb); +} + + +static void dtm0_ut_log_tx_bulk_parallel_do(struct m0_be_tx_bulk *tb, + struct m0_be_tx *tx, + struct m0_be_op *op, + void *datum, + void *user, + uint64_t worker_index, + uint64_t partition) +{ + struct dtm0_ut_log_ctx *lctx = datum; + struct m0_dtm0_redo *redo = user; + struct m0_fid p_sdev_fid; + + (void) worker_index; + (void) partition; + + p_sdev_fid = M0_FID_TINIT(M0_CONF__SDEV_FT_ID, 1, 2); + + m0_be_op_active(op); + m0_dtm0_log_redo_add(&lctx->dol, tx, redo, &p_sdev_fid); + m0_be_op_done(op); + + dtm0_ut_redo_put(redo); +} + +static void dtm0_ut_log_tx_bulk_parallel_done(struct m0_be_tx_bulk *tb, + void *datum, + void *user, + uint64_t worker_index, + uint64_t partition) +{ +} + +M0_INTERNAL void dtm0_ut_log_mp_init(struct dtm0_ut_log_mp_ctx *lmp_ctx, + struct dtm0_ut_log_ctx *lctx) +{ + struct m0_be_tx_bulk *tb; + struct m0_be_tx_bulk_cfg *tb_cfg; + struct m0_be_op *op; + int rc; + + M0_ALLOC_PTR(tb); + M0_UT_ASSERT(tb != NULL); + + M0_ALLOC_PTR(tb_cfg); + M0_UT_ASSERT(tb_cfg != NULL); + + M0_ALLOC_PTR(op); + M0_UT_ASSERT(op != NULL); + + m0_be_op_init(op); + + *tb_cfg = (struct m0_be_tx_bulk_cfg) { + .tbc_q_cfg = { + .bqc_q_size_max = MPSC_NR_REC_TOTAL / 2, + .bqc_producers_nr_max = 1, + }, + .tbc_workers_nr = 100, + .tbc_partitions_nr = 1, + .tbc_work_items_per_tx_max = 1, + .tbc_datum = lctx, + .tbc_do = + &dtm0_ut_log_tx_bulk_parallel_do, + .tbc_done = + &dtm0_ut_log_tx_bulk_parallel_done, + .tbc_dom = &lctx->ut_be.but_dom, + }; + + rc = m0_be_tx_bulk_init(tb, tb_cfg); + M0_UT_ASSERT(rc == 0); + + lmp_ctx->op = op; + lmp_ctx->lctx = lctx; + lmp_ctx->tb_cfg = tb_cfg; + lmp_ctx->tb = tb; +} + +M0_INTERNAL void dtm0_ut_log_mp_run(struct dtm0_ut_log_mp_ctx *lmp_ctx) +{ + m0_be_tx_bulk_run(lmp_ctx->tb, lmp_ctx->op); + dtm0_ut_log_produce_redo_add(lmp_ctx->tb, lmp_ctx->lctx); +} + +M0_INTERNAL void dtm0_ut_log_mp_fini(struct dtm0_ut_log_mp_ctx *lmp_ctx) +{ + int rc; + rc = m0_be_tx_bulk_status(lmp_ctx->tb); + M0_UT_ASSERT(rc == 0); + + m0_be_op_fini(lmp_ctx->op); + m0_free(lmp_ctx->op); + + m0_be_tx_bulk_fini(lmp_ctx->tb); + m0_free(lmp_ctx->tb); + m0_free(lmp_ctx->tb_cfg); +} + #undef M0_TRACE_SUBSYSTEM /** @} end of dtm0 group */ diff --git a/dtm0/ut/helper.h b/dtm0/ut/helper.h index d58b04ea231..57a5630eb66 100644 --- a/dtm0/ut/helper.h +++ b/dtm0/ut/helper.h @@ -59,6 +59,40 @@ struct m0_ut_dtm0_helper { M0_INTERNAL void m0_ut_dtm0_helper_init(struct m0_ut_dtm0_helper *udh); M0_INTERNAL void m0_ut_dtm0_helper_fini(struct m0_ut_dtm0_helper *udh); +struct dtm0_ut_log_ctx { + struct m0_be_ut_backend ut_be; + struct m0_be_ut_seg ut_seg; + struct m0_dtm0_domain_cfg dod_cfg; + struct m0_dtm0_log dol; +}; + +M0_INTERNAL struct dtm0_ut_log_ctx *dtm0_ut_log_init(void); +M0_INTERNAL void dtm0_ut_log_fini(struct dtm0_ut_log_ctx *lctx); + +M0_INTERNAL struct m0_dtm0_redo *dtm0_ut_redo_get(int timestamp); +M0_INTERNAL void dtm0_ut_redo_put(struct m0_dtm0_redo *redo); + +enum { + MPSC_TS_BASE = 0, +}; + +enum { + MPSC_NR_REC_TOTAL = 0x100, +}; + +struct dtm0_ut_log_mp_ctx { + struct m0_be_tx_bulk_cfg *tb_cfg; + struct m0_be_tx_bulk *tb; + struct m0_be_op *op; + struct dtm0_ut_log_ctx *lctx; +}; + +M0_INTERNAL void dtm0_ut_log_mp_init(struct dtm0_ut_log_mp_ctx *lmp_ctx, + struct dtm0_ut_log_ctx *lctx); + +M0_INTERNAL void dtm0_ut_log_mp_run(struct dtm0_ut_log_mp_ctx *lmp_ctx); + +M0_INTERNAL void dtm0_ut_log_mp_fini(struct dtm0_ut_log_mp_ctx *lmp_ctx); /** @} end of dtm0 group */ #endif /* __MOTR_DTM0_UT_HELPER_H__ */ diff --git a/dtm0/ut/log.c b/dtm0/ut/log.c index 61b4b043735..4276f1a095a 100644 --- a/dtm0/ut/log.c +++ b/dtm0/ut/log.c @@ -30,6 +30,240 @@ #include "dtm0/log.h" +#include "ut/ut.h" /* M0_UT_ASSERT */ +#include "lib/memory.h" /* M0_ALLOC_PTR */ +#include "lib/misc.h" /* m0_rnd64 */ +#include "be/ut/helper.h" /* m0_be_ut_backend_init */ +#include "be/domain.h" /* m0_be_domain_seg_first */ +#include "conf/objs/common.h" /* M0_CONF__SDEV_FT_ID */ +#include "fid/fid.h" /* M0_FID_TINIT */ + +#include "dtm0/domain.h" /* m0_dtm0_domain_cfg */ +#include "dtm0/cfg_default.h" /* m0_dtm0_domain_cfg_default_dup */ +#include "dtm0/dtm0.h" /* m0_dtm0_redo */ +#include "be/tx_bulk.h" /* m0_be_tx_bulk */ +#include "dtm0/ut/helper.h" /* dtm0_ut_log_ctx */ + +enum { + M0_DTM0_UT_LOG_SIMPLE_REDO_SIZE = 0x100, +}; + +/* TODO: add dtm0_ut_log_init-fini */ + +void m0_dtm0_ut_log_simple(void) +{ + enum { + TS_BASE = 0x100, + NR_OPER = 0x10, + NR_REC_PER_OPER = 0x10, + }; + struct m0_dtm0_redo *redo; + struct m0_buf redo_buf = {}; + struct m0_fid p_sdev_fid; + uint64_t seed = 42; + int rc; + int i; + int j; + struct m0_dtx0_id dtx0_id; + struct dtm0_ut_log_ctx *lctx; + bool successful; + + M0_ALLOC_PTR(redo); + M0_UT_ASSERT(redo != NULL); + rc = m0_buf_alloc(&redo_buf, M0_DTM0_UT_LOG_SIMPLE_REDO_SIZE); + M0_UT_ASSERT(rc == 0); + + for (i = 0; i < redo_buf.b_nob; ++i) + ((char *)redo_buf.b_addr)[i] = m0_rnd64(&seed) & 0xff; + p_sdev_fid = M0_FID_TINIT(M0_CONF__SDEV_FT_ID, 1, 2); + *redo = (struct m0_dtm0_redo){ + .dtr_descriptor = { + .dtd_id = { + .dti_timestamp = TS_BASE, + .dti_originator_sdev_fid = p_sdev_fid, + }, + .dtd_participants = { + .dtpa_participants_nr = 1, + .dtpa_participants = &p_sdev_fid, + }, + }, + .dtr_payload = { + .dtp_type = M0_DTX0_PAYLOAD_BLOB, + .dtp_data = { + .ab_count = 1, + .ab_elems = &redo_buf, + }, + }, + }; + + lctx = dtm0_ut_log_init(); + + for (i = 0; i < NR_OPER; ++i) { + rc = m0_dtm0_log_open(&lctx->dol, &lctx->dod_cfg.dodc_log); + M0_UT_ASSERT(rc == 0); + for (j = 0; j < NR_REC_PER_OPER; ++j) { + redo->dtr_descriptor.dtd_id.dti_timestamp = j + TS_BASE; + M0_BE_UT_TRANSACT(&lctx->ut_be, tx, cred, + m0_dtm0_log_redo_add_credit(&lctx->dol, + redo, + &cred), + rc = m0_dtm0_log_redo_add(&lctx->dol, + tx, + redo, + &p_sdev_fid)); + M0_UT_ASSERT(rc == 0); + } + for (j = 0; j < NR_REC_PER_OPER + 1; ++j) { + if (j == NR_REC_PER_OPER) + m0_dtm0_log_end(&lctx->dol); + successful = j == NR_REC_PER_OPER; + M0_BE_OP_SYNC(op, + m0_dtm0_log_p_get_none_left(&lctx->dol, + &op, + &dtx0_id, + &successful)); + M0_UT_ASSERT(equi(successful, j < NR_REC_PER_OPER)); + if (!successful) + break; + redo->dtr_descriptor.dtd_id.dti_timestamp = j + TS_BASE; + M0_UT_ASSERT(m0_dtx0_id_eq(&dtx0_id, + &redo->dtr_descriptor.dtd_id)); + + M0_BE_UT_TRANSACT(&lctx->ut_be, tx, cred, + m0_dtm0_log_prune_credit(&lctx->dol, + &cred), + m0_dtm0_log_prune(&lctx->dol, tx, + &redo->dtr_descriptor.dtd_id)); + } + m0_dtm0_log_close(&lctx->dol); + } + + + dtm0_ut_log_fini(lctx); + + m0_buf_free(&redo_buf); + m0_free(redo); +} + +/* + * TODO: Add "re-create" test. Create log, try to create it once again. + */ + +static void dtm0_ut_log_remove(struct dtm0_ut_log_ctx *lctx, + int timestamp, int nr) +{ + int i; + struct m0_dtx0_id dtx0_id; + bool *seen; + bool successful; + + M0_ALLOC_ARR(seen, nr); + M0_UT_ASSERT(seen != NULL); + M0_UT_ASSERT(m0_forall(i, nr, !seen[i])); + + for (i = 0; i < nr; ++i) { + M0_BE_OP_SYNC(op, + m0_dtm0_log_p_get_none_left(&lctx->dol, &op, + &dtx0_id, &successful)); + M0_UT_ASSERT(successful); + seen[dtx0_id.dti_timestamp - timestamp] = true; + M0_BE_UT_TRANSACT(&lctx->ut_be, tx, cred, + m0_dtm0_log_prune_credit(&lctx->dol, &cred), + m0_dtm0_log_prune(&lctx->dol, tx, &dtx0_id)); + } + + M0_UT_ASSERT(m0_forall(i, nr, seen[i])); +} + + +void m0_dtm0_ut_log_mpsc(void) +{ + struct dtm0_ut_log_ctx *lctx; + struct dtm0_ut_log_mp_ctx lmp_ctx; + int rc; + + lctx = dtm0_ut_log_init(); + M0_UT_ASSERT(lctx != NULL); + + rc = m0_dtm0_log_open(&lctx->dol, &lctx->dod_cfg.dodc_log); + M0_UT_ASSERT(rc == 0); + + dtm0_ut_log_mp_init(&lmp_ctx, lctx); + dtm0_ut_log_mp_run(&lmp_ctx); + m0_be_op_wait(lmp_ctx.op); + dtm0_ut_log_mp_fini(&lmp_ctx); + + dtm0_ut_log_remove(lctx, MPSC_TS_BASE, MPSC_NR_REC_TOTAL); + + m0_dtm0_log_close(&lctx->dol); + dtm0_ut_log_fini(lctx); +} + +/* TODO: (before landing) + * Goals: + * 1. Land new DTM0 log code in main. + * 2. The code must always be enabled. + * 3. No visible changes (from Motr client user's perspective). + * 4. Apply real load to the log. + * + * 1. Add a test where log_end() works fine before we are awaiting + * for the latest and after we are awaiting for the latest. + * 2. Empty log + end == !successful. + * 3. Simple pruner (remove after added), concurrency == 1 (single FOM, single + * record). + * x4. Pruner UTs: + * x4.1. Add to the log, run pruner, check if log is empty. + * x4.2. One tx_bulk that adds records to the log, run pruner. + * x4.3. Run pruner, empty log, nothing happens. + * x4.4. Add many log records, run pruner, some record may still be present + * in the log. + * x4.5. Run pruner, run tx_bulk, wait until log is empty, wait 0.5 sec, + * repeat (tx_bulk, wait until empty). + * x5. Run pruner in DTM0 domain (init/start/stop/fini) by default. + * 6. DTX0 for server, and call it from CAS. + * 7. Random dtxid if CAS request has empty dtx descriptor. + * + * 6-7, options: + * ENABLE_DTM0: + * Server: Use old dtm0 code. + * Client: send correct txd. + * !ENABLE_DTM0: + * NODTM0: + * Server: Use non-dtm0 code. + * Client: send empty txd. + * !NODTM0: + * Server: Use new-dtm0 code. + * Client: send random txd. + * + * [2.1] + * log_add() + * log_end() + * log_wait() -> ok + * log_wait() -> fail + * + * [2.2] + * log_add() + * log_wait() -> ok + * log_end() + * log_wait() -> fail + * + * [3] + * log_end() + * log_wait() -> fail + * + * TODO: (after landing) + * 1. Wait until be tx becomes PERSISTENT. + * 2. Simple pruner, concurrency > 1 (many FOMs). + * 3. Simple pruner, batch delete. + * 4. DTX0 for client is always enabled at data structure level but + * the state transitions may be disabled by NODTM. If that is not possible, + * then just generate random dtxid. + * 5. Pruner: do not add records to FOL. + * 6. s/dod/dodc in DTM0 log. + * 7. Co-fom service or DTM fom service? single service for coroutines? + * + * Send Pmsg from CAS when dtx is PERSISTENT already. + */ #undef M0_TRACE_SUBSYSTEM diff --git a/dtm0/ut/main.c b/dtm0/ut/main.c index a7baab0fb66..0dc682d1109 100644 --- a/dtm0/ut/main.c +++ b/dtm0/ut/main.c @@ -1452,22 +1452,42 @@ static void remach_cli_evict_mixed(void) } extern void m0_dtm0_ut_drlink_simple(void); -extern void m0_dtm0_ut_domain_init_fini(void); +extern void m0_dtm0_ut_volatile_domain_init_fini(void); +extern void m0_dtm0_ut_log_simple(void); +extern void m0_dtm0_ut_log_mpsc(void); +extern void m0_dtm0_ut_pruner_init_fini(void); +extern void m0_dtm0_ut_pruner_start_stop(void); +extern void m0_dtm0_ut_pruner_one(void); +extern void m0_dtm0_ut_pruner_two(void); +extern void m0_dtm0_ut_pruner_many_left(void); +extern void m0_dtm0_ut_pruner_mpsc(void); +extern void m0_dtm0_ut_pruner_mpsc_many(void); +extern void m0_dtm0_ut_persistent_domain_init_fini(void); struct m0_ut_suite dtm0_ut = { .ts_name = "dtm0-ut", .ts_tests = { - { "xcode", cas_xcode_test }, - { "drlink-simple", &m0_dtm0_ut_drlink_simple }, - { "domain_init-fini", &m0_dtm0_ut_domain_init_fini }, - { "remach-init-fini", remach_init_fini }, - { "remach-start-stop", remach_start_stop }, - { "remach-boot-cluster-ss", remach_boot_cluster_ss }, - { "remach-boot-cluster-cs", remach_boot_cluster_cs }, - { "remach-reboot-server", remach_reboot_server }, - { "remach-reboot-twice", remach_reboot_twice }, - { "remach-boot-real-log", remach_boot_real_log }, - { "remach-real-log-replay", remach_real_log_replay }, + { "xcode", cas_xcode_test }, + { "drlink-simple", &m0_dtm0_ut_drlink_simple }, + { "domain_init-fini", &m0_dtm0_ut_volatile_domain_init_fini }, + { "log-simple", &m0_dtm0_ut_log_simple }, + { "log-mpsc", &m0_dtm0_ut_log_mpsc }, + { "pruner_init-fini", &m0_dtm0_ut_pruner_init_fini }, + { "pruner_start-stop", &m0_dtm0_ut_pruner_start_stop }, + { "pruner_one", &m0_dtm0_ut_pruner_one }, + { "pruner_two", &m0_dtm0_ut_pruner_two }, + { "pruner_many_left", &m0_dtm0_ut_pruner_many_left }, + { "pruner_mpsc", &m0_dtm0_ut_pruner_mpsc }, + { "pruner_mpsc_many", &m0_dtm0_ut_pruner_mpsc_many }, + { "domain_full_init-fini", &m0_dtm0_ut_persistent_domain_init_fini }, + { "remach-init-fini", remach_init_fini }, + { "remach-start-stop", remach_start_stop }, + { "remach-boot-cluster-ss", remach_boot_cluster_ss }, + { "remach-boot-cluster-cs", remach_boot_cluster_cs }, + { "remach-reboot-server", remach_reboot_server }, + { "remach-reboot-twice", remach_reboot_twice }, + { "remach-boot-real-log", remach_boot_real_log }, + { "remach-real-log-replay", remach_real_log_replay }, { "remach-rec-mark-empty", remach_rec_mark_empty }, { "remach-rec-mark-nonempty", remach_rec_mark_nonempty }, { "remach-client-eviction-empty-log", diff --git a/dtm0/ut/pruner.c b/dtm0/ut/pruner.c index 7690d612259..b80dbf0927b 100644 --- a/dtm0/ut/pruner.c +++ b/dtm0/ut/pruner.c @@ -29,7 +29,249 @@ #include "lib/trace.h" #include "dtm0/pruner.h" +#include "ut/ut.h" +#include "lib/memory.h" /* M0_ALLOC_PTR */ +#include "dtm0/domain.h" /* m0_dtm0_domain_cfg */ +#include "dtm0/cfg_default.h" /* m0_dtm0_domain_cfg_default_dup */ +#include "reqh/reqh.h" /* M0_REQH_INIT */ +#include "dtm0/ut/helper.h" /* dtm0_ut_log_ctx */ +#include "dtm0/dtm0.h" /* m0_dtm0_redo */ +void m0_dtm0_ut_reqh_init(struct m0_reqh **preqh, struct m0_be_seg *be_seg) +{ + struct m0_reqh *reqh; + int rc; + + M0_ALLOC_PTR(reqh); + M0_UT_ASSERT(reqh != NULL); + + rc = M0_REQH_INIT(reqh, + .rhia_dtm = (void*)1, + .rhia_mdstore = (void*)1, + .rhia_db = be_seg, + .rhia_fid = &g_process_fid); + M0_UT_ASSERT(rc == 0); + m0_reqh_start(reqh); + + *preqh = reqh; +} + +void m0_dtm0_ut_reqh_fini(struct m0_reqh **preqh) +{ + struct m0_reqh *reqh = *preqh; + + m0_reqh_services_terminate(reqh); + reqh->rh_beseg = NULL; + m0_reqh_fini(reqh); + m0_free(reqh); + *preqh = NULL; +} + +void m0_dtm0_ut_cfs_init(struct m0_co_fom_service **pcfs, + struct m0_reqh *reqh) +{ + struct m0_co_fom_service *cfs; + int rc; + M0_ALLOC_PTR(cfs); + M0_UT_ASSERT(cfs != NULL); + rc = m0_co_fom_service_init(cfs, reqh); + M0_UT_ASSERT(rc == 0); + *pcfs = cfs; +} + +void m0_dtm0_ut_cfs_fini(struct m0_co_fom_service **pcfs) +{ + struct m0_co_fom_service *cfs = *pcfs; + m0_co_fom_service_fini(cfs); + m0_free(cfs); + *pcfs = NULL; +} + +struct dtm0_ut_pruner_ctx { + struct m0_dtm0_pruner *pruner; + struct m0_reqh *reqh; + struct m0_co_fom_service *cfs; + struct dtm0_ut_log_ctx *lctx; +}; + +static void dtm0_ut_pruner_init(struct dtm0_ut_pruner_ctx *pctx) +{ + struct m0_dtm0_domain_cfg *cfg; + int rc; + + pctx->lctx = dtm0_ut_log_init(); + cfg = &pctx->lctx->dod_cfg; + rc = m0_dtm0_log_open(&pctx->lctx->dol, &cfg->dodc_log); + M0_UT_ASSERT(rc == 0); + + m0_dtm0_ut_reqh_init(&pctx->reqh, cfg->dodc_log.dlc_seg); + m0_dtm0_ut_cfs_init(&pctx->cfs, pctx->reqh); + + M0_ALLOC_PTR(pctx->pruner); + M0_UT_ASSERT(pctx->pruner != NULL); + + cfg->dodc_pruner.dpc_cfs = pctx->cfs; + cfg->dodc_pruner.dpc_dol = &pctx->lctx->dol; + + rc = m0_dtm0_pruner_init(pctx->pruner, &cfg->dodc_pruner); + M0_UT_ASSERT(rc == 0); +} + +static void dtm0_ut_pruner_start(struct dtm0_ut_pruner_ctx *ctx) +{ + m0_dtm0_pruner_start(ctx->pruner); +} + +static void dtm0_ut_pruner_stop(struct dtm0_ut_pruner_ctx *pctx) +{ + m0_dtm0_pruner_stop(pctx->pruner); +} + +static void dtm0_ut_pruner_fini(struct dtm0_ut_pruner_ctx *pctx) +{ + m0_dtm0_pruner_fini(pctx->pruner); + m0_free(pctx->pruner); + pctx->pruner = NULL; + + m0_dtm0_ut_cfs_fini(&pctx->cfs); + m0_dtm0_ut_reqh_fini(&pctx->reqh); + + m0_dtm0_log_close(&pctx->lctx->dol); + dtm0_ut_log_fini(pctx->lctx); +} + +void m0_dtm0_ut_pruner_init_fini(void) +{ + struct dtm0_ut_pruner_ctx ctx = {}; + dtm0_ut_pruner_init(&ctx); + dtm0_ut_pruner_fini(&ctx); +} + +void m0_dtm0_ut_pruner_start_stop(void) +{ + struct dtm0_ut_pruner_ctx ctx = {}; + + dtm0_ut_pruner_init(&ctx); + dtm0_ut_pruner_start(&ctx); + m0_dtm0_log_end(&ctx.lctx->dol); + dtm0_ut_pruner_stop(&ctx); + dtm0_ut_pruner_fini(&ctx); +} + +enum { + DTM0_UT_REDO_TS_BASE = 1, +}; + +static void dtm0_ut_log_redo_add(struct dtm0_ut_log_ctx *lctx, int timestamp) +{ + struct m0_dtm0_redo *redo; + int rc; + struct m0_fid *p_sdev_fid; + + + redo = dtm0_ut_redo_get(DTM0_UT_REDO_TS_BASE); + M0_UT_ASSERT(redo != NULL); + p_sdev_fid = &redo->dtr_descriptor.dtd_id.dti_originator_sdev_fid; + redo->dtr_descriptor.dtd_id.dti_timestamp = timestamp; + M0_BE_UT_TRANSACT(&lctx->ut_be, tx, cred, + m0_dtm0_log_redo_add_credit(&lctx->dol, + redo, + &cred), + rc = m0_dtm0_log_redo_add(&lctx->dol, tx, redo, + p_sdev_fid)); + M0_UT_ASSERT(rc == 0); + dtm0_ut_redo_put(redo); +} + +void m0_dtm0_ut_pruner_one(void) +{ + struct dtm0_ut_pruner_ctx ctx = {}; + + dtm0_ut_pruner_init(&ctx); + dtm0_ut_pruner_start(&ctx); + dtm0_ut_log_redo_add(ctx.lctx, DTM0_UT_REDO_TS_BASE); + m0_dtm0_log_end(&ctx.lctx->dol); + dtm0_ut_pruner_stop(&ctx); + M0_UT_ASSERT(m0_dtm0_log_is_empty(&ctx.lctx->dol)); + dtm0_ut_pruner_fini(&ctx); +} + +void m0_dtm0_ut_pruner_two(void) +{ + struct dtm0_ut_pruner_ctx ctx = {}; + + dtm0_ut_pruner_init(&ctx); + dtm0_ut_pruner_start(&ctx); + dtm0_ut_log_redo_add(ctx.lctx, DTM0_UT_REDO_TS_BASE); + dtm0_ut_log_redo_add(ctx.lctx, DTM0_UT_REDO_TS_BASE + 1); + m0_dtm0_log_end(&ctx.lctx->dol); + dtm0_ut_pruner_stop(&ctx); + M0_UT_ASSERT(m0_dtm0_log_is_empty(&ctx.lctx->dol)); + dtm0_ut_pruner_fini(&ctx); +} + +void m0_dtm0_ut_pruner_many_left(void) +{ + struct dtm0_ut_pruner_ctx ctx = {}; + const int dtx_nr = 0x100; + int i; + + dtm0_ut_pruner_init(&ctx); + for (i = 0; i < dtx_nr; ++i) + dtm0_ut_log_redo_add(ctx.lctx, DTM0_UT_REDO_TS_BASE + i); + dtm0_ut_pruner_start(&ctx); + m0_dtm0_log_end(&ctx.lctx->dol); + dtm0_ut_pruner_stop(&ctx); + M0_UT_ASSERT(!m0_dtm0_log_is_empty(&ctx.lctx->dol)); + dtm0_ut_pruner_fini(&ctx); +} + +void m0_dtm0_ut_pruner_mpsc(void) +{ + struct dtm0_ut_pruner_ctx ctx[1] = {}; + struct dtm0_ut_log_mp_ctx lmp_ctx; + + dtm0_ut_pruner_init(ctx); + dtm0_ut_pruner_start(ctx); + + dtm0_ut_log_mp_init(&lmp_ctx, ctx->lctx); + dtm0_ut_log_mp_run(&lmp_ctx); + m0_be_op_wait(lmp_ctx.op); + dtm0_ut_log_mp_fini(&lmp_ctx); + + M0_UT_ASSERT(!m0_dtm0_log_is_empty(&ctx->lctx->dol)); + m0_nanosleep(M0_MKTIME(0, 100), NULL); + M0_UT_ASSERT(!m0_dtm0_log_is_empty(&ctx->lctx->dol)); + + m0_dtm0_log_end(&ctx->lctx->dol); + dtm0_ut_pruner_stop(ctx); + dtm0_ut_pruner_fini(ctx); +} + +void m0_dtm0_ut_pruner_mpsc_many(void) +{ + struct dtm0_ut_pruner_ctx ctx[1] = {}; + struct dtm0_ut_log_mp_ctx lmp_ctx; + int run; + const int nr_runs = 4; + + dtm0_ut_pruner_init(ctx); + dtm0_ut_pruner_start(ctx); + + for (run = 0; run < nr_runs; ++run) { + dtm0_ut_log_mp_init(&lmp_ctx, ctx->lctx); + dtm0_ut_log_mp_run(&lmp_ctx); + m0_be_op_wait(lmp_ctx.op); + dtm0_ut_log_mp_fini(&lmp_ctx); + + while (!m0_dtm0_log_is_empty(&ctx->lctx->dol)) + m0_nanosleep(M0_MKTIME(0, 100), NULL); + } + + m0_dtm0_log_end(&ctx->lctx->dol); + dtm0_ut_pruner_stop(ctx); + dtm0_ut_pruner_fini(ctx); +} #undef M0_TRACE_SUBSYSTEM /** @} end of dtm0 group */ diff --git a/fop/fom.c b/fop/fom.c index 5a0aabebd0b..0cda5f0ce1d 100644 --- a/fop/fom.c +++ b/fop/fom.c @@ -353,6 +353,7 @@ static bool hung_fom_notify(const struct m0_fom *fom) M0_DTM0_REDO_OPCODE, M0_DTM0_RLINK_OPCODE, M0_DTM0_RECOVERY_FOM_OPCODE, + M0_DTM0_PRUNER_OPCODE, M0_FDMI_SOURCE_DOCK_OPCODE))) return true; @@ -1400,6 +1401,7 @@ void m0_fom_init(struct m0_fom *fom, const struct m0_fom_type *fom_type, fom->fo_ops = ops; fom->fo_transitions = 0; fom->fo_local = false; + fom->fo_local_update = false; m0_fom_callback_init(&fom->fo_cb); runq_tlink_init(fom); diff --git a/fop/fom.h b/fop/fom.h index 43b17e5fe16..08c9e04f018 100644 --- a/fop/fom.h +++ b/fop/fom.h @@ -501,6 +501,12 @@ struct m0_fom { * e.g., undo or redo during recovery. */ bool fo_local; + + /** + * Set when local fom is used to execute a local operation + * under a be tx. See fom_is_update. + */ + bool fo_local_update; /** Pointer to service instance. */ struct m0_reqh_service *fo_service; /** diff --git a/fop/fom_generic.c b/fop/fom_generic.c index d70dcd591e6..72504c74f86 100644 --- a/fop/fom_generic.c +++ b/fop/fom_generic.c @@ -99,6 +99,13 @@ M0_EXPORTED(m0_rpc_item_generic_reply_rc); static bool fom_is_update(const struct m0_fom *fom) { + /* + * Local FOMs may not have an FOP attached to it. + * Use a separate flag to see if we need to open tx + * for that FOM. + */ + if (fom->fo_local) + return fom->fo_local_update; /* The rest of condition will always work for non-DTM0 services. */ return !m0_dtm0_is_a_volatile_dtm(fom->fo_service) && m0_rpc_item_is_update(m0_fop_to_rpc_item(fom->fo_fop)); diff --git a/lib/buf.h b/lib/buf.h index f1c4670cfc0..3a1da419f2b 100644 --- a/lib/buf.h +++ b/lib/buf.h @@ -43,7 +43,7 @@ struct m0_buf { struct m0_bufs { uint32_t ab_count; struct m0_buf *ab_elems; -} M0_XCA_SEQUENCE M0_XCA_DOMAIN(conf|rpc); +} M0_XCA_SEQUENCE M0_XCA_DOMAIN(conf|rpc|be); /** * Initialisers for struct m0_buf. diff --git a/m0t1fs/linux_kernel/st/common_service_fids_inc.sh b/m0t1fs/linux_kernel/st/common_service_fids_inc.sh index c9c23fe3ae1..2f649323be7 100644 --- a/m0t1fs/linux_kernel/st/common_service_fids_inc.sh +++ b/m0t1fs/linux_kernel/st/common_service_fids_inc.sh @@ -36,3 +36,4 @@ ADDB_CAS_FID_CON='^s|14' # addb for CAS service DIXR_FID_CON='^s|15' # DIX repare DIXB_FID_CON='^s|16' # DIX rebalance FDMI_FID_CON='^s|17' # fdmi service +DTM_FID_CON='^s|18' # dtm service diff --git a/m0t1fs/linux_kernel/st/m0t1fs_common_inc.sh b/m0t1fs/linux_kernel/st/m0t1fs_common_inc.sh index 3793d832410..20b1f42b5e3 100755 --- a/m0t1fs/linux_kernel/st/m0t1fs_common_inc.sh +++ b/m0t1fs/linux_kernel/st/m0t1fs_common_inc.sh @@ -350,7 +350,7 @@ function build_conf() local site_count=1 local PROC_FID_CONT='^r|1' local MD_REDUNDANCY=1 - local m0t1fs_ep="$lnet_nid:12345:33:1" + local m0t1fs_ep="$lnet_nid:12345:34:101" local nr_ios=${#IOSEP[*]} if [ $SINGLE_NODE -eq 1 ] ; then @@ -392,6 +392,7 @@ function build_conf() local CONFD="$CONF_FID_CON:0" local HA_SVC_ID='^s|1:6' local FIS_SVC_ID='^s|1:7' + local DTM_SVC_ID='^s|1:8' local SITEID='^S|1:6' local RACKID='^a|1:6' local ENCLID='^e|1:7' @@ -412,6 +413,7 @@ function build_conf() local MDCTRLVID="^j|2:$(($pool_width + 4))" local M0T1FS_RMID="^s|1:101" + local M0T1FS_DTMID="^s|1:102" local M0T1FS_PROCID="^r|1:100" local NODES="$NODE" @@ -422,7 +424,8 @@ function build_conf() local PROC_OBJS local M0D=0 local M0T1FS_RM="{0x73| (($M0T1FS_RMID), @M0_CST_RMS, [1: \"${m0t1fs_ep}\"], [0], [0])}" - local M0T1FS_PROC="{0x72| (($M0T1FS_PROCID), [1:3], 0, 0, 0, 0, \"${m0t1fs_ep}\", [1: $M0T1FS_RMID])}" + local M0T1FS_DTM="{0x73| (($M0T1FS_DTMID), @M0_CST_DTM0, [1: \"${m0t1fs_ep}\"], [1: "\"origin:in-volatile\""], [0])}" + local M0T1FS_PROC="{0x72| (($M0T1FS_PROCID), [1:3], 0, 0, 0, 0, \"${m0t1fs_ep}\", [2: $M0T1FS_DTMID, $M0T1FS_RMID])}" PROC_OBJS="$PROC_OBJS${PROC_OBJS:+, }\n $M0T1FS_PROC" PROC_NAMES="$PROC_NAMES${PROC_NAMES:+, }$M0T1FS_PROCID" @@ -461,8 +464,10 @@ function build_conf() local SNS_REP_OBJ="{0x73| (($SNS_REP_NAME), @M0_CST_SNS_REP, [1: $iosep], [0], [0])}" local SNS_REB_OBJ="{0x73| (($SNS_REB_NAME), @M0_CST_SNS_REB, [1: $iosep], [0], [0])}" local RM_NAME="$RMS_FID_CON:$M0D" + local DTM_NAME="$DTM_FID_CON:$M0D" local RM_OBJ="{0x73| (($RM_NAME), @M0_CST_RMS, [1: $iosep], [0], [0])}" - local NAMES_NR=5 + local DTM_OBJ="{0x73| (($DTM_NAME), @M0_CST_DTM0, [1: $iosep], [1: "\"origin:in-volatile\""], [0])}" + local NAMES_NR=6 if [ $ENABLE_CAS -eq 1 ] ; then local DIX_REP_NAME="$DIXR_FID_CON:$i" local DIX_REB_NAME="$DIXB_FID_CON:$i" @@ -471,14 +476,14 @@ function build_conf() local DIX_REP_OBJ="{0x73| (($DIX_REP_NAME), @M0_CST_DIX_REP, [1: $iosep], [0], [0])}" local DIX_REB_OBJ="{0x73| (($DIX_REB_NAME), @M0_CST_DIX_REB, [1: $iosep], [0], [0])}" local CAS_OBJS="$CAS_OBJ, \n $DIX_REP_OBJ, \n $DIX_REB_OBJ" - NAMES_NR=8 + NAMES_NR=9 fi PROC_NAME="$PROC_FID_CONT:$M0D" - IOS_NAMES[$i]="$IOS_NAME, $ADDB_NAME, $SNS_REP_NAME, $SNS_REB_NAME, \ + IOS_NAMES[$i]="$IOS_NAME, $ADDB_NAME, $SNS_REP_NAME, $SNS_REB_NAME, $DTM_NAME, \ $RM_NAME${CAS_NAME:+,} $CAS_NAME, $DIX_REP_NAME, $DIX_REB_NAME" PROC_OBJ="{0x72| (($PROC_NAME), [1:3], 0, 0, 0, 0, $iosep, [$NAMES_NR: ${IOS_NAMES[$i]}])}" - IOS_OBJS="$IOS_OBJS${IOS_OBJS:+, }\n $IOS_OBJ, \n $ADDB_OBJ, \ + IOS_OBJS="$IOS_OBJS${IOS_OBJS:+, }\n $IOS_OBJ, \n $ADDB_OBJ, \n $DTM_OBJ, \ \n $SNS_REP_OBJ, \n $SNS_REB_OBJ, \n $RM_OBJ${CAS_OBJS:+, \n} $CAS_OBJS" PROC_OBJS="$PROC_OBJS${PROC_OBJS:+, }\n $PROC_OBJ" # +1 here for process object @@ -493,12 +498,14 @@ function build_conf() local MDS_OBJ="{0x73| (($MDS_NAME), @M0_CST_MDS, [1: $mdsep], [0], [0])}" local ADDB_OBJ="{0x73| (($ADDB_NAME), @M0_CST_ADDB2, [1: $mdsep], [0], [0])}" local RM_NAME="$RMS_FID_CON:$M0D" + local DTM_NAME="$DTM_FID_CON:$M0D" local RM_OBJ="{0x73| (($RM_NAME), @M0_CST_RMS, [1: $mdsep], [0], [0])}" + local DTM_OBJ="{0x73| (($DTM_NAME), @M0_CST_DTM0, [1: $mdsep], [1: "\"origin:in-volatile\""], [0])}" PROC_NAME="$PROC_FID_CONT:$M0D" - MDS_NAMES[$i]="$MDS_NAME, $ADDB_NAME, $RM_NAME" - MDS_OBJS="$MDS_OBJS${MDS_OBJS:+,} \n $MDS_OBJ, \n $ADDB_OBJ, \n $RM_OBJ" - PROC_OBJ="{0x72| (($PROC_NAME), [1:3], 0, 0, 0, 0, $mdsep, [3: ${MDS_NAMES[$i]}])}" + MDS_NAMES[$i]="$MDS_NAME, $ADDB_NAME, $RM_NAME, $DTM_NAME" + MDS_OBJS="$MDS_OBJS${MDS_OBJS:+,} \n $MDS_OBJ, \n $ADDB_OBJ, \n $RM_OBJ, \n $DTM_OBJ" + PROC_OBJ="{0x72| (($PROC_NAME), [1:3], 0, 0, 0, 0, $mdsep, [4: ${MDS_NAMES[$i]}])}" PROC_OBJS="$PROC_OBJS, \n $PROC_OBJ" PROC_NAMES="$PROC_NAMES, $PROC_NAME" done @@ -549,16 +556,22 @@ function build_conf() RM_NAME="$RMS_FID_CON:$M0D" RM_OBJ="{0x73| (($RM_NAME), @M0_CST_RMS, [1: $HA_ENDPOINT], [0], [0])}" RM_OBJS="$RM_OBJS${RM_OBJS:+,} \n $RM_OBJ" + DTM_NAME="$DTM_FID_CON:$M0D" + DTM_OBJ="{0x73| (($DTM_NAME), @M0_CST_DTM0, [1: $HA_ENDPOINT], [1: "\"origin:in-volatile\""], [0])}" + DTM_OBJS="$DTM_OBJS${DTM_OBJS:+,} \n $DTM_OBJ" PROC_OBJ="{0x72| (($PROC_NAME), [1:3], 0, 0, 0, 0, "${HA_ENDPOINT}", - [3: $HA_SVC_ID, $FIS_SVC_ID, $RM_NAME])}" + [4: $HA_SVC_ID, $FIS_SVC_ID, $DTM_NAME, $RM_NAME])}" PROC_OBJS="$PROC_OBJS, \n $PROC_OBJ" PROC_NAMES="$PROC_NAMES, $PROC_NAME" PROC_NAME="$PROC_FID_CONT:$((M0D++))" RM_NAME="$RMS_FID_CON:$M0D" RM_OBJ="{0x73| (($RM_NAME), @M0_CST_RMS, [1: $CONFD_ENDPOINT], [0], [0])}" RM_OBJS="$RM_OBJS${RM_OBJS:+,} \n $RM_OBJ" + DTM_NAME="$DTM_FID_CON:$M0D" + DTM_OBJ="{0x73| (($DTM_NAME), @M0_CST_DTM0, [1: $CONFD_ENDPOINT], [1: "\"origin:in-volatile\""], [0])}" + DTM_OBJS="$DTM_OBJS${DTM_OBJS:+,} \n $DTM_OBJ" PROC_OBJ="{0x72| (($PROC_NAME), [1:3], 0, 0, 0, 0, "${CONFD_ENDPOINT}", - [2: $CONFD, $RM_NAME])}" + [3: $CONFD, $DTM_NAME, $RM_NAME])}" PROC_OBJS="$PROC_OBJS, \n $PROC_OBJ" PROC_NAMES="$PROC_NAMES, $PROC_NAME" local SITE="{0x53| (($SITEID), [1: $RACKID], [$pvers_count: $PVER_IDS])}" @@ -655,7 +668,7 @@ function build_conf() # Here "15" configuration objects includes services excluding ios & mds, # pools, racks, enclosures, controllers and their versioned objects. echo -e " -[$(($IOS_OBJS_NR + $((${#mdservices[*]} * 5)) + $NR_IOS_DEVS + 19 +[$(($IOS_OBJS_NR + $((${#mdservices[*]} * 6)) + $NR_IOS_DEVS + 19 + 3 + $MD_OBJ_COUNT + $PVER1_OBJ_COUNT + 5 + $DIX_PVER_OBJ_COUNT + $FDMI_ITEMS_NR)): {0x74| (($ROOT), 1, (11, 22), $MDPOOLID, $IMETA_PVER, $MD_REDUNDANCY, [1: \"$pool_width $nr_data_units $nr_parity_units $nr_spare_units\"], @@ -670,12 +683,14 @@ function build_conf() {0x73| (($HA_SVC_ID), @M0_CST_HA, [1: $HA_ENDPOINT], [0], [0])}, {0x73| (($FIS_SVC_ID), @M0_CST_FIS, [1: $HA_ENDPOINT], [0], [0])}, $M0T1FS_RM, + $M0T1FS_DTM, $FDMI_GROUP $FDMI_FILTER $FDMI_FILTER2 $MDS_OBJS, $IOS_OBJS, $RM_OBJS, + $DTM_OBJS, $IOS_DEVS, $SITE, $RACK, diff --git a/m0t1fs/linux_kernel/st/m0t1fs_server_inc.sh b/m0t1fs/linux_kernel/st/m0t1fs_server_inc.sh index bf65bb31f58..92853aed058 100644 --- a/m0t1fs/linux_kernel/st/m0t1fs_server_inc.sh +++ b/m0t1fs/linux_kernel/st/m0t1fs_server_inc.sh @@ -171,7 +171,7 @@ servers_stop() # shutdown services. mds should be stopped last, because # other ioservices may have connections to mdservice. - local pids=$(pgrep "$prog") + local pids=$(pgrep "$prog" | sort -r) echo === pids of services: $pids === echo "Shutting down services one by one. mdservice is the last." local delay=5 @@ -337,7 +337,7 @@ EOF DIR=$MOTR_M0T1FS_TEST_DIR/ha rm -rf $DIR mkdir -p $DIR - opts="$common_opts -T ad -e $XPRT:${lnet_nid}:${HA_EP%:*:*}:$MKFS_PORTAL:1 \ + opts="$common_opts -T linux -e $XPRT:${lnet_nid}:${HA_EP%:*:*}:$MKFS_PORTAL:1 \ -c $CONFDB" cmd="cd $DIR && exec $prog_mkfs -F $opts |& tee -a m0mkfs.log" echo $cmd @@ -391,7 +391,7 @@ EOF # spawn ha agent proc_fid="'<"$PROC_FID_CNTR:$ha_key">'" - opts="$common_opts -T ad -e $XPRT:${lnet_nid}:$HA_EP \ + opts="$common_opts -T linux -e $XPRT:${lnet_nid}:$HA_EP \ -c $CONFDB -f $proc_fid ${FI_OPT:-} -H ${lnet_nid}:$HA_EP" DIR=$MOTR_M0T1FS_TEST_DIR/ha cmd="cd $DIR && exec $prog_start $opts |& tee -a m0d.log" diff --git a/motr/client_init.c b/motr/client_init.c index 851244491fd..827484fb7f6 100644 --- a/motr/client_init.c +++ b/motr/client_init.c @@ -40,6 +40,7 @@ #include "net/lnet/lnet_core_types.h" /* M0_NET_LNET_NIDSTR_SIZE */ #include "dtm0/service.h" /* m0_dtm0_service_find */ #include "dtm0/helper.h" /* m0_dtm_client_service_start */ +#include "dtm0/cfg_default.h" /* m0_dtm0_domain_cfg_default_dup */ #include "motr/io.h" /* io_sm_conf */ #include "motr/client.h" @@ -1436,8 +1437,9 @@ static int initlift_addb2(struct m0_sm *mach) static int initlift_dtm0(struct m0_sm *mach) { - int rc = 0; - struct m0_client *m0c; + int rc = 0; + struct m0_client *m0c; + struct m0_dtm0_domain_cfg cfg; M0_ENTRY(); M0_PRE(mach != NULL); @@ -1446,7 +1448,11 @@ static int initlift_dtm0(struct m0_sm *mach) M0_ASSERT(m0c_invariant(m0c)); if (m0c->m0c_initlift_direction == STARTUP) { - rc = m0_dtm0_domain_init(&m0c->m0c_dtm0_domain, NULL); + rc = m0_dtm0_domain_cfg_default_dup(&cfg, false); + if (rc != 0) + return M0_RC(rc); + cfg.dod_reqh = &m0c->m0c_reqh; + rc = m0_dtm0_domain_init(&m0c->m0c_dtm0_domain, &cfg); if (rc != 0) { initlift_fail(rc, m0c); return M0_RC(initlift_get_next_floor(m0c)); diff --git a/motr/init.c b/motr/init.c index d9a3b8e3a73..a52122d6228 100644 --- a/motr/init.c +++ b/motr/init.c @@ -47,6 +47,7 @@ #include "fol/fol.h" #include "dtm/dtm.h" #include "dtm0/service.h" +#include "dtm0/dtm0.h" #include "reqh/reqh.h" #include "lib/timer.h" #include "fid/fid.h" @@ -244,6 +245,8 @@ struct init_fini_call subsystem[] = { { &m0_fdmi_init, &m0_fdmi_fini, "fdmi" }, { &m0_fol_fdmi_src_init, &m0_fol_fdmi_src_fini, "fol_fdmi_source" }, { &m0_dtm0_stype_init, &m0_dtm0_stype_fini, "dtm0"}, + { &m0_cfs_register, &m0_cfs_unregister, "co_fom"}, + { &m0_dtm0_mod_init, &m0_dtm0_mod_fini, "dtm0_mod"}, #endif }; diff --git a/motr/setup.c b/motr/setup.c index 733d2445025..0c18a60e989 100644 --- a/motr/setup.c +++ b/motr/setup.c @@ -66,7 +66,8 @@ #include "ioservice/io_service.h" /* m0_ios_net_buffer_pool_size_set */ #include "stob/linux.h" #include "conf/ha.h" /* m0_conf_ha_process_event_post */ -#include "dtm0/helper.h" /* m0_dtm0_log_create */ +#include "dtm0/helper.h" /* m0_dtm0_old_log_create */ +#include "dtm0/cfg_default.h" /* m0_dtm0_domain_cfg_default_dup */ /** @addtogroup m0d @@ -1332,7 +1333,7 @@ static int cs_storage_prepare(struct m0_reqh_context *rctx, bool erase) rc = rc ?: m0_mdstore_create(&rctx->rc_mdstore, grp, &rctx->rc_cdom_id, bedom, rctx->rc_beseg) - ?: m0_dtm0_log_create(grp, bedom, rctx->rc_beseg); + ?: m0_dtm0_old_log_create(grp, bedom, rctx->rc_beseg); if (rc != 0) goto end; dom = rctx->rc_mdstore.md_dom; @@ -1695,9 +1696,16 @@ static int cs_storage_setup(struct m0_motr *cctx) return M0_ERR(rc); } -static int cs_dtm0_init(struct m0_reqh_context *rctx) +static int cs_dtm0_init(struct m0_reqh_context *rctx, bool mkfs) { - return m0_dtm0_domain_init(&rctx->rc_dtm0_domain, NULL); + struct m0_dtm0_domain_cfg cfg; + int rc; + + rc = m0_dtm0_domain_cfg_default_dup(&cfg, mkfs); + if (rc != 0) + return rc; + cfg.dod_reqh = &rctx->rc_reqh; + return m0_dtm0_domain_init(&rctx->rc_dtm0_domain, &cfg); } static void cs_dtm0_fini(struct m0_reqh_context *rctx) @@ -2671,7 +2679,7 @@ static int cs_level_enter(struct m0_module *module) case CS_LEVEL_STORAGE_SETUP: return M0_RC(cs_storage_setup(cctx)); case CS_LEVEL_DTM0_INIT: - return M0_RC(cs_dtm0_init(rctx)); + return M0_RC(cs_dtm0_init(rctx, cctx->cc_mkfs)); case CS_LEVEL_RWLOCK_UNLOCK: m0_rwlock_write_unlock(&cctx->cc_rwlock); return M0_RC(0); diff --git a/motr/st/utils/motr_local_conf.sh b/motr/st/utils/motr_local_conf.sh index 87ee6907347..4f94b2003db 100644 --- a/motr/st/utils/motr_local_conf.sh +++ b/motr/st/utils/motr_local_conf.sh @@ -50,5 +50,5 @@ if [ X"$MOTR_PROF_OPT" == X ]; then fi if [ X"$MOTR_PROC_FID" == X ]; then - MOTR_PROC_FID=0x7200000000000000:0 + MOTR_PROC_FID=0x7200000000000001:64 fi diff --git a/rpc/rpc_opcodes.h b/rpc/rpc_opcodes.h index cb0ba05b60f..ceae54ead8d 100644 --- a/rpc/rpc_opcodes.h +++ b/rpc/rpc_opcodes.h @@ -359,6 +359,7 @@ enum M0_RPC_OPCODES { M0_DTM0_RLINK_OPCODE = 1073, M0_FDMI_SOURCE_DOCK_TIMER_OPCODE = 1074, M0_DTM0_RECOVERY_FOM_OPCODE = 1075, + M0_DTM0_PRUNER_OPCODE = 1076, M0_OPCODES_NR = 2048 } M0_XCA_ENUM; diff --git a/scripts/install/opt/seagate/cortx/motr/sanity/motr_sanity.sh b/scripts/install/opt/seagate/cortx/motr/sanity/motr_sanity.sh index 674a2759f4c..56ff3980619 100755 --- a/scripts/install/opt/seagate/cortx/motr/sanity/motr_sanity.sh +++ b/scripts/install/opt/seagate/cortx/motr/sanity/motr_sanity.sh @@ -207,7 +207,7 @@ generate_endpoints() ios1_fid='0x7200000000000001:3' echo "Ioservice 1 FID: $ios1_fid" - process_fid='0x7200000000000000:0' + process_fid='0x7200000000000001:5' echo "Process FID: $process_fid" } diff --git a/scripts/install/usr/libexec/cortx-motr/motr-cleanup b/scripts/install/usr/libexec/cortx-motr/motr-cleanup index 3f29ed4b16f..641f4366730 100755 --- a/scripts/install/usr/libexec/cortx-motr/motr-cleanup +++ b/scripts/install/usr/libexec/cortx-motr/motr-cleanup @@ -39,7 +39,7 @@ source $service_funcs motr_services="" # motr-cleanup is excluded from the list -for s in m0d@* m0t1fs@* motr-client motr-kernel motr-mkfs motr-mkfs@* motr-server-confd motr-server-ha motr-server@* motr motr-singlenode motr-trace@*; do +for s in m0d@* m0t1fs@* motr-client motr-kernel motr-mkfs motr-mkfs@* motr-server@* motr-server-confd motr-server-ha motr motr-singlenode motr-trace@*; do motr_services="$motr_services $s.service" done @@ -58,6 +58,10 @@ for s in $(systemctl list-units --failed $motr_services | awk '/failed/ {if ($4 systemctl reset-failed $s done +systemctl stop *mkfs.slice +systemctl stop *trace.slice +systemctl stop *server.slice + m0_log 'Done.' data_dir=${MOTR_M0D_DATA_DIR:-$MOTR_M0D_DEFAULT_DATA_DIR} diff --git a/scripts/install/usr/libexec/cortx-motr/motr-service.functions b/scripts/install/usr/libexec/cortx-motr/motr-service.functions index 0c42a275090..a81cf95fa4f 100644 --- a/scripts/install/usr/libexec/cortx-motr/motr-service.functions +++ b/scripts/install/usr/libexec/cortx-motr/motr-service.functions @@ -13,7 +13,6 @@ set -e set -E - sysconfig_dir='/etc/sysconfig' [ -d /etc/sysconfig ] || sysconfig_dir='/etc/default' @@ -376,6 +375,7 @@ m0_get_ep_of() local ios_tid=401 local cas_tid=501 local fdmi_tid=601 + local client_tid=601 local portal=$(_portal_id $1) local ep_str="$lnid:$lpid:$portal:" else @@ -386,6 +386,7 @@ m0_get_ep_of() local ios_tid=3401 local cas_tid=3501 local fdmi_tid=3601 + local client_tid=3701 local ep_str="$lnid@" fi @@ -416,6 +417,11 @@ m0_get_ep_of() fi done + if [[ $service == "client" ]] ; then + ep=${MOTR_HA_EP:-$ep_str$((client_tid++))} + found=true + fi + if ! $found ; then m0_echo_err "cannot find endpoint of '$service' on the" \ " ${node:+current} node" @@ -675,6 +681,9 @@ m0_build_service_fid() dix-rebalance) cid=12 ;; fdmi) cid=13 ;; iscs) cid=14 ;; + dtm) cid=15 ;; + dtmc) cid=16 ;; + rmsc) cid=17 ;; esac printf %s:%d "^s|$(( $cid + $node_idx * 16 ))" $sidx @@ -797,6 +806,9 @@ _build_process_names() names="$names${names:+, }$name" ((++key)) || true done + name="$proc_fid_con:$key" + names="$names${names:+, }$name" + ((++key)) || true done echo -e "$names" @@ -812,6 +824,7 @@ _build_process_objects() local rms_nr=$(m0_get_global_services rms | wc -w) for node in $(m0_get_nodes) ; do + local epc=$(m0_get_ep_of client $node) for s in $(m0_get_services $node) ; do local ep=$(m0_get_ep_of $s $node) local sname="$(m0_build_service_fid $s $node_idx)" @@ -819,6 +832,7 @@ _build_process_objects() local rms="$(m0_build_service_fid "rms$idx" $node_idx)" local fdmi="$(m0_build_service_fid "fdmi$idx" $node_idx)" local iscs="$(m0_build_service_fid "iscs$idx" $node_idx)" + local dtm="$(m0_build_service_fid "dtm$idx" $node_idx)" proc_name="$proc_fid_con:$idx" @@ -839,14 +853,20 @@ _build_process_objects() repair="$(m0_build_service_fid ${s/$svc_name/${prefix}-repair} $node_idx)" rebalance="$(m0_build_service_fid ${s/$svc_name/${prefix}-rebalance} $node_idx)" - obj="{0x72| (($proc_name), [1:3], 0, 0, 0, 0, "\"$ep\"", [7: $sname, $repair, $rebalance, $rms, $addb, $fdmi, $iscs])}" + obj="{0x72| (($proc_name), [1:3], 0, 0, 0, 0, "\"$ep\"", [8: $sname, $repair, $rebalance, $rms, $addb, $fdmi, $iscs, $dtm])}" else - obj="{0x72| (($proc_name), [1:3], 0, 0, 0, 0, "\"$ep\"", [5: $sname, $rms, $addb, $fdmi, $iscs])}" + obj="{0x72| (($proc_name), [1:3], 0, 0, 0, 0, "\"$ep\"", [6: $sname, $rms, $addb, $fdmi, $iscs, $dtm])}" fi objs="$objs${objs:+, \n} $obj" ((idx++)) || true done + local dtmc="$(m0_build_service_fid "dtmc$idx" $node_idx)" + local rmsc="$(m0_build_service_fid "rmsc$idx" $node_idx)" + proc_name="$proc_fid_con:$idx" + obj="{0x72| (($proc_name), [1:3], 0, 0, 0, 0, "\"$epc\"", [2: $rmsc, $dtmc])}" + objs="$objs${objs:+, \n} $obj" + ((idx++)) || true ((node_idx++)) || true done @@ -873,6 +893,7 @@ _type_id() dix-repair) type_id=@M0_CST_DIX_REP ;; dix-rebalance) type_id=@M0_CST_DIX_REB ;; iscs) type_id=@M0_CST_ISCS ;; + dtm) type_id=@M0_CST_DTM0 ;; # cas*) type_id=11 ;; # dix-repair) type_id=12 ;; @@ -892,6 +913,7 @@ _build_service_objects() local rms_nr=$(m0_get_global_services rms | wc -w) for node in $(m0_get_nodes) ; do + local lep=$(m0_get_ep_of "client" $node) for s in $(m0_get_services $node) ; do local ep=$(m0_get_ep_of $s $node) local type_id=$(_type_id $s) @@ -932,6 +954,10 @@ _build_service_objects() type_id=$(_type_id "iscs") objs="$objs${objs:+,\n }{0x73| (($iscs), $type_id, [1: "\"$ep\""], [0], [0])}" + dtm="$(m0_build_service_fid "dtm$idx" $node_idx)" + type_id=$(_type_id "dtm") + objs="$objs${objs:+,\n }{0x73| (($dtm), $type_id, [1: "\"$ep\""], [1: "\"origin:in-volatile\""], [0])}" + if [[ $s == ios* || $s == cas* ]] ; then local repair local rebalance @@ -957,6 +983,13 @@ _build_service_objects() fi ((idx++)) || true done + dtmc="$(m0_build_service_fid "dtmc$idx" $node_idx)" + type_id=$(_type_id "dtm") + objs="$objs${objs:+,\n }{0x73| (($dtmc), $type_id, [1: "\"$lep\""], [1: "\"origin:in-volatile\""], [0])}" + rmsc="$(m0_build_service_fid "rmsc$idx" $node_idx)" + type_id=$(_type_id "rms") + objs="$objs${objs:+,\n }{0x73| (($rmsc), $type_id, [1: "\"$lep\""], [0], [0])}" + ((idx++)) || true ((node_idx++)) || true done @@ -1052,12 +1085,13 @@ m0_build_local_conf() local proc_names=$(_build_process_names $proc_fid_con) local proc_objs=$(_build_process_objects $proc_fid_con) local procs=$(m0_get_global_services | wc -w) + local client_procs=$(m0_get_nodes | wc -w) local service_objs=$(_build_service_objects) local services=$(echo "$service_objs" | wc -l) local cas_objs_nr m0_cas_enabled && cas_objs_nr=$(($ipool_width * 3 + 6)) || cas_objs_nr=0 local local_conf=$(cat <