From be46f9589b48bbc1f49b72db4aad9487b6624a18 Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Thu, 29 Aug 2024 13:32:37 -0600 Subject: [PATCH 1/4] DAOS-16445 client: Try cycling different method We've noticed that with 8GB files we fill space pretty quickly. Just trying another method to see if it works any better. Basic idea is two fold: 1. Use a large prime number as the increment as we cycle through the oids. 2. Randomize which one we start with on a per mount basis. Required-githooks: true Signed-off-by: Jeff Olivier --- src/client/dfs/dfs_internal.h | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/client/dfs/dfs_internal.h b/src/client/dfs/dfs_internal.h index 99f2fb8cde2..42ba1d860bf 100644 --- a/src/client/dfs/dfs_internal.h +++ b/src/client/dfs/dfs_internal.h @@ -102,6 +102,18 @@ /** MAX value for the HI OID */ #define MAX_OID_HI ((1UL << 32) - 1) +/** Use a large prime for cycling through all oids */ +#define OID_INC (999999937) + +static inline void +oid_inc(daos_obj_id_t *oid) +{ + oid->hi = (oid->hi + OID_INC) & MAX_OID_HI; + /** Skip reserved entries */ + if (oid->lo == RESERVED_LO && oid->hi <= 1) + oid->hi += OID_INC; +} + typedef uint64_t dfs_magic_t; typedef uint16_t dfs_sb_ver_t; typedef uint16_t dfs_layout_ver_t; @@ -164,6 +176,8 @@ struct dfs { daos_handle_t coh; /** refcount on cont handle that through the DFS API */ uint32_t coh_refcount; + /** The last oid.hi in the sequence */ + uint32_t last_hi; /** Transaction handle epoch. DAOS_EPOCH_MAX for DAOS_TX_NONE */ daos_epoch_t th_epoch; /** Transaction handle */ @@ -343,7 +357,7 @@ oid_gen(dfs_t *dfs, daos_oclass_id_t oclass, bool file, daos_obj_id_t *oid) D_MUTEX_LOCK(&dfs->lock); /** If we ran out of local OIDs, alloc one from the container */ - if (dfs->oid.hi >= MAX_OID_HI) { + if (dfs->oid.hi == dfs->last_hi) { /** Allocate an OID for the namespace */ rc = daos_cont_alloc_oids(dfs->coh, 1, &dfs->oid.lo, NULL); if (rc) { @@ -351,12 +365,14 @@ oid_gen(dfs_t *dfs, daos_oclass_id_t oclass, bool file, daos_obj_id_t *oid) D_MUTEX_UNLOCK(&dfs->lock); return daos_der2errno(rc); } - dfs->oid.hi = 0; + /** Start such that dfs->last_hi will be final value */ + dfs->oid.hi = dfs->last_hi; } /** set oid and lo, bump the current hi value */ oid->lo = dfs->oid.lo; - oid->hi = dfs->oid.hi++; + oid_inc(&dfs->oid); + oid->hi = dfs->oid.hi; D_MUTEX_UNLOCK(&dfs->lock); /** if a regular file, use UINT64 typed dkeys for the array object */ From 79e8165a5bb5423f96a08ce15dff3e5a1f473026 Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Fri, 23 Aug 2024 22:04:44 -0600 Subject: [PATCH 2/4] DAOS-16445 client: Try cycling different method We've noticed that with 8GB files we fill space pretty quickly. Just trying another method to see if it works any better. Basic idea is two fold: 1. Use a large prime number as the increment as we cycle through the oids. 2. Randomize which one we start with on a per mount basis. Required-githooks: true Signed-off-by: Jeff Olivier --- src/client/dfs/mnt.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/client/dfs/mnt.c b/src/client/dfs/mnt.c index a73fafe34df..dc955200ab2 100644 --- a/src/client/dfs/mnt.c +++ b/src/client/dfs/mnt.c @@ -685,6 +685,11 @@ dfs_mount(daos_handle_t poh, daos_handle_t coh, int flags, dfs_t **_dfs) /** if RW, allocate an OID for the namespace */ if (amode == O_RDWR) { + dfs->last_hi = (unsigned int)d_rand(); + /** Avoid potential conflict with SB or ROOT */ + if (dfs->last_hi <= 1) + dfs->last_hi += OID_INC; + rc = daos_cont_alloc_oids(coh, 1, &dfs->oid.lo, NULL); if (rc) { D_ERROR("daos_cont_alloc_oids() Failed, " DF_RC "\n", DP_RC(rc)); @@ -695,10 +700,9 @@ dfs_mount(daos_handle_t poh, daos_handle_t coh, int flags, dfs_t **_dfs) * if this is the first time we allocate on this container, * account 0 for SB, 1 for root obj. */ - if (dfs->oid.lo == RESERVED_LO) - dfs->oid.hi = ROOT_HI + 1; - else - dfs->oid.hi = 0; + dfs->oid.hi = dfs->last_hi; + /** Increment so that dfs->last_hi is the last value */ + oid_inc(&dfs->oid); } dfs->mounted = DFS_MOUNT; @@ -1023,7 +1027,7 @@ dfs_global2local(daos_handle_t poh, daos_handle_t coh, int flags, d_iov_t glob, /** allocate a new oid on the next file or dir creation */ dfs->oid.lo = 0; - dfs->oid.hi = MAX_OID_HI; + dfs->oid.hi = dfs->last_hi; rc = D_MUTEX_INIT(&dfs->lock, NULL); if (rc != 0) { From 1b5a38c4be5cf4e16e580ebdce67a8c2c050218a Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Thu, 29 Aug 2024 12:46:03 -0600 Subject: [PATCH 3/4] Add a function users can use to cycle oids Required-githooks: true Signed-off-by: Jeff Olivier --- src/client/dfs/dfs_internal.h | 19 +++---------------- src/client/dfs/mnt.c | 8 ++------ src/include/daos_obj.h | 17 ++++++++++++++++- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/client/dfs/dfs_internal.h b/src/client/dfs/dfs_internal.h index 42ba1d860bf..7425fc2f00d 100644 --- a/src/client/dfs/dfs_internal.h +++ b/src/client/dfs/dfs_internal.h @@ -99,21 +99,6 @@ /** Max recursion depth for symlinks */ #define DFS_MAX_RECURSION 40 -/** MAX value for the HI OID */ -#define MAX_OID_HI ((1UL << 32) - 1) - -/** Use a large prime for cycling through all oids */ -#define OID_INC (999999937) - -static inline void -oid_inc(daos_obj_id_t *oid) -{ - oid->hi = (oid->hi + OID_INC) & MAX_OID_HI; - /** Skip reserved entries */ - if (oid->lo == RESERVED_LO && oid->hi <= 1) - oid->hi += OID_INC; -} - typedef uint64_t dfs_magic_t; typedef uint16_t dfs_sb_ver_t; typedef uint16_t dfs_layout_ver_t; @@ -371,7 +356,9 @@ oid_gen(dfs_t *dfs, daos_oclass_id_t oclass, bool file, daos_obj_id_t *oid) /** set oid and lo, bump the current hi value */ oid->lo = dfs->oid.lo; - oid_inc(&dfs->oid); + daos_obj_oid_cycle(&dfs->oid); + if (unlikely(dfs->oid.lo == RESERVED_LO && dfs->oid.hi <= 1)) + daos_obj_oid_cycle(&dfs->oid); /* Avoid reserved oids */ oid->hi = dfs->oid.hi; D_MUTEX_UNLOCK(&dfs->lock); diff --git a/src/client/dfs/mnt.c b/src/client/dfs/mnt.c index dc955200ab2..d270be1e414 100644 --- a/src/client/dfs/mnt.c +++ b/src/client/dfs/mnt.c @@ -688,7 +688,7 @@ dfs_mount(daos_handle_t poh, daos_handle_t coh, int flags, dfs_t **_dfs) dfs->last_hi = (unsigned int)d_rand(); /** Avoid potential conflict with SB or ROOT */ if (dfs->last_hi <= 1) - dfs->last_hi += OID_INC; + dfs->last_hi = 2; rc = daos_cont_alloc_oids(coh, 1, &dfs->oid.lo, NULL); if (rc) { @@ -696,13 +696,9 @@ dfs_mount(daos_handle_t poh, daos_handle_t coh, int flags, dfs_t **_dfs) D_GOTO(err_root, rc = daos_der2errno(rc)); } - /* - * if this is the first time we allocate on this container, - * account 0 for SB, 1 for root obj. - */ dfs->oid.hi = dfs->last_hi; /** Increment so that dfs->last_hi is the last value */ - oid_inc(&dfs->oid); + daos_obj_oid_cycle(&dfs->oid); } dfs->mounted = DFS_MOUNT; diff --git a/src/include/daos_obj.h b/src/include/daos_obj.h index 316d1b5547b..267763a2ba4 100644 --- a/src/include/daos_obj.h +++ b/src/include/daos_obj.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2015-2023 Intel Corporation. + * (C) Copyright 2015-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -564,6 +564,21 @@ daos_obj_generate_oid(daos_handle_t coh, daos_obj_id_t *oid, enum daos_otype_t type, daos_oclass_id_t cid, daos_oclass_hints_t hints, uint32_t args); + +/** + * This function, if called 2^32 times will set oid->hi to every unique 32-bit + * value. The caller is responsible for setting the initial value, tracking the + * final value, and avoiding any values that are otherwise reserved. + * + * \param[in, out] oid oid to cycle + */ +static inline void +daos_obj_oid_cycle(daos_obj_id_t *oid) +{ + /** Uses a large prime number to guarantee hitting every unique value */ + oid->hi = (oid->hi + 999999937) & UINT_MAX; +} + /** * Open an DAOS object. * From ab1c12bffb8dcab51ea00d4505f0177d4b5e6c4b Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Thu, 29 Aug 2024 14:46:52 -0600 Subject: [PATCH 4/4] Fix clang-format issues Required-githooks: true Signed-off-by: Jeff Olivier --- src/include/daos_obj.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/include/daos_obj.h b/src/include/daos_obj.h index 267763a2ba4..52b15ade40b 100644 --- a/src/include/daos_obj.h +++ b/src/include/daos_obj.h @@ -564,7 +564,6 @@ daos_obj_generate_oid(daos_handle_t coh, daos_obj_id_t *oid, enum daos_otype_t type, daos_oclass_id_t cid, daos_oclass_hints_t hints, uint32_t args); - /** * This function, if called 2^32 times will set oid->hi to every unique 32-bit * value. The caller is responsible for setting the initial value, tracking the