DAOS-16445 client: Try cycling different method

We've noticed that with 8GB files we fill space pretty quickly.

Just trying another method to see if it works any better.

Basic idea is two fold:

1. Use a large prime number as the increment as we cycle through the
   oids.
2. Randomize which one we start with on a per mount basis.

Required-githooks: true

Signed-off-by: Jeff Olivier <jeffolivier@google.com>

src/client/dfs/dfs_internal.h

@@ -102,6 +102,18 @@
 /** MAX value for the HI OID */
 #define MAX_OID_HI         ((1UL << 32) - 1)
 
+/** Use a large prime for cycling through all oids */
+#define OID_INC            (999999937)
+
+static inline void
+oid_inc(daos_obj_id_t *oid)
+{
+	oid->hi = (oid->hi + OID_INC) & MAX_OID_HI;
+	/** Skip reserved entries */
+	if (oid->lo == RESERVED_LO && oid->hi <= 1)
+		oid->hi += OID_INC;
+}
+
 typedef uint64_t dfs_magic_t;
 typedef uint16_t dfs_sb_ver_t;
 typedef uint16_t dfs_layout_ver_t;
@@ -162,6 +174,8 @@ struct dfs {
 	daos_handle_t        coh;
 	/** refcount on cont handle that through the DFS API */
 	uint32_t             coh_refcount;
+	/** The last oid.hi in the sequence */
+	uint32_t             last_hi;
 	/** Object ID reserved for this DFS (see oid_gen below) */
 	daos_obj_id_t        oid;
 	/** superblock object OID */
@@ -337,20 +351,22 @@ oid_gen(dfs_t *dfs, daos_oclass_id_t oclass, bool file, daos_obj_id_t *oid)
 
 	D_MUTEX_LOCK(&dfs->lock);
 	/** If we ran out of local OIDs, alloc one from the container */
-	if (dfs->oid.hi >= MAX_OID_HI) {
+	if (dfs->oid.hi == dfs->last_hi) {
 		/** Allocate an OID for the namespace */
 		rc = daos_cont_alloc_oids(dfs->coh, 1, &dfs->oid.lo, NULL);
 		if (rc) {
 			D_ERROR("daos_cont_alloc_oids() Failed (%d)\n", rc);
 			D_MUTEX_UNLOCK(&dfs->lock);
 			return daos_der2errno(rc);
 		}
-		dfs->oid.hi = 0;
+		/** Start such that dfs->last_hi will be final value */
+		dfs->oid.hi = dfs->last_hi;
 	}
 
 	/** set oid and lo, bump the current hi value */
 	oid->lo = dfs->oid.lo;
-	oid->hi = dfs->oid.hi++;
+	oid_inc(&dfs->oid);
+	oid->hi = dfs->oid.hi;
 	D_MUTEX_UNLOCK(&dfs->lock);
 
 	/** if a regular file, use UINT64 typed dkeys for the array object */

src/client/dfs/mnt.c

@@ -683,6 +683,11 @@ dfs_mount(daos_handle_t poh, daos_handle_t coh, int flags, dfs_t **_dfs)
 
 	/** if RW, allocate an OID for the namespace */
 	if (amode == O_RDWR) {
+		dfs->last_hi = (unsigned int)d_rand();
+		/** Avoid potential conflict with SB or ROOT */
+		if (dfs->last_hi <= 1)
+			dfs->last_hi += OID_INC;
+
 		rc = daos_cont_alloc_oids(coh, 1, &dfs->oid.lo, NULL);
 		if (rc) {
 			D_ERROR("daos_cont_alloc_oids() Failed, " DF_RC "\n", DP_RC(rc));
@@ -693,10 +698,9 @@ dfs_mount(daos_handle_t poh, daos_handle_t coh, int flags, dfs_t **_dfs)
 		 * if this is the first time we allocate on this container,
 		 * account 0 for SB, 1 for root obj.
 		 */
-		if (dfs->oid.lo == RESERVED_LO)
-			dfs->oid.hi = ROOT_HI + 1;
-		else
-			dfs->oid.hi = 0;
+		dfs->oid.hi = dfs->last_hi;
+		/** Increment so that dfs->last_hi is the last value */
+		oid_inc(&dfs->oid);
 	}
 
 	dfs->mounted = DFS_MOUNT;
@@ -1016,7 +1020,7 @@ dfs_global2local(daos_handle_t poh, daos_handle_t coh, int flags, d_iov_t glob,
 
 	/** allocate a new oid on the next file or dir creation */
 	dfs->oid.lo = 0;
-	dfs->oid.hi = MAX_OID_HI;
+	dfs->oid.hi = dfs->last_hi;
 
 	rc = D_MUTEX_INIT(&dfs->lock, NULL);
 	if (rc != 0) {