Enhance test_index_all to print state & status

We also move the state/status checks to 45 seconds past the minute since
the cron jobs run on the minute and we give them 3/4 of the minute to
run before we perform a check (makes it so that in the server logs we
get overlapping API calls with the cron job logs making easier to read
the logs).

lib/pbench/test/functional/server/test_put.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+from datetime import datetime, timedelta
 from http import HTTPStatus
 import os.path
 from pathlib import Path
@@ -134,6 +135,7 @@ def check_indexed(server_client: PbenchServerClient, datasets):
                     assert status["index"] == "ok"
                     indexed.append(dataset)
                 else:
+                    print(f"\t\tstate={state!r} status={status!r}")
                     not_indexed.append(dataset)
         except HTTPError as exc:
             pytest.fail(
@@ -177,27 +179,37 @@ def test_index_all(server_client: PbenchServerClient, login_user):
             )
 
         # For each dataset we find, poll the state until it reaches Indexed
-        # state, or until we time out.
-        now = start = time.time()
-        timeout = start + (60.0 * 10.0)
+        # state, or until we time out.  Since the cron jobs run once a minute
+        # and they start on the minute, we make our 1st check 45 seconds into
+        # the next minute, and then check at 45 seconds past the minute until
+        # we reached 5 minutes past the original start time.
+        oneminute = timedelta(minutes=1)
+        now = start = datetime.utcnow()
+        timeout = start + timedelta(minutes=5)
+        target_int = (
+            datetime(now.year, now.month, now.day, now.hour, now.minute)
+            + oneminute
+            + timedelta(seconds=45)
+        )
 
         while not_indexed:
-            print(f"[{now - start:0.2f}] Checking ...")
+            print(f"[{(now - start).total_seconds():0.2f}] Checking ...")
             not_indexed, indexed = TestPut.check_indexed(server_client, not_indexed)
             for dataset in indexed:
                 tp = dataset.metadata["server.tarball-path"]
                 if os.path.basename(tp) not in tarball_names:
                     continue
                 count += 1
                 print(f"    Indexed {tp}")
+            now = datetime.utcnow()
             if not not_indexed or now >= timeout:
                 break
-            time.sleep(30.0)  # sleep for half a minute
-            now = time.time()
-        assert (
-            not not_indexed
-        ), f"Timed out after {now - start} seconds; unindexed datasets: " + ", ".join(
-            d.metadata["server.tarball-path"] for d in not_indexed
+            time.sleep((target_int - now).total_seconds())
+            target_int += oneminute
+            now = datetime.utcnow()
+        assert not not_indexed, (
+            f"Timed out after {(now - start).total_seconds()} seconds; unindexed datasets: "
+            + ", ".join(d.metadata["server.tarball-path"] for d in not_indexed)
         )
         assert (
             len(tarball_names) == count