Updates test file wrapper to deterministically detect file write comp…

…letion (elastic#176115)

Closes elastic#119267

## Summary

Attempts to deterministically detect when a file is written in entirety
in order to resolve flaky test issues where parsed JSON is incomplete.

Flaky Test Runner:
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5015

(cherry picked from commit f9125ba)

# Conflicts:
#	x-pack/test/security_api_integration/tests/anonymous/login.ts
#	x-pack/test/security_api_integration/tests/audit/audit_log.ts
#	x-pack/test/security_api_integration/tests/audit/file_wrapper.ts
#	x-pack/test/security_api_integration/tests/kerberos/kerberos_login.ts
#	x-pack/test/security_api_integration/tests/oidc/authorization_code_flow/oidc_auth.ts
#	x-pack/test/security_api_integration/tests/pki/pki_auth.ts
#	x-pack/test/security_api_integration/tests/saml/saml_login.ts
#	x-pack/test/security_api_integration/tests/token/audit.ts

x-pack/test/security_api_integration/tests/audit/audit_log.ts

@@ -12,6 +12,7 @@ import { RetryService } from '../../../../../test/common/services/retry';
 import { FtrProviderContext } from '../../ftr_provider_context';
 
 class FileWrapper {
+  delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
   constructor(private readonly path: string, private readonly retry: RetryService) {}
   async reset() {
     // "touch" each file to ensure it exists and is empty before each test
@@ -35,10 +36,15 @@ class FileWrapper {
     });
   }
   // writing in a file is an async operation. we use this method to make sure logs have been written.
-  async isNotEmpty() {
-    const content = await this.read();
-    const line = content[0];
-    return line.length > 0;
+  async isWritten() {
+    // attempt at determinism - wait for the size of the file to stop changing.
+    await this.retry.waitForWithTimeout(`file '${this.path}' to be written`, 5000, async () => {
+      const sizeBefore = Fs.statSync(this.path).size;
+      await this.delay(500);
+      const sizeAfter = Fs.statSync(this.path).size;
+      return sizeAfter === sizeBefore;
+    });
+    return Fs.statSync(this.path).size > 0;
   }
 }
 
@@ -57,8 +63,7 @@ export default function ({ getService }: FtrProviderContext) {
 
     it('logs audit events when reading and writing saved objects', async () => {
       await supertest.get('/audit_log?query=param').set('kbn-xsrf', 'foo').expect(204);
-      await retry.waitFor('logs event in the dest file', async () => await logFile.isNotEmpty());
-
+      await logFile.isWritten();
       const content = await logFile.readJSON();
 
       const httpEvent = content.find((c) => c.event.action === 'http_request');
@@ -94,8 +99,7 @@ export default function ({ getService }: FtrProviderContext) {
           params: { username, password },
         })
         .expect(200);
-      await retry.waitFor('logs event in the dest file', async () => await logFile.isNotEmpty());
-
+      await logFile.isWritten();
       const content = await logFile.readJSON();
 
       const loginEvent = content.find((c) => c.event.action === 'user_login');
@@ -116,8 +120,7 @@ export default function ({ getService }: FtrProviderContext) {
           params: { username, password: 'invalid_password' },
         })
         .expect(401);
-      await retry.waitFor('logs event in the dest file', async () => await logFile.isNotEmpty());
-
+      await logFile.isWritten();
       const content = await logFile.readJSON();
 
       const loginEvent = content.find((c) => c.event.action === 'user_login');