Lock x.py build state

Prevent spurious build failures and other bugs caused by parallel runs of
x.py. We back the lock with sqlite, so that we have a cross-platform locking
strategy, and which can be done nearly first in the build process (from Python),
which helps move the lock as early as possible.

src/bootstrap/bootstrap.py

@@ -12,9 +12,38 @@
 import sys
 import tarfile
 import tempfile
+import sqlite3
 
 from time import time
 
+# Acquire a lock on the build directory to make sure that
+# we don't cause a race condition while building
+# Lock is created in `build_dir/lock.db`
+def acquire_lock(build_dir):
+    path = os.path.join(build_dir, "lock.db")
+    try:
+        con = sqlite3.Connection(path, timeout=0)
+        curs = con.cursor()
+        curs.execute("BEGIN EXCLUSIVE")
+        # The lock is released when the cursor is dropped
+        return curs
+    # If the database is busy then lock has already been acquired
+    # so we wait for the lock.
+    # We retry every quarter second so that execution is passed back to python
+    # so that it can handle signals
+    except sqlite3.OperationalError:
+        del con
+        del curs
+        print("Waiting for lock on build directory")
+        con = sqlite3.Connection(path, timeout=0.25)
+        curs = con.cursor()
+        while True:
+            try:
+                curs.execute("BEGIN EXCLUSIVE")
+            except sqlite3.OperationalError:
+                pass
+        return curs
+
 def support_xz():
     try:
         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
@@ -1225,6 +1254,12 @@ def bootstrap(help_triggered):
     build.set_dist_environment(data["dist_server"])
 
     build.build = args.build or build.build_triple()
+
+    # Acquire the lock before doing any build actions
+    # The lock is released when `lock` is dropped
+    if not os.path.exists(build.build_dir):
+        os.makedirs(build.build_dir)
+    lock = acquire_lock(build.build_dir)
     build.update_submodules()
 
     # Fetch/build the bootstrap