Fix directory creation race condition in Folder and SandboxFolder (…

…#4912)

The `Folder` and `SandboxFolder` classes of `aiida.common.folders` used
the following paradigm to create the required folders:

    if not os.path.exists(filepath):
        os.makedirs(filepath)

However, this is susceptible to a race condition. If two processes call
the same piece of code almost at the same time, they may both evaluate
the conditional to be True if the filepath does not yet exist, but one
of the two will actually get to the creation first, causing the second
process to except with a `FileExistsError`.

The solution is to replace it with `os.makedirs(filepath, exist_ok=True)`
which will swallow the exception if the path already exists.

Cherry-pick: dc686c5

aiida/common/folders.py

@@ -342,8 +342,7 @@ def create(self):
         It is always safe to call it, it will do nothing if the folder
         already exists.
         """
-        if not self.exists():
-            os.makedirs(self.abspath, mode=self.mode_dir)
+        os.makedirs(self.abspath, mode=self.mode_dir, exist_ok=True)
 
     def replace_with_folder(self, srcdir, move=False, overwrite=False):
         """This routine copies or moves the source folder 'srcdir' to the local folder pointed to by this Folder.
@@ -370,8 +369,7 @@ def replace_with_folder(self, srcdir, move=False, overwrite=False):
 
         # Create parent dir, if needed, with the right mode
         pardir = os.path.dirname(self.abspath)
-        if not os.path.exists(pardir):
-            os.makedirs(pardir, mode=self.mode_dir)
+        os.makedirs(pardir, mode=self.mode_dir, exist_ok=True)
 
         if move:
             shutil.move(srcdir, self.abspath)
@@ -417,8 +415,7 @@ def __init__(self, sandbox_in_repo=True):
         # First check if the sandbox folder already exists
         if sandbox_in_repo:
             sandbox = os.path.join(get_profile().repository_path, 'sandbox')
-            if not os.path.exists(sandbox):
-                os.makedirs(sandbox)
+            os.makedirs(sandbox, exist_ok=True)
             abspath = tempfile.mkdtemp(dir=sandbox)
         else:
             abspath = tempfile.mkdtemp()