[T-242] Docker for alpine and centos binaries

.gitignore

@@ -3,3 +3,5 @@ build/
 prebuilds/
 node_modules/
 token.txt
+.DS_Store
+.idea

Makefile

@@ -0,0 +1,8 @@
+.PHONY: build run
+
+current_dir = $(shell pwd)
+
+docker-build:
+	docker build -t jasperdm/centos-watchman -f ./docker-agents/centos.Dockerfile .
+	docker build -t jasperdm/alpine-watchman -f ./docker-agents/alpine.Dockerfile .
+

azure-pipelines-template.yml

@@ -2,17 +2,16 @@ jobs:
 - job: ${{ parameters.name }}
   pool: 
     vmImage: ${{ parameters.vmImage }}
-  ${{ if eq(parameters.name, 'Linux') }}:
-    container: jotadrilo/watchman:latest
+  ${{ if or(eq(parameters.name, 'Alpine'), eq(parameters.name, 'CentOS7')) }}:
+    container: ${{ parameters.container }}
   strategy:
     matrix:
-      node_8_x:
-        node_version: 8.x
-        publish_artifacts: false
       node_10_x:
         node_version: 10.x
+        publish_artifacts: false
+      node_12_x:
+        node_version: 12.x
         publish_artifacts: true
-    maxParallel: 3
   steps:
   - task: NodeTool@0
     inputs:
@@ -28,14 +27,8 @@ jobs:
 
   - ${{ if eq(parameters.name, 'Windows') }}:
     - script: |
-        curl -sSf -L -o watchman.zip https://github.com/facebook/watchman/suites/296618337/artifacts/249606
-      displayName: Download Watchman
-    - task: ExtractFiles@1
-      inputs:
-        archiveFilePatterns: 'watchman.zip'
-        destinationFolder: $(Agent.HomeDirectory)
-        cleanDestinationFolder: false
-      displayName: Extract Watchman
+        cp -rf $(System.DefaultWorkingDirectory)\watchman $(Agent.HomeDirectory)
+      displayName: Copy Watchman
     - script: |
         set PATH=%PATH%;$(Agent.HomeDirectory)\watchman\windows\bin
         echo "##vso[task.setvariable variable=PATH]%PATH%"
@@ -46,12 +39,10 @@ jobs:
     displayName: Install dependencies
   - script: npm test
     displayName: Run tests
+  - script: npm run prebuild
+    displayName: Test prebuild
 
   - ${{ if startsWith(variables['Build.SourceBranch'], 'refs/tags/v') }}:
-    - script: npm run prebuild
-      displayName: Prebuild
-      condition: and(succeeded(), eq(variables['publish_artifacts'], 'true'))
-
     - task: PublishBuildArtifacts@1
       displayName: Publish Artifacts
       condition: and(succeeded(), eq(variables['publish_artifacts'], 'true'))

azure-pipelines.yml

@@ -14,8 +14,15 @@ jobs:
 
   - template: azure-pipelines-template.yml
     parameters:
-      name: Linux
+      name: Alpine
       vmImage: ubuntu-16.04
+      container: jasperdm/alpine-watchman:latest
+
+  - template: azure-pipelines-template.yml
+    parameters:
+      name: CentOS7
+      vmImage: ubuntu-16.04
+      container: jasperdm/centos-watchman:latest
 
   - template: azure-pipelines-template.yml
     parameters:

binding.gyp

@@ -35,7 +35,7 @@
             "src/watchman/WatchmanBackend.cc",
             "src/shared/BruteForceBackend.cc",
             "src/linux/InotifyBackend.cc",
-            "src/unix/fts.cc"
+            "src/unix/legacy.cc"
           ],
           "defines": [
             "WATCHMAN",

docker-agents/alpine.Dockerfile

@@ -0,0 +1,53 @@
+FROM node:12-alpine
+
+RUN apk update
+RUN apk upgrade
+
+# Azure Pipelines stuff
+RUN apk add --no-cache --virtual .pipeline-deps readline linux-pam \
+  && apk add bash sudo shadow \
+  && apk del .pipeline-deps
+
+RUN apk add --no-cache \
+  libcrypto1.1 \
+  libgcc \ 
+  libstdc++ \ 
+  ca-certificates  \ 
+  openssl \
+  automake \
+  autoconf \
+  bash \
+  build-base \
+  libtool \
+  linux-headers \
+  openssl-dev \
+  python-dev \
+  g++ \ 
+  gcc \
+  git \
+  fts-dev \
+  python
+
+ENV WATCHMAN_VERSION=4.9.0 \
+  WATCHMAN_SHA256=1f6402dc70b1d056fffc3748f2fdcecff730d8843bb6936de395b3443ce05322
+
+RUN cd /tmp \
+ && wget -O watchman.tar.gz "https://github.com/facebook/watchman/archive/v${WATCHMAN_VERSION}.tar.gz" \
+ && echo "$WATCHMAN_SHA256 *watchman.tar.gz" | sha256sum -c - \
+ && tar -xz -f watchman.tar.gz -C /tmp/ \
+ && rm -rf watchman.tar.gz
+
+RUN cd /tmp/watchman-${WATCHMAN_VERSION} \
+ && ./autogen.sh \
+ && ./configure --enable-lenient \
+ && make \
+ && make install \
+ && cd $HOME \
+ && rm -rf /tmp/*
+
+RUN watchman --version
+RUN command -v bash
+
+# Set Node runtime as it's using musl
+LABEL "com.azure.dev.pipelines.agent.handler.node.path"="/usr/local/bin/node"
+CMD [ "node" ]

docker-agents/centos.Dockerfile

@@ -0,0 +1,37 @@
+FROM centos/devtoolset-7-toolchain-centos7
+
+USER 0
+
+ENV WATCHMAN_VERSION=4.9.0 \
+  WATCHMAN_SHA256=1f6402dc70b1d056fffc3748f2fdcecff730d8843bb6936de395b3443ce05322
+
+RUN yum install -y \
+  wget \
+  libgcc \ 
+  libstdc++ \ 
+  ca-certificates  \ 
+  openssl \
+  automake \
+  autoconf \
+  bash \
+  libtool \
+  openssl-devel \
+  python-devel \
+  gcc \
+  git
+
+RUN cd /tmp \
+ && wget -O watchman.tar.gz "https://github.com/facebook/watchman/archive/v${WATCHMAN_VERSION}.tar.gz" \
+ && echo "$WATCHMAN_SHA256 *watchman.tar.gz" | sha256sum -c - \
+ && tar -xz -f watchman.tar.gz -C /tmp/ \
+ && rm -rf watchman.tar.gz
+
+RUN cd /tmp/watchman-${WATCHMAN_VERSION} \
+ && ./autogen.sh \
+ && ./configure \
+ && make \
+ && make install \
+ && cd $HOME \
+ && rm -rf /tmp/*
+
+RUN watchman --version

src/unix/fts.cc

@@ -1,5 +1,11 @@
 #include <string>
-#define __THROW // weird error on linux
+
+// weird error on linux
+#ifdef __THROW
+#undef __THROW
+#endif
+#define __THROW
+
 #include <fts.h>
 #include "../DirTree.hh"
 #include "../shared/BruteForceBackend.hh"

src/unix/legacy.cc

@@ -0,0 +1,67 @@
+#include <string>
+
+// weird error on linux
+#ifdef __THROW
+#undef __THROW
+#endif
+#define __THROW
+
+#ifdef _LIBC
+# include <include/sys/stat.h>
+#else
+# include <sys/stat.h>
+#endif
+#include <dirent.h>
+#include <unistd.h>
+
+#include "../DirTree.hh"
+#include "../shared/BruteForceBackend.hh"
+
+#define CONVERT_TIME(ts) ((uint64_t)ts.tv_sec * 1000000000 + ts.tv_nsec)
+#if __APPLE__
+#define st_mtim st_mtimespec
+#endif
+#define ISDOT(a) (a[0] == '.' && (!a[1] || (a[1] == '.' && !a[2])))
+
+void iterateDir(Watcher &watcher, const std::shared_ptr <DirTree> tree, const char *relative, int parent_fd, const std::string dirname) {
+    int open_flags = (O_RDONLY | O_CLOEXEC | O_DIRECTORY | O_NOCTTY | O_NONBLOCK | O_NOFOLLOW);
+    int new_fd = openat(parent_fd, relative, open_flags);
+
+    struct stat rootAttributes;
+    fstatat(new_fd, ".", &rootAttributes, AT_SYMLINK_NOFOLLOW);
+    tree->add(dirname, CONVERT_TIME(rootAttributes.st_mtim), true);
+
+    if (DIR *dir = fdopendir(new_fd)) {
+        while (struct dirent *ent = (errno = 0, readdir(dir))) {
+            if (ISDOT(ent->d_name)) continue;
+
+            std::string fullPath = dirname + "/" + ent->d_name;
+
+            if (watcher.mIgnore.count(fullPath) == 0) {
+                struct stat attrib;
+                fstatat(new_fd, ent->d_name, &attrib, AT_SYMLINK_NOFOLLOW);
+                bool isDir = ent->d_type == DT_DIR;
+
+                if (isDir) {
+                    iterateDir(watcher, tree, ent->d_name, new_fd, fullPath);
+                } else {
+                    tree->add(fullPath, CONVERT_TIME(attrib.st_mtim), isDir);
+                }
+            }
+        }
+    }
+
+    close(new_fd);
+
+    if (errno) {
+        throw WatcherError(strerror(errno), &watcher);
+    }
+}
+
+void BruteForceBackend::readTree(Watcher &watcher, std::shared_ptr <DirTree> tree) {
+    int fd = open(watcher.mDir.c_str(), O_RDONLY);
+    if (fd) {
+        iterateDir(watcher, tree, ".", fd, watcher.mDir);
+        close(fd);
+    }
+}

test/watcher.js

@@ -606,22 +606,24 @@ describe('watcher', () => {
     });
   });
 
-  describe('watchman errors', () => {
-    it('should emit an error when watchman dies', async () => {
-      let dir = path.join(fs.realpathSync(require('os').tmpdir()), Math.random().toString(31).slice(2));
-      fs.mkdirpSync(dir);
-      await new Promise(resolve => setTimeout(resolve, 100));
-
-      let p = new Promise(resolve => {
-        watcher.subscribe(dir, (err, events) => {
-          setImmediate(() => resolve(err));
-        }, {backend: 'watchman'});
-      });
+  if (backends.includes('watchman')) {
+    describe('watchman errors', () => {
+      it('should emit an error when watchman dies', async () => {
+        let dir = path.join(fs.realpathSync(require('os').tmpdir()), Math.random().toString(31).slice(2));
+        fs.mkdirpSync(dir);
+        await new Promise(resolve => setTimeout(resolve, 100));
+
+        let p = new Promise(resolve => {
+          watcher.subscribe(dir, (err, events) => {
+            setImmediate(() => resolve(err));
+          }, {backend: 'watchman'});
+        });
 
-      execSync('watchman shutdown-server');
+        execSync('watchman shutdown-server');
 
-      let err = await p;
-      assert(err, 'No error was emitted');
+        let err = await p;
+        assert(err, 'No error was emitted');
+      });
     });
-  });
+  }
 });