chore: split core, http api server and cli out from ipfs module (#3288)

Decomposes the main ipfs module into separate core, http api server and cli modules which encapsulate the different parts of ipfs. This allows us to: 1. Omit modules/code/dependencies if they aren't going to be used 2. Reduces the amount of tests being run in series The average time of a CI job has gone from just under 20 minutes to just over 10 minutes. Outliers here are electron tests (which should be fixed by #3251) and webworker tests which will need further investigation. Fixes #2678 Fixes #2877
ipfs · Sep 28, 2020 · 796d7ef · 796d7ef
1 parent 8699cba
commit 796d7ef
Show file tree

Hide file tree

Showing 667 changed files with 3,258 additions and 5,603 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -9,7 +9,6 @@ branches:
   - /^release\/.*$/
 
 stages:
-  - check
   - test
   - release-rc
   - test-external
@@ -38,14 +37,16 @@ addons:
       # https://stackoverflow.com/questions/57903415/travis-ci-chrome-62-instead-of-77
       - dpkg
   chrome: stable
+  firefox: latest
 
 before_install:
   # prevents windows error: npm ERR! ... git-sh-setup: file not found
   - if [ "$TRAVIS_OS_NAME" = "windows" ]; then export PATH=/c/PROGRA~1/Git/usr/bin:/c/PROGRA~1/Git/mingw64/libexec/git-core:$PATH ; fi
   # only run jobs in packages that have changed since master in PR builds
   - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then export RUN_SINCE='--since master' ; fi
 
-script: npm run test:node -- $RUN_SINCE -- -- --timeout 10000 --bail
+script:
+  - npm run test:node -- $RUN_SINCE -- -- --timeout 10000 --bail
 
 jobs:
   allow_failures:
@@ -57,74 +58,181 @@ jobs:
     - name: external - orbit-db
     - name: external - ipfs-log
     - name: external - sidetree
+
   include:
-    - stage: check
+    # manual install step, we do this to cache the installed files for subsequent steps
+    - stage: test
+      name: lint
       script:
-        - npm run build -- $RUN_SINCE --scope={ipfs,ipfs-http-client,ipfs-message-port-*}
-        - npm run dep-check -- $RUN_SINCE -- -- -- -i electron-webrtc
         - npm run lint -- $RUN_SINCE --concurrency 1
 
+    - stage: test
+      name: dep-check
+      script:
+        - npm run dep-check -- $RUN_SINCE -- -- -- -i electron-webrtc
+
     - stage: test
       name: chrome
-      addons:
-        chrome: stable
       script:
         - npm run test:browser -- $RUN_SINCE -- -- --bail
 
     - stage: test
       name: chrome webworker
-      addons:
-        chrome: stable
       script:
-        - npm run test:webworker -- $RUN_SINCE -- -- --bail
+        - npm run test:webworker -- $RUN_SINCE -- -- --bail --timeout 60000
 
     - stage: test
       name: firefox
-      addons:
-        firefox: latest
       script:
         - npm run test:browser -- $RUN_SINCE -- -- --bail --browsers FirefoxHeadless
 
     - stage: test
       name: firefox webworker
-      addons:
-        firefox: latest
       script:
-        - npm run test:webworker -- $RUN_SINCE -- -- --bail --browsers FirefoxHeadless
+        - npm run test:webworker -- $RUN_SINCE -- -- --bail --browsers FirefoxHeadless --timeout 60000
 
     - stage: test
       name: electron-main
       os: osx
       script:
-        - npm run test:electron-main -- $RUN_SINCE -- -- --bail
+        - npm run test:electron-main -- $RUN_SINCE -- -- --bail --timeout 60000
 
     - stage: test
       name: electron-renderer
       os: osx
       script:
-        - npm run test:electron-renderer -- $RUN_SINCE -- -- --bail
+        - npm run test:electron-renderer -- $RUN_SINCE -- -- --bail --timeout 60000
 
     - stage: test
       name: interop - node
       script:
-        - npm run test:interop:node -- $RUN_SINCE -- -- --bail
+        - npm run test:interop -- $RUN_SINCE -- -- -- -t node --bail
 
     - stage: test
       name: interop - browser
       script:
-        - npm run test:interop:browser -- $RUN_SINCE -- -- --bail
+        - npm run test:interop -- $RUN_SINCE -- -- -- -t browser --bail
 
     - stage: test
       name: interop - electron-main
       os: osx
       script:
-        - npm run test:interop:electron-main -- $RUN_SINCE -- -- --bail --timeout 10000
+        - npm run test:interop -- $RUN_SINCE -- -- -- -t electron-main -f ./test/node.js --bail --timeout 60000
 
     - stage: test
       name: interop - electron-renderer
       os: osx
       script:
-        - npm run test:interop:electron-renderer -- $RUN_SINCE -- -- --bail --timeout 10000
+        - npm run test:interop -- $RUN_SINCE -- -- -- -t electron-renderer -f ./test/browser.js -bail --timeout 60000
+
+    - stage: test
+      name: js-ipfs interface tests - node
+      script:
+        - npm run test:interface:core -- $RUN_SINCE -- -- --bail -t node
+
+    - stage: test
+      name: js-ipfs interface tests - chrome
+      script:
+        - npm run test:interface:core -- $RUN_SINCE -- -- --bail -t browser
+
+    - stage: test
+      name: js-ipfs interface tests - chrome webworker
+      script:
+        - npm run test:interface:core -- $RUN_SINCE -- -- --bail -t webworker --timeout 60000
+
+    - stage: test
+      name: js-ipfs interface tests - firefox
+      script:
+        - npm run test:interface:core -- $RUN_SINCE -- -- --bail -t browser --browsers FirefoxHeadless
+
+    - stage: test
+      name: js-ipfs interface tests - firefox webworker
+      script:
+        - npm run test:interface:core -- $RUN_SINCE -- -- --bail -t webworker --browsers FirefoxHeadless --timeout 60000
+
+    - stage: test
+      name: js-ipfs interface tests - electron main
+      os: osx
+      script:
+        - npm run test:interface:core -- $RUN_SINCE -- -- --bail -t electron-main --timeout 60000
+
+    - stage: test
+      name: js-ipfs interface tests - electron renderer
+      os: osx
+      script:
+        - npm run test:interface:core -- $RUN_SINCE -- -- --bail -t electron-renderer --timeout 60000
+
+    - stage: test
+      name: http-api-client interface tests vs go-ipfs - node
+      script:
+        - npm run test:interface:http-go -- $RUN_SINCE -- -- --bail -t node
+
+    - stage: test
+      name: http-api-client interface tests vs go-ipfs - chrome
+      script:
+        - npm run test:interface:http-go -- $RUN_SINCE -- -- --bail -t browser
+
+    - stage: test
+      name: http-api-client interface tests vs go-ipfs - chrome webworker
+      script:
+        - npm run test:interface:http-go -- $RUN_SINCE -- -- --bail -t webworker --timeout 60000
+
+    - stage: test
+      name: http-api-client interface tests vs go-ipfs - firefox
+      script:
+        - npm run test:interface:http-go -- $RUN_SINCE -- -- --bail -t browser --browsers FirefoxHeadless
+
+    - stage: test
+      name: http-api-client interface tests vs go-ipfs - firefox webworker
+      script:
+        - npm run test:interface:http-go -- $RUN_SINCE -- -- --bail -t webworker --browsers FirefoxHeadless --timeout 60000
+
+    - stage: test
+      name: http-api-client interface tests vs js-ipfs - node
+      script:
+        - npm run test:interface:http-js -- $RUN_SINCE -- -- --bail -t node
+
+    - stage: test
+      name: http-api-client interface tests vs js-ipfs - chrome
+      script:
+        - npm run test:interface:http-js -- $RUN_SINCE -- -- --bail -t browser
+
+    - stage: test
+      name: http-api-client interface tests vs js-ipfs - chrome webworker
+      script:
+        - npm run test:interface:http-js -- $RUN_SINCE -- -- --bail -t webworker --timeout 60000
+
+    - stage: test
+      name: http-api-client interface tests vs js-ipfs - firefox
+      script:
+        - npm run test:interface:http-js -- $RUN_SINCE -- -- --bail -t browser --browsers FirefoxHeadless
+
+    - stage: test
+      name: http-api-client interface tests vs js-ipfs - firefox webworker
+      script:
+        - npm run test:interface:http-js -- $RUN_SINCE -- -- --bail -t webworker --browsers FirefoxHeadless --timeout 60000
+
+    - stage: test
+      name: http-api-client interface tests vs js-ipfs - electron main
+      os: osx
+      script:
+        - npm run test:interface:http-js -- $RUN_SINCE -- -- --bail -t electron-main --timeout 60000
+
+    - stage: test
+      name: http-api-client interface tests vs js-ipfs - electron renderer
+      os: osx
+      script:
+        - npm run test:interface:http-js -- $RUN_SINCE -- -- --bail -t electron-renderer --timeout 60000
+
+    - stage: test
+      name: ipfs-message-port-client interface tests - chrome
+      script:
+        - npm run test:interface:message-port-client -- $RUN_SINCE -- -- --bail -t browser
+
+    - stage: test
+      name: ipfs-message-port-client interface tests - firefox
+      script:
+        - npm run test:interface:message-port-client -- $RUN_SINCE -- -- --bail -t browser --browsers FirefoxHeadless
 
     - stage: test
       name: examples

diff --git a/examples/browser-exchange-files/package.json b/examples/browser-exchange-files/package.json
@@ -19,7 +19,7 @@
   },
   "dependencies": {
     "ipfs": "^0.50.1",
-    "it-all": "^1.0.1",
+    "it-all": "^1.0.4",
     "test-ipfs-example": "^2.0.3"
   },
   "browser": {

diff --git a/examples/browser-exchange-files/test.js b/examples/browser-exchange-files/test.js
@@ -12,7 +12,7 @@ const df = createFactory({
   ipfsHttpModule: require('ipfs-http-client')
 }, {
   js: {
-    ipfsBin: require.resolve('ipfs/src/cli/bin.js')
+    ipfsBin: require.resolve('ipfs/src/cli.js')
   }
 })
 const {

diff --git a/examples/browser-http-client-upload-file/test.js b/examples/browser-http-client-upload-file/test.js
@@ -5,7 +5,7 @@ const execa = require('execa')
 const { createFactory } = require('ipfsd-ctl')
 const df = createFactory({
   ipfsHttpModule: require('ipfs-http-client'),
-  ipfsBin: require.resolve('ipfs/src/cli/bin.js')
+  ipfsBin: require.resolve('ipfs/src/cli.js')
 })
 const {
   startServer

diff --git a/examples/browser-ipns-publish/package.json b/examples/browser-ipns-publish/package.json
@@ -17,15 +17,15 @@
     "ipfs-http-client": "^47.0.0",
     "ipfs-utils": "^3.0.0",
     "ipns": "^0.8.0",
-    "it-last": "^1.0.2",
+    "it-last": "^1.0.4",
     "p-retry": "^4.2.0",
     "uint8arrays": "^1.1.0"
   },
   "browserslist": [
     "last 2 versions and not dead and > 2%"
   ],
   "devDependencies": {
-    "delay": "^4.3.0",
+    "delay": "^4.4.0",
     "execa": "^4.0.0",
     "ipfsd-ctl": "^7.0.0",
     "go-ipfs": "^0.6.0",

diff --git a/examples/browser-readablestream/package.json b/examples/browser-readablestream/package.json
@@ -21,7 +21,7 @@
   },
   "dependencies": {
     "ipfs": "^0.50.1",
-    "it-to-stream": "^0.1.1",
+    "it-to-stream": "^0.1.2",
     "videostream": "^3.2.0"
   }
 }
diff --git a/examples/circuit-relaying/package.json b/examples/circuit-relaying/package.json
@@ -14,7 +14,7 @@
   "author": "Dmitriy Ryajov <dryajov@gmail.com>",
   "license": "MIT",
   "dependencies": {
-    "delay": "^4.3.0",
+    "delay": "^4.4.0",
     "ipfs": "^0.50.1",
     "ipfs-pubsub-room": "^2.0.1",
     "uint8arrays": "^1.1.0"

diff --git a/examples/custom-ipfs-repo/package.json b/examples/custom-ipfs-repo/package.json
@@ -13,7 +13,7 @@
     "datastore-fs": "^2.0.0",
     "ipfs": "^0.50.1",
     "ipfs-repo": "^6.0.3",
-    "it-all": "^1.0.1"
+    "it-all": "^1.0.4"
   },
   "devDependencies": {
     "execa": "^4.0.0",

diff --git a/examples/explore-ethereum-blockchain/test.js b/examples/explore-ethereum-blockchain/test.js
@@ -8,7 +8,7 @@ const df = createFactory({
   ipfsHttpModule: require('ipfs-http-client')
 }, {
   js: {
-    ipfsBin: require.resolve('ipfs/src/cli/bin.js')
+    ipfsBin: require.resolve('ipfs/src/cli.js')
   }
 })
 

diff --git a/examples/http-client-browser-pubsub/test.js b/examples/http-client-browser-pubsub/test.js
@@ -7,7 +7,7 @@ const df = createFactory({
   ipfsHttpModule: require('ipfs-http-client'),
 }, {
   js: {
-    ipfsBin: require.resolve('ipfs/src/cli/bin.js')
+    ipfsBin: require.resolve('ipfs/src/cli.js')
   },
   go: {
     ipfsBin: require('go-ipfs').path(),

diff --git a/examples/http-client-bundle-webpack/test.js b/examples/http-client-bundle-webpack/test.js
@@ -5,7 +5,7 @@ const execa = require('execa')
 const { createFactory } = require('ipfsd-ctl')
 const df = createFactory({
   ipfsHttpModule: require('ipfs-http-client'),
-  ipfsBin: require.resolve('ipfs/src/cli/bin.js')
+  ipfsBin: require.resolve('ipfs/src/cli.js')
 })
 const {
   startServer

diff --git a/examples/ipfs-101/package.json b/examples/ipfs-101/package.json
@@ -11,7 +11,7 @@
   "license": "MIT",
   "dependencies": {
     "ipfs": "^0.50.1",
-    "it-all": "^1.0.1",
+    "it-all": "^1.0.4",
     "uint8arrays": "^1.1.0"
   },
   "devDependencies": {

diff --git a/examples/running-multiple-nodes/test.js b/examples/running-multiple-nodes/test.js
@@ -25,7 +25,7 @@ async function startCliNode () {
       IPFS_PATH: repoDir
     }
   }
-  const ipfs = require.resolve('ipfs/src/cli/bin.js')
+  const ipfs = require.resolve('ipfs/src/cli.js')
 
   await execa(ipfs, ['init'], opts)
   await execa(ipfs, ['config', 'Addresses.Swarm', '--json', JSON.stringify([`/ip4/0.0.0.0/tcp/0`, `/ip4/127.0.0.1/tcp/0/ws`])], opts)

diff --git a/examples/traverse-ipld-graphs/package.json b/examples/traverse-ipld-graphs/package.json
@@ -15,7 +15,7 @@
   "dependencies": {
     "cids": "^1.0.0",
     "ipfs": "^0.50.1",
-    "ipld-block": "^0.10.0",
+    "ipld-block": "^0.10.1",
     "ipld-dag-pb": "^0.20.0",
     "multihashing-async": "^2.0.1"
   }

diff --git a/package.json b/package.json
@@ -4,6 +4,7 @@
   "description": "JavaScript implementation of the IPFS specification",
   "scripts": {
     "postinstall": "lerna bootstrap",
+    "link": "lerna link",
     "reset": "lerna run clean && rm -rf packages/*/node_modules node_modules",
     "test": "lerna run test",
     "test:node": "lerna run test:node",
@@ -14,10 +15,11 @@
     "test:electron-renderer": "lerna run test:electron-renderer",
     "test:external": "lerna run test:external",
     "test:cli": "lerna run test:cli",
-    "test:interop:node": "lerna run test:interop:node",
-    "test:interop:browser": "lerna run test:interop:browser",
-    "test:interop:electron-main": "lerna run test:interop:electron-main",
-    "test:interop:electron-renderer": "lerna run test:interop:electron-renderer",
+    "test:interop": "lerna run test:interop",
+    "test:interface:core": "lerna run test:interface:core",
+    "test:interface:http-go": "lerna run test:interface:http-go",
+    "test:interface:http-js": "lerna run test:interface:http-js",
+    "test:interface:message-port-client": "lerna run test:interface:message-port-client",
     "coverage": "lerna run coverage",
     "build": "lerna run build",
     "clean": "lerna run clean",
-Original file line number
+Diff line change
@@ Expand Up / @@ -12,7 +12,7 @@ const df = createFactory({ @@
       ipfsHttpModule: require('ipfs-http-client')
     }, {
       js: {
-        ipfsBin: require.resolve('ipfs/src/cli/bin.js')
+        ipfsBin: require.resolve('ipfs/src/cli.js')
       }
     })
     const {
@@ Expand Down @@