Upgrade Framework to Yarn v3 (redwoodjs#3154)

* setup and install yarn berry

* futz'n to get it work'n

* upgrade @babel/parser

* trying to run eslint to from tasks/framework-tools

* upgrade to yarn v3.0.0

* upgrade all task projects

* e2e install fix

* fix lint and tooling scripts

* lint fix

* changes per review comments

* fix test

* use yarn cache GH workflow

* upgrade cypress

* fix local cypress run-e2e

* stashed: experimenting with execa process

* enable preferInteractive

I ran yarn dlx doctor (something like that), and we're relying on
hoisting an awful lot.
preferInteractive will help us clean things up as we add deps.

* Update yarn v3.0.2

yarn dlx @yarnpkgs/doctor wasn't working for me with yarn v3.0.1 and the version of the workspace plugin we had

* Revert "stashed: experimenting with execa process"

This reverts commit fe20ee1.

* update yarn.lock files

* build:test-project working 🚀

remove comment

* use demandCommand instead

* draft code for subDir check

* Add SubDir check

* add Yarn TypeScript plugin

* remove cross-env from yarn scripts

* GH Actions: fix command, add (update) yarn caching

* update yarn.lock

* fix yarn cache on CI

* added yarn constraints plugin

* CI fix

another CI fix attempt

CI: trying more things

might be the one

nope

* Update global declaration for useAuth

* Move auth global type augmentation to ambient.d.ts

* Also update web tsconfig to use ambient

* Also do ambient type thingies for router

* misc TS fixes and lint cleanup

* CI: add yarn constraints and dedupe checks

* allow package scripts build and test

* fix dependency constraintes dedupe

* update yarn.lock

* fix build TS and codemod package errors

* update yarn.lock

* use jscodeshift in root

* remove jscodeshift from root package.json

* align loose option for babel plugins

* add yarn seciton to contributing

* add babel plugin

* update contributing toc

* update Yarn section in CONTRIBUTING

* add yarn cache to GH Workflow publish canary

* move dep to devDep per comment review

Co-authored-by: jtoar <dominic.saadi@hey.com>
Co-authored-by: Daniel Choudhury <dannychoudhury@gmail.com>

.github/workflows/build-eslint-jest.yaml

@@ -21,14 +21,28 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
-      # - name: Cache "node_modules"
-      #   uses: actions/cache@v2
-      #   with:
-      #     path: '**/node_modules'
-      #     key: node_modules_${{ runner.os }}_${{ hashFiles('**/yarn.lock') }}
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn config get cacheFolder)"
+
+      - name: Cache yarn
+        uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: yarn-${{ hashFiles('yarn.lock') }}
+          restore-keys: |
+            yarn-
 
       - name: Install dependencies
-        run: yarn install --frozen-lockfile
+        run: yarn install --immutable
+
+      - name: 'Check Yarn constraints (fix w/ "yarn constraints --fix")'
+        run: yarn constraints
+
+      - name: 'Check for duplicate dependencies (fix w/ "yarn dedupe")'
+        if: steps.yarn-cache.outputs.cache-hit != 'true'
+        run: yarn dedupe --check
 
       - name: Build
         run: yarn build

.github/workflows/create-pr-artifact.yaml

@@ -21,7 +21,7 @@ jobs:
           node-version: 14
 
       - name: Install dependencies
-        run: yarn install --frozen-lockfile
+        run: yarn install --immutable
 
       - name: Build
         run: yarn build

.github/workflows/e2e.yaml

@@ -23,14 +23,21 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
-      - name: Cache "node_modules"
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn config get cacheFolder)"
+
+      - name: Cache yarn
         uses: actions/cache@v2
+        id: yarn-cache
         with:
-          path: '**/node_modules'
-          key: node_modules_${{ runner.os }}_${{ hashFiles('**/yarn.lock') }}
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: yarn-${{ hashFiles('yarn.lock') }}
+          restore-keys: |
+            yarn-
 
-      - name: Install framework modules
-        run: yarn install --frozen-lockfile
+      - name: Install dependencies
+        run: yarn install --immutable
 
       - name: Create a temporary directory
         id: createpath

.github/workflows/publish-npm-canary.yaml

@@ -18,12 +18,34 @@ jobs:
         uses: actions/setup-node@v2
         with:
           node-version: '14'
+
       - name: Print node and yarn versions
         run: |
           node --version
           yarn --version
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn config get cacheFolder)"
+
+      - name: Cache yarn
+        uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: yarn-${{ hashFiles('yarn.lock') }}
+          restore-keys: |
+            yarn-
+
       - name: Install dependencies
-        run: yarn install --frozen-lockfile --check-files
+        run: yarn install --immutable
+
+      - name: 'Check Yarn constraints (fix w/ "yarn constraints --fix")'
+        run: yarn constraints
+
+      - name: 'Check for duplicate dependencies (fix w/ "yarn dedupe")'
+        if: steps.yarn-cache.outputs.cache-hit != 'true'
+        run: yarn dedupe --check
 
       - name: Build
         run: yarn build

.github/workflows/publish-npm-rc.yaml

@@ -21,7 +21,7 @@ jobs:
           node-version: '14'
 
       - name: Install dependencies
-        run: yarn install --frozen-lockfile --check-files
+        run: yarn install --immutable --check-files
 
       - name: Build
         run: yarn build

.gitignore

@@ -11,3 +11,11 @@ tasks/e2e/cypress/fixtures/example.json
 tmp/
 .vscode/*
 blog-test-project/*
+.yarn/*
+**/.yarn/install-state.gz
+!.yarn/patches
+!.yarn/releases
+!.yarn/plugins
+!.yarn/sdks
+!.yarn/versions
+.pnp.*

.yarnrc.yml

@@ -0,0 +1,19 @@
+compressionLevel: 0
+
+enableGlobalCache: true
+
+nmMode: hardlinks-local
+
+nodeLinker: node-modules
+
+plugins:
+  - path: .yarn/plugins/@yarnpkg/plugin-workspace-tools.cjs
+    spec: "@yarnpkg/plugin-workspace-tools"
+  - path: .yarn/plugins/@yarnpkg/plugin-typescript.cjs
+    spec: "@yarnpkg/plugin-typescript"
+  - path: .yarn/plugins/@yarnpkg/plugin-constraints.cjs
+    spec: "@yarnpkg/plugin-constraints"
+
+preferInteractive: true
+
+yarnPath: .yarn/releases/yarn-3.0.2.cjs

CONTRIBUTING.md

@@ -6,6 +6,7 @@ Before interacting with the Redwood community, please read and understand our [C
 
 **Table of Contents**
 
+- [Contributing](#contributing)
   - [Code Organization](#code-organization)
   - [Local Setup](#local-setup)
     - [Redwood Framework](#redwood-framework)
@@ -15,7 +16,14 @@ Before interacting with the Redwood community, please read and understand our [C
   - [Browser-based Setup](#browser-based-setup)
   - [Integration Tests](#integration-tests)
   - [Releases](#releases)
-    - [Troubleshooting](#troubleshooting)
+  - [Yarn v3: Tips and Troubleshooting](#yarn-v3-tips-and-troubleshooting)
+    - [Migrating from yarn v1 to yarn v3](#migrating-from-yarn-v1-to-yarn-v3)
+    - [New Yarn Commands and Utilities](#new-yarn-commands-and-utilities)
+    - [Added to CI: dedupe and constraints](#added-to-ci-dedupe-and-constraints)
+    - [About Yarn v3](#about-yarn-v3)
+      - [Benefits](#benefits)
+      - [New Files](#new-files)
+      - [Advanced Cases](#advanced-cases)
 
 ## Code Organization
 
@@ -196,6 +204,95 @@ This...
   3) Commits, Tags, and Pushes to GH
   4) and finally publishes all packages to NPM.
 
-### Troubleshooting
-
 If something went wrong you can use `yarn lerna publish from-package` to publish the packages that aren't already in the registry.
+
+## Yarn v3: Tips and Troubleshooting
+
+### Migrating from yarn v1 to yarn v3
+As of `v0.37`, the Redwood Framework has moved from yarn `v1` to yarn `v3`.
+
+If you already have a local copy of the Redwood Framework, or if you're switching between branches that are using different versions, **you'll have to run**:
+```
+git clean -fxd -e .env
+yarn install
+```
+...and then you'll be good to go.
+
+> Note: Yarn v3 is installed in the directory, while Yarn v1 is installed globally. This allows us to switch as needed per branch and/or project.
+### New Yarn Commands and Utilities
+**`yarn add --interactive`**
+Reuse the specified package from other workspaces in the project. Example:
+```
+yarn workspace create-redwood-app add -i rimraf
+```
+
+> Note: Interactivity is enabled by default
+
+For example, if we're using `yarn add` to add a dependency to a workspace (say `packages/codemods`), and we already have that dependency in another worksapce (say `packages/api-server`), yarn will ask us if we want to use the same version:
+
+```
+redwood/packages/codemods$ yarn add yargs
+? Which range do you want to use? …
+❯ Reuse yargs@16.2.0 (originally used by @redwoodjs/api-server@0.37.2 and 2 others)
+  Use yargs@^17.2.1 (resolved from latest)
+```
+
+**`yarn workspaces foreach ...`**
+This is a command from the workspaces plugin. Runs the command across all workspaces. Example:
+```
+yarn workspaces foreach -i  -v some-package
+```
+-v: outputs the package name the command is currently running against
+
+### Added to CI: dedupe and constraints
+**`yarn dedupe --check`**
+> Duplicates are defined as descriptors with overlapping ranges being resolved and
+locked to different locators. They are a natural consequence of Yarn's
+deterministic installs, but they can sometimes pile up and unnecessarily
+increase the size of your project.
+> This command dedupes dependencies in the current project using different
+strategies (only one is implemented at the moment):
+
+**`yarn constraints`**
+See new file `constraints.pro` for repo config
+- https://yarnpkg.com/features/constraints
+- Reference from Babel project: https://github.com/babel/babel/blob/main/constraints.pro
+
+### About Yarn v3
+Aside from a few plugins, we aren't using most of it's advanced features (like [PnP](https://yarnpkg.com/features/pnp)) yet.
+
+So besides the output in your terminal looking different, not much else is.
+
+> We may explore things like PnP in the future.
+> We just have to take things one step at a time since we're trying to release `v1`.
+#### Benefits
+
+Some of the benefit yarn `v3` brings us as we prepare for `v1` are:
+
+- faster CI (up to 50% faster)
+  - most importantly, no more Windows timeouts!
+- faster yarn installs
+- better dependency guarantees
+
+One of the most significant changes in yarn `v3` is it's stance on [hoisting](https://yarnpkg.com/advanced/lexicon/#hoisting).
+
+#### New Files
+
+Yarn `v3` keeps more of itself in the repo than before.
+For exmaple, yarn `v3` isn't installed globally, but on a per-project basis.
+
+Here's a quick overview of some of the new yarn-related files in this repo:
+
+| File             | Description                                                          |
+|:-----------------|:---------------------------------------------------------------------|
+| `.yarnrc.yml`    | Yarn's configuration file                                            |
+| `.yarn/plugins`  | Where installed [plugins](https://yarnpkg.com/features/plugins) live |
+| `.yarn/releases` | The `yarn v3` binaries themselves                                    |
+
+#### Advanced Cases
+If needed, there's more information in [this PR #3154 comment](https://github.com/redwoodjs/redwood/pull/3154#issue-957115489) about special cases:
+- "Binary hoisting" is no longer allowed
+- Specifying Yarn v1 binary (when necessary)
+- `yarn dlx`
+- Set `YARN_IGNORE_PATH=1` to ignore local yarn version settings.
+- how "postinstall" script works

babel.config.js

@@ -51,8 +51,10 @@ module.exports = {
      * https://github.com/tc39/proposal-decorators
      **/
     ['@babel/plugin-proposal-decorators', { legacy: true }],
+    // The "loose" option must be the same for all three of these plugins.
     ['@babel/plugin-proposal-class-properties', { loose: true }],
     ['@babel/plugin-proposal-private-methods', { loose: true }],
+    ['@babel/plugin-proposal-private-property-in-object', { loose: true }],
   ],
   overrides: [
     // ** WEB PACKAGES **

constraints.pro

@@ -0,0 +1,14 @@
+% Yarn Constraints https://yarnpkg.com/features/constraints
+% check with "yarn constraints" (fix w/ "yarn constraints --fix")
+% reference for other constraints: https://github.com/babel/babel/blob/main/constraints.pro
+
+% Enforces that a dependency doesn't appear in both `dependencies` and `devDependencies`
+gen_enforced_dependency(WorkspaceCwd, DependencyIdent, null, 'devDependencies') :-
+  workspace_has_dependency(WorkspaceCwd, DependencyIdent, _, 'devDependencies'),
+  workspace_has_dependency(WorkspaceCwd, DependencyIdent, _, 'dependencies').
+
+% Prevent two workspaces from depending on conflicting versions of a same dependency
+gen_enforced_dependency(WorkspaceCwd, DependencyIdent, DependencyRange2, DependencyType) :-
+  workspace_has_dependency(WorkspaceCwd, DependencyIdent, DependencyRange, DependencyType),
+  workspace_has_dependency(OtherWorkspaceCwd, DependencyIdent, DependencyRange2, DependencyType2),
+  DependencyRange \= DependencyRange2.

package.json

@@ -7,7 +7,13 @@
     "packages/*"
   ],
   "devDependencies": {
+    "@babel/cli": "7.15.4",
+    "@babel/node": "7.15.4",
+    "eslint": "7.32.0",
+    "jest": "27.2.0",
     "npm-packlist": "3.0.0",
+    "rimraf": "3.0.2",
+    "typescript": "4.4.3",
     "typescript-transform-paths": "3.3.1"
   },
   "resolutions": {
@@ -22,7 +28,7 @@
     "vscode-languageserver-textdocument": "1.0.1"
   },
   "scripts": {
-    "framework": "yarn --cwd ./tasks/framework-tools",
+    "framework": "yarn ./tasks/framework-tools",
     "build": "yarn framework build",
     "build:js": "yarn framework build:js",
     "build:types": "yarn framework build:types",
@@ -35,5 +41,6 @@
     "postinstall": "yarn framework install",
     "build:link": "node ./tasks/build-and-copy",
     "build:test-project": "node ./tasks/test-project/test-project"
-  }
+  },
+  "packageManager": "yarn@3.0.2"
 }

packages/api-server/package.json

@@ -13,6 +13,7 @@
   "main": "dist/handler",
   "license": "MIT",
   "dependencies": {
+    "@babel/plugin-transform-runtime": "7.15.0",
     "ansi-colors": "4.1.1",
     "body-parser": "1.19.0",
     "chokidar": "3.5.2",
@@ -28,16 +29,19 @@
     "youch-terminal": "1.1.1"
   },
   "devDependencies": {
+    "@babel/cli": "7.15.4",
     "@types/aws-lambda": "8.10.83",
     "@types/express": "4.17.13",
     "@types/lodash.escape": "4.0.6",
     "@types/morgan": "1.9.3",
     "@types/qs": "6.9.7",
-    "@types/stack-trace": "0.0.29"
+    "@types/stack-trace": "0.0.29",
+    "jest": "27.2.0",
+    "typescript": "4.4.3"
   },
   "scripts": {
     "build": "yarn build:js",
-    "prepublishOnly": "yarn cross-env NODE_ENV=production yarn build",
+    "prepublishOnly": "NODE_ENV=production yarn build",
     "build:js": "babel src -d dist --extensions \".js,.ts,.tsx\"",
     "build:watch": "nodemon --watch src --ext \"js,ts,tsx\" --ignore dist --exec \"yarn build && yarn fix:permissions\"",
     "fix:permissions": "chmod +x dist/index.js; chmod +x dist/watch.js"

packages/api/package.json

@@ -22,13 +22,16 @@
     "uuid": "8.3.2"
   },
   "devDependencies": {
+    "@babel/cli": "7.15.4",
     "@redwoodjs/auth": "0.37.2",
     "@types/crypto-js": "4.0.2",
     "@types/jsonwebtoken": "8.5.5",
     "@types/lodash.merge": "4.6.6",
     "@types/lodash.omitby": "4.6.6",
     "@types/md5": "2.3.1",
-    "split2": "3.2.2"
+    "jest": "27.2.0",
+    "split2": "3.2.2",
+    "typescript": "4.4.3"
   },
   "jest": {
     "testPathIgnorePatterns": [
@@ -37,7 +40,7 @@
   },
   "scripts": {
     "build": "yarn build:js && yarn build:types",
-    "prepublishOnly": "yarn cross-env NODE_ENV=production yarn build",
+    "prepublishOnly": "NODE_ENV=production yarn build",
     "build:js": "babel src -d dist --extensions \".js,.ts,.tsx\"",
     "build:types": "tsc --build --verbose",
     "build:watch": "nodemon --watch src --ext \"js,ts,tsx\" --ignore dist --exec \"yarn build\"",