Move /deps/node_modules to /data/olympia/node_modules

Remove django_node_assets

TMP: fix node_modules missing

Fix the static serving

.dockerignore

@@ -61,3 +61,6 @@ docker-bake.hcl
 docker-compose*.yml
 Dockerfile*
 Makefile-os
+
+# Ignore all .git directories in any subdirectory
+.git/

Dockerfile

@@ -97,14 +97,13 @@ ENV PIP_CACHE_DIR=/deps/cache/
 ENV PIP_SRC=/deps/src/
 ENV PYTHONUSERBASE=/deps
 ENV PATH=$PYTHONUSERBASE/bin:$PATH
-ENV NPM_CONFIG_PREFIX=/deps/
 ENV NPM_CACHE_DIR=/deps/cache/npm
 ENV NPM_DEBUG=true
 # Set python path to the project root and src to resolve olympia modules correctly
 ENV PYTHONPATH=${HOME}:${HOME}/src
 
 ENV PIP_COMMAND="python3 -m pip"
-ENV NPM_ARGS="--prefix ${NPM_CONFIG_PREFIX} --cache ${NPM_CACHE_DIR} --loglevel verbose"
+ENV NPM_ARGS="--cache ${NPM_CACHE_DIR} --loglevel verbose"
 
 # All we need in "base" is pip to be installed
 #this let's other layers install packages using the correct version.
@@ -135,8 +134,8 @@ RUN \
     # Files required to install pip dependencies
     --mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
     # Files required to install npm dependencies
-    --mount=type=bind,source=package.json,target=/deps/package.json \
-    --mount=type=bind,source=package-lock.json,target=/deps/package-lock.json \
+    --mount=type=bind,source=package.json,target=${HOME}/package.json \
+    --mount=type=bind,source=package-lock.json,target=${HOME}/package-lock.json \
     # Mounts for caching dependencies
     --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
     --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
@@ -176,6 +175,9 @@ RUN \
     --mount=type=bind,src=src,target=${HOME}/src \
     --mount=type=bind,src=Makefile-docker,target=${HOME}/Makefile-docker \
     --mount=type=bind,src=manage.py,target=${HOME}/manage.py \
+    --mount=type=bind,src=package.json,target=${HOME}/package.json \
+    --mount=type=bind,src=package-lock.json,target=${HOME}/package-lock.json \
+    --mount=type=bind,src=vite.config.js,target=${HOME}/vite.config.js \
 <<EOF
 echo "from olympia.lib.settings_base import *" > settings_local.py
 DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets

Makefile-docker

@@ -8,8 +8,6 @@ export PYTHON_COMMAND=python3
 export PIP_COMMAND=$(PYTHON_COMMAND) -m pip
 APP=src/olympia/
 
-NODE_MODULES := $(NPM_CONFIG_PREFIX)node_modules/
-
 REQUIRED_FILES := \
 	Makefile \
 	Makefile-os \
@@ -73,6 +71,7 @@ data_load:
 
 .PHONY: update_assets
 update_assets:
+	npm $(NPM_ARGS) run build
 	# Copy files required in compress_assets to the static folder
 	# If changing this here, make sure to adapt tests in amo/test_commands.py
 	$(PYTHON_COMMAND) manage.py compress_assets
@@ -94,7 +93,7 @@ setup-ui-tests:
 lint: ## lint the code
 	ruff check .
 	ruff format --check .
-	NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- prettier --check '**'
+	npm exec $(NPM_ARGS) -- prettier --check '**'
 	curlylint src/
 
 lint-codestyle: lint
@@ -199,15 +198,15 @@ test_failed: ## rerun the failed tests from the previous run
 
 .PHONY: run_js_tests
 run_js_tests: ## Run the JavaScript test suite (requires compiled/compressed assets).
-	NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- jest tests/js
+	npm exec $(NPM_ARGS) -- jest tests/js
 
 .PHONY: watch_js_tests
 watch_js_tests: ## Run+watch the JavaScript test suite (requires compiled/compressed assets).
-	NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- jest --watch
+	npm exec $(NPM_ARGS) -- jest --watch
 
 .PHONY: format
 format: ## Autoformat our codebase.
-	NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- prettier --write '**'
+	npm exec $(NPM_ARGS) -- prettier --write '**'
 	ruff check --fix-only .
 	ruff format .
 

docker-compose.yml

@@ -49,6 +49,16 @@ services:
       - ${HOST_MOUNT_SOURCE:?}deps:/deps
       - data_site_static:/data/olympia/site-static
       - ${HOST_MOUNT_SOURCE:?}storage:/data/olympia/storage
+
+  static:
+    <<: *olympia
+    ports:
+      - "5173:5173"
+    volumes:
+      - ${HOST_MOUNT_SOURCE:?}:/data/olympia
+      - ${HOST_MOUNT_SOURCE:?}deps:/deps
+    command: npm run dev -d
+
   worker:
     <<: *olympia
     command: [
@@ -103,8 +113,7 @@ services:
     image: nginx
     volumes:
       - data_nginx:/etc/nginx/conf.d
-      - ${HOST_MOUNT_SOURCE:?}:/srv
-      - data_site_static:/srv/site-static
+      - data_site_static:/srv/static
       - ${HOST_MOUNT_SOURCE:?}storage:/srv/storage
     ports:
       - "80:80"

docker/nginx/addons.conf

@@ -10,13 +10,27 @@ server {
         alias /srv/storage/;
     }
 
-    location /static/ {
-         alias /srv/static/;
+    # TODO: the issue is that vite is trying to import files which converts to urls
+    # that are seved by /static/ and not /static/vite/ so the issue is with how
+    # vite is configuring the urls internally.
 
-         # Fallback to the uwsgi server if the file is not found in the static files directory.
-         # This will happen for vendor files from pytnon or npm dependencies that won't be available
-         # in the static files directory.
-         error_page 404 = @olympia;
+    location /static/vite/ {
+        proxy_pass http://static:5173/static/;
+        proxy_pass_header Server;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_intercept_errors on;
+        add_header X-Served-By "static" always;
+        error_page 404 = @olympia;
+        error_page 502 = @olympia;
+    }
+
+    location /static/ {
+        root /srv;
+        try_files $uri @olympia;
+        add_header X-Served-By "nginx-direct" always;
     }
 
     location /user-media/ {
@@ -50,6 +64,8 @@ server {
         uwsgi_param   X-Real-IP             $remote_addr;
         uwsgi_param   X-Forwarded-For       $proxy_add_x_forwarded_for;
         uwsgi_param   X-Forwarded-Protocol  ssl;
+
+        add_header X-Served-By "olympia" always;
     }
 
     location @frontendamo {

docs/topics/development/dependency_management.md

@@ -66,9 +66,7 @@ Ensure to comment in the requirements file above transitive dependencies which d
 
 ## Node.js Dependencies
 
-Node.js dependencies are managed using npm. Similar to Python dependencies, Node.js dependencies are installed into the `/deps` directory.
-
-- **Environment Variables**: Environment variables are set for Node.js CLIs to ensure that dependencies are installed in the `/deps` directory. This includes setting paths for `NPM_CONFIG_PREFIX` and `NPM_CACHE_DIR`.
+Node.js dependencies are managed using npm.
 
 - **Caching Mechanism**: Node.js dependencies are also cached using Docker build stages. Internal npm cache folders are cached to avoid re-downloading packages unnecessarily.
 

package.json

@@ -5,6 +5,12 @@
   "engines": {
     "node": ">= 18.18"
   },
+  "type": "module",
+  "scripts": {
+    "dev": "vite -d",
+    "build": "vite build",
+    "preview": "vite preview"
+  },
   "dependencies": {
     "@claviska/jquery-minicolors": "2.3.6",
     "addons-linter": "7.6.0",
@@ -31,6 +37,7 @@
     "jest-environment-jsdom": "^29.7.0",
     "lodash": "^4.17.21",
     "prettier": "^3.3.3",
-    "terser": "^5.36.0"
+    "terser": "^5.36.0",
+    "vite": "^6.0.3"
   }
 }

requirements/prod.txt

@@ -1205,6 +1205,9 @@ watchdog[watchmedo]==3.0.0 \
     --hash=sha256:c9d8c8ec7efb887333cf71e328e39cffbf771d8f8f95d308ea4125bf5f90ba64 \
     --hash=sha256:d00e6be486affb5781468457b21a6cbe848c33ef43f9ea4a73b4882e5f188a44 \
     --hash=sha256:d429c2430c93b7903914e4db9a966c7f2b068dd2ebdd2fa9b9ce094c7d459f33
+django-vite==3.0.5 \
+    --hash=sha256:049b74f38c999cbfcf0e2c21b254c2e059bb97bfd7e4049caf2d0f9fba0b482f \
+    --hash=sha256:431c1212e7627adc20666d150578f1a8983f043e90f3905778fb3c5c0ffe6963
 django-node-assets==0.9.14 \
     --hash=sha256:80cbe3d10521808309712b2aa5ef6d69799bbcafef844cf7f223d3c93f405768 \
     --hash=sha256:d5b5c472136084d533268f52ab77897327863a102e25c81f484aae85eb806987

scripts/install_deps.py

@@ -6,14 +6,14 @@
 import sys
 
 
-def copy_package_json():
-    """Copy package.json files to deps directory if they exist."""
-    try:
-        shutil.copy('/data/olympia/package.json', '/deps')
-        shutil.copy('/data/olympia/package-lock.json', '/deps')
-    except (IOError, OSError):
-        pass  # Ignore if files don't exist or can't be copied
+def clean_dir(dir_path, filter):
+    if not os.path.exists(dir_path):
+        return
 
+    for item in os.listdir(dir_path):
+        item_path = os.path.join(dir_path, item)
+        if os.path.isdir(item_path) and item not in filter:
+            shutil.rmtree(item_path)
 
 def main(targets):
     # Constants
@@ -38,16 +38,11 @@ def main(targets):
     # installed or in the host ./deps directory before running this script
     if 'local' not in DOCKER_TAG or OLYMPIA_DEPS == 'production':
         print('Removing existing deps')
-        for item in os.listdir('/deps'):
-            item_path = os.path.join('/deps', item)
-            if os.path.isdir(item_path) and item != 'cache':
-                shutil.rmtree(item_path)
+        clean_dir('/deps', ['cache'])
+        clean_dir('/data/olympia/node_modules', [])
     else:
         print('Updating existing deps')
 
-    # Copy package.json files
-    copy_package_json()
-
     # Prepare the includes lists
     pip_includes = []
     npm_includes = []

settings.py

@@ -9,6 +9,7 @@
 import os
 from urllib.parse import urlparse
 
+from olympia.core.utils import get_version_json
 from olympia.lib.settings_base import *  # noqa
 
 
@@ -198,3 +199,11 @@ def insert_debug_toolbar_middleware(middlewares):
 }
 
 ENABLE_ADMIN_MLBF_UPLOAD = True
+
+print('DJANGO_VITE banana', get_version_json().get('target'))
+
+
+# Override the dev mode from the base settings depending on if we are in a production image
+DJANGO_VITE_DEVMODE = get_version_json().get('target') != 'production'
+DJANGO_VITE["default"]["dev_mode"] = DJANGO_VITE_DEVMODE
+DJANGO_VITE["default"]["static_url_prefix"] = "/static/vite/" if DJANGO_VITE_DEVMODE else ""

src/olympia/core/apps.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import os
 import subprocess
@@ -120,6 +121,29 @@ def static_check(app_configs, **kwargs):
             )
         )
 
+    if not os.path.exists(settings.STATIC_BUILD_MANIFEST_PATH):
+        errors.append(
+            Error(
+                f'Static build manifest file does not exist: {settings.STATIC_BUILD_MANIFEST_PATH}',
+                id='setup.E003',
+            )
+        )
+    else:
+        with open(settings.DJANGO_VITE['default']['manifest_path'], 'r') as f:
+            manifest = json.load(f)
+
+            for name, asset in manifest.items():
+                # Assets compiled by vite are in the static root directory
+                # after running collectstatic. So we should look there.
+                path = os.path.join(settings.STATIC_ROOT, asset['file'])
+                if not os.path.exists(path):
+                    errors.append(
+                        Error(
+                            f'Static asset {name} does not exist at expected path: {path}',
+                            id='setup.E003',
+                        )
+                    )
+
     return errors
 
 

src/olympia/devhub/templates/devhub/index.html

@@ -3,6 +3,10 @@
 <!DOCTYPE html>
 <html lang="{{ LANG }}" dir="{{ DIR }}">
   <head>
+    <!-- vite -->
+    {{ vite_hmr_client() }}
+    # TODO: cannot resolve the asset in prod mode
+    {{ vite_asset('js/blue.js') }}
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     {% if not settings.ENGAGE_ROBOTS %}

src/olympia/lib/settings_base.py

@@ -98,15 +98,15 @@ def path(*folders):
 # LESS CSS OPTIONS (Debug only).
 LESS_PREPROCESS = True  # Compile LESS with Node, rather than client-side JS?
 LESS_LIVE_REFRESH = False  # Refresh the CSS on save?
-LESS_BIN = env('LESS_BIN', default='/deps/node_modules/less/bin/lessc')
+LESS_BIN = env('LESS_BIN', default=path('node_modules/less/bin/lessc'))
 
 # Path to cleancss (our CSS minifier).
 CLEANCSS_BIN = env(
-    'CLEANCSS_BIN', default='/deps/node_modules/clean-css-cli/bin/cleancss'
+    'CLEANCSS_BIN', default=path('node_modules/clean-css-cli/bin/cleancss')
 )
 
 # Path to our JS minifier.
-JS_MINIFIER_BIN = env('JS_MINIFIER_BIN', default='/deps/node_modules/terser/bin/terser')
+JS_MINIFIER_BIN = env('JS_MINIFIER_BIN', default=path('node_modules/terser/bin/terser'))
 
 # rsvg-convert is used to save our svg static theme previews to png
 RSVG_CONVERT_BIN = env('RSVG_CONVERT_BIN', default='rsvg-convert')
@@ -116,7 +116,7 @@ def path(*folders):
 
 # Path to our addons-linter binary
 ADDONS_LINTER_BIN = env(
-    'ADDONS_LINTER_BIN', default='/deps/node_modules/addons-linter/bin/addons-linter'
+    'ADDONS_LINTER_BIN', default=path('node_modules/addons-linter/bin/addons-linter')
 )
 # --enable-background-service-worker linter flag value
 ADDONS_LINTER_ENABLE_SERVICE_WORKER = False
@@ -376,6 +376,10 @@ def get_db_config(environ_var, atomic_requests=True):
             path('src/olympia/templates'),
         ),
         'OPTIONS': {
+            'globals': {
+                'vite_hmr_client': 'django_vite.templatetags.django_vite.vite_hmr_client',
+                'vite_asset': 'django_vite.templatetags.django_vite.vite_asset',
+            },
             # http://jinja.pocoo.org/docs/dev/extensions/#newstyle-gettext
             'newstyle_gettext': True,
             # Match our regular .html and .txt file endings except
@@ -562,6 +566,7 @@ def get_db_config(environ_var, atomic_requests=True):
     'rangefilter',
     'django_recaptcha',
     'drf_yasg',
+    'django_vite',
     'django_node_assets',
     # Django contrib apps
     'django.contrib.admin',
@@ -1328,11 +1333,14 @@ def read_only_mode(env):
     'django_node_assets.finders.NodeModulesFinder',
 )
 
-NODE_MODULES_ROOT = os.path.join('/', 'deps', 'node_modules')
-NODE_PACKAGE_JSON = os.path.join('/', 'deps', 'package.json')
+NODE_MODULES_ROOT = path('node_modules')
+NODE_PACKAGE_JSON = path('package.json')
 NODE_PACKAGE_MANAGER_INSTALL_OPTIONS = ['--dry-run']
 
-STATIC_BUILD_PATH = os.path.join('/', 'data', 'olympia', 'static-build')
+STATIC_BUILD_PATH = path('static-build')
+# The manifest file is created in static-build but copied into the static root
+# so we should expect to find it at /<static_root/<static_build>/manifest.json
+STATIC_BUILD_MANIFEST_PATH = path(STATIC_BUILD_PATH, 'manifest.json')
 
 STATICFILES_DIRS = (
     path('static'),
@@ -1619,3 +1627,13 @@ def read_only_mode(env):
 TESTING_ENV = False
 
 ENABLE_ADMIN_MLBF_UPLOAD = False
+
+# TODO: we need to make this work for production environments as well.
+DJANGO_VITE = {
+  "default": {
+    # Always use prod mode in the base settings. We can override this in settings.py
+    "dev_mode": False,
+    # "static_url_prefix": "",
+    "manifest_path": STATIC_BUILD_MANIFEST_PATH,
+  }
+}

static/css/index.css

@@ -0,0 +1,3 @@
+* {
+  color: red;
+}

static/css/index.css.js

@@ -0,0 +1 @@
+import './index.css';

static/js/blue.css

@@ -0,0 +1,3 @@
+* {
+  color: blue !important;
+}