Merge pull request #117 from vikejs/phonzammi/dev

feat: add support for checking crawlers/bots & integrate renderToStringAsync()

examples/full/package.json

@@ -9,7 +9,7 @@
   "dependencies": {
     "node-fetch": "^3.3.2",
     "solid-js": "^1.8.21",
-    "vike": "^0.4.191",
+    "vike": "^0.4.195",
     "vike-solid": "workspace:*"
   },
   "devDependencies": {

examples/minimal/package.json

@@ -6,7 +6,7 @@
   },
   "dependencies": {
     "solid-js": "^1.8.21",
-    "vike": "^0.4.191",
+    "vike": "^0.4.195",
     "vike-solid": "workspace:*"
   },
   "devDependencies": {

examples/solid-query/.test-dev.test.ts

@@ -0,0 +1,2 @@
+import { testRun } from "./.testRun";
+testRun("pnpm run dev");

examples/solid-query/.test-preview.test.ts

@@ -0,0 +1,2 @@
+import { testRun } from "./.testRun";
+testRun("pnpm run preview");

examples/solid-query/.testRun.ts

@@ -0,0 +1,70 @@
+export { testRun };
+
+import { test, expect, run, page, partRegex, getServerUrl, autoRetry } from "@brillout/test-e2e";
+const dataHk = partRegex`data-hk=${/[0-9-]+/}`;
+
+function testRun(cmd: `pnpm run ${"dev" | "preview"}`) {
+  run(cmd);
+
+  const content = "Return of the Jedi";
+  const loading = "Loading movies...";
+  const titleDefault = "My Vike + Solid App";
+  const titleOverriden = "6 Star Wars movies";
+  const titleAsScript = `<script>document.title = "${titleOverriden}"</script>`;
+  const description = partRegex`<meta ${dataHk} name="description" content="List of 6 Star Wars movies.">`;
+  test("HTML (as user)", async () => {
+    const html = await fetchAsUser("/");
+    expect(html).toContain(content);
+    expect(html).toContain(loading);
+    expect(html).toContain(titleAsScript);
+    expect(getTitle(html)).toBe(titleDefault);
+    expect(html.split("<title>").length).toBe(2);
+    expect(html).not.toMatch(description);
+  });
+  test("HTML (as bot)", async () => {
+    const html = await fetchAsBot("/");
+    expect(html).toContain(content);
+    expect(html).not.toContain(loading);
+    expect(html).not.toContain(titleAsScript);
+    expect(getTitle(html)).toBe(titleOverriden);
+    expect(html.split("<title>").length).toBe(2);
+    expect(html).toMatch(description);
+  });
+  test("DOM", async () => {
+    await page.goto(getServerUrl() + "/");
+    const body = await page.textContent("body");
+    // Playwright seems to await the HTML stream
+    expect(body).not.toContain(loading);
+    expect(body).toContain(content);
+    await testCounter();
+  });
+}
+
+function getTitle(html: string) {
+  const title = html.match(/<title>(.*?)<\/title>/i)?.[1];
+  return title;
+}
+
+async function testCounter() {
+  // autoRetry() for awaiting client-side code loading & executing
+  await autoRetry(
+    async () => {
+      expect(await page.textContent("button")).toBe("Counter 0");
+      await page.click("button");
+      expect(await page.textContent("button")).toContain("Counter 1");
+    },
+    { timeout: 5 * 1000 },
+  );
+}
+
+async function fetchAsBot(pathname: string) {
+  return await fetchHtml(pathname, "curl/8.5.0");
+}
+async function fetchAsUser(pathname: string) {
+  return await fetchHtml(pathname, "chrome");
+}
+async function fetchHtml(pathname: string, userAgent: string) {
+  const response = await fetch(getServerUrl() + pathname, { headers: { ["User-Agent"]: userAgent } });
+  const html = await response.text();
+  return html;
+}

examples/solid-query/package.json

@@ -9,7 +9,7 @@
     "@tanstack/solid-query": "5.52.2",
     "node-fetch": "^3.3.2",
     "solid-js": "^1.8.22",
-    "vike": "^0.4.191",
+    "vike": "^0.4.195",
     "vike-solid": "workspace:^",
     "vike-solid-query": "workspace:^"
   },

examples/solid-query/pages/+config.ts

@@ -8,6 +8,6 @@ export default {
 
   passToClient: ["routeParams"],
   stream: true,
-  injectScriptsAt: "STREAM",
+  injectScriptsAt: "HTML_STREAM",
   extends: [vikeSolid, vikeSolidQuery],
 } satisfies Config;

examples/solid-query/pages/index/Movies.tsx

@@ -17,7 +17,7 @@ export function Movies() {
   };
 
   return (
-    <QueryBoundary query={query} loadingFallback={<p>Loading movies ...</p>}>
+    <QueryBoundary query={query} loadingFallback={"Loading movies..."}>
       {(movies) => (
         <>
           <Config title={`${movies.length} Star Wars movies`} />

packages/vike-solid-query/package.json

@@ -13,7 +13,7 @@
   "peerDependencies": {
     "@tanstack/solid-query": ">=5.0.0",
     "solid-js": "^1.8.7",
-    "vike-solid": ">=0.7.3"
+    "vike-solid": ">=0.7.4"
   },
   "devDependencies": {
     "@brillout/release-me": "^0.4.0",
@@ -26,8 +26,8 @@
     "solid-js": "^1.8.22",
     "tsup": "^8.2.4",
     "typescript": "^5.5.4",
-    "vike": "^0.4.193",
-    "vike-solid": "^0.7.3",
+    "vike": "^0.4.195",
+    "vike-solid": "^0.7.4",
     "vite": "5.4.2"
   },
   "exports": {

packages/vike-solid/+config.ts

@@ -6,7 +6,7 @@ import { ssrEffect } from "./integration/ssrEffect.js";
 const config = {
   name: "vike-solid",
   require: {
-    vike: ">=0.4.191",
+    vike: ">=0.4.195",
   },
 
   // https://vike.dev/onRenderHtml

packages/vike-solid/hooks/useConfig/useConfig-server.ts

@@ -1,7 +1,7 @@
 export { useConfig };
 import type { PageContext } from "vike/types";
 import type { PageContextInternal } from "../../types/PageContext.js";
-import type { ConfigFromHook } from "../../types/Config.js";
+import type { ConfigFromHook, Stream } from "../../types/Config.js";
 import { usePageContext } from "../usePageContext.js";
 import { getPageContext } from "vike/getPageContext";
 import { objectKeys } from "../../utils/objectKeys.js";
@@ -20,7 +20,14 @@ function useConfig(): (config: ConfigFromHook) => void {
 
   // Component
   pageContext = usePageContext();
-  return (config: ConfigFromHook) => setPageContextConfigFromHook(config, pageContext);
+  return (config: ConfigFromHook) => {
+    if (!pageContext._headAlreadySet) {
+      setPageContextConfigFromHook(config, pageContext);
+    } else {
+      // <head> already sent to the browser => send DOM-manipulating scripts during HTML streaming
+      apply(config, pageContext._stream!);
+    }
+  };
 }
 
 const configsClientSide = ["title"];
@@ -44,3 +51,11 @@ function setPageContextConfigFromHook(config: ConfigFromHook, pageContext: PageC
     }
   });
 }
+
+function apply(config: ConfigFromHook, stream: Stream) {
+  const { title } = config;
+  if (title) {
+    const htmlSnippet = `<script>document.title = ${JSON.stringify(title)}</script>`;
+    stream.write(htmlSnippet);
+  }
+}

packages/vike-solid/integration/onRenderHtml.tsx

@@ -1,5 +1,5 @@
 // https://vike.dev/onRenderHtml
-import { generateHydrationScript, renderToStream, renderToString } from "solid-js/web";
+import { generateHydrationScript, renderToStream, renderToString, renderToStringAsync } from "solid-js/web";
 import { dangerouslySkipEscape, escapeInject, stampPipe } from "vike/server";
 import { getHeadSetting } from "./getHeadSetting.js";
 import { getPageElement } from "./getPageElement.js";
@@ -10,6 +10,7 @@ import type { PageContextInternal } from "../types/PageContext.js";
 import type { Head } from "../types/Config.js";
 import type { JSX } from "solid-js/jsx-runtime";
 import { isCallable } from "../utils/isCallable.js";
+import isBot from "isbot-fast";
 
 export { onRenderHtml };
 
@@ -18,7 +19,7 @@ type TPipe = Parameters<typeof stampPipe>[0];
 const onRenderHtml: OnRenderHtmlAsync = async (
   pageContext: PageContextServer & PageContextInternal,
 ): ReturnType<OnRenderHtmlAsync> => {
-  const pageHtml = getPageHtml(pageContext);
+  const pageHtml = await getPageHtml(pageContext);
 
   const headHtml = getHeadHtml(pageContext);
 
@@ -40,16 +41,32 @@ const onRenderHtml: OnRenderHtmlAsync = async (
     </html>`;
 };
 
-function getPageHtml(pageContext: PageContextServer) {
+async function getPageHtml(pageContext: PageContextServer & PageContextInternal) {
   let pageHtml: string | ReturnType<typeof dangerouslySkipEscape> | TPipe = "";
+  const userAgent: string | undefined =
+    pageContext.headers?.["user-agent"] ||
+    // TODO/eventually: remove old way of acccessing the User Agent header.
+    // @ts-ignore
+    pageContext.userAgent;
+
   if (pageContext.Page) {
-    if (!pageContext.config.stream) {
+    if (userAgent && isBot(userAgent)) {
+      pageHtml = dangerouslySkipEscape(await renderToStringAsync(() => getPageElement(pageContext)));
+    } else if (!pageContext.config.stream) {
       pageHtml = dangerouslySkipEscape(renderToString(() => getPageElement(pageContext)));
     } else if (pageContext.config.stream === "web") {
-      pageHtml = renderToStream(() => getPageElement(pageContext)).pipeTo;
+      pageHtml = renderToStream(() => getPageElement(pageContext), {
+        onCompleteShell(info) {
+          pageContext._stream ??= info;
+        },
+      }).pipeTo;
       stampPipe(pageHtml, "web-stream");
     } else {
-      pageHtml = renderToStream(() => getPageElement(pageContext)).pipe;
+      pageHtml = renderToStream(() => getPageElement(pageContext), {
+        onCompleteShell(info) {
+          pageContext._stream ??= info;
+        },
+      }).pipe;
       stampPipe(pageHtml, "node-stream");
     }
   }

packages/vike-solid/package.json

@@ -11,11 +11,12 @@
     "release:commit": "LANG=en_US release-me commit"
   },
   "dependencies": {
+    "isbot-fast": "^1.2.0",
     "vite-plugin-solid": "^2.10.2"
   },
   "peerDependencies": {
     "solid-js": "^1.8.7",
-    "vike": ">=0.4.191",
+    "vike": ">=0.4.195",
     "vite": ">=5.0.0"
   },
   "devDependencies": {
@@ -34,7 +35,7 @@
     "solid-js": "^1.8.22",
     "tslib": "^2.7.0",
     "typescript": "^5.5.4",
-    "vike": "^0.4.193",
+    "vike": "^0.4.195",
     "vite": "^5.4.2"
   },
   "exports": {

packages/vike-solid/types/Config.ts

@@ -186,3 +186,4 @@ export type ConfigFromHook = PickWithoutGetter<
 >;
 export type ConfigFromHookResolved = Omit<ConfigFromHook, ConfigsCumulative> &
   Pick<Vike.ConfigResolved, ConfigsCumulative>;
+export type Stream = { write: (v: string) => void };

packages/vike-solid/types/PageContext.ts

@@ -1,5 +1,5 @@
 import type { JSX } from "solid-js";
-import type { ConfigFromHookResolved } from "./Config";
+import type { ConfigFromHookResolved, Stream } from "./Config";
 
 // https://vike.dev/pageContext#typescript
 declare global {
@@ -15,4 +15,5 @@ declare global {
 export type PageContextInternal = {
   _configFromHook?: ConfigFromHookResolved;
   _headAlreadySet?: boolean;
+  _stream?: Stream;
 };

packages/vike-solid/types/isBot.d.ts

@@ -0,0 +1,4 @@
+declare module "isbot-fast" {
+  function isBot(userAgent: string): boolean;
+  export = isBot;
+}