#487176 added robots.txt implementation (#946)

* #487176 added robots.txt implementation

Added robots.txt implementation, services for sitercore-jss/sitecore-jss-nextjs, sample for create-sitecore-jss

* #487176: refactoring and fixed comments

* #487176: added site.js and site.d.ts

* #487176 added robots.txt implementation

Added robots.txt implementation, services for sitercore-jss/sitecore-jss-nextjs, sample for create-sitecore-jss

* #487176: refactoring and fixed comments

* #487176 added robots.txt implementation

Added robots.txt implementation, services for sitercore-jss/sitecore-jss-nextjs, sample for create-sitecore-jss

* #487176: refactoring and fixed comments

* #487176 added robots.txt implementation

Added robots.txt implementation, services for sitercore-jss/sitecore-jss-nextjs, sample for create-sitecore-jss

* #487176: refactoring and fixed comments

* #487176: removed unused vars in unit test

* #487176: added site.d.ts into exclude

* #487176: fixed lint errors

* #487176 added extend rewrites for next.config

* #487176 added a new line

.gitignore

@@ -22,4 +22,7 @@ lerna-debug.log
 coverage/
 /badges
 packages/*/*.tgz
-samples/
+samples/
+
+.idea
+.next

packages/create-sitecore-jss/src/templates/nextjs-sxa/src/components/pages/api/robots.ts

@@ -0,0 +1,21 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import config from 'temp/config';
+import { GraphQLRobotsService } from '@sitecore-jss/sitecore-jss-nextjs';
+
+const robotsApi = async (_req: NextApiRequest, res: NextApiResponse): Promise<void> => {
+  // Ensure response is text/html
+  res.setHeader('Content-Type', 'text/html;charset=utf-8');
+
+  // create robots graphql service
+  const robotsService = new GraphQLRobotsService({
+    endpoint: config.graphQLEndpoint,
+    apiKey: config.sitecoreApiKey,
+    siteName: config.jssAppName,
+  });
+
+  const robotsResult = await robotsService.fetchRobots();
+
+  return res.status(200).send(robotsResult);
+};
+
+export default robotsApi;

packages/create-sitecore-jss/src/templates/nextjs-sxa/src/lib/next-config/plugins/robots.js

@@ -0,0 +1,16 @@
+const robotsPlugin = (nextConfig = {}) => {
+  return Object.assign({}, nextConfig, {
+    async rewrites() {
+      return [
+        ...await nextConfig.rewrites(),
+        // robots route
+        {
+          source: '/robots.txt',
+          destination: '/api/robots',
+        },
+      ];
+    },
+  });
+};
+
+module.exports = robotsPlugin;

packages/sitecore-jss-nextjs/src/index.ts

@@ -51,6 +51,11 @@ export {
   RestDictionaryService,
   RestDictionaryServiceConfig,
 } from '@sitecore-jss/sitecore-jss/i18n';
+export {
+  RobotsQueryResult,
+  GraphQLRobotsService,
+  GraphQLRobotsServiceConfig,
+} from '@sitecore-jss/sitecore-jss/site';
 export { GraphQLRequestClient } from '@sitecore-jss/sitecore-jss';
 
 export {

packages/sitecore-jss/site.d.ts

@@ -0,0 +1 @@
+export * from './types/site/index';

packages/sitecore-jss/site.js

@@ -0,0 +1 @@
+module.exports = require('./dist/cjs/site/index');

packages/sitecore-jss/src/debug.ts

@@ -27,4 +27,5 @@ export default Object.freeze({
   dictionary: debug(`${rootNamespace}:dictionary`),
   experienceEditor: debug(`${rootNamespace}:editing`),
   sitemap: debug(`${rootNamespace}:sitemap`),
+  robots: debug(`${rootNamespace}:robots`),
 });

packages/sitecore-jss/src/site/graphql-robots-service.test.ts

@@ -0,0 +1,63 @@
+import { expect } from 'chai';
+import nock from 'nock';
+import { GraphQLRobotsService, siteNameError } from './graphql-robots-service';
+
+const robotsQueryResultNull = {
+  site: {
+    siteInfo: null,
+  },
+};
+
+describe('GraphQLRobotsService', () => {
+  const endpoint = 'http://site';
+  const apiKey = 'some-api-key';
+  const siteName = 'site-name';
+
+  afterEach(() => {
+    nock.cleanAll();
+  });
+
+  const mockRobotsRequest = (siteName?: string) => {
+    nock(endpoint)
+      .post('/')
+      .reply(
+        200,
+        siteName
+          ? {
+              data: {
+                site: {
+                  siteInfo: {
+                    robots: siteName,
+                  },
+                },
+              },
+            }
+          : {
+              data: robotsQueryResultNull,
+            }
+      );
+  };
+
+  describe('Fetch robots.txt', () => {
+    it('should get error if robots.txt has empty sitename', async () => {
+      mockRobotsRequest();
+
+      const service = new GraphQLRobotsService({ endpoint, apiKey, siteName: '' });
+      await service.fetchRobots().catch((error: Error) => {
+        expect(error.message).to.equal(siteNameError);
+      });
+
+      return expect(nock.isDone()).to.be.false;
+    });
+
+    it('should get robots.txt', async () => {
+      mockRobotsRequest(siteName);
+
+      const service = new GraphQLRobotsService({ endpoint, apiKey, siteName });
+      const robots = await service.fetchRobots();
+      expect(robots).to.equal(siteName);
+
+      return expect(nock.isDone()).to.be.true;
+    });
+  });
+});

packages/sitecore-jss/src/site/graphql-robots-service.ts

@@ -0,0 +1,92 @@
+import { GraphQLClient, GraphQLRequestClient } from '../graphql';
+import debug from '../debug';
+
+// The default query for request robots.txt
+const defaultQuery = /* GraphQL */ `
+  query RobotsQuery($siteName: String!) {
+    site {
+      siteInfo(site: $siteName) {
+        robots
+      }
+    }
+  }
+`;
+
+/** @private */
+export const siteNameError = 'The siteName cannot be empty';
+
+export type GraphQLRobotsServiceConfig = {
+  /**
+   * Your Graphql endpoint
+   */
+  endpoint: string;
+  /**
+   * The API key to use for authentication
+   */
+  apiKey: string;
+  /**
+   * The JSS application name
+   */
+  siteName: string;
+};
+
+/**
+ * The schema of data returned in response to robots.txt request
+ */
+export type RobotsQueryResult = { site: { siteInfo: { robots: string } } };
+
+/**
+ * Service that fetch the robots.txt data using Sitecore's GraphQL API.
+ */
+export class GraphQLRobotsService {
+  private graphQLClient: GraphQLClient;
+
+  protected get query(): string {
+    return defaultQuery;
+  }
+
+  /**
+   * Creates an instance of graphQL robots.txt service with the provided options
+   * @param {GraphQLRobotsServiceConfig} options instance
+   */
+  constructor(public options: GraphQLRobotsServiceConfig) {
+    this.graphQLClient = this.getGraphQLClient();
+  }
+
+  /**
+   * Fetch a data of robots.txt from API
+   * @returns text of robots.txt
+   * @throws {Error} if the siteName is empty.
+   */
+  async fetchRobots(): Promise<string> {
+    const siteName: string = this.options.siteName;
+
+    if (!siteName) {
+      throw new Error(siteNameError);
+    }
+
+    const robotsResult: Promise<RobotsQueryResult> = this.graphQLClient.request(this.query, {
+      siteName,
+    });
+    try {
+      return robotsResult.then((result: RobotsQueryResult) => {
+        return result?.site?.siteInfo?.robots;
+      });
+    } catch (e) {
+      return Promise.reject(e);
+    }
+  }
+
+  /**
+   * Gets a GraphQL client that can make requests to the API. Uses graphql-request as the default
+   * library for fetching graphql data (@see GraphQLRequestClient). Override this method if you
+   * want to use something else.
+   * @returns {GraphQLClient} implementation
+   */
+  protected getGraphQLClient(): GraphQLClient {
+    return new GraphQLRequestClient(this.options.endpoint, {
+      apiKey: this.options.apiKey,
+      debugger: debug.robots,
+    });
+  }
+}

packages/sitecore-jss/src/site/index.ts

@@ -0,0 +1,5 @@
+export {
+  RobotsQueryResult,
+  GraphQLRobotsService,
+  GraphQLRobotsServiceConfig,
+} from './graphql-robots-service';

packages/sitecore-jss/tsconfig.json

@@ -37,6 +37,7 @@
     "graphql.d.ts",
     "i18n.d.ts",
     "tracking.d.ts",
+    "site.d.ts",
     "src/**/*.test.ts",
     "src/testData/*",
   ]