feat: introduce audit for broken backlinks SITES-18417

src/backlinks/handler.js

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import {
+  internalServerError, noContent, notFound, ok,
+} from '@adobe/spacecat-shared-http-utils';
+import AhrefsAPIClient from '../support/ahrefs-client.js';
+import { retrieveSiteBySiteId } from '../utils/data-access.js';
+
+export default async function auditBrokenBacklinks(message, context) {
+  const { type, url: siteId, auditContext } = message;
+  const { dataAccess, log, sqs } = context;
+  const {
+    AUDIT_RESULTS_QUEUE_URL: queueUrl,
+  } = context.env;
+
+  try {
+    log.info(`Received ${type} audit request for siteId: ${siteId}`);
+
+    const site = await retrieveSiteBySiteId(dataAccess, siteId, log);
+    if (!site) {
+      return notFound('Site not found');
+    }
+
+    const auditConfig = site.getAuditConfig();
+    if (auditConfig.auditsDisabled()) {
+      log.info(`Audits disabled for site ${siteId}`);
+      return ok();
+    }
+
+    if (auditConfig.getAuditTypeConfig(type)?.disabled()) {
+      log.info(`Audit type ${type} disabled for site ${siteId}`);
+      return ok();
+    }
+
+    const ahrefsAPIClient = AhrefsAPIClient.createFrom(context);
+    const data = await ahrefsAPIClient.getBrokenBacklinks(siteId);
+
+    const auditResult = {
+      brokenBacklinks: data.backlinks,
+    };
+    const auditData = {
+      siteId: site.getId(),
+      auditedAt: new Date().toISOString(),
+      auditResult,
+    };
+
+    await dataAccess.addAudit(auditData);
+
+    await sqs.sendMessage(queueUrl, {
+      type,
+      url: siteId,
+      auditContext,
+      auditResult,
+    });
+
+    log.info(`Successfully audited ${siteId} for ${type} type audit`);
+    return noContent();
+  } catch (e) {
+    return internalServerError(`Internal server error: ${e.message}`);
+  }
+}

src/index.js

@@ -21,13 +21,15 @@ import apex from './apex/handler.js';
 import cwv from './cwv/handler.js';
 import lhs from './lhs/handler.js';
 import notfound from './notfound/handler.js';
+import backlinks from './backlinks/handler.js';
 
 const HANDLERS = {
   apex,
   cwv,
   'lhs-mobile': lhs,
   'lhs-desktop': lhs,
   404: notfound,
+  'broken-backlinks': backlinks,
 };
 
 /**

src/support/ahrefs-client.js

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { isValidUrl } from '@adobe/spacecat-shared-utils';
+import { fetch } from './utils.js';
+
+export default class AhrefsAPIClient {
+  static createFrom(context) {
+    const { AHREFS_API_BASE_URL: apiBaseUrl, AHREFS_API_KEY: apiKey } = context.env;
+    return new AhrefsAPIClient({ apiBaseUrl, apiKey });
+  }
+
+  constructor(config) {
+    const { apiKey, apiBaseUrl } = config;
+
+    if (!isValidUrl(apiBaseUrl)) {
+      throw new Error(`Invalid Ahrefs API Base URL: ${apiBaseUrl}`);
+    }
+
+    this.apiBaseUrl = apiBaseUrl;
+    this.apiKey = apiKey;
+  }
+
+  async sendRequest(endpoint, queryParams = {}) {
+    const queryParamsKeys = Object.keys(queryParams);
+    const queryString = queryParamsKeys.length > 0
+      ? `?${queryParamsKeys
+        .map((key) => `${encodeURIComponent(key)}=${encodeURIComponent(queryParams[key])}`)
+        .join('&')}` : '';
+
+    const response = await fetch(`${this.apiBaseUrl}${endpoint}${queryString}`, {
+      method: 'GET',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${this.apiKey}`,
+      },
+    });
+
+    if (!response.ok) {
+      throw new Error(`Ahrefs API request failed with status: ${response.status}`);
+    }
+
+    try {
+      return await response.json();
+    } catch (e) {
+      throw new Error(`Error parsing Ahrefs API response: ${e.message}`);
+    }
+  }
+
+  async getBrokenBacklinks(url) {
+    const filter = {
+      and: [
+        { field: 'is_dofollow', is: ['eq', 1] },
+        { field: 'is_content', is: ['eq', 1] },
+        { field: 'domain_rating_source', is: ['gte', 29.5] },
+        { field: 'traffic_domain', is: ['gte', 500] },
+        { field: 'links_external', is: ['lte', 300] },
+      ],
+    };
+
+    const queryParams = {
+      select: [
+        'title',
+        'url_from',
+        'url_to',
+      ].join(','),
+      limit: 50,
+      mode: 'prefix',
+      order_by: 'traffic_domain:desc',
+      target: url,
+      output: 'json',
+      where: JSON.stringify(filter),
+    };
+
+    return this.sendRequest('/site-explorer/broken-backlinks', queryParams);
+  }
+}

test/audits/backlinks.test.js

@@ -0,0 +1,154 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/* eslint-env mocha */
+
+import { createSite } from '@adobe/spacecat-shared-data-access/src/models/site.js';
+
+import chai from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+import sinon from 'sinon';
+import sinonChai from 'sinon-chai';
+import nock from 'nock';
+import auditBrokenBacklinks from '../../src/backlinks/handler.js';
+
+chai.use(sinonChai);
+chai.use(chaiAsPromised);
+const { expect } = chai;
+
+describe('Backlinks Tests', () => {
+  let message;
+  let context;
+  let mockLog;
+  let mockDataAccess;
+
+  const sandbox = sinon.createSandbox();
+
+  const siteData = {
+    id: 'site1',
+    baseURL: 'https://adobe.com',
+  };
+
+  const site = createSite(siteData);
+
+  const auditResult = {
+    backlinks: [
+      {
+        title: 'backlink title',
+        url_from: 'url-from',
+        url_to: 'url-to',
+      },
+      {
+        title: 'backlink title 2',
+        url_from: 'url-from-2',
+        url_to: 'url-to-2',
+      },
+    ],
+  };
+
+  beforeEach(() => {
+    mockDataAccess = {
+      getSiteByID: sinon.stub().resolves(site),
+      addAudit: sinon.stub(),
+    };
+
+    message = {
+      type: 'broken-backlinks',
+      url: 'site1',
+    };
+
+    mockLog = {
+      info: sinon.spy(),
+      warn: sinon.spy(),
+      error: sinon.spy(),
+    };
+
+    context = {
+      log: mockLog,
+      env: {
+        AHREFS_API_BASE_URL: 'https://ahrefs.com',
+        AHREFS_API_KEY: 'ahrefs-token',
+        AUDIT_RESULTS_QUEUE_URL: 'queueUrl',
+      },
+      dataAccess: mockDataAccess,
+      sqs: {
+        sendMessage: sandbox.stub().resolves(),
+      },
+    };
+  });
+
+  afterEach(() => {
+    nock.cleanAll();
+    sinon.restore();
+  });
+
+  it('should successfully perform an audit to detect broken backlinks', async () => {
+    nock('https://ahrefs.com')
+      .get(/.*/)
+      .reply(200, auditResult);
+
+    const response = await auditBrokenBacklinks(message, context);
+
+    expect(response.status).to.equal(204);
+    expect(mockDataAccess.addAudit).to.have.been.calledOnce;
+    expect(context.sqs.sendMessage).to.have.been.calledOnce;
+    expect(context.log.info).to.have.been.calledWith('Successfully audited site1 for broken-backlinks type audit');
+  });
+
+  it('returns a 404 when site does not exist', async () => {
+    mockDataAccess.getSiteByID.resolves(null);
+
+    const response = await auditBrokenBacklinks(message, context);
+
+    expect(response.status).to.equal(404);
+  });
+
+  it('returns a 200 when site audits are disabled', async () => {
+    const siteWithDisabledAudits = createSite({
+      ...siteData,
+      auditConfig: { auditsDisabled: true },
+    });
+
+    mockDataAccess.getSiteByID.resolves(siteWithDisabledAudits);
+
+    const response = await auditBrokenBacklinks(message, context);
+
+    expect(response.status).to.equal(200);
+    expect(mockLog.info).to.have.been.calledTwice;
+    expect(mockLog.info).to.have.been.calledWith('Audits disabled for site site1');
+  });
+
+  it('returns a 200 when audits for type are disabled', async () => {
+    const siteWithDisabledAudits = createSite({
+      ...siteData,
+      auditConfig: { auditsDisabled: false, auditTypeConfigs: { 'broken-backlinks': { disabled: true } } },
+    });
+
+    mockDataAccess.getSiteByID.resolves(siteWithDisabledAudits);
+
+    const response = await auditBrokenBacklinks(message, context);
+
+    expect(response.status).to.equal(200);
+    expect(mockLog.info).to.have.been.calledWith('Audit type broken-backlinks disabled for site site1');
+  });
+
+  it('should handle audit api errors gracefully', async () => {
+    nock('https://ahrefs.com')
+      .get(/.*/)
+      .reply(500);
+
+    const response = await auditBrokenBacklinks(message, context);
+
+    expect(response.status).to.equal(500);
+    expect(context.sqs.sendMessage).to.not.have.been.called;
+  });
+});