Fixes and improvements to meta tags

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## Unreleased
+
+### New features
+
+#### Exclude individual pages from search engine indexes
+
+You can now exclude individual pages from search engine indexes by including `prevent_indexing: true` in the frontmatter for the page.
+
+This was added in [pull request #192: Fixes and improvements to meta tags](https://github.com/alphagov/tech-docs-gem/pull/192).
+
+### Fixes
+
+We’ve made fixes to the Tech Docs Gem in the following pull requests:
+
+- [#192: Fixes and improvements to meta tags](https://github.com/alphagov/tech-docs-gem/pull/192)
+
 ## 2.0.13
 
 - [Pull request #189: Update orange code highlight colour to meet minimum AA colour contrast ratio criterion](https://github.com/alphagov/tech-docs-gem/pull/189)

example/source/index.html.md.erb

@@ -1,5 +1,6 @@
 ---
 title: GOV.UK Documentation Example
+description: Example of a documentation homepage
 old_paths:
  - /something/old-as-well.html
 ---

example/source/prevent-index-page.html.md

@@ -0,0 +1,10 @@
+---
+title: Un-indexed Page
+prevent_indexing: true
+hide_in_navigation: true
+---
+
+# Un-indexed page
+
+This page should not be indexed by search engines, because it contains a 
+`<meta name="robots" content="noindex">` tag.

lib/govuk_tech_docs/meta_tags.rb

@@ -8,12 +8,8 @@ def initialize(config, current_page)
     def tags
       all_tags = {
         "description" => page_description,
-        "og:description" => page_description,
-        "og:image" => page_image,
-        "og:site_name" => site_name,
-        "og:title" => page_title,
-        "og:type" => "object",
-        "og:url" => canonical_url,
+        "google-site-verification" => google_site_verification,
+        "robots" => robots,
         "twitter:card" => "summary",
         "twitter:domain" => URI.parse(host).host,
         "twitter:image" => page_image,
@@ -24,6 +20,21 @@ def tags
       Hash[all_tags.select { |_k, v| v }]
     end
 
+    # OpenGraph uses the non-standard property attribute instead of name, so we
+    # return these separately so we can output them correctly.
+    def opengraph_tags
+      all_opengraph_tags = {
+        "og:description" => page_description,
+        "og:image" => page_image,
+        "og:site_name" => site_name,
+        "og:title" => page_title,
+        "og:type" => "object",
+        "og:url" => canonical_url,
+      }
+
+      Hash[all_opengraph_tags.select { |_k, v| v }]
+    end
+
     def browser_title
       [page_title, site_name].select(&:present?).uniq.join(" - ")
     end
@@ -45,11 +56,19 @@ def site_name
     end
 
     def page_description
-      locals[:description] || frontmatter.description
+      locals[:description] || frontmatter[:description]
     end
 
     def page_title
-      locals[:title] || frontmatter.title
+      locals[:title] || frontmatter[:title]
+    end
+
+    def robots
+      "noindex" if config[:tech_docs][:prevent_indexing] || frontmatter[:prevent_indexing]
+    end
+
+    def google_site_verification
+      config[:tech_docs][:google_site_verification]
     end
 
     def host

lib/source/layouts/core.erb

@@ -4,21 +4,18 @@
     <meta content="IE=edge" http-equiv="X-UA-Compatible">
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
-    <% if config[:tech_docs][:prevent_indexing] %>
-      <meta name="robots" content="noindex">
-    <% end %>
 
     <title><%= meta_tags.browser_title %></title>
 
     <%= stylesheet_link_tag :manifest %>
 
     <link rel="canonical" href="<%= meta_tags.canonical_url %>">
 
-    <% if config[:tech_docs][:google_site_verification] %>
-      <meta name="google-site-verification" content="<%= config[:tech_docs][:google_site_verification] %>" />
+    <% meta_tags.tags.each do |name, content| %>
+      <%= tag :meta, name: name, content: content %>
     <% end %>
 
-    <% meta_tags.tags.each do |property, content| %>
+    <% meta_tags.opengraph_tags.each do |property, content| %>
       <%= tag :meta, property: property, content: content %>
     <% end %>
 

spec/features/integration_spec.rb

@@ -34,6 +34,9 @@
 
     when_i_view_a_page_with_no_sidebar
     then_there_is_no_sidebar
+
+    when_i_view_a_page_with_prevent_indexing
+    then_there_is_a_robots_noindex_metatag
   end
 
   def when_the_site_is_created
@@ -58,6 +61,7 @@ def then_there_is_a_sidebar
 
   def and_there_are_proper_meta_tags
     expect(page).to have_title "GOV.UK Documentation Example - My First Service"
+    expect(page).to have_css 'meta[name="description"]', visible: false
     expect(page).to have_css 'meta[property="og:site_name"]', visible: false
   end
 
@@ -124,4 +128,12 @@ def when_i_view_a_page_with_no_sidebar
   def then_there_is_no_sidebar
     expect(page).to have_no_css "div.app-pane__toc"
   end
+
+  def when_i_view_a_page_with_prevent_indexing
+    visit "/prevent-index-page.html"
+  end
+
+  def then_there_is_a_robots_noindex_metatag
+    expect(page).to have_css 'meta[name="robots"][content="noindex"]', visible: false
+  end
 end

spec/govuk_tech_docs/meta_tags_spec.rb

@@ -24,67 +24,93 @@ def generate_title(site_name:, page_title:)
       )
 
       current_page = double("current_page",
-                            data: double("page_frontmatter", description: "The description.", title: page_title),
+                            data: { description: "The description.", title: page_title },
                             metadata: { locals: {} })
 
       GovukTechDocs::MetaTags.new(config, current_page).browser_title
     end
   end
 
   describe "#tags" do
-    it "returns all the extra meta tags" do
+    it "returns standard meta tags" do
       config = generate_config(
         host: "https://www.example.org",
         service_name: "Foo",
         full_service_name: "Test Site",
       )
 
       current_page = double("current_page",
-                            data: double("page_frontmatter", description: "The description.", title: "The Title"),
+                            data: { description: "The description.", title: "The Title" },
                             url: "/foo.html",
                             metadata: { locals: {} })
 
       tags = GovukTechDocs::MetaTags.new(config, current_page).tags
 
-      expect(tags).to eql("description" => "The description.",
-        "og:description" => "The description.",
-        "og:image" => "https://www.example.org/images/govuk-large.png",
-        "og:site_name" => "Test Site",
-        "og:title" => "The Title",
-        "og:type" => "object",
-        "og:url" => "https://www.example.org/foo.html",
+      expect(tags).to eql(
+        "description" => "The description.",
         "twitter:card" => "summary",
         "twitter:domain" => "www.example.org",
         "twitter:image" => "https://www.example.org/images/govuk-large.png",
         "twitter:title" => "The Title - Test Site",
-        "twitter:url" => "https://www.example.org/foo.html")
+        "twitter:url" => "https://www.example.org/foo.html",
+      )
     end
 
-    it "uses the local variable as page description for proxied pages" do
+    it "adds a noindex robots tag when the site config prevents indexing" do
+      config = generate_config(
+        prevent_indexing: true,
+      )
+
       current_page = double("current_page",
-                            data: double("page_frontmatter", description: "The description.", title: "The Title"),
+                            data: {},
                             url: "/foo.html",
-                            metadata: { locals: { description: "The local variable description." } })
+                            metadata: { locals: {} })
+
+      tags = GovukTechDocs::MetaTags.new(config, current_page).tags
+
+      expect(tags["robots"]).to eql("noindex")
+    end
+
+    it "adds a noindex robots tag when the page frontmatter prevents indexing" do
+      current_page = double("current_page",
+                            data: { prevent_indexing: true },
+                            url: "/foo.html",
+                            metadata: { locals: {} })
 
       tags = GovukTechDocs::MetaTags.new(generate_config, current_page).tags
 
-      expect(tags["description"]).to eql("The local variable description.")
+      expect(tags["robots"]).to eql("noindex")
     end
 
-    it "uses the local variable as page title for proxied pages" do
+    it "adds a google site validation meta tag when provided in config" do
+      config = generate_config(
+        google_site_verification: "LEGIT-VALIDATION-TOKEN",
+      )
+
       current_page = double("current_page",
-                            data: double("page_frontmatter", description: "The description.", title: "The Title"),
+                            data: {},
                             url: "/foo.html",
-                            metadata: { locals: { title: "The local variable title." } })
+                            metadata: { locals: {} })
+
+      tags = GovukTechDocs::MetaTags.new(config, current_page).tags
+
+      expect(tags["google-site-verification"]).to eql("LEGIT-VALIDATION-TOKEN")
+    end
+
+    it "uses the local variable as page description for proxied pages" do
+      current_page = double("current_page",
+                            data: { description: "The description." },
+                            url: "/foo.html",
+                            metadata: { locals: { description: "The local variable description." } })
 
       tags = GovukTechDocs::MetaTags.new(generate_config, current_page).tags
 
-      expect(tags["og:title"]).to eql("The local variable title.")
+      expect(tags["description"]).to eql("The local variable description.")
     end
 
     it "works even when no config is set" do
       current_page = double("current_page",
-                            data: double("page_frontmatter", description: "The description.", title: "The Title"),
+                            data: {},
                             url: "/foo.html",
                             metadata: { locals: { title: "The local variable title." } })
 
@@ -96,6 +122,43 @@ def generate_title(site_name:, page_title:)
     end
   end
 
+  describe "#opengraph_tags" do
+    it "returns opengraph meta tags" do
+      config = generate_config(
+        host: "https://www.example.org",
+        service_name: "Foo",
+        full_service_name: "Test Site",
+      )
+
+      current_page = double("current_page",
+                            data: { description: "The description.", title: "The Title" },
+                            url: "/foo.html",
+                            metadata: { locals: {} })
+
+      og_tags = GovukTechDocs::MetaTags.new(config, current_page).opengraph_tags
+
+      expect(og_tags).to eql(
+        "og:description" => "The description.",
+        "og:image" => "https://www.example.org/images/govuk-large.png",
+        "og:site_name" => "Test Site",
+        "og:title" => "The Title",
+        "og:type" => "object",
+        "og:url" => "https://www.example.org/foo.html",
+      )
+    end
+
+    it "uses the local variable as page title for proxied pages" do
+      current_page = double("current_page",
+                            data: { description: "The description." },
+                            url: "/foo.html",
+                            metadata: { locals: { title: "The local variable title." } })
+
+      tags = GovukTechDocs::MetaTags.new(generate_config, current_page).opengraph_tags
+
+      expect(tags["og:title"]).to eql("The local variable title.")
+    end
+  end
+
   def generate_config(config = {})
     {
       tech_docs: {