Chris

scrapers/apa.json

@@ -18,6 +18,63 @@
       "selector": "//meta[@name='citation_date']",
       "attribute": "content"
     },
+  "url": "web.a.ebscohost.com",
+  "elements": {
+    "publisher": {
+      "selector": "//meta[@name='DC.Publisher']",
+      "attribute": "content"
+    },
+
+    "journal_name": {
+      "selector": "//meta[@name='citation_journal_title']",
+      "attribute": "content"
+    },
+
+    "journal_issn": {
+      "selector": "//meta[@name='citation_issn']",
+      "attribute": "content"
+    },
+
+    "volume": {
+      "selector": "//meta[@name='citation_volume']",
+      "attribute": "content"
+    },
+
+    "issue": {
+      "selector": "//meta[@name='citation_issue']",
+      "attribute": "content"
+    },
+
+    "firstpage": {
+      "selector": "//meta[@name='citation_firstpage']",
+      "attribute": "content"
+    },
+
+    "title": {
+      "selector": "//title",
+      "attribute": "content"
+    },
+
+    "keywords": {
+      "selector": "//ul[contains(@class, 'kwd-group')]",
+      "attribute": "text"
+    },
+
+    "author_name": {
+      "selector": "//meta[@name='citation_author']",
+      "attribute": "content"
+    },
+
+    "author_institution": {
+      "selector": "//meta[@name='citation_author_institution']",
+      "attribute": "content"
+    },
+
+    "date_published": {
+      "selector": "//*[@id='slugline']/cite/span[1]",
+      "attribute": "text"
+    },
+
     "doi": {
       "selector": "//meta[@name='citation_doi']",
       "attribute": "content"
@@ -30,19 +87,52 @@
       "selector": "//meta[@name='citation_publisher']",
       "attribute": "content"
     },
+
+    "pmid": {
+      "selector": "//meta[@name='citation_pmid']",
+      "attribute": "content"
+    },
+
+    "copyright": {
+      "selector": "//*[@id='copyright-statement-1']",
+      "attribute": "html"
+    },
+
+    "fulltext_html": {
+      "selector": "//meta[@name='citation_fulltext_html_url']",
+      "attribute": "content",
+      "download": {
+        "rename": "fulltext.html"
+      }
+    },
+
     "fulltext_pdf": {
       "selector": "//meta[@name='citation_pdf_url']",
       "attribute": "content",
       "download": {
         "rename": "fulltext.pdf"
       }
     },
+
     "fulltext_html": {
       "selector": "//meta[@name='citation_fulltext_pdf_url']",
       "attribute": "content",
       "download": {
         "rename": "fulltext.html"
       }
+    },
+
+    "abstract_html": {
+      "selector": "//meta[@name='citation_abstract_html_url']",
+      "attribute": "content",
+      "download": {
+        "rename": "abstract.html"
+      }
+    },
+
+    "competing_interests_html": {
+      "selector": "//*[contains(@class, 'fn-conflict')]",
+      "attribute": "text"
     }
   }
 }

scrapers/elsevier.json

@@ -0,0 +1,25 @@
+{
+  "url": "sciencedirect\\.com",
+  "elements": {
+    "title": {
+      "selector": "//title",
+      "attribute": "content"
+    },
+    "fulltext_html": {
+      "selector": "//link[contains(@rel, 'canonical')]",
+      "attribute": "href",
+      "download": {
+        "rename": "fulltext.html"
+      }
+    },
+
+    "fulltext_pdf": {
+      "selector": "//*[contains(@id, 'pdfLink')]",
+      "attribute": "href",
+      "download": {
+        "rename": "fulltext.pdf"
+      }
+    }
+  }
+}
+

scrapers/sage.json

@@ -1,44 +1,103 @@
 {
-  "url": ".*sagepub.*\\.com",
-  "headless": true,
+  "url": ".*\\.sagepub\\.com",
   "elements": {
     "publisher": {
       "selector": "//meta[@name='DC.Publisher']",
       "attribute": "content"
     },
+
+    "journal_name": {
+      "selector": "//meta[@name='citation_journal_title']",
+      "attribute": "content"
+    },
+
+    "journal_issn": {
+      "selector": "//meta[@name='citation_issn']",
+      "attribute": "content"
+    },
+
+    "volume": {
+      "selector": "//meta[@name='citation_volume']",
+      "attribute": "content"
+    },
+
+    "issue": {
+      "selector": "//meta[@name='citation_issue']",
+      "attribute": "content"
+    },
+
+    "firstpage": {
+      "selector": "//meta[@name='citation_firstpage']",
+      "attribute": "content"
+    },
+
     "title": {
-      "selector": "//meta[@name='DC.Title']",
+      "selector": "//meta[@name='citation_title']",
       "attribute": "content"
     },
-    "authors": {
+
+    "keywords": {
+      "selector": "//ul[contains(@class, 'kwd-group')]",
+      "attribute": "text"
+    },
+
+    "author_name": {
       "selector": "//meta[@name='citation_author']",
       "attribute": "content"
     },
-    "date": {
-      "selector": "//meta[@name='citation_online_date']",
+
+    "author_institution": {
+      "selector": "//meta[@name='citation_author_institution']",
       "attribute": "content"
     },
+
+    "date_published": {
+      "selector": "//*[@id='slugline']/cite/span[1]",
+      "attribute": "text"
+    },
+
     "doi": {
       "selector": "//meta[@name='citation_doi']",
       "attribute": "content"
     },
-    "issn": {
-      "selector": "//meta[@name='citation_issn']",
+
+    "pmid": {
+      "selector": "//meta[@name='citation_pmid']",
       "attribute": "content"
     },
+
+    "copyright": {
+      "selector": "//*[@id='copyright-statement-1']",
+      "attribute": "html"
+    },
+
+    "fulltext_html": {
+      "selector": "//meta[@name='citation_fulltext_html_url']",
+      "attribute": "content",
+      "download": {
+        "rename": "fulltext.html"
+      }
+    },
+
     "fulltext_pdf": {
       "selector": "//meta[@name='citation_pdf_url']",
       "attribute": "content",
       "download": {
         "rename": "fulltext.pdf"
       }
     },
-    "fulltext_html": {
-      "selector": "//meta[@name='citation_fulltext_html_url']",
+
+    "abstract_html": {
+      "selector": "//meta[@name='citation_abstract_html_url']",
       "attribute": "content",
       "download": {
-        "rename": "fulltext.html"
+        "rename": "abstract.html"
       }
+    },
+
+    "competing_interests_html": {
+      "selector": "//*[contains(@class, 'fn-conflict')]",
+      "attribute": "text"
     }
   }
-}
+}

scrapers/springer.json

@@ -1,11 +1,38 @@
 {
   "url": ".*springer.*\\.com",
   "headless": true,
+
   "elements": {
     "publisher": {
       "selector": "//meta[@name='citation_publisher']",
       "attribute": "content"
     },
+
+    "journal_name": {
+      "selector": "//meta[@name='citation_journal_title']",
+      "attribute": "content"
+    },
+
+    "journal_issn": {
+      "selector": "//meta[@name='citation_issn']",
+      "attribute": "content"
+    },
+
+    "volume": {
+      "selector": "//meta[@name='citation_volume']",
+      "attribute": "content"
+    },
+
+    "issue": {
+      "selector": "//meta[@name='citation_issue']",
+      "attribute": "content"
+    },
+
+    "firstpage": {
+      "selector": "//meta[@name='citation_firstpage']",
+      "attribute": "content"
+    },
+
     "title": {
       "selector": "//meta[@name='citation_title']",
       "attribute": "content"
@@ -18,21 +45,48 @@
       "selector": "//meta[@name='citation_online_date']",
       "attribute": "content"
     },
+
+    "author_name": {
+      "selector": "//meta[@name='citation_author']",
+      "attribute": "content"
+    },
+
+    "author_institution": {
+      "selector": "//meta[@name='citation_author_institution']",
+      "attribute": "content"
+    },
+
+    "date_published": {
+      "selector": "//meta[@name='citation_cover_date']",
+      "attribute": "text"
+    },
+
     "doi": {
       "selector": "//meta[@name='citation_doi']",
       "attribute": "content"
     },
+
     "issn": {
       "selector": "//meta[@name='citation_issn']",
       "attribute": "content"
     },
+
+    "fulltext_html": {
+      "selector": "//meta[contains(@name, 'fulltext_html')]",
+      "attribute": "content",
+      "download": {
+        "rename": "fulltext.html"
+      }
+    },
+
     "fulltext_pdf": {
       "selector": "//meta[@name='citation_pdf_url']",
       "attribute": "content",
       "download": {
         "rename": "fulltext.pdf"
       }
     },
+
     "fulltext_html": {
       "selector": "//meta[@name='citation_fulltext_html_url']",
       "attribute": "content",
@@ -42,3 +96,13 @@
     }
   }
 }
+
+    "abstract_html": {
+      "selector": "//meta[@name='citation_abstract_html_url']",
+      "attribute": "content",
+      "download": {
+        "rename": "abstract.html"
+      }
+    }
+  }
+}