feat(search): Add search using Algolia

config.js

@@ -7,3 +7,9 @@ export const condition = {
     { pinned: false },
   ],
 }
+
+export const algolia = {
+  appId: "GQCS86ZYQX",
+  indexName: "dnh-posts",
+  searchKey: "b952957ee76d25524f9a31bf5a108313",
+}

data/algolia.js

@@ -0,0 +1,50 @@
+import algoliasearch from "algoliasearch"
+import { algolia } from "../config"
+import cheerio from "cheerio"
+
+const log = require("debug")("dnh:agolia")
+const { appId, indexName } = algolia
+const adminKey = process.env.ALGOLIA_ADMIN_KEY
+
+const breakHTMLIntoNodes = (html) => {
+  const $ = cheerio.load(html)
+  const result = []
+
+  $("p").each((i, elem) => {
+    result.push({
+      tag_name: "p",
+      selector: i + 1,
+      content: $(elem).text(),
+    })
+  })
+
+  return result
+}
+export default async function sendToAlgolia(posts) {
+
+  const data = posts.map(({ content, meta }) => {
+    const nodes = breakHTMLIntoNodes(content)
+    const baseObject = {
+      ...meta,
+    }
+
+    return nodes.map((node) => ({
+      objectID: `${ baseObject.id }_${ node.tag_name }_${ node.selector }`,
+      ...baseObject,
+      ...node,
+    }))
+  })
+  // Flatten array
+  .reduce((result, node) => result.concat(node), [])
+
+  const client = algoliasearch(appId, adminKey)
+  const index = client.initIndex(indexName)
+  index.saveObjects(data)
+  .then(() => {
+    log("Sent data to Agolia")
+  })
+  .catch((error) => {
+    log(error)
+    log("Error while sending data to Agolia")
+  })
+}

data/get-all-posts.js → data/get.js

@@ -2,9 +2,9 @@ import dnh from "./api"
 import access from "safe-access"
 import parseQuery from "./utils/parse-query"
 
-const log = require("debug")("dnh:get-all-posts")
+const log = require("debug")("dnh:get")
 
-const getAllPosts = ({ users, posts }) => (
+export const getAllPosts = ({ users, posts }) => (
   new Promise((resolve, reject) => {
     const getAllPosts = (query = {}) => {
       dnh.getPostFromCategory("share/writes", query)
@@ -33,4 +33,11 @@ const getAllPosts = ({ users, posts }) => (
   })
 )
 
-export default getAllPosts
+export const getAndUpdateRawPost = async function (post, collection) {
+  if (!post.raw) {
+    const response = await dnh.getRawPost(post.id)
+    post.raw = response.body
+    collection.update(post)
+    log("downloaded post " + post.id)
+  }
+}

data/save-to-file.js → data/process.js

@@ -1,18 +1,9 @@
-import { writeFile } from "fs-promise"
-import { join } from "path"
 import joinUri from "join-uri"
 import toSimpleUnicode from "vietnamese-unicode-toolkit"
 import stripAccentMarks from "./utils/stripAccentMarks"
-import _ from "lodash"
-
-const template = (meta, content) => (
-`---json
-${ JSON.stringify(meta)}
----
+import renderer from "./remark-renderer"
 
-${ content }
-`
-)
+import _ from "lodash"
 
 const getTopicOriginalPosterId = (post) => (
   post.posters
@@ -32,13 +23,10 @@ const normalizeTags = (tags = []) => {
   return _.uniq(a)
 }
 
-export default async function saveToFile(
+export default function process(
   post,
-  distPath,
   usersCollection
 ) {
-  const filePath = join(distPath, post.id.toString() + ".md")
-
   // Get original posters information
   const userId = getTopicOriginalPosterId(post)
   const user = usersCollection.by("id", userId)
@@ -55,6 +43,8 @@ export default async function saveToFile(
     title: toSimpleUnicode(post.title),
     tags: post.tags && normalizeTags(post.tags),
     date: post.created_at,
+    views: post.views,
+    likes: post.like_count,
     ...user && {
       author: {
         username: user.username,
@@ -63,8 +53,10 @@ export default async function saveToFile(
     },
   }
 
-  const content = toSimpleUnicode(post.raw)
-  const fileContent = template(meta, content)
+  const content = renderer(toSimpleUnicode(post.raw))
 
-  await writeFile(filePath, fileContent)
+  return {
+    meta,
+    content,
+  }
 }

scripts/remark-renderer.js → data/remark-renderer.js

@@ -1,34 +1,17 @@
 import remark from "remark"
 import slug from "remark-slug"
 import emoji from "remark-gemoji-to-emoji"
-import autoLinkHeadings from "remark-autolink-headings"
 import highlight from "remark-highlight.js"
 import html from "remark-html"
 import { baseUrl } from "../config"
 import relToAbs from "rel-to-abs"
 
 export default (text) => {
   const resultHtml = remark
-    // https://github.com/wooorm/remark-slug
     .use(slug)
-
-    // https://github.com/ben-eb/remark-autolink-headings
-    .use(autoLinkHeadings, {
-      attributes: {
-        class: "statinamic-HeadingAnchor",
-      },
-      template: "#",
-    })
-
     .use(emoji)
-
-    // https://github.com/wooorm/remark-html
     .use(html, { entities: "escape" })
-
-    // https://github.com/ben-eb/remark-highlight.js
     .use(highlight)
-
-    // render
     .process(text, {
       commonmark: false,
       pedantic: true,

data/run.js

@@ -1,23 +1,16 @@
-import { join } from "path"
 import fsp from "fs-promise"
-import getAllPosts from "./get-all-posts"
+import { join } from "path"
+import { condition } from "../config"
 import { async as database } from "./db"
 import dnh from "./api"
-import { condition } from "../config"
-import saveToFile from "./save-to-file"
+import { getAllPosts, getAndUpdateRawPost } from "./get"
+import process from "./process"
+import save from "./save"
+import algolia from "./algolia"
 
 const log = require("debug")("dnh:run")
 const logStats = require("debug")("dnh:stats")
 
-const getAndUpdateRawPost = async function (post, collection) {
-  if (!post.raw) {
-    const response = await dnh.getRawPost(post.id)
-    post.raw = response.body
-    collection.update(post)
-    log("downloaded post " + post.id)
-  }
-}
-
 export default async function () {
   // TODO: Is it safe to do this ?
   const { db, users, posts } = await database()
@@ -35,23 +28,36 @@ export default async function () {
       .map((key) => getAndUpdateRawPost(data[key], posts))
     )
 
+    /**
+     * Make dirs
+     */
     const contentDir = join(__dirname, "../content")
     const postsDir = join(contentDir, "posts")
     await fsp.ensureDir(contentDir)
     await fsp.emptyDir(contentDir)
     await fsp.mkdirs(postsDir)
     log("Created content folder")
 
-    await fsp.copy(
-      join(__dirname, "../content-holder"),
-      contentDir
-    )
+    await fsp.copy(join(__dirname, "../content-holder"), contentDir)
     log("Copied content from content-holder/ to content/")
 
-    await Promise.all(
-      Object
+    /**
+     * Process data
+     */
+    const processedData = Object
       .keys(data)
-      .map((key) => saveToFile(data[key], postsDir, users))
+      .map((key) => process(data[key], users))
+
+    /**
+     * Send to Algolia
+     */
+    await algolia(processedData)
+    /**
+     * Save to files
+     */
+    await Promise.all(
+      processedData
+      .map((item) => save(item, postsDir))
     )
     log("Saved posts to markdown files")
 

data/save.js

@@ -0,0 +1,22 @@
+import { writeFile } from "fs-promise"
+import { join } from "path"
+
+const template = (meta, content) => (
+`---json
+${ JSON.stringify(meta)}
+---
+
+${ content }
+`
+)
+
+export default async function saveToFile(
+  fileData,
+  distPath,
+) {
+  const { meta, content } = fileData
+  const filePath = join(distPath, fileData.meta.id.toString() + ".md")
+  const fileContent = template(meta, content)
+
+  await writeFile(filePath, fileContent)
+}

package.json

@@ -92,6 +92,9 @@
     "whatwg-fetch": "^0.11.0"
   },
   "dependencies": {
+    "algolia-react-input": "^1.0.9",
+    "algoliasearch": "^3.13.1",
+    "cheerio": "^0.20.0",
     "debug": "^2.2.0",
     "fs-promise": "^0.5.0",
     "got": "^6.2.0",

scripts/webpack.config.babel.js

@@ -2,7 +2,6 @@ import path from "path"
 
 import webpack from "webpack"
 import ExtractTextPlugin from "extract-text-webpack-plugin"
-import renderer from "./remark-renderer"
 
 export default ({ config, pkg }) => ({
   ...config.dev && {
@@ -59,7 +58,7 @@ export default ({ config, pkg }) => ({
   statinamic: {
     loader: {
       context: path.join(config.cwd, config.source),
-      renderer,
+      renderer: (html) => html,
       feedsOptions: {
         title: pkg.name,
         site_url: pkg.homepage,

web_modules/LayoutContainer/index.js

@@ -28,10 +28,6 @@ export default class Layout extends Component {
               rel: "stylesheet",
               href: "https://fonts.googleapis.com/css?family=Roboto+Condensed:400,400italic,700,700italic|Open+Sans:300|Roboto+Slab:400,300&subset=latin,vietnamese",
             },
-            {
-              rel: "stylesheet",
-              href: "https://maxcdn.bootstrapcdn.com/font-awesome/4.5.0/css/font-awesome.min.css",
-            },
             {
               rel: "icon",
               type: "image/png",

web_modules/components/PostEntry/index.js

@@ -0,0 +1,41 @@
+import React, { PropTypes } from "react"
+import Link from "statinamic/lib/Link"
+import Time from "../Time"
+import styles from "./styles.css"
+
+const PostEntry = ({ post }) => ((
+  <div className={ styles.postEntry }>
+    <Time
+      format="MMMM YYYY"
+      time={ post.date }
+      className={ styles.time }
+    />
+    <span style={ { padding: "5px" } } />
+    {
+      post.tags && post.tags.map((tag) => (
+        <span key={ tag } className={ styles.tag }>{ tag }</span>
+      ))
+    }
+    <h2>
+      <Link
+        style={ { borderBottom: "none" } }
+        to={ post.__url }
+      >
+        <span dangerouslySetInnerHTML={ { __html: post.title } } />
+      </Link>
+    </h2>
+    <p dangerouslySetInnerHTML={ { __html: post.description } } />
+    <Link
+      className={ styles.readmore }
+      to={ post.__url }
+    >
+      { "Read" }
+    </Link>
+  </div>
+))
+
+PostEntry.propTypes = {
+  post: PropTypes.object.isRequired,
+}
+
+export default PostEntry