From 21b07d842a3fd40a5ee362483ba1906cfcd09b55 Mon Sep 17 00:00:00 2001 From: Khoa Nguyen Date: Mon, 4 Apr 2016 02:50:34 +0700 Subject: [PATCH] feat(search): Add search using Algolia --- config.js | 6 +++ data/algolia.js | 50 +++++++++++++++++++ data/{get-all-posts.js => get.js} | 13 +++-- data/{save-to-file.js => process.js} | 28 ++++------- {scripts => data}/remark-renderer.js | 17 ------- data/run.js | 46 +++++++++-------- data/save.js | 22 ++++++++ package.json | 3 ++ scripts/remark-img-rel-to-abs.js | 35 ------------- scripts/webpack.config.babel.js | 3 +- web_modules/LayoutContainer/index.js | 4 -- web_modules/components/PostEntry/index.js | 41 +++++++++++++++ .../PostEntry/styles.css} | 4 ++ .../components/SearchResult/algolia.svg | 21 ++++++++ web_modules/components/SearchResult/index.js | 44 ++++++++++++++++ .../components/SearchResult/styles.css | 11 ++++ web_modules/components/SidebarLayout/index.js | 47 ++++++++++++++++- .../components/SidebarLayout/styles.css | 7 +++ web_modules/components/SidebarLeft/index.js | 7 ++- web_modules/components/SidebarSocial/index.js | 40 --------------- .../components/SidebarSocial/style.css | 41 --------------- web_modules/layouts/Homepage/index.js | 38 ++------------ web_modules/layouts/Post/index.js | 2 +- web_modules/utils/algolia.js | 9 ++++ 24 files changed, 319 insertions(+), 220 deletions(-) create mode 100644 data/algolia.js rename data/{get-all-posts.js => get.js} (70%) rename data/{save-to-file.js => process.js} (72%) rename {scripts => data}/remark-renderer.js (57%) create mode 100644 data/save.js delete mode 100644 scripts/remark-img-rel-to-abs.js create mode 100644 web_modules/components/PostEntry/index.js rename web_modules/{layouts/Homepage/style.css => components/PostEntry/styles.css} (96%) create mode 100644 web_modules/components/SearchResult/algolia.svg create mode 100644 web_modules/components/SearchResult/index.js create mode 100644 web_modules/components/SearchResult/styles.css delete mode 100644 web_modules/components/SidebarSocial/index.js delete mode 100644 web_modules/components/SidebarSocial/style.css create mode 100644 web_modules/utils/algolia.js diff --git a/config.js b/config.js index 6562aeb..38acdf0 100644 --- a/config.js +++ b/config.js @@ -7,3 +7,9 @@ export const condition = { { pinned: false }, ], } + +export const algolia = { + appId: "GQCS86ZYQX", + indexName: "dnh-posts", + searchKey: "b952957ee76d25524f9a31bf5a108313", +} diff --git a/data/algolia.js b/data/algolia.js new file mode 100644 index 0000000..07dbab6 --- /dev/null +++ b/data/algolia.js @@ -0,0 +1,50 @@ +import algoliasearch from "algoliasearch" +import { algolia } from "../config" +import cheerio from "cheerio" + +const log = require("debug")("dnh:agolia") +const { appId, indexName } = algolia +const adminKey = process.env.ALGOLIA_ADMIN_KEY + +const breakHTMLIntoNodes = (html) => { + const $ = cheerio.load(html) + const result = [] + + $("p").each((i, elem) => { + result.push({ + tag_name: "p", + selector: i + 1, + content: $(elem).text(), + }) + }) + + return result +} +export default async function sendToAlgolia(posts) { + + const data = posts.map(({ content, meta }) => { + const nodes = breakHTMLIntoNodes(content) + const baseObject = { + ...meta, + } + + return nodes.map((node) => ({ + objectID: `${ baseObject.id }_${ node.tag_name }_${ node.selector }`, + ...baseObject, + ...node, + })) + }) + // Flatten array + .reduce((result, node) => result.concat(node), []) + + const client = algoliasearch(appId, adminKey) + const index = client.initIndex(indexName) + index.saveObjects(data) + .then(() => { + log("Sent data to Agolia") + }) + .catch((error) => { + log(error) + log("Error while sending data to Agolia") + }) +} diff --git a/data/get-all-posts.js b/data/get.js similarity index 70% rename from data/get-all-posts.js rename to data/get.js index 605dac3..c25ebeb 100644 --- a/data/get-all-posts.js +++ b/data/get.js @@ -2,9 +2,9 @@ import dnh from "./api" import access from "safe-access" import parseQuery from "./utils/parse-query" -const log = require("debug")("dnh:get-all-posts") +const log = require("debug")("dnh:get") -const getAllPosts = ({ users, posts }) => ( +export const getAllPosts = ({ users, posts }) => ( new Promise((resolve, reject) => { const getAllPosts = (query = {}) => { dnh.getPostFromCategory("share/writes", query) @@ -33,4 +33,11 @@ const getAllPosts = ({ users, posts }) => ( }) ) -export default getAllPosts +export const getAndUpdateRawPost = async function (post, collection) { + if (!post.raw) { + const response = await dnh.getRawPost(post.id) + post.raw = response.body + collection.update(post) + log("downloaded post " + post.id) + } +} diff --git a/data/save-to-file.js b/data/process.js similarity index 72% rename from data/save-to-file.js rename to data/process.js index de6f245..2353eaf 100644 --- a/data/save-to-file.js +++ b/data/process.js @@ -1,18 +1,9 @@ -import { writeFile } from "fs-promise" -import { join } from "path" import joinUri from "join-uri" import toSimpleUnicode from "vietnamese-unicode-toolkit" import stripAccentMarks from "./utils/stripAccentMarks" -import _ from "lodash" - -const template = (meta, content) => ( -`---json -${ JSON.stringify(meta)} ---- +import renderer from "./remark-renderer" -${ content } -` -) +import _ from "lodash" const getTopicOriginalPosterId = (post) => ( post.posters @@ -32,13 +23,10 @@ const normalizeTags = (tags = []) => { return _.uniq(a) } -export default async function saveToFile( +export default function process( post, - distPath, usersCollection ) { - const filePath = join(distPath, post.id.toString() + ".md") - // Get original posters information const userId = getTopicOriginalPosterId(post) const user = usersCollection.by("id", userId) @@ -55,6 +43,8 @@ export default async function saveToFile( title: toSimpleUnicode(post.title), tags: post.tags && normalizeTags(post.tags), date: post.created_at, + views: post.views, + likes: post.like_count, ...user && { author: { username: user.username, @@ -63,8 +53,10 @@ export default async function saveToFile( }, } - const content = toSimpleUnicode(post.raw) - const fileContent = template(meta, content) + const content = renderer(toSimpleUnicode(post.raw)) - await writeFile(filePath, fileContent) + return { + meta, + content, + } } diff --git a/scripts/remark-renderer.js b/data/remark-renderer.js similarity index 57% rename from scripts/remark-renderer.js rename to data/remark-renderer.js index 0b52f7b..78e8775 100644 --- a/scripts/remark-renderer.js +++ b/data/remark-renderer.js @@ -1,7 +1,6 @@ import remark from "remark" import slug from "remark-slug" import emoji from "remark-gemoji-to-emoji" -import autoLinkHeadings from "remark-autolink-headings" import highlight from "remark-highlight.js" import html from "remark-html" import { baseUrl } from "../config" @@ -9,26 +8,10 @@ import relToAbs from "rel-to-abs" export default (text) => { const resultHtml = remark - // https://github.com/wooorm/remark-slug .use(slug) - - // https://github.com/ben-eb/remark-autolink-headings - .use(autoLinkHeadings, { - attributes: { - class: "statinamic-HeadingAnchor", - }, - template: "#", - }) - .use(emoji) - - // https://github.com/wooorm/remark-html .use(html, { entities: "escape" }) - - // https://github.com/ben-eb/remark-highlight.js .use(highlight) - - // render .process(text, { commonmark: false, pedantic: true, diff --git a/data/run.js b/data/run.js index a18adc2..6c7a1df 100644 --- a/data/run.js +++ b/data/run.js @@ -1,23 +1,16 @@ -import { join } from "path" import fsp from "fs-promise" -import getAllPosts from "./get-all-posts" +import { join } from "path" +import { condition } from "../config" import { async as database } from "./db" import dnh from "./api" -import { condition } from "../config" -import saveToFile from "./save-to-file" +import { getAllPosts, getAndUpdateRawPost } from "./get" +import process from "./process" +import save from "./save" +import algolia from "./algolia" const log = require("debug")("dnh:run") const logStats = require("debug")("dnh:stats") -const getAndUpdateRawPost = async function (post, collection) { - if (!post.raw) { - const response = await dnh.getRawPost(post.id) - post.raw = response.body - collection.update(post) - log("downloaded post " + post.id) - } -} - export default async function () { // TODO: Is it safe to do this ? const { db, users, posts } = await database() @@ -35,6 +28,9 @@ export default async function () { .map((key) => getAndUpdateRawPost(data[key], posts)) ) + /** + * Make dirs + */ const contentDir = join(__dirname, "../content") const postsDir = join(contentDir, "posts") await fsp.ensureDir(contentDir) @@ -42,16 +38,26 @@ export default async function () { await fsp.mkdirs(postsDir) log("Created content folder") - await fsp.copy( - join(__dirname, "../content-holder"), - contentDir - ) + await fsp.copy(join(__dirname, "../content-holder"), contentDir) log("Copied content from content-holder/ to content/") - await Promise.all( - Object + /** + * Process data + */ + const processedData = Object .keys(data) - .map((key) => saveToFile(data[key], postsDir, users)) + .map((key) => process(data[key], users)) + + /** + * Send to Algolia + */ + await algolia(processedData) + /** + * Save to files + */ + await Promise.all( + processedData + .map((item) => save(item, postsDir)) ) log("Saved posts to markdown files") diff --git a/data/save.js b/data/save.js new file mode 100644 index 0000000..d596fc5 --- /dev/null +++ b/data/save.js @@ -0,0 +1,22 @@ +import { writeFile } from "fs-promise" +import { join } from "path" + +const template = (meta, content) => ( +`---json +${ JSON.stringify(meta)} +--- + +${ content } +` +) + +export default async function saveToFile( + fileData, + distPath, +) { + const { meta, content } = fileData + const filePath = join(distPath, fileData.meta.id.toString() + ".md") + const fileContent = template(meta, content) + + await writeFile(filePath, fileContent) +} diff --git a/package.json b/package.json index 72ac9f6..ec34e1a 100644 --- a/package.json +++ b/package.json @@ -92,6 +92,9 @@ "whatwg-fetch": "^0.11.0" }, "dependencies": { + "algolia-react-input": "^1.0.9", + "algoliasearch": "^3.13.1", + "cheerio": "^0.20.0", "debug": "^2.2.0", "fs-promise": "^0.5.0", "got": "^6.2.0", diff --git a/scripts/remark-img-rel-to-abs.js b/scripts/remark-img-rel-to-abs.js deleted file mode 100644 index 4822fb3..0000000 --- a/scripts/remark-img-rel-to-abs.js +++ /dev/null @@ -1,35 +0,0 @@ -import visit from "unist-util-visit" -import joinUri from "join-uri" - -// https://github.com/auth0/rel-to-abs/blob/48dcf4a56daba6b1014f6fb0d734ad866f0c9faf/index.js#L4-L9 -const regex = /^(https?|file|ftps?|mailto|javascript|data:image\/[^;]{2,9};):/i -const convert = (baseUrl, currentUrl) => { - if (!currentUrl || regex.test(currentUrl)) { - return currentUrl - } - return joinUri(baseUrl, currentUrl) -} - -const getTransformer = (baseUrl) => { - const transformer = (ast) => { - visit(ast, "image", (node) => { - console.log(node) - if (node.url) { - node.url = convert(baseUrl, node.url) - } - }) - } - - return transformer -} - -/** - * Expose. - */ -export default function(remark, options = {}) { - if (!options.baseUrl) { - throw new Error("You muse define baseUrl") - } - - return getTransformer(options.baseUrl) -} diff --git a/scripts/webpack.config.babel.js b/scripts/webpack.config.babel.js index c61817c..36e4747 100644 --- a/scripts/webpack.config.babel.js +++ b/scripts/webpack.config.babel.js @@ -2,7 +2,6 @@ import path from "path" import webpack from "webpack" import ExtractTextPlugin from "extract-text-webpack-plugin" -import renderer from "./remark-renderer" export default ({ config, pkg }) => ({ ...config.dev && { @@ -59,7 +58,7 @@ export default ({ config, pkg }) => ({ statinamic: { loader: { context: path.join(config.cwd, config.source), - renderer, + renderer: (html) => html, feedsOptions: { title: pkg.name, site_url: pkg.homepage, diff --git a/web_modules/LayoutContainer/index.js b/web_modules/LayoutContainer/index.js index 7cff3e4..aa7c955 100644 --- a/web_modules/LayoutContainer/index.js +++ b/web_modules/LayoutContainer/index.js @@ -28,10 +28,6 @@ export default class Layout extends Component { rel: "stylesheet", href: "https://fonts.googleapis.com/css?family=Roboto+Condensed:400,400italic,700,700italic|Open+Sans:300|Roboto+Slab:400,300&subset=latin,vietnamese", }, - { - rel: "stylesheet", - href: "https://maxcdn.bootstrapcdn.com/font-awesome/4.5.0/css/font-awesome.min.css", - }, { rel: "icon", type: "image/png", diff --git a/web_modules/components/PostEntry/index.js b/web_modules/components/PostEntry/index.js new file mode 100644 index 0000000..cb93039 --- /dev/null +++ b/web_modules/components/PostEntry/index.js @@ -0,0 +1,41 @@ +import React, { PropTypes } from "react" +import Link from "statinamic/lib/Link" +import Time from "../Time" +import styles from "./styles.css" + +const PostEntry = ({ post }) => (( +
+
+)) + +PostEntry.propTypes = { + post: PropTypes.object.isRequired, +} + +export default PostEntry diff --git a/web_modules/layouts/Homepage/style.css b/web_modules/components/PostEntry/styles.css similarity index 96% rename from web_modules/layouts/Homepage/style.css rename to web_modules/components/PostEntry/styles.css index 284cd4b..6ce9d22 100644 --- a/web_modules/layouts/Homepage/style.css +++ b/web_modules/components/PostEntry/styles.css @@ -65,3 +65,7 @@ border-bottom: 1px solid; } } + +:global .hl { + background: #D3E8F6; +} diff --git a/web_modules/components/SearchResult/algolia.svg b/web_modules/components/SearchResult/algolia.svg new file mode 100644 index 0000000..64ce5db --- /dev/null +++ b/web_modules/components/SearchResult/algolia.svg @@ -0,0 +1,21 @@ + + + + Algolia_logo_bg-white + Created with Sketch. + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/web_modules/components/SearchResult/index.js b/web_modules/components/SearchResult/index.js new file mode 100644 index 0000000..d6b5cfb --- /dev/null +++ b/web_modules/components/SearchResult/index.js @@ -0,0 +1,44 @@ +import React, { Component, PropTypes } from "react" +import PostEntry from "../PostEntry" +import styles from "./styles.css" +import AlgoliaLogo from "./algolia.svg" +import SVG from "react-svg-inline" + +export default class SearchResult extends Component { + static propTypes = { + hits: PropTypes.array.isRequired, + }; + + render() { + const hits = this.props.hits + + return ( +
+

Kết quả tìm kiếm

+ { + hits.length <= 0 && + Không có kết quả nào. Hãy thử tìm với từ khóa khác + } + { + hits.length > 0 && + this.props.hits.map((post, key) => { + const data = { + ...post, + title: post._highlightResult.title.value, + description: post._highlightResult.content.value, + __url: `/${ post.route}/`, + } + + return ( + + ) + }) + } +

+
+ ) + } +} diff --git a/web_modules/components/SearchResult/styles.css b/web_modules/components/SearchResult/styles.css new file mode 100644 index 0000000..b149991 --- /dev/null +++ b/web_modules/components/SearchResult/styles.css @@ -0,0 +1,11 @@ +.title { + margin: 0; +} + +.algolia { + text-align: right; + + & svg { + max-width: 20%; + } +} diff --git a/web_modules/components/SidebarLayout/index.js b/web_modules/components/SidebarLayout/index.js index a41ac19..9318f89 100644 --- a/web_modules/components/SidebarLayout/index.js +++ b/web_modules/components/SidebarLayout/index.js @@ -1,19 +1,62 @@ import React, { Component, PropTypes } from "react" import SidebarLeft from "../SidebarLeft" import styles from "./styles.css" +import { client as algoliaClient, indexName } from "../../utils/algolia" +import AlgoliaInput from "algolia-react-input" +import SearchResult from "../SearchResult" export default class SidebarLayout extends Component { static propTypes = { children: PropTypes.node.isRequired, }; + constructor(props) { + super(props) + + this.state = { + hits: undefined, + } + } + + onError = () => { + console.log("onError", arguments) + } + onResults = (content) => { + this.setState({ hits: content.hits }) + } + + onEmptyField = () => { + this.setState({ hits: undefined }) + } + render() { + const SearchInput = ( + + ) + + const hits = this.state.hits + return (
- +
- { this.props.children } + { + typeof hits !== "undefined" && + + } + { + typeof hits === "undefined" && + this.props.children + }
diff --git a/web_modules/components/SidebarLayout/styles.css b/web_modules/components/SidebarLayout/styles.css index 9afe132..fee5f1d 100644 --- a/web_modules/components/SidebarLayout/styles.css +++ b/web_modules/components/SidebarLayout/styles.css @@ -25,3 +25,10 @@ padding: 25px 20px; } } + +.searchInput { + width: 100%; + font-size: 1rem; + padding: 0.7rem 0.5rem; + margin: 0.5rem 0; +} diff --git a/web_modules/components/SidebarLeft/index.js b/web_modules/components/SidebarLeft/index.js index cef9690..b20f096 100644 --- a/web_modules/components/SidebarLeft/index.js +++ b/web_modules/components/SidebarLeft/index.js @@ -2,7 +2,6 @@ import React, { PropTypes } from "react" import Link from "statinamic/lib/Link" import SidebarNav from "../SidebarNav" import LogoImg from "./logo-full.png" -import Social from "../SidebarSocial" import styles from "./style.css" class SidebarLeft extends React.Component { @@ -10,6 +9,10 @@ class SidebarLeft extends React.Component { metadata: PropTypes.object.isRequired, }; + static propTypes = { + searchInput: PropTypes.node.isRequired, + }; + render() { const { pkg: { config }, @@ -28,10 +31,10 @@ class SidebarLeft extends React.Component {

{ config.siteDescr }

+ { this.props.searchInput }
-

© All rights reserved.

diff --git a/web_modules/components/SidebarSocial/index.js b/web_modules/components/SidebarSocial/index.js deleted file mode 100644 index fdd685c..0000000 --- a/web_modules/components/SidebarSocial/index.js +++ /dev/null @@ -1,40 +0,0 @@ -import React, { Component, PropTypes } from "react" -import styles from "./style.css" - -export default class SidebarSocial extends Component { - static contextTypes = { - metadata: PropTypes.object.isRequired, - }; - - render() { - const { - pkg: { config }, - } = this.context.metadata - - const twitter = config.twitter - const vk = config.vk - const rss = config.rss - const email = config.email - const github = config.github - const telegram = config.telegram - - return ( -
-
    -
    • -
    • -
    • -
-
    -
    • -
  • - -
    • -
-
    -
    • -
-
- ) - } -} diff --git a/web_modules/components/SidebarSocial/style.css b/web_modules/components/SidebarSocial/style.css deleted file mode 100644 index 8ddb96b..0000000 --- a/web_modules/components/SidebarSocial/style.css +++ /dev/null @@ -1,41 +0,0 @@ -.wrapper { - margin-top: 30px; -} - -.wrapper ul { - list-style: none; - padding: 0; - margin: 10px 0; - clear: fix-legacy; -} - -.wrapper ul > li { - float: left; - margin-right: 5px; - text-align: center; - line-height: 21px; - height: 24px; - width: 24px; - border-radius: 3px; - background: #f4edde; -} - -.wrapper ul > li:hover { - background: #f4efe8; -} - -.wrapper ul > li > a { - border-bottom: 0; - line-height: 21px; - cursor: pointer; -} - -.wrapper ul > li > a > i { - color: #333; - font-size: 14px; - line-height: 21px; -} - -.wrapper ul > li:hover a > i { - color: #222; -} diff --git a/web_modules/layouts/Homepage/index.js b/web_modules/layouts/Homepage/index.js index 18fac30..f26bcd2 100644 --- a/web_modules/layouts/Homepage/index.js +++ b/web_modules/layouts/Homepage/index.js @@ -1,11 +1,9 @@ import React, { Component, PropTypes } from "react" -import Link from "statinamic/lib/Link" -import Time from "../../components/Time" + import enhanceCollection from "statinamic/lib/enhance-collection" import Helmet from "react-helmet" import SidebarLayout from "../../components/SidebarLayout" - -import styles from "./style.css" +import PostEntry from "../../components/PostEntry" export default class Homepage extends Component { static contextTypes = { @@ -23,37 +21,7 @@ export default class Homepage extends Component { sort: "date", reverse: true, }).map((post) => ( -
-
+ )) return ( diff --git a/web_modules/layouts/Post/index.js b/web_modules/layouts/Post/index.js index e01c9fb..8ac1157 100644 --- a/web_modules/layouts/Post/index.js +++ b/web_modules/layouts/Post/index.js @@ -62,7 +62,7 @@ export default class Post extends Component { />
-
+

{ head.title }

diff --git a/web_modules/utils/algolia.js b/web_modules/utils/algolia.js new file mode 100644 index 0000000..e522cc0 --- /dev/null +++ b/web_modules/utils/algolia.js @@ -0,0 +1,9 @@ +/** + * Init algolia client search + */ + +import algoliasearch from "algoliasearch" +import { algolia } from "../../config" + +export const client = algoliasearch(algolia.appId, algolia.searchKey) +export const indexName = algolia.indexName