diff --git a/.gitignore b/.gitignore index 8d87b1d..e197b14 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ node_modules/* +articles/* +test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ddb1d4..f8d42ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +1.0.0 +===== + +* Ground up rewrite for node 0.8. Code is now much more maintainable, readable, + and organized. + * New architecture: reed -> APIs -> redis connector/filesystem helper/file + processor. +* No more blocking code in the library (not necessarily dependencies). +* Articles no longer need to have dashes in the filename. +* Reed will now watch files that end in ".markdown" as well as ".md". +* Reed now properly detects file additions, updates, and removals that happened + while it was not running. +* `reed.index` and `reed.refresh` methods deprecated. They can still be called but will only + emit a warning. They will be removed in the next version. +* `reed.removeAll` method is now atomic (Redis MULTI). +* Method blocking is now much more efficient by using a queue instead of what + amounted to fancy spinlock. + 0.9.8 ===== diff --git a/README.md b/README.md index 41f38c2..ab13766 100644 --- a/README.md +++ b/README.md @@ -176,13 +176,6 @@ Reed exposes the following functions: * `removeAll(callback)`: Removes all blog posts. The callback is called after all posts have been deleted, and receives `error` if there was an error during deletion. **This deletion is not transactional!** -* `index(callback)`: Forces a full refresh of the opened directory. This should - usually not be necessary, as reed should automatically take care of posts - being added and updated. The callback receives `error` if indexing was - prematurely interrupted by an error. -* `refresh()`: Forces a refresh of the Redis index, removing any entries that - are no longer present on the filesystem. This should usually not be necessary, - as reed should handle this internally. **Note**: `get`, `list`, `index`, `remove`, and `removeAll` asynchronously block until reed is in a ready state. This means they can be called before diff --git a/lib/blog-connector.js b/lib/blog-connector.js new file mode 100644 index 0000000..66776ca --- /dev/null +++ b/lib/blog-connector.js @@ -0,0 +1,240 @@ +var async = require('async'), + conn = require('./redis-connection'), + fileProcessor = require('./file-processor'), + keyManager = require('./keymanager').KeyManager; + +//the redis client +var client; + +//constants +exports.upsertResult = { + UPDATE: 0, + NEW: 1, + NONE: 2 +}; + +/* + * Open a connection to redis. + * + * cfg - the configuration to use. + * callback - callback receives (error, success). success is true if the connection was + * opened or is already open. + */ +exports.open = function(cfg, callback) { + keyManager.open(cfg, function(err) { + if (err) return callback(err); + conn.open(cfg, function(err, redisClient) { + if (err) return callback(err, false); + client = redisClient; + callback(err, false); + }); + }); +} + +exports.close = function() { + conn.close(); + keyManager.close(); +} + +exports.listPosts = function(callback) { + client.zrevrange(keyManager.blogDates, 0, -1, function(err, titles) { + if (err) return callback(err); + callback(null, titles); + }); +} + +exports.getPost = function(title, callback) { + keyManager.toPostFilenameFromTitle(title, function(err, filename) { + if (err) return callback(err); + if (filename == null) return callback(new Error('Post not found: ' + title)); + exports.getPostByFilename(filename, callback); + }); +} + +exports.getPostByFilename = function(filename, callback) { + var key = keyManager.toPostKeyFromFilename(filename); + + client.hgetall(key, function(err, hash) { + if (typeof hash !== "undefined" && hash != null && Object.keys(hash).length > 0) { + var post = hash.post; + var metadata = {}; + try { + metadata = JSON.parse(hash.metadata); + } + catch (parseErr) { + //no good metadata - ignore + } + + if (typeof metadata.lastModified !== 'undefined') { + metadata.lastModified = new Date(metadata.lastModified); + } + + callback(null, true, metadata, hash.post); + } + else { + callback(new Error('Post not found: ' + filename), false); + } + }); +} + +exports.insertPost = function(filename, callback) { + fileProcessor.process(filename, function(err, postDate, metadata, post) { + if (err) return callback(err); + + var ptr = keyManager.toPostPointer(filename); + var key = keyManager.toPostKeyFromFilename(filename); + var title = keyManager.toTitle(filename); + + metadataString = JSON.stringify(metadata); + client.sadd(keyManager.blogIndex, filename, function(err) { + client.zadd(keyManager.blogDates, postDate, title, function(err) { + client.set(ptr, filename, function(err) { + client.hset(key, 'metadata', metadataString, function() { + client.hset(key, 'post', post, callback); + }); + }); + }); + }); + }); +} + +exports.upsertPost = function(filename, callback) { + var returnValue; + exports.getPostByFilename(filename, function(err, found, metadata, post) { + if (found) { + //compare last modified times. + fileProcessor.getLastModified(filename, function(err, lastModified) { + if (err) return callback(err); + + if (lastModified.getTime() > metadata.lastModified.getTime()) { + exports.updatePost(filename, function(err) { + if (err) return callback(err); + callback(null, exports.upsertResult.UPDATE); + }); + } + else { + //no need to do anything at all. + process.nextTick(function() { + callback(null, exports.upsertResult.NONE); + }); + } + }); + } + else { + //brand new. + exports.insertPost(filename, function(err) { + if (err) return callback(err); + callback(null, exports.upsertResult.NEW); + }); + } + }); +} + +exports.updatePost = function(filename, callback) { + //for now this can delegate to insert since redis does insert/overwrite. + //might need it later if there need to be special rules for updates. + exports.insertPost(filename, callback); +} + +exports.removePost = function(filename, callback) { + var ptr = keyManager.toPostPointer(filename); + var key = keyManager.toPostKeyFromFilename(filename); + var title = keyManager.toTitle(filename); + + client.del(ptr, function(err) { + if (err) return callback(err); + + client.del(key, function(err) { + if (err) return callback(err); + + client.zrem(keyManager.blogDates, title, function(err) { + if (err) return callback(err); + + client.srem(keyManager.blogIndex, filename, function(err) { + if (err) return callback(err); + callback(null, filename); + }); + }); + }); + }); +} + +exports.removePostByTitle = function(title, callback) { + keyManager.toPostFilenameFromTitle(title, function(err, filename) { + if (err) return callback(err); + exports.removePost(filename, callback); + }); +} + +exports.removeAllPosts = function(callback) { + exports.listPosts(function(err, titles) { + if (err) return callback(err); + + //stuff that's easy to delete. + var tran = client.multi(); + tran.del(keyManager.blogDates); + tran.del(keyManager.blogIndex); + tran.del(keyManager.blogNewIndex); + + //Need to acquire all of the post filenames asyncly from redis, + //so use the async library to (readably) get them all into the multi. + var tasks = []; + titles.forEach(function(title) { + tasks.push(function(cb) { + keyManager.toPostFilenameFromTitle(title, function(err, filename) { + var key = keyManager.toPostKeyFromFilename(filename); + var ptr = keyManager.toPostPointer(filename); + tran.del(key); + tran.del(ptr); + cb(err); + }); + }); + }); + + async.parallel(tasks, function(err) { + if (err) return callback(err); + tran.exec(function(err, replies) { + if (err) return callback(err); + callback(null); + }); + }); + }); +} + +exports.cleanup = function(newIndex, callback) { + var t1 = [], t2 = []; + + //create a temporary "new index" set in redis. + newIndex.forEach(function(value) { + t1.push(function(cb) { + client.sadd(keyManager.blogNewIndex, value, cb); + }); + }); + + async.parallel(t1, function(err) { + if (err) return callback(err); + + client.sdiff(keyManager.blogIndex, keyManager.blogNewIndex, function(err, removedFilenames) { + if (err) return callback(err); + + //remove all deleted keys from the index and system. + removedFilenames.forEach(function(filename) { + t2.push(function(cb) { + exports.removePost(filename, function(err) { + if (err) cb(err); + client.srem(keyManager.blogIndex, filename, cb); + }); + }); + }); + + async.parallel(t2, function(err) { + if (err) return callback(err); + + client.del(keyManager.blogNewIndex, function(err) { + if (err) return callback(err); + callback(null, removedFilenames); + }); + }); + }); + }); +} diff --git a/lib/blog.js b/lib/blog.js new file mode 100644 index 0000000..a4a527a --- /dev/null +++ b/lib/blog.js @@ -0,0 +1,284 @@ +var hound = require('hound'), + events = require('events'), + util = require('util'), + path = require('path'), + async = require('async'), + ru = require('./reed-util'), + redis = require('./blog-connector'), + keyManager = require('./keymanager').KeyManager, + FilesystemHelper = require('./filesystem-helper').FilesystemHelper; + +//constants +var upsertResult = redis.upsertResult; + +//singleton to enable events. +//user code interacts with it through exports.on method. +function ReedBlog() { } +util.inherits(ReedBlog, events.EventEmitter); + +var blog = new ReedBlog(); + +//redis configuration (default to start, but can change) +//set by parent reed module. +var cfg = {}; + +//directory to watch +var dir; + +//misc objects: file watcher, filesystem helper, etc. +var watcher; +var fsh; + +//states +var open = false; +var ready = false; + +//method queue for when methods are called without the redis connection open. +var queue = []; + +//Private methods. +function watch() { + watcher = hound.watch(dir); + + watcher.on('create', function(filename, stats) { + if (ru.isMarkdown(filename, stats)) { + filename = path.resolve(process.cwd(), filename); + redis.insertPost(filename, function(err) { + if (err) return blog.emit('error', err); + var title = keyManager.toTitle(filename); + blog.emit('add', title); + }); + } + }); + + watcher.on('change', function(filename, stats) { + if (ru.isMarkdown(filename, stats)) { + filename = path.resolve(process.cwd(), filename); + console.log(filename); + redis.updatePost(filename, function(err) { + if (err) return blog.emit('error', err); + var title = keyManager.toTitle(filename); + blog.emit('update', title); + }); + } + }); + + watcher.on('delete', function(filename) { + if (ru.isMarkdownFilename(filename)) { + filename = path.resolve(process.cwd(), filename); + redis.removePost(filename, function(err) { + if (err) return blog.emit('error', err); + blog.emit('remove', filename); + }); + } + }); +} + +function initDirectory(callback) { + fsh.readMarkdown(dir, function(err, files) { + if (err) return callback(err); + + var newIndex = []; + var tasks = []; + files.forEach(function(filename) { + tasks.push(function(cb) { + var fullpath = path.resolve(dir, filename); + redis.upsertPost(fullpath, function(err, result) { + if (err) cb(err); + if (result == upsertResult.NEW) { + var title = keyManager.toTitle(filename); + blog.emit('add', title); + } + else if (result == upsertResult.UPDATE) { + var title = keyManager.toTitle(filename); + blog.emit('update', title); + } + + newIndex.push(fullpath); + cb(null); + }); + }); + }); + + async.parallel(tasks, function(err) { + if (err) return callback(err); + + redis.cleanup(newIndex, function(err, removedFilenames) { + if (err) return callback(err); + + removedFilenames.forEach(function(filename) { + blog.emit('remove', filename); + }); + + callback(null); + }); + }); + }); +} + +//Connection methods. +blog.configure = function(config) { + //selectively overwrite default config properties. this way the user + //only needs to override what's necessary. + for (prop in config) { + if (config[prop]) { + cfg[prop] = config[prop]; + } + } +} + +blog.open = function(directory) { + if (open === true || ready === true) { + throw new Error('reed already open on ' + dir); + } + + if (typeof directory !== 'string') { + throw new Error('Must specify directory to read from'); + } + + dir = directory; + + redis.open(cfg, function(err, success) { + if (err) return blog.emit('error', err); + + fsh = new FilesystemHelper(directory); + initDirectory(function(err) { + if (err) return blog.emit('error', err); + watch(); + open = true; + ready = true; + + //handle any queued method calls. + queue.forEach(function(queuedCall) { + queuedCall(); + }); + + queue = []; + blog.emit('ready'); + }); + }); +} + +blog.close = function() { + if (!open || !ready) { + throw new Error('reed is not open.'); + } + + redis.close(); + watcher.clear(); + ready = false; + open = false; + queue = []; +} + +//Data manipulation methods. +blog.get = function(title, callback) { + if (!open) return queue.push(function() { + blog.get(title, callback); + }); + + redis.getPost(title, function(err, found, metadata, post) { + if (err) return callback(err); + + if (found) { + callback(null, metadata, post); + } + else { + callback(new Error('Could not find post: ' + title)); + } + }); +} + +blog.getMetadata = function(title, callback) { + if (!open) return queue.push(function() { + blog.getMetadata(title, callback); + }); + + blog.get(title, function(err, metadata, post) { + if (err) return callback(err); + callback(null, metadata); + }); +} + +blog.all = function(callback) { + if (!open) return queue.push(function() { + blog.all(callback); + }); + + blog.list(function(err, titles) { + if (err) return callback(err); + + //create the series to load all posts asyncly in order. + var getAllPosts = []; + titles.forEach(function(title) { + getAllPosts.push(function(cb) { + blog.get(title, function(err, metadata, htmlContent) { + var post = { + metadata: metadata, + htmlContent: htmlContent + }; + + cb(err, post); + }); + }); + }); + + //get all the posts. + async.series(getAllPosts, function(err, posts) { + callback(null, posts); + }); + }); +} + +blog.list = function(callback) { + if (!open) return queue.push(function() { + blog.list(callback); + }); + + redis.listPosts(function(err, titles) { + if (err) return callback(err); + callback(null, titles); + }); +} + +blog.remove = function(title, callback) { + if (!open) return queue.push(function() { + blog.remove(title, callback); + }); + + redis.removePostByTitle(title, function(err, filename) { + if (err) return callback(err); + + fsh.remove(filename, function(err) { + if (err) return callback(err); + callback(null); + }); + }); +} + +blog.removeAll = function(callback) { + if (!open) return queue.push(function() { + blog.removeAll(callback); + }); + + redis.removeAllPosts(function(err) { + if (err) return callback(err); + + fsh.removeAllPosts(function(err) { + if (err) return callback(err); + callback(null); + }); + }); +} + +//Deprecated methods +blog.index = function(callback) { + console.log('index is deprecated and will be removed in the next version.'); +} + +blog.refresh = function() { + console.log('refresh is deprecated and will be removed in the next version.'); +} + +//The module itself is an event-based object. +module.exports = blog; diff --git a/lib/file-processor.js b/lib/file-processor.js new file mode 100644 index 0000000..3656a98 --- /dev/null +++ b/lib/file-processor.js @@ -0,0 +1,56 @@ +var fs = require('fs'), + parseMarkdown = require("node-markdown").Markdown; + +//taken from wheat -- MIT license +function preProcess(markdown) { + if (!(typeof markdown === 'string')) { + markdown = markdown.toString(); + } + + var props = {}; + + // Parse out headers + var match; + while(match = markdown.match(/^([a-z]+):\s*(.*)\s*\n/i)) { + var name = match[1]; + name = name[0].toLowerCase() + name.substring(1); + var value = match[2]; + markdown = markdown.substr(match[0].length); + props[name] = value; + } + + props.markdown = markdown; + return props; +} + +//Callback receives: +// err +// postDate - string version of the post date (from getTime()) +// metadata +// post content - the HTML +exports.process = function(filename, callback) { + fs.readFile(filename, function(err, data) { + if (err) return callback(err); + + if (typeof data === 'undefined') { + return callback(new Error('No data for ' + filename)); + } + + var metadata = preProcess(data.toString()); + var post = parseMarkdown(metadata.markdown); + + fs.stat(filename, function(err, stats) { + if (err) return callback(err); + var postDate = stats.mtime.getTime(); + metadata.lastModified = postDate; + callback(null, postDate, metadata, post); + }); + }); +} + +exports.getLastModified = function(filename, callback) { + fs.stat(filename, function(err, stats) { + if (err) return callback(err); + callback(null, stats.mtime); + }); +} diff --git a/lib/filesystem-helper.js b/lib/filesystem-helper.js new file mode 100644 index 0000000..27cbcad --- /dev/null +++ b/lib/filesystem-helper.js @@ -0,0 +1,56 @@ +var fs = require('fs'), + path = require('path'), + async = require('async'), + ru = require('./reed-util'); + +function FilesystemHelper(directory) { + this.dir = directory; +} + +FilesystemHelper.prototype.exists = function(filename, callback) { + fs.exists(this.dir + filename, callback); +} + +FilesystemHelper.prototype.readMarkdown = function(dir, callback) { + fs.readdir(dir, function(err, files) { + if (err) return callback(err); + var self = this; + var filenames = []; + + var tasks = []; + files.forEach(function(file) { + tasks.push(function(cb) { + var fullpath = path.join(dir, file); + fs.stat(fullpath, function(err, stats) { + if (err) return callback(err); + if (ru.isMarkdown(file, stats)) { + filenames.push(file); + } + + cb(null); + }); + }); + }); + + async.parallel(tasks, function(err) { + if (err) return callback(err); + callback(null, filenames); + }); + }); +} + +FilesystemHelper.prototype.remove = function(filename, callback) { + fs.unlink(filename, function(err) { + if (typeof callback !== 'undefined') callback(unlinkErr); + }); +} + +FilesystemHelper.prototype.removeAll = function() { + this.readMarkdown(this.dir, function(files) { + files.forEach(function(filename) { + this.remove(filename); + }); + }); +} + +exports.FilesystemHelper = FilesystemHelper; diff --git a/lib/keymanager.js b/lib/keymanager.js new file mode 100644 index 0000000..04fff89 --- /dev/null +++ b/lib/keymanager.js @@ -0,0 +1,78 @@ +var path = require('path'), + S = require('string') + conn = require('./redis-connection'); + +//redis open/close methods +function open(cfg, callback) { + conn.open(cfg, function(err, redisClient) { + if (err) return callback(err, false); + client = redisClient; + callback(err, false); + }); +} + +function close() { + conn.close(); +} + +//redis client +var client; + +var keyManager = { + blogIndex: 'reed:blog:index', + blogNewIndex: 'reed:blog:newindex', + blogDates: 'reed:blog:dates', + pagesIndex: 'reed:pages:index', + pagesNewIndex: 'reed:pages:newindex', + + open: open, + close: close, + + toPostKeyFromFilename: function(filename) { + if (!S(filename).startsWith('reed:blog:')) { + return 'reed:blog:' + filename; + } + else { + return filename; + } + }, + + toPagesKeyFromFilename: function(filename) { + if (!S(filename).startsWith('reed:pages:')) { + return 'reed:pages:' + filename; + } + else { + return filename; + } + }, + + toPostFilenameFromTitle: function(title, callback) { + client.get('reed:blogpointer:' + title, function(err, key) { + if (err) return callback(err); + callback(null, key); + }); + }, + + toPagesFilenameFromTitle: function(title, callback) { + client.get('reed:pagespointer:' + title, function(err, key) { + if (err) return callback(err); + callback(null, key); + }); + }, + + toTitle: function(filename) { + var ext = path.extname(filename); + var title = path.basename(filename, ext); + return title; + }, + + toPostPointer: function(filename) { + return 'reed:blogpointer:' + keyManager.toTitle(filename); + }, + + toPagesPointer: function(filename) { + return 'reed:pagespointer:' + keyManager.toTitle(filename); + } +}; + +exports.KeyManager = keyManager; diff --git a/lib/pages-connector.js b/lib/pages-connector.js new file mode 100644 index 0000000..cfd2987 --- /dev/null +++ b/lib/pages-connector.js @@ -0,0 +1,145 @@ +var async = require('async'), + conn = require('./redis-connection'), + fileProcessor = require('./file-processor'), + keyManager = require('./keymanager').KeyManager; + +//the redis client +var client; + +exports.open = function(cfg, callback) { + keyManager.open(cfg, function(err) { + if (err) return callback(err, false); + conn.open(cfg, function(err, redisClient) { + if (err) return callback(err, false); + client = redisClient; + callback(err, false); + }); + }); +} + +exports.close = function() { + conn.close(); + keyManager.close(); +} + +exports.getPageFilenameForTitle = function(title, callback) { + keyManager.toPagesFilenameFromTitle(title, function(err, filename) { + if (err) return callback(err); + callback(null, filename); + }); +} + +exports.getPage = function(title, callback) { + keyManager.toPagesFilenameFromTitle(title, function(err, filename) { + if (err) return callback(err); + exports.getPageByFilename(filename, callback); + }); +} + +exports.getPageByFilename = function(filename, callback) { + var key = keyManager.toPagesKeyFromFilename(filename); + + client.hgetall(key, function(err, hash) { + if (typeof(hash) !== 'undefined' && hash != null && Object.keys(hash).length > 0) { + var post = hash.post; + if (typeof callback !== "undefined") { + var metadata = {}; + try { + metadata = JSON.parse(hash.metadata); + } + catch (parseErr) { + //no good metadata - ignore + } + + if (typeof metadata.lastModified !== 'undefined') { + metadata.lastModified = new Date(metadata.lastModified); + } + + callback(null, true, metadata, hash.post); + } + } + else { + callback(new Error('Page not found: ' + filename), false); + } + }); +} + +exports.insertPage = function(filename, callback) { + fileProcessor.process(filename, function(err, postDate, metadata, post) { + if (err) return callback(err); + + var ptr = keyManager.toPagesPointer(filename); + var key = keyManager.toPagesKeyFromFilename(filename); + var title = keyManager.toTitle(filename); + + metadataString = JSON.stringify(metadata); + client.sadd(keyManager.pagesIndex, filename, function(err) { + client.set(ptr, filename, function(err) { + client.hset(key, 'metadata', metadataString, function() { + client.hset(key, 'post', post, callback); + }); + }); + }); + }); +} + +exports.updatePage = function(filename, callback) { + exports.insertPage(filename, callback); +} + +exports.removePage = function(filename, callback) { + var ptr = keyManager.toPagesPointer(filename); + var key = keyManager.toPagesKeyFromFilename(filename); + var title = keyManager.toTitle(filename); + + client.del(ptr, function(err) { + if (err) return callback(err); + + client.del(key, function(err) { + if (err) return callback(err); + + client.srem(keyManager.pagesIndex, filename, function(err) { + if (err) return callback(err); + callback(null, filename); + }); + }); + }); +} + +exports.cleanupPages = function(newIndex, callback) { + var t1 = [], t2 = []; + + //create a temporary "new index" set in redis. + newIndex.forEach(function(value) { + t1.push(function(cb) { + client.sadd(keyManager.pagesNewIndex, value, cb); + }); + }); + + async.parallel(t1, function(err) { + if (err) return callback(err); + + client.sdiff(keyManager.pagesIndex, keyManager.pagesNewIndex, function(err, removedFilenames) { + if (err) return callback(err); + + //remove all deleted keys from the index and system. + removedFilenames.forEach(function(filename) { + t2.push(function(cb) { + exports.removePage(filename, function(err) { + if (err) cb(err); + client.srem(keyManager.pagesIndex, filename, cb); + }); + }); + }); + + async.parallel(t2, function(err) { + if (err) return callback(err); + + client.del(keyManager.pagesNewIndex, function(err) { + if (err) return callback(err); + callback(null, removedFilenames); + }); + }); + }); + }); +} diff --git a/lib/pages.js b/lib/pages.js new file mode 100644 index 0000000..d7bd978 --- /dev/null +++ b/lib/pages.js @@ -0,0 +1,174 @@ +var util = require('util'), + events = require('events'), + path = require('path'), + async = require('async'), + hound = require('hound'), + redis = require('./pages-connector'), + FilesystemHelper = require('./filesystem-helper').FilesystemHelper; + +//singleton to enable events. +//user code interacts with it through exports.on method. +function ReedPages() { } +util.inherits(ReedPages, events.EventEmitter); + +var pages = new ReedPages(); + +//constants +var upsertResult = redis.upsertResult; + +//directory to watch +var dir; + +//redis configuration (default to start, but can change) +//set by parent reed module. +var cfg = {}; + +//states +var open = false; +var ready = false; + +var fsh; + +//methods queued because the connection isn't open +var queue = []; + +function watch() { + watcher = hound.watch(dir); + + watcher.on('create', function(filename, stats) { + if (ru.isMarkdown(filename, stats)) { + filename = path.resolve(process.cwd(), filename); + redis.insertPage(filename, function(err) { + if (err) return pages.emit('error', err); + }); + } + }); + + watcher.on('change', function(filename, stats) { + if (ru.isMarkdown(filename, stats)) { + filename = path.resolve(process.cwd(), filename); + console.log(filename); + redis.updatePage(filename, function(err) { + if (err) return pages.emit('error', err); + }); + } + }); + + watcher.on('delete', function(filename) { + if (ru.isMarkdownFilename(filename)) { + filename = path.resolve(process.cwd(), filename); + redis.removePage(filename, function(err) { + if (err) return pages.emit('error', err); + }); + } + }); +} + +function initDirectory(callback) { + fsh.readMarkdown(dir, function(err, files) { + if (err) return callback(err); + + var newIndex = []; + var tasks = []; + files.forEach(function(filename) { + tasks.push(function(cb) { + var fullpath = path.resolve(dir, filename); + redis.insertPage(fullpath, function(err, result) { + if (err) cb(err); + newIndex.push(fullpath); + cb(null); + }); + }); + }); + + async.parallel(tasks, function(err) { + if (err) return callback(err); + + redis.cleanupPages(newIndex, function(err, removedFilenames) { + if (err) return callback(err); + callback(null); + }); + }); + }); +} + +pages.configure = function(config) { + //selectively overwrite default config properties. this way the user + //only needs to override what's necessary. + for (prop in config) { + if (config[prop]) { + cfg[prop] = config[prop]; + } + } +} + +pages.open = function(directory, callback) { + if (open === true || ready === true) { + throw new Error('reed pages already open on ' + dir); + } + + if (typeof directory !== 'string') { + throw new Error('Must specify directory to read from'); + } + + dir = directory; + redis.open(cfg, function(err, success) { + if (err) return callback(err); + + fsh = new FilesystemHelper(directory); + initDirectory(function(err) { + if (err) return pages.emit('error', err); + watch(); + open = true; + ready = true; + + //handle any queued method calls. + queue.forEach(function(queuedCall) { + queuedCall(); + }); + + queue = []; + callback(null); + }); + }); +} + +pages.close = function() { + if (!open || !ready) { + throw new Error('reed pages is not open.'); + } + + redis.close(); + open = false; + ready = false; + queue = []; +} + +pages.get = function(title, callback) { + if (!open) return queue.push(function() { + pages.get(title, callback); + }); + + redis.getPage(title, function(err, found, metadata, page) { + if (err) return callback(err); + if (found) { + callback (null, metadata, page); + } + else { + callback(new Error('Could not find page: ' + title)); + } + }); +} + +pages.remove = function(title, callback) { + if (!open) return queue.push(function() { + pages.remove(title, callback); + }); + + redis.removePage(title, function(err) { + callback(err); + }); +} + +//Export +module.exports = pages; diff --git a/lib/redis-connection.js b/lib/redis-connection.js new file mode 100644 index 0000000..53d912b --- /dev/null +++ b/lib/redis-connection.js @@ -0,0 +1,53 @@ +var redis = require('redis'); + +var connections = 0; + +//The redis connection. +var client; +var open = false; + +exports.open = function(cfg, callback) { + connections++; + //already open? + if (typeof client !== 'undefined' && open) { + return process.nextTick(function() { + callback(null, client); + }); + } + + client = redis.createClient(cfg.port, cfg.host); + open = true; + + //authentication may cause redis to fail + //this ensures we see the problem if it occurs + client.on('error', function(errMsg) { + connections--; + open = false; + callback(errMsg); + }); + + if (cfg.password) { + //if we are to auth we need to wait on callback before + //starting to do work against redis + return client.auth(cfg.password, function (err) { + if (err) return callback(err); + callback(null, client); + }); + + } + else { + //no auth, just start + return process.nextTick(function() { + callback(null, client); + }); + } +} + +exports.close = function() { + connections--; + + if (connections == 0) { + client.quit(); + open = false; + } +} diff --git a/lib/reed-util.js b/lib/reed-util.js new file mode 100644 index 0000000..caf386e --- /dev/null +++ b/lib/reed-util.js @@ -0,0 +1,7 @@ +exports.isMarkdownFilename = function(filename) { + return exports.endsWith(file, '.md') || exports.endsWith(file, '.markdown'); +} + +exports.isMarkdown = function(file, stats) { + return stats.isFile() && (exports.endsWith(file, '.md') || exports.endsWith(file, '.markdown')); +} diff --git a/lib/reed.js b/lib/reed.js index aad6c07..e8ea3a7 100644 --- a/lib/reed.js +++ b/lib/reed.js @@ -1,786 +1,20 @@ -var fs = require("fs"), - path = require("path"), - util = require("util"), - events = require("events"), - redis = require("redis"), - async = require("async"), - md = require("node-markdown").Markdown; - -//singleton to enable events. -//user code interacts with it through exports.on method. -function Reed() { } -util.inherits(Reed, events.EventEmitter); - -var reed = new Reed(); - -//the config object. -var cfg; - -//the redis client. -var client; - -//directory to look for markdown files in. -//set by exports.open and exports.pages.open -var dir; -var pagesDir; - -//if we are open, further calls to open() will be rejected. -//if we are ready, allow calls to the methods. -var open = false; -var ready = false; -var pagesOpen = false; -var pagesReady = false; - -function endsWith(str, text) { - return str.substr(-text.length) === text; - return str.substr(-text.length) === text; -} - -function startsWith(str, text) { - return str.substr(0, text.length) === text; -} - -function redisInsert(key, date, processedData, post, callback) { - processedData = JSON.stringify(processedData); - client.zadd("blog:dates", date, key, function(err) { - if (startsWith(key, "blog:") == false) { - key = "blog:" + key; - } - client.hset(key, "metadata", processedData, function() { - client.hset(key, "post", post, callback); - }); - }); -} - -function redisPageInsert(key, date, processedData, post, callback) { - processedData = JSON.stringify(processedData); - - if (startsWith(key, "page:") == false) { - key = "page:" + key; - } - - client.hset(key, "metadata", processedData, function() { - client.hset(key, "post", post, callback); - }); -} - -function redisDelete(title, callback) { - var filename = toFilename(title); - var key = toKey(title); - - client.del(key, function(err) { - if (err) { callback(err); return; } - client.zrem("blog:dates", filename, function(err) { - if (err) { - callback(err); - } - else { - callback(null); - } - }); - }); -} - -function redisPageDelete(title, callback) { - var key = toPagesKey(title); - - client.del(key, function(err) { - if (err) { - callback(err); - } - else { - callback(null); - } - }); -} - -//taken from wheat -- MIT license -function preProcess(markdown) { - if (!(typeof markdown === 'string')) { - markdown = markdown.toString(); - } - - var props = {}; - - // Parse out headers - var match; - while(match = markdown.match(/^([a-z]+):\s*(.*)\s*\n/i)) { - var name = match[1]; - name = name[0].toLowerCase() + name.substring(1); - var value = match[2]; - markdown = markdown.substr(match[0].length); - props[name] = value; - } - props.markdown = markdown; - - return props; -} - -function readMarkdown(directory, callback) { - fs.readdir(directory, function(err, files) { - if (typeof files === "undefined") { - callback(err); - return; - } - - files = files.filter(function(el, i, arr) { - return endsWith(el, ".md"); - }); - - callback(null, files); - }); -} - -function loadFromFilesystem(filename, callback) { - fs.readFile(filename, function(err, data) { - if (err) return callback(err); - - if (typeof(data) === "undefined") { - if (typeof callback !== "undefined") callback(err); - return; - } - - var metadata = preProcess(data.toString()); - metadata.id = toID(filename); - var post = md(metadata.markdown); - - var postDate = fs.statSync(filename).mtime.getTime(); - metadata.lastModified = postDate; - - redisInsert(filename, postDate, metadata, post, function() { - metadata.lastModified = new Date(metadata.lastModified); - if (watched[filename] !== true) { - reed.emit("add", metadata, post); - } - - watch(filename); - - if (typeof callback !== "undefined") - callback(null, metadata, post); - }); - }); -} - -function loadPageFromFilesystem(filename, callback) { - fs.readFile(filename, function(err, data) { - if (err) return callback(err); - - if (typeof(data) === "undefined") { - if (typeof callback !== "undefined") callback(err); - return; - } - - var metadata = preProcess(data.toString()); - metadata.id = toID(filename); - var post = md(metadata.markdown); - - var postDate = fs.statSync(filename).mtime.getTime(); - metadata.lastModified = postDate; - - redisPageInsert(filename, postDate, metadata, post, function() { - metadata.lastModified = new Date(metadata.lastModified); - watchPage(filename); - - if (typeof callback !== "undefined") - callback(null, metadata, post); - }); - }); -} - -var watched = {}; -function watch(filename) { - if (watched[filename] !== true) { - watched[filename] = true; - fs.watchFile(filename, function(curr, prev) { - var exists = path.existsSync(filename); - - if (exists) { - loadFromFilesystem(filename, function(err, metadata, htmlContent) { - if (err != null) { - reed.emit("error", err); - } - else { - reed.emit("update", metadata, htmlContent); - } - }); - } - else { - redisDelete(filename, function(err) { - delete watched[filename]; - fs.unwatchFile(filename); - - if (err) { - reed.emit("error", err); - } - else { - reed.emit("remove", filename); - } - }); - } - }); - } -} - -var watchedPages = {}; -function watchPage(filename) { - if (watchedPages[filename] !== true) { - watchedPages[filename] = true; - fs.watchFile(filename, function(curr, prev) { - var exists = path.existsSync(filename); - - if (exists) { - loadPageFromFilesystem(filename, function(err, metadata, htmlContent) { - if (err != null) { - reed.emit("error", err); - } - }); - } - else { - redisPageDelete(filename, function(err) { - delete watched[filename]; - fs.unwatchFile(filename); - - if (err) { - reed.emit("error", err); - } - }); - } - }); - } -} - -function findNewFiles(firstTime) { - readMarkdown(dir, function(err, files) { - if (typeof files === "undefined") { - reed.emit("error", err); - return; - } - - //if the directory is empty, we are automatically ready. - if (files.length === 0) { - ready = true; - reed.emit("ready"); - } - - var c = 0; - files.forEach(function(file) { - file = toFilename(file); - if (watched[file] !== true) { - //new file? - reed.getMetadata(file, function(err, md) { - if (err != null) { - reed.emit("error", err); - return; - } - - fs.stat(file, function(err, stats) { - if (err != null) { - reed.emit("error", err); - return; - } - - if (md.lastModified !== stats.mtime.getTime()) { - //new file! - loadFromFilesystem(file, function(err) { - if (err != null) { - reed.emit("error", err); - } - - //emit ready event if we are done. - if (firstTime && c + 1 === files.length) { - ready = true; - reed.emit("ready"); - } - c++; - }); - } - else { - watch(file); - //emit ready event if we are done. - if (firstTime && c + 1 === files.length) { - ready = true; - reed.emit("ready"); - } - c++; - } - }); - }); - } - }); //end foreach - }); -} - -function toFilename(title) { - var filename = title.replace(" ", "-"); - if (!endsWith(filename, ".md")) filename += ".md"; - if (!startsWith(filename, dir)) filename = dir + "/" + filename; - return filename; -} - -function toKey(title) { - var filename = toFilename(title); - return "blog:" + filename; -} - -function toPagesFilename(title) { - var filename = title.replace(" ", "-"); - if (!endsWith(filename, ".md")) filename += ".md"; - if (!startsWith(filename, pagesDir)) filename = pagesDir + "/" + filename; - return filename; -} - -function toPagesKey(title) { - var filename = toPagesFilename(title); - return "page:" + filename; -} - -function toID(filename) { - var start = filename.lastIndexOf("/") + 1; - var id = filename.substring(start, filename.length - 3); - return id; -} - -function readyCheck(fn, args) { - if (ready) { - return true; - } - else { - process.nextTick(function() { - fn.apply(module, args); - }); - - return false; - } -} - -function pagesReadyCheck(fn, args) { - if (pagesReady) { - return true; - } - else { - process.nextTick(function() { - fn.apply(module, args); - }); - - return false; - } -} - -function openRedis(callback) { - //already open? - if (typeof client !== 'undefined') { - process.nextTick(function() { - callback(false); - }); - } - - //declare all props that we want for redis - var redisConf = { - host: '127.0.0.1', - port: 6379, - password: null - }; - - if (cfg) { - //only take the props we are interested in - //this way the user can specify only - //host or only port leaving the rest - // as defaults - for (prop in redisConf) { - if (cfg[prop]) { - redisConf[prop] = cfg[prop]; - } - } - } - - client = redis.createClient(redisConf.port, redisConf.host); - - //authentication may cause redis to fail - //this ensures we see the problem if it occurs - client.on('error', function(errMsg) { - reed.emit('error', errMsg); - }); - - if (redisConf.password) { - //if we are to auth we need to wait on callback before - //starting to do work against redis - client.auth(redisConf.password, function (err) { - if (err) return reed.emit('error', err); - callback(true); - }); - - } - else { - //no auth, just start - process.nextTick(function() { - callback(true); - }); - } -} - -reed.getMetadata = function(title, callback) { - //convert to key/filename - var filename = toFilename(title); - var key = toKey(title); - - //first hit redis - client.hget(key, "metadata", function(err, metadata) { - try { - metadata = JSON.parse(metadata); - } - catch (parseErr) { - //if we can't understand it we don't want it - metadata = {}; - } - if (typeof(metadata) !== "undefined" && metadata != null && Object.keys(metadata).length > 0) { - if (typeof callback !== "undefined") - callback(null, metadata); - } - else { - //now hit filesystem. - loadFromFilesystem(filename, callback); - } - }); -} - -reed.get = function get(title, callback) { - if (!readyCheck(get, arguments)) return; - - //convert to key/filename - var filename = toFilename(title); - var key = toKey(title); - - //first hit redis - client.hgetall(key, function(err, hash) { - var post = hash.post; - if (typeof(hash) !== "undefined" && Object.keys(hash).length > 0) { - if (typeof callback !== "undefined") { - var metadata = {}; - try { - metadata = JSON.parse(hash.metadata); - } - catch (parseErr) { - //no good metadata - ignore - } - - if (typeof metadata.lastModified !== 'undefined') { - metadata.lastModified = new Date(metadata.lastModified); - } - callback(null, metadata, hash.post); - } - } - else { - //now hit filesystem. - loadFromFilesystem(filename, callback); - } - }); -} - -reed.index = function index(callback) { - if (!readyCheck(index, arguments)) return; - - readMarkdown(dir, function(err, files) { - if (typeof files === "undefined") { - callback(err); - return; - } - - var mdFilter = function(el, i, arr) { - return endsWith(el, ".md"); - } - - var c = 0; - var skip = false; - files = files.filter(mdFilter); - for (var x = 0; x < files.length; x++) { - if (skip) break; //probably shouldn't hit this, but who knows. - var file = toFilename(files[x]); - loadFromFilesystem(file, function(err) { - if (skip) return; //SHOULD hit this though. - if (err != null) { - if (typeof(callback) !== "undefined") { - skip = true; - callback(err); - return; - } - } - else { - if (c + 1 == files.length) { - if (typeof(callback) !== "undefined") - callback(null); - return; - } - - c++; - } - }); - } - }); -} - -reed.list = function list(callback) { - if (!readyCheck(list, arguments)) return; - - var titles = []; - client.zrevrange("blog:dates", 0, -1, function(err, keys) { - keys.forEach(function(key) { - //5 = "blog:", length - 3 = ".md" - var title = key.substring(key.lastIndexOf('/') + 1, key.length - 3); - title = title.replace("-", " "); - titles.push(title); - }); - - if (typeof callback !== "undefined") - callback(null, titles); - }); -} - -reed.all = function(callback) { - reed.list(function(err, titles) { - //create the series to load all posts asyncly in order. - var getAllPosts = []; - titles.forEach(function(title) { - getAllPosts.push(function(cb) { - reed.get(title, function(err, metadata, htmlContent) { - var post = { - metadata: metadata, - htmlContent: htmlContent - }; - - cb(err, post); - }); - }); - }); - - //get all the posts. - async.series(getAllPosts, function(err, posts) { - callback(null, posts); - }); - }); -} - -reed.refresh = function refresh() { - if (!readyCheck(refresh, arguments)) return; - - reed.list(function(err, titles) { - if (err) { reed.emit("error", err); return; } - - //create async tasks. - //this tabular monster is jut a fully async version of what happens - //to watched files: if the file no longer exists, it is removed from - //redis and a remove event is emitted. - var tasks = []; - titles.forEach(function(title) { - tasks.push(function(callback) { - path.exists(toFilename(title), function(exists) { - if (!exists) { - redisDelete(toFilename(title), function(err) { - if (err) { - callback(err); - } - else { - //since the file was removed when reed was - //not running, we must emit the remove event. - reed.emit("remove", toFilename(title)); - callback(null); - } - }); - } - }); - }); - }); - - async.parallel(tasks, function(err) { - if (err) { - reed.emit("error", err); - } - }); - }); -} - -reed.remove = function remove(title, callback) { - if (!readyCheck(remove, arguments)) return; - - var key = toKey(title); - var filename = toFilename(title); - redisDelete(key, function(err) { - if (err) { callback(err); return; } - fs.unlink(filename, function(unlinkErr) { - //no need to emit a remove event here because - //the file watching emits it for us when the - //file is deleted. - callback(unlinkErr); - }); - }); -} - -reed.removeAll = function removeAll(callback) { - if (!readyCheck(removeAll, arguments)) return; - - function delegate(key) { - return function(callback) { - client.del(key, function(err) { - if (err != null) callback(err); - else callback(null); - }); - } - } - - var funcs = []; - for (file in watched) { - funcs.push(delegate(toKey(file))); - } - - funcs.push(delegate("blog:dates")); - - async.waterfall(funcs, function(err) { - if (!err && typeof callback !== 'undefined') { - callback(null); - } - else if (typeof callback !== 'undefined') { - callback(err); - } - }); -} - -reed.configure = function(config) { - cfg = config; -} - -reed.open = function(directory) { - if (open === true || ready === true) { - throw new Error("reed already open on " + dir); - } - else { - open = true; - } - - if (typeof(directory) !== "string") { - throw new Error("Must specify directory to read from"); - } - - dir = directory; - - openRedis(function() { - fs.watchFile(dir, function(curr, prev) { - findNewFiles(); - }); - - reed.refresh(); - - process.nextTick(function() { - findNewFiles(true); - }); - }); -} - -reed.close = function() { - //only quit if pages is not in use. - if (!pagesReady) { - client.quit(); - } - - for (file in watched) { - if (watched[file] === true) { - fs.unwatchFile(file); - } - } - - watched = {}; - fs.unwatchFile(dir); - open = false; - ready = false; -} - -//Pages functionality -reed.pages = {}; -reed.pages.open = function(directory, callback) { - if (pagesOpen === true || pagesReady === true) { - throw new Error("reed already open on " + dir); - } - else { - pagesOpen = true; - pagesReady = true; - } - - if (typeof(directory) !== "string") { - throw new Error("Must specify directory to read from"); - } - - openRedis(function() { - pagesDir = directory; - reed.emit("pagesReady"); - if (typeof callback == 'function') { - callback(); - } - }); -} - -reed.pages.get = function getPage(title, callback) { - if (!pagesReadyCheck(getPage, arguments)) return; - - //convert to key/filename - var filename = toPagesFilename(title); - var key = toPagesKey(title); - - //first hit redis - client.hgetall(key, function(err, hash) { - var post = hash.post; - if (typeof(hash) !== "undefined" && Object.keys(hash).length > 0) { - if (typeof callback !== "undefined") { - var metadata = {}; - try { - metadata = JSON.parse(hash.metadata); - } - catch (parseErr) { - //no good metadata - ignore - } - - if (typeof metadata.lastModified !== 'undefined') { - metadata.lastModified = new Date(metadata.lastModified); - } - callback(null, metadata, hash.post); - } - } - else { - //now hit filesystem. - loadPageFromFilesystem(filename, callback); - } - }); -} - -reed.pages.remove = function remove(title, callback) { - if (!pagesReadyCheck(remove, arguments)) return; - - var key = toPagesKey(title); - var filename = toPagesFilename(title); - - fs.unlink(filename, function(unlinkErr) { - if (unlinkErr) return callback(unlinkErr); - redisPageDelete(key, function(err) { - if (err) return callback(err); - callback(null); - }); - }); -} - -reed.pages.close = function() { - //only quit if blog part is not in use. - if (!ready) { - client.quit(); - } - - for (file in watchedPages) { - if (watchedPages[file] === true) { - fs.unwatchFile(file); - } - } - - watchedPages = {}; - pagesOpen = false; - pagesReady = false; -} - -//Export an event emitting object that also has the various control methods -//on it. -module.exports = reed; +var util = require('util'), + events = require('events'), + blog = require('./blog'), + pages = require('./pages'); + +//default redis configuration. +var cfg = { + host: '127.0.0.1', + port: 6379, + password: null +}; + +blog.configure(cfg); +pages.configure(cfg); +pages.on('error', function(err) { + blog.emit('error', err); +}); + +module.exports = blog; +module.exports.pages = pages; diff --git a/package.json b/package.json index 26fe4b7..85d6de7 100644 --- a/package.json +++ b/package.json @@ -4,14 +4,14 @@ "email": "rei@thermetics.net", "description": "Redis + markdown blogging/website core", "tags": [ "redis", "blog" ], - "version": "0.9.8", + "version": "1.0.0", "homepage": "http://www.agnos.is/", "repository": { "type": "git", "url": "git://github.com/ProjectMoon/reed.git" }, "engines": { - "node": ">= 0.4.0" + "node": ">= 0.8.0" }, "directories": { "lib": "./lib" @@ -20,7 +20,9 @@ "dependencies": { "redis": "", "node-markdown": "", - "async": "" + "async": "", + "hound": "", + "string": "" }, "devDependencies": { "vows": ""