cookie 模拟登陆

rockdai · Nov 5, 2014 · 53e6463 · 53e6463
1 parent e0f1201
commit 53e6463
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 1 deletion.
diff --git a/config.js b/config.js
@@ -1,6 +1,7 @@
 var config = {
   mongodb_url: process.env.MONGOHQ_URL || 'mongodb://localhost/haixiu',
   port: process.env.PORT || 3000,
+  douban_cookie: 'viewed="3590768_1016272"; ll="108296"; bid="hWcTdvDyOYI"; ct=y; _ga=GA1.2.1167184662.1412271513; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1415179087%2C%22https%3A%2F%2Fwww.google.com%2F%22%5D; __utmt=1; as="http://www.douban.com/"; dbcl2="105659582:Xb5v0vuj9OM"; ck="GKUf"; _pk_id.100001.8cb4=d9658059b6096e90.1412332268.27.1415180767.1415172287.; _pk_ses.100001.8cb4=*; push_noty_num=0; push_doumail_num=0; __utma=30149280.1167184662.1412271513.1415166517.1415179088.44; __utmb=30149280.29.10.1415179088; __utmc=30149280; __utmz=30149280.1415162297.42.25.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); __utmv=30149280.10565',
 };
 
 exports = module.exports = config;
diff --git a/crawler.js b/crawler.js
@@ -5,6 +5,7 @@ var cheerio = require('cheerio');
 var model = require('./model');
 var Post = model.Post;
 var eventproxy = require('eventproxy');
+var config = require('./config');
 
 var q = async.queue(function (task, callback) {
   var postInfo = task;
@@ -22,6 +23,7 @@ var q = async.queue(function (task, callback) {
   ep.all('fetch_author', function () {
     superagent.get(postInfo.author_url)
       .set('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36')
+      .set('Cookie', config.douban_cookie)
       .end(ep.done(function (res) {
         if (res.status !== 200) {
           console.error(403, postInfo.author_url);
@@ -48,7 +50,8 @@ var q = async.queue(function (task, callback) {
   ep.all('got_author', function () {
     callback();
   });
-}, 3);
+// 并发数
+}, 1);
 
 function fetchHaixiuzu() {
   var ep = new eventproxy();
@@ -57,6 +60,7 @@ function fetchHaixiuzu() {
   });
   superagent.get('https://database.duoshuo.com/api/threads/listPosts.json?thread_key=haixiuzu&page=1&limit=100')
     .set('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36')
+    .set('Cookie', config.douban_cookie)
     .end(ep.done(function (res) {
       var json = JSON.parse(res.text);
       var parentPosts = json.parentPosts;