ditect_spam.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from requests_oauthlib import OAuth1Session\n",
    "from requests.exceptions import ConnectionError, ReadTimeout, SSLError\n",
    "from elasticsearch import Elasticsearch\n",
    "import json, datetime, time, pytz, re, sys,traceback, pymongo\n",
    "from pymongo import MongoClient\n",
    "from collections import defaultdict\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "KEYS = { # 自分のアカウントで入手したキーを下記に記載\n",
    "        'consumer_key':'en35U31mW3P9EAqZvR6KXkZ2h',\n",
    "        'consumer_secret':'JWoHyJU5KRnbeWjb17hVHAQsiPFgwGwvBkZSkRsARrApZqNEKd',\n",
    "        'access_token':'868744152753647617-ei1ptGYVwJqgv2zy7f09fPVuFrsYbLY',\n",
    "        'access_secret':'VHQQ1g1MDTUMRufrF2mojkxiig47qEc7ghcGj32fyMX1f',\n",
    "       }\n",
    "twitter = None\n",
    "connect = None\n",
    "db      = None\n",
    "tweetdata = None\n",
    "meta    = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def initialize(): # twitter接続情報や、mongoDBへの接続処理等initial処理実行\n",
    "    global twitter, twitter, connect, db, tweetdata, meta\n",
    "    twitter = OAuth1Session(KEYS['consumer_key'],KEYS['consumer_secret'],\n",
    "                            KEYS['access_token'],KEYS['access_secret'])\n",
    "#   connect = Connection('localhost', 27017)     # Connection classは廃止されたのでMongoClientに変更 \n",
    "    connect = MongoClient('localhost', 27017)\n",
    "    db = connect.trend\n",
    "    tweetdata = db.recipe_tweetdata\n",
    "    meta = db.recipe_metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 文字列を日本時間2タイムゾーンを合わせた日付型で返す\n",
    "def str_to_date_jp(str_date):\n",
    "    dts = datetime.datetime.strptime(str_date,'%a %b %d %H:%M:%S +0000 %Y')\n",
    "    return pytz.utc.localize(dts).astimezone(pytz.timezone('Asia/Tokyo'))\n",
    "\n",
    "# 現在時刻をUNIX Timeで返す\n",
    "def now_unix_time():\n",
    "    return time.mktime(datetime.datetime.now().timetuple())\n",
    "    \n",
    "# 日付の文字列をDatetime型で返す\n",
    "def str_to_date_jp(str_date):\n",
    "    dts = datetime.datetime.strptime(str_date,'%a %b %d %H:%M:%S +0000 %Y')\n",
    "    return pytz.utc.localize(dts).astimezone(pytz.timezone('Asia/Tokyo'))\n",
    "\n",
    "# UTCの日付文字列を日本時間にしてDatetime型で返す\n",
    "def utc_str_to_jp_str(str_date):\n",
    "    dts = datetime.datetime.strptime(str_date,'%a %b %d %H:%M:%S +0000 %Y')\n",
    "    return pytz.utc.localize(dts).astimezone(pytz.timezone('Asia/Tokyo')).strftime(\"%Y/%m/%d %H:%M:%S\")\n",
    "\n",
    "\n",
    "def str_to_date(str_date):\n",
    "    dts = datetime.datetime.strptime(str_date,'%Y-%m-%d %H:%M:%S')\n",
    "    return pytz.utc.localize(dts)\n",
    "\n",
    "def str_to_date_jp_utc(str_date):\n",
    "    return datetime.datetime.strptime(str_date,'%Y-%m-%d %H:%M:%S') - datetime.timedelta(hours=9)\n",
    "\n",
    "def date_to_Japan_time(dts):\n",
    "    return pytz.utc.localize(dts).astimezone(pytz.timezone('Asia/Tokyo'))\n",
    "\n",
    "def date_to_Japan_time_str(dts):\n",
    "    return pytz.utc.localize(dts).astimezone(pytz.timezone('Asia/Tokyo')).strftime(\"%Y/%m/%d %H:%M:%S\")\n",
    "\n",
    "def date_to_str(dt):\n",
    "    return dt.strftime(\"%Y/%m/%d %H:%M:%S\")\n",
    "\n",
    "def str_to_unix_date_jp(str_date):\n",
    "    dts = datetime.datetime.strptime(str_date,'%a %b %d %H:%M:%S +0000 %Y')\n",
    "    dt = pytz.utc.localize(dts).astimezone(pytz.timezone('Asia/Tokyo'))\n",
    "    return time.mktime(dt.timetuple())\n",
    "\n",
    "def unix_time_to_datetime(int_date):\n",
    "    return datetime.datetime.fromtimestamp(int_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "def select_outlier_retweet_num_per_hour(from_str_datetime_jp, limit=120):\n",
    "    '''\n",
    "    from_str_datetime_jp: １時間枠の開始時刻\n",
    "    limit: この数を超えてリツイートされたものを検出する\n",
    "    e.g. select_outlier_tweet_num_per_hour(\"2015-03-18 22:00:00\")\n",
    "    '''\n",
    "    result_list = []\n",
    "    from_date = str_to_date_jp_utc(from_str_datetime_jp)\n",
    "    to_date = str_to_date_jp_utc(from_str_datetime_jp) + datetime.timedelta(hours=1) \n",
    "\n",
    "    for d in tweetdata.find({'retweeted_status':{\"$ne\": None},'created_datetime':{\"$gte\":from_date, \"$lt\":to_date}},\\\n",
    "                            {'user':1,'text':1,'entities':1, 'created_at':1, 'id':1}):\n",
    "        mensioned_username = \"\"\n",
    "        if len(d['entities']['user_mentions'])!=0:\n",
    "            mensioned_username = d['entities']['user_mentions'][0]['screen_name']\n",
    "\n",
    "        result_list.append({\"created_at\":utc_str_to_jp_str(d['created_at']),\\\n",
    "                            \"screen_name\":d['user']['screen_name'],\\\n",
    "                            \"referred_name\":mensioned_username,\\\n",
    "                            \"text\":d['text'].replace('\\n',' ')\\\n",
    "                            })\n",
    "\n",
    "    name_dict = defaultdict(int)\n",
    "    for r in result_list:\n",
    "        name_dict[r['referred_name']] += 1  \n",
    "\n",
    "    s = sorted(name_dict.items(),key=lambda x: x[1],reverse=True) # リツイート回数でソート\n",
    "    return s[0:int(np.sum(list(map(lambda x: 1 if x[1]>limit else 0 ,s))))] # リツイート元ユーザー名, リツイート回数(limitを超えたもの)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2017-06-23 02:00:00 [('okumuratorucc', 29), ('ApAopoo', 17), ('okayado1215', 8), ('CChannel_tv', 7), ('gomio123', 6), ('map_u_chin', 6), ('colabo_yumeno', 5), ('yako_FLPR3', 4), ('yolu831', 4), ('akita11', 3), ('Shinsuke1', 3), ('usa_sable', 3), ('jupi420gab', 3), ('sorasorayaada', 3), ('shunichitakano', 3), ('skchammm', 3), ('110Hongo', 3), ('yukakuramoti', 3), ('onoderamasaru', 2), ('euro_tour', 2), ('Hortensia_PR', 2), ('2chourou', 2), ('onshokukiko', 2), ('odencentury', 2), ('LecoleVantan2T', 2), ('etwaaru', 2), ('aichi_34', 2), ('uswow02', 2), ('costmary_net', 2), ('cvk_kd', 2), ('moa__tsukino', 2), ('matsujun5213', 2), ('aZOf0woXmJCSz6J', 2), ('Potetosalad73', 2), ('uni_mafumafu', 2), ('nao_kakaomame', 2), ('sukisukiaraisan', 2), ('jaja_uma_', 2), ('blendykotya', 2), ('910123mai2', 2), ('the_Vambi', 2), ('J_League', 2), ('eda_neko', 2), ('SasimiAlexander', 2), ('mahya99', 2), ('iseebitarou_com', 2), ('toshi_seapara', 2)]\n",
      "2017-06-23 03:00:00 [('onoda_kimi', 16), ('okumuratorucc', 12), ('CChannel_tv', 8), ('ApAopoo', 8), ('kairow_rvg', 7), ('TheVambi', 6), ('onoderamasaru', 5), ('yako_FLPR3', 4), ('kametaro1986', 4), ('unCondensation', 4), ('vgvd', 3), ('seizinnzeppa', 3), ('colabo_yumeno', 3), ('moa__tsukino', 3), ('etwaaru', 3), ('onshokukiko', 3), ('cvk_kd', 3), ('MikanOsaka', 2), ('lin_empty', 2), ('sondborel', 2), ('una113_idol', 2), ('canotics', 2), ('ECYhW', 2), ('1000_MARU', 2), ('sugiurake4649', 2), ('tennenkaeru', 2), ('sorasorayaada', 2), ('jaja_uma_', 2), ('HiderohSahara', 2), ('tmt481', 2), ('PolandTravel_jp', 2), ('enjoy_tokyo', 2), ('yuu_yuugekisyu', 2), ('koikoi_west', 2), ('Tastemade_japan', 2), ('map_u_chin', 2), ('marisolove753', 2), ('kumo___atm', 2)]\n"
     ]
    }
   ],
   "source": [
    "start_date = str_to_date_jp_utc(\"2017-06-23 11:00:00\")\n",
    "to_date    = str_to_date_jp_utc(\"2017-06-23 13:00:00\")\n",
    "d_diff = (to_date - start_date)\n",
    "d_hours = d_diff.days * 24 + d_diff.seconds/float(3600)\n",
    "\n",
    "for i in range(int(d_hours)):\n",
    "    d = (start_date + datetime.timedelta(hours=i)).strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "    result = select_outlier_retweet_num_per_hour(d, limit=1)\n",
    "    if len(result) > 0:\n",
    "        print (d, result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "s = sorted(name_dict.items(),key=lambda x:x[1],reverse=True) # リツイート回数でソート\n",
    "t=s[0:int(np.sum(list(map(lambda x: 1 if x[1]>2 else 0 ,s))))] # リツイート元ユーザー名, リツイート回数(limitを超えたもの)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('okumuratorucc', 29),\n",
       " ('ApAopoo', 17),\n",
       " ('okayado1215', 8),\n",
       " ('CChannel_tv', 7),\n",
       " ('gomio123', 6),\n",
       " ('map_u_chin', 6),\n",
       " ('colabo_yumeno', 5),\n",
       " ('yako_FLPR3', 4),\n",
       " ('yolu831', 4),\n",
       " ('akita11', 3),\n",
       " ('Shinsuke1', 3),\n",
       " ('usa_sable', 3),\n",
       " ('jupi420gab', 3),\n",
       " ('sorasorayaada', 3),\n",
       " ('shunichitakano', 3),\n",
       " ('skchammm', 3),\n",
       " ('110Hongo', 3),\n",
       " ('yukakuramoti', 3)]"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "key(name_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'int'>, {'strawberry': 2, 'grape': 2, 'orange': 1})\n",
      "['strawberry', 'grape', 'orange']\n"
     ]
    }
   ],
   "source": [
    "fruit = ['strawberry','grape','orange','strawberry','grape']\n",
    "d = defaultdict(int)\n",
    "for f in fruit:\n",
    "    d[f] += 1 #デフォルト値がintで初期化されているのでインクリメントできる\n",
    "print(d)\n",
    "print (sorted(d, key = lambda x:x[1], reverse=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('okumuratorucc', 29),\n",
       " ('ApAopoo', 17),\n",
       " ('okayado1215', 8),\n",
       " ('CChannel_tv', 7),\n",
       " ('gomio123', 6),\n",
       " ('map_u_chin', 6),\n",
       " ('colabo_yumeno', 5),\n",
       " ('yako_FLPR3', 4),\n",
       " ('yolu831', 4),\n",
       " ('akita11', 3),\n",
       " ('Shinsuke1', 3),\n",
       " ('usa_sable', 3),\n",
       " ('jupi420gab', 3),\n",
       " ('sorasorayaada', 3),\n",
       " ('shunichitakano', 3),\n",
       " ('skchammm', 3),\n",
       " ('110Hongo', 3),\n",
       " ('yukakuramoti', 3),\n",
       " ('onoderamasaru', 2),\n",
       " ('euro_tour', 2),\n",
       " ('Hortensia_PR', 2),\n",
       " ('2chourou', 2),\n",
       " ('onshokukiko', 2),\n",
       " ('odencentury', 2),\n",
       " ('LecoleVantan2T', 2),\n",
       " ('etwaaru', 2),\n",
       " ('aichi_34', 2),\n",
       " ('uswow02', 2),\n",
       " ('costmary_net', 2),\n",
       " ('cvk_kd', 2),\n",
       " ('moa__tsukino', 2),\n",
       " ('matsujun5213', 2),\n",
       " ('aZOf0woXmJCSz6J', 2),\n",
       " ('Potetosalad73', 2),\n",
       " ('uni_mafumafu', 2),\n",
       " ('nao_kakaomame', 2),\n",
       " ('sukisukiaraisan', 2),\n",
       " ('jaja_uma_', 2),\n",
       " ('blendykotya', 2),\n",
       " ('910123mai2', 2),\n",
       " ('the_Vambi', 2),\n",
       " ('J_League', 2),\n",
       " ('eda_neko', 2),\n",
       " ('SasimiAlexander', 2),\n",
       " ('mahya99', 2),\n",
       " ('iseebitarou_com', 2),\n",
       " ('toshi_seapara', 2),\n",
       " ('oja_map', 1),\n",
       " ('yuzpon5', 1),\n",
       " ('animatechannel', 1),\n",
       " ('ICYokohamaGrand', 1),\n",
       " ('mm__neko225', 1),\n",
       " ('_nappanda_', 1),\n",
       " ('event_checker', 1),\n",
       " ('panda3091', 1),\n",
       " ('chunma04', 1),\n",
       " ('mika_berry', 1),\n",
       " ('kairow_rvg', 1),\n",
       " ('nikkeikansai', 1),\n",
       " ('YUI05525296', 1),\n",
       " ('HiderohSahara', 1),\n",
       " ('ichigain880', 1),\n",
       " ('future8market', 1),\n",
       " ('yonakiishi', 1),\n",
       " ('taklamahiro', 1),\n",
       " ('511mC', 1),\n",
       " ('m_kyounoryouri', 1),\n",
       " ('utaite__love__', 1),\n",
       " ('Brecht', 1),\n",
       " ('maruskitchen01', 1),\n",
       " ('nasurie', 1),\n",
       " ('gusinraisan', 1),\n",
       " ('konnyaku_in_box', 1),\n",
       " ('michmon39', 1),\n",
       " ('Reborn_Art_Fes', 1),\n",
       " ('BJWUTO', 1),\n",
       " ('kaeru_ojyousama', 1),\n",
       " ('chirashi_uraura', 1),\n",
       " ('nekoze_aomori', 1),\n",
       " ('Suzumixxx', 1),\n",
       " ('_tmdd', 1),\n",
       " ('Tonerico_swt', 1),\n",
       " ('sf_raw21', 1),\n",
       " ('beergirl_net', 1),\n",
       " ('MomentsJapan', 1),\n",
       " ('5a2bbf4545ef4f8', 1),\n",
       " ('riochi0612', 1),\n",
       " ('yamazaki_haruka', 1),\n",
       " ('chisaki_ann', 1),\n",
       " ('sakeko', 1),\n",
       " ('imirixoxo', 1),\n",
       " ('KAZUYA_drums', 1),\n",
       " ('kbrakn1209', 1),\n",
       " ('cult666', 1),\n",
       " ('nijiirokun', 1),\n",
       " ('yuu_bcats', 1),\n",
       " ('Wl9uZ', 1),\n",
       " ('wabisabiya', 1),\n",
       " ('hkazano', 1),\n",
       " ('curry_HoweverRT', 1),\n",
       " ('crescent_tulb', 1),\n",
       " ('katsuse_m', 1),\n",
       " ('matsudora124', 1),\n",
       " ('satomi16532882', 1),\n",
       " ('Yuis0603', 1),\n",
       " ('MxSxD_CAFE', 1),\n",
       " ('Vulpes_No9', 1),\n",
       " ('6m_9k', 1),\n",
       " ('Tamagawa_GHQ', 1),\n",
       " ('agro_ecology', 1),\n",
       " ('dearlovingyuki', 1),\n",
       " ('naym_59', 1),\n",
       " ('godfatherpart7', 1),\n",
       " ('mimiaijp', 1),\n",
       " ('twinrail', 1),\n",
       " ('sakkurusan', 1),\n",
       " ('Kkorepo', 1),\n",
       " ('nitiyoutwitt', 1),\n",
       " ('k_____101226', 1),\n",
       " ('bazaarcafe1', 1),\n",
       " ('Sakura__ShrimP', 1),\n",
       " ('ichiban_KIRIN', 1),\n",
       " ('manpuku1975', 1),\n",
       " ('j44KMPlXzmq1CRb', 1),\n",
       " ('fallindebu', 1),\n",
       " ('girlsqb_shiina', 1),\n",
       " ('nambakeizai', 1),\n",
       " ('lilygakusaki', 1),\n",
       " ('kikoushi70', 1),\n",
       " ('char_so', 1),\n",
       " ('BloombergJapan', 1),\n",
       " ('oriibuoiruspain', 1),\n",
       " ('zengyuai', 1),\n",
       " ('48vandange', 1),\n",
       " ('TaSteMaDefan', 1),\n",
       " ('sss_ueno', 1),\n",
       " ('kumo___atm', 1),\n",
       " ('asdr_g', 1),\n",
       " ('wired_jp', 1),\n",
       " ('naname728', 1),\n",
       " ('livedoornews', 1),\n",
       " ('rambada0214', 1),\n",
       " ('owarai_natalie', 1),\n",
       " ('sirabee_news', 1),\n",
       " ('shochann1', 1),\n",
       " ('ItteQ_genmato', 1),\n",
       " ('marisolove753', 1),\n",
       " ('keikeikei9', 1),\n",
       " ('masayachiba', 1),\n",
       " ('uraurx', 1),\n",
       " ('rukoimo', 1),\n",
       " ('moamoa_media', 1),\n",
       " ('muteki77', 1),\n",
       " ('YafengCheng', 1),\n",
       " (1, 0),\n",
       " (2, 0),\n",
       " (0, 0)]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = sorted(name_dict.items(),key=lambda x:x[1],reverse=True) # リツイート回数でソート\n",
    "s"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}