From 0407f40190581bb7edfc68fd8e344caba34a6b09 Mon Sep 17 00:00:00 2001
From: Will Feng <willfeng@fb.com>
Date: Wed, 7 Jun 2017 17:36:09 -0400
Subject: [PATCH 1/6] Created MTurkAgent

---
 parlai/mturk/core/agents.py                   | 202 ++++++++++++++
 parlai/mturk/core/data_model.py               |  25 +-
 parlai/mturk/core/handler_template.py         |  60 +++--
 parlai/mturk/core/manage_hit.py               | 247 ------------------
 parlai/mturk/core/mturk_index.html            |  20 +-
 parlai/mturk/core/setup_aws.py                |  10 +-
 .../mturk/tasks/multi_agent_dialog/sample.py  |  22 ++
 7 files changed, 294 insertions(+), 292 deletions(-)
 create mode 100644 parlai/mturk/core/agents.py
 delete mode 100644 parlai/mturk/core/manage_hit.py
 create mode 100644 parlai/mturk/tasks/multi_agent_dialog/sample.py

diff --git a/parlai/mturk/core/agents.py b/parlai/mturk/core/agents.py
new file mode 100644
index 00000000000..0fd58687eac
--- /dev/null
+++ b/parlai/mturk/core/agents.py
@@ -0,0 +1,202 @@
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+
+from parlai.core.agents import Agent
+from parlai.core.worlds import display_messages
+
+import os
+import time
+from datetime import datetime
+import random
+import string
+import webbrowser
+import json
+import requests
+from parlai.core.agents import create_agent_from_shared
+from .setup_aws import setup_aws, check_mturk_balance, create_hit_type, create_hit_with_hit_type, setup_aws_credentials
+
+
+def _get_new_messages(json_api_endpoint_url, task_group_id, conversation_id, after_message_id, excluded_agent_id=None, included_agent_id=None):
+    params = {
+        'method_name': 'get_new_messages',
+        'task_group_id': task_group_id,
+        'last_message_id': after_message_id,
+        'conversation_id': conversation_id,
+    }
+    if excluded_agent_id:
+        params['excluded_agent_id'] = excluded_agent_id
+    if included_agent_id:
+        params['included_agent_id'] = included_agent_id
+
+    request = requests.get(json_api_endpoint_url, params=params)
+    return json.loads(request.json())
+
+def _send_new_message(json_api_endpoint_url, task_group_id, conversation_id, agent_id, message_text=None, reward=None, episode_done=False):
+    post_data_dict = {
+        'method_name': 'send_new_message',
+        'task_group_id': task_group_id,
+        'conversation_id': conversation_id,
+        'cur_agent_id': agent_id,
+        'episode_done': episode_done,
+    }
+    if message_text:
+        post_data_dict['text'] = message_text
+    if reward:
+        post_data_dict['reward'] = reward
+
+    request = requests.post(json_api_endpoint_url, data=json.dumps(post_data_dict))
+    return json.loads(request.json())
+
+def _get_review_status_count(json_api_endpoint_url, task_group_id, conversation_id, review_status, requester_key):
+    params = {
+        'method_name': 'get_review_status_count',
+        'task_group_id': task_group_id,
+        'conversation_id': conversation_id,
+        'review_status': review_status,
+        'requester_key': requester_key
+    }
+    request = requests.get(json_api_endpoint_url, params=params)
+    return request.json()
+
+class MTurkAgent(Agent):
+
+    skip_init = False
+    html_api_endpoint_url = None
+    json_api_endpoint_url = None
+    requester_key_gt = None
+    
+    def __init__(self, opt, shared=None):
+        super().__init__(opt)
+
+        self.id = opt['agent_id']
+        self.task_name = opt['task_name']
+        self.is_sandbox = opt['is_sandbox']
+        self.conversation_id = opt['conversation_id']
+        self.mturk_agent_ids = opt['mturk_agent_ids']
+
+        self.hit_reward = opt.get('reward', 0.5)
+        self.hit_title = opt.get('hit_title', 'hit title')
+        self.hit_description = opt.get('hit_description', 'hit description')
+        self.hit_keywords = opt.get('hit_keywords', 'hitkeywords')
+        self.task_description = "'''" + opt.get('task_description', 'hit task description') + "'''"
+
+        self.last_message_id = 0
+
+        if not self.__class__.skip_init:
+            print("\nYou are going to allow workers from Amazon Mechanical Turk to chat with your dialog model running on your local machine.\nDuring this process, Internet connection is required, and you should turn off your computer's auto-sleep feature.\n")
+            key_input = input("Please press Enter to continue... ")
+            print("")
+            
+        setup_aws_credentials()
+
+        if not check_mturk_balance(num_hits=1, hit_reward=self.hit_reward, is_sandbox=self.is_sandbox):
+            return
+            
+        if not self.__class__.skip_init:
+            print('Setting up MTurk backend...')
+            html_api_endpoint_url, json_api_endpoint_url, requester_key_gt = setup_aws(self.task_description, 1, self.is_sandbox)
+            self.__class__.html_api_endpoint_url = html_api_endpoint_url
+            self.__class__.json_api_endpoint_url = json_api_endpoint_url
+            self.__class__.requester_key_gt = requester_key_gt
+
+        self.__class__.skip_init = True
+        self.html_api_endpoint_url = self.__class__.html_api_endpoint_url
+        self.json_api_endpoint_url = self.__class__.json_api_endpoint_url
+        self.requester_key_gt = self.__class__.requester_key_gt
+
+        self.task_group_id = str(self.task_name) + '_' + str(self.conversation_id)
+
+        print('Creating HITs...')
+        hit_type_id = create_hit_type(
+            hit_title=self.hit_title,
+            hit_description=self.hit_description + ' (ID: ' + self.task_group_id + ', Role: ' + self.id + ')',
+            hit_keywords=self.hit_keywords,
+            hit_reward=self.hit_reward,
+            is_sandbox=self.is_sandbox
+        )
+        mturk_chat_url = self.html_api_endpoint_url + "?method_name=chat_index&task_group_id="+str(self.task_group_id)+"&conversation_id="+str(self.conversation_id)+"&cur_agent_id="+str(self.id)
+        mturk_page_url = create_hit_with_hit_type(
+            page_url=mturk_chat_url,
+            hit_type_id=hit_type_id,
+            is_sandbox=self.is_sandbox
+        )
+
+        print("MTurk setup done.\n")
+        print("Link to your HIT: " + mturk_page_url + "\n")
+        print("Waiting for Turkers to complete the tasks... (Please don't close your laptop or put your computer into sleep or standby mode.)\n")
+
+    def observe(self, msg):
+        if msg['id'] not in self.mturk_agent_ids: # If the message sender is an mturk agent, then there is no need to upload this message to db since it's already been done on the message sender side.
+            agent_last_message_in_db = None
+            conversation_dict = _get_new_messages(
+                json_api_endpoint_url=self.json_api_endpoint_url, 
+                task_group_id=self.task_group_id,
+                conversation_id=self.conversation_id,
+                after_message_id=self.last_message_id,
+                included_agent_id=msg['id'])['conversation_dict']
+            if self.conversation_id in conversation_dict:
+                agent_last_message_in_db = conversation_dict[self.conversation_id][0]
+                agent_last_message_in_db.pop('message_id', None)
+                if 'episode_done' not in msg:
+                    msg['episode_done'] = False
+                if agent_last_message_in_db == msg:
+                    return
+
+            _send_new_message(
+                json_api_endpoint_url=self.json_api_endpoint_url,
+                task_group_id=self.task_group_id,
+                conversation_id=self.conversation_id,
+                agent_id=msg['id'],
+                message_text=msg.get('text', None),
+                reward=msg.get('reward', None),
+                episode_done=msg.get('episode_done', False),
+            )
+
+    def act(self):
+        while True:
+            ret = _get_new_messages(
+                json_api_endpoint_url=self.json_api_endpoint_url,
+                task_group_id=self.task_group_id,
+                conversation_id=self.conversation_id,
+                after_message_id=self.last_message_id,
+                included_agent_id=self.id
+            )
+            conversation_dict = ret['conversation_dict']
+            
+            if str(self.conversation_id) in conversation_dict:
+                new_last_message_id = ret['last_message_id']
+                if new_last_message_id:
+                    self.last_message_id = new_last_message_id
+
+                new_messages = conversation_dict[str(self.conversation_id)]
+
+                print(display_messages([new_messages[0]]))
+                return new_messages[0]
+
+            time.sleep(1)
+
+    def epoch_done(self):
+        return False
+
+    def shutdown(self):
+        if _get_review_status_count(json_api_endpoint_url=self.json_api_endpoint_url, task_group_id=self.task_group_id, conversation_id=self.conversation_id, review_status='approved', requester_key=self.requester_key_gt) + \
+            _get_review_status_count(json_api_endpoint_url=self.json_api_endpoint_url, task_group_id=self.task_group_id, conversation_id=self.conversation_id, review_status='rejected', requester_key=self.requester_key_gt) > 0:
+            return
+        else:
+            while _get_review_status_count(json_api_endpoint_url=self.json_api_endpoint_url, task_group_id=self.task_group_id, conversation_id=self.conversation_id, review_status='pending', requester_key=self.requester_key_gt) < len(self.mturk_agent_ids):
+                time.sleep(2)
+
+            mturk_agent_ids_string = str(self.mturk_agent_ids).replace("'", '''"''')
+            mturk_approval_url = self.html_api_endpoint_url + "?method_name=approval_index&task_group_id="+str(self.task_group_id)+"&conversation_id="+str(self.conversation_id)+"&mturk_agent_ids="+mturk_agent_ids_string+"&requester_key="+self.requester_key_gt
+            print("\nAll HITs are done! Please go to the following link to approve/reject them (or they will be auto-approved in 4 weeks if no action is taken):\n")
+            print(mturk_approval_url)
+            print("")
+
+            # Loop for checking approval status
+            while _get_review_status_count(json_api_endpoint_url=self.json_api_endpoint_url, task_group_id=self.task_group_id, conversation_id=self.conversation_id, review_status='pending', requester_key=self.requester_key_gt) > 0:
+                time.sleep(2)
+
+            print("All reviews are done!")
\ No newline at end of file
diff --git a/parlai/mturk/core/data_model.py b/parlai/mturk/core/data_model.py
index 9f204440926..f93e3579bee 100644
--- a/parlai/mturk/core/data_model.py
+++ b/parlai/mturk/core/data_model.py
@@ -100,7 +100,7 @@ def send_new_message(db_session, task_group_id, conversation_id, agent_id, messa
     return new_message_object
 
 
-def get_new_messages(db_session, task_group_id, conversation_id=None, after_message_id=None, excluded_agent_id=None, populate_meta_info=False):
+def get_new_messages(db_session, task_group_id, conversation_id=None, after_message_id=None, excluded_agent_id=None, included_agent_id=None, populate_meta_info=False):
     """
     Return:
     conversation_dict = {
@@ -123,6 +123,10 @@ def get_new_messages(db_session, task_group_id, conversation_id=None, after_mess
     if not after_message_id:
         after_message_id = -1
 
+    included_agent_ids = []
+    if included_agent_id:
+        included_agent_ids = [included_agent_id]
+
     excluded_agent_ids = []
     if excluded_agent_id:
         excluded_agent_ids = [excluded_agent_id]
@@ -130,6 +134,8 @@ def get_new_messages(db_session, task_group_id, conversation_id=None, after_mess
     last_message_id = None
 
     query = db_session.query(Message).filter(Message.task_group_id==task_group_id).filter(Message.id > after_message_id)
+    if len(included_agent_ids) > 0:
+        query = query.filter(Message.agent_id.in_(included_agent_ids))
     if len(excluded_agent_ids) > 0:
         query = query.filter(~Message.agent_id.in_(excluded_agent_ids))
     if conversation_id:
@@ -164,7 +170,12 @@ def get_new_messages(db_session, task_group_id, conversation_id=None, after_mess
 
 
 def set_hit_info(db_session, task_group_id, conversation_id, assignment_id, hit_id, worker_id, is_sandbox, approval_status='pending'):
-    existing_object = db_session.query(MTurkHITInfo).filter(MTurkHITInfo.task_group_id==task_group_id).filter(MTurkHITInfo.conversation_id==conversation_id).first()
+    existing_object = db_session.query(MTurkHITInfo) \
+                        .filter(MTurkHITInfo.task_group_id==task_group_id) \
+                        .filter(MTurkHITInfo.conversation_id==conversation_id) \
+                        .filter(MTurkHITInfo.assignment_id==assignment_id) \
+                        .filter(MTurkHITInfo.hit_id==hit_id) \
+                        .first()
     if not existing_object:
         new_hit_info_object = MTurkHITInfo(
             task_group_id=task_group_id,
@@ -187,12 +198,12 @@ def set_hit_info(db_session, task_group_id, conversation_id, assignment_id, hit_
         db_session.commit()
 
 
-def get_hit_info(db_session, task_group_id, conversation_id):
-    existing_object = db_session.query(MTurkHITInfo).filter(MTurkHITInfo.task_group_id==task_group_id).filter(MTurkHITInfo.conversation_id==conversation_id).first()
-    return existing_object
+def get_all_matching_hit_infos(db_session, task_group_id, conversation_id):
+    matching_hit_infos = list(db_session.query(MTurkHITInfo).filter(MTurkHITInfo.task_group_id==task_group_id).filter(MTurkHITInfo.conversation_id==conversation_id).all())
+    return matching_hit_infos
 
-def get_pending_review_count(db_session, task_group_id):
-    return db_session.query(MTurkHITInfo).filter(MTurkHITInfo.task_group_id==task_group_id).filter(MTurkHITInfo.approval_status=='pending').count()
+def get_review_status_count(db_session, task_group_id, conversation_id, review_status):
+    return db_session.query(MTurkHITInfo).filter(MTurkHITInfo.task_group_id==task_group_id).filter(MTurkHITInfo.conversation_id==conversation_id).filter(MTurkHITInfo.approval_status==review_status).count()
 
 def get_all_review_status(db_session, task_group_id):
     return db_session.query(MTurkHITInfo).filter(MTurkHITInfo.task_group_id==task_group_id).order_by(MTurkHITInfo.conversation_id).all()
\ No newline at end of file
diff --git a/parlai/mturk/core/handler_template.py b/parlai/mturk/core/handler_template.py
index ce5a0d3b0d0..b4155a438ac 100755
--- a/parlai/mturk/core/handler_template.py
+++ b/parlai/mturk/core/handler_template.py
@@ -112,6 +112,7 @@ def get_new_messages(event, context):
         if 'conversation_id' in event['query']:
             conversation_id = int(event['query']['conversation_id'])
         excluded_agent_id = event['query'].get('excluded_agent_id', None)
+        included_agent_id = event['query'].get('included_agent_id', None)
 
         conversation_dict, new_last_message_id = data_model.get_new_messages(
             db_session=db_session, 
@@ -119,6 +120,7 @@ def get_new_messages(event, context):
             conversation_id=conversation_id,
             after_message_id=last_message_id,
             excluded_agent_id=excluded_agent_id,
+            included_agent_id=included_agent_id,
             populate_meta_info=True
         )
 
@@ -179,12 +181,12 @@ def approval_index(event, context):
 
             task_group_id = event['query']['task_group_id']
             conversation_id = event['query']['conversation_id']
-            cur_agent_id = event['query']['cur_agent_id']
+            mturk_agent_ids = event['query']['mturk_agent_ids']
 
             template_context = {}
             template_context['task_group_id'] = task_group_id
             template_context['conversation_id'] = conversation_id
-            template_context['cur_agent_id'] = cur_agent_id
+            template_context['mturk_agent_ids'] = mturk_agent_ids
             template_context['task_description'] = task_description
             template_context['is_cover_page'] = False
             template_context['is_approval_page'] = True
@@ -212,39 +214,41 @@ def review_hit(event, context):
             conversation_id = int(params['conversation_id'])
             action = params['action'] # 'approve' or 'reject'
 
-            hit_info = data_model.get_hit_info(
+            hit_infos = data_model.get_all_matching_hit_infos(
                 db_session=db_session, 
                 task_group_id=task_group_id, 
                 conversation_id=conversation_id
             )
 
-            if hit_info:
-                assignment_id = hit_info.assignment_id
-                client = boto3.client(
-                    service_name = 'mturk', 
-                    region_name = 'us-east-1',
-                    endpoint_url = 'https://mturk-requester-sandbox.us-east-1.amazonaws.com'
-                )
-                # Region is always us-east-1
-                if not hit_info.is_sandbox:
-                    client = boto3.client(service_name = 'mturk', region_name='us-east-1')
-
-                if action == 'approve':
-                    client.approve_assignment(AssignmentId=assignment_id)
-                    hit_info.approval_status = 'approved'
-                elif action == 'reject':
-                    client.reject_assignment(AssignmentId=assignment_id, RequesterFeedback='')
-                    hit_info.approval_status = 'rejected'
-                db_session.add(hit_info)
-                db_session.commit()
+            if len(hit_infos) > 0:
+                for hit_info in hit_infos:
+                    assignment_id = hit_info.assignment_id
+                    client = boto3.client(
+                        service_name = 'mturk', 
+                        region_name = 'us-east-1',
+                        endpoint_url = 'https://mturk-requester-sandbox.us-east-1.amazonaws.com'
+                    )
+                    # Region is always us-east-1
+                    if not hit_info.is_sandbox:
+                        client = boto3.client(service_name = 'mturk', region_name='us-east-1')
+
+                    if action == 'approve':
+                        client.approve_assignment(AssignmentId=assignment_id)
+                        hit_info.approval_status = 'approved'
+                    elif action == 'reject':
+                        client.reject_assignment(AssignmentId=assignment_id, RequesterFeedback='')
+                        hit_info.approval_status = 'rejected'
+                    db_session.add(hit_info)
+                    db_session.commit()
+
         except KeyError:
             raise Exception('400')
 
-def get_pending_review_count(event, context):
+def get_review_status_count(event, context):
     if event['method'] == 'GET':
         """
         Handler for getting the number of pending reviews.
-        Expects <requester_key>, <task_group_id> as query parameters.
+        Expects <requester_key>, <task_group_id>, <conversation_id> as query parameters.
         """
         try:
             requester_key = event['query']['requester_key']
@@ -252,9 +256,13 @@ def get_pending_review_count(event, context):
                 raise Exception('403')
 
             task_group_id = event['query']['task_group_id']
-            return data_model.get_pending_review_count(
+            conversation_id = event['query']['conversation_id']
+            review_status = event['query']['review_status']
+            return data_model.get_review_status_count(
                 db_session=db_session,
-                task_group_id=task_group_id
+                task_group_id=task_group_id,
+                conversation_id=conversation_id,
+                review_status=review_status
             )
         except KeyError:
             raise Exception('400')
diff --git a/parlai/mturk/core/manage_hit.py b/parlai/mturk/core/manage_hit.py
deleted file mode 100644
index 244a22cd7b9..00000000000
--- a/parlai/mturk/core/manage_hit.py
+++ /dev/null
@@ -1,247 +0,0 @@
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree. An additional grant
-# of patent rights can be found in the PATENTS file in the same directory.
-
-import os
-import time
-from datetime import datetime
-import random
-import string
-import webbrowser
-import json
-import requests
-from parlai.core.agents import create_agent_from_shared
-from .setup_aws import setup_aws, check_mturk_balance, create_hit_type, create_hit_with_hit_type, setup_aws_credentials
-
-
-def _get_random_alphanumeric_string(N):
-    return ''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(N))
-
-
-def _setup_relay(task_config, num_hits, is_sandbox):
-    """Sets up relay server
-    """
-    # set up relay server
-    html_api_endpoint_url, json_api_endpoint_url, requester_key_gt = setup_aws(task_config, num_hits, is_sandbox)
-
-    return html_api_endpoint_url, json_api_endpoint_url, requester_key_gt
-
-def _send_new_message(json_api_endpoint_url, task_group_id, conversation_id, agent_id, message_text=None, reward=None, episode_done=False):
-    post_data_dict = {
-        'method_name': 'send_new_message',
-        'task_group_id': task_group_id,
-        'conversation_id': conversation_id,
-        'cur_agent_id': agent_id,
-        'episode_done': episode_done,
-    }
-    if message_text:
-        post_data_dict['text'] = message_text
-    if reward:
-        post_data_dict['reward'] = reward
-
-    request = requests.post(json_api_endpoint_url, data=json.dumps(post_data_dict))
-    return json.loads(request.json())
-
-def _get_new_messages(json_api_endpoint_url, task_group_id, after_message_id, excluded_agent_id=None):
-    params = {
-        'method_name': 'get_new_messages',
-        'task_group_id': task_group_id,
-        'last_message_id': after_message_id,
-    }
-    if excluded_agent_id:
-        params['excluded_agent_id'] = excluded_agent_id
-
-    request = requests.get(json_api_endpoint_url, params=params)
-    return json.loads(request.json())
-
-def _get_pending_review_count(json_api_endpoint_url, task_group_id, requester_key):
-    params = {
-        'method_name': 'get_pending_review_count',
-        'task_group_id': task_group_id,
-        'requester_key': requester_key
-    }
-    request = requests.get(json_api_endpoint_url, params=params)
-    return request.json()
-
-def _get_all_review_status(json_api_endpoint_url, task_group_id, requester_key):
-    params = {
-        'method_name': 'get_all_review_status',
-        'task_group_id': task_group_id,
-        'requester_key': requester_key
-    }
-    request = requests.get(json_api_endpoint_url, params=params)
-    return request.json()
-
-def create_hits(opt, task_config, task_module_name, bot, chat_page_only=False):
-    num_hits = opt['num_hits']
-    hit_reward = opt['reward']
-    is_sandbox = opt['is_sandbox']
-    verbose = opt['verbose']
-
-    print("\nYou are going to allow workers from Amazon Mechanical Turk to chat with your dialog model running on your local machine.\nDuring this process, Internet connection is required, and you should turn off your computer's auto-sleep feature.\n")
-    key_input = input("Please press Enter to continue... ")
-    print("")
-
-    setup_aws_credentials()
-    if not check_mturk_balance(num_hits=num_hits, hit_reward=hit_reward, is_sandbox=is_sandbox):
-        return
-
-    task_group_id = str(int(time.time())) + '_' + _get_random_alphanumeric_string(10) # Random string to further avoid collision
-
-    print('Setting up MTurk backend...')
-    html_api_endpoint_url, json_api_endpoint_url, requester_key_gt = _setup_relay(task_config, num_hits, is_sandbox)
-
-    approval_index_url_template = html_api_endpoint_url + "?method_name=approval_index&task_group_id={{task_group_id}}&conversation_id=1&cur_agent_id={{cur_agent_id}}&requester_key="+requester_key_gt
-
-    worker_agent_id = task_config['worker_agent_id']
-    bot_agent_id = bot.getID()
-    cids = range(1, num_hits+1)
-    cid_map = {cid: i for i, cid in enumerate(cids)}
-    c_done_map = {cid: False for cid in cids}
-    logs = {cid: [] for cid in cids}
-
-    shared = bot.share()
-    bots = []
-    last_message_id = -1
-
-    # If the bot needs to send the first message of the conversation, it will send it here
-    for cid in cids:
-        new_bot = create_agent_from_shared(shared)
-        new_bot.conversation_id = cid
-        bots.append(new_bot)
-        response = new_bot.act()
-        if response:
-            if response.get('episode_done', False):
-                c_done_map[cid] = True
-            if verbose:
-                print('Conversation '+str(cid)+' - Bot says: ' + str(response))
-            logs[cid].append(response)
-            new_message = _send_new_message(
-                json_api_endpoint_url=json_api_endpoint_url,
-                task_group_id=task_group_id,
-                conversation_id=cid,
-                agent_id=bot_agent_id,
-                message_text=response.get('text', None),
-                reward=response.get('reward', None),
-                episode_done=response.get('episode_done', False),
-            )
-            if new_message['message_id'] > last_message_id:
-                last_message_id = new_message['message_id']
-
-    hits_created = False
-    conversations_remaining = set(cids)
-
-    # Main loop for polling and handling new messages
-    while len(conversations_remaining) > 0:
-        ret = _get_new_messages(
-            json_api_endpoint_url=json_api_endpoint_url,
-            task_group_id=task_group_id,
-            after_message_id=last_message_id,
-            excluded_agent_id=bot_agent_id,
-        )
-        conversation_dict = ret['conversation_dict']
-        new_last_message_id = ret['last_message_id']
-
-        if new_last_message_id:
-            last_message_id = new_last_message_id
-
-        time.sleep(1)
-
-        for conversation_id, new_messages in conversation_dict.items():
-            conversation_id = int(conversation_id)
-            if conversation_id in conversations_remaining and len(new_messages) > 0:
-                agent = bots[cid_map[conversation_id]]
-                for new_message in new_messages:
-                    if verbose:
-                        print('Conversation '+str(conversation_id)+' - Bot received: ' + str(new_message))
-                    logs[conversation_id].append(new_message)
-                    agent.observe(new_message)
-                    if new_message.get('episode_done', False) or c_done_map[conversation_id]:
-                        # We're done here
-                        conversations_remaining.remove(conversation_id)
-                        print('Conversation '+str(conversation_id)+' is DONE!\n')
-                    else:
-                        # Agent still needs to reply
-                        response = agent.act()
-                        if response:
-                            if response.get('episode_done', False):
-                                c_done_map[conversation_id] = True
-                            if verbose:
-                                print('Conversation '+str(conversation_id)+' - Bot says: ' + str(response))
-                            logs[conversation_id].append(response)
-                            _send_new_message(
-                                json_api_endpoint_url=json_api_endpoint_url,
-                                task_group_id=task_group_id,
-                                conversation_id=conversation_id,
-                                agent_id=bot_agent_id,
-                                message_text=response.get('text', None),
-                                reward=response.get('reward', None),
-                                episode_done=response.get('episode_done', False),
-                            )
-
-        # We don't create new HITs until this point, so that the HIT page will always have the conversation fully populated.
-        if not hits_created:
-            print('Creating HITs...')
-            hit_type_id = create_hit_type(
-                hit_title=task_config['hit_title'],
-                hit_description=task_config['hit_description'] + ' (ID: ' + task_group_id + ')',
-                hit_keywords=task_config['hit_keywords'],
-                hit_reward=hit_reward,
-                is_sandbox=is_sandbox
-            )
-            mturk_chat_url = None
-            mturk_page_url = None
-            for cid in cids:
-                mturk_chat_url = html_api_endpoint_url + "?method_name=chat_index&task_group_id="+str(task_group_id)+"&conversation_id="+str(cid)+"&cur_agent_id="+str(worker_agent_id)
-                if not chat_page_only:
-                    mturk_page_url = create_hit_with_hit_type(
-                        page_url=mturk_chat_url,
-                        hit_type_id=hit_type_id,
-                        is_sandbox=is_sandbox
-                    )
-
-            print("MTurk setup done.\n")
-            if chat_page_only:
-                webbrowser.open(mturk_chat_url)
-            else:
-                print("Link to your HIT: " + mturk_page_url + "\n")
-                print("Waiting for Turkers to complete the tasks... (Please don't close your laptop or put your computer into sleep or standby mode.)\n")
-            hits_created = True
-
-    while _get_pending_review_count(json_api_endpoint_url=json_api_endpoint_url, task_group_id=task_group_id, requester_key=requester_key_gt) != num_hits:
-        time.sleep(2)
-
-    mturk_approval_url = html_api_endpoint_url + "?method_name=approval_index&task_group_id="+str(task_group_id)+"&conversation_id=1&cur_agent_id="+worker_agent_id+"&requester_key="+requester_key_gt
-    print("\nAll HITs are done! Please go to the following link to approve/reject them (or they will be auto-approved in 4 weeks if no action is taken):\n")
-    print(mturk_approval_url)
-    print("")
-
-    approval_status_dict = {cid: '' for cid in cids}
-    # Loop for checking approval status
-    while _get_pending_review_count(json_api_endpoint_url=json_api_endpoint_url, task_group_id=task_group_id, requester_key=requester_key_gt) > 0:
-        time.sleep(2)
-
-    print("Approvals are done!")
-
-    for hit_info in _get_all_review_status(json_api_endpoint_url=json_api_endpoint_url, task_group_id=task_group_id, requester_key=requester_key_gt):
-        conversation_id = hit_info['conversation_id']
-        approval_status_dict[conversation_id] = hit_info['approval_status']
-
-    logs_approved = {cid:log for (cid,log) in logs.items() if approval_status_dict[cid] == 'approved'}
-    logs_rejected = {cid:log for (cid,log) in logs.items() if approval_status_dict[cid] == 'rejected'}
-
-    # Saving logs to file
-    # Log format: {conversation_id: [list of messages in the conversation]}
-    mturk_log_path = opt['mturk_log_path']
-    task_group_path = os.path.join(mturk_log_path,
-                                   task_module_name + '_' +
-                                   datetime.now().strftime('%Y-%m-%d_%H:%M:%S'))
-    os.makedirs(task_group_path)
-    with open(os.path.join(task_group_path, 'approved.json'), 'w') as fout:
-        fout.write(json.dumps(logs_approved))
-    with open(os.path.join(task_group_path, 'rejected.json'), 'w') as fout:
-        fout.write(json.dumps(logs_rejected))
-
-    print("All conversations are saved to "+opt['mturk_log_path']+" in JSON format.\n")
diff --git a/parlai/mturk/core/mturk_index.html b/parlai/mturk/core/mturk_index.html
index 3ab11c89838..a6494c9ceae 100755
--- a/parlai/mturk/core/mturk_index.html
+++ b/parlai/mturk/core/mturk_index.html
@@ -180,6 +180,7 @@ <h1>Live Chat</h1>
                 show_new_messages_on_UI(new_messages);
                 if (done_after_responding) {
                     update_for_response_type('done');
+                    task_done = true;
                 } else {
                     check_done(new_messages);
                 }
@@ -242,10 +243,15 @@ <h1>Live Chat</h1>
     });
 
     var mturk_submit_url = `{{mturk_submit_url}}`;
-    var verbose_log = false;
+    var verbose_log = true;
     var task_group_id = `{{task_group_id}}`;
     var conversation_id = parseInt(`{{conversation_id}}`);
     var cur_agent_id = `{{cur_agent_id}}`;
+    var mturk_agent_ids = `{{mturk_agent_ids}}`;
+    if (mturk_agent_ids) {
+        mturk_agent_ids = JSON.parse(mturk_agent_ids);
+        console.log(mturk_agent_ids);
+    }
     var is_cover_page = (`{{is_cover_page}}` === 'True') ? true : false;
     var is_approval_page = (`{{is_approval_page}}` === 'True') ? true : false;
     var num_hits = parseInt(`{{num_hits}}`);
@@ -254,7 +260,7 @@ <h1>Live Chat</h1>
     var messages_processed = {};
     var messages_shown = {};
     var done_after_responding = false;
-    var self_display_name = 'You';
+    var task_done = false;
 
     function show_new_messages_on_UI(new_messages) {
         for (var i = 0; i < new_messages.length; i++) {
@@ -269,7 +275,7 @@ <h1>Live Chat</h1>
             }
 
             if (!(message_id in messages_shown)) {
-                if (agent_id != cur_agent_id) {
+                if ((!is_approval_page && agent_id != cur_agent_id) || (is_approval_page && $.inArray(agent_id, mturk_agent_ids) == -1)) {
                     $('div#message_thread').append(`
                         <div class="row" style="margin-left: 0; margin-right: 0">
                             <div class="alert alert-warning" role="alert" style="float: left; display:table">
@@ -287,7 +293,7 @@ <h1>Live Chat</h1>
                         $('div#message_thread').append(`
                         <div class="row" style="margin-left: 0; margin-right: 0">
                             <div class="alert alert-info" role="alert" style="float: right; display:table">
-                                <span style="font-size: 16px"><b>`+self_display_name+`</b>: `+reward_text+`</span>
+                                <span style="font-size: 16px"><b>`+agent_id+`</b>: `+reward_text+`</span>
                             </div>
                         </div>
                         `);
@@ -295,7 +301,7 @@ <h1>Live Chat</h1>
                         $('div#message_thread').append(`
                         <div class="row" style="margin-left: 0; margin-right: 0">
                             <div class="alert alert-info" role="alert" style="float: right; display:table">
-                                <span style="font-size: 16px"><b>`+self_display_name+`</b>: `+message_text+`</span>
+                                <span style="font-size: 16px"><b>`+agent_id+`</b>: `+message_text+`</span>
                             </div>
                         </div>
                         `);
@@ -308,7 +314,7 @@ <h1>Live Chat</h1>
             $("div#message_thread").css("display", "");
             scroll_conversation_to_bottom();
         }
-        if (received_message_from_other_agents) {
+        if (received_message_from_other_agents && (!task_done)) {
             $("div#response-type-text-input").css("display", "");
             $("input#id_text_input").focus();
         }
@@ -362,6 +368,7 @@ <h1>Live Chat</h1>
                 show_new_messages_on_UI(new_messages);
                 if (done_after_responding) {
                     update_for_response_type('done');
+                    task_done = true;
                 } else {
                     check_done(new_messages);
                 }
@@ -456,7 +463,6 @@ <h1>Live Chat</h1>
         $("input#id_text_input").focus();
         update_for_response_type('text_input');
         if (is_approval_page) {
-            self_display_name = 'Worker';
             $("div#left-pane").css("min-height", $(window).height());
         }
         if (is_cover_page) {
diff --git a/parlai/mturk/core/setup_aws.py b/parlai/mturk/core/setup_aws.py
index e165ccb9aee..67bc4eb4593 100644
--- a/parlai/mturk/core/setup_aws.py
+++ b/parlai/mturk/core/setup_aws.py
@@ -228,7 +228,7 @@ def setup_rds():
 
     return host
 
-def setup_relay_server_api(mturk_submit_url, rds_host, task_config, is_sandbox, num_hits, requester_key_gt, should_clean_up_after_upload=True):
+def setup_relay_server_api(mturk_submit_url, rds_host, task_description, is_sandbox, num_hits, requester_key_gt, should_clean_up_after_upload=False):
     # Dynamically generate handler.py file, and then create zip file
     print("Lambda: Preparing relay server code...")
 
@@ -253,7 +253,7 @@ def setup_relay_server_api(mturk_submit_url, rds_host, task_config, is_sandbox,
         "requester_key_gt = \'" + requester_key_gt + "\'\n" + \
         "num_hits = " + str(num_hits) + "\n" + \
         "is_sandbox = " + str(is_sandbox) + "\n" + \
-        'task_description = ' + task_config['task_description'])
+        'task_description = ' + task_description)
     with open(os.path.join(parent_dir, lambda_server_directory_name, 'handler.py'), 'w') as handler_file:
         handler_file.write(handler_file_string)
     create_zip_file(
@@ -600,7 +600,7 @@ def setup_all_dependencies(lambda_server_directory_name):
     call(command, stdout=devnull, stderr=devnull)
 
     # Set up psycopg2
-    command = "git clone https://github.com/yf225/awslambda-psycopg2.git".split(" ")
+    command = "git clone https://github.com/jkehler/awslambda-psycopg2.git".split(" ")
     call(command, stdout=devnull, stderr=devnull)
     shutil.copytree("./awslambda-psycopg2/with_ssl_support/psycopg2", parent_dir+'/'+lambda_server_directory_name+"/psycopg2")
     shutil.rmtree("./awslambda-psycopg2")
@@ -632,13 +632,13 @@ def create_zip_file(lambda_server_directory_name, lambda_server_zip_file_name, f
     if verbose:
         print("Done!")
 
-def setup_aws(task_config, num_hits, is_sandbox):
+def setup_aws(task_description, num_hits, is_sandbox):
     mturk_submit_url = 'https://workersandbox.mturk.com/mturk/externalSubmit'
     if not is_sandbox:
         mturk_submit_url = 'https://www.mturk.com/mturk/externalSubmit'
     requester_key_gt = get_requester_key()
     rds_host = setup_rds()
-    html_api_endpoint_url, json_api_endpoint_url = setup_relay_server_api(mturk_submit_url, rds_host, task_config, is_sandbox, num_hits, requester_key_gt)
+    html_api_endpoint_url, json_api_endpoint_url = setup_relay_server_api(mturk_submit_url, rds_host, task_description, is_sandbox, num_hits, requester_key_gt)
 
     return html_api_endpoint_url, json_api_endpoint_url, requester_key_gt
 
diff --git a/parlai/mturk/tasks/multi_agent_dialog/sample.py b/parlai/mturk/tasks/multi_agent_dialog/sample.py
new file mode 100644
index 00000000000..4a590b29ed4
--- /dev/null
+++ b/parlai/mturk/tasks/multi_agent_dialog/sample.py
@@ -0,0 +1,22 @@
+import time
+from parlai.mturk.core.agents import MTurkAgent
+from parlai.agents.local_human.local_human import LocalHumanAgent
+from parlai.core.worlds import MultiAgentDialogWorld
+
+conversation_id = str(int(time.time())) # Use the same conversation_id for all mturk agents
+task_name = 'multi_agent_dialog'
+mturk_agent_ids = ['mturk_agent_1', 'mturk_agent_2']
+mturk_agent_1 = MTurkAgent({'agent_id':'mturk_agent_1', 'task_name':task_name, 'is_sandbox':True, 'conversation_id':conversation_id, 'mturk_agent_ids':mturk_agent_ids})
+mturk_agent_2 = MTurkAgent({'agent_id':'mturk_agent_2', 'task_name':task_name, 'is_sandbox':True, 'conversation_id':conversation_id, 'mturk_agent_ids':mturk_agent_ids})
+
+human_agent_1 = LocalHumanAgent(opt=None)
+human_agent_1.id = 'human_1'
+human_agent_2 = LocalHumanAgent(opt=None)
+human_agent_2.id = 'human_2'
+
+world = MultiAgentDialogWorld({'task':'test'}, [human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2])
+
+while not world.epoch_done():
+    world.parley()
+
+world.shutdown()
\ No newline at end of file

From ffe501e4dc6aefa9541fa2b1be80cad310544988 Mon Sep 17 00:00:00 2001
From: Will Feng <willfeng@fb.com>
Date: Wed, 7 Jun 2017 18:25:10 -0400
Subject: [PATCH 2/6] avoid conflicting AWS setup

---
 parlai/mturk/core/agents.py                     |  2 +-
 parlai/mturk/core/setup_aws.py                  | 12 +++++++-----
 parlai/mturk/tasks/multi_agent_dialog/sample.py |  6 +++---
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/parlai/mturk/core/agents.py b/parlai/mturk/core/agents.py
index 0fd58687eac..02b7f204e74 100644
--- a/parlai/mturk/core/agents.py
+++ b/parlai/mturk/core/agents.py
@@ -178,7 +178,7 @@ def act(self):
 
             time.sleep(1)
 
-    def epoch_done(self):
+    def episode_done(self):
         return False
 
     def shutdown(self):
diff --git a/parlai/mturk/core/setup_aws.py b/parlai/mturk/core/setup_aws.py
index 5e54b3bb57e..3e380c11f3b 100644
--- a/parlai/mturk/core/setup_aws.py
+++ b/parlai/mturk/core/setup_aws.py
@@ -14,21 +14,23 @@
 import json
 import webbrowser
 import hashlib
+import getpass
 from botocore.exceptions import ClientError
 from botocore.exceptions import ProfileNotFound
 
 aws_profile_name = 'parlai_mturk'
 region_name = 'us-west-2'
+user_name = getpass.getuser()
 
 iam_role_name = 'parlai_relay_server'
-lambda_function_name = 'parlai_relay_server'
+lambda_function_name = 'parlai_relay_server_' + user_name
 lambda_permission_statement_id = 'lambda-permission-statement-id'
-api_gateway_name = 'ParlaiRelayServer'
+api_gateway_name = 'ParlaiRelayServer_' + user_name
 endpoint_api_name_html = 'html'  # For GET-ing HTML
 endpoint_api_name_json = 'json'  # For GET-ing and POST-ing JSON
 
-rds_db_instance_identifier = 'parlai-mturk-db'
-rds_db_name = 'parlai_mturk_db'
+rds_db_instance_identifier = 'parlai-mturk-db-' + user_name
+rds_db_name = 'parlai_mturk_db_' + user_name
 rds_username = 'parlai_user'
 rds_password = 'parlai_user_password'
 rds_security_group_name = 'parlai-mturk-db-security-group'
@@ -228,7 +230,7 @@ def setup_rds():
 
     return host
 
-def setup_relay_server_api(mturk_submit_url, rds_host, task_description, is_sandbox, num_hits, requester_key_gt, should_clean_up_after_upload=False):
+def setup_relay_server_api(mturk_submit_url, rds_host, task_description, is_sandbox, num_hits, requester_key_gt, should_clean_up_after_upload=True):
     # Dynamically generate handler.py file, and then create zip file
     print("Lambda: Preparing relay server code...")
 
diff --git a/parlai/mturk/tasks/multi_agent_dialog/sample.py b/parlai/mturk/tasks/multi_agent_dialog/sample.py
index 4a590b29ed4..f0372e2086e 100644
--- a/parlai/mturk/tasks/multi_agent_dialog/sample.py
+++ b/parlai/mturk/tasks/multi_agent_dialog/sample.py
@@ -14,9 +14,9 @@
 human_agent_2 = LocalHumanAgent(opt=None)
 human_agent_2.id = 'human_2'
 
-world = MultiAgentDialogWorld({'task':'test'}, [human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2])
+world = MultiAgentDialogWorld({'task':task_name}, [human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2])
 
-while not world.epoch_done():
+while not world.episode_done():
     world.parley()
 
-world.shutdown()
\ No newline at end of file
+world.shutdown()

From a3e1b9ff48de1f610ade4ec44f25966dca5cb880 Mon Sep 17 00:00:00 2001
From: Will Feng <willfeng@fb.com>
Date: Thu, 8 Jun 2017 01:14:18 -0400
Subject: [PATCH 3/6] adopted sample tasks to new design

---
 parlai/mturk/core/agents.py                   | 32 +++++-----
 parlai/mturk/core/handler_template.py         |  2 +
 parlai/mturk/core/mturk_index.html            | 25 ++++++--
 parlai/mturk/run_mturk.py                     | 26 --------
 parlai/mturk/tasks/model_evaluator/agents.py  | 58 -----------------
 parlai/mturk/tasks/model_evaluator/run.py     | 53 +++++++++++++++
 .../tasks/model_evaluator/task_config.py      |  6 --
 parlai/mturk/tasks/model_evaluator/worlds.py  | 46 +++++++++++++
 parlai/mturk/tasks/multi_agent_dialog/run.py  | 54 ++++++++++++++++
 .../mturk/tasks/multi_agent_dialog/sample.py  | 22 -------
 .../tasks/multi_agent_dialog/task_config.py   | 39 +++++++++++
 parlai/mturk/tasks/qa_data_collection/run.py  | 50 +++++++++++++++
 .../tasks/qa_data_collection/task_config.py   |  5 --
 .../{agents.py => worlds.py}                  | 64 +++++++++----------
 14 files changed, 309 insertions(+), 173 deletions(-)
 delete mode 100644 parlai/mturk/run_mturk.py
 delete mode 100644 parlai/mturk/tasks/model_evaluator/agents.py
 create mode 100644 parlai/mturk/tasks/model_evaluator/run.py
 create mode 100644 parlai/mturk/tasks/model_evaluator/worlds.py
 create mode 100644 parlai/mturk/tasks/multi_agent_dialog/run.py
 delete mode 100644 parlai/mturk/tasks/multi_agent_dialog/sample.py
 create mode 100644 parlai/mturk/tasks/multi_agent_dialog/task_config.py
 create mode 100644 parlai/mturk/tasks/qa_data_collection/run.py
 rename parlai/mturk/tasks/qa_data_collection/{agents.py => worlds.py} (54%)

diff --git a/parlai/mturk/core/agents.py b/parlai/mturk/core/agents.py
index 02b7f204e74..0a40cb7a767 100644
--- a/parlai/mturk/core/agents.py
+++ b/parlai/mturk/core/agents.py
@@ -72,21 +72,21 @@ def __init__(self, opt, shared=None):
         super().__init__(opt)
 
         self.id = opt['agent_id']
-        self.task_name = opt['task_name']
+        self.task_name = opt['task']
         self.is_sandbox = opt['is_sandbox']
         self.conversation_id = opt['conversation_id']
         self.mturk_agent_ids = opt['mturk_agent_ids']
-
-        self.hit_reward = opt.get('reward', 0.5)
-        self.hit_title = opt.get('hit_title', 'hit title')
-        self.hit_description = opt.get('hit_description', 'hit description')
-        self.hit_keywords = opt.get('hit_keywords', 'hitkeywords')
-        self.task_description = "'''" + opt.get('task_description', 'hit task description') + "'''"
+        self.all_agent_ids = opt['all_agent_ids']
+        self.hit_reward = opt['reward']
+        self.hit_title = opt['hit_title']
+        self.hit_description = opt['hit_description']
+        self.hit_keywords = opt['hit_keywords']
+        self.task_description = opt['task_description']
 
         self.last_message_id = 0
 
         if not self.__class__.skip_init:
-            print("\nYou are going to allow workers from Amazon Mechanical Turk to chat with your dialog model running on your local machine.\nDuring this process, Internet connection is required, and you should turn off your computer's auto-sleep feature.\n")
+            print("\nYou are going to allow workers from Amazon Mechanical Turk to be an agent in ParlAI.\nDuring this process, Internet connection is required, and you should turn off your computer's auto-sleep feature.\n")
             key_input = input("Please press Enter to continue... ")
             print("")
             
@@ -101,6 +101,7 @@ def __init__(self, opt, shared=None):
             self.__class__.html_api_endpoint_url = html_api_endpoint_url
             self.__class__.json_api_endpoint_url = json_api_endpoint_url
             self.__class__.requester_key_gt = requester_key_gt
+            print("MTurk setup done.\n")
 
         self.__class__.skip_init = True
         self.html_api_endpoint_url = self.__class__.html_api_endpoint_url
@@ -117,20 +118,19 @@ def __init__(self, opt, shared=None):
             hit_reward=self.hit_reward,
             is_sandbox=self.is_sandbox
         )
-        mturk_chat_url = self.html_api_endpoint_url + "?method_name=chat_index&task_group_id="+str(self.task_group_id)+"&conversation_id="+str(self.conversation_id)+"&cur_agent_id="+str(self.id)
+        all_agent_ids_string = str(self.all_agent_ids).replace("'", '''"''')
+        mturk_chat_url = self.html_api_endpoint_url + "?method_name=chat_index&task_group_id="+str(self.task_group_id)+"&conversation_id="+str(self.conversation_id)+"&all_agent_ids="+all_agent_ids_string+"&cur_agent_id="+str(self.id)
         mturk_page_url = create_hit_with_hit_type(
             page_url=mturk_chat_url,
             hit_type_id=hit_type_id,
             is_sandbox=self.is_sandbox
         )
 
-        print("MTurk setup done.\n")
-        print("Link to your HIT: " + mturk_page_url + "\n")
-        print("Waiting for Turkers to complete the tasks... (Please don't close your laptop or put your computer into sleep or standby mode.)\n")
+        print("Link to HIT for " + self.id + ": " + mturk_page_url + "\n")
+        print("Waiting for Turkers to respond... (Please don't close your laptop or put your computer into sleep or standby mode.)\n")
 
     def observe(self, msg):
         if msg['id'] not in self.mturk_agent_ids: # If the message sender is an mturk agent, then there is no need to upload this message to db since it's already been done on the message sender side.
-            agent_last_message_in_db = None
             conversation_dict = _get_new_messages(
                 json_api_endpoint_url=self.json_api_endpoint_url, 
                 task_group_id=self.task_group_id,
@@ -173,10 +173,9 @@ def act(self):
 
                 new_messages = conversation_dict[str(self.conversation_id)]
 
-                print(display_messages([new_messages[0]]))
                 return new_messages[0]
 
-            time.sleep(1)
+            time.sleep(1) # Wait for 1 second, so that we are not polling too frequently.
 
     def episode_done(self):
         return False
@@ -186,6 +185,7 @@ def shutdown(self):
             _get_review_status_count(json_api_endpoint_url=self.json_api_endpoint_url, task_group_id=self.task_group_id, conversation_id=self.conversation_id, review_status='rejected', requester_key=self.requester_key_gt) > 0:
             return
         else:
+            # Loop to ensure all HITs are done
             while _get_review_status_count(json_api_endpoint_url=self.json_api_endpoint_url, task_group_id=self.task_group_id, conversation_id=self.conversation_id, review_status='pending', requester_key=self.requester_key_gt) < len(self.mturk_agent_ids):
                 time.sleep(2)
 
@@ -195,7 +195,7 @@ def shutdown(self):
             print(mturk_approval_url)
             print("")
 
-            # Loop for checking approval status
+            # Loop for checking review status
             while _get_review_status_count(json_api_endpoint_url=self.json_api_endpoint_url, task_group_id=self.task_group_id, conversation_id=self.conversation_id, review_status='pending', requester_key=self.requester_key_gt) > 0:
                 time.sleep(2)
 
diff --git a/parlai/mturk/core/handler_template.py b/parlai/mturk/core/handler_template.py
index b4155a438ac..240f4ff53c9 100755
--- a/parlai/mturk/core/handler_template.py
+++ b/parlai/mturk/core/handler_template.py
@@ -53,6 +53,7 @@ def chat_index(event, context):
         try:
             task_group_id = event['query']['task_group_id']
             conversation_id = event['query']['conversation_id']
+            all_agent_ids = event['query']['all_agent_ids']
             cur_agent_id = event['query']['cur_agent_id']
             assignment_id = event['query']['assignmentId'] # from mturk
 
@@ -63,6 +64,7 @@ def chat_index(event, context):
                 template_context['task_group_id'] = task_group_id
                 template_context['conversation_id'] = conversation_id
                 template_context['cur_agent_id'] = cur_agent_id
+                template_context['all_agent_ids'] = all_agent_ids
                 template_context['task_description'] = task_description
                 template_context['mturk_submit_url'] = mturk_submit_url
                 template_context['is_cover_page'] = False
diff --git a/parlai/mturk/core/mturk_index.html b/parlai/mturk/core/mturk_index.html
index a6494c9ceae..7513b73098b 100755
--- a/parlai/mturk/core/mturk_index.html
+++ b/parlai/mturk/core/mturk_index.html
@@ -243,14 +243,25 @@ <h1>Live Chat</h1>
     });
 
     var mturk_submit_url = `{{mturk_submit_url}}`;
-    var verbose_log = true;
+    var verbose_log = false;
     var task_group_id = `{{task_group_id}}`;
     var conversation_id = parseInt(`{{conversation_id}}`);
     var cur_agent_id = `{{cur_agent_id}}`;
     var mturk_agent_ids = `{{mturk_agent_ids}}`;
     if (mturk_agent_ids) {
         mturk_agent_ids = JSON.parse(mturk_agent_ids);
-        console.log(mturk_agent_ids);
+    }
+    var all_agent_ids = `{{all_agent_ids}}`;
+    var cur_agent_index = null;
+    var previous_agent_id = null;
+    if (all_agent_ids) {
+        all_agent_ids = JSON.parse(all_agent_ids);
+        cur_agent_index = all_agent_ids.indexOf(cur_agent_id);
+        if (cur_agent_index == 0) {
+            previous_agent_id = all_agent_ids[all_agent_ids.length-1];
+        } else {
+            previous_agent_id = all_agent_ids[cur_agent_index-1];
+        }
     }
     var is_cover_page = (`{{is_cover_page}}` === 'True') ? true : false;
     var is_approval_page = (`{{is_approval_page}}` === 'True') ? true : false;
@@ -314,8 +325,8 @@ <h1>Live Chat</h1>
             $("div#message_thread").css("display", "");
             scroll_conversation_to_bottom();
         }
-        if (received_message_from_other_agents && (!task_done)) {
-            $("div#response-type-text-input").css("display", "");
+        if (received_message_from_other_agents && (!task_done) && new_messages[new_messages.length-1]['id'] === previous_agent_id) {
+            update_for_response_type('text_input');
             $("input#id_text_input").focus();
         }
     }
@@ -461,7 +472,11 @@ <h1>Live Chat</h1>
         $("form#mturk_submit_form input#workerId").val(get_url_parameter('workerId'));
         $("form#mturk_submit_form").attr("action", mturk_submit_url);
         $("input#id_text_input").focus();
-        update_for_response_type('text_input');
+        if (cur_agent_index == 0) {
+            update_for_response_type('text_input');
+        } else {
+            update_for_response_type('idle');
+        }
         if (is_approval_page) {
             $("div#left-pane").css("min-height", $(window).height());
         }
diff --git a/parlai/mturk/run_mturk.py b/parlai/mturk/run_mturk.py
deleted file mode 100644
index f4f55ab1b27..00000000000
--- a/parlai/mturk/run_mturk.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree. An additional grant
-# of patent rights can be found in the PATENTS file in the same directory.
-from parlai.core.params import ParlaiParser
-from core import manage_hit
-
-argparser = ParlaiParser(False, False)
-argparser.add_parlai_data_path()
-argparser.add_mturk_args()
-
-opt = argparser.parse_args()
-
-task_module_name = 'parlai.mturk.tasks.' + opt['task']
-Agent = __import__(task_module_name+'.agents', fromlist=['']).default_agent_class
-task_config = __import__(task_module_name+'.task_config', fromlist=['']).task_config
-
-print("Creating HIT tasks for "+task_module_name+" ...")
-
-manage_hit.create_hits(
-	opt=opt,
-	task_config=task_config,
-	task_module_name=task_module_name,
-	bot=Agent(opt=opt),
-)
\ No newline at end of file
diff --git a/parlai/mturk/tasks/model_evaluator/agents.py b/parlai/mturk/tasks/model_evaluator/agents.py
deleted file mode 100644
index 43f68f43109..00000000000
--- a/parlai/mturk/tasks/model_evaluator/agents.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree. An additional grant
-# of patent rights can be found in the PATENTS file in the same directory.
-import copy
-import importlib
-from parlai.core.agents import Agent
-from parlai.core.agents import create_agent
-from parlai.core.worlds import create_task
-
-class ModelEvaluatorAgent(Agent):
-    """
-    MTurk agent for evaluating a dialog model's performance given a context.
-    Assumes the context is a context from a given task, e.g.
-    from SQuAD, CBT, etc.
-    """
-    def __init__(self, opt, shared=None):
-        self.opt = copy.deepcopy(opt)
-        self.id = 'Model Evaluator'
-
-        # The dialog model we will evaluate
-        agent_opt = {}
-        agent_opt['model'] = 'ir_baseline'
-        agent = create_agent(agent_opt)
-
-        # The task that we will evaluate the dialog model on
-        task_opt = {}
-        task_opt['datatype'] = 'test'
-        task_opt['datapath'] = opt['datapath']
-        task_opt['task'] = '#MovieDD-Reddit'
-        self.world = create_task(task_opt, agent)
-
-    def observe(self, observation):
-        self.observation = observation
-
-        # The rating given by turker
-        # Because we only have one turn in this conversation, we don't need to track turn_index
-        # print(self.observation)
-        return observation
-
-    def act(self):
-        # All agents act once in the world
-        self.world.parley()
-
-        ad = {}
-        # Show the dialog model's response to the context, and ask the turker to rate the response
-        ad['text'] = (
-            self.world.get_acts()[0]['text'] + "\n\n" +
-            "How would you rate the following response (from 0 to 10):\n\n" +
-            self.world.get_acts()[1]['text'])
-
-        # TODO: deal with multi-turn dialogs, for now we will just deal
-        # with 1-turn dialogs in this task.
-        ad['episode_done'] = True  # self.world.episode_done()
-        return ad
-
-default_agent_class = ModelEvaluatorAgent
diff --git a/parlai/mturk/tasks/model_evaluator/run.py b/parlai/mturk/tasks/model_evaluator/run.py
new file mode 100644
index 00000000000..4c631722a84
--- /dev/null
+++ b/parlai/mturk/tasks/model_evaluator/run.py
@@ -0,0 +1,53 @@
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+from parlai.core.params import ParlaiParser
+from parlai.mturk.tasks.model_evaluator.worlds import ModelEvaluatorWorld
+from parlai.mturk.core.agents import MTurkAgent
+from parlai.core.worlds import create_task
+from task_config import task_config
+import time
+import os
+
+
+def main():
+    argparser = ParlaiParser(False, False)
+    argparser.add_parlai_data_path()
+    argparser.add_mturk_args()
+
+    # The dialog model we want to evaluate
+    from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
+    IrBaselineAgent.add_cmdline_args(argparser)
+    opt = argparser.parse_args()
+    opt['task'] = os.path.basename(os.getcwd())
+    
+    model_agent = IrBaselineAgent(opt=opt)
+
+    # The task that we will evaluate the dialog model on
+    task_opt = {}
+    task_opt['datatype'] = 'test'
+    task_opt['datapath'] = opt['datapath']
+    task_opt['task'] = '#MovieDD-Reddit'
+    task_world = create_task(task_opt, model_agent)
+
+    # Create the MTurk agent which provides a chat interface to the Turker
+    opt.update(task_config)
+    mturk_agent_id = 'Worker'
+    opt['agent_id'] = mturk_agent_id
+    opt['mturk_agent_ids'] = [mturk_agent_id]
+    opt['all_agent_ids'] = [ModelEvaluatorWorld.evaluator_agent_id, mturk_agent_id]
+    opt['conversation_id'] = str(int(time.time()))
+
+    mturk_agent = MTurkAgent(opt=opt)
+
+    world = ModelEvaluatorWorld(opt=opt, task_world=task_world, mturk_agent=mturk_agent)
+
+    while not world.episode_done():
+        world.parley()
+
+    world.shutdown()
+
+if __name__ == '__main__':
+    main()
diff --git a/parlai/mturk/tasks/model_evaluator/task_config.py b/parlai/mturk/tasks/model_evaluator/task_config.py
index 315d7c1aca2..a78dcbcf4a2 100644
--- a/parlai/mturk/tasks/model_evaluator/task_config.py
+++ b/parlai/mturk/tasks/model_evaluator/task_config.py
@@ -6,7 +6,6 @@
 
 task_config = {}
 
-
 """A short and descriptive title about the kind of task the HIT contains.
 On the Amazon Mechanical Turk web site, the HIT title appears in search results,
 and everywhere the HIT is mentioned.
@@ -27,11 +26,6 @@
 task_config['hit_keywords'] = 'chat,dialog,rating'
 
 
-"""A short name indicating the turker's role in the conversation.
-"""
-task_config['worker_agent_id'] = 'Teacher'
-
-
 """A detailed task description that will be shown on the HIT task preview page
 and on the left side of the chat page. Supports HTML formatting.
 """
diff --git a/parlai/mturk/tasks/model_evaluator/worlds.py b/parlai/mturk/tasks/model_evaluator/worlds.py
new file mode 100644
index 00000000000..cece81f32c8
--- /dev/null
+++ b/parlai/mturk/tasks/model_evaluator/worlds.py
@@ -0,0 +1,46 @@
+from parlai.core.worlds import World, validate
+
+
+class ModelEvaluatorWorld(World):
+    """
+    World for letting Turkers evaluate a dialog model's performance given a context.
+    Assumes the context is a context from a given task, e.g. from SQuAD, CBT, etc.
+    """
+
+    evaluator_agent_id = 'Model Evaluator'
+
+    def __init__(self, opt, task_world, mturk_agent):
+        self.task_world = task_world
+        self.mturk_agent = mturk_agent
+        self.episodeDone = False
+
+    def parley(self):
+        self.task_world.parley()
+
+        ad = {}
+        # Show the dialog model's response to the context, and ask the turker to rate the response
+        ad['id'] = self.__class__.evaluator_agent_id
+        ad['text'] = (
+            self.task_world.get_acts()[0]['text'] + "\n\n" +
+            "How would you rate the following response (from 0 to 10):\n\n" +
+            self.task_world.get_acts()[1]['text'])
+
+        # TODO: deal with multi-turn dialogs, for now we will just deal
+        # with 1-turn dialogs in this task.
+        ad['episode_done'] = True  # self.world.episode_done()
+        
+        self.mturk_agent.observe(validate(ad))
+        rating = self.mturk_agent.act()
+
+        self.episodeDone = True
+
+    def episode_done(self):
+        return self.episodeDone
+
+    def report(self):
+        # TODO: Add logging code here
+        pass
+
+    def shutdown(self):
+        self.task_world.shutdown()
+        self.mturk_agent.shutdown()
diff --git a/parlai/mturk/tasks/multi_agent_dialog/run.py b/parlai/mturk/tasks/multi_agent_dialog/run.py
new file mode 100644
index 00000000000..13fe5de401e
--- /dev/null
+++ b/parlai/mturk/tasks/multi_agent_dialog/run.py
@@ -0,0 +1,54 @@
+import os
+import time
+from parlai.core.params import ParlaiParser
+from parlai.mturk.core.agents import MTurkAgent
+from parlai.agents.local_human.local_human import LocalHumanAgent
+from parlai.core.worlds import MultiAgentDialogWorld
+from task_config import task_config
+
+"""
+This task consists of two local human agents and two MTurk agents,
+chatting with each other in a free-form format.
+You can end the conversation by sending a message ending with
+`[DONE]` from human_1.
+"""
+def main():
+    argparser = ParlaiParser(False, False)
+    argparser.add_parlai_data_path()
+    argparser.add_mturk_args()
+    opt = argparser.parse_args()
+    opt['task'] = os.path.basename(os.getcwd())
+
+    mturk_agent_1_id = 'mturk_agent_1'
+    mturk_agent_2_id = 'mturk_agent_2'
+    human_agent_1_id = 'human_1'
+    human_agent_2_id = 'human_2'
+
+    # Create the MTurk agents
+    opt.update(task_config)
+    opt['conversation_id'] = str(int(time.time()))
+    
+    opt['mturk_agent_ids'] = [mturk_agent_1_id, mturk_agent_2_id]
+    opt['all_agent_ids'] = [human_agent_1_id, human_agent_2_id, mturk_agent_1_id, mturk_agent_2_id]
+
+    opt['agent_id'] = mturk_agent_1_id
+    mturk_agent_1 = MTurkAgent(opt=opt)
+
+    opt['agent_id'] = mturk_agent_2_id
+    mturk_agent_2 = MTurkAgent(opt=opt)
+
+    # Create the local human agents
+    human_agent_1 = LocalHumanAgent(opt=None)
+    human_agent_1.id = human_agent_1_id
+    human_agent_2 = LocalHumanAgent(opt=None)
+    human_agent_2.id = human_agent_2_id
+
+    world = MultiAgentDialogWorld(opt=opt, agents=[human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2])
+
+    while not world.episode_done():
+        world.parley()
+
+    world.shutdown()
+
+if __name__ == '__main__':
+    main()
diff --git a/parlai/mturk/tasks/multi_agent_dialog/sample.py b/parlai/mturk/tasks/multi_agent_dialog/sample.py
deleted file mode 100644
index f0372e2086e..00000000000
--- a/parlai/mturk/tasks/multi_agent_dialog/sample.py
+++ /dev/null
@@ -1,22 +0,0 @@
-import time
-from parlai.mturk.core.agents import MTurkAgent
-from parlai.agents.local_human.local_human import LocalHumanAgent
-from parlai.core.worlds import MultiAgentDialogWorld
-
-conversation_id = str(int(time.time())) # Use the same conversation_id for all mturk agents
-task_name = 'multi_agent_dialog'
-mturk_agent_ids = ['mturk_agent_1', 'mturk_agent_2']
-mturk_agent_1 = MTurkAgent({'agent_id':'mturk_agent_1', 'task_name':task_name, 'is_sandbox':True, 'conversation_id':conversation_id, 'mturk_agent_ids':mturk_agent_ids})
-mturk_agent_2 = MTurkAgent({'agent_id':'mturk_agent_2', 'task_name':task_name, 'is_sandbox':True, 'conversation_id':conversation_id, 'mturk_agent_ids':mturk_agent_ids})
-
-human_agent_1 = LocalHumanAgent(opt=None)
-human_agent_1.id = 'human_1'
-human_agent_2 = LocalHumanAgent(opt=None)
-human_agent_2.id = 'human_2'
-
-world = MultiAgentDialogWorld({'task':task_name}, [human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2])
-
-while not world.episode_done():
-    world.parley()
-
-world.shutdown()
diff --git a/parlai/mturk/tasks/multi_agent_dialog/task_config.py b/parlai/mturk/tasks/multi_agent_dialog/task_config.py
new file mode 100644
index 00000000000..3a9b2e7b4dc
--- /dev/null
+++ b/parlai/mturk/tasks/multi_agent_dialog/task_config.py
@@ -0,0 +1,39 @@
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+
+task_config = {}
+
+"""A short and descriptive title about the kind of task the HIT contains.
+On the Amazon Mechanical Turk web site, the HIT title appears in search results,
+and everywhere the HIT is mentioned.
+"""
+task_config['hit_title'] = 'Chat with three other people'
+
+
+"""A description includes detailed information about the kind of task the HIT contains.
+On the Amazon Mechanical Turk web site, the HIT description appears in the expanded
+view of search results, and in the HIT and assignment screens.
+"""
+task_config['hit_description'] = 'Chat with three other people.'
+
+
+"""One or more words or phrases that describe the HIT, separated by commas.
+On MTurk website, these words are used in searches to find HITs.
+"""
+task_config['hit_keywords'] = 'chat,dialog'
+
+
+"""A detailed task description that will be shown on the HIT task preview page
+and on the left side of the chat page. Supports HTML formatting.
+"""
+task_config['task_description'] = \
+'''\'\'\'
+In this task, you are going to chat with three other people.<br><br>
+
+If you are ready, please click "Accept HIT" to start this task.
+\'\'\''''
+
+
diff --git a/parlai/mturk/tasks/qa_data_collection/run.py b/parlai/mturk/tasks/qa_data_collection/run.py
new file mode 100644
index 00000000000..2daf73be9ce
--- /dev/null
+++ b/parlai/mturk/tasks/qa_data_collection/run.py
@@ -0,0 +1,50 @@
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+from parlai.core.params import ParlaiParser
+from parlai.mturk.tasks.qa_data_collection.worlds import QADataCollectionWorld
+from parlai.mturk.core.agents import MTurkAgent
+from task_config import task_config
+import time
+import os
+import importlib
+
+
+def main():
+    argparser = ParlaiParser(False, False)
+    argparser.add_parlai_data_path()
+    argparser.add_mturk_args()
+    opt = argparser.parse_args()
+    opt['task'] = os.path.basename(os.getcwd())
+
+    # Initialize a SQuAD teacher agent, which we will later get context from
+    module_name = 'parlai.tasks.squad.agents'
+    class_name = 'DefaultTeacher'
+    my_module = importlib.import_module(module_name)
+    task_class = getattr(my_module, class_name)
+    task_opt = {}
+    task_opt['datatype'] = 'train'
+    task_opt['datapath'] = opt['datapath']
+    task = task_class(task_opt)
+
+    # Create the MTurk agent which provides a chat interface to the Turker
+    opt.update(task_config)
+    mturk_agent_id = 'Worker'
+    opt['agent_id'] = mturk_agent_id
+    opt['mturk_agent_ids'] = [mturk_agent_id]
+    opt['all_agent_ids'] = [QADataCollectionWorld.collector_agent_id, mturk_agent_id]
+    opt['conversation_id'] = str(int(time.time()))
+
+    mturk_agent = MTurkAgent(opt=opt)
+
+    world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent)
+
+    while not world.episode_done():
+        world.parley()
+
+    world.shutdown()
+
+if __name__ == '__main__':
+    main()
diff --git a/parlai/mturk/tasks/qa_data_collection/task_config.py b/parlai/mturk/tasks/qa_data_collection/task_config.py
index ba7b2b9b067..7e23158704e 100644
--- a/parlai/mturk/tasks/qa_data_collection/task_config.py
+++ b/parlai/mturk/tasks/qa_data_collection/task_config.py
@@ -27,11 +27,6 @@
 task_config['hit_keywords'] = 'chat,question,answer'
 
 
-"""A short name indicating the turker's role in the conversation.
-"""
-task_config['worker_agent_id'] = 'Teacher'
-
-
 """A detailed task description that will be shown on the HIT task preview page
 and on the left side of the chat page. Supports HTML formatting.
 """
diff --git a/parlai/mturk/tasks/qa_data_collection/agents.py b/parlai/mturk/tasks/qa_data_collection/worlds.py
similarity index 54%
rename from parlai/mturk/tasks/qa_data_collection/agents.py
rename to parlai/mturk/tasks/qa_data_collection/worlds.py
index 5e3caaa334e..a3a9522206a 100644
--- a/parlai/mturk/tasks/qa_data_collection/agents.py
+++ b/parlai/mturk/tasks/qa_data_collection/worlds.py
@@ -3,49 +3,28 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree. An additional grant
 # of patent rights can be found in the PATENTS file in the same directory.
-import copy
-import importlib
-from parlai.core.agents import Agent
-from parlai.core.agents import create_agent
+from parlai.core.worlds import World, validate
 
-class QADataCollectionAgent(Agent):
+
+class QADataCollectionWorld(World):
     """
-    MTurk agent for recording a turker's question and answer given a context.
+    World for recording a turker's question and answer given a context.
     Assumes the context is a random context from a given task, e.g.
     from SQuAD, CBT, etc.
     """
-    def __init__(self, opt, shared=None):
-        self.opt = copy.deepcopy(opt)
-        self.id = 'QA Collector'
-        self.turn_index = -1
 
-        # Initialize a SQuAD teacher agent, which we will later get context from
-        module_name = 'parlai.tasks.squad.agents'
-        class_name = 'DefaultTeacher'
-        my_module = importlib.import_module(module_name)
-        task_class = getattr(my_module, class_name)
-        task_opt = {}
-        task_opt['datatype'] = 'train'
-        task_opt['datapath'] = opt['datapath']
-        self.task = task_class(task_opt)
+    collector_agent_id = 'QA Collector'
 
-    def observe(self, observation):
-        self.observation = observation
+    def __init__(self, opt, task, mturk_agent):
+        self.task = task
+        self.mturk_agent = mturk_agent
+        self.episodeDone = False
+        self.turn_index = -1
 
-        if self.turn_index == 0:
-            # Turker's question, from the first turn
-            # print(self.observation)
-            pass
-        elif self.turn_index == 1:
-            # Turker's answer, from the second turn
-            # print(self.observation)
-            pass
-        return observation
-
-    def act(self):
+    def parley(self):
         self.turn_index = (self.turn_index + 1) % 2; # Each turn starts from the QA Collector agent
         ad = { 'episode_done': False }
-        ad['id'] = self.id
+        ad['id'] = self.__class__.collector_agent_id
 
         if self.turn_index == 0:
             # At the first turn, the QA Collector agent provides the context and
@@ -59,6 +38,9 @@ def act(self):
             ad['text'] = (context +
                         '\n\nPlease provide a question given this context.')
 
+            self.mturk_agent.observe(validate(ad))
+            self.question = self.mturk_agent.act() # Can log the turker's question here
+
         if self.turn_index == 1:
             # At the second turn, the QA Collector collects the turker's question from the first turn,
             # and then prompts the turker to provide the answer
@@ -68,6 +50,18 @@ def act(self):
 
             ad['episode_done'] = True  # end of episode
 
-        return ad
+            self.mturk_agent.observe(validate(ad))
+            self.answer = self.mturk_agent.act() # Can log the turker's answer here
+
+            self.episodeDone = True
+
+    def episode_done(self):
+        return self.episodeDone
+
+    def report(self):
+        # TODO: Add logging code here
+        pass
 
-default_agent_class = QADataCollectionAgent
+    def shutdown(self):
+        self.task.shutdown()
+        self.mturk_agent.shutdown()

From 36eb71c0281d78b23c2363a70a3eff04da3365c9 Mon Sep 17 00:00:00 2001
From: Will Feng <willfeng@fb.com>
Date: Thu, 8 Jun 2017 22:32:27 -0400
Subject: [PATCH 4/6] moved task world creation inside ModelEvaluatorWorld

---
 parlai/mturk/tasks/model_evaluator/run.py    | 5 +----
 parlai/mturk/tasks/model_evaluator/worlds.py | 7 +++----
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/parlai/mturk/tasks/model_evaluator/run.py b/parlai/mturk/tasks/model_evaluator/run.py
index 4c631722a84..4c2db99e501 100644
--- a/parlai/mturk/tasks/model_evaluator/run.py
+++ b/parlai/mturk/tasks/model_evaluator/run.py
@@ -6,7 +6,6 @@
 from parlai.core.params import ParlaiParser
 from parlai.mturk.tasks.model_evaluator.worlds import ModelEvaluatorWorld
 from parlai.mturk.core.agents import MTurkAgent
-from parlai.core.worlds import create_task
 from task_config import task_config
 import time
 import os
@@ -22,7 +21,6 @@ def main():
     IrBaselineAgent.add_cmdline_args(argparser)
     opt = argparser.parse_args()
     opt['task'] = os.path.basename(os.getcwd())
-    
     model_agent = IrBaselineAgent(opt=opt)
 
     # The task that we will evaluate the dialog model on
@@ -30,7 +28,6 @@ def main():
     task_opt['datatype'] = 'test'
     task_opt['datapath'] = opt['datapath']
     task_opt['task'] = '#MovieDD-Reddit'
-    task_world = create_task(task_opt, model_agent)
 
     # Create the MTurk agent which provides a chat interface to the Turker
     opt.update(task_config)
@@ -42,7 +39,7 @@ def main():
 
     mturk_agent = MTurkAgent(opt=opt)
 
-    world = ModelEvaluatorWorld(opt=opt, task_world=task_world, mturk_agent=mturk_agent)
+    world = ModelEvaluatorWorld(opt=opt, model_agent=model_agent, task_opt=task_opt, mturk_agent=mturk_agent)
 
     while not world.episode_done():
         world.parley()
diff --git a/parlai/mturk/tasks/model_evaluator/worlds.py b/parlai/mturk/tasks/model_evaluator/worlds.py
index cece81f32c8..6d7a26d78e3 100644
--- a/parlai/mturk/tasks/model_evaluator/worlds.py
+++ b/parlai/mturk/tasks/model_evaluator/worlds.py
@@ -1,5 +1,4 @@
-from parlai.core.worlds import World, validate
-
+from parlai.core.worlds import World, validate, create_task
 
 class ModelEvaluatorWorld(World):
     """
@@ -9,8 +8,8 @@ class ModelEvaluatorWorld(World):
 
     evaluator_agent_id = 'Model Evaluator'
 
-    def __init__(self, opt, task_world, mturk_agent):
-        self.task_world = task_world
+    def __init__(self, opt, model_agent, task_opt, mturk_agent):
+        self.task_world = create_task(task_opt, model_agent)
         self.mturk_agent = mturk_agent
         self.episodeDone = False
 

From 4f9b3f0fa10af76d470d89dcf5baa5c193427462 Mon Sep 17 00:00:00 2001
From: Will Feng <willfeng@fb.com>
Date: Thu, 8 Jun 2017 23:23:21 -0400
Subject: [PATCH 5/6] updated MTurk tutorial and README

---
 README.md                 |  26 ++++-----
 docs/source/mturk_new.rst | 120 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 132 insertions(+), 14 deletions(-)
 create mode 100644 docs/source/mturk_new.rst

diff --git a/README.md b/README.md
index f06fcac44cf..40d6d73bfe4 100644
--- a/README.md
+++ b/README.md
@@ -264,35 +264,33 @@ To add your own task:
 
 An important part of ParlAI is seamless integration with Mechanical Turk for data collection, training and evaluation.
 Human Turkers are also viewed as agents in ParlAI and hence person-person, person-bot, or multiple people and bots in group chat can all converse within the standard framework, switching out the roles as desired with no code changes to the agents. This is because Turkers also receive and send via a (pretty printed) version of the same interface, using the fields of the observation/action dict.
-We provide two examples in the first release, collecting data, and human evaluation of a bot.
+We currently provide three examples, which are collecting data, human evaluation of a bot, and round-robin chat between local humans and remote Turkers.
 
 <p align=center><img width="100%" src="docs/source/\_static/img/mturk.png" /></p>
 
 The mturk library contains the following directories and files:
 
 - **core**: this directory contains the core code for setting up AWS backend that supports the MTurk chat interface, and code for HIT creation and approval.
-- **tasks**: this directory contains two sample MTurk tasks that are provided in the first release.
+- **tasks**: this directory contains three sample MTurk tasks.
   - **_qa\_data\_collection_**: get questions and answers from turkers, given a random paragraph from SQuAD.
-  - **_model\_evaluator_**: evaluate the information retrieval baseline model on the Reddit movie dialog dataset.
-- **run_mturk.py**: file for calling mturk core code with user-specified task module, dialog model agent, number of HITs, and reward for each HIT.
+  - **_model\_evaluator_**: ask turkers to evaluate the information retrieval baseline model on the Reddit movie dialog dataset.
+  - **_multi\_agent\_dialog_**: round-robin chat between two local human agents and two Turkers.
 
-To run sample MTurk task and agent:
-- In __run\_mturk.py__, uncomment the task module and the agent class you want to use
-- For `create_hits` method, change `num_hits` and `hit_reward` if needed. Set `is_sandbox` to `True` if you want to run the sample in MTurk sandbox only, or set it to `False` to allow turkers to work on it and potentially get paid for it.
-- Run `python run_mturk.py`
+To run an MTurk task:
+- Go into the directory for the task you want to run.
+- Run `python run.py -nh <num_hits> -r <reward> [--sandbox]/[--live]`, with `<num_hits>` and `<reward>` set appropriately. Use `--sandbox` to run the task in MTurk sandbox mode before pushing it live.
 
-To add your own MTurk task and dialog model:
+To add your own MTurk task:
 - create a new folder within the mturk/tasks directory for your new task
 - implement __task\_config.py__, with at least the following fields in the task_config dictionary:
   - `hit_title`: a short and descriptive title about the kind of task the HIT contains. On the Amazon Mechanical Turk web site, the HIT title appears in search results, and everywhere the HIT is mentioned.
   - `hit_description`: a description includes detailed information about the kind of task the HIT contains. On the Amazon Mechanical Turk web site, the HIT description appears in the expanded view of search results, and in the HIT and assignment screens.
   - `hit_keywords`: one or more words or phrases that describe the HIT, separated by commas. On MTurk website, these words are used in searches to find HITs.
-  - `worker_agent_id`: a short name indicating the turker's role in the conversation.
   - `task_description`: a detailed task description that will be shown on the HIT task preview page and on the left side of the chat page. Supports HTML formatting.
-- implement __agents.py__, with at least an agent class that extends from Agent
-  - write your own `__init__()` method that wraps your dialog model agent. (Please see mturk/tasks/model_evaluator/agents.py file for a concrete example.)
-  - write your own `act()` method that returns your dialog model's response as well as helpful text to the turker for what action they should take next.
-- import your task module and agent class in __run\_mturk.py__ file, and then run `python run_mturk.py`.
+- implement __run.py__, with code for setting up and running the world where `MTurkAgent` lives in.
+- (Optional) implement __worlds.py__, with a world class that extends from `World`.
+
+Please see [the tutorial](http://parl.ai/static/docs/mturk.html) to learn more about the MTurk examples and how to create and run your own task.
 
 ## Support
 If you have any questions, bug reports or feature requests, please don't hesitate to post on our [Github Issues page](https://github.com/facebookresearch/ParlAI/issues).
diff --git a/docs/source/mturk_new.rst b/docs/source/mturk_new.rst
new file mode 100644
index 00000000000..5cec6b111ed
--- /dev/null
+++ b/docs/source/mturk_new.rst
@@ -0,0 +1,120 @@
+..
+  Copyright (c) 2017-present, Facebook, Inc.
+  All rights reserved.
+  This source code is licensed under the BSD-style license found in the
+  LICENSE file in the root directory of this source tree. An additional grant
+  of patent rights can be found in the PATENTS file in the same directory.
+
+Using Mechanical Turk
+=====================
+
+In ParlAI, you can use Amazon Mechanical Turk for **data collection**, **training** and **evaluation** of your dialog model. 
+
+Human Turkers are viewed as just another type of agent in ParlAI, and hence person-to-person, person-to-bot, or multiple people and bots in group chat can all talk to each other within the same framework. 
+
+The human Turkers communicate in observation/action dict format, the same as all other agents in ParlAI. During the conversation, the message that human Turkers receive is rendered on the live chat webpage in a pretty printed format, similar to the following:
+
+.. figure:: _static/img/mturk-small.png
+   :align: center
+
+   Example: Human Turker participating in a QA data collection task
+
+Each MTurk task has at least one human Turker that connects to ParlAI via the Mechanical Turk Live Chat interface, encapsulated as an ``MTurkAgent`` object.
+
+Each MTurk task also consists of a ``World`` where all agents live and interact within.
+
+Example Tasks
+-------------
+
+We provide a few examples of using Mechanical Turk with ParlAI:
+
+- `QA Data Collection <https://github.com/facebookresearch/ParlAI/blob/master/parlai/mturk/tasks/qa_data_collection/>`__: collect questions and answers from Turkers, given a random Wikipedia paragraph from SQuAD.
+- `Model Evaluator <https://github.com/facebookresearch/ParlAI/blob/master/parlai/mturk/tasks/model_evaluator/>`__: ask Turkers to evaluate the information retrieval baseline model on the Reddit movie dialog dataset.
+- `Multi-Agent Dialog <https://github.com/facebookresearch/ParlAI/blob/master/parlai/mturk/tasks/multi_agent_dialog/>`__: round-robin chat between two local human agents and two Turkers.
+
+Task 1: Collecting Data
+^^^^^^^^^^^^^^^^^^^^^^^
+
+One of the biggest use cases of Mechanical Turk is to collect natural language data from human Turkers. 
+
+As an example, the `QA Data Collection task <https://github.com/facebookresearch/ParlAI/blob/master/parlai/mturk/tasks/qa_data_collection/>`__ does the following:
+
+1. Pick a random Wikipedia paragraph from SQuAD dataset.
+2. Ask a Turker to provide a question given the paragraph.
+3. Ask the same Turker to provide an answer to their question.
+
+In ``QADataCollectionWorld``, there are two agents: one is the human Turker (``MTurkAgent``), the other is the task agent (``DefaultTeacher`` from SQuAD) that provides the Wikipedia paragraph.
+
+The ``QADataCollectionWorld`` uses ``turn_index`` to denote what stage the conversation is at. One *turn* means that the world has been called ``parley()`` once.
+
+After two turns, the task is finished, and the Turker's work is submitted for your review.
+
+
+Task 2: Evaluating a Dialog Model
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can easily evaluate your dialog model's performance with human Turkers using ParlAI. As an example, the `Model Evaluator task <https://github.com/facebookresearch/ParlAI/blob/master/parlai/mturk/tasks/model_evaluator/>`__ does the following:
+
+1. Initialize a task world with a dialog model agent (`ir_baseline <https://github.com/facebookresearch/ParlAI/blob/master/parlai/agents/ir_baseline/agents.py#L111>`__) and a dataset (`MovieDD-Reddit <https://github.com/facebookresearch/ParlAI/blob/master/parlai/tasks/moviedialog/agents.py#L57>`__).
+2. Let all the agents in the task world ``observe()`` and ``act()`` once, by calling ``parley()`` on the world.
+3. Ask the human Turker to rate the dialog model agent's response on a scale of 0-10.
+
+In ``ModelEvaluatorWorld``, there are two main components: one is the ``task_world`` that contains the task and the dialog model we are evaluating, the other is the ``MTurkAgent`` which is an interface to the human Turker.
+
+Note that since the human Turker speaks only once to provide the rating, the ``ModelEvaluatorWorld`` doesn't need to use ``turn_index`` to keep track of the turns. 
+
+After one turn, the task is finished, and the Turker's work is submitted for your review.
+
+
+Creating Your Own Task
+----------------------
+
+ParlAI provides a generic MTurk dialog interface that one can use to implement any kind of dialog tasks. To create your own task, start with reading the tutorials on the provided examples, and then copy and modify the example ``worlds.py``, ``run.py`` and ``task_config.py`` files to create your task. 
+
+A few things to keep in mind:
+
+1. To end a conversation, you should send a message with ``episode_done = True`` from the first non-MTurk agent, and the conversation is ended after all MTurk agents respond.
+2. Make sure to test your dialog task using MTurk's sandbox mode before pushing it live, by using the ``--sandbox`` flag (enabled by default) when running ``run.py``.
+
+
+Running a Task
+--------------
+
+If you have not used Mechanical Turk before, you will need an MTurk Requester Account and an AWS account (these are two separate accounts). Follow the steps below:
+
+- Sign up for an AWS account at `aws.amazon.com <https://aws.amazon.com/>`__
+
+- Sign up for an MTurk account at `requester.mturk.com <https://requester.mturk.com/>`__
+
+- Go to the developer tab (`https://requester.mturk.com/developer <https://requester.mturk.com/developer>`__) and link your AWS account to your MTurk account (Step 2 on that screen)
+
+- MTurk also has a “Sandbox” which is a test version of the MTurk marketplace. You can use it to test publishing and completing tasks without paying any money. ParlAI supports the Sandbox. To use the Sandbox, you need to sign up for a `Sandbox account <http://requestersandbox.mturk.com/>`__. You will then also need to `link your AWS account <http://requestersandbox.mturk.com/developer>`__ to your Sandbox account. In order to test faster, you will also want to create a `Sandbox Worker account <http://workersandbox.mturk.com/>`__. You can then view tasks your publish from ParlAI and complete them yourself.
+
+- ParlAI will connect to your AWS account and set up some supporting resources including a Lambda function, an API Gateway and an RDS database. It will also use your AWS account to connect to the MTurk API. In order to do this, it will require credentials to access your AWS account. To set this up, you will need to create an `IAM user <https://console.aws.amazon.com/iam/>`__ with programmatic access and an AdministratorAccess policy. You can learn more about how to set up IAM users `here <http://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html>`__. Once you have created the account, keep its access key and the secret key handy as you will need it next.
+
+Then, to run an MTurk task, first ensure that the task directory is in `parlai/mturk/tasks/ <https://github.com/facebookresearch/ParlAI/blob/master/parlai/mturk/tasks/>`__. Then, run its ``run.py`` file with proper flags:
+
+.. code-block:: python
+
+    python run.py -nh <num_hits> -r <reward> [--sandbox]/[--live]
+
+E.g. to create 2 HITs for the `QA Data Collection <https://github.com/facebookresearch/ParlAI/blob/master/parlai/mturk/tasks/qa_data_collection/>`__ example with $0.05 each in sandbox mode, first go into the task directory and then run:
+
+.. code-block:: python
+
+    python run.py -nh 2 -r 0.05 --sandbox
+
+Please make sure to test your task in MTurk sandbox mode first (``--sandbox``) before pushing it live (``--live``).
+
+
+Reviewing Turker's Work
+-----------------------
+
+After all HITs are completed, you will be provided a webpage link to review them. 
+
+If you don't take any action in 4 weeks, all HITs will be auto-approved and Turkers will be paid.
+
+
+-------
+
+\* Turker icon credit: `Amazon Mechanical Turk <https://requester.mturk.com/>`__. Robot icon credit: `Icons8 <https://icons8.com/>`__.
\ No newline at end of file

From 8e8fa0cbdf71b03ad0b350cbe893661c3413cb90 Mon Sep 17 00:00:00 2001
From: Will Feng <willfeng@fb.com>
Date: Thu, 8 Jun 2017 23:32:16 -0400
Subject: [PATCH 6/6] updated wordings in README

---
 README.md | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 40d6d73bfe4..afc0ae6eb74 100644
--- a/README.md
+++ b/README.md
@@ -263,14 +263,16 @@ To add your own task:
 ### MTurk
 
 An important part of ParlAI is seamless integration with Mechanical Turk for data collection, training and evaluation.
+
 Human Turkers are also viewed as agents in ParlAI and hence person-person, person-bot, or multiple people and bots in group chat can all converse within the standard framework, switching out the roles as desired with no code changes to the agents. This is because Turkers also receive and send via a (pretty printed) version of the same interface, using the fields of the observation/action dict.
-We currently provide three examples, which are collecting data, human evaluation of a bot, and round-robin chat between local humans and remote Turkers.
+
+We currently provide three examples: collecting data, human evaluation of a bot, and round-robin chat between local humans and remote Turkers.
 
 <p align=center><img width="100%" src="docs/source/\_static/img/mturk.png" /></p>
 
-The mturk library contains the following directories and files:
+The mturk library contains the following directories:
 
-- **core**: this directory contains the core code for setting up AWS backend that supports the MTurk chat interface, and code for HIT creation and approval.
+- **core**: this directory contains the core code for setting up AWS backend that supports the MTurk chat interface, code for HIT creation and approval, and the wrapper class `MTurkAgent` which encapsulates the MTurk interface into a standard `Agent` class.
 - **tasks**: this directory contains three sample MTurk tasks.
   - **_qa\_data\_collection_**: get questions and answers from turkers, given a random paragraph from SQuAD.
   - **_model\_evaluator_**: ask turkers to evaluate the information retrieval baseline model on the Reddit movie dialog dataset.
@@ -282,7 +284,7 @@ To run an MTurk task:
 
 To add your own MTurk task:
 - create a new folder within the mturk/tasks directory for your new task
-- implement __task\_config.py__, with at least the following fields in the task_config dictionary:
+- implement __task\_config.py__, with at least the following fields in the `task_config` dictionary:
   - `hit_title`: a short and descriptive title about the kind of task the HIT contains. On the Amazon Mechanical Turk web site, the HIT title appears in search results, and everywhere the HIT is mentioned.
   - `hit_description`: a description includes detailed information about the kind of task the HIT contains. On the Amazon Mechanical Turk web site, the HIT description appears in the expanded view of search results, and in the HIT and assignment screens.
   - `hit_keywords`: one or more words or phrases that describe the HIT, separated by commas. On MTurk website, these words are used in searches to find HITs.
@@ -290,7 +292,7 @@ To add your own MTurk task:
 - implement __run.py__, with code for setting up and running the world where `MTurkAgent` lives in.
 - (Optional) implement __worlds.py__, with a world class that extends from `World`.
 
-Please see [the tutorial](http://parl.ai/static/docs/mturk.html) to learn more about the MTurk examples and how to create and run your own task.
+Please see [the MTurk tutorial](http://parl.ai/static/docs/mturk.html) to learn more about the MTurk examples and how to create and run your own task.
 
 ## Support
 If you have any questions, bug reports or feature requests, please don't hesitate to post on our [Github Issues page](https://github.com/facebookresearch/ParlAI/issues).