React frontend (facebookresearch#1281)

* Loading react code in the worker frontend

* first working implementation for react frontend

* feature parity

* Abstracting socket handler out from the main app

* Fixing mturk submit, infra for custom views

* Add demo task for react server

* Simplify package logic

* bugfixes pre PR

* needing package for legacy

* Manifest and copyrights

* addressing comments

* fixing test cases to refer to legacy mock

Author: JackUrb

MANIFEST.in

@@ -1,3 +1,4 @@
-include parlai/mturk/core/server/html/*
-include parlai/mturk/core/server/server.js
-include parlai/mturk/core/server/package.json
+include parlai/mturk/core/server_legacy/html/*
+include parlai/mturk/core/server_legacy/server.js
+include parlai/mturk/core/server_legacy/package.json
+include parlai/mturk/core/react_server/*

parlai/mturk/core/agents.py

@@ -648,3 +648,15 @@ def shutdown(self, timeout=None, direct_submit=False):
                 self.db_logger.log_submit_assignment(
                     self.worker_id, self.assignment_id)
             return did_complete
+
+    def update_agent_id(self, agent_id):
+        """Workaround used to force an update to an agent_id on the front-end
+        to render the correct react components for onboarding and waiting
+        worlds. Only really used in special circumstances where different
+        agents need different onboarding worlds.
+        """
+        self.mturk_manager.worker_manager.change_agent_conversation(
+            agent=self,
+            conversation_id=self.conversation_id,
+            new_agent_id=agent_id,
+        )

parlai/mturk/core/mturk_data_handler.py

@@ -545,7 +545,7 @@ def log_worker_note(self, worker_id, assignment_id, note):
             except Exception as e:
                 print(repr(e))
 
-    def get_all_worker_data(self, start=0, count=30):
+    def get_all_worker_data(self, start=0, count=100):
         """get all the worker data for all worker_ids."""
         with self.table_access_condition:
             conn = self._get_connection()
@@ -600,7 +600,7 @@ def get_worker_assignment_pairing(self, worker_id, assignment_id):
             results = c.fetchone()
             return results
 
-    def get_all_run_data(self, start=0, count=30):
+    def get_all_run_data(self, start=0, count=100):
         """get all the run data for all task_group_ids."""
         with self.table_access_condition:
             conn = self._get_connection()

parlai/mturk/core/mturk_manager.py

@@ -764,6 +764,68 @@ def is_task_world(self, conversation_id):
 
     # Manager Lifecycle Functions #
 
+    def populate_legacy_task_files(self, task_directory_path):
+        # Poplulate files to copy over to the server
+        if not self.task_files_to_copy:
+            self.task_files_to_copy = []
+        if not task_directory_path:
+            task_directory_path = os.path.join(
+                self.opt['parlai_home'],
+                'parlai',
+                'mturk',
+                'tasks',
+                self.opt['task']
+            )
+        self.task_files_to_copy.append(
+            os.path.join(task_directory_path, 'html', 'cover_page.html'))
+        try:
+            for file_name in os.listdir(os.path.join(
+                    task_directory_path, 'html')):
+                self.task_files_to_copy.append(os.path.join(
+                    task_directory_path, 'html', file_name
+                ))
+        except FileNotFoundError:  # noqa F821 we don't support python2
+            # No html dir exists
+            pass
+        for mturk_agent_id in self.mturk_agent_ids + ['onboarding']:
+            self.task_files_to_copy.append(os.path.join(
+                task_directory_path,
+                'html',
+                '{}_index.html'.format(mturk_agent_id)
+            ))
+
+    def populate_task_files(self, task_directory_path):
+        # Poplulate files to copy over to the server
+        if not self.task_files_to_copy:
+            self.task_files_to_copy = {
+                'static': [],
+                'components': [],
+                'css': [],
+            }
+        if not task_directory_path:
+            task_directory_path = os.path.join(
+                self.opt['parlai_home'],
+                'parlai',
+                'mturk',
+                'tasks',
+                self.opt['task']
+            )
+        self.task_files_to_copy['static'].append(os.path.join(
+            task_directory_path, 'frontend', 'static', 'cover_page.html'))
+        try:
+            frontend_contents = os.listdir(
+                os.path.join(task_directory_path, 'frontend'))
+            for dir in frontend_contents:
+                if dir in self.task_files_to_copy:
+                    for file_name in os.listdir(os.path.join(
+                            task_directory_path, 'frontend', dir)):
+                        self.task_files_to_copy[dir].append(os.path.join(
+                            task_directory_path, 'frontend', dir, file_name
+                        ))
+        except FileNotFoundError:  # noqa F821 we don't support python2
+            # No frontend dir exists
+            pass
+
     def setup_server(self, task_directory_path=None):
         """Prepare the MTurk server for the new HIT we would like to submit"""
         assert self.task_state >= self.STATE_CREATED
@@ -884,34 +946,6 @@ def setup_server(self, task_directory_path=None):
             unique_worker=self.is_unique,
             is_sandbox=self.opt['is_sandbox']
         )
-        # Poplulate files to copy over to the server
-        if not self.task_files_to_copy:
-            self.task_files_to_copy = []
-        if not task_directory_path:
-            task_directory_path = os.path.join(
-                self.opt['parlai_home'],
-                'parlai',
-                'mturk',
-                'tasks',
-                self.opt['task']
-            )
-        self.task_files_to_copy.append(
-            os.path.join(task_directory_path, 'html', 'cover_page.html'))
-        try:
-            for file_name in os.listdir(os.path.join(
-                    task_directory_path, 'html')):
-                self.task_files_to_copy.append(os.path.join(
-                    task_directory_path, 'html', file_name
-                ))
-        except FileNotFoundError:  # noqa F821 we don't support python2
-            # No html dir exists
-            pass
-        for mturk_agent_id in self.mturk_agent_ids + ['onboarding']:
-            self.task_files_to_copy.append(os.path.join(
-                task_directory_path,
-                'html',
-                '{}_index.html'.format(mturk_agent_id)
-            ))
 
         # Setup the server with a likely-unique app-name
         task_name = '{}-{}'.format(str(uuid.uuid4())[:8], self.opt['task'])
@@ -921,11 +955,18 @@ def setup_server(self, task_directory_path=None):
             heroku_team = self.opt['heroku_team']
         else:
             heroku_team = None
-        self.server_url = server_utils.setup_server(self.server_task_name,
-                                                    self.task_files_to_copy,
-                                                    self.opt['local'],
-                                                    heroku_team,
-                                                    self.opt['hobby'])
+
+        if self.opt.get('frontend_version', 0) < 1:
+            self.populate_legacy_task_files(task_directory_path)
+            self.server_url = server_utils.setup_legacy_server(
+                self.server_task_name, self.task_files_to_copy,
+                self.opt['local'], heroku_team, self.opt['hobby'])
+        else:
+            self.populate_task_files(task_directory_path)
+            self.server_url = server_utils.setup_server(
+                self.server_task_name, self.task_files_to_copy,
+                self.opt['local'], heroku_team, self.opt['hobby'])
+
         shared_utils.print_and_log(logging.INFO, self.server_url)
 
         shared_utils.print_and_log(logging.INFO, "MTurk server setup done.\n",

parlai/mturk/core/react_server/.babelrc

@@ -0,0 +1,4 @@
+{
+  "presets": ["@babel/env", "@babel/preset-react"],
+  "plugins": ["@babel/plugin-proposal-class-properties"]
+}