diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b2d584f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +expdj/experiment_repo +static/ diff --git a/.gitignore b/.gitignore index 1286136..2e98fa6 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ compose/nginx/ssl* *.cred /expdj/experiment_repo/* +postgres-data/ diff --git a/Dockerfile b/Dockerfile index 4e6df8f..5ebf043 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,8 @@ RUN apt-get update && apt-get install -y \ libhdf5-dev \ libgeos-dev \ openssl \ - wget + wget\ + git RUN mkdir /code WORKDIR /code diff --git a/expdj/apps/experiments/static/js/jquery_boostrap_modal_steps.js b/expdj/apps/experiments/static/js/jquery_boostrap_modal_steps.js index 28c273f..f5253f6 100644 --- a/expdj/apps/experiments/static/js/jquery_boostrap_modal_steps.js +++ b/expdj/apps/experiments/static/js/jquery_boostrap_modal_steps.js @@ -8,7 +8,7 @@ btnCancelHtml: 'Cancel', btnPreviousHtml: 'Previous', btnNextHtml: 'Next', - btnLastStepHtml: 'Start Experiment', + btnLastStepHtml: 'Next', disableNextButton: false, completeCallback: function(){ $("#start_experiment_button").click(); diff --git a/expdj/apps/experiments/views.py b/expdj/apps/experiments/views.py index 8d12229..3f1c43a 100644 --- a/expdj/apps/experiments/views.py +++ b/expdj/apps/experiments/views.py @@ -322,11 +322,11 @@ def get_battery_intro(battery, show_advertisement=True): if battery.advertisement is not None: instruction_forms.append( {"title": "Advertisement", "html": battery.advertisement}) - if battery.consent is not None: - instruction_forms.append({"title": "Consent", "html": battery.consent}) if battery.instructions is not None: instruction_forms.append( {"title": "Instructions", "html": battery.instructions}) + if battery.consent is not None: + instruction_forms.append({"title": "Consent", "html": battery.consent}) return instruction_forms diff --git a/expdj/apps/turk/api_views.py b/expdj/apps/turk/api_views.py index 5fa7eee..8a9c647 100644 --- a/expdj/apps/turk/api_views.py +++ b/expdj/apps/turk/api_views.py @@ -1,15 +1,48 @@ +from django.core.exceptions import ObjectDoesNotExist +from django.http import Http404 from django.http.response import HttpResponseForbidden -from rest_framework import exceptions, generics, permissions, viewsets +from rest_framework import exceptions, generics, viewsets +from rest_framework.permissions import AllowAny +from rest_framework.views import APIView +from rest_framework.response import Response from expdj.apps.experiments.models import Battery -from expdj.apps.turk.models import Result, Worker +from expdj.apps.turk.models import Assignment, Result, Worker, HIT from expdj.apps.turk.serializers import ResultSerializer +from expdj.apps.turk.tasks import updated_assign_experiment_credit +from expdj.apps.turk.utils import get_worker_experiments, get_connection, get_credentials class BatteryResultAPIList(generics.ListAPIView): serializer_class = ResultSerializer def get_queryset(self): battery_id = self.kwargs.get('bid') - if (Battery.objects.get(pk=battery_id).owner is not self.request.user.pk): + if (Battery.objects.get(pk=battery_id).owner.id is not self.request.user.pk): raise exceptions.PermissionDenied() return Result.objects.filter(battery__id=battery_id) + +class WorkerExperiments(APIView): + permission_classes = (AllowAny,) + def get(self, request, worker_id, hit_id): + try: + hit = HIT.objects.get(mturk_id=hit_id) + worker = Worker.objects.get(id=worker_id) + # assignment = Assignment.objects.get(hit__mturk_id=hit_id) + all_assignments = Assignment.objects.filter(worker_id=worker_id, hit__battery_id=hit.battery_id) + except ObjectDoesNotExist: + raise Http404 + exps = list(get_worker_experiments(worker, hit.battery)) + exps = [x.template.name for x in exps] + submit = False + status = 'Not Submitted' + + marked_complete = all_assignments.filter(completed=True).count() > 0 + + if len(exps) == 0 and not marked_complete: + all_assignments.filter(completed=False).update(completed=True) + submit = True + updated_assign_experiment_credit.apply_async([worker_id, hit.battery_id, hit_id], countdown=60) + elif len(exps) == 0 and marked_complete: + status = 'Submit Attempted' + + return Response({'experiments': exps, 'assignment_status': status, 'submit': submit}) diff --git a/expdj/apps/turk/models.py b/expdj/apps/turk/models.py index a9edf2c..2a86c67 100644 --- a/expdj/apps/turk/models.py +++ b/expdj/apps/turk/models.py @@ -3,13 +3,6 @@ import collections import datetime -import boto -from boto.mturk.price import Price -from boto.mturk.qualification import (AdultRequirement, LocaleRequirement, - NumberHitsApprovedRequirement, - PercentAssignmentsApprovedRequirement, - Qualifications, Requirement) -from boto.mturk.question import ExternalQuestion from django.contrib.auth.models import User from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models @@ -23,7 +16,7 @@ from expdj.apps.turk.utils import (amazon_string_to_datetime, get_connection, get_credentials, get_time_difference, to_dict) -from expdj.settings import BASE_DIR, DOMAIN_NAME +from expdj.settings import DOMAIN_NAME def init_connection_callback(sender, **signal_args): @@ -301,7 +294,7 @@ def disable(self): """ # Check for new results and cache a copy in Django model self.update(do_update_assignments=True) - self.connection.dispose_hit(self.mturk_id) + self.connection.delete_hit(HITId=self.mturk_id) def dispose(self): """Dispose of a HIT that is no longer needed. @@ -339,34 +332,38 @@ def dispose(self): self.mturk_id, assignment.mturk_id)) # Checks pass. Dispose of HIT and update status - self.connection.dispose_hit(self.mturk_id) + self.connection.delete_hit(HITId=self.mturk_id) self.update() def expire(self): """Expire a HIT that is no longer needed as Mechanical Turk service""" if not self.has_connection(): self.generate_connection() - self.connection.expire_hit(self.mturk_id) + # expire hit no longer a function in boto3 + # self.connection.expire_hit(self.mturk_id) + self.connection.update_expiration_for_hit(HITId=self.mturk_id, ExpireAt=datetime.datetime.now()) self.update() + ''' Does not appear to be in use currently def extend(self, assignments_increment=None, expiration_increment=None): - """Increase the maximum assignments or extend the expiration date""" + # Increase the maximum assignments or extend the expiration date if not self.has_connection(): self.generate_connection() - self.connection.extend_hit(self.mturk_id, - assignments_increment=assignments_increment, - expiration_increment=expiration_increment) + self.connection.update_expiration_for_hit( + HITId=self.mturk_id, + ExpireAt=self. self.update() + ''' def set_reviewing(self, revert=None): - """Toggle HIT status between Reviewable and Reviewing""" + """ Toggle HIT status between Reviewable and Reviewing """ if not self.has_connection(): self.generate_connection() - self.connection.set_reviewing(self.mturk_id, revert=revert) + self.connection.update_hit_review_status(HITId=self.mturk_id, Revert=revert) self.update() def generate_connection(self): - # Get the aws access id from the credentials file + """ Get the aws access id from the credentials file """ AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY_ID = get_credentials( battery=self.battery) self.connection = get_connection( @@ -378,76 +375,75 @@ def has_connection(self): return False def send_hit(self): - - # First check for qualifications - qualifications = Qualifications() + """ Collect all the settings and call api to submit hit to aws """ + qualifications = [] if self.qualification_adult: - qualifications.add(AdultRequirement("EqualTo", 1)) - else: - qualifications.add(AdultRequirement("EqualTo", 0)) - if self.qualification_custom not in [None, ""]: - qualifications.add( - Requirement( - self.qualification_custom, - self.qualification_custom_operator, - self.qualification_custom_value, - required_to_preview=True)) + qualifications.append({ + "QualificationTypeId": "00000000000000000060", + "Comparator": "EqualTo", + "IntegerValues": [1], + "ActionsGuarded": "DiscoverPreviewAndAccept" + }) if self.qualification_number_hits_approved is not None: - qual_number_hits = NumberHitsApprovedRequirement( - "GreaterThan", self.qualification_number_hits_approved) - qualifications.add(qual_number_hits) - if self.qualification_percent_assignments_approved is not None: - qual_perc_approved = PercentAssignmentsApprovedRequirement( - "GreaterThan", self.qualification_percent_assignments_approved) - qualifications.add(qual_perc_approved) + qualifications.append({ + "QualificationTypeId": "00000000000000000040", + "Comparator": "GreaterThanOrEqualTo", + "IntegerValues": [self.qualification_number_hits_approve], + "ActionsGuarded": "DiscoverPreviewAndAccept" + }) + if self.qualification_custom not in [None, ""]: + qualifications.append({ + "QualificationTypeId": self.qualification_custom, + "Comparator": self.qualification_custom_operator, + "IntegerValues": [self.qualification_custom_value], + "ActionsGuarded": "Accept" + }) if self.qualification_locale != 'None': - qualifications.add( - LocaleRequirement( - "EqualTo", - self.qualification_locale)) - + qualifications.append({ + "QualificationTypeId": "00000000000000000071", + "Comparator": "EqualTo", + "LocaleValues": [{ + "Country": self.qualification_locale, + }], + "ActionsGuarded": "DiscoverPreviewAndAccept" + }) + if self.qualification_percent_assignments_approved is not None: + qualifications.append({ + "QualificationTypeId": "000000000000000000L0", + "Comparator": "GreaterThanOrEqualTo", + "IntegerValues": [self.qualification_percent_assignments_approved], + "ActionsGuarded": "DiscoverPreviewAndAccept" + }) # Domain name must be https url = "%s/turk/%s" % (DOMAIN_NAME, self.id) frame_height = 900 - questionform = ExternalQuestion(url, frame_height) - - if len(qualifications.requirements) > 0: - result = self.connection.create_hit( - title=self.title, - description=self.description, - keywords=self.keywords, - duration=datetime.timedelta( - self.assignment_duration_in_hours / 24.0), - lifetime=datetime.timedelta( - self.lifetime_in_hours / 24.0), - max_assignments=self.max_assignments, - question=questionform, - qualifications=qualifications, - reward=Price( - amount=self.reward), - response_groups=( - 'Minimal', - 'HITDetail'))[0] + question_xml = ''' + + {} + 0 +'''.format(url) + + settings = { + 'Title': self.title, + 'Description': self.description, + 'Keywords': self.keywords, + 'AssignmentDurationInSeconds': int(self.assignment_duration_in_hours * 60 * 60), + 'LifetimeInSeconds': int(self.lifetime_in_hours * 60 * 60), + 'MaxAssignments': self.max_assignments, + 'Question': question_xml, + 'Reward': str(self.reward) + } + ''' no equivelent in boto3? + 'response_groups': ( + 'Minimal', + 'HITDetail') + ''' + settings['QualificationRequirements'] = qualifications - else: - result = self.connection.create_hit( - title=self.title, - description=self.description, - keywords=self.keywords, - duration=datetime.timedelta( - self.assignment_duration_in_hours / 24.0), - lifetime=datetime.timedelta( - self.lifetime_in_hours / 24.0), - max_assignments=self.max_assignments, - question=questionform, - reward=Price( - amount=self.reward), - response_groups=( - 'Minimal', - 'HITDetail'))[0] + result = self.connection.create_hit(**settings)['HIT'] # Update our hit object with the aws HIT - self.mturk_id = result.HITId + self.mturk_id = result['HITId'] # When we generate the hit, we won't have any assignments to update self.update(mturk_hit=result) @@ -480,27 +476,27 @@ def update(self, mturk_hit=None, do_update_assignments=False): """ self.generate_connection() if mturk_hit is None or not hasattr(mturk_hit, "HITStatus"): - hit = self.connection.get_hit(self.mturk_id)[0] + hit = self.connection.get_hit(HITId=self.mturk_id)['HIT'] else: - assert isinstance(mturk_hit, boto.mturk.connection.HIT) + # assert isinstance(mturk_hit, boto.mturk.connection.HIT) hit = mturk_hit - self.status = HIT.reverse_status_lookup[hit.HITStatus] - self.reward = hit.Amount - self.assignment_duration_in_seconds = hit.AssignmentDurationInSeconds - self.auto_approval_delay_in_seconds = hit.AutoApprovalDelayInSeconds - self.max_assignments = hit.MaxAssignments - self.creation_time = amazon_string_to_datetime(hit.CreationTime) - self.description = hit.Description - self.title = hit.Title - self.hit_type_id = hit.HITTypeId - self.keywords = hit.Keywords - if hasattr(self, 'NumberOfAssignmentsCompleted'): - self.number_of_assignments_completed = hit.NumberOfAssignmentsCompleted - if hasattr(self, 'NumberOfAssignmentsAvailable'): - self.number_of_assignments_available = hit.NumberOfAssignmentsAvailable - if hasattr(self, 'NumberOfAssignmentsPending'): - self.number_of_assignments_pending = hit.NumberOfAssignmentsPending + self.status = HIT.reverse_status_lookup[hit['HITStatus']] + self.reward = hit['Reward'] + self.assignment_duration_in_seconds = hit['AssignmentDurationInSeconds'] + self.auto_approval_delay_in_seconds = hit['AutoApprovalDelayInSeconds'] + self.max_assignments = hit['MaxAssignments'] + self.creation_time = hit['CreationTime'] + self.description = hit['Description'] + self.title = hit['Title'] + self.hit_type_id = hit['HITTypeId'] + self.keywords = hit['Keywords'] + if 'NumberOfAssignmentsCompleted' in hit.keys(): + self.number_of_assignments_completed = hit['NumberOfAssignmentsCompleted'] + if 'NumberOfAssignmentsAvailable' in hit.keys(): + self.number_of_assignments_available = hit['NumberOfAssignmentsAvailable'] + if 'NumberOfAssignmentsPending' in hit.keys(): + self.number_of_assignments_pending = hit['NumberOfAssignmentsPending'] # 'CurrencyCode', 'Reward', 'Expiration', 'expired'] self.save() @@ -510,13 +506,15 @@ def update(self, mturk_hit=None, do_update_assignments=False): def update_assignments(self, page_number=1, page_size=10, update_all=True): self.generate_connection() - assignments = self.connection.get_assignments(self.mturk_id, - page_size=page_size, - page_number=page_number) + assignments = self.connection.list_assignments_for_hit( + HITId=self.mturk_id, + MaxResults=page_size, + NextToken=page_number + )['Assignments'] for mturk_assignment in assignments: - assert mturk_assignment.HITId == self.mturk_id + assert mturk_assignment['HITId'] == self.mturk_id djurk_assignment = Assignment.objects.get_or_create( - mturk_id=mturk_assignment.AssignmentId, hit=self)[0] + mturk_id=mturk_assignment['AssignmentId'], hit=self)[0] djurk_assignment.update(mturk_assignment, hit=self) if update_all and int(assignments.PageNumber) *\ page_size < int(assignments.TotalNumResults): @@ -564,13 +562,17 @@ class Assignment(models.Model): blank=True, help_text="The status of the assignment") auto_approval_time = models.DateTimeField(null=True, blank=True, help_text=( - "If results have been submitted, this is the date and time, in UTC, the results of the assignment are considered approved automatically if they have not already been explicitly approved or rejected by the requester")) + "If results have been submitted, this is the date and time, in UTC, "\ + "the results of the assignment are considered approved automatically"\ + "if they have not already been explicitly approved or rejected by the requester")) accept_time = models.DateTimeField(null=True, blank=True, help_text=( "The date and time, in UTC, the Worker accepted the assignment")) submit_time = models.DateTimeField(null=True, blank=True, help_text=( - "If the Worker has submitted results, this is the date and time, in UTC, the assignment was submitted")) + "If the Worker has submitted results, this is the date and time, in "\ + "UTC, the assignment was submitted")) approval_time = models.DateTimeField(null=True, blank=True, help_text=( - "If requester has approved the results, this is the date and time, in UTC, the results were approved")) + "If requester has approved the results, this is the date and time, in "\ + "UTC, the results were approved")) rejection_time = models.DateTimeField(null=True, blank=True, help_text=( "If requester has rejected the results, this is the date and time, in UTC, the results were rejected")) deadline = models.DateTimeField(null=True, blank=True, help_text=( @@ -592,25 +594,29 @@ def create(self): def approve(self, feedback=None): """Thin wrapper around Boto approve function.""" self.hit.generate_connection() - self.hit.connection.approve_assignment( - self.mturk_id, feedback=feedback) + kargs = {'AssignmentId': self.mturk_id} + # if feedback == None or feedback == '': + # kargs['RequesterFeedback'] = feedback + + self.hit.connection.approve_assignment(**kargs) self.update() def reject(self, feedback=None): """Thin wrapper around Boto reject function.""" self.hit.generate_connection() - self.hit.connection.reject_assignment(self.mturk_id, feedback=feedback) + self.hit.connection.reject_assignment(AssignmentId=self.mturk_id, RequesterFeedback=feedback) self.update() def bonus(self, value=0.0, feedback=None): """Thin wrapper around Boto bonus function.""" self.hit.generate_connection() - self.hit.connection.grant_bonus( - self.worker_id, - self.mturk_id, - bonus_price=boto.mturk.price.Price(amount=value), - reason=feedback) + self.hit.connection.send_bonus( + WorkerId='self.worker_id', + AssignmentId='self.mturk_id', + BonusAmount=str(value), + Reason=feedback + ) self.update() def update(self, mturk_assignment=None, hit=None): @@ -627,42 +633,43 @@ def update(self, mturk_assignment=None, hit=None): assignment = None if mturk_assignment is None: - hit = self.hit.connection.get_hit(self.hit.mturk_id)[0] - for a in self.hit.connection.get_assignments(hit.HITId): + hit = self.hit.connection.get_hit(HITId=self.hit.mturk_id)['HIT'] + for a in self.hit.connection.list_assignments_for_hit(HITId=hit['HITId'])['Assignments']: # While we have the query, we may as well update - if a.AssignmentId == self.mturk_id: + if a['AssignmentId'] == self.mturk_id: # That's this record. Hold onto so we can update below assignment = a else: other_assignments = Assignment.objects.filter( - mturk_id=a.AssignmentId) + mturk_id=a['AssignmentId']) # Amazon can reuse Assignment ids, so there is an # occasional duplicate for other_assignment in other_assignments: - if other_assignment.worker_id == a.WorkerId: + if other_assignment.worker_id == a['WorkerId']: other_assignment.update(a) else: + ''' assert isinstance( mturk_assignment, boto.mturk.connection.Assignment) + ''' assignment = mturk_assignment if assignment is not None: - self.status = self.reverse_status_lookup[assignment.AssignmentStatus] - self.worker_id = get_worker(assignment.WorkerId) - self.submit_time = amazon_string_to_datetime(assignment.SubmitTime) - self.accept_time = amazon_string_to_datetime(assignment.AcceptTime) - self.auto_approval_time = amazon_string_to_datetime( - assignment.AutoApprovalTime) - self.submit_time = amazon_string_to_datetime(assignment.SubmitTime) + self.status = self.reverse_status_lookup[assignment['AssignmentStatus']] + self.worker = get_worker(assignment['WorkerId']) + self.submit_time = assignment['SubmitTime'] + self.accept_time = assignment['AcceptTime'] + self.auto_approval_time = assignment['AutoApprovalTime'] + self.submit_time = assignment['SubmitTime'] # Different response groups for query - if hasattr(assignment, 'RejectionTime'): - self.rejection_time = amazon_string_to_datetime( - assignment.RejectionTime) - if hasattr(assignment, 'ApprovalTime'): - self.approval_time = amazon_string_to_datetime( - assignment.ApprovalTime) + # if hasattr(assignment, 'RejectionTime'): + if 'RejectionTime' in assignment.keys(): + self.rejection_time = assignment['RejectionTime'] + # if hasattr(assignment, 'ApprovalTime'): + if 'ApprovalTime' in assignment.keys(): + self.approval_time = assignment['ApprovalTime'] self.save() @@ -675,7 +682,9 @@ def __repr__(self): class Result(models.Model): - '''A result holds a battery id and an experiment template, to keep track of the battery/experiment combinations that a worker has completed''' + '''A result holds a battery id and an experiment template, to keep track + of the battery/experiment combinations that a worker has completed + ''' taskdata = JSONField( null=True, blank=True, load_kwargs={ 'object_pairs_hook': collections.OrderedDict}) @@ -741,7 +750,8 @@ class Result(models.Model): (True, 'Granted')), default=False, - verbose_name="the function assign_experiment_credit has been run to allocate credit for this result") + verbose_name="the function assign_experiment_credit has been run to " \ + "allocate credit for this result") class Meta: verbose_name = "Result" @@ -779,7 +789,13 @@ class Bonus(models.Model): help_text="dictionary of experiments with bonus amounts", load_kwargs={ 'object_pairs_hook': collections.OrderedDict}) - # {u'test_task': {'description': u'performance_var True EQUALS True', 'experiment_id': 113, 'amount': 3.0} # amount in dollars/cents + ''' + {u'test_task': { + 'description': u'performance_var True EQUALS True', + 'experiment_id': 113, + 'amount': 3.0 + } # amount in dollars/cents + ''' granted = models.BooleanField( choices=( (False, @@ -794,10 +810,11 @@ def __unicode__(self): return "<%s_%s>" % (self.battery, self.worker) def calculate_bonus(self): + ''' calculate bonus regardless of experiment_id key ''' if self.amounts is not None: amounts = dict(self.amounts) total = 0 - for experiment_id, record in amounts.iteritems(): + for _, record in amounts.iteritems(): if "amount" in record: total = total + record["amount"] return total diff --git a/expdj/apps/turk/tasks.py b/expdj/apps/turk/tasks.py index 9a6846e..9ed386e 100644 --- a/expdj/apps/turk/tasks.py +++ b/expdj/apps/turk/tasks.py @@ -3,7 +3,6 @@ import os import numpy -from boto.mturk.price import Price from celery import Celery, shared_task from django.conf import settings from django.utils import timezone @@ -15,6 +14,7 @@ from expdj.apps.experiments.utils import get_experiment_type from expdj.apps.turk.models import (HIT, Assignment, Blacklist, Bonus, Result, get_worker) +from expdj.apps.turk.utils import get_worker_experiments, get_connection, get_credentials from expdj.settings import TURK os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'expdj.settings') @@ -59,6 +59,47 @@ def assign_experiment_credit(worker_id): result.assignment.save() grant_bonus(result.id) +@shared_task +def updated_assign_experiment_credit(worker_id, battery_id, hit_id): + ''' We want to approve an asignment for a given worker/battery if: + 1) No assignment is marked as complete + 2) And no assignment at the AWS API is marked as approved + 3) And there is at least one assignment at the AWS API that is approved + ''' + battery = Battery.objects.get(id=battery_id) + hit = HIT.objects.get(mturk_id=hit_id) + all_assignments = Assignment.objects.filter(worker_id=worker_id, hit__battery_id=battery_id) + AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY_ID = get_credentials(battery) + conn = get_connection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY_ID, hit) + submitted = [] + approved = False + for assignment in all_assignments: + try: + aws_assignment = conn.get_assignment(AssignmentId=assignment.mturk_id)['Assignment'] + if aws_assignment['AssignmentStatus'] == 'Submitted': + submitted.append(assignment) + if aws_assignment['AssignmentStatus'] == 'Approved': + approved = True + continue + except Exception as e: + print(e) + continue + + if len(submitted) == 0: + return + + if not approved: + submitted[0].approve() + submitted[0].completed = True + submitted[0].save() + submitted = submitted[1:] + + for x in submitted: + print("attemtpting to reject {}".format(x.mturk_id)) + response = conn.reject_assignment( + AssignmentId=x.mturk_id, + RequesterFeedback='Already attempted to approve another HIT for the same set of experiments' + ) @shared_task def check_blacklist(result_id): @@ -173,10 +214,9 @@ def grant_bonus(result_id): try: bonus = Bonus.objects.get(worker=worker, battery=battery) amount = bonus.calculate_bonus() - price = Price(amount) reason = get_bonus_reason(bonus) result.assignment.hit.connection.grant_bonus( - worker.id, result.assignment.mturk_id, price, reason) + worker.id, result.assignment.mturk_id, amount, reason) bonus.granted = True bonus.save() except Bonus.DoesNotExist: diff --git a/expdj/apps/turk/templates/experiments/mturk_battery.html b/expdj/apps/turk/templates/experiments/mturk_battery.html index 7f788dc..7934301 100644 --- a/expdj/apps/turk/templates/experiments/mturk_battery.html +++ b/expdj/apps/turk/templates/experiments/mturk_battery.html @@ -18,7 +18,7 @@ {% include "experiments/serve_battery_runjs.html" %} -{% if amazon_host == "mechanicalturk.amazonaws.com" %} +{% if amazon_host == "mturk-requester.us-east-1.amazonaws.com" %}
{% else %} diff --git a/expdj/apps/turk/templates/games/mturk_battery.html b/expdj/apps/turk/templates/games/mturk_battery.html index bee311b..fcd5a43 100644 --- a/expdj/apps/turk/templates/games/mturk_battery.html +++ b/expdj/apps/turk/templates/games/mturk_battery.html @@ -28,7 +28,7 @@ }); -{% if amazon_host == "mechanicalturk.amazonaws.com" %} +{% if amazon_host == "mturk-requester.us-east-1.amazonaws.com" %} {% else %} diff --git a/expdj/apps/turk/templates/surveys/worker_finished.html b/expdj/apps/turk/templates/surveys/worker_finished.html index 4129330..3f44352 100644 --- a/expdj/apps/turk/templates/surveys/worker_finished.html +++ b/expdj/apps/turk/templates/surveys/worker_finished.html @@ -32,7 +32,7 @@

Thank you for your participation!

-{% if amazon_host == "mechanicalturk.amazonaws.com" %} +{% if amazon_host == "mturk-requester.us-east-1.amazonaws.com" %} {% else %} diff --git a/expdj/apps/turk/templates/turk/serve_battery_intro.html b/expdj/apps/turk/templates/turk/serve_battery_intro.html index 36bfbfa..99aef9c 100644 --- a/expdj/apps/turk/templates/turk/serve_battery_intro.html +++ b/expdj/apps/turk/templates/turk/serve_battery_intro.html @@ -22,13 +22,16 @@ var complete = "1"; $("#next_button").click(function() { - if (complete == "complete"){ - + if (complete == "3" || complete == "complete"){ {% if assignment_id %} window.open("{{ start_url }}") + // window.location.assign("{{ start_url }}") {% else %} window.open("{% url 'not_consent_view' %}",fullscreen=1) + // window.location.assign("{% url 'not_consent_view' %}") {% endif %} + $("#next_button").addClass("hidden") + $("#previous_button").addClass("hidden") } complete = $("#next_button").attr("data-step") }); diff --git a/expdj/apps/turk/templates/turk/status_monitor.html b/expdj/apps/turk/templates/turk/status_monitor.html new file mode 100644 index 0000000..cf3e4ba --- /dev/null +++ b/expdj/apps/turk/templates/turk/status_monitor.html @@ -0,0 +1,36 @@ +
+ Experiments left in battery:

+ Current assignment status:

+ + {% if sandbox %} + + {% else %} + + {% endif %} + + + + + + +
diff --git a/expdj/apps/turk/tests.py b/expdj/apps/turk/tests.py index 85bcc4d..d56bd08 100644 --- a/expdj/apps/turk/tests.py +++ b/expdj/apps/turk/tests.py @@ -8,14 +8,21 @@ import boto import django from django.test import TestCase -from django.test.utils import override_settings +from expdj.apps.turk.utils import (PRODUCTION_HOST, PRODUCTION_WORKER_URL, + SANDBOX_HOST, SANDBOX_WORKER_URL, + amazon_string_to_datetime, + get_connection, get_host, get_worker_url, + is_sandbox) + +''' from cogpheno.apps.turk.utils import (PRODUCTION_HOST, PRODUCTION_WORKER_URL, SANDBOX_HOST, SANDBOX_WORKER_URL, InvalidDjurkSettings, amazon_string_to_datetime, get_connection, get_host, get_worker_url, is_sandbox) +''' """Basic unit tests for Turk App""" diff --git a/expdj/apps/turk/urls.py b/expdj/apps/turk/urls.py index b9c230c..8e2fc66 100644 --- a/expdj/apps/turk/urls.py +++ b/expdj/apps/turk/urls.py @@ -2,7 +2,7 @@ from django.views.generic.base import TemplateView from expdj.apps.experiments.views import sync -from expdj.apps.turk.api_views import BatteryResultAPIList +from expdj.apps.turk.api_views import BatteryResultAPIList, WorkerExperiments from expdj.apps.turk.views import (clone_hit, contact_worker, delete_hit, edit_hit, end_assignment, expire_hit, finished_view, hit_detail, manage_hit, @@ -60,8 +60,15 @@ url(r'^worker/contact/(?P\d+)', contact_worker, name='contact_worker'), # New API - url(r'^new_api/results/(?P\d+)/$', + url( + r'^new_api/results/(?P\d+)/$', BatteryResultAPIList.as_view(), name='battery_result_api_list' - ) + ), + url( + r'^new_api/worker_experiments/(?P[A-Za-z0-9]+)/(?P[A-Za-z0-9]+)/$', + WorkerExperiments.as_view(), + name='worker_experiments' + ), + ] diff --git a/expdj/apps/turk/utils.py b/expdj/apps/turk/utils.py index 28657bb..de3637e 100644 --- a/expdj/apps/turk/utils.py +++ b/expdj/apps/turk/utils.py @@ -4,9 +4,7 @@ import os import pandas -from boto.mturk.connection import MTurkConnection -from boto.mturk.price import Price -from boto.mturk.question import ExternalQuestion +import boto3 from django.conf import settings from expdj.apps.experiments.models import Experiment @@ -21,8 +19,10 @@ def to_dict(input_ordered_dict): return json.loads(json.dumps(input_ordered_dict)) -PRODUCTION_HOST = u'mechanicalturk.amazonaws.com' -SANDBOX_HOST = u'mechanicalturk.sandbox.amazonaws.com' +# PRODUCTION_HOST = u'mechanicalturk.amazonaws.com' +PRODUCTION_HOST= u'mturk-requester.us-east-1.amazonaws.com' +# SANDBOX_HOST = u'mechanicalturk.sandbox.amazonaws.com' +SANDBOX_HOST = u'mturk-requester-sandbox.us-east-1.amazonaws.com' PRODUCTION_WORKER_URL = u'https://www.mturk.com' SANDBOX_WORKER_URL = u'https://workersandbox.mturk.com' @@ -98,13 +98,14 @@ def get_connection(aws_access_key_id, aws_secret_access_key, hit=None): """Create connection based upon settings/configuration parameters""" host = get_host(hit) - debug = get_debug(hit) - - return MTurkConnection( + # debug = get_debug(hit) + return boto3.client( + 'mturk', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, - host=host, - debug=debug) + endpoint_url='https://' + host, + region_name='us-east-1' + ) def get_app_url(): diff --git a/expdj/apps/turk/views.py b/expdj/apps/turk/views.py index cca5631..d9ab741 100644 --- a/expdj/apps/turk/views.py +++ b/expdj/apps/turk/views.py @@ -11,6 +11,7 @@ HttpResponseNotAllowed, HttpResponseRedirect) from django.shortcuts import (get_object_or_404, redirect, render, render_to_response) +from django.template import Context, Template from django.utils import timezone from django.views.decorators.csrf import ensure_csrf_cookie from expfactory.battery import get_experiment_run, get_load_static @@ -29,7 +30,7 @@ get_unique_experiments) from expdj.apps.turk.utils import (get_connection, get_credentials, get_host, get_worker_experiments, get_worker_url) -from expdj.settings import BASE_DIR, MEDIA_ROOT, STATIC_ROOT +from expdj.settings import BASE_DIR, DOMAIN_NAME, MEDIA_ROOT, STATIC_ROOT media_dir = os.path.join(BASE_DIR, MEDIA_ROOT) @@ -70,7 +71,6 @@ def get_amazon_variables(request): "hit_id": hit_id, "turk_submit_to": turk_submit_to} - @login_required def manage_hit(request, bid, hid): '''manage_hit shows details about workers that have completed / not completed a HIT @@ -169,9 +169,11 @@ def serve_hit(request, hid): # worker time runs out to allocate credit if already_created: assignment.accept_time = datetime.now() + ''' we now attempt to assign credit when we tell them its ok to submit if hit.assignment_duration_in_hours is not None: assign_experiment_credit.apply_async( [worker.id], countdown=360 * (hit.assignment_duration_in_hours)) + ''' assignment.save() # Does the worker have experiments remaining for the hit? @@ -204,6 +206,7 @@ def serve_hit(request, hid): # Add variables to the context aws["amazon_host"] = host + aws["sandbox"] = hit.sandbox aws["uniqueId"] = result.id # If this is the last experiment, the finish button will link to a @@ -240,8 +243,14 @@ def preview_hit(request, hid): hit = get_hit(hid, request) battery = hit.battery context = get_amazon_variables(request) - - context["instruction_forms"] = get_battery_intro(battery) + intro = get_battery_intro(battery) + status_template = Template('{% include "turk/status_monitor.html" %}') + context["status_url"] = "{}/new_api/worker_experiments/{}/{}/".format(DOMAIN_NAME, context["worker_id"], hit.mturk_id) + context["sandbox"] = hit.sandbox + status_context = Context(context) + intro.append({'title': 'Assignment Status', 'html': status_template.render(status_context)}) + + context["instruction_forms"] = intro context["hit_uid"] = hid context["start_url"] = "/accept/%s/?assignmentId=%s&workerId=%s&turkSubmitTo=%s&hitId=%s" % ( hid, context["assignment_id"], context["worker_id"], context["turk_submit_to"], context["hit_id"]) diff --git a/requirements.txt b/requirements.txt index 73e6950..b0e9817 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ requests-oauthlib jsonfield expfactory<3 boto3 -boto==2.1.1 +boto django-dbbackup<2.3 cognitiveatlas dropbox==1.6 @@ -34,7 +34,8 @@ Pillow python-openid django-sendfile django-polymorphic -celery[redis] +redis<3 +celery>=3.1.15,<4.0 django-celery django-cleanup django-chosen