Skip to content

Commit 2aaa5d0

Browse files
burnerleeburnerlee
authored
Fix #305: Allow users to add custom submission metadata attributes from CLI(#315)
* Allow users to add submission metadata from cli * change error message on wrong answer format * fix flake issues * fix code formatting and variable names * fix minor code formatting issue * add feature to leave non-required fields empty * Change message format to ask for user inputs * Integrate attribute submission in setup test * fix flake issues * fix camel case to snake case
1 parent 921da6f commit 2aaa5d0

File tree

6 files changed

+353
-45
lines changed

6 files changed

+353
-45
lines changed

‎evalai/challenges.py

+150-11
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,14 @@
22
import json
33

44
from click import style
5+
from click.utils import echo
56

67
from evalai.utils.auth import get_host_url
7-
from evalai.utils.common import Date, notify_user, upload_file_using_presigned_url
8+
from evalai.utils.common import (
9+
Date,
10+
notify_user,
11+
upload_file_using_presigned_url,
12+
)
813
from evalai.utils.challenges import (
914
display_all_challenge_list,
1015
display_future_challenge_list,
@@ -17,7 +22,10 @@
1722
display_challenge_phase_split_list,
1823
display_leaderboard,
1924
)
20-
from evalai.utils.submissions import display_my_submission_details
25+
from evalai.utils.submissions import (
26+
display_my_submission_details,
27+
get_submission_meta_attributes,
28+
)
2129
from evalai.utils.teams import participate_in_a_challenge
2230
from evalai.utils.submissions import make_submission
2331
from evalai.utils.urls import URLS
@@ -205,11 +213,19 @@ def participate(ctx, team):
205213
"""
206214
Invoked by running `evalai challenge CHALLENGE participate TEAM`
207215
"""
208-
terms_and_conditions_page_url = "{}{}".format(get_host_url(), URLS.terms_and_conditions_page.value)
209-
terms_and_conditions_page_url = terms_and_conditions_page_url.format(ctx.challenge_id)
210-
message = "Please refer challenge terms and conditions here: {}" \
211-
"\n\nBy agreeing to participate in the challenge, you are agreeing to terms and conditions." \
212-
"\n\nDo you accept challenge terms and conditions?".format(terms_and_conditions_page_url)
216+
terms_and_conditions_page_url = "{}{}".format(
217+
get_host_url(), URLS.terms_and_conditions_page.value
218+
)
219+
terms_and_conditions_page_url = terms_and_conditions_page_url.format(
220+
ctx.challenge_id
221+
)
222+
message = (
223+
"Please refer challenge terms and conditions here: {}"
224+
"\n\nBy agreeing to participate in the challenge, you are agreeing to terms and conditions."
225+
"\n\nDo you accept challenge terms and conditions?".format(
226+
terms_and_conditions_page_url
227+
)
228+
)
213229
if click.confirm(message):
214230
participate_in_a_challenge(ctx.challenge_id, team)
215231
else:
@@ -224,7 +240,10 @@ def participate(ctx, team):
224240
@click.option("--public", is_flag=True)
225241
@click.option("--private", is_flag=True)
226242
@click.option(
227-
"--file", type=click.File("rb"), required=True, help="File path to the submission or annotation file"
243+
"--file",
244+
type=click.File("rb"),
245+
required=True,
246+
help="File path to the submission or annotation file",
228247
)
229248
def submit(ctx, file, annotation, large, public, private):
230249
"""
@@ -264,18 +283,138 @@ def submit(ctx, file, annotation, large, public, private):
264283
style("Method Name", fg="yellow"), type=str, default=""
265284
)
266285
submission_metadata["method_description"] = click.prompt(
267-
style("Method Description", fg="yellow"), type=str, default=""
286+
style("Method Description", fg="yellow"),
287+
type=str,
288+
default="",
268289
)
269290
submission_metadata["project_url"] = click.prompt(
270291
style("Project URL", fg="yellow"), type=str, default=""
271292
)
272293
submission_metadata["publication_url"] = click.prompt(
273294
style("Publication URL", fg="yellow"), type=str, default=""
274295
)
296+
submission_meta_attributes = get_submission_meta_attributes(
297+
ctx.challenge_id, ctx.phase_id
298+
)
299+
submission_attribute_metadata = []
300+
if (
301+
submission_meta_attributes
302+
and len(submission_meta_attributes) > 0
303+
):
304+
if click.confirm(
305+
"Do you want to include the Submission Metadata?"
306+
):
307+
for attribute in submission_meta_attributes:
308+
attribute_type = attribute["type"]
309+
attribute_name = attribute["name"]
310+
attribute_description = attribute["description"]
311+
attribute_required = attribute.get("required")
312+
if attribute_required:
313+
attribute_name = attribute_name + '*'
314+
value = None
315+
message = "{} ({})".format(
316+
attribute_name, attribute_description
317+
)
318+
if attribute_type == "text":
319+
while True:
320+
value = click.prompt(
321+
style(message, fg="yellow"),
322+
type=str,
323+
default="",
324+
)
325+
if not attribute_required or value != "":
326+
break
327+
echo(
328+
"Error: {} is a required field".format(
329+
attribute["name"]
330+
)
331+
)
332+
if attribute_type == "boolean":
333+
while True:
334+
value = click.prompt(
335+
style(message, fg="yellow"), type=bool, default=""
336+
)
337+
if not attribute_required or value != "":
338+
break
339+
echo(
340+
"Error: {} is a required field".format(
341+
attribute["name"]
342+
)
343+
)
344+
if attribute_type == "radio":
345+
while True:
346+
value = click.prompt(
347+
style(
348+
"{}:\nChoices:{}".format(
349+
message, attribute["options"]
350+
),
351+
fg="yellow",
352+
),
353+
type=click.Choice(attribute["options"]),
354+
default=""
355+
)
356+
if not attribute_required or value != "":
357+
break
358+
echo(
359+
"Error: {} is a required field".format(
360+
attribute["name"]
361+
)
362+
)
363+
if attribute_type == "checkbox":
364+
option_chosen = True
365+
while option_chosen:
366+
value = []
367+
choices = click.prompt(
368+
style(
369+
"{}:\nChoices(separated by comma):{}".format(
370+
message, attribute["options"]
371+
),
372+
fg="yellow",
373+
),
374+
type=str,
375+
show_default=False,
376+
default=""
377+
)
378+
if choices != "":
379+
choices = [
380+
choice.strip(" ")
381+
for choice in choices.split(",")
382+
]
383+
else:
384+
choices = []
385+
option_chosen = False
386+
if attribute_required and len(choices) == 0:
387+
echo(
388+
"Error: {} is a required field. Please select atleast one option".format(
389+
attribute["name"]
390+
)
391+
)
392+
option_chosen = True
393+
for choice in choices:
394+
if choice in attribute["options"]:
395+
value.append(choice)
396+
option_chosen = False
397+
else:
398+
echo(
399+
"Error: Choose correct value(s) from the given options only"
400+
)
401+
option_chosen = True
402+
break
403+
submission_attribute_metadata.append(
404+
{attribute_name: value}
405+
)
275406
if large:
276-
upload_file_using_presigned_url(ctx.phase_id, file, "submission", submission_metadata)
407+
upload_file_using_presigned_url(
408+
ctx.phase_id, file, "submission", submission_metadata
409+
)
277410
else:
278-
make_submission(ctx.challenge_id, ctx.phase_id, file, submission_metadata)
411+
make_submission(
412+
ctx.challenge_id,
413+
ctx.phase_id,
414+
file,
415+
submission_metadata,
416+
submission_attribute_metadata,
417+
)
279418

280419

281420
challenge.add_command(phase)

‎evalai/utils/submissions.py

+63-8
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import json
12
import requests
23
import sys
34

@@ -18,18 +19,72 @@
1819
requests.packages.urllib3.disable_warnings()
1920

2021

21-
def make_submission(challenge_id, phase_id, file, submission_metadata={}):
22+
def get_submission_meta_attributes(challenge_id, phase_id):
23+
"""
24+
Function to get all submission_meta_attributes for a challenge phase
25+
26+
Parameters:
27+
challenge_id (int): id of challenge to which submission is made
28+
phase_id (int): id of challenge phase to which submission is made
29+
30+
Returns:
31+
list: list of objects of submission meta attributes
32+
"""
33+
url = "{}{}".format(get_host_url(), URLS.challenge_phase_detail.value)
34+
url = url.format(challenge_id, phase_id)
35+
headers = get_request_header()
36+
try:
37+
response = requests.get(url, headers=headers)
38+
response.raise_for_status()
39+
except requests.exceptions.HTTPError as err:
40+
if response.status_code in EVALAI_ERROR_CODES:
41+
validate_token(response.json())
42+
echo(
43+
style(
44+
"\nError: {}\n"
45+
"\nUse `evalai challenges` to fetch the active challenges.\n"
46+
"\nUse `evalai challenge CHALLENGE phases` to fetch the "
47+
"active phases.\n".format(response.json()),
48+
fg="red",
49+
bold=True,
50+
)
51+
)
52+
else:
53+
echo(err)
54+
sys.exit(1)
55+
except requests.exceptions.RequestException:
56+
echo(
57+
style(
58+
"\nCould not establish a connection to EvalAI."
59+
" Please check the Host URL.\n",
60+
bold=True,
61+
fg="red",
62+
)
63+
)
64+
sys.exit(1)
65+
response = response.json()
66+
return response["submission_meta_attributes"]
67+
68+
69+
def make_submission(
70+
challenge_id,
71+
phase_id,
72+
file,
73+
submission_metadata={},
74+
submission_attribute_metadata={},
75+
):
2276
"""
2377
Function to submit a file to a challenge
2478
"""
2579
url = "{}{}".format(get_host_url(), URLS.make_submission.value)
2680
url = url.format(challenge_id, phase_id)
27-
2881
headers = get_request_header()
2982
input_file = {"input_file": file}
30-
data = {"status": "submitting"}
83+
data = {
84+
"status": "submitting",
85+
"submission_metadata": json.dumps(submission_attribute_metadata),
86+
}
3187
data = dict(data, **submission_metadata)
32-
3388
try:
3489
response = requests.post(
3590
url, headers=headers, files=input_file, data=data
@@ -80,7 +135,7 @@ def make_submission(challenge_id, phase_id, file, submission_metadata={}):
80135
response["id"]
81136
),
82137
bold=True,
83-
fg="white"
138+
fg="white",
84139
)
85140
)
86141

@@ -105,7 +160,7 @@ def pretty_print_my_submissions_data(submissions, start_date, end_date):
105160
style(
106161
"\nSorry, you have not made any submissions to this challenge phase.\n",
107162
bold=True,
108-
fg="red"
163+
fg="red",
109164
)
110165
)
111166
sys.exit(1)
@@ -135,7 +190,7 @@ def pretty_print_my_submissions_data(submissions, start_date, end_date):
135190
style(
136191
"\nSorry, no submissions were made during this time period.\n",
137192
bold=True,
138-
fg="red"
193+
fg="red",
139194
)
140195
)
141196
sys.exit(1)
@@ -271,7 +326,7 @@ def display_submission_result(submission_id):
271326
"""
272327
try:
273328
response = submission_details_request(submission_id).json()
274-
echo(requests.get(response['submission_result_file']).text)
329+
echo(requests.get(response["submission_result_file"]).text)
275330
except requests.exceptions.MissingSchema:
276331
echo(
277332
style(

‎test.json

Whitespace-only changes.

0 commit comments

Comments
 (0)