Skip to content

Commit 0e88ce0

Browse files
tmorrelltmorrell
committed
Black formatting
1 parent fb8d161 commit 0e88ce0

File tree

5 files changed

+457
-401
lines changed

5 files changed

+457
-401
lines changed

‎caltechdata_api/caltechdata_edit.py

Lines changed: 31 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,23 @@
11
from requests import session
2-
import json,copy
2+
import json, copy
33
from caltechdata_api import customize_schema
44
from caltechdata_api import send_s3
55

6-
def caltechdata_edit(token,ids,metadata={},files={},delete={},production=False):
7-
'''Including files will only replaces files if they have the same name
6+
7+
def caltechdata_edit(token, ids, metadata={}, files={}, delete={}, production=False):
8+
"""Including files will only replaces files if they have the same name
89
The delete option will delete any existing files with a given file extension
9-
There are more file operations that could be implemented'''
10+
There are more file operations that could be implemented"""
1011

11-
#If files is a string - change to single value array
12+
# If files is a string - change to single value array
1213
if isinstance(files, str) == True:
1314
files = [files]
1415
if isinstance(ids, int):
1516
ids = [str(ids)]
1617
if isinstance(ids, str):
1718
ids = [ids]
1819

19-
headers = {
20-
'Authorization' : 'Bearer %s' % token,
21-
'Content-type': 'application/json'
22-
}
20+
headers = {"Authorization": "Bearer %s" % token, "Content-type": "application/json"}
2321

2422
if production == True:
2523
url = "https://data.caltech.edu/submit/api/edit/"
@@ -32,7 +30,7 @@ def caltechdata_edit(token,ids,metadata={},files={},delete={},production=False):
3230
metadata = customize_schema.customize_schema(copy.deepcopy(metadata))
3331

3432
for idv in ids:
35-
metadata['id'] = idv
33+
metadata["id"] = idv
3634

3735
if files:
3836
# Files to delete
@@ -41,79 +39,76 @@ def caltechdata_edit(token,ids,metadata={},files={},delete={},production=False):
4139
existing = c.get(api_url + idv)
4240
file_info = existing.json()["metadata"]
4341
fids = []
44-
for f in files: #Check if new files match existing
45-
if 'electronic_location_and_access' in file_info:
42+
for f in files: # Check if new files match existing
43+
if "electronic_location_and_access" in file_info:
4644
for ex in file_info["electronic_location_and_access"]:
47-
name = ex['electronic_name'][0]
48-
fu = ex['uniform_resource_identifier'].split('/')[-2]
45+
name = ex["electronic_name"][0]
46+
fu = ex["uniform_resource_identifier"].split("/")[-2]
4947
if name == f:
5048
fids.append(fu)
5149
for d in delete:
5250
if name == d:
5351
fids.append(fu)
54-
if name.split('.')[-1] == d:
52+
if name.split(".")[-1] == d:
5553
fids.append(fu)
5654
if len(fids) > 0:
57-
fjson = {'delete': fids}
55+
fjson = {"delete": fids}
5856

5957
# upload new
6058
print(files)
6159
fileinfo = [send_s3(f, token, production) for f in files]
6260

63-
fjson['new'] = fileinfo
64-
metadata['files'] = fjson
61+
fjson["new"] = fileinfo
62+
metadata["files"] = fjson
6563

66-
dat = json.dumps({'record': metadata})
64+
dat = json.dumps({"record": metadata})
6765

68-
#outf = open('out.json','w')
69-
#outf.write(dat)
66+
# outf = open('out.json','w')
67+
# outf.write(dat)
7068

7169
c = session()
7270
response = c.post(url, headers=headers, data=dat)
7371
print(response.text)
7472

75-
def caltechdata_add(token,ids,metadata={},files={},production=False):
76-
'''Adds file'''
7773

78-
#If files is a string - change to single value array
74+
def caltechdata_add(token, ids, metadata={}, files={}, production=False):
75+
"""Adds file"""
76+
77+
# If files is a string - change to single value array
7978
if isinstance(ids, int):
8079
ids = [str(ids)]
8180
if isinstance(ids, str):
8281
ids = [ids]
8382

8483
if production == True:
8584
url = "https://data.caltech.edu/submit/api/edit/"
86-
api_url = "https://data.caltech.edu/api/record/"
85+
api_url = "https://data.caltech.edu/api/record/"
8786
else:
8887
url = "https://cd-sandbox.tind.io/submit/api/edit/"
8988
api_url = "https://cd-sandbox.tind.io/api/record/"
9089

91-
headers = {
92-
'Authorization' : 'Bearer %s' % token,
93-
'Content-type': 'application/json'
94-
}
90+
headers = {"Authorization": "Bearer %s" % token, "Content-type": "application/json"}
9591

9692
if metadata:
9793
metadata = customize_schema.customize_schema(copy.deepcopy(metadata))
9894

9995
fjson = {}
10096

10197
for idv in ids:
102-
metadata['id'] = idv
98+
metadata["id"] = idv
10399

104100
if files:
105101
# upload new
106102
fileinfo = [send_s3(f, token, production) for f in files]
107103

108-
fjson['new'] = fileinfo
109-
metadata['files'] = fjson
104+
fjson["new"] = fileinfo
105+
metadata["files"] = fjson
110106

111-
dat = json.dumps({'record': metadata})
107+
dat = json.dumps({"record": metadata})
112108

113-
#outf = open('out.json','w')
114-
#outf.write(dat)
109+
# outf = open('out.json','w')
110+
# outf.write(dat)
115111

116112
c = session()
117113
response = c.post(url, headers=headers, data=dat)
118114
return response.text
119-

‎caltechdata_api/caltechdata_write.py

Lines changed: 53 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -3,74 +3,78 @@
33
import json, copy
44
import os
55

6-
def send_s3(filepath,token,production=False):
7-
6+
7+
def send_s3(filepath, token, production=False):
8+
89
if production == True:
910
s3surl = "https://data.caltech.edu/tindfiles/sign_s3/"
1011
chkurl = "https://data.caltech.edu/tindfiles/md5_s3"
1112
else:
1213
s3surl = "https://cd-sandbox.tind.io/tindfiles/sign_s3/"
1314
chkurl = "https://cd-sandbox.tind.io/tindfiles/md5_s3"
1415

15-
headers = { 'Authorization' : 'Bearer %s' % token }
16+
headers = {"Authorization": "Bearer %s" % token}
1617

1718
c = session()
1819

1920
print(s3surl)
2021
print(headers)
21-
response = c.get(s3surl,headers=headers)
22+
response = c.get(s3surl, headers=headers)
2223
jresp = response.json()
23-
data = jresp['data']
24-
25-
bucket = jresp['bucket']
26-
key = data['fields']['key']
27-
policy = data['fields']['policy']
28-
aid = data['fields']['AWSAccessKeyId']
29-
signature = data['fields']['signature']
30-
url = data['url']
31-
32-
infile = open(filepath,'rb')
33-
size = infile.seek(0,2)
34-
infile.seek(0,0) #reset at beginning
35-
36-
s3headers = { 'Host' : bucket+'.s3.amazonaws.com',\
37-
'Date' : 'date',\
38-
'x-amz-acl' : 'public-read',\
39-
'Access-Control-Allow-Origin' : '*' }
40-
41-
form = ( ( 'key', key )
42-
, ("acl", "public-read")
43-
, ('AWSAccessKeyID', aid)
44-
, ('policy', policy)
45-
, ('signature', signature)
46-
, ('file', infile ))
24+
data = jresp["data"]
25+
26+
bucket = jresp["bucket"]
27+
key = data["fields"]["key"]
28+
policy = data["fields"]["policy"]
29+
aid = data["fields"]["AWSAccessKeyId"]
30+
signature = data["fields"]["signature"]
31+
url = data["url"]
32+
33+
infile = open(filepath, "rb")
34+
size = infile.seek(0, 2)
35+
infile.seek(0, 0) # reset at beginning
36+
37+
s3headers = {
38+
"Host": bucket + ".s3.amazonaws.com",
39+
"Date": "date",
40+
"x-amz-acl": "public-read",
41+
"Access-Control-Allow-Origin": "*",
42+
}
43+
44+
form = (
45+
("key", key),
46+
("acl", "public-read"),
47+
("AWSAccessKeyID", aid),
48+
("policy", policy),
49+
("signature", signature),
50+
("file", infile),
51+
)
4752

4853
c = session()
49-
response = c.post(url,files=form, headers=s3headers)
54+
response = c.post(url, files=form, headers=s3headers)
5055
print(response)
51-
if(response.text):
56+
if response.text:
5257
raise Exception(response.text)
5358

54-
print(chkurl+'/'+bucket+'/'+key+'/')
59+
print(chkurl + "/" + bucket + "/" + key + "/")
5560
print(headers)
56-
response = c.get(chkurl+'/'+bucket+'/'+key+'/',headers=headers)
61+
response = c.get(chkurl + "/" + bucket + "/" + key + "/", headers=headers)
5762
print(response)
5863
md5 = response.json()["md5"]
59-
filename = filepath.split('/')[-1]
64+
filename = filepath.split("/")[-1]
65+
66+
fileinfo = {"url": key, "filename": filename, "md5": md5, "size": size}
6067

61-
fileinfo = { "url" : key,\
62-
"filename" : filename,\
63-
"md5" : md5,"size" : size }
68+
return fileinfo
6469

65-
return(fileinfo)
6670

67-
def caltechdata_write(metadata,token,files=[],production=False):
71+
def caltechdata_write(metadata, token, files=[], production=False):
6872

69-
#If files is a string - change to single value array
73+
# If files is a string - change to single value array
7074
if isinstance(files, str) == True:
7175
files = [files]
7276

73-
fileinfo=[]
77+
fileinfo = []
7478

7579
for f in files:
7680
fileinfo.append(send_s3(f, token, production))
@@ -80,20 +84,18 @@ def caltechdata_write(metadata,token,files=[],production=False):
8084
else:
8185
url = "https://cd-sandbox.tind.io/submit/api/create/"
8286

83-
headers = {
84-
'Authorization' : 'Bearer %s' % token,
85-
'Content-type': 'application/json'
86-
}
87+
headers = {"Authorization": "Bearer %s" % token, "Content-type": "application/json"}
8788

8889
newdata = customize_schema.customize_schema(copy.deepcopy(metadata))
89-
newdata['files'] = fileinfo
90-
if 'doi' not in newdata:
91-
#We want tind to generate the identifier
92-
newdata['final_actions'] = [{"type":"create_doi",\
93-
"parameters":{"type":"records","field":"doi"}}]
90+
newdata["files"] = fileinfo
91+
if "doi" not in newdata:
92+
# We want tind to generate the identifier
93+
newdata["final_actions"] = [
94+
{"type": "create_doi", "parameters": {"type": "records", "field": "doi"}}
95+
]
9496

95-
dat = json.dumps({'record': newdata})
97+
dat = json.dumps({"record": newdata})
9698

9799
c = session()
98-
response = c.post(url,headers=headers,data=dat)
100+
response = c.post(url, headers=headers, data=dat)
99101
return response.text

0 commit comments

Comments
 (0)