Skip to content

Commit e94a60f

Browse files
jnolisjnolis
committed
black lint
1 parent cc2cbe5 commit e94a60f

4 files changed

+108
-61
lines changed

‎examples/pytorch/01-start-with-pytorch.ipynb

+31-16
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,16 @@
2828
"metadata": {},
2929
"outputs": [],
3030
"source": [
31-
"import uuid # noqa\n",
31+
"import uuid # noqa\n",
3232
"import datetime\n",
33-
"import pickle # noqa\n",
33+
"import pickle # noqa\n",
3434
"import json\n",
35-
"import torch # noqa\n",
35+
"import torch # noqa\n",
3636
"import torch.nn as nn\n",
3737
"import torch.optim as optim\n",
3838
"import numpy as np\n",
3939
"import urllib.request\n",
40-
"import pandas as pd # noqa\n",
40+
"import pandas as pd # noqa\n",
4141
"from torch.utils.data import Dataset, DataLoader"
4242
]
4343
},
@@ -68,13 +68,16 @@
6868
" def get_substrings(in_str):\n",
6969
" # add the stop character to the end of the name, then generate all the partial names\n",
7070
" in_str = in_str + \"+\"\n",
71-
" res = [in_str[0: j] for j in range(1, len(in_str) + 1)]\n",
71+
" res = [in_str[0:j] for j in range(1, len(in_str) + 1)]\n",
7272
" return res\n",
73+
"\n",
7374
" pet_names_expanded = [get_substrings(name) for name in pet_names]\n",
7475
" pet_names_expanded = [item for sublist in pet_names_expanded for item in sublist]\n",
7576
" pet_names_characters = [list(name) for name in pet_names_expanded]\n",
76-
" pet_names_padded = [name[-(str_len + 1):] for name in pet_names_characters]\n",
77-
" pet_names_padded = [list((str_len + 1 - len(characters)) * \"*\") + characters for characters in pet_names_padded]\n",
77+
" pet_names_padded = [name[-(str_len + 1) :] for name in pet_names_characters]\n",
78+
" pet_names_padded = [\n",
79+
" list((str_len + 1 - len(characters)) * \"*\") + characters for characters in pet_names_padded\n",
80+
" ]\n",
7881
" pet_names_numeric = [[characters.index(char) for char in name] for name in pet_names_padded]\n",
7982
"\n",
8083
" # the final x and y data to use for training the model. Note that the x data needs to be one-hot encoded\n",
@@ -158,7 +161,9 @@
158161
" loss = criterion(batch_y_pred.transpose(1, 2), batch_y)\n",
159162
" loss.backward()\n",
160163
" optimizer.step()\n",
161-
" print(f\"{datetime.datetime.now().isoformat()} - epoch {epoch} complete - loss {loss.item()}\")\n",
164+
" print(\n",
165+
" f\"{datetime.datetime.now().isoformat()} - epoch {epoch} complete - loss {loss.item()}\"\n",
166+
" )\n",
162167
" return model"
163168
]
164169
},
@@ -203,27 +208,37 @@
203208
" device = torch.device(0)\n",
204209
" in_progress_name = []\n",
205210
" next_letter = \"\"\n",
206-
" while(not next_letter == \"+\" and len(in_progress_name) < 30):\n",
211+
" while not next_letter == \"+\" and len(in_progress_name) < 30:\n",
207212
" # prep the data to run in the model again\n",
208213
" in_progress_name_padded = in_progress_name[-str_len:]\n",
209-
" in_progress_name_padded = list((str_len - len(in_progress_name_padded)) * \"*\") + in_progress_name_padded\n",
214+
" in_progress_name_padded = (\n",
215+
" list((str_len - len(in_progress_name_padded)) * \"*\") + in_progress_name_padded\n",
216+
" )\n",
210217
" in_progress_name_numeric = [characters.index(char) for char in in_progress_name_padded]\n",
211218
" in_progress_name_tensor = torch.tensor(in_progress_name_numeric, device=device)\n",
212-
" in_progress_name_tensor = torch.nn.functional.one_hot(in_progress_name_tensor, num_classes=len(characters)).float()\n",
219+
" in_progress_name_tensor = torch.nn.functional.one_hot(\n",
220+
" in_progress_name_tensor, num_classes=len(characters)\n",
221+
" ).float()\n",
213222
" in_progress_name_tensor = torch.unsqueeze(in_progress_name_tensor, 0)\n",
214223
"\n",
215224
" # get the probabilities of each possible next character by running the model\n",
216225
" with torch.no_grad():\n",
217226
" next_letter_probabilities = model(in_progress_name_tensor)\n",
218227
"\n",
219228
" next_letter_probabilities = next_letter_probabilities[0, -1, :]\n",
220-
" next_letter_probabilities = torch.nn.functional.softmax(next_letter_probabilities, dim=0).detach().cpu().numpy()\n",
229+
" next_letter_probabilities = (\n",
230+
" torch.nn.functional.softmax(next_letter_probabilities, dim=0).detach().cpu().numpy()\n",
231+
" )\n",
221232
" next_letter_probabilities = next_letter_probabilities[1:]\n",
222-
" next_letter_probabilities = [p / sum(next_letter_probabilities) for p in next_letter_probabilities]\n",
233+
" next_letter_probabilities = [\n",
234+
" p / sum(next_letter_probabilities) for p in next_letter_probabilities\n",
235+
" ]\n",
223236
"\n",
224237
" # determine what the actual letter is\n",
225-
" next_letter = characters[np.random.choice(len(characters) - 1, p=next_letter_probabilities) + 1]\n",
226-
" if(next_letter != \"+\"):\n",
238+
" next_letter = characters[\n",
239+
" np.random.choice(len(characters) - 1, p=next_letter_probabilities) + 1\n",
240+
" ]\n",
241+
" if next_letter != \"+\":\n",
227242
" # if the next character isn't stop add the latest generated character to the name and continue\n",
228243
" in_progress_name.append(next_letter)\n",
229244
" # turn the list of characters into a single string\n",
@@ -290,4 +305,4 @@
290305
},
291306
"nbformat": 4,
292307
"nbformat_minor": 4
293-
}
308+
}

‎examples/pytorch/02-start-with-dask.ipynb

+2-2
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
"@dask.delayed\n",
4949
"def lazy_exponent(args):\n",
5050
" x, y = args\n",
51-
" '''Define a lazily evaluating function'''\n",
51+
" \"\"\"Define a lazily evaluating function\"\"\"\n",
5252
" return x ** y"
5353
]
5454
},
@@ -117,4 +117,4 @@
117117
},
118118
"nbformat": 4,
119119
"nbformat_minor": 4
120-
}
120+
}

‎examples/pytorch/03-start-with-pytorch-dask-multiple-models.ipynb

+36-25
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,26 @@
2222
"metadata": {},
2323
"outputs": [],
2424
"source": [
25-
"import uuid # noqa\r\n",
26-
"import datetime\r\n",
27-
"import pickle # noqa\r\n",
28-
"import json\r\n",
29-
"import torch\r\n",
30-
"import torch.nn as nn\r\n",
31-
"import torch.optim as optim\r\n",
32-
"import numpy as np # noqa\r\n",
33-
"import urllib.request\r\n",
34-
"import pandas as pd\r\n",
25+
"import uuid # noqa\n",
26+
"\n",
27+
"import datetime\n",
28+
"\n",
29+
"import pickle # noqa\n",
30+
"\n",
31+
"import json\n",
32+
"\n",
33+
"import torch\n",
34+
"\n",
35+
"import torch.nn as nn\n",
36+
"\n",
37+
"import torch.optim as optim\n",
38+
"\n",
39+
"import numpy as np # noqa\n",
40+
"\n",
41+
"import urllib.request\n",
42+
"\n",
43+
"import pandas as pd\n",
44+
"\n",
3545
"from torch.utils.data import Dataset, DataLoader"
3646
]
3747
},
@@ -55,13 +65,16 @@
5565
" def get_substrings(in_str):\n",
5666
" # add the stop character to the end of the name, then generate all the partial names\n",
5767
" in_str = in_str + \"+\"\n",
58-
" res = [in_str[0: j] for j in range(1, len(in_str) + 1)]\n",
68+
" res = [in_str[0:j] for j in range(1, len(in_str) + 1)]\n",
5969
" return res\n",
70+
"\n",
6071
" pet_names_expanded = [get_substrings(name) for name in pet_names]\n",
6172
" pet_names_expanded = [item for sublist in pet_names_expanded for item in sublist]\n",
6273
" pet_names_characters = [list(name) for name in pet_names_expanded]\n",
63-
" pet_names_padded = [name[-(str_len + 1):] for name in pet_names_characters]\n",
64-
" pet_names_padded = [list((str_len + 1 - len(characters)) * \"*\") + characters for characters in pet_names_padded]\n",
74+
" pet_names_padded = [name[-(str_len + 1) :] for name in pet_names_characters]\n",
75+
" pet_names_padded = [\n",
76+
" list((str_len + 1 - len(characters)) * \"*\") + characters for characters in pet_names_padded\n",
77+
" ]\n",
6578
" pet_names_numeric = [[characters.index(char) for char in name] for name in pet_names_padded]\n",
6679
"\n",
6780
" # the final x and y data to use for training the model. Note that the x data needs to be one-hot encoded\n",
@@ -184,13 +197,15 @@
184197
" loss = criterion(batch_y_pred.transpose(1, 2), batch_y)\n",
185198
" loss.backward()\n",
186199
" optimizer.step()\n",
187-
" logger.info(f\"{datetime.datetime.now().isoformat()} - batch {i} - batch_size {batch_size} - lr {lr} - epoch {epoch} complete - loss {loss.item()}\")\n",
200+
" logger.info(\n",
201+
" f\"{datetime.datetime.now().isoformat()} - batch {i} - batch_size {batch_size} - lr {lr} - epoch {epoch} complete - loss {loss.item()}\"\n",
202+
" )\n",
188203
" new_results = {\n",
189204
" \"batch_size\": batch_size,\n",
190205
" \"lr\": lr,\n",
191206
" \"epoch\": epoch,\n",
192207
" \"loss\": loss.item(),\n",
193-
" \"elapsed_time_sec\": (datetime.datetime.now() - training_start_time).total_seconds()\n",
208+
" \"elapsed_time_sec\": (datetime.datetime.now() - training_start_time).total_seconds(),\n",
194209
" }\n",
195210
" results.append(new_results)\n",
196211
" return results"
@@ -236,19 +251,15 @@
236251
"source": [
237252
"results_concatenated = [item for sublist in results for item in sublist]\n",
238253
"results_df = pd.DataFrame.from_dict(results_concatenated)\n",
239-
"results_df['experiment'] = \"bs=\" + results_df['batch_size'].astype(str) + \" lr=\" + results_df['lr'].astype(str)\n",
254+
"results_df[\"experiment\"] = (\n",
255+
" \"bs=\" + results_df[\"batch_size\"].astype(str) + \" lr=\" + results_df[\"lr\"].astype(str)\n",
256+
")\n",
240257
"\n",
241258
"sns.set_theme()\n",
242259
"\n",
243-
"sns.relplot(\n",
244-
" data=results_df,\n",
245-
" x=\"epoch\", y=\"loss\", col=\"experiment\", kind=\"line\"\n",
246-
")\n",
260+
"sns.relplot(data=results_df, x=\"epoch\", y=\"loss\", col=\"experiment\", kind=\"line\")\n",
247261
"\n",
248-
"sns.relplot(\n",
249-
" data=results_df,\n",
250-
" x=\"elapsed_time_sec\", y=\"loss\", col=\"experiment\", kind=\"line\"\n",
251-
")"
262+
"sns.relplot(data=results_df, x=\"elapsed_time_sec\", y=\"loss\", col=\"experiment\", kind=\"line\")"
252263
]
253264
}
254265
],
@@ -273,4 +284,4 @@
273284
},
274285
"nbformat": 4,
275286
"nbformat_minor": 5
276-
}
287+
}

‎examples/pytorch/04-start-with-pytorch-dask-single-model.ipynb

+39-18
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
"import torch.optim as optim\n",
3838
"import numpy as np\n",
3939
"import urllib.request\n",
40-
"import pandas as pd # noqa\n",
40+
"import pandas as pd # noqa\n",
4141
"from torch.utils.data import Dataset, DataLoader"
4242
]
4343
},
@@ -61,13 +61,16 @@
6161
" def get_substrings(in_str):\n",
6262
" # add the stop character to the end of the name, then generate all the partial names\n",
6363
" in_str = in_str + \"+\"\n",
64-
" res = [in_str[0: j] for j in range(1, len(in_str) + 1)]\n",
64+
" res = [in_str[0:j] for j in range(1, len(in_str) + 1)]\n",
6565
" return res\n",
66+
"\n",
6667
" pet_names_expanded = [get_substrings(name) for name in pet_names]\n",
6768
" pet_names_expanded = [item for sublist in pet_names_expanded for item in sublist]\n",
6869
" pet_names_characters = [list(name) for name in pet_names_expanded]\n",
69-
" pet_names_padded = [name[-(str_len + 1):] for name in pet_names_characters]\n",
70-
" pet_names_padded = [list((str_len + 1 - len(characters)) * \"*\") + characters for characters in pet_names_padded]\n",
70+
" pet_names_padded = [name[-(str_len + 1) :] for name in pet_names_characters]\n",
71+
" pet_names_padded = [\n",
72+
" list((str_len + 1 - len(characters)) * \"*\") + characters for characters in pet_names_padded\n",
73+
" ]\n",
7174
" pet_names_numeric = [[characters.index(char) for char in name] for name in pet_names_padded]\n",
7275
"\n",
7376
" # the final x and y data to use for training the model. Note that the x data needs to be one-hot encoded\n",
@@ -180,7 +183,9 @@
180183
"\n",
181184
" for epoch in range(num_epochs):\n",
182185
" # the logger here logs to the dask log of each work, for easy debugging\n",
183-
" logger.info(f\"Worker {worker_rank} - {datetime.datetime.now().isoformat()} - Beginning epoch {epoch}\")\n",
186+
" logger.info(\n",
187+
" f\"Worker {worker_rank} - {datetime.datetime.now().isoformat()} - Beginning epoch {epoch}\"\n",
188+
" )\n",
184189
"\n",
185190
" # this ensures the data is reshuffled each epoch\n",
186191
" sampler.set_epoch(epoch)\n",
@@ -195,15 +200,21 @@
195200
" loss.backward()\n",
196201
" optimizer.step()\n",
197202
"\n",
198-
" logger.info(f\"Worker {worker_rank} - {datetime.datetime.now().isoformat()} - epoch {epoch} - batch {i} - batch complete - loss {loss.item()}\")\n",
203+
" logger.info(\n",
204+
" f\"Worker {worker_rank} - {datetime.datetime.now().isoformat()} - epoch {epoch} - batch {i} - batch complete - loss {loss.item()}\"\n",
205+
" )\n",
199206
"\n",
200207
" # the first rh call saves a json file with the loss from the worker at the end of the epoch\n",
201208
" rh.submit_result(\n",
202209
" f\"logs/data_{worker_rank}_{epoch}.json\",\n",
203-
" json.dumps({'loss': loss.item(),\n",
204-
" 'time': datetime.datetime.now().isoformat(),\n",
205-
" 'epoch': epoch,\n",
206-
" 'worker': worker_rank})\n",
210+
" json.dumps(\n",
211+
" {\n",
212+
" \"loss\": loss.item(),\n",
213+
" \"time\": datetime.datetime.now().isoformat(),\n",
214+
" \"epoch\": epoch,\n",
215+
" \"worker\": worker_rank,\n",
216+
" }\n",
217+
" ),\n",
207218
" )\n",
208219
" # this saves the model. We only need to do it for one worker (so we picked worker 0)\n",
209220
" if worker_rank == 0:\n",
@@ -283,27 +294,37 @@
283294
"def generate_name(model, characters, str_len):\n",
284295
" in_progress_name = []\n",
285296
" next_letter = \"\"\n",
286-
" while(not next_letter == \"+\" and len(in_progress_name) < 30):\n",
297+
" while not next_letter == \"+\" and len(in_progress_name) < 30:\n",
287298
" # prep the data to run in the model again\n",
288299
" in_progress_name_padded = in_progress_name[-str_len:]\n",
289-
" in_progress_name_padded = list((str_len - len(in_progress_name_padded)) * \"*\") + in_progress_name_padded\n",
300+
" in_progress_name_padded = (\n",
301+
" list((str_len - len(in_progress_name_padded)) * \"*\") + in_progress_name_padded\n",
302+
" )\n",
290303
" in_progress_name_numeric = [characters.index(char) for char in in_progress_name_padded]\n",
291304
" in_progress_name_tensor = torch.tensor(in_progress_name_numeric)\n",
292-
" in_progress_name_tensor = torch.nn.functional.one_hot(in_progress_name_tensor, num_classes=len(characters)).float()\n",
305+
" in_progress_name_tensor = torch.nn.functional.one_hot(\n",
306+
" in_progress_name_tensor, num_classes=len(characters)\n",
307+
" ).float()\n",
293308
" in_progress_name_tensor = torch.unsqueeze(in_progress_name_tensor, 0)\n",
294309
"\n",
295310
" # get the probabilities of each possible next character by running the model\n",
296311
" with torch.no_grad():\n",
297312
" next_letter_probabilities = model(in_progress_name_tensor)\n",
298313
"\n",
299314
" next_letter_probabilities = next_letter_probabilities[0, -1, :]\n",
300-
" next_letter_probabilities = torch.nn.functional.softmax(next_letter_probabilities, dim=0).detach().cpu().numpy()\n",
315+
" next_letter_probabilities = (\n",
316+
" torch.nn.functional.softmax(next_letter_probabilities, dim=0).detach().cpu().numpy()\n",
317+
" )\n",
301318
" next_letter_probabilities = next_letter_probabilities[1:]\n",
302-
" next_letter_probabilities = [p / sum(next_letter_probabilities) for p in next_letter_probabilities]\n",
319+
" next_letter_probabilities = [\n",
320+
" p / sum(next_letter_probabilities) for p in next_letter_probabilities\n",
321+
" ]\n",
303322
"\n",
304323
" # determine what the actual letter is\n",
305-
" next_letter = characters[np.random.choice(len(characters) - 1, p=next_letter_probabilities) + 1]\n",
306-
" if(next_letter != \"+\"):\n",
324+
" next_letter = characters[\n",
325+
" np.random.choice(len(characters) - 1, p=next_letter_probabilities) + 1\n",
326+
" ]\n",
327+
" if next_letter != \"+\":\n",
307328
" # if the next character isn't stop add the latest generated character to the name and continue\n",
308329
" in_progress_name.append(next_letter)\n",
309330
" # turn the list of characters into a single string\n",
@@ -390,4 +411,4 @@
390411
},
391412
"nbformat": 4,
392413
"nbformat_minor": 4
393-
}
414+
}

0 commit comments

Comments
 (0)