Skip to content

Commit cd866ea

Browse files
gustavocidornelaswhoseoyster
authored andcommitted
Polish OpenAIMonitor and add it to the API reference
1 parent 986cb22 commit cd866ea

File tree

4 files changed

+169
-146
lines changed

4 files changed

+169
-146
lines changed

docs/source/reference/monitoring.rst

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,24 @@ Openlayer project is what enables the monitoring mode.
3333

3434
Publishing production data
3535
----------------------------
36-
Production data is published to an inference pipeline on the Openlayer platform using
37-
the methods below.
36+
37+
LLMs
38+
^^^^
39+
40+
If you are using an OpenAI LLM, you can simply switch monitoring on and off with a
41+
single line of code.
42+
43+
.. autosummary::
44+
:toctree: api/
45+
:template: class.rst
46+
47+
openlayer.llm_monitors.OpenAIMonitor
48+
49+
Traditional ML models
50+
^^^^^^^^^^^^^^^^^^^^^
51+
52+
For traditional ML models and other LLM providers, you can publish production data with
53+
the following methods.
3854

3955
.. autosummary::
4056
:toctree: api/

examples/monitoring/quickstart/llms/openai_llm_monitor.ipynb

Lines changed: 13 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
},
2424
{
2525
"cell_type": "code",
26-
"execution_count": 1,
26+
"execution_count": null,
2727
"id": "f3f4fa13",
2828
"metadata": {},
2929
"outputs": [],
@@ -44,7 +44,7 @@
4444
},
4545
{
4646
"cell_type": "code",
47-
"execution_count": 2,
47+
"execution_count": null,
4848
"id": "e60584fa",
4949
"metadata": {},
5050
"outputs": [],
@@ -64,19 +64,10 @@
6464
},
6565
{
6666
"cell_type": "code",
67-
"execution_count": 8,
67+
"execution_count": null,
6868
"id": "2cb5ce9b",
6969
"metadata": {},
70-
"outputs": [
71-
{
72-
"name": "stdout",
73-
"output_type": "stream",
74-
"text": [
75-
"Monitoring is already on!\n",
76-
"To stop it, call `stop_monitoring`.\n"
77-
]
78-
}
79-
],
70+
"outputs": [],
8071
"source": [
8172
"openai_monitor.start_monitoring()"
8273
]
@@ -91,7 +82,7 @@
9182
},
9283
{
9384
"cell_type": "code",
94-
"execution_count": 4,
85+
"execution_count": null,
9586
"id": "fb648049-00bd-447c-8feb-ecf794d45ba7",
9687
"metadata": {},
9788
"outputs": [],
@@ -103,18 +94,10 @@
10394
},
10495
{
10596
"cell_type": "code",
106-
"execution_count": 5,
97+
"execution_count": null,
10798
"id": "7d1e8c74-c667-44a7-b8ea-8f5b83049ea7",
10899
"metadata": {},
109-
"outputs": [
110-
{
111-
"name": "stdout",
112-
"output_type": "stream",
113-
"text": [
114-
"Data published!\n"
115-
]
116-
}
117-
],
100+
"outputs": [],
118101
"source": [
119102
"completion = openai.ChatCompletion.create(\n",
120103
" model=\"gpt-3.5-turbo\",\n",
@@ -127,18 +110,10 @@
127110
},
128111
{
129112
"cell_type": "code",
130-
"execution_count": 6,
113+
"execution_count": null,
131114
"id": "58dd9603",
132115
"metadata": {},
133-
"outputs": [
134-
{
135-
"name": "stdout",
136-
"output_type": "stream",
137-
"text": [
138-
"Data published!\n"
139-
]
140-
}
141-
],
116+
"outputs": [],
142117
"source": [
143118
"completion = openai.ChatCompletion.create(\n",
144119
" model=\"gpt-3.5-turbo\",\n",
@@ -159,89 +134,20 @@
159134
},
160135
{
161136
"cell_type": "code",
162-
"execution_count": 7,
137+
"execution_count": null,
163138
"id": "27bb2bdc",
164139
"metadata": {},
165-
"outputs": [
166-
{
167-
"data": {
168-
"text/html": [
169-
"<div>\n",
170-
"<style scoped>\n",
171-
" .dataframe tbody tr th:only-of-type {\n",
172-
" vertical-align: middle;\n",
173-
" }\n",
174-
"\n",
175-
" .dataframe tbody tr th {\n",
176-
" vertical-align: top;\n",
177-
" }\n",
178-
"\n",
179-
" .dataframe thead th {\n",
180-
" text-align: right;\n",
181-
" }\n",
182-
"</style>\n",
183-
"<table border=\"1\" class=\"dataframe\">\n",
184-
" <thead>\n",
185-
" <tr style=\"text-align: right;\">\n",
186-
" <th></th>\n",
187-
" <th>input</th>\n",
188-
" <th>output</th>\n",
189-
" <th>tokens</th>\n",
190-
" <th>latency</th>\n",
191-
" </tr>\n",
192-
" </thead>\n",
193-
" <tbody>\n",
194-
" <tr>\n",
195-
" <th>0</th>\n",
196-
" <td>How are you doing today?</td>\n",
197-
" <td>Thank you for asking! As an AI, I don't have f...</td>\n",
198-
" <td>62</td>\n",
199-
" <td>6400.212049</td>\n",
200-
" </tr>\n",
201-
" <tr>\n",
202-
" <th>1</th>\n",
203-
" <td>Is Python strongly typed?</td>\n",
204-
" <td>Yes, Python is considered a strongly typed lan...</td>\n",
205-
" <td>68</td>\n",
206-
" <td>7226.889133</td>\n",
207-
" </tr>\n",
208-
" </tbody>\n",
209-
"</table>\n",
210-
"</div>"
211-
],
212-
"text/plain": [
213-
" input \\\n",
214-
"0 How are you doing today? \n",
215-
"1 Is Python strongly typed? \n",
216-
"\n",
217-
" output tokens latency \n",
218-
"0 Thank you for asking! As an AI, I don't have f... 62 6400.212049 \n",
219-
"1 Yes, Python is considered a strongly typed lan... 68 7226.889133 "
220-
]
221-
},
222-
"execution_count": 7,
223-
"metadata": {},
224-
"output_type": "execute_result"
225-
}
226-
],
140+
"outputs": [],
227141
"source": [
228142
"openai_monitor.data"
229143
]
230144
},
231145
{
232146
"cell_type": "code",
233-
"execution_count": 9,
147+
"execution_count": null,
234148
"id": "f9139f2b",
235149
"metadata": {},
236-
"outputs": [
237-
{
238-
"name": "stdout",
239-
"output_type": "stream",
240-
"text": [
241-
"Monitoring stopped.\n"
242-
]
243-
}
244-
],
150+
"outputs": [],
245151
"source": [
246152
"openai_monitor.stop_monitoring()"
247153
]

examples/monitoring/quickstart/discriminative-ml/monitoring-quickstart.ipynb renamed to examples/monitoring/quickstart/traditional-ml/monitoring-quickstart.ipynb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"id": "ef55abc9",
66
"metadata": {},
77
"source": [
8-
"[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/openlayer-ai/examples-gallery/blob/main/monitoring/quickstart/discriminative-ml/monitoring-quickstart.ipynb)\n",
8+
"[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/openlayer-ai/examples-gallery/blob/main/monitoring/quickstart/traditional-ml/monitoring-quickstart.ipynb)\n",
99
"\n",
1010
"\n",
1111
"# <a id=\"top\">Monitoring quickstart</a>\n",

0 commit comments

Comments
 (0)