Skip to content

Commit d877b4c

Browse files
efd6efd6
authored
box_events: fix handling of large cursor offsets (#14319)
When a cursor stream offset is large — at least 1e6, the template renders the value in e-notation. This is a consequence of the cursor being stored as JSON and so being contaminated by JS number semantics. Another threshold exists at 0x1p53 (4.5e15) where we lose exact integer representation. We do see values as large as 3.0e16, so we are beyond this value and cannot rely on numeric value representation at all. This is exacerbated by the fact that the input converts from string to integer values via float64. To resolve this, explicitly convert the offset to an integer when rendering the value into the parameter, and accept that we may either recollect or miss documents from the API.
1 parent e35e24a commit d877b4c

File tree

5 files changed

+332
-6
lines changed

5 files changed

+332
-6
lines changed

‎packages/box_events/_dev/deploy/docker/files/config.yml

Lines changed: 322 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,20 @@ rules:
1616
Content-Type:
1717
- "application/json"
1818
body: |-
19-
{"access_token":"c3FIOG9vSGV4VHo4QzAyg5T1JvNnJoZ3ExaVNyQWw6WjRsanRKZG5lQk9qUE1BVQ","expires_in":3600,"token_type":"bearer","refresh_token":"c3FIOG9vSGV4VHo4QzAyg5T1JvNnJoZ3ExaVNyQWw6WjRsanRKZG5lQk9qUE1BVQ","issued_token_type":"urn:ietf:params:oauth:token-type:access_token"}
19+
{{ minify_json `
20+
{
21+
"access_token": "c3FIOG9vSGV4VHo4QzAyg5T1JvNnJoZ3ExaVNyQWw6WjRsanRKZG5lQk9qUE1BVQ",
22+
"expires_in": 3600,
23+
"token_type": "bearer",
24+
"refresh_token": "c3FIOG9vSGV4VHo4QzAyg5T1JvNnJoZ3ExaVNyQWw6WjRsanRKZG5lQk9qUE1BVQ",
25+
"issued_token_type": "urn:ietf:params:oauth:token-type:access_token"
26+
}
27+
`}}
2028
- path: /2.0/events
2129
methods: [GET]
2230
query_params:
2331
stream_type: all
32+
stream_position: null
2433
request_headers:
2534
Authorization:
2635
- "Bearer c3FIOG9vSGV4VHo4QzAyg5T1JvNnJoZ3ExaVNyQWw6WjRsanRKZG5lQk9qUE1BVQ"
@@ -30,4 +39,315 @@ rules:
3039
Content-Type:
3140
- "application/json; charset=utf-8"
3241
body: |-
33-
{"chunk_size":2,"entries":[{"source":null,"created_by":{"type":"user","id":"2","name":"Unknown User","login":""},"action_by":null,"created_at":"2019-12-20T11:38:56-08:00","event_id":"97f1b31f-f143-4777-81f8-1b557b39ca33","event_type":"SHIELD_ALERT","ip_address":"10.1.2.3","type":"event","session_id":null,"additional_details":{"shield_alert":{"rule_category":"Anomalous Download","rule_id":123,"rule_name":"Anomalous Download Rule","risk_score":77,"alert_summary":{"description":"Significant increase in download content week over week, 9200% (25.04 MB) more than last week 12 additional files downloaded week over week)","download_delta_size":"25 Mb","download_delta_percent":9200,"historical_period":{"date_range":{"start_date":"2019-12-01T01:01:00-08:00","end_date":"2019-12-08T01:01:00-08:00"},"download_size":"0 Mb","downloaded_files_count":1},"anomaly_period":{"date_range":{"start_date":"2019-12-08T01:01:00-08:00","end_date":"2019-12-15T01:01:00-08:00"},"download_size":"25 Mb","downloaded_files_count":13},"download_ips":[{"ip":"1.128.0.0"},{"ip":"175.16.199.0"}]},"alert_id":444,"priority":"medium","user":{"id":567,"name":"Some user","email":"[email protected]"},"link":"https://cloud.app.box.com/master/shield/alerts/444","created_at":"2019-12-20T11:38:16-08:00"}}},{"created_at":"2022-06-27T05:09:40-07:00","created_by":{"id":"19530772260","login":"[email protected]","name":"Elastic Integrations","type":"user"},"event_id":"e1cb161d5fbd3f3a80fd560f39a0f52a2cff3db9","event_type":"ITEM_CREATE","recorded_at":"2022-06-27T05:09:41-07:00","session_id":"rzraadh3n273zc5f","source":{"content_created_at":"2022-06-27T05:09:40-07:00","content_modified_at":"2022-06-27T05:09:40-07:00","created_at":"2022-06-27T05:09:40-07:00","created_by":{"id":"19530772260","login":"[email protected]","name":"Elastic Integrations","type":"user"},"description":"","etag":"0","folder_upload_email":null,"id":"166233012413","item_status":"active","modified_at":"2022-06-27T05:09:40-07:00","modified_by":{"id":"19530772260","login":"[email protected]","name":"Elastic Integrations","type":"user"},"name":"Platform App Diagnostics run on 2022-06-27 05-09-38 PDT","owned_by":{"id":"19530772260","login":"[email protected]","name":"Elastic Integrations","type":"user"},"parent":{"etag":"0","id":"166232910591","name":"Box Reports","sequence_id":"0","type":"folder"},"path_collection":{"entries":[{"etag":null,"id":"0","name":"All Files","sequence_id":null,"type":"folder"},{"etag":"0","id":"166232910591","name":"Box Reports","sequence_id":"0","type":"folder"}],"total_count":2},"purged_at":null,"sequence_id":"0","shared_link":null,"size":0,"synced":false,"trashed_at":null,"type":"folder"},"type":"event"}],"next_stream_position":1152922976252290800}
42+
{{ minify_json `
43+
{
44+
"chunk_size": 2,
45+
"entries": [
46+
{
47+
"source": null,
48+
"created_by": {
49+
"type": "user",
50+
"id": "2",
51+
"name": "Unknown User",
52+
"login": ""
53+
},
54+
"action_by": null,
55+
"created_at": "2019-12-20T11:38:56-08:00",
56+
"event_id": "97f1b31f-f143-4777-81f8-1b557b39ca31",
57+
"event_type": "SHIELD_ALERT",
58+
"ip_address": "10.1.2.3",
59+
"type": "event",
60+
"session_id": null,
61+
"additional_details": {
62+
"shield_alert": {
63+
"rule_category": "Anomalous Download",
64+
"rule_id": 123,
65+
"rule_name": "Anomalous Download Rule",
66+
"risk_score": 77,
67+
"alert_summary": {
68+
"description": "Significant increase in download content week over week, 9200% (25.04 MB) more than last week 12 additional files downloaded week over week)",
69+
"download_delta_size": "25 Mb",
70+
"download_delta_percent": 9200,
71+
"historical_period": {
72+
"date_range": {
73+
"start_date": "2019-12-01T01:01:00-08:00",
74+
"end_date": "2019-12-08T01:01:00-08:00"
75+
},
76+
"download_size": "0 Mb",
77+
"downloaded_files_count": 1
78+
},
79+
"anomaly_period": {
80+
"date_range": {
81+
"start_date": "2019-12-08T01:01:00-08:00",
82+
"end_date": "2019-12-15T01:01:00-08:00"
83+
},
84+
"download_size": "25 Mb",
85+
"downloaded_files_count": 13
86+
},
87+
"download_ips": [
88+
{
89+
"ip": "1.128.0.0"
90+
},
91+
{
92+
"ip": "175.16.199.0"
93+
}
94+
]
95+
},
96+
"alert_id": 444,
97+
"priority": "medium",
98+
"user": {
99+
"id": 567,
100+
"name": "Some user",
101+
"email": "[email protected]"
102+
},
103+
"link": "https://cloud.app.box.com/master/shield/alerts/444",
104+
"created_at": "2019-12-20T11:38:16-08:00"
105+
}
106+
}
107+
},
108+
{
109+
"created_at": "2022-06-27T05:09:40-07:00",
110+
"created_by": {
111+
"id": "19530772260",
112+
"login": "[email protected]",
113+
"name": "Elastic Integrations",
114+
"type": "user"
115+
},
116+
"event_id": "e1cb161d5fbd3f3a80fd560f39a0f52a2cff3db9",
117+
"event_type": "ITEM_CREATE",
118+
"recorded_at": "2022-06-27T05:09:41-07:00",
119+
"session_id": "rzraadh3n273zc5f",
120+
"source": {
121+
"content_created_at": "2022-06-27T05:09:40-07:00",
122+
"content_modified_at": "2022-06-27T05:09:40-07:00",
123+
"created_at": "2022-06-27T05:09:40-07:00",
124+
"created_by": {
125+
"id": "19530772260",
126+
"login": "[email protected]",
127+
"name": "Elastic Integrations",
128+
"type": "user"
129+
},
130+
"description": "",
131+
"etag": "0",
132+
"folder_upload_email": null,
133+
"id": "166233012413",
134+
"item_status": "active",
135+
"modified_at": "2022-06-27T05:09:40-07:00",
136+
"modified_by": {
137+
"id": "19530772260",
138+
"login": "[email protected]",
139+
"name": "Elastic Integrations",
140+
"type": "user"
141+
},
142+
"name": "Platform App Diagnostics run on 2022-06-27 05-09-38 PDT",
143+
"owned_by": {
144+
"id": "19530772260",
145+
"login": "[email protected]",
146+
"name": "Elastic Integrations",
147+
"type": "user"
148+
},
149+
"parent": {
150+
"etag": "0",
151+
"id": "166232910591",
152+
"name": "Box Reports",
153+
"sequence_id": "0",
154+
"type": "folder"
155+
},
156+
"path_collection": {
157+
"entries": [
158+
{
159+
"etag": null,
160+
"id": "0",
161+
"name": "All Files",
162+
"sequence_id": null,
163+
"type": "folder"
164+
},
165+
{
166+
"etag": "0",
167+
"id": "166232910591",
168+
"name": "Box Reports",
169+
"sequence_id": "0",
170+
"type": "folder"
171+
}
172+
],
173+
"total_count": 2
174+
},
175+
"purged_at": null,
176+
"sequence_id": "0",
177+
"shared_link": null,
178+
"size": 0,
179+
"synced": false,
180+
"trashed_at": null,
181+
"type": "folder"
182+
},
183+
"type": "event"
184+
}
185+
],
186+
"next_stream_position": 1152922976252290800
187+
}
188+
`}}
189+
- path: /2.0/events
190+
methods: [GET]
191+
query_params:
192+
stream_type: all
193+
# This is a consequence of loss of exact representation of ints in floats.
194+
# There is no good way to deal with this. The least worst approach to fix
195+
# it without changing the cursor state store's type handling is to have
196+
# a nextafter template helper. That is obviously a terrible solution.
197+
stream_position: 1152922976252290816
198+
request_headers:
199+
Authorization:
200+
- "Bearer c3FIOG9vSGV4VHo4QzAyg5T1JvNnJoZ3ExaVNyQWw6WjRsanRKZG5lQk9qUE1BVQ"
201+
responses:
202+
- status_code: 200
203+
headers:
204+
Content-Type:
205+
- "application/json; charset=utf-8"
206+
body: |-
207+
{{ minify_json `
208+
{
209+
"chunk_size": 2,
210+
"entries": [
211+
{
212+
"source": null,
213+
"created_by": {
214+
"type": "user",
215+
"id": "2",
216+
"name": "Unknown User",
217+
"login": ""
218+
},
219+
"action_by": null,
220+
"created_at": "2019-12-20T11:38:56-08:00",
221+
"event_id": "97f1b31f-f143-4777-81f8-1b557b39ca32",
222+
"event_type": "SHIELD_ALERT",
223+
"ip_address": "10.1.2.3",
224+
"type": "event",
225+
"session_id": null,
226+
"additional_details": {
227+
"shield_alert": {
228+
"rule_category": "Anomalous Download",
229+
"rule_id": 123,
230+
"rule_name": "Anomalous Download Rule",
231+
"risk_score": 77,
232+
"alert_summary": {
233+
"description": "Significant increase in download content week over week, 9200% (25.04 MB) more than last week 12 additional files downloaded week over week)",
234+
"download_delta_size": "25 Mb",
235+
"download_delta_percent": 9200,
236+
"historical_period": {
237+
"date_range": {
238+
"start_date": "2019-12-01T01:01:00-08:00",
239+
"end_date": "2019-12-08T01:01:00-08:00"
240+
},
241+
"download_size": "0 Mb",
242+
"downloaded_files_count": 1
243+
},
244+
"anomaly_period": {
245+
"date_range": {
246+
"start_date": "2019-12-08T01:01:00-08:00",
247+
"end_date": "2019-12-15T01:01:00-08:00"
248+
},
249+
"download_size": "25 Mb",
250+
"downloaded_files_count": 13
251+
},
252+
"download_ips": [
253+
{
254+
"ip": "1.128.0.0"
255+
},
256+
{
257+
"ip": "175.16.199.0"
258+
}
259+
]
260+
},
261+
"alert_id": 444,
262+
"priority": "medium",
263+
"user": {
264+
"id": 567,
265+
"name": "Some user",
266+
"email": "[email protected]"
267+
},
268+
"link": "https://cloud.app.box.com/master/shield/alerts/444",
269+
"created_at": "2019-12-20T11:38:16-08:00"
270+
}
271+
}
272+
},
273+
{
274+
"created_at": "2022-06-27T05:09:40-07:00",
275+
"created_by": {
276+
"id": "19530772260",
277+
"login": "[email protected]",
278+
"name": "Elastic Integrations",
279+
"type": "user"
280+
},
281+
"event_id": "e1cb161d5fbd3f3a80fd560f39a0f52a2cff3db8",
282+
"event_type": "ITEM_CREATE",
283+
"recorded_at": "2022-06-27T05:09:41-07:00",
284+
"session_id": "rzraadh3n273zc5f",
285+
"source": {
286+
"content_created_at": "2022-06-27T05:09:40-07:00",
287+
"content_modified_at": "2022-06-27T05:09:40-07:00",
288+
"created_at": "2022-06-27T05:09:40-07:00",
289+
"created_by": {
290+
"id": "19530772260",
291+
"login": "[email protected]",
292+
"name": "Elastic Integrations",
293+
"type": "user"
294+
},
295+
"description": "",
296+
"etag": "0",
297+
"folder_upload_email": null,
298+
"id": "166233012413",
299+
"item_status": "active",
300+
"modified_at": "2022-06-27T05:09:40-07:00",
301+
"modified_by": {
302+
"id": "19530772260",
303+
"login": "[email protected]",
304+
"name": "Elastic Integrations",
305+
"type": "user"
306+
},
307+
"name": "Platform App Diagnostics run on 2022-06-27 05-09-38 PDT",
308+
"owned_by": {
309+
"id": "19530772260",
310+
"login": "[email protected]",
311+
"name": "Elastic Integrations",
312+
"type": "user"
313+
},
314+
"parent": {
315+
"etag": "0",
316+
"id": "166232910591",
317+
"name": "Box Reports",
318+
"sequence_id": "0",
319+
"type": "folder"
320+
},
321+
"path_collection": {
322+
"entries": [
323+
{
324+
"etag": null,
325+
"id": "0",
326+
"name": "All Files",
327+
"sequence_id": null,
328+
"type": "folder"
329+
},
330+
{
331+
"etag": "0",
332+
"id": "166232910591",
333+
"name": "Box Reports",
334+
"sequence_id": "0",
335+
"type": "folder"
336+
}
337+
],
338+
"total_count": 2
339+
},
340+
"purged_at": null,
341+
"sequence_id": "0",
342+
"shared_link": null,
343+
"size": 0,
344+
"synced": false,
345+
"trashed_at": null,
346+
"type": "folder"
347+
},
348+
"type": "event"
349+
}
350+
],
351+
"next_stream_position": 2152922976252290800
352+
}
353+
`}}

‎packages/box_events/changelog.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "2.14.1"
3+
changes:
4+
- description: Fix handling of large cursor offsets.
5+
type: bugfix
6+
link: https://github.com/elastic/integrations/pull/14319
27
- version: "2.14.0"
38
changes:
49
- description: Update and expand ECS `user.*` field mappings.

‎packages/box_events/data_stream/events/_dev/test/system/test-httpjson-config.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,11 @@ vars:
66
client_secret: '8CN4J1ULy6pyR6XA6U8pAYm1CBUNONc7'
77
box_subject_id: '19630872360'
88
api_url: 'http://{{Hostname}}:{{Port}}'
9+
enable_request_tracer: true
910
# correspond to data_stream
1011
data_stream:
1112
vars:
13+
interval: 10s
1214
stream_type: 'all'
13-
enable_request_tracer: true
1415
assert:
15-
hit_count: 2
16+
hit_count: 4

‎packages/box_events/data_stream/events/agent/stream/httpjson.yml.hbs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ request.method: "GET"
1717
request.transforms:
1818
- set:
1919
target: url.params.stream_position
20-
value: '[[.cursor.next_stream_position]]'
20+
value: '[[toInt .cursor.next_stream_position]]'
2121
{{#if stream_type}}
2222
- set:
2323
target: url.params.stream_type

0 commit comments

Comments
 (0)