diff --git a/packages/box_events/_dev/deploy/docker/files/config.yml b/packages/box_events/_dev/deploy/docker/files/config.yml index 7a002095739..e430335a0bc 100644 --- a/packages/box_events/_dev/deploy/docker/files/config.yml +++ b/packages/box_events/_dev/deploy/docker/files/config.yml @@ -16,11 +16,20 @@ rules: Content-Type: - "application/json" body: |- - {"access_token":"c3FIOG9vSGV4VHo4QzAyg5T1JvNnJoZ3ExaVNyQWw6WjRsanRKZG5lQk9qUE1BVQ","expires_in":3600,"token_type":"bearer","refresh_token":"c3FIOG9vSGV4VHo4QzAyg5T1JvNnJoZ3ExaVNyQWw6WjRsanRKZG5lQk9qUE1BVQ","issued_token_type":"urn:ietf:params:oauth:token-type:access_token"} + {{ minify_json ` + { + "access_token": "c3FIOG9vSGV4VHo4QzAyg5T1JvNnJoZ3ExaVNyQWw6WjRsanRKZG5lQk9qUE1BVQ", + "expires_in": 3600, + "token_type": "bearer", + "refresh_token": "c3FIOG9vSGV4VHo4QzAyg5T1JvNnJoZ3ExaVNyQWw6WjRsanRKZG5lQk9qUE1BVQ", + "issued_token_type": "urn:ietf:params:oauth:token-type:access_token" + } + `}} - path: /2.0/events methods: [GET] query_params: stream_type: all + stream_position: null request_headers: Authorization: - "Bearer c3FIOG9vSGV4VHo4QzAyg5T1JvNnJoZ3ExaVNyQWw6WjRsanRKZG5lQk9qUE1BVQ" @@ -30,4 +39,315 @@ rules: Content-Type: - "application/json; charset=utf-8" body: |- - {"chunk_size":2,"entries":[{"source":null,"created_by":{"type":"user","id":"2","name":"Unknown User","login":""},"action_by":null,"created_at":"2019-12-20T11:38:56-08:00","event_id":"97f1b31f-f143-4777-81f8-1b557b39ca33","event_type":"SHIELD_ALERT","ip_address":"10.1.2.3","type":"event","session_id":null,"additional_details":{"shield_alert":{"rule_category":"Anomalous Download","rule_id":123,"rule_name":"Anomalous Download Rule","risk_score":77,"alert_summary":{"description":"Significant increase in download content week over week, 9200% (25.04 MB) more than last week 12 additional files downloaded week over week)","download_delta_size":"25 Mb","download_delta_percent":9200,"historical_period":{"date_range":{"start_date":"2019-12-01T01:01:00-08:00","end_date":"2019-12-08T01:01:00-08:00"},"download_size":"0 Mb","downloaded_files_count":1},"anomaly_period":{"date_range":{"start_date":"2019-12-08T01:01:00-08:00","end_date":"2019-12-15T01:01:00-08:00"},"download_size":"25 Mb","downloaded_files_count":13},"download_ips":[{"ip":"1.128.0.0"},{"ip":"175.16.199.0"}]},"alert_id":444,"priority":"medium","user":{"id":567,"name":"Some user","email":"some@user.com"},"link":"https://cloud.app.box.com/master/shield/alerts/444","created_at":"2019-12-20T11:38:16-08:00"}}},{"created_at":"2022-06-27T05:09:40-07:00","created_by":{"id":"19530772260","login":"info@elastic.co","name":"Elastic Integrations","type":"user"},"event_id":"e1cb161d5fbd3f3a80fd560f39a0f52a2cff3db9","event_type":"ITEM_CREATE","recorded_at":"2022-06-27T05:09:41-07:00","session_id":"rzraadh3n273zc5f","source":{"content_created_at":"2022-06-27T05:09:40-07:00","content_modified_at":"2022-06-27T05:09:40-07:00","created_at":"2022-06-27T05:09:40-07:00","created_by":{"id":"19530772260","login":"info@elastic.co","name":"Elastic Integrations","type":"user"},"description":"","etag":"0","folder_upload_email":null,"id":"166233012413","item_status":"active","modified_at":"2022-06-27T05:09:40-07:00","modified_by":{"id":"19530772260","login":"info@elastic.co","name":"Elastic Integrations","type":"user"},"name":"Platform App Diagnostics run on 2022-06-27 05-09-38 PDT","owned_by":{"id":"19530772260","login":"info@elastic.co","name":"Elastic Integrations","type":"user"},"parent":{"etag":"0","id":"166232910591","name":"Box Reports","sequence_id":"0","type":"folder"},"path_collection":{"entries":[{"etag":null,"id":"0","name":"All Files","sequence_id":null,"type":"folder"},{"etag":"0","id":"166232910591","name":"Box Reports","sequence_id":"0","type":"folder"}],"total_count":2},"purged_at":null,"sequence_id":"0","shared_link":null,"size":0,"synced":false,"trashed_at":null,"type":"folder"},"type":"event"}],"next_stream_position":1152922976252290800} + {{ minify_json ` + { + "chunk_size": 2, + "entries": [ + { + "source": null, + "created_by": { + "type": "user", + "id": "2", + "name": "Unknown User", + "login": "" + }, + "action_by": null, + "created_at": "2019-12-20T11:38:56-08:00", + "event_id": "97f1b31f-f143-4777-81f8-1b557b39ca31", + "event_type": "SHIELD_ALERT", + "ip_address": "10.1.2.3", + "type": "event", + "session_id": null, + "additional_details": { + "shield_alert": { + "rule_category": "Anomalous Download", + "rule_id": 123, + "rule_name": "Anomalous Download Rule", + "risk_score": 77, + "alert_summary": { + "description": "Significant increase in download content week over week, 9200% (25.04 MB) more than last week 12 additional files downloaded week over week)", + "download_delta_size": "25 Mb", + "download_delta_percent": 9200, + "historical_period": { + "date_range": { + "start_date": "2019-12-01T01:01:00-08:00", + "end_date": "2019-12-08T01:01:00-08:00" + }, + "download_size": "0 Mb", + "downloaded_files_count": 1 + }, + "anomaly_period": { + "date_range": { + "start_date": "2019-12-08T01:01:00-08:00", + "end_date": "2019-12-15T01:01:00-08:00" + }, + "download_size": "25 Mb", + "downloaded_files_count": 13 + }, + "download_ips": [ + { + "ip": "1.128.0.0" + }, + { + "ip": "175.16.199.0" + } + ] + }, + "alert_id": 444, + "priority": "medium", + "user": { + "id": 567, + "name": "Some user", + "email": "some@user.com" + }, + "link": "https://cloud.app.box.com/master/shield/alerts/444", + "created_at": "2019-12-20T11:38:16-08:00" + } + } + }, + { + "created_at": "2022-06-27T05:09:40-07:00", + "created_by": { + "id": "19530772260", + "login": "info@elastic.co", + "name": "Elastic Integrations", + "type": "user" + }, + "event_id": "e1cb161d5fbd3f3a80fd560f39a0f52a2cff3db9", + "event_type": "ITEM_CREATE", + "recorded_at": "2022-06-27T05:09:41-07:00", + "session_id": "rzraadh3n273zc5f", + "source": { + "content_created_at": "2022-06-27T05:09:40-07:00", + "content_modified_at": "2022-06-27T05:09:40-07:00", + "created_at": "2022-06-27T05:09:40-07:00", + "created_by": { + "id": "19530772260", + "login": "info@elastic.co", + "name": "Elastic Integrations", + "type": "user" + }, + "description": "", + "etag": "0", + "folder_upload_email": null, + "id": "166233012413", + "item_status": "active", + "modified_at": "2022-06-27T05:09:40-07:00", + "modified_by": { + "id": "19530772260", + "login": "info@elastic.co", + "name": "Elastic Integrations", + "type": "user" + }, + "name": "Platform App Diagnostics run on 2022-06-27 05-09-38 PDT", + "owned_by": { + "id": "19530772260", + "login": "info@elastic.co", + "name": "Elastic Integrations", + "type": "user" + }, + "parent": { + "etag": "0", + "id": "166232910591", + "name": "Box Reports", + "sequence_id": "0", + "type": "folder" + }, + "path_collection": { + "entries": [ + { + "etag": null, + "id": "0", + "name": "All Files", + "sequence_id": null, + "type": "folder" + }, + { + "etag": "0", + "id": "166232910591", + "name": "Box Reports", + "sequence_id": "0", + "type": "folder" + } + ], + "total_count": 2 + }, + "purged_at": null, + "sequence_id": "0", + "shared_link": null, + "size": 0, + "synced": false, + "trashed_at": null, + "type": "folder" + }, + "type": "event" + } + ], + "next_stream_position": 1152922976252290800 + } + `}} + - path: /2.0/events + methods: [GET] + query_params: + stream_type: all + # This is a consequence of loss of exact representation of ints in floats. + # There is no good way to deal with this. The least worst approach to fix + # it without changing the cursor state store's type handling is to have + # a nextafter template helper. That is obviously a terrible solution. + stream_position: 1152922976252290816 + request_headers: + Authorization: + - "Bearer c3FIOG9vSGV4VHo4QzAyg5T1JvNnJoZ3ExaVNyQWw6WjRsanRKZG5lQk9qUE1BVQ" + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json; charset=utf-8" + body: |- + {{ minify_json ` + { + "chunk_size": 2, + "entries": [ + { + "source": null, + "created_by": { + "type": "user", + "id": "2", + "name": "Unknown User", + "login": "" + }, + "action_by": null, + "created_at": "2019-12-20T11:38:56-08:00", + "event_id": "97f1b31f-f143-4777-81f8-1b557b39ca32", + "event_type": "SHIELD_ALERT", + "ip_address": "10.1.2.3", + "type": "event", + "session_id": null, + "additional_details": { + "shield_alert": { + "rule_category": "Anomalous Download", + "rule_id": 123, + "rule_name": "Anomalous Download Rule", + "risk_score": 77, + "alert_summary": { + "description": "Significant increase in download content week over week, 9200% (25.04 MB) more than last week 12 additional files downloaded week over week)", + "download_delta_size": "25 Mb", + "download_delta_percent": 9200, + "historical_period": { + "date_range": { + "start_date": "2019-12-01T01:01:00-08:00", + "end_date": "2019-12-08T01:01:00-08:00" + }, + "download_size": "0 Mb", + "downloaded_files_count": 1 + }, + "anomaly_period": { + "date_range": { + "start_date": "2019-12-08T01:01:00-08:00", + "end_date": "2019-12-15T01:01:00-08:00" + }, + "download_size": "25 Mb", + "downloaded_files_count": 13 + }, + "download_ips": [ + { + "ip": "1.128.0.0" + }, + { + "ip": "175.16.199.0" + } + ] + }, + "alert_id": 444, + "priority": "medium", + "user": { + "id": 567, + "name": "Some user", + "email": "some@user.com" + }, + "link": "https://cloud.app.box.com/master/shield/alerts/444", + "created_at": "2019-12-20T11:38:16-08:00" + } + } + }, + { + "created_at": "2022-06-27T05:09:40-07:00", + "created_by": { + "id": "19530772260", + "login": "info@elastic.co", + "name": "Elastic Integrations", + "type": "user" + }, + "event_id": "e1cb161d5fbd3f3a80fd560f39a0f52a2cff3db8", + "event_type": "ITEM_CREATE", + "recorded_at": "2022-06-27T05:09:41-07:00", + "session_id": "rzraadh3n273zc5f", + "source": { + "content_created_at": "2022-06-27T05:09:40-07:00", + "content_modified_at": "2022-06-27T05:09:40-07:00", + "created_at": "2022-06-27T05:09:40-07:00", + "created_by": { + "id": "19530772260", + "login": "info@elastic.co", + "name": "Elastic Integrations", + "type": "user" + }, + "description": "", + "etag": "0", + "folder_upload_email": null, + "id": "166233012413", + "item_status": "active", + "modified_at": "2022-06-27T05:09:40-07:00", + "modified_by": { + "id": "19530772260", + "login": "info@elastic.co", + "name": "Elastic Integrations", + "type": "user" + }, + "name": "Platform App Diagnostics run on 2022-06-27 05-09-38 PDT", + "owned_by": { + "id": "19530772260", + "login": "info@elastic.co", + "name": "Elastic Integrations", + "type": "user" + }, + "parent": { + "etag": "0", + "id": "166232910591", + "name": "Box Reports", + "sequence_id": "0", + "type": "folder" + }, + "path_collection": { + "entries": [ + { + "etag": null, + "id": "0", + "name": "All Files", + "sequence_id": null, + "type": "folder" + }, + { + "etag": "0", + "id": "166232910591", + "name": "Box Reports", + "sequence_id": "0", + "type": "folder" + } + ], + "total_count": 2 + }, + "purged_at": null, + "sequence_id": "0", + "shared_link": null, + "size": 0, + "synced": false, + "trashed_at": null, + "type": "folder" + }, + "type": "event" + } + ], + "next_stream_position": 2152922976252290800 + } + `}} diff --git a/packages/box_events/changelog.yml b/packages/box_events/changelog.yml index f004c97ddd1..0bf79e943da 100644 --- a/packages/box_events/changelog.yml +++ b/packages/box_events/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.14.1" + changes: + - description: Fix handling of large cursor offsets. + type: bugfix + link: https://github.com/elastic/integrations/pull/14319 - version: "2.14.0" changes: - description: Update and expand ECS `user.*` field mappings. diff --git a/packages/box_events/data_stream/events/_dev/test/system/test-httpjson-config.yml b/packages/box_events/data_stream/events/_dev/test/system/test-httpjson-config.yml index 074f117a3e3..57578c3fb70 100644 --- a/packages/box_events/data_stream/events/_dev/test/system/test-httpjson-config.yml +++ b/packages/box_events/data_stream/events/_dev/test/system/test-httpjson-config.yml @@ -6,10 +6,11 @@ vars: client_secret: '8CN4J1ULy6pyR6XA6U8pAYm1CBUNONc7' box_subject_id: '19630872360' api_url: 'http://{{Hostname}}:{{Port}}' + enable_request_tracer: true # correspond to data_stream data_stream: vars: + interval: 10s stream_type: 'all' - enable_request_tracer: true assert: - hit_count: 2 + hit_count: 4 diff --git a/packages/box_events/data_stream/events/agent/stream/httpjson.yml.hbs b/packages/box_events/data_stream/events/agent/stream/httpjson.yml.hbs index 76d1684b005..8baa94aea60 100644 --- a/packages/box_events/data_stream/events/agent/stream/httpjson.yml.hbs +++ b/packages/box_events/data_stream/events/agent/stream/httpjson.yml.hbs @@ -17,7 +17,7 @@ request.method: "GET" request.transforms: - set: target: url.params.stream_position - value: '[[.cursor.next_stream_position]]' + value: '[[toInt .cursor.next_stream_position]]' {{#if stream_type}} - set: target: url.params.stream_type diff --git a/packages/box_events/manifest.yml b/packages/box_events/manifest.yml index 6830c63fdd9..d7cdd808cd5 100644 --- a/packages/box_events/manifest.yml +++ b/packages/box_events/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: box_events title: Box Events -version: "2.14.0" +version: "2.14.1" description: "Collect logs from Box with Elastic Agent" type: integration categories: