entityanalytics_okta,okta: record okta domain into host.name in ingested documents (#13721)

This change affects two packages, entityanalytics_okta and okta in a
coordinated way.

entityanalytics_okta:

Rather than waiting for a stack release this adds the okta_domain value
via a beats processor, placing it in a location that is unlikely to
collide with data in the input's event data in the future. Move this to
host.name in ingest, falling back to reasonable location on failure.

okta:

This records the Okta Domain value from the Okta API URL provided by the
configuration. This is chosen over using the okta_domain value since
the URL is always present, so this simplifies the logic. The URL is
passed outside the event.original to avoid collision. This means that
it does not end up in the event.original. The actual Okta Domain is
obtain from the URL in the ingest pipeline and then placed gingerly in
host.name if possible, falling back to the same locations used in the
entityanalytics_okta integration in order to harmonise the two
integrations. The work to do this is done last in the pipeline to allow
an unlikely failure to not interfere with other parts of the pipeline,
but to provide an informative error in that case that that does
happen.

Author: efd6

packages/entityanalytics_okta/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.3.0"
+  changes:
+    - description: Retain Okta Domain value in `host.name` where possible.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13721
 - version: "2.2.1"
   changes:
     - description: Fix default request trace enabled behavior.

packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-device.json

@@ -96,6 +96,7 @@
                     }
                 ]
             },
+            "okta_domain": "trial-xxxxxxx-admin.okta.com",
             "device": {
                 "id": "guo4a5u7YAHhjXrMK0g4"
             },

packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-device.json-expected.json

@@ -71,11 +71,14 @@
                     "host"
                 ],
                 "kind": "asset",
-                "original": "{\"input\":{\"type\":\"entity-analytics\"},\"@timestamp\":\"2023-06-13T07:12:17.341Z\",\"ecs\":{\"version\":\"8.11.0\"},\"event\":{\"kind\":\"asset\"},\"device\":{\"id\":\"guo4a5u7YAHhjXrMK0g4\"},\"okta\":{\"resourceAlternateId\":null,\"lastUpdated\":\"2019-10-02T18:03:07.000Z\",\"resourceDisplayName\":{\"sensitive\":false,\"value\":\"Example Device name 1\"},\"resourceId\":\"guo4a5u7YAHhjXrMK0g4\",\"_links\":{\"activate\":{\"hints\":{\"allow\":[\"POST\"]},\"href\":\"https://localhost/api/v1/devices/guo4a5u7YAHhjXrMK0g4/lifecycle/activate\"},\"self\":{\"hints\":{\"allow\":[\"GET\",\"PATCH\",\"PUT\"]},\"href\":\"https://localhost/api/v1/devices/guo4a5u7YAHhjXrMK0g4\"},\"users\":{\"hints\":{\"allow\":[\"GET\"]},\"href\":\"https://localhost/api/v1/devices/guo4a5u7YAHhjXrMK0g4/users\"}},\"created\":\"2019-10-02T18:03:07.000Z\",\"profile\":{\"serialNumber\":\"XXDDRFCFRGF3M8MD6D\",\"displayName\":\"Example Device name 1\",\"registered\":true,\"diskEncryptionType\":\"ALL_INTERNAL_VOLUMES\",\"secureHardwarePresent\":false,\"platform\":\"WINDOWS\",\"sid\":\"S-1-11-111\"},\"id\":\"guo4a5u7YAHhjXrMK0g4\",\"users\":[{\"lastLogin\":\"2013-06-24T17:39:19.000Z\",\"lastUpdated\":\"2013-06-27T16:35:28.000Z\",\"passwordChanged\":\"2013-06-24T16:39:19.000Z\",\"credentials\":{\"provider\":{\"name\":\"OKTA\",\"type\":\"OKTA\"},\"recovery_question\":{}},\"created\":\"2013-06-24T16:39:18.000Z\",\"profile\":{\"profileUrl\":\"http://www.example.com/profile\",\"lastName\":\"Brock\",\"zipCode\":\"94107\",\"preferredLanguage\":\"en-US\",\"city\":\"San Francisco\",\"costCenter\":\"10\",\"displayName\":\"Isaac Brock\",\"nickName\":\"issac\",\"secondEmail\":\"[email protected]\",\"login\":\"[email protected]\",\"title\":\"Director\",\"employeeNumber\":\"187\",\"division\":\"R&D\",\"firstName\":\"Isaac\",\"primaryPhone\":\"+1-555-514-1337\",\"mobilePhone\":\"+1-555-415-1337\",\"streetAddress\":\"301 Brannan St.\",\"countryCode\":\"US\",\"organization\":\"Okta\",\"state\":\"CA\",\"userType\":\"Employee\",\"department\":\"Engineering\",\"email\":\"[email protected]\"},\"statusChanged\":\"2013-06-24T16:39:19.000Z\",\"id\":\"00ub0oNGTSWTBKOLGLNR\",\"activated\":\"2013-06-24T16:39:19.000Z\",\"status\":\"ACTIVE\"}],\"resourceType\":\"UDDevice\",\"status\":\"ACTIVE\"},\"tags\":[\"preserve_original_event\",\"preserve_duplicate_custom_fields\"],\"_index\":\"logs-entityanalytics_okta.entity-default\",\"_id\":\"_id\",\"_version\":-3}",
+                "original": "{\"input\":{\"type\":\"entity-analytics\"},\"@timestamp\":\"2023-06-13T07:12:17.341Z\",\"ecs\":{\"version\":\"8.11.0\"},\"okta_domain\":\"trial-xxxxxxx-admin.okta.com\",\"event\":{\"kind\":\"asset\"},\"device\":{\"id\":\"guo4a5u7YAHhjXrMK0g4\"},\"okta\":{\"resourceAlternateId\":null,\"lastUpdated\":\"2019-10-02T18:03:07.000Z\",\"resourceDisplayName\":{\"sensitive\":false,\"value\":\"Example Device name 1\"},\"resourceId\":\"guo4a5u7YAHhjXrMK0g4\",\"_links\":{\"activate\":{\"hints\":{\"allow\":[\"POST\"]},\"href\":\"https://localhost/api/v1/devices/guo4a5u7YAHhjXrMK0g4/lifecycle/activate\"},\"self\":{\"hints\":{\"allow\":[\"GET\",\"PATCH\",\"PUT\"]},\"href\":\"https://localhost/api/v1/devices/guo4a5u7YAHhjXrMK0g4\"},\"users\":{\"hints\":{\"allow\":[\"GET\"]},\"href\":\"https://localhost/api/v1/devices/guo4a5u7YAHhjXrMK0g4/users\"}},\"created\":\"2019-10-02T18:03:07.000Z\",\"profile\":{\"serialNumber\":\"XXDDRFCFRGF3M8MD6D\",\"displayName\":\"Example Device name 1\",\"registered\":true,\"diskEncryptionType\":\"ALL_INTERNAL_VOLUMES\",\"secureHardwarePresent\":false,\"platform\":\"WINDOWS\",\"sid\":\"S-1-11-111\"},\"id\":\"guo4a5u7YAHhjXrMK0g4\",\"users\":[{\"lastLogin\":\"2013-06-24T17:39:19.000Z\",\"lastUpdated\":\"2013-06-27T16:35:28.000Z\",\"passwordChanged\":\"2013-06-24T16:39:19.000Z\",\"credentials\":{\"provider\":{\"name\":\"OKTA\",\"type\":\"OKTA\"},\"recovery_question\":{}},\"created\":\"2013-06-24T16:39:18.000Z\",\"profile\":{\"profileUrl\":\"http://www.example.com/profile\",\"lastName\":\"Brock\",\"zipCode\":\"94107\",\"preferredLanguage\":\"en-US\",\"city\":\"San Francisco\",\"costCenter\":\"10\",\"displayName\":\"Isaac Brock\",\"nickName\":\"issac\",\"secondEmail\":\"[email protected]\",\"login\":\"[email protected]\",\"title\":\"Director\",\"employeeNumber\":\"187\",\"division\":\"R&D\",\"firstName\":\"Isaac\",\"primaryPhone\":\"+1-555-514-1337\",\"mobilePhone\":\"+1-555-415-1337\",\"streetAddress\":\"301 Brannan St.\",\"countryCode\":\"US\",\"organization\":\"Okta\",\"state\":\"CA\",\"userType\":\"Employee\",\"department\":\"Engineering\",\"email\":\"[email protected]\"},\"statusChanged\":\"2013-06-24T16:39:19.000Z\",\"id\":\"00ub0oNGTSWTBKOLGLNR\",\"activated\":\"2013-06-24T16:39:19.000Z\",\"status\":\"ACTIVE\"}],\"resourceType\":\"UDDevice\",\"status\":\"ACTIVE\"},\"tags\":[\"preserve_original_event\",\"preserve_duplicate_custom_fields\"],\"_index\":\"logs-entityanalytics_okta.entity-default\",\"_id\":\"_id\",\"_version\":-3}",
                 "type": [
                     "info"
                 ]
             },
+            "host": {
+                "name": "trial-xxxxxxx-admin.okta.com"
+            },
             "input": {
                 "type": "entity-analytics"
             },

packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-user.json

@@ -44,6 +44,7 @@
                     "recovery_question": {}
                 }
             },
+            "okta_domain": "trial-xxxxxxx-admin.okta.com",
             "groups": [
                 {
                     "id": "OGYzMDMwYjFmODBiNjli",

packages/entityanalytics_okta/data_stream/entity/_dev/test/pipeline/test-user.json-expected.json

@@ -83,12 +83,15 @@
                     "iam"
                 ],
                 "kind": "asset",
-                "original": "{\"input\":{\"type\":\"entity-analytics\"},\"@timestamp\":\"2023-06-13T07:12:17.341Z\",\"ecs\":{\"version\":\"8.11.0\"},\"groups\":[{\"profile\":{\"name\":\"Everyone\",\"description\":\"All users in your organization\"},\"id\":\"OGYzMDMwYjFmODBiNjli\"}],\"event\":{\"kind\":\"asset\"},\"okta\":{\"lastLogin\":\"2013-06-24T17:39:19.000Z\",\"lastUpdated\":\"2013-06-27T16:35:28.000Z\",\"passwordChanged\":\"2013-06-24T16:39:19.000Z\",\"credentials\":{\"provider\":{\"name\":\"OKTA\",\"type\":\"OKTA\"},\"recovery_question\":{}},\"created\":\"2013-06-24T16:39:18.000Z\",\"profile\":{\"profileUrl\":\"http://www.example.com/profile\",\"lastName\":\"Brock\",\"zipCode\":\"94107\",\"preferredLanguage\":\"en-US\",\"city\":\"San Francisco\",\"costCenter\":\"10\",\"displayName\":\"Isaac Brock\",\"nickName\":\"issac\",\"secondEmail\":\"[email protected]\",\"login\":\"[email protected]\",\"title\":\"Director\",\"employeeNumber\":\"187\",\"division\":\"R&D\",\"firstName\":\"Isaac\",\"primaryPhone\":\"+1-555-514-1337\",\"mobilePhone\":\"+1-555-415-1337\",\"streetAddress\":\"301 Brannan St.\",\"countryCode\":\"US\",\"organization\":\"Okta\",\"state\":\"CA\",\"userType\":\"Employee\",\"department\":\"Engineering\",\"email\":\"[email protected]\"},\"statusChanged\":\"2013-06-24T16:39:19.000Z\",\"id\":\"00ub0oNGTSWTBKOLGLNR\",\"activated\":\"2013-06-24T16:39:19.000Z\",\"status\":\"ACTIVE\"},\"user\":{\"id\":\"00u5tvodynDjUCNKn697\"},\"tags\":[\"preserve_original_event\",\"preserve_duplicate_custom_fields\"],\"_index\":\"logs-entityanalytics_okta.entity-default\",\"_id\":\"_id\",\"_version\":-3}",
+                "original": "{\"input\":{\"type\":\"entity-analytics\"},\"@timestamp\":\"2023-06-13T07:12:17.341Z\",\"ecs\":{\"version\":\"8.11.0\"},\"groups\":[{\"profile\":{\"name\":\"Everyone\",\"description\":\"All users in your organization\"},\"id\":\"OGYzMDMwYjFmODBiNjli\"}],\"okta_domain\":\"trial-xxxxxxx-admin.okta.com\",\"event\":{\"kind\":\"asset\"},\"okta\":{\"lastLogin\":\"2013-06-24T17:39:19.000Z\",\"lastUpdated\":\"2013-06-27T16:35:28.000Z\",\"passwordChanged\":\"2013-06-24T16:39:19.000Z\",\"credentials\":{\"provider\":{\"name\":\"OKTA\",\"type\":\"OKTA\"},\"recovery_question\":{}},\"created\":\"2013-06-24T16:39:18.000Z\",\"profile\":{\"profileUrl\":\"http://www.example.com/profile\",\"lastName\":\"Brock\",\"zipCode\":\"94107\",\"preferredLanguage\":\"en-US\",\"city\":\"San Francisco\",\"costCenter\":\"10\",\"displayName\":\"Isaac Brock\",\"nickName\":\"issac\",\"secondEmail\":\"[email protected]\",\"login\":\"[email protected]\",\"title\":\"Director\",\"employeeNumber\":\"187\",\"division\":\"R&D\",\"firstName\":\"Isaac\",\"primaryPhone\":\"+1-555-514-1337\",\"mobilePhone\":\"+1-555-415-1337\",\"streetAddress\":\"301 Brannan St.\",\"countryCode\":\"US\",\"organization\":\"Okta\",\"state\":\"CA\",\"userType\":\"Employee\",\"department\":\"Engineering\",\"email\":\"[email protected]\"},\"statusChanged\":\"2013-06-24T16:39:19.000Z\",\"id\":\"00ub0oNGTSWTBKOLGLNR\",\"activated\":\"2013-06-24T16:39:19.000Z\",\"status\":\"ACTIVE\"},\"user\":{\"id\":\"00u5tvodynDjUCNKn697\"},\"tags\":[\"preserve_original_event\",\"preserve_duplicate_custom_fields\"],\"_index\":\"logs-entityanalytics_okta.entity-default\",\"_id\":\"_id\",\"_version\":-3}",
                 "type": [
                     "user",
                     "info"
                 ]
             },
+            "host": {
+                "name": "trial-xxxxxxx-admin.okta.com"
+            },
             "input": {
                 "type": "entity-analytics"
             },

packages/entityanalytics_okta/data_stream/entity/agent/stream/entity-analytics.yml.hbs

@@ -33,7 +33,11 @@ tags:
 {{#contains "forwarded" tags}}
 publisher_pipeline.disable_host: true
 {{/contains}}
-{{#if processors}}
 processors:
+{{#if processors}}
 {{processors}}
 {{/if}}
+- add_fields:
+    target: ''
+    fields:
+      okta_domain: {{okta_domain}}

packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/default.yml

@@ -23,6 +23,15 @@ processors:
       name: '{{ IngestPipeline "device" }}'
       tag: pipeline_to_device
       if: ctx.device?.id != null
+  - rename:
+      field: okta_domain
+      target_field: host.name
+      ignore_missing: true
+      on_failure:
+        - rename:
+            field: okta_domain
+            target_field: okta.okta_domain
+            ignore_failure: true
 
   - script:
       lang: painless

packages/entityanalytics_okta/data_stream/entity/sample_event.json

@@ -1,38 +1,41 @@
 {
-    "@timestamp": "2025-02-17T01:32:37.018Z",
+    "@timestamp": "2025-04-29T23:45:56.269Z",
     "agent": {
-        "ephemeral_id": "5565e14c-c3d1-4168-9860-fb280f704fad",
-        "id": "f1b6848f-87f5-4d0e-8dae-49fb70d285f6",
-        "name": "elastic-agent-11615",
+        "ephemeral_id": "039aec05-a412-489e-a506-5f4fdf78b738",
+        "id": "31bbbbfc-763f-4e44-959f-8e97a01f2dbd",
+        "name": "elastic-agent-68980",
         "type": "filebeat",
         "version": "8.15.0"
     },
     "data_stream": {
         "dataset": "entityanalytics_okta.entity",
-        "namespace": "71124",
+        "namespace": "87097",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "f1b6848f-87f5-4d0e-8dae-49fb70d285f6",
+        "id": "31bbbbfc-763f-4e44-959f-8e97a01f2dbd",
         "snapshot": false,
         "version": "8.15.0"
     },
     "event": {
         "action": "started",
         "agent_id_status": "verified",
         "dataset": "entityanalytics_okta.entity",
-        "ingested": "2025-02-17T01:32:38Z",
+        "ingested": "2025-04-29T23:45:57Z",
         "kind": "asset",
-        "start": "2025-02-17T01:32:37.018Z"
+        "start": "2025-04-29T23:45:56.269Z"
+    },
+    "host": {
+        "name": "trial-xxxxxxx-admin.okta.com"
     },
     "input": {
         "type": "entity-analytics"
     },
     "labels": {
-        "identity_source": "entity-analytics-entityanalytics_okta.entity-e600b1a8-23ab-4aa5-9694-d245bc06b6ed"
+        "identity_source": "entity-analytics-entityanalytics_okta.entity-aa7bbba7-39b6-4bad-80f5-6ed793be029b"
     },
     "tags": [
         "preserve_original_event",

packages/entityanalytics_okta/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: entityanalytics_okta
 title: Okta Entity Analytics
-version: "2.2.1"
+version: "2.3.0"
 description: "Collect Identities from Okta with Elastic Agent."
 type: integration
 categories:

packages/okta/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.7.0"
+  changes:
+    - description: Retain Okta Domain value in `host.name` where possible.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13721
 - version: "3.6.1"
   changes:
     - description: Fix default request trace enabled behavior.