Merge branch 'main' into fix-login-401

ingestion/tests/cli_e2e/test_cli_vertica.py

@@ -74,7 +74,7 @@ def view_column_lineage_count(self) -> int:
         return 2
 
     def expected_lineage_node(self) -> str:
-        return "e2e_vertica.VMart.public.vendor_dimension_v"
+        return "e2e_vertica.VMart.public.vendor_dimension"
 
     @staticmethod
     def fqn_created_table() -> str:

openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md

@@ -46,6 +46,23 @@ You can refer to the following guide to get more details about the backup and re
   {% /inlineCallout %}
 {% /inlineCalloutContainer %}
 
+## Understanding the "Running" State in OpenMetadata
+
+In OpenMetadata, the **"Running"** state indicates that the OpenMetadata server has received a response from Airflow confirming that a workflow is in progress. However, if Airflow unexpectedly stops or crashes before it can send a failure status update through the **Failure Callback**, OpenMetadata remains unaware of the workflow’s actual state. As a result, the workflow may appear to be stuck in **"Running"** even though it is no longer executing.  
+
+This situation can also occur during an OpenMetadata upgrade. If an ingestion pipeline was running at the time of the upgrade and the process caused Airflow to shut down, OpenMetadata would not receive any further updates from Airflow. Consequently, the pipeline status remains **"Running"** indefinitely.
+
+{% image
+  src="/images/v1.6/deployment/upgrade/running-state-in-openmetadata.png"
+  alt="Running State in OpenMetadata"
+  caption="Running State in OpenMetadata" /%}
+
+### Expected Steps to Resolve  
+To resolve this issue:  
+- Ensure that Airflow is restarted properly after an unexpected shutdown.  
+- Manually update the pipeline status if necessary.  
+- Check Airflow logs to verify if the DAG execution was interrupted.  
+
 ### Update `sort_buffer_size` (MySQL) or `work_mem` (Postgres)
 
 Before running the migrations, it is important to update these parameters to ensure there are no runtime errors.

openmetadata-docs/content/partials/v1.7/deployment/upgrade/upgrade-prerequisites.md

@@ -46,6 +46,23 @@ You can refer to the following guide to get more details about the backup and re
   {% /inlineCallout %}
 {% /inlineCalloutContainer %}
 
+## Understanding the "Running" State in OpenMetadata
+
+In OpenMetadata, the **"Running"** state indicates that the OpenMetadata server has received a response from Airflow confirming that a workflow is in progress. However, if Airflow unexpectedly stops or crashes before it can send a failure status update through the **Failure Callback**, OpenMetadata remains unaware of the workflow’s actual state. As a result, the workflow may appear to be stuck in **"Running"** even though it is no longer executing.  
+
+This situation can also occur during an OpenMetadata upgrade. If an ingestion pipeline was running at the time of the upgrade and the process caused Airflow to shut down, OpenMetadata would not receive any further updates from Airflow. Consequently, the pipeline status remains **"Running"** indefinitely.
+
+{% image
+  src="/images/v1.7/deployment/upgrade/running-state-in-openmetadata.png"
+  alt="Running State in OpenMetadata"
+  caption="Running State in OpenMetadata" /%}
+
+### Expected Steps to Resolve
+To resolve this issue:  
+- Ensure that Airflow is restarted properly after an unexpected shutdown.  
+- Manually update the pipeline status if necessary.  
+- Check Airflow logs to verify if the DAG execution was interrupted.
+
 ### Update `sort_buffer_size` (MySQL) or `work_mem` (Postgres)
 
 Before running the migrations, it is important to update these parameters to ensure there are no runtime errors.

openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/processors/DataInsightsEntityEnricherProcessor.java

@@ -4,6 +4,7 @@
 import static org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils.END_TIMESTAMP_KEY;
 import static org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils.START_TIMESTAMP_KEY;
 import static org.openmetadata.service.apps.bundles.insights.workflows.dataAssets.DataAssetsWorkflow.ENTITY_TYPE_FIELDS_KEY;
+import static org.openmetadata.service.search.SearchIndexUtils.parseFollowers;
 import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_TYPE_KEY;
 import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.TIMESTAMP_KEY;
 import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats;
@@ -231,10 +232,8 @@ private Map<String, Object> enrichEntity(
     oCustomProperties.ifPresent(
         o -> entityMap.put(String.format("%sCustomProperty", entityType), o));
 
-    // Remove 'changeDescription' field
-    entityMap.remove("changeDescription");
-    // Remove 'sampleData'
-    entityMap.remove("sampleData");
+    // Parse Followers:
+    entityMap.put("followers", parseFollowers(entity.getFollowers()));
 
     return entityMap;
   }

openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java

@@ -61,7 +61,8 @@ public void jobToBeExecuted(JobExecutionContext jobExecutionContext) {
               .withTimestamp(jobStartTime)
               .withRunType(runType)
               .withStatus(AppRunRecord.Status.RUNNING)
-              .withScheduleInfo(jobApp.getAppSchedule());
+              .withScheduleInfo(jobApp.getAppSchedule())
+              .withConfig(JsonUtils.getMap(jobApp.getAppConfiguration()));
 
       boolean update = false;
       if (jobExecutionContext.isRecovering()) {

openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java

@@ -251,7 +251,7 @@ public ResultList<App> list(
                     mediaType = "application/json",
                     schema = @Schema(implementation = AppRunList.class)))
       })
-  public Response listAppRuns(
+  public ResultList<AppRunRecord> listAppRuns(
       @Context UriInfo uriInfo,
       @Context SecurityContext securityContext,
       @Parameter(description = "Name of the App", schema = @Schema(type = "string"))
@@ -281,9 +281,7 @@ public Response listAppRuns(
           Long endTs) {
     App installation = repository.getByName(uriInfo, name, repository.getFields("id,pipelines"));
     if (installation.getAppType().equals(AppType.Internal)) {
-      return Response.status(Response.Status.OK)
-          .entity(repository.listAppRuns(installation, limitParam, offset))
-          .build();
+      return repository.listAppRuns(installation, limitParam, offset);
     }
     if (!installation.getPipelines().isEmpty()) {
       EntityReference pipelineRef = installation.getPipelines().get(0);
@@ -292,13 +290,27 @@ public Response listAppRuns(
       IngestionPipeline ingestionPipeline =
           ingestionPipelineRepository.get(
               uriInfo, pipelineRef.getId(), ingestionPipelineRepository.getFields(FIELD_OWNERS));
-      return Response.ok(
-              ingestionPipelineRepository.listPipelineStatus(
-                  ingestionPipeline.getFullyQualifiedName(), startTs, endTs),
-              MediaType.APPLICATION_JSON_TYPE)
-          .build();
+      return ingestionPipelineRepository
+          .listPipelineStatus(ingestionPipeline.getFullyQualifiedName(), startTs, endTs)
+          .map(pipelineStatus -> convertPipelineStatus(installation, pipelineStatus));
     }
-    throw new IllegalArgumentException("App does not have an associated pipeline.");
+    throw new IllegalArgumentException("App does not have a scheduled deployment");
+  }
+
+  private static AppRunRecord convertPipelineStatus(App app, PipelineStatus pipelineStatus) {
+    return new AppRunRecord()
+        .withAppId(app.getId())
+        .withAppName(app.getName())
+        .withExecutionTime(pipelineStatus.getStartDate())
+        .withEndTime(pipelineStatus.getEndDate())
+        .withStatus(
+            switch (pipelineStatus.getPipelineState()) {
+              case QUEUED -> AppRunRecord.Status.PENDING;
+              case SUCCESS -> AppRunRecord.Status.SUCCESS;
+              case FAILED, PARTIAL_SUCCESS -> AppRunRecord.Status.FAILED;
+              case RUNNING -> AppRunRecord.Status.RUNNING;
+            })
+        .withConfig(pipelineStatus.getConfig());
   }
 
   @GET
@@ -617,7 +629,7 @@ public Response create(
     limits.enforceLimits(
         securityContext,
         getResourceContext(),
-        new OperationContext(Entity.APPLICATION, MetadataOperation.CREATE));
+        new OperationContext(APPLICATION, MetadataOperation.CREATE));
     if (SCHEDULED_TYPES.contains(app.getScheduleType())) {
       ApplicationHandler.getInstance()
           .installApplication(

openmetadata-service/src/main/java/org/openmetadata/service/util/ResultList.java

@@ -17,6 +17,8 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
 import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 import javax.validation.constraints.NotNull;
 import org.openmetadata.schema.system.EntityError;
 import org.openmetadata.schema.type.Paging;
@@ -95,6 +97,11 @@ public ResultList(List<T> data, Integer offset, int total) {
     paging = new Paging().withBefore(null).withAfter(null).withTotal(total).withOffset(offset);
   }
 
+  /* Conveniently map the data to another type without the need to create a new ResultList */
+  public <S> ResultList<S> map(Function<T, S> mapper) {
+    return new ResultList<>(data.stream().map(mapper).collect(Collectors.toList()), paging);
+  }
+
   public ResultList(List<T> data, Integer offset, Integer limit, Integer total) {
     this.data = data;
     paging =
@@ -106,6 +113,17 @@ public ResultList(List<T> data, Integer offset, Integer limit, Integer total) {
             .withLimit(limit);
   }
 
+  public ResultList(List<T> data, Paging other) {
+    this.data = data;
+    paging =
+        new Paging()
+            .withBefore(null)
+            .withAfter(null)
+            .withTotal(other.getTotal())
+            .withOffset(other.getOffset())
+            .withLimit(other.getLimit());
+  }
+
   public ResultList(
       List<T> data, List<EntityError> errors, String beforeCursor, String afterCursor, int total) {
     this.data = data;

openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json

@@ -34,7 +34,8 @@
         "active",
         "activeError",
         "stopped",
-        "success"
+        "success",
+        "pending"
       ]
     },
     "runType": {
@@ -63,6 +64,10 @@
     },
     "scheduleInfo": {
       "$ref": "./app.json#/definitions/appSchedule"
+    },
+    "config": {
+      "descripton": "The configuration used for this application run. It's type will be based on the application type. Old runs might not be compatible with schema of app configuration.",
+      "$ref": "../../type/basic.json#/definitions/map"
     }
   },
   "additionalProperties": false

openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json

@@ -43,6 +43,10 @@
         "status": {
           "description": "Ingestion Pipeline summary status. Informed at the end of the execution.",
           "$ref": "status.json#/definitions/ingestionStatus"
+        },
+        "config": {
+          "description": "Pipeline configuration for this particular execution.",
+          "$ref": "../../../type/basic.json#/definitions/map"
         }
       },
       "additionalProperties": false

openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts

@@ -381,7 +381,9 @@ test.describe('Activity feed', () => {
 
     // Task 1 - Resolved the task
 
+    const resolveTask2 = page.waitForResponse('/api/v1/feed/tasks/*/resolve');
     await page.getByText('Accept Suggestion').click();
+    await resolveTask2;
 
     await toastNotification(page, /Task resolved successfully/);
 

openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/AdvancedSearch.spec.ts

@@ -13,6 +13,10 @@
 import test from '@playwright/test';
 import { SidebarItem } from '../../constant/sidebar';
 import { EntityDataClass } from '../../support/entity/EntityDataClass';
+import { TableClass } from '../../support/entity/TableClass';
+import { Glossary } from '../../support/glossary/Glossary';
+import { GlossaryTerm } from '../../support/glossary/GlossaryTerm';
+import { UserClass } from '../../support/user/UserClass';
 import {
   FIELDS,
   OPERATOR,
@@ -28,6 +32,10 @@ test.describe.configure({
   timeout: 4 * 60 * 1000,
 });
 
+const user = new UserClass();
+const table = new TableClass(undefined, 'Regular');
+let glossaryEntity: Glossary;
+
 test.describe('Advanced Search', { tag: '@advanced-search' }, () => {
   // use the admin user to login
   test.use({ storageState: 'playwright/.auth/admin.json' });
@@ -37,6 +45,20 @@ test.describe('Advanced Search', { tag: '@advanced-search' }, () => {
   test.beforeAll('Setup pre-requests', async ({ browser }) => {
     const { page, apiContext, afterAction } = await createNewPage(browser);
     await EntityDataClass.preRequisitesForTests(apiContext);
+    await user.create(apiContext);
+    glossaryEntity = new Glossary(undefined, [
+      {
+        id: user.responseData.id,
+        type: 'user',
+        name: user.responseData.name,
+        displayName: user.responseData.displayName,
+      },
+    ]);
+    const glossaryTermEntity = new GlossaryTerm(glossaryEntity);
+
+    await glossaryEntity.create(apiContext);
+    await glossaryTermEntity.create(apiContext);
+    await table.create(apiContext);
 
     // Add Owner & Tag to the table
     await EntityDataClass.table1.visitEntityPage(page);
@@ -156,8 +178,8 @@ test.describe('Advanced Search', { tag: '@advanced-search' }, () => {
         EntityDataClass.topic1.entity.messageSchema.schemaFields[1].name,
       ],
       'dataModel.columns.name.keyword': [
-        EntityDataClass.dashboard1.dataModel.columns[0].name,
-        EntityDataClass.dashboard1.dataModel.columns[1].name,
+        EntityDataClass.container1.entity.dataModel.columns[0].name,
+        EntityDataClass.container2.entity.dataModel.columns[1].name,
       ],
       dataModelType: [
         EntityDataClass.dashboard1.dataModel.dataModelType,
@@ -178,14 +200,12 @@ test.describe('Advanced Search', { tag: '@advanced-search' }, () => {
       'responseSchema.schemaFields.name.keyword': [
         EntityDataClass.apiCollection1.apiEndpoint.responseSchema
           .schemaFields[0].name,
-        EntityDataClass.apiCollection1.apiEndpoint.responseSchema
-          .schemaFields[1].name,
+        'errors',
       ],
       'requestSchema.schemaFields.name.keyword': [
         EntityDataClass.apiCollection1.apiEndpoint.requestSchema.schemaFields[0]
           .name,
-        EntityDataClass.apiCollection1.apiEndpoint.requestSchema.schemaFields[1]
-          .name,
+        'photoUrls',
       ],
       'name.keyword': [
         EntityDataClass.table1.entity.name,
@@ -196,6 +216,12 @@ test.describe('Advanced Search', { tag: '@advanced-search' }, () => {
         EntityDataClass.dashboardDataModel2.entity.project,
       ],
       status: ['Approved', 'In Review'],
+      tableType: ['View', 'Regular'],
+      entityType: ['dashboard', 'pipeline'],
+      'charts.displayName.keyword': [
+        EntityDataClass.dashboard1.charts.displayName,
+        EntityDataClass.dashboard2.charts.displayName,
+      ],
     };
 
     await afterAction();
@@ -204,6 +230,9 @@ test.describe('Advanced Search', { tag: '@advanced-search' }, () => {
   test.afterAll('Cleanup', async ({ browser }) => {
     const { apiContext, afterAction } = await createNewPage(browser);
     await EntityDataClass.postRequisitesForTests(apiContext);
+    await glossaryEntity.delete(apiContext);
+    await user.delete(apiContext);
+    await table.delete(apiContext);
     await afterAction();
   });
 

openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Topic.spec.ts

@@ -45,6 +45,6 @@ test.describe('Topic entity specific tests ', () => {
   test('Topic page should show schema tab with count', async ({ page }) => {
     await topic.visitEntityPage(page);
 
-    await expect(page.getByRole('tab', { name: 'Schema' })).toContainText('1');
+    await expect(page.getByRole('tab', { name: 'Schema' })).toContainText('2');
   });
 });

openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ProfilerConfigurationPage.spec.ts

@@ -19,7 +19,7 @@ import { SidebarItem } from '../../constant/sidebar';
 import { AdminClass } from '../../support/user/AdminClass';
 import { UserClass } from '../../support/user/UserClass';
 import { performAdminLogin } from '../../utils/admin';
-import { redirectToHomePage } from '../../utils/common';
+import { redirectToHomePage, toastNotification } from '../../utils/common';
 import { sidebarClick } from '../../utils/sidebar';
 
 const user = new UserClass();
@@ -123,7 +123,8 @@ test.describe('Profiler Configuration Page', () => {
         );
       });
 
-      await expect(adminPage.getByRole('alert').first()).toHaveText(
+      await toastNotification(
+        adminPage,
         /Profiler Configuration updated successfully./
       );
     });

openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/SearchIndexApplication.spec.ts

@@ -90,6 +90,22 @@ test('Search Index Application', async ({ page }) => {
     await verifyLastExecutionRun(page);
   });
 
+  await test.step('View App Run Config', async () => {
+    await page.getByTestId('app-historical-config').click();
+    await page.waitForSelector('[role="dialog"].ant-modal');
+
+    await expect(page.locator('[role="dialog"].ant-modal')).toBeVisible();
+
+    await expect(page.locator('.ant-modal-title')).toContainText(
+      'Search Indexing Configuration'
+    );
+
+    await page.click('[data-testid="app-run-config-close"]');
+    await page.waitForSelector('[role="dialog"].ant-modal', {
+      state: 'detached',
+    });
+  });
+
   await test.step('Edit application', async () => {
     await page.click('[data-testid="edit-button"]');
     await page.waitForSelector('[data-testid="schedular-card-container"]');