fix(ge_profiler): support nonnull_count for complex types (#14631)

Author: ligfx

metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py

@@ -307,7 +307,6 @@ def _is_single_row_query_method(query: Any) -> bool:
         "get_column_max",
         "get_column_mean",
         "get_column_stdev",
-        "get_column_nonnull_count",
         "get_column_unique_count",
     }
     CONSTANT_ROW_QUERY_METHODS = {
@@ -331,6 +330,7 @@ def _is_single_row_query_method(query: Any) -> bool:
 
     FIRST_PARTY_SINGLE_ROW_QUERY_METHODS = {
         "get_column_unique_count_dh_patch",
+        "_get_column_cardinality",
     }
 
     # We'll do this the inefficient way since the arrays are pretty small.
@@ -497,7 +497,20 @@ def _get_column_cardinality(
         self, column_spec: _SingleColumnSpec, column: str
     ) -> None:
         try:
-            nonnull_count = self.dataset.get_column_nonnull_count(column)
+            # Don't use Great Expectations get_column_nonnull_count because it
+            # generates this SQL:
+            #
+            #   sum(CASE WHEN (mycolumn IN (NULL) OR mycolumn IS NULL) THEN 1 ELSE 0 END)
+            #
+            # which fails for complex types (such as Databricks maps) that don't
+            # support the IN operator.
+            nonnull_count = convert_to_json_serializable(
+                self.dataset.engine.execute(
+                    sa.select(sa.func.count(sa.column(column))).select_from(
+                        self.dataset._table
+                    )
+                ).scalar()
+            )
             column_spec.nonnull_count = nonnull_count
         except Exception as e:
             logger.debug(