feat(native): support custom schemas in native sidecar function registry

Author: Joe-Abraham

presto-native-execution/presto_cpp/main/PrestoServer.cpp

@@ -1669,6 +1669,18 @@ void PrestoServer::registerSidecarEndpoints() {
          proxygen::ResponseHandler* downstream) {
         http::sendOkResponse(downstream, getFunctionsMetadata());
       });
+  httpServer_->registerGet(
+      R"(/v1/functions/([^/]+))",
+      [](proxygen::HTTPMessage* /*message*/,
+         const std::vector<std::string>& pathMatch) {
+        return new http::CallbackRequestHandler(
+            [catalog = pathMatch[1]](
+                proxygen::HTTPMessage* /*message*/,
+                std::vector<std::unique_ptr<folly::IOBuf>>& /*body*/,
+                proxygen::ResponseHandler* downstream) {
+              http::sendOkResponse(downstream, getFunctionsMetadata(catalog));
+            });
+      });
   httpServer_->registerPost(
       "/v1/velox/plan",
       [server = this](

presto-native-execution/presto_cpp/main/connectors/CMakeLists.txt

@@ -20,4 +20,6 @@ if(PRESTO_ENABLE_ARROW_FLIGHT_CONNECTOR)
 endif()
 
 target_link_libraries(presto_connectors presto_velox_expr_conversion
-                      velox_type_fbhive)
+                      velox_type_fbhive presto_hive_functions)
+
+add_subdirectory(hive)

presto-native-execution/presto_cpp/main/connectors/PrestoToVeloxConnector.cpp

@@ -13,11 +13,16 @@
  */
 
 #include "presto_cpp/main/connectors/PrestoToVeloxConnector.h"
+#include <mutex>
+#include "presto_cpp/main/connectors/hive/functions/HiveFunctionRegistration.h"
+#include "presto_cpp/main/connectors/hive/functions/InitcapFunction.h"
+#include "presto_cpp/main/functions/dynamic_registry/DynamicFunctionRegistrar.h"
 #include "presto_cpp/main/types/PrestoToVeloxExpr.h"
 #include "presto_cpp/main/types/TypeParser.h"
 #include "presto_cpp/presto_protocol/connector/hive/HiveConnectorProtocol.h"
 #include "presto_cpp/presto_protocol/connector/iceberg/IcebergConnectorProtocol.h"
 #include "presto_cpp/presto_protocol/connector/tpch/TpchConnectorProtocol.h"
+#include "velox/functions/FunctionRegistry.h"
 
 #include <velox/type/fbhive/HiveTypeParser.h>
 #include "velox/connectors/hive/HiveConnector.h"
@@ -52,6 +57,14 @@ void registerPrestoToVeloxConnector(
       connectorName);
   protocol::registerConnectorProtocol(
       connectorName, std::move(connectorProtocol));
+
+  // Register hive-specific functions when hive catalog is detected.
+  // Delegate to generic Hive native function registrar which is idempotent.
+  if (connectorName ==
+          velox::connector::hive::HiveConnectorFactory::kHiveConnectorName ||
+      connectorName == std::string("hive-hadoop2")) {
+    hive::functions::registerHiveNativeFunctions();
+  }
 }
 
 void unregisterPrestoToVeloxConnector(const std::string& connectorName) {

presto-native-execution/presto_cpp/main/connectors/hive/CMakeLists.txt

@@ -0,0 +1,13 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_subdirectory(functions)

presto-native-execution/presto_cpp/main/connectors/hive/functions/CMakeLists.txt

@@ -0,0 +1,13 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+add_library(presto_hive_functions HiveFunctionRegistration.cpp)
+target_link_libraries(presto_hive_functions velox_functions_prestosql)

presto-native-execution/presto_cpp/main/connectors/hive/functions/HiveFunctionRegistration.cpp

@@ -0,0 +1,38 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "presto_cpp/main/connectors/hive/functions/HiveFunctionRegistration.h"
+
+#include <mutex>
+
+#include "presto_cpp/main/connectors/hive/functions/InitcapFunction.h"
+#include "presto_cpp/main/functions/dynamic_registry/DynamicFunctionRegistrar.h"
+#include "velox/functions/FunctionRegistry.h"
+
+namespace facebook::presto::hive::functions {
+
+namespace {
+void registerHiveFunctions() {
+  // Register functions under the 'hive.default' namespace.
+  facebook::presto::registerPrestoFunction<InitCapFunction, Varchar, Varchar>(
+      "initcap", "hive.default");
+}
+} // namespace
+
+void registerHiveNativeFunctions() {
+  static std::once_flag once;
+  std::call_once(once, []() { registerHiveFunctions(); });
+}
+
+} // namespace facebook::presto::hive::functions

presto-native-execution/presto_cpp/main/connectors/hive/functions/HiveFunctionRegistration.h

@@ -0,0 +1,23 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+namespace facebook::presto::hive::functions {
+
+// Registers Hive-specific native functions into the 'hive.default' namespace.
+// This method is safe to call multiple times; it performs one-time registration
+// guarded by an internal call_once.
+void registerHiveNativeFunctions();
+
+} // namespace facebook::presto::hive::functions

presto-native-execution/presto_cpp/main/connectors/hive/functions/InitcapFunction.h

@@ -0,0 +1,53 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "velox/functions/Macros.h"
+#include "velox/functions/lib/string/StringImpl.h"
+
+using namespace facebook::velox;
+using namespace facebook::velox::functions;
+namespace facebook::presto::hive::functions {
+
+/// The InitCapFunction capitalizes the first character of each word in a
+/// string, and lowercases the rest.
+template <typename T>
+struct InitCapFunction {
+  VELOX_DEFINE_FUNCTION_TYPES(T);
+
+  static constexpr bool is_default_ascii_behavior = true;
+
+  FOLLY_ALWAYS_INLINE void call(
+      out_type<Varchar>& result,
+      const arg_type<Varchar>& input) {
+    stringImpl::initcap<
+        /*strictSpace=*/false,
+        /*isAscii=*/false,
+        /*turkishCasing=*/true,
+        /*greekFinalSigma=*/true>(result, input);
+  }
+
+  FOLLY_ALWAYS_INLINE void callAscii(
+      out_type<Varchar>& result,
+      const arg_type<Varchar>& input) {
+    stringImpl::initcap<
+        /*strictSpace=*/false,
+        /*isAscii=*/true,
+        /*turkishCasing=*/true,
+        /*greekFinalSigma=*/true>(result, input);
+  }
+};
+
+} // namespace facebook::presto::hive::functions

presto-native-execution/presto_cpp/main/functions/FunctionMetadata.cpp

@@ -319,4 +319,66 @@ json getFunctionsMetadata() {
   return j;
 }
 
+json getFunctionsMetadata(const std::string& catalog) {
+  json j;
+
+  // Get metadata for all registered scalar functions in velox.
+  const auto signatures = getFunctionSignatures();
+  static const std::unordered_set<std::string> kBlockList = {
+      "row_constructor", "in", "is_null"};
+  // Exclude aggregate companion functions (extract aggregate companion
+  // functions are registered as vector functions).
+  const auto aggregateFunctions = exec::aggregateFunctions().copy();
+  for (const auto& entry : signatures) {
+    const auto name = entry.first;
+    // Skip internal functions. They don't have any prefix.
+    if (kBlockList.count(name) != 0 ||
+        name.find("$internal$") != std::string::npos ||
+        getScalarMetadata(name).companionFunction) {
+      continue;
+    }
+
+    const auto parts = getFunctionNameParts(name);
+    if (parts[0] != catalog) {
+      continue;
+    }
+    const auto schema = parts[1];
+    const auto function = parts[2];
+    j[function] = buildScalarMetadata(name, schema, entry.second);
+  }
+
+  // Get metadata for all registered aggregate functions in velox.
+  for (const auto& entry : aggregateFunctions) {
+    if (!aggregateFunctions.at(entry.first).metadata.companionFunction) {
+      const auto name = entry.first;
+      const auto parts = getFunctionNameParts(name);
+      if (parts[0] != catalog) {
+        continue;
+      }
+      const auto schema = parts[1];
+      const auto function = parts[2];
+      j[function] =
+          buildAggregateMetadata(name, schema, entry.second.signatures);
+    }
+  }
+
+  // Get metadata for all registered window functions in velox. Skip aggregates
+  // as they have been processed.
+  const auto& functions = exec::windowFunctions();
+  for (const auto& entry : functions) {
+    if (aggregateFunctions.count(entry.first) == 0) {
+      const auto name = entry.first;
+      const auto parts = getFunctionNameParts(entry.first);
+      if (parts[0] != catalog) {
+        continue;
+      }
+      const auto schema = parts[1];
+      const auto function = parts[2];
+      j[function] = buildWindowMetadata(name, schema, entry.second.signatures);
+    }
+  }
+
+  return j;
+}
+
 } // namespace facebook::presto

presto-native-execution/presto_cpp/main/functions/FunctionMetadata.h

@@ -21,4 +21,7 @@ namespace facebook::presto {
 // Returns metadata for all registered functions as json.
 nlohmann::json getFunctionsMetadata();
 
+// Returns metadata for all registered functions filtered by catalog as json.
+nlohmann::json getFunctionsMetadata(const std::string& catalog);
+
 } // namespace facebook::presto