diff --git a/sdk-gen/subprojects/java/api/build.gradle b/sdk-gen/subprojects/java/api/build.gradle index 4583d223..7063b9f2 100644 --- a/sdk-gen/subprojects/java/api/build.gradle +++ b/sdk-gen/subprojects/java/api/build.gradle @@ -19,6 +19,7 @@ dependencies { runtimeOnly group: 'org.apache.arrow', name: 'arrow-memory-netty', version: project['arrow.version'], transitive: false runtimeOnly group: 'org.apache.arrow', name: 'arrow-memory-netty-buffer-patch', version: project['arrow.version'], transitive: false // End of flight-core + api project(':wdp-connect-sdk-gen-java-sdk-connector-api') api project(':wdp-connect-sdk-gen-java-api-models') api group: 'org.apache.commons', name: 'commons-pool2', version: project['commons.pool2.version'] api group: 'org.apache.commons', name: 'commons-text', version: project['commons.text.version'] diff --git a/sdk-gen/subprojects/java/api/src/main/java/com/ibm/connect/sdk/api/AbstractSdkConnectorFlightProducer.java b/sdk-gen/subprojects/java/api/src/main/java/com/ibm/connect/sdk/api/AbstractSdkConnectorFlightProducer.java new file mode 100644 index 00000000..695bc90a --- /dev/null +++ b/sdk-gen/subprojects/java/api/src/main/java/com/ibm/connect/sdk/api/AbstractSdkConnectorFlightProducer.java @@ -0,0 +1,93 @@ +/* *************************************************** */ +/* */ +/* (C) Copyright IBM Corp. 2026 */ +/* */ +/* *************************************************** */ +package com.ibm.connect.sdk.api; + +import java.util.Collections; + +import com.ibm.wdp.connect.common.sdk.api.models.ConnectionProperties; +import com.ibm.wdp.connect.common.sdk.api.models.CustomFlightDatasourceTypes; +import com.ibm.wdp.connect.sdk.connector.SdkConnectorFactory; + +/** + * Convenience base class for Flight producers that use {@link SdkConnectorFactory}. + * + *

Subclasses implement only {@link #getSdkConnectorFactory()} — the old + * {@link ConnectorFactory} path is satisfied by a no-op factory whose datasource types + * are derived from the SDK factory. The SDK connector path is active whenever + * {@link #getSdkConnectorFactory()} returns non-null. + * + *

Usage: + * 

+ *   public class MyFlightProducer extends AbstractSdkConnectorFlightProducer {
+ *       {@literal @}Override
+ *       protected SdkConnectorFactory getSdkConnectorFactory() {
+ *           return MyConnectorFactory.getInstance();
+ *       }
+ *   }
+ *
+ */ +public abstract class AbstractSdkConnectorFlightProducer extends ConnectorFlightProducer +{ + /** + * {@inheritDoc} + * + *

Returns a no-op {@link ConnectorFactory} whose datasource types are derived from + * the SDK connector factory. The old connector path is never invoked when + * {@link #getSdkConnectorFactory()} returns non-null. + */ + @Override + protected ConnectorFactory getConnectorFactory() + { + return new NoOpConnectorFactory(getSdkConnectorFactory()); + } + + /** + * {@inheritDoc} + */ + @Override + protected abstract SdkConnectorFactory getSdkConnectorFactory(); + + // ---- inner class ---- + + /** + * A no-op {@link ConnectorFactory} that exposes the datasource types from the SDK factory + * but throws {@link UnsupportedOperationException} on {@link #createConnector}. + * + *

This satisfies the {@link ConnectorFlightProducer} constructor requirement. + * It is never actually called because the SDK connector path short-circuits all operations + * when {@code sdkConnectorFactory != null}. + */ + private static final class NoOpConnectorFactory extends PooledConnectorFactory + { + private final SdkConnectorFactory sdkFactory; + + NoOpConnectorFactory(SdkConnectorFactory sdkFactory) + { + super(); + this.sdkFactory = sdkFactory; + } + + @Override + public CustomFlightDatasourceTypes getDatasourceTypes() + { + final CustomFlightDatasourceTypes types = new CustomFlightDatasourceTypes(); + // Return empty list — datasource type listing for SDK producers goes through + // doAction(ACTION_LIST_DATASOURCE_TYPES) which uses connectorFactory.getDatasourceTypes(). + // SDK producers that want to expose types should override doAction or provide + // their own CustomFlightDatasourceTypes mapping. + types.setDatasourceTypes(Collections.emptyList()); + return types; + } + + @Override + protected Connector createNewConnector(String datasourceTypeName, ConnectionProperties properties) + { + throw new UnsupportedOperationException( + "NoOpConnectorFactory.createNewConnector should never be called when sdkConnectorFactory is set. " + + "datasourceTypeName=" + datasourceTypeName); + } + } +} diff --git a/sdk-gen/subprojects/java/api/src/main/java/com/ibm/connect/sdk/api/ConnectorFlightProducer.java b/sdk-gen/subprojects/java/api/src/main/java/com/ibm/connect/sdk/api/ConnectorFlightProducer.java index b5ee712f..ae207236 100644 --- a/sdk-gen/subprojects/java/api/src/main/java/com/ibm/connect/sdk/api/ConnectorFlightProducer.java +++ b/sdk-gen/subprojects/java/api/src/main/java/com/ibm/connect/sdk/api/ConnectorFlightProducer.java @@ -9,6 +9,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; @@ -49,6 +50,20 @@ import com.ibm.wdp.connect.common.sdk.api.models.CustomFlightAssetsCriteria; import com.ibm.wdp.connect.common.sdk.api.models.CustomFlightDatasourceType; import com.ibm.wdp.connect.common.sdk.api.models.CustomFlightDatasourceTypes; +import com.ibm.wdp.connect.sdk.connector.ArrowBatchReader; +import com.ibm.wdp.connect.sdk.connector.ArrowBatchWriter; +import com.ibm.wdp.connect.sdk.connector.AssetDescriptor; +import com.ibm.wdp.connect.sdk.connector.ColumnarArrowBatchReader; +import com.ibm.wdp.connect.sdk.connector.ColumnarArrowBatchWriter; +import com.ibm.wdp.connect.sdk.connector.ConnectionProperties; +import com.ibm.wdp.connect.sdk.connector.DiscoveryCriteria; +import com.ibm.wdp.connect.sdk.connector.SdkColumnarSourceInteraction; +import com.ibm.wdp.connect.sdk.connector.SdkColumnarTargetInteraction; +import com.ibm.wdp.connect.sdk.connector.SdkConnector; +import com.ibm.wdp.connect.sdk.connector.SdkConnectorFactory; +import com.ibm.wdp.connect.sdk.connector.SdkDiscoveryInteraction; +import com.ibm.wdp.connect.sdk.connector.SdkSourceInteraction; +import com.ibm.wdp.connect.sdk.connector.SdkTargetInteraction; /** * An abstract Flight producer for connectors. @@ -58,6 +73,7 @@ public abstract class ConnectorFlightProducer implements FlightProducer private static final Logger LOGGER = getLogger(ConnectorFlightProducer.class); private static final String UNKNOWN_VERSION = "unknown"; + private static final int DEFAULT_BATCH_SIZE = 1000; /** * Action type to check the health of the service and return its version. @@ -103,10 +119,15 @@ public abstract class ConnectorFlightProducer implements FlightProducer private final CustomFlightDatasourceTypes datasourceTypes; /** - * A factory for creating connectors. + * A factory for creating connectors (old path). */ private final ConnectorFactory connectorFactory; + /** + * A factory for creating SDK-style connectors (new path); null when not overridden. + */ + private final SdkConnectorFactory sdkConnectorFactory; + private final ModelMapper modelMapper; private final FlightDescriptorCache descriptorCache; private final BufferAllocator rootAllocator; @@ -117,6 +138,7 @@ public abstract class ConnectorFlightProducer implements FlightProducer public ConnectorFlightProducer() { connectorFactory = getConnectorFactory(); + sdkConnectorFactory = getSdkConnectorFactory(); datasourceTypes = connectorFactory.getDatasourceTypes(); modelMapper = new ModelMapper(); descriptorCache = new FlightDescriptorCache(); @@ -130,6 +152,21 @@ public ConnectorFlightProducer() */ abstract protected ConnectorFactory getConnectorFactory(); + /** + * Returns a factory for creating SDK-style connectors, or null if not used. + * + *

Subclasses that want to use the SDK connector path should override this method + * to return their {@link SdkConnectorFactory}. When non-null, all Flight operations + * (getStream, getFlightInfo, acceptPut, listFlights) will use the SDK path instead of + * the legacy {@link ConnectorFactory} path. + * + * @return an {@link SdkConnectorFactory}, or null (default) + */ + protected SdkConnectorFactory getSdkConnectorFactory() + { + return null; + } + /** * {@inheritDoc} */ @@ -145,6 +182,34 @@ public void getStream(CallContext context, Ticket ticket, ServerStreamListener l throw new IllegalArgumentException(ApiMsgs.NO_FLIGHT_DESCRIPTOR_FOR_TICKET.format()); } final CustomFlightAssetDescriptor asset = modelMapper.fromBytes(descriptor.getCommand(), CustomFlightAssetDescriptor.class); + + // --- SDK connector path --- + if (sdkConnectorFactory != null) { + final AssetDescriptor assetDescriptor = toAssetDescriptor(asset); + try (SdkConnector connector = sdkConnectorFactory.createConnector( + asset.getDatasourceTypeName(), toConnectionProperties(asset.getConnectionProperties()))) { + connector.connect(); + try (SdkSourceInteraction interaction = connector.getSourceInteraction(assetDescriptor, ticket)) { + final Schema schema = interaction.getSchema(); + if (interaction instanceof SdkColumnarSourceInteraction) { + final int batchSize = getBatchSize(asset); + try (ColumnarArrowBatchWriter writer = new ColumnarArrowBatchWriter(schema, rootAllocator, batchSize)) { + ((SdkColumnarSourceInteraction) interaction).stream(writer); + streamBatches(writer.batches(), schema, listener, bpStrategy); + } + } else { + final int batchSize = getBatchSize(asset); + try (ArrowBatchWriter writer = new ArrowBatchWriter(schema, rootAllocator, batchSize)) { + interaction.stream(writer); + streamBatches(writer.batches(), schema, listener, bpStrategy); + } + } + } + } + return; + } + + // --- Legacy connector path --- try (Connector connector = connectorFactory.createConnector(asset.getDatasourceTypeName(), asset.getConnectionProperties())) { connector.connect(); @@ -201,6 +266,33 @@ public void listFlights(CallContext context, Criteria criteria, StreamListener connector = sdkConnectorFactory.createConnector( + assetsCriteria.getDatasourceTypeName(), toConnectionProperties(assetsCriteria.getConnectionProperties()))) { + connector.connect(); + try (SdkDiscoveryInteraction discovery = connector.getDiscoveryInteraction(discoveryCriteria)) { + final List sdkAssets + = discovery.discoverAssets(discoveryCriteria); + for (final AssetDescriptor sdkAsset : sdkAssets) { + final CustomFlightAssetDescriptor asset = fromAssetDescriptor(sdkAsset, + assetsCriteria.getDatasourceTypeName(), + assetsCriteria.getConnectionProperties()); + completeAsset(asset); + final FlightDescriptor flightDescriptor = FlightDescriptor.command(modelMapper.toBytes(asset)); + final Schema schema = connector.getSchema(sdkAsset); + final FlightInfo flightInfo = createFlightInfo(flightDescriptor, schema, Collections.emptyList()); + listener.onNext(flightInfo); + } + } + } + listener.onCompleted(); + return; + } + + // --- Legacy connector path --- try (Connector connector = connectorFactory.createConnector(assetsCriteria.getDatasourceTypeName(), assetsCriteria.getConnectionProperties())) { connector.connect(); @@ -245,6 +337,23 @@ public FlightInfo getFlightInfo(CallContext context, FlightDescriptor descriptor try { ThreadLocale.setLocale(context); final CustomFlightAssetDescriptor asset = modelMapper.fromBytes(descriptor.getCommand(), CustomFlightAssetDescriptor.class); + + // --- SDK connector path --- + if (sdkConnectorFactory != null) { + final AssetDescriptor assetDescriptor = toAssetDescriptor(asset); + try (SdkConnector connector = sdkConnectorFactory.createConnector( + asset.getDatasourceTypeName(), toConnectionProperties(asset.getConnectionProperties()))) { + connector.connect(); + try (SdkSourceInteraction interaction = connector.getSourceInteraction(assetDescriptor, null)) { + final Schema schema = interaction.getSchema(); + asset.setFields(Utils.getAssetFields(schema)); + final List tickets = interaction.getTickets(); + return createFlightInfo(FlightDescriptor.command(modelMapper.toBytes(asset)), schema, tickets); + } + } + } + + // --- Legacy connector path --- try (Connector connector = connectorFactory.createConnector(asset.getDatasourceTypeName(), asset.getConnectionProperties())) { connector.connect(); @@ -289,6 +398,41 @@ public Runnable acceptPut(CallContext context, FlightStream flightStream, Stream throw new IllegalArgumentException(ApiMsgs.MISSING_PARTITION_INDEX.format()); } asset.setFields(Utils.getAssetFields(flightStream.getSchema())); + + // --- SDK connector path --- + if (sdkConnectorFactory != null) { + final AssetDescriptor assetDescriptor = toAssetDescriptor(asset); + try (SdkConnector connector = sdkConnectorFactory.createConnector( + asset.getDatasourceTypeName(), toConnectionProperties(asset.getConnectionProperties()))) { + connector.connect(); + try (SdkTargetInteraction interaction = connector.getTargetInteraction(assetDescriptor)) { + if (asset.getPartitionCount() == null || asset.getPartitionCount() == 1) { + interaction.setup(); + } + // Collect all incoming batches from the FlightStream + final List batches = new ArrayList<>(); + while (flightStream.next()) { + batches.add(flightStream.getRoot()); + } + if (interaction instanceof SdkColumnarTargetInteraction) { + try (ColumnarArrowBatchReader reader = new ColumnarArrowBatchReader(batches)) { + ((SdkColumnarTargetInteraction) interaction).consume(reader); + } + } else { + try (ArrowBatchReader reader = new ArrowBatchReader(batches)) { + interaction.consume(reader); + } + } + if (asset.getPartitionCount() == null || asset.getPartitionCount() == 1) { + interaction.wrapup(); + } + } + } + ackStream.onCompleted(); + return; + } + + // --- Legacy connector path --- try (Connector connector = connectorFactory.createConnector(asset.getDatasourceTypeName(), asset.getConnectionProperties())) { connector.connect(); @@ -446,4 +590,98 @@ public void listActions(CallContext context, StreamListener listener } } + // ---- private helpers ---- + + /** + * Streams batches from the given iterator to the Flight listener, handling backpressure. + */ + private void streamBatches(Iterator batches, Schema schema, + ServerStreamListener listener, BackpressureStrategy bpStrategy) throws Exception + { + try (VectorSchemaRoot vectorSchemaRoot = VectorSchemaRoot.create(schema, rootAllocator)) { + final VectorLoader loader = new VectorLoader(vectorSchemaRoot); + listener.start(vectorSchemaRoot); + while (batches.hasNext()) { + final VectorSchemaRoot batch = batches.next(); + final VectorUnloader unloader = new VectorUnloader(batch); + loader.load(unloader.getRecordBatch()); + WaitResult wr; + while ((wr = bpStrategy.waitForListener(5000)) == WaitResult.TIMEOUT) { + LOGGER.info("Waiting for ready from client"); + } + if (wr == WaitResult.CANCELLED) { + break; + } + listener.putNext(); + vectorSchemaRoot.clear(); + } + if (listener.isCancelled()) { + LOGGER.info("Stream has been cancelled"); + listener.error(CallStatus.CANCELLED.withDescription("Stream cancelled.").toRuntimeException()); + } else { + listener.completed(); + } + } + } + + /** + * Translates a {@link CustomFlightAssetDescriptor} to an {@link AssetDescriptor}. + */ + private static AssetDescriptor toAssetDescriptor(CustomFlightAssetDescriptor src) + { + final int batchSize = src.getBatchSize() != null ? src.getBatchSize() : DEFAULT_BATCH_SIZE; + return new AssetDescriptor( + src.getId(), + src.getName(), + src.getPath(), + src.getDatasourceTypeName(), + src.getConnectionProperties(), // ConnectionProperties extends HashMap + Boolean.TRUE.equals(src.isHasChildren()), + batchSize); + } + + /** + * Translates a {@link CustomFlightAssetsCriteria} to a {@link DiscoveryCriteria}. + */ + private static DiscoveryCriteria toDiscoveryCriteria(CustomFlightAssetsCriteria src) + { + final ConnectionProperties connProps = toConnectionProperties(src.getConnectionProperties()); + return new DiscoveryCriteria(src.getPath(), src.getDatasourceTypeName(), connProps); + } + + /** + * Translates a model SDK-API {@code ConnectionProperties} to the connector-SDK + * {@link ConnectionProperties}. + * + *

The model type extends {@code HashMap} and is passed directly as the + * backing map. The parameter must be fully-qualified because both packages expose a class + * named {@code ConnectionProperties}. + */ + private static ConnectionProperties toConnectionProperties( + com.ibm.wdp.connect.common.sdk.api.models.ConnectionProperties src) + { + return new ConnectionProperties(src); + } + + /** + * Translates an {@link AssetDescriptor} back to a {@link CustomFlightAssetDescriptor}. + */ + private static CustomFlightAssetDescriptor fromAssetDescriptor( + AssetDescriptor src, String datasourceTypeName, + com.ibm.wdp.connect.common.sdk.api.models.ConnectionProperties connectionProperties) + { + final CustomFlightAssetDescriptor descriptor = new CustomFlightAssetDescriptor(); + descriptor.setId(src.getId()); + descriptor.setName(src.getName()); + descriptor.setPath(src.getPath()); + descriptor.setDatasourceTypeName(datasourceTypeName); + descriptor.setConnectionProperties(connectionProperties); + descriptor.setHasChildren(src.hasChildren()); + return descriptor; + } + + private static int getBatchSize(CustomFlightAssetDescriptor asset) + { + return (asset.getBatchSize() != null && asset.getBatchSize() > 0) ? asset.getBatchSize() : DEFAULT_BATCH_SIZE; + } } diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/build.gradle b/sdk-gen/subprojects/java/connectors_forge_rest/impl/build.gradle index d8686879..b394d848 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/build.gradle +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/build.gradle @@ -80,6 +80,7 @@ dependencies { api project(':' + rootProject.name + '-java-api-models') api project(':' + rootProject.name + '-java-api') implementation project(':' + rootProject.name + '-java-util') + implementation project(':wdp-connect-sdk-gen-java-sdk-connector-api') // JSON processing for REST API calls (Java 11+ HttpClient is used, no need for Apache HttpClient) implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: project['jackson.version'] diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/AuthenticationType.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/AuthenticationType.java index f35c123c..44c7e1ac 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/AuthenticationType.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/AuthenticationType.java @@ -21,31 +21,21 @@ public enum AuthenticationType private final String value; - AuthenticationType(String value) - { - this.value = value; - } - - public String getValue() - { - return value; - } + AuthenticationType(String value) { this.value = value; } + public String getValue() { return value; } public static AuthenticationType fromValue(String value) { if (value == null) { throw new IllegalArgumentException( - "Authentication type cannot be null. Valid values are: " + validValues()); + "Authentication type cannot be null. Valid values are: " + validValues()); } - final String normalizedValue = value.toLowerCase(Locale.ENGLISH); for (final AuthenticationType type : values()) { - if (type.value.equals(normalizedValue)) { - return type; - } + if (type.value.equals(normalizedValue)) { return type; } } throw new IllegalArgumentException( - "Invalid authentication type: '" + value + "'. Valid values are: " + validValues()); + "Invalid authentication type: '" + value + "'. Valid values are: " + validValues()); } public static String validValues() @@ -55,5 +45,3 @@ public static String validValues() .collect(Collectors.joining(", ")); } } - -// Made with Bob \ No newline at end of file diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/ForgeSchemaBuilder.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/ForgeSchemaBuilder.java new file mode 100644 index 00000000..1e3f796a --- /dev/null +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/ForgeSchemaBuilder.java @@ -0,0 +1,168 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.connect.restconnector; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; + +/** + * Builds an Arrow {@link Schema} from a list of {@link RestFieldDefinition} objects. + * + *

Replaces the old {@code RestFieldTypeMapper} which produced + * {@code CustomFlightAssetField} instances (from the SDK model framework). This class produces + * Arrow types directly, with no dependency on the SDK model framework. + * + *

Supported type strings (case-insensitive): + *

+ */ +public class ForgeSchemaBuilder +{ + /** Matches types with a length parameter, e.g. VarChar(50) or LongVarChar(2000). */ + private static final Pattern TYPE_WITH_LENGTH = Pattern.compile("^(\\w+)\\((\\d+)\\)$"); + + private ForgeSchemaBuilder() + { + // utility class + } + + /** + * Builds an Arrow {@link Schema} from the given list of field definitions. + * + * @param fieldDefs + * the field definitions from the JSON configuration file + * @return the Arrow schema + */ + public static Schema buildSchema(List fieldDefs) + { + final List fields = new ArrayList<>(fieldDefs.size()); + for (final RestFieldDefinition fieldDef : fieldDefs) { + fields.add(toArrowField(fieldDef)); + } + return new Schema(fields); + } + + /** + * Converts a single {@link RestFieldDefinition} to an Arrow {@link Field}. + * + * @param fieldDef + * the field definition + * @return the Arrow field + */ + public static Field toArrowField(RestFieldDefinition fieldDef) + { + final boolean nullable = !fieldDef.isNotNull(); + final ArrowType arrowType = toArrowType(fieldDef.getTypeString()); + return new Field(fieldDef.getName(), new FieldType(nullable, arrowType, null), null); + } + + // ---- private helpers ---- + + private static ArrowType toArrowType(String typeString) + { + final String trimmed = typeString.trim(); + + // Check for types with length parameter: VarChar(N), LongVarChar(N), etc. + final Matcher lengthMatcher = TYPE_WITH_LENGTH.matcher(trimmed); + if (lengthMatcher.matches()) { + final String baseType = lengthMatcher.group(1).toLowerCase(Locale.ENGLISH); + switch (baseType) { + case "varchar": + case "nvarchar": + case "char": + case "nchar": + case "longvarchar": + case "longnvarchar": + case "clob": + case "nclob": + return ArrowType.Utf8.INSTANCE; + case "varbinary": + case "binary": + case "blob": + return ArrowType.Binary.INSTANCE; + default: + return ArrowType.Utf8.INSTANCE; + } + } + + // Simple types without length + final String typeLower = trimmed.toLowerCase(Locale.ENGLISH); + switch (typeLower) { + case "integer": + case "int": + return new ArrowType.Int(32, true); + case "bigint": + return new ArrowType.Int(64, true); + case "smallint": + return new ArrowType.Int(16, true); + case "tinyint": + return new ArrowType.Int(8, true); + case "boolean": + case "bool": + case "bit": + return ArrowType.Bool.INSTANCE; + case "date": + return new ArrowType.Date(DateUnit.DAY); + case "timestamp": + case "datetime": + return new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"); + case "time": + return new ArrowType.Time(TimeUnit.MILLISECOND, 32); + case "double": + case "float8": + return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE); + case "float": + case "real": + case "float4": + return new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE); + case "decimal": + case "numeric": + return new ArrowType.Decimal(38, 10, 128); + case "json": + case "jsonb": + case "array": + case "object": + case "varchar": + case "nvarchar": + case "char": + case "nchar": + case "longvarchar": + case "longnvarchar": + case "clob": + case "text": + return ArrowType.Utf8.INSTANCE; + default: + // Unknown type — default to Utf8 + return ArrowType.Utf8.INSTANCE; + } + } +} diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/JsonToArrowStream.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/JsonToArrowStream.java new file mode 100644 index 00000000..dee4eec1 --- /dev/null +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/JsonToArrowStream.java @@ -0,0 +1,564 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.connect.restconnector; + +import static org.slf4j.LoggerFactory.getLogger; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.sql.Date; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.format.DateTimeParseException; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import org.slf4j.Logger; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.ibm.wdp.connect.sdk.connector.RowWriter; + +/** + * Fetches data from a REST API endpoint and streams each JSON object to a {@link RowWriter}. + * + *

This is the forge REST engine's JSON-to-Arrow streaming component. It replaces the old + * {@code JsonToRecordStream} — the only change is that instead of creating {@code Record} objects + * it calls {@link RowWriter#startRow()}, {@link RowWriter#set(String, Object)}, + * and {@link RowWriter#endRow()} for each JSON object. + * + *

The HTTP connection is opened lazily. The JSON response is parsed using Jackson's streaming + * API to avoid loading the entire response into memory. + * + *

Supports all pagination strategies in {@link PaginationType}. + */ +public class JsonToArrowStream implements Closeable +{ + private static final Logger LOGGER = getLogger(JsonToArrowStream.class); + + private static final int HTTP_TIMEOUT_SECONDS = 60; + private static final int HTTP_OK = 200; + private static final int MAX_PAGES = 10000; + + private final String baseUrl; + private final String dataPath; + private final List fieldDefs; + private final Map authHeaders; + private final PaginationConfig paginationConfig; + private final ObjectMapper objectMapper; + + /** + * Creates a streaming instance for the given URL and field definitions. + * + * @param url + * the full URL to fetch (e.g. "https://api.spacexdata.com/v4/rockets") + * @param fieldDefs + * the field definitions that define the expected fields and their types + */ + public JsonToArrowStream(String url, List fieldDefs) + { + this(url, null, fieldDefs, null, null); + } + + /** + * Creates a streaming instance with authentication headers. + * + * @param url + * the full URL to fetch + * @param fieldDefs + * the field definitions + * @param authHeaders + * optional authentication headers (may be null) + */ + public JsonToArrowStream(String url, List fieldDefs, Map authHeaders) + { + this(url, null, fieldDefs, authHeaders, null); + } + + /** + * Creates a streaming instance with data path and authentication headers. + * + * @param url + * the full URL to fetch + * @param dataPath + * optional JSON path to the data array (e.g. "result" for {"result": [...]}) + * @param fieldDefs + * the field definitions + * @param authHeaders + * optional authentication headers (may be null) + */ + public JsonToArrowStream(String url, String dataPath, List fieldDefs, + Map authHeaders) + { + this(url, dataPath, fieldDefs, authHeaders, null); + } + + /** + * Creates a streaming instance with full configuration including pagination support. + * + * @param url + * the base URL to fetch + * @param dataPath + * optional JSON path to the data array + * @param fieldDefs + * the field definitions + * @param authHeaders + * optional authentication headers (may be null) + * @param paginationConfig + * optional pagination configuration (may be null for non-paginated APIs) + */ + public JsonToArrowStream(String url, String dataPath, List fieldDefs, + Map authHeaders, PaginationConfig paginationConfig) + { + this.baseUrl = url; + this.dataPath = dataPath; + this.fieldDefs = fieldDefs; + this.authHeaders = authHeaders; + this.paginationConfig = paginationConfig; + this.objectMapper = new ObjectMapper(); + } + + /** + * Fetches all data from the configured URL and streams it into the provided {@link RowWriter}. + * + *

For each JSON object in the response, calls: + *

    + *
  1. {@link RowWriter#startRow()}
    2. + *
  2. {@link RowWriter#set(String, Object)} for each field
    3. + *
  3. {@link RowWriter#endRow()}
    4. + *
+ * + * @param writer + * the row writer to receive the data + * @throws IOException + * if an HTTP or JSON error occurs + * @throws InterruptedException + * if the thread is interrupted during an HTTP request + */ + public void streamTo(RowWriter writer) throws IOException, InterruptedException + { + final HttpClient httpClient = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(HTTP_TIMEOUT_SECONDS)) + .build(); + + // Pagination state + int currentOffset = paginationConfig != null ? paginationConfig.getInitialOffset() : 0; + int currentPage = paginationConfig != null ? paginationConfig.getInitialPage() : 1; + String nextCursor = null; + String nextPageUrl = null; + int totalPagesFetched = 0; + + do { + if (totalPagesFetched >= MAX_PAGES) { + LOGGER.warn("Reached maximum page limit of {}", MAX_PAGES); + break; + } + + final String url = buildCurrentPageUrl(currentOffset, currentPage, nextCursor, nextPageUrl); + LOGGER.debug("Fetching page {}: {}", totalPagesFetched + 1, url); + + final HttpRequest request = buildRequest(url); + final HttpResponse response = httpClient.send(request, BodyHandlers.ofInputStream()); + + if (response.statusCode() != HTTP_OK) { + throw new IOException("HTTP request failed with status " + response.statusCode()); + } + + final java.net.http.HttpHeaders headers = response.headers(); + + // For link_header pagination, extract next URL from response headers + if (paginationConfig != null && "link_header".equals(paginationConfig.getType())) { + nextPageUrl = extractLinkHeader(headers); + } + + totalPagesFetched++; + int recordsInPage = 0; + + // Cursor and next_url need full parse to extract metadata + if (paginationConfig != null && + ("cursor".equals(paginationConfig.getType()) || "next_url".equals(paginationConfig.getType()))) { + + final JsonNode rootNode = objectMapper.readTree(response.body()); + + if ("cursor".equals(paginationConfig.getType())) { + final JsonNode cursorNode = extractJsonPath(rootNode, paginationConfig.getNextCursorPath()); + nextCursor = (cursorNode != null && !cursorNode.isNull()) ? cursorNode.asText() : null; + } else { + final JsonNode urlNode = extractJsonPath(rootNode, paginationConfig.getNextUrlPath()); + nextPageUrl = (urlNode != null && !urlNode.isNull()) ? urlNode.asText() : null; + } + + JsonNode dataNode = rootNode; + if (dataPath != null && !dataPath.isEmpty()) { + dataNode = extractJsonPath(rootNode, dataPath); + if (dataNode == null) { + throw new IOException("Data path '" + dataPath + "' not found in JSON response"); + } + } + + recordsInPage = streamNodeToWriter(dataNode, writer); + + } else { + // Streaming parse for offset/page/link_header/no-pagination + final JsonFactory factory = new JsonFactory(); + try (JsonParser jsonParser = factory.createParser(response.body())) { + recordsInPage = streamParserToWriter(jsonParser, writer); + } + } + + // Advance pagination state + if (paginationConfig != null) { + if ("offset".equals(paginationConfig.getType())) { + currentOffset += paginationConfig.getPageSize(); + } else if ("page".equals(paginationConfig.getType())) { + currentPage++; + } + } + + // Determine if there are more pages + if (!hasMorePages(paginationConfig, recordsInPage, nextCursor, nextPageUrl, totalPagesFetched)) { + break; + } + + } while (true); + } + + /** {@inheritDoc} */ + @Override + public void close() + { + // No persistent resources to close in this stateless implementation + } + + // ---- private helpers ---- + + private int streamNodeToWriter(JsonNode dataNode, RowWriter writer) throws IOException + { + int count = 0; + if (dataNode.isArray()) { + for (final JsonNode item : dataNode) { + if (item.isObject()) { + writeObjectToWriter((ObjectNode) item, writer); + count++; + } + } + } else if (dataNode.isObject()) { + writeObjectToWriter((ObjectNode) dataNode, writer); + count = 1; + } + return count; + } + + private int streamParserToWriter(JsonParser jsonParser, RowWriter writer) throws IOException + { + int count = 0; + JsonToken firstToken = jsonParser.nextToken(); + + if (dataPath != null && !dataPath.isEmpty()) { + if (firstToken == JsonToken.START_ARRAY) { + // Root is an array (e.g. NBP API returns [{...}]) — fall back to tree-mode + // so that extractJsonPath can handle numeric index segments like "0.rates". + // JsonParser is already positioned AT START_ARRAY so readTree reads the full array. + final JsonNode rootNode = objectMapper.readTree(jsonParser); + final JsonNode dataNode = extractJsonPath(rootNode, dataPath); + if (dataNode == null) { + throw new IOException("Data path '" + dataPath + "' not found in JSON response"); + } + return streamNodeToWriter(dataNode, writer); + } + if (firstToken != JsonToken.START_OBJECT) { + throw new IOException("Expected START_OBJECT or START_ARRAY at root when dataPath is specified, got: " + firstToken); + } + boolean found = false; + while (jsonParser.nextToken() != JsonToken.END_OBJECT) { + if (dataPath.equals(jsonParser.currentName())) { + firstToken = jsonParser.nextToken(); + found = true; + break; + } + jsonParser.skipChildren(); + } + if (!found) { + throw new IOException("Data path '" + dataPath + "' not found in JSON response"); + } + } + + if (firstToken == JsonToken.START_ARRAY) { + JsonToken token; + while ((token = jsonParser.nextToken()) != JsonToken.END_ARRAY && token != null) { + if (token == JsonToken.START_OBJECT) { + final ObjectNode objectNode = objectMapper.readTree(jsonParser); + writeObjectToWriter(objectNode, writer); + count++; + } + } + } else if (firstToken == JsonToken.START_OBJECT) { + final ObjectNode objectNode = objectMapper.readTree(jsonParser); + writeObjectToWriter(objectNode, writer); + count = 1; + } + + return count; + } + + private void writeObjectToWriter(ObjectNode objectNode, RowWriter writer) throws IOException + { + writer.startRow(); + for (final RestFieldDefinition fieldDef : fieldDefs) { + final String fieldName = fieldDef.getName(); + final JsonNode valueNode = fieldName.contains(".") + ? getNestedValue(objectNode, fieldName) + : objectNode.get(fieldName); + writer.set(fieldName, convertValue(valueNode, fieldDef)); + } + writer.endRow(); + } + + private static JsonNode getNestedValue(ObjectNode objectNode, String flattenedName) + { + final int dotIndex = flattenedName.indexOf('.'); + if (dotIndex < 0) { + return objectNode.get(flattenedName); + } + final String parentKey = flattenedName.substring(0, dotIndex); + final String remainingPath = flattenedName.substring(dotIndex + 1); + final JsonNode parentNode = objectNode.get(parentKey); + if (parentNode == null || !parentNode.isObject()) { + return null; + } + if (remainingPath.contains(".")) { + return getNestedValue((ObjectNode) parentNode, remainingPath); + } + return parentNode.get(remainingPath); + } + + private static Object convertValue(JsonNode node, RestFieldDefinition fieldDef) + { + if (node == null || node.isNull()) { + return null; + } + if (node.isObject() || node.isArray()) { + return node.toString(); + } + + final String baseType = extractBaseType(fieldDef.getTypeString().toLowerCase(Locale.ENGLISH)); + + switch (baseType) { + case "integer": + case "int": + case "smallint": + case "tinyint": + return node.isNumber() ? node.intValue() : parseIntSafe(node.asText()); + case "bigint": + return node.isNumber() ? node.longValue() : parseLongSafe(node.asText()); + case "boolean": + case "bool": + case "bit": + return node.isBoolean() ? node.booleanValue() : Boolean.parseBoolean(node.asText()); + case "double": + case "float8": + case "float": + case "real": + case "float4": + return node.isNumber() ? node.doubleValue() : parseDoubleSafe(node.asText()); + case "date": + return parseDateSafe(node.asText()); + case "timestamp": + case "datetime": + return parseTimestampSafe(node.asText()); + case "json": + case "jsonb": + case "array": + case "object": + return node.toString(); + default: + return node.asText(); + } + } + + private static String extractBaseType(String typeLower) + { + final int parenIdx = typeLower.indexOf('('); + return parenIdx >= 0 ? typeLower.substring(0, parenIdx).trim() : typeLower.trim(); + } + + private static Integer parseIntSafe(String text) + { + if (text == null || text.isEmpty()) return null; + try { return Integer.parseInt(text.trim()); } + catch (NumberFormatException e) { LOGGER.warn("Cannot parse integer: '{}'", text); return null; } + } + + private static Long parseLongSafe(String text) + { + if (text == null || text.isEmpty()) return null; + try { return Long.parseLong(text.trim()); } + catch (NumberFormatException e) { LOGGER.warn("Cannot parse long: '{}'", text); return null; } + } + + private static Double parseDoubleSafe(String text) + { + if (text == null || text.isEmpty()) return null; + try { return Double.parseDouble(text.trim()); } + catch (NumberFormatException e) { LOGGER.warn("Cannot parse double: '{}'", text); return null; } + } + + private static Date parseDateSafe(String text) + { + if (text == null || text.isEmpty()) return null; + try { return Date.valueOf(LocalDate.parse(text.trim())); } + catch (DateTimeParseException e) { LOGGER.warn("Cannot parse date: '{}'", text); return null; } + } + + private static Timestamp parseTimestampSafe(String text) + { + if (text == null || text.isEmpty()) return null; + try { return Timestamp.from(Instant.parse(text.trim())); } + catch (DateTimeParseException e) { + try { return Timestamp.valueOf(text.trim().replace("T", " ").replaceAll("\\.\\d+Z?$", "")); } + catch (Exception ex) { LOGGER.warn("Cannot parse timestamp: '{}'", text); return null; } + } + } + + private HttpRequest buildRequest(String url) + { + final HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(Duration.ofSeconds(HTTP_TIMEOUT_SECONDS)) + .header("Accept", "application/json") + .header("User-Agent", "CP4D-REST-Connector/1.0") + .GET(); + if (authHeaders != null) { + authHeaders.forEach(builder::header); + } + return builder.build(); + } + + private String buildCurrentPageUrl(int currentOffset, int currentPage, String nextCursor, String nextPageUrl) + { + if (paginationConfig == null) { + return baseUrl; + } + + final String type = paginationConfig.getType(); + final boolean hasParams = baseUrl.contains("?"); + final String sep = hasParams ? "&" : "?"; + final StringBuilder sb = new StringBuilder(baseUrl); + + switch (type) { + case "offset": + sb.append(sep).append(paginationConfig.getOffsetParam()).append('=').append(currentOffset); + if (paginationConfig.getLimitParam() != null) { + sb.append('&').append(paginationConfig.getLimitParam()).append('=').append(paginationConfig.getPageSize()); + } + break; + case "page": + sb.append(sep).append(paginationConfig.getPageParam()).append('=').append(currentPage); + if (paginationConfig.getLimitParam() != null) { + sb.append('&').append(paginationConfig.getLimitParam()).append('=').append(paginationConfig.getPageSize()); + } + break; + case "cursor": + if (nextCursor != null && !nextCursor.isEmpty()) { + sb.append(sep).append(paginationConfig.getCursorParam()).append('=').append(nextCursor); + if (paginationConfig.getLimitParam() != null) { + sb.append('&').append(paginationConfig.getLimitParam()).append('=').append(paginationConfig.getPageSize()); + } + } else if (paginationConfig.getLimitParam() != null) { + sb.append(sep).append(paginationConfig.getLimitParam()).append('=').append(paginationConfig.getPageSize()); + } + break; + case "link_header": + case "next_url": + if (nextPageUrl != null && !nextPageUrl.isEmpty()) { + return nextPageUrl; + } + if (paginationConfig.getLimitParam() != null) { + sb.append(sep).append(paginationConfig.getLimitParam()).append('=').append(paginationConfig.getPageSize()); + } + break; + default: + break; + } + + return sb.toString(); + } + + private static boolean hasMorePages(PaginationConfig config, int recordsInPage, String nextCursor, + String nextPageUrl, int totalPagesFetched) + { + if (config == null || totalPagesFetched >= MAX_PAGES) { + return false; + } + switch (config.getType()) { + case "offset": + case "page": + return recordsInPage >= config.getPageSize(); + case "cursor": + return nextCursor != null && !nextCursor.isEmpty(); + case "link_header": + case "next_url": + return nextPageUrl != null && !nextPageUrl.isEmpty(); + default: + return false; + } + } + + private static String extractLinkHeader(java.net.http.HttpHeaders headers) + { + final java.util.Optional linkHeader = headers.firstValue("Link"); + if (!linkHeader.isPresent()) { + return null; + } + for (final String link : linkHeader.get().split(",")) { + if (link.contains("rel=\"next\"") || link.contains("rel='next'")) { + final int start = link.indexOf('<') + 1; + final int end = link.indexOf('>'); + if (start > 0 && end > start) { + return link.substring(start, end); + } + } + } + return null; + } + + private static JsonNode extractJsonPath(JsonNode node, String path) + { + if (path == null || path.isEmpty()) { + return node; + } + JsonNode current = node; + for (final String segment : path.split("\\.")) { + if (current == null || current.isNull()) { + return null; + } + if (current.isArray()) { + // ArrayNode.get(String) returns null — must use get(int) + try { + current = current.get(Integer.parseInt(segment)); + } catch (NumberFormatException e) { + return null; // non-numeric key on an array — path not found + } + } else { + current = current.get(segment); + } + } + return current; + } +} diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/JsonToRecordStream.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/JsonToRecordStream.java deleted file mode 100644 index 1935b218..00000000 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/JsonToRecordStream.java +++ /dev/null @@ -1,940 +0,0 @@ -/* *************************************************** */ - -/* (C) Copyright IBM Corp. 2026 */ - -/* *************************************************** */ -package com.ibm.connect.restconnector; - -import static org.slf4j.LoggerFactory.getLogger; - -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.net.http.HttpResponse.BodyHandlers; -import java.sql.Date; -import java.sql.Timestamp; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDate; -import java.time.format.DateTimeParseException; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.NoSuchElementException; - -import org.slf4j.Logger; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.ibm.connect.sdk.api.Record; - -/** - * A streaming iterator that fetches data from a REST API endpoint and converts - * each JSON object in the response array into a {@link Record}. - * - *

The HTTP connection is opened lazily on the first call to {@link #hasNext()}. - * The JSON response is parsed using Jackson's streaming API to avoid loading - * the entire response into memory. - * - *

The response is expected to be a JSON array of objects. Each object is - * converted to a {@link Record} with values in the order defined by the - * field definitions. - */ -public class JsonToRecordStream implements Iterator, Closeable -{ - private static final Logger LOGGER = getLogger(JsonToRecordStream.class); - - private static final int HTTP_TIMEOUT_SECONDS = 60; - private static final int HTTP_OK = 200; - private static final int MAX_PAGES = 10000; // Safety limit to prevent infinite loops - - private final String baseUrl; - private final String dataPath; - private final List fieldDefs; - private final java.util.Map authHeaders; - private final PaginationConfig paginationConfig; - private final ObjectMapper objectMapper; - - private HttpClient httpClient; - private InputStream responseStream; - private JsonParser jsonParser; - private java.net.http.HttpHeaders lastResponseHeaders; - - private Record nextRecord; - private boolean initialized; - private boolean done; - - // Pagination state - private int currentOffset; - private int currentPage; - private int recordsInCurrentPage; - private String nextCursor; - private String nextPageUrl; - private int totalPagesFetched; - - /** - * Creates a streaming record iterator for the given URL and field definitions. - * - * @param url - * the full URL to fetch (e.g. "https://api.spacexdata.com/v4/rockets") - * @param fieldDefs - * the field definitions that define the expected fields and their types - */ - public JsonToRecordStream(String url, List fieldDefs) - { - this(url, null, fieldDefs, null, null); - } - - /** - * Creates a streaming record iterator for the given URL, field definitions, and authentication headers. - * - * @param url - * the full URL to fetch (e.g. "https://api.spacexdata.com/v4/rockets") - * @param fieldDefs - * the field definitions that define the expected fields and their types - * @param authHeaders - * optional authentication headers to include in the HTTP request (may be null) - */ - public JsonToRecordStream(String url, List fieldDefs, java.util.Map authHeaders) - { - this(url, null, fieldDefs, authHeaders, null); - } - - /** - * Creates a streaming record iterator for the given URL, data path, field definitions, and authentication headers. - * - * @param url - * the full URL to fetch (e.g. "https://api.spacexdata.com/v4/rockets") - * @param dataPath - * optional JSON path to the data array (e.g. "result" for {"result": [...]}) - * @param fieldDefs - * the field definitions that define the expected fields and their types - * @param authHeaders - * optional authentication headers to include in the HTTP request (may be null) - */ - public JsonToRecordStream(String url, String dataPath, List fieldDefs, java.util.Map authHeaders) - { - this(url, dataPath, fieldDefs, authHeaders, null); - } - - /** - * Creates a streaming record iterator with full configuration including pagination support. - * - * @param url - * the base URL to fetch (e.g. "https://api.spacexdata.com/v4/rockets") - * @param dataPath - * optional JSON path to the data array (e.g. "result" for {"result": [...]}) - * @param fieldDefs - * the field definitions that define the expected fields and their types - * @param authHeaders - * optional authentication headers to include in the HTTP request (may be null) - * @param paginationConfig - * optional pagination configuration (may be null for non-paginated APIs) - */ - public JsonToRecordStream(String url, String dataPath, List fieldDefs, - java.util.Map authHeaders, PaginationConfig paginationConfig) - { - this.baseUrl = url; - this.dataPath = dataPath; - this.fieldDefs = fieldDefs; - this.authHeaders = authHeaders; - this.paginationConfig = paginationConfig; - this.objectMapper = new ObjectMapper(); - this.initialized = false; - this.done = false; - - // Initialize pagination state - this.recordsInCurrentPage = 0; - this.totalPagesFetched = 0; - if (paginationConfig != null) { - this.currentOffset = paginationConfig.getInitialOffset(); - this.currentPage = paginationConfig.getInitialPage(); - } - } - - /** - * {@inheritDoc} - */ - @Override - public boolean hasNext() - { - if (!initialized) { - try { - initialize(); - } - catch (Exception e) { - LOGGER.error("Failed to initialize HTTP stream", e); - done = true; - return false; - } - } - if (nextRecord != null) { - return true; - } - if (done) { - return false; - } - try { - nextRecord = advance(); - } - catch (Exception e) { - LOGGER.error("Error reading next record from stream", e); - done = true; - return false; - } - return nextRecord != null; - } - - /** - * {@inheritDoc} - */ - @Override - public Record next() - { - if (!hasNext()) { - throw new NoSuchElementException("No more records in stream"); - } - final Record record = nextRecord; - nextRecord = null; - return record; - } - - /** - * {@inheritDoc} - */ - @Override - public void close() throws IOException - { - done = true; - if (jsonParser != null) { - try { - jsonParser.close(); - } - catch (IOException e) { - LOGGER.warn("Error closing JSON parser", e); - } - } - if (responseStream != null) { - try { - responseStream.close(); - } - catch (IOException e) { - LOGGER.warn("Error closing HTTP response stream", e); - } - } - } - - /** - * Opens the HTTP connection and positions the JSON parser at the first object. - */ - private void initialize() throws IOException, InterruptedException - { - // Build the initial URL with pagination parameters if configured - final String currentUrl = buildCurrentPageUrl(); - - LOGGER.info("Opening HTTP stream"); - httpClient = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(HTTP_TIMEOUT_SECONDS)) - .build(); - - final HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() - .uri(URI.create(currentUrl)) - .timeout(Duration.ofSeconds(HTTP_TIMEOUT_SECONDS)) - .header("Accept", "application/json") - .header("User-Agent", "CP4D-REST-Connector/1.0"); - - // Add authentication headers if provided - if (authHeaders != null && !authHeaders.isEmpty()) { - for (final java.util.Map.Entry header : authHeaders.entrySet()) { - requestBuilder.header(header.getKey(), header.getValue()); - LOGGER.debug("Adding authentication header: {}", header.getKey()); - } - } - - final HttpRequest request = requestBuilder.GET().build(); - - final HttpResponse response = httpClient.send(request, BodyHandlers.ofInputStream()); - - if (response.statusCode() != HTTP_OK) { - throw new IOException("HTTP request failed with status " + response.statusCode()); - } - - // Store response headers for Link header pagination - lastResponseHeaders = response.headers(); - - // Extract next page URL from Link header if using link_header pagination - if (paginationConfig != null && "link_header".equals(paginationConfig.getType())) { - nextPageUrl = extractLinkHeader(lastResponseHeaders); - } - - responseStream = response.body(); - totalPagesFetched++; - - // For cursor and next_url pagination, we need to parse the full response to extract metadata - if (paginationConfig != null && - ("cursor".equals(paginationConfig.getType()) || "next_url".equals(paginationConfig.getType()))) { - parseResponseWithMetadata(); - } else { - final JsonFactory factory = new JsonFactory(); - jsonParser = factory.createParser(responseStream); - // Navigate to the data array - navigateToDataArray(); - } - - // Only pre-fetch if not already initialized (single object case) - if (!initialized) { - initialized = true; - // Pre-fetch the first record - nextRecord = advance(); - } - } - - /** - * Navigates the JSON parser to the data array. - * Handles both root-level arrays and nested arrays specified by dataPath. - * - * @throws IOException if an I/O error occurs or the expected structure is not found - */ - private void navigateToDataArray() throws IOException - { - // Advance to the first token - JsonToken firstToken = jsonParser.nextToken(); - - // If dataPath is specified, navigate to the nested array - if (dataPath != null && !dataPath.isEmpty()) { - LOGGER.debug("Navigating to configured nested data path"); - if (firstToken != JsonToken.START_OBJECT) { - throw new IOException("Expected START_OBJECT at root when dataPath is specified, but got: " + firstToken); - } - - // Navigate through the JSON structure to find the data path - boolean found = false; - while (jsonParser.nextToken() != JsonToken.END_OBJECT) { - final String fieldName = jsonParser.currentName(); - - if (dataPath.equals(fieldName)) { - // Move to the value of this field - firstToken = jsonParser.nextToken(); - found = true; - LOGGER.debug("Found configured data path, token type: {}", firstToken); - break; - } - // Skip this field's value - jsonParser.skipChildren(); - } - - if (!found) { - throw new IOException("Data path '" + dataPath + "' not found in JSON response"); - } - } - - if (firstToken == JsonToken.START_ARRAY) { - // JSON array response — advance past the START_ARRAY token - // The parser is now positioned before the first object - LOGGER.debug("Response is a JSON array"); - } else if (firstToken == JsonToken.START_OBJECT) { - // Single JSON object response — we'll read it as one record - // Push back by not advancing — we'll handle it in advance() - LOGGER.debug("Response is a single JSON object"); - // We need to re-read this object, so we set a flag - // We handle this by reading the current object directly - final Record record = readCurrentObject(); - nextRecord = record; - done = true; // Only one record - initialized = true; - } else { - throw new IOException("Unexpected JSON token at start of response: " + firstToken); - } - } - - /** - * Advances the parser to the next JSON object and converts it to a Record. - * Handles pagination by fetching the next page when the current page is exhausted. - * - * @return the next Record, or null if there are no more records - */ - private Record advance() throws IOException - { - if (done) { - return null; - } - - // Peek at the next token - final JsonToken token = jsonParser.nextToken(); - if (token == null || token == JsonToken.END_ARRAY) { - // Current page exhausted - check if there are more pages - if (paginationConfig != null && hasMorePages()) { - try { - fetchNextPage(); - // Recursively call advance() to read from the new page - return advance(); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - LOGGER.error("Interrupted while fetching next page", e); - done = true; - return null; - } - } - done = true; - return null; - } - - if (token != JsonToken.START_OBJECT) { - LOGGER.warn("Expected START_OBJECT but got: {}", token); - done = true; - return null; - } - - // Successfully read an object - increment counter for this page - recordsInCurrentPage++; - return readCurrentObject(); - } - - /** - * Checks if there are more pages to fetch based on pagination configuration. - * - * @return true if more pages should be fetched, false otherwise - */ - private boolean hasMorePages() - { - // Safety check: don't fetch more than MAX_PAGES - if (totalPagesFetched >= MAX_PAGES) { - LOGGER.warn("Reached maximum page limit of {}", MAX_PAGES); - return false; - } - - final String type = paginationConfig.getType(); - - if ("offset".equals(type) || "page".equals(type)) { - // For offset/page pagination, continue if we got a full page - // If we got fewer records than page_size, we've reached the end - return recordsInCurrentPage >= paginationConfig.getPageSize(); - } - - if ("cursor".equals(type)) { - // For cursor pagination, continue if we have a next cursor - return nextCursor != null && !nextCursor.isEmpty(); - } - - if ("link_header".equals(type) || "next_url".equals(type)) { - // For link_header/next_url pagination, continue if we have a next URL - return nextPageUrl != null && !nextPageUrl.isEmpty(); - } - - return false; - } - - /** - * Fetches the next page of data by closing the current stream and opening a new one. - * - * @throws IOException if an I/O error occurs - * @throws InterruptedException if the thread is interrupted - */ - private void fetchNextPage() throws IOException, InterruptedException - { - LOGGER.debug("Fetching next page (current page: {}, records in page: {})", - totalPagesFetched, recordsInCurrentPage); - - // Close current parser and stream - if (jsonParser != null) { - jsonParser.close(); - } - if (responseStream != null) { - responseStream.close(); - } - - // Update pagination state based on type - final String type = paginationConfig.getType(); - if ("offset".equals(type)) { - currentOffset += paginationConfig.getPageSize(); - } else if ("page".equals(type)) { - currentPage++; - } - // For cursor/link_header/next_url, the state is already updated - - // Reset page record counter - recordsInCurrentPage = 0; - - // Build URL for next page - final String nextUrl = buildCurrentPageUrl(); - LOGGER.debug("Fetching next page"); - - // Make HTTP request - final HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() - .uri(URI.create(nextUrl)) - .timeout(Duration.ofSeconds(HTTP_TIMEOUT_SECONDS)) - .header("Accept", "application/json") - .header("User-Agent", "CP4D-REST-Connector/1.0") - .GET(); - - // Add authentication headers - if (authHeaders != null && !authHeaders.isEmpty()) { - for (final java.util.Map.Entry header : authHeaders.entrySet()) { - requestBuilder.header(header.getKey(), header.getValue()); - } - } - - final HttpRequest request = requestBuilder.build(); - final HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); - - if (response.statusCode() != HTTP_OK) { - throw new IOException("HTTP request failed with status code: " + response.statusCode()); - } - - // Store response headers for Link header extraction - lastResponseHeaders = response.headers(); - - // Extract next page URL from Link header if using link_header pagination - if ("link_header".equals(type)) { - nextPageUrl = extractLinkHeader(lastResponseHeaders); - LOGGER.debug("Extracted next page URL from Link header"); - } - - // Open new stream - responseStream = response.body(); - - // Increment page counter - totalPagesFetched++; - - // For cursor and next_url pagination, parse the full response to extract metadata - if ("cursor".equals(type) || "next_url".equals(type)) { - parseResponseWithMetadata(); - } else { - // For other pagination types, use streaming parser - final JsonFactory factory = new JsonFactory(); - jsonParser = factory.createParser(responseStream); - // Navigate to the data array - navigateToDataArray(); - } - } - - /** - * Parses the full JSON response to extract both data and pagination metadata. - * Used for cursor and next_url pagination types where metadata is in the response body. - * - * @throws IOException if an I/O error occurs - */ - private void parseResponseWithMetadata() throws IOException - { - // Parse the entire response into a JsonNode to extract metadata - final com.fasterxml.jackson.databind.JsonNode rootNode = objectMapper.readTree(responseStream); - - final String type = paginationConfig.getType(); - - if ("cursor".equals(type)) { - // Extract next cursor from the response - final String cursorPath = paginationConfig.getNextCursorPath(); - if (cursorPath != null) { - final com.fasterxml.jackson.databind.JsonNode cursorNode = extractJsonPath(rootNode, cursorPath); - if (cursorNode != null && !cursorNode.isNull()) { - nextCursor = cursorNode.asText(); - LOGGER.debug("Extracted next cursor"); - } else { - nextCursor = null; - LOGGER.debug("No next cursor found in response"); - } - } - } else if ("next_url".equals(type)) { - // Extract next URL from the response - final String nextUrlPath = paginationConfig.getNextUrlPath(); - if (nextUrlPath != null) { - final com.fasterxml.jackson.databind.JsonNode urlNode = extractJsonPath(rootNode, nextUrlPath); - if (urlNode != null && !urlNode.isNull()) { - nextPageUrl = urlNode.asText(); - LOGGER.debug("Extracted next URL"); - } else { - nextPageUrl = null; - LOGGER.debug("No next URL found in response"); - } - } - } - - // Now extract the data array from the parsed response - com.fasterxml.jackson.databind.JsonNode dataNode = rootNode; - if (dataPath != null && !dataPath.isEmpty()) { - dataNode = extractJsonPath(rootNode, dataPath); - if (dataNode == null) { - throw new IOException("Data path '" + dataPath + "' not found in JSON response"); - } - } - - // Convert the data node to a JSON string and create a new parser from it - final String dataJson = objectMapper.writeValueAsString(dataNode); - final JsonFactory factory = new JsonFactory(); - jsonParser = factory.createParser(dataJson); - - // Position parser at the start of the array - final JsonToken firstToken = jsonParser.nextToken(); - if (firstToken == JsonToken.START_ARRAY) { - LOGGER.debug("Positioned at data array"); - } else if (firstToken == JsonToken.START_OBJECT) { - // Single object response - final Record record = readCurrentObject(); - nextRecord = record; - done = true; - initialized = true; - } else { - throw new IOException("Unexpected JSON token in data: " + firstToken); - } - } - - /** - * Extracts a value from a JSON node using a dot-separated path. - * For example, "pagination.next_cursor" will navigate through the JSON structure. - * - * @param node the root JSON node - * @param path the dot-separated path (e.g., "pagination.next_cursor") - * @return the value at the path, or null if not found - */ - private com.fasterxml.jackson.databind.JsonNode extractJsonPath( - com.fasterxml.jackson.databind.JsonNode node, String path) - { - if (path == null || path.isEmpty()) { - return node; - } - - com.fasterxml.jackson.databind.JsonNode current = node; - final String[] segments = path.split("\\."); - - for (final String segment : segments) { - if (current == null || current.isNull()) { - return null; - } - current = current.get(segment); - } - - return current; - } - - /** - * Reads the current JSON object (parser positioned at START_OBJECT) and - * converts it to a Record with values in field definition order. - * - * @return the Record - */ - private Record readCurrentObject() throws IOException - { - // Read the entire JSON object into an ObjectNode for random field access - final ObjectNode objectNode = objectMapper.readTree(jsonParser); - - final Record record = new Record(fieldDefs.size()); - for (final RestFieldDefinition fieldDef : fieldDefs) { - final String fieldName = fieldDef.getName(); - final com.fasterxml.jackson.databind.JsonNode valueNode; - - // Check if this is a flattened field (contains dot indicating nested path) - if (fieldName.contains(".")) { - valueNode = getNestedValue(objectNode, fieldName); - } else { - valueNode = objectNode.get(fieldName); - } - - record.appendValue(convertValue(valueNode, fieldDef)); - } - return record; - } - - /** - * Retrieves a value from a nested JSON object using dot-separated path. - * For example, "rates.currency" will look for objectNode.get("rates").get("currency") - * - * @param objectNode the root JSON object - * @param flattenedName the flattened field name (e.g. "rates.currency") - * @return the nested value, or null if not found - */ - private com.fasterxml.jackson.databind.JsonNode getNestedValue(ObjectNode objectNode, String flattenedName) - { - final int dotIndex = flattenedName.indexOf('.'); - if (dotIndex < 0) { - return objectNode.get(flattenedName); - } - - final String parentKey = flattenedName.substring(0, dotIndex); - final String remainingPath = flattenedName.substring(dotIndex + 1); - - final com.fasterxml.jackson.databind.JsonNode parentNode = objectNode.get(parentKey); - if (parentNode == null || !parentNode.isObject()) { - return null; - } - - // Recursively handle deeper nesting - if (remainingPath.contains(".")) { - return getNestedValue((ObjectNode) parentNode, remainingPath); - } else { - return parentNode.get(remainingPath); - } - } - - /** - * Converts a JSON node value to the appropriate Java Serializable type - * based on the field definition's type string. - * - * @param node - * the JSON node (may be null if field is absent) - * @param fieldDef - * the field definition - * @return the Java value, or null if the node is null or JSON null - */ - private java.io.Serializable convertValue(com.fasterxml.jackson.databind.JsonNode node, RestFieldDefinition fieldDef) - { - if (node == null || node.isNull()) { - return null; - } - - // For objects and arrays, serialize back to JSON string - if (node.isObject() || node.isArray()) { - return node.toString(); - } - - final String typeLower = fieldDef.getTypeString().toLowerCase(Locale.ENGLISH); - - // Handle types with length parameter (e.g. VarChar(50)) - final String baseType = extractBaseType(typeLower); - - switch (baseType) { - case "integer": - case "int": - case "smallint": - case "tinyint": - if (node.isNumber()) { - return node.intValue(); - } - return parseIntSafe(node.asText()); - - case "bigint": - if (node.isNumber()) { - return node.longValue(); - } - return parseLongSafe(node.asText()); - - case "boolean": - case "bool": - case "bit": - if (node.isBoolean()) { - return node.booleanValue(); - } - return Boolean.parseBoolean(node.asText()); - - case "double": - case "float8": - case "float": - case "real": - case "float4": - if (node.isNumber()) { - return node.doubleValue(); - } - return parseDoubleSafe(node.asText()); - - case "date": - return parseDateSafe(node.asText()); - - case "timestamp": - case "datetime": - return parseTimestampSafe(node.asText()); - - case "json": - case "jsonb": - case "array": - case "object": - return node.toString(); - - default: - // varchar, longvarchar, and all other string types - return node.asText(); - } - } - - /** - * Extracts the base type name from a type string that may include a length parameter. - * E.g. "varchar(50)" → "varchar", "integer" → "integer" - */ - private static String extractBaseType(String typeLower) - { - final int parenIdx = typeLower.indexOf('('); - if (parenIdx >= 0) { - return typeLower.substring(0, parenIdx).trim(); - } - return typeLower.trim(); - } - - private static Integer parseIntSafe(String text) - { - if (text == null || text.isEmpty()) { - return null; - } - try { - return Integer.parseInt(text.trim()); - } - catch (NumberFormatException e) { - LOGGER.warn("Cannot parse integer value: '{}'", text); - return null; - } - } - - private static Long parseLongSafe(String text) - { - if (text == null || text.isEmpty()) { - return null; - } - try { - return Long.parseLong(text.trim()); - } - catch (NumberFormatException e) { - LOGGER.warn("Cannot parse long value: '{}'", text); - return null; - } - } - - private static Double parseDoubleSafe(String text) - { - if (text == null || text.isEmpty()) { - return null; - } - try { - return Double.parseDouble(text.trim()); - } - catch (NumberFormatException e) { - LOGGER.warn("Cannot parse double value: '{}'", text); - return null; - } - } - - private static Date parseDateSafe(String text) - { - if (text == null || text.isEmpty()) { - return null; - } - try { - // Try ISO date format: "2006-03-24" - return Date.valueOf(LocalDate.parse(text.trim())); - } - catch (DateTimeParseException e) { - LOGGER.warn("Cannot parse date value: '{}'", text); - return null; - } - } - - private static Timestamp parseTimestampSafe(String text) - { - if (text == null || text.isEmpty()) { - return null; - } - try { - // Try ISO-8601 format: "2006-03-24T18:30:00.000Z" - return Timestamp.from(Instant.parse(text.trim())); - } - catch (DateTimeParseException e) { - // Try without timezone - try { - return Timestamp.valueOf(text.trim().replace("T", " ").replaceAll("\\.\\d+Z?$", "")); - } - catch (Exception ex) { - LOGGER.warn("Cannot parse timestamp value: '{}'", text); - return null; - } - } - } - - /** - * Builds the URL for the current page based on pagination configuration. - * - * @return the URL to fetch - */ - private String buildCurrentPageUrl() - { - if (paginationConfig == null) { - return baseUrl; - } - - final String type = paginationConfig.getType(); - final StringBuilder urlBuilder = new StringBuilder(baseUrl); - - // Check if URL already has query parameters - final boolean hasParams = baseUrl.contains("?"); - final String separator = hasParams ? "&" : "?"; - - if ("offset".equals(type)) { - urlBuilder.append(separator); - urlBuilder.append(paginationConfig.getOffsetParam()).append('=').append(currentOffset); - if (paginationConfig.getLimitParam() != null) { - urlBuilder.append('&').append(paginationConfig.getLimitParam()).append('=').append(paginationConfig.getPageSize()); - } - } - else if ("page".equals(type)) { - urlBuilder.append(separator); - urlBuilder.append(paginationConfig.getPageParam()).append('=').append(currentPage); - if (paginationConfig.getLimitParam() != null) { - urlBuilder.append('&').append(paginationConfig.getLimitParam()).append('=').append(paginationConfig.getPageSize()); - } - } - else if ("cursor".equals(type)) { - if (nextCursor != null && !nextCursor.isEmpty()) { - urlBuilder.append(separator); - urlBuilder.append(paginationConfig.getCursorParam()).append('=').append(nextCursor); - } - if (paginationConfig.getLimitParam() != null) { - urlBuilder.append(nextCursor != null ? '&' : separator.charAt(0)); - urlBuilder.append(paginationConfig.getLimitParam()).append('=').append(paginationConfig.getPageSize()); - } - } - else if ("link_header".equals(type)) { - // For link_header, use nextPageUrl if available, otherwise use base URL with page size - if (nextPageUrl != null && !nextPageUrl.isEmpty()) { - return nextPageUrl; - } - if (paginationConfig.getLimitParam() != null) { - urlBuilder.append(separator); - urlBuilder.append(paginationConfig.getLimitParam()).append('=').append(paginationConfig.getPageSize()); - } - } - else if ("next_url".equals(type)) { - // For next_url, use nextPageUrl if available, otherwise use base URL with page size - if (nextPageUrl != null && !nextPageUrl.isEmpty()) { - return nextPageUrl; - } - if (paginationConfig.getLimitParam() != null) { - urlBuilder.append(separator); - urlBuilder.append(paginationConfig.getLimitParam()).append('=').append(paginationConfig.getPageSize()); - } - } - - return urlBuilder.toString(); - } - - /** - * Extracts the next page URL from Link header (RFC 5988). - * - * @param headers the HTTP response headers - * @return the next page URL, or null if not found - */ - private String extractLinkHeader(java.net.http.HttpHeaders headers) - { - final java.util.Optional linkHeader = headers.firstValue("Link"); - if (!linkHeader.isPresent()) { - return null; - } - - // Parse Link header: ; rel="next" - final String[] links = linkHeader.get().split(","); - for (final String link : links) { - if (link.contains("rel=\"next\"") || link.contains("rel='next'")) { - final int start = link.indexOf('<') + 1; - final int end = link.indexOf('>'); - if (start > 0 && end > start) { - return link.substring(start, end); - } - } - } - return null; - } - -} - -// Made with Bob diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/PaginationConfig.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/PaginationConfig.java index 5a0f69a2..af1f1238 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/PaginationConfig.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/PaginationConfig.java @@ -10,8 +10,8 @@ * *

Supports multiple pagination strategies: *

    - *
  • offset: Offset-based pagination (e.g., ?offset=0&limit=100)
    • - *
  • page: Page-based pagination (e.g., ?page=1&per_page=50)
    • + *
  • offset: Offset-based pagination (e.g., ?offset=0&limit=100)
    • + *
  • page: Page-based pagination (e.g., ?page=1&per_page=50)
    • *
  • cursor: Cursor-based pagination with next cursor in response
    • *
  • link_header: Link header pagination (RFC 5988)
    • *
  • next_url: Next URL in response body
    • @@ -70,115 +70,44 @@ public PaginationConfig(PaginationType type, String offsetParam, String pagePara this.nextUrlPath = nextUrlPath; } - /** - * Returns the pagination type. - * - * @return the type (offset, page, cursor, link_header, next_url) - */ + /** Returns the pagination type string. */ public String getType() { return type != null ? type.getValue() : null; } - /** - * Returns the pagination type enum. - * - * @return the pagination type enum - */ + /** Returns the pagination type enum. */ public PaginationType getTypeEnum() { return type; } - /** - * Returns the query parameter name for offset. - * - * @return the offset parameter name (e.g., "offset", "skip") - */ - public String getOffsetParam() - { - return offsetParam; - } + /** Returns the query parameter name for offset. */ + public String getOffsetParam() { return offsetParam; } - /** - * Returns the query parameter name for page number. - * - * @return the page parameter name (e.g., "page", "page_number") - */ - public String getPageParam() - { - return pageParam; - } + /** Returns the query parameter name for page number. */ + public String getPageParam() { return pageParam; } - /** - * Returns the query parameter name for page size/limit. - * - * @return the limit parameter name (e.g., "limit", "per_page", "size") - */ - public String getLimitParam() - { - return limitParam; - } + /** Returns the query parameter name for page size/limit. */ + public String getLimitParam() { return limitParam; } - /** - * Returns the number of items per page. - * - * @return the page size - */ - public int getPageSize() - { - return pageSize; - } + /** Returns the number of items per page. */ + public int getPageSize() { return pageSize; } - /** - * Returns the initial offset value. - * - * @return the initial offset (typically 0) - */ - public int getInitialOffset() - { - return initialOffset; - } + /** Returns the initial offset value. */ + public int getInitialOffset() { return initialOffset; } - /** - * Returns the initial page number. - * - * @return the initial page (typically 1) - */ - public int getInitialPage() - { - return initialPage; - } + /** Returns the initial page number. */ + public int getInitialPage() { return initialPage; } - /** - * Returns the query parameter name for cursor. - * - * @return the cursor parameter name (e.g., "cursor", "next_token") - */ - public String getCursorParam() - { - return cursorParam; - } + /** Returns the query parameter name for cursor. */ + public String getCursorParam() { return cursorParam; } - /** - * Returns the JSON path to the next cursor in the response. - * - * @return the next cursor path (e.g., "pagination.next", "meta.next_token") - */ - public String getNextCursorPath() - { - return nextCursorPath; - } + /** Returns the JSON path to the next cursor in the response. */ + public String getNextCursorPath() { return nextCursorPath; } - /** - * Returns the JSON path to the next URL in the response. - * - * @return the next URL path (e.g., "pagination.next_url", "links.next") - */ - public String getNextUrlPath() - { - return nextUrlPath; - } + /** Returns the JSON path to the next URL in the response. */ + public String getNextUrlPath() { return nextUrlPath; } @Override public String toString() @@ -190,5 +119,3 @@ public String toString() + "', nextCursorPath='" + nextCursorPath + "', nextUrlPath='" + nextUrlPath + "'}"; } } - -// Made with Bob \ No newline at end of file diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/PaginationType.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/PaginationType.java index bb924889..0f8a5ef5 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/PaginationType.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/PaginationType.java @@ -22,27 +22,15 @@ public enum PaginationType private final String value; - PaginationType(String value) - { - this.value = value; - } - - public String getValue() - { - return value; - } + PaginationType(String value) { this.value = value; } + public String getValue() { return value; } public static PaginationType fromValue(String value) { - if (value == null) { - return null; - } - + if (value == null) { return null; } final String normalizedValue = value.toLowerCase(Locale.ENGLISH); for (final PaginationType type : values()) { - if (type.value.equals(normalizedValue)) { - return type; - } + if (type.value.equals(normalizedValue)) { return type; } } return null; } @@ -54,5 +42,3 @@ public static String validValues() .collect(Collectors.joining(", ")); } } - -// Made with Bob \ No newline at end of file diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestApiMapping.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestApiMapping.java index 5eb8e6c8..8f9b67ba 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestApiMapping.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestApiMapping.java @@ -54,106 +54,32 @@ public RestApiMapping(String connectorName, String connectorLabel, String connec this.metadata = metadata != null ? Collections.unmodifiableMap(new LinkedHashMap<>(metadata)) : Collections.emptyMap(); } - /** - * Returns the connector name. - * - * @return the connector name - */ - public String getConnectorName() - { - return connectorName; - } + public String getConnectorName() { return connectorName; } + public String getConnectorLabel() { return connectorLabel; } + public String getConnectorDescription() { return connectorDescription; } + public String getBaseUrl() { return baseUrl; } + public String getAuthenticationType() { return authenticationType.getValue(); } + public AuthenticationType getAuthenticationTypeEnum() { return authenticationType; } + public Map getTables() { return tables; } /** - * Returns the connector label. - * - * @return the connector label - */ - public String getConnectorLabel() - { - return connectorLabel; - } - - /** - * Returns the connector description. - * - * @return the connector description - */ - public String getConnectorDescription() - { - return connectorDescription; - } - - /** - * Returns the base URL for all API calls. - * - * @return the base URL (e.g. "https://api.spacexdata.com:443") - */ - public String getBaseUrl() - { - return baseUrl; - } - - /** - * Returns the authentication type for this API. - * - * @return the authentication type: "none", "api_key", "oauth2", or "basic" - */ - public String getAuthenticationType() - { - return authenticationType.getValue(); - } - - /** - * Returns the authentication type enum for this API. - * - * @return the authentication type enum - */ - public AuthenticationType getAuthenticationTypeEnum() - { - return authenticationType; - } - - /** - * Returns the map of table name to table definition. - * - * @return an unmodifiable map of table definitions keyed by table name - */ - public Map getTables() - { - return tables; - } - - /** - * Returns the table definition for the given table name (case-insensitive lookup). - * - * @param tableName - * the table name to look up - * @return the table definition, or null if not found + * Returns the table definition for the given table name. + * Lookup order: + * 1. Exact match (preserves the original case from the JSON DSL key) + * 2. Case-insensitive linear scan (supports callers that normalise to upper/lower case) */ public RestTableDefinition getTable(String tableName) { - if (tableName == null) { - return null; - } - // Try exact match first - final RestTableDefinition def = tables.get(tableName); - if (def != null) { - return def; + if (tableName == null) { return null; } + final RestTableDefinition exact = tables.get(tableName); + if (exact != null) { return exact; } + for (final Map.Entry entry : tables.entrySet()) { + if (entry.getKey().equalsIgnoreCase(tableName)) { return entry.getValue(); } } - // Try upper-case match - return tables.get(tableName.toUpperCase(java.util.Locale.ENGLISH)); + return null; } - /** - * Returns the metadata map from the "$metadata" section of the connector configuration. - * - * @return an unmodifiable map of metadata (connector_source, target_service, connector_type, etc.) - */ - public Map getMetadata() - { - return metadata; - } + public Map getMetadata() { return metadata; } @Override public String toString() @@ -163,5 +89,3 @@ public String toString() + ", metadata=" + metadata + "}"; } } - -// Made with Bob diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestApiMappingLoader.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestApiMappingLoader.java index 99678172..86044d94 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestApiMappingLoader.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestApiMappingLoader.java @@ -8,6 +8,8 @@ import static org.slf4j.LoggerFactory.getLogger; import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -64,12 +66,8 @@ public class RestApiMappingLoader private static final String KEY_MODIFIER = "$key"; private static final String NOTNULL_MODIFIER = "$notnull"; private static final String ARRAY_SUFFIX = "[]"; - - // Array of all supported modifiers for easy extension - private static final String[] ALL_MODIFIERS = { - KEY_MODIFIER, - NOTNULL_MODIFIER - }; + + private static final String[] ALL_MODIFIERS = { KEY_MODIFIER, NOTNULL_MODIFIER }; private RestApiMappingLoader() { @@ -88,7 +86,31 @@ private RestApiMappingLoader() public static RestApiMapping load(String filePath) throws IOException { LOGGER.info("Loading REST API configuration from: {}", filePath); - final String content = new String(Files.readAllBytes(Paths.get(filePath))); + final String content = new String(Files.readAllBytes(Paths.get(filePath)), StandardCharsets.UTF_8); + return parse(content); + } + + /** + * Loads and parses a JSON configuration from an {@link InputStream}. + * + *

    This overload is intended for configs bundled as classpath resources, e.g.: + * 

    +     *   InputStream is = getClass().getResourceAsStream("/forge/mappings/my-connector.json");
+     *   RestApiMapping mapping = RestApiMappingLoader.load(is);
+     *
    + * + *

    The caller is responsible for closing the stream. + * + * @param inputStream + * the input stream to read the JSON configuration from; must not be null + * @return the parsed {@link RestApiMapping} + * @throws IOException + * if the stream cannot be read or the JSON cannot be parsed + */ + public static RestApiMapping load(InputStream inputStream) throws IOException + { + LOGGER.info("Loading REST API configuration from InputStream"); + final String content = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8); return parse(content); } @@ -111,7 +133,8 @@ public static RestApiMapping parse(String jsonContent) throws IOException final Map metadataMap = parseMetadata(root); LOGGER.info("Loaded REST API configuration: connectorName='{}', authenticationType='{}', {} tables, metadata={}", - metadata.connectorName, authenticationType.getValue(), tables.size(), metadataMap.isEmpty() ? "none" : metadataMap.keySet()); + metadata.connectorName, authenticationType.getValue(), tables.size(), + metadataMap.isEmpty() ? "none" : metadataMap.keySet()); return new RestApiMapping(metadata.connectorName, metadata.connectorLabel, metadata.connectorDescription, baseUrl, authenticationType, tables, metadataMap); } @@ -124,34 +147,25 @@ private static ConnectorMetadata parseConnectorMetadata(JsonNode root) return new ConnectorMetadata(connectorName, connectorLabel, connectorDescription); } - /** - * Parses the $metadata section from the JSON configuration. - * - * @param root - * the root JSON node - * @return a map of metadata key-value pairs, or an empty map if no metadata is present - */ private static Map parseMetadata(JsonNode root) { final Map metadata = new LinkedHashMap<>(); final JsonNode metadataNode = root.get(METADATA_KEY); - + if (metadataNode != null && metadataNode.isObject()) { final Iterator> fields = metadataNode.fields(); while (fields.hasNext()) { final Map.Entry field = fields.next(); - final String key = field.getKey(); final JsonNode value = field.getValue(); - if (value != null && !value.isNull()) { - metadata.put(key, value.asText()); + metadata.put(field.getKey(), value.asText()); } } LOGGER.debug("Parsed metadata: {}", metadata); } else { LOGGER.debug("No $metadata section found in configuration"); } - + return metadata; } @@ -213,7 +227,6 @@ private static RestTableEntry parseTable(Map.Entry tableEntry) final List fields = parseFields(tableNode, ""); logParsedTable(tableName, dataPath, paginationConfig, fields.size()); - return new RestTableEntry(tableName, new RestTableDefinition(path, dataPath, paginationConfig, fields)); } @@ -227,7 +240,8 @@ private static String parseTablePath(String tableName, JsonNode tableNode) return pathNode.get(0).asText(); } - private static void logParsedTable(String tableName, String dataPath, PaginationConfig paginationConfig, int fieldCount) + private static void logParsedTable(String tableName, String dataPath, PaginationConfig paginationConfig, + int fieldCount) { if (paginationConfig != null && dataPath != null) { LOGGER.debug("Loaded table '{}' with data path '{}', pagination type '{}', and {} fields", @@ -279,31 +293,22 @@ private static List parseFields(JsonNode tableNode, String final String rawKey = fieldEntry.getKey(); final JsonNode fieldValue = fieldEntry.getValue(); - // Skip special keys that start with $ if (rawKey.startsWith("$")) { continue; } - // Check if this is a nested object field (key ends with []) final boolean isNestedObject = rawKey.endsWith(ARRAY_SUFFIX); - final String baseFieldName = isNestedObject ? rawKey.substring(0, rawKey.length() - ARRAY_SUFFIX.length()) : rawKey; + final String baseFieldName = isNestedObject + ? rawKey.substring(0, rawKey.length() - ARRAY_SUFFIX.length()) : rawKey; final String fieldName = prefix + baseFieldName; if (isNestedObject && fieldValue.isObject()) { - // Nested object with [] — flatten its fields with dot separator - final List nestedFields = parseFields(fieldValue, fieldName + "."); - fields.addAll(nestedFields); + fields.addAll(parseFields(fieldValue, fieldName + ".")); } else { - // Simple field with type string like "VARCHAR,$key,$notnull" or "INTEGER" final String rawType = fieldValue.asText(); - - // Parse modifiers final boolean isKey = rawType.contains(KEY_MODIFIER); final boolean isNotNull = rawType.contains(NOTNULL_MODIFIER); - - // Remove all modifiers from the type string final String typeString = removeModifiers(rawType); - fields.add(new RestFieldDefinition(fieldName, typeString, isKey, isNotNull)); } } @@ -311,36 +316,17 @@ private static List parseFields(JsonNode tableNode, String return fields; } - /** - * Removes all known modifiers from a type string. - *

    - * This method handles modifiers in any position (beginning, middle, or end) - * and with or without surrounding commas. - * - * @param rawType the raw type string with potential modifiers (e.g., "VARCHAR,$key,$notnull") - * @return the clean type string without modifiers (e.g., "VARCHAR") - */ private static String removeModifiers(String rawType) { String result = rawType; - - // Remove each modifier in all possible positions for (final String modifier : ALL_MODIFIERS) { - result = result.replace("," + modifier, ""); // Remove ",modifier" - result = result.replace(modifier + ",", ""); // Remove "modifier," - result = result.replace(modifier, ""); // Remove standalone "modifier" + result = result.replace("," + modifier, ""); + result = result.replace(modifier + ",", ""); + result = result.replace(modifier, ""); } - return result.trim(); } - /** - * Parses the pagination configuration from a table JSON node. - * - * @param tableNode - * the JSON object representing a table - * @return the pagination configuration, or null if no pagination is configured - */ private static PaginationConfig parsePaginationConfig(JsonNode tableNode) { final JsonNode paginationNode = tableNode.get(PAGINATION_KEY); @@ -414,7 +400,8 @@ private static PaginationType parsePaginationType(JsonNode paginationNode) return paginationType; } - private static String parseRequiredPaginationField(JsonNode paginationNode, String fieldName, String warningMessage) + private static String parseRequiredPaginationField(JsonNode paginationNode, String fieldName, + String warningMessage) { final String fieldValue = parseOptionalText(paginationNode, fieldName); if (fieldValue == null) { @@ -449,5 +436,3 @@ private RestTableEntry(String tableName, RestTableDefinition tableDefinition) } } } - -// Made with Bob diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestConnector.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestConnector.java index a31f5579..f3d00d67 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestConnector.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestConnector.java @@ -23,19 +23,26 @@ import com.ibm.wdp.connect.common.sdk.api.models.CustomFlightAssetDescriptor; import com.ibm.wdp.connect.common.sdk.api.models.CustomFlightAssetsCriteria; import com.ibm.wdp.connect.common.sdk.api.models.DiscoveredAssetType; +import com.ibm.wdp.connect.sdk.connector.AssetDescriptor; +import com.ibm.wdp.connect.sdk.connector.DiscoveryCriteria; +import com.ibm.wdp.connect.sdk.connector.SdkConnector; /** * An Arrow-based connector for connecting to a REST API data source. * + *

    Implements both the legacy {@link Connector} interface (for backward compatibility with + * existing SDK tooling) and the new {@link SdkConnector} interface (for the Arrow-native path + * through {@link AbstractSdkConnectorFlightProducer}). + * *

    The connector reads a JSON configuration file that describes the API endpoints * and their field schemas. It uses this configuration to discover assets and read data * from the REST API in a streaming fashion. - * + * *

    Each connector instance is associated with a specific datasource type (connector name) * and loads its configuration from the factory's cache. */ -@SuppressWarnings({ "PMD.AvoidDollarSigns", "PMD.ClassNamingConventions" }) -public class RestConnector implements Connector +public class RestConnector implements Connector, + SdkConnector { private static final Logger LOGGER = getLogger(RestConnector.class); @@ -43,7 +50,7 @@ public class RestConnector implements ConnectorHierarchical discovery: - *

      - *
    • Path "/" - returns all table names as containers (no fields)
      • - *
    • Path "/{tableName}" - returns the specific table as a dataset with fields
      • - *
    */ @Override public List discoverAssets(CustomFlightAssetsCriteria criteria) throws Exception @@ -118,7 +165,6 @@ public List discoverAssets(CustomFlightAssetsCriter final List assets = new ArrayList<>(); if ("/".equals(path)) { - // Root discovery: return all tables as containers (no fields) for (final Map.Entry entry : apiMapping.getTables().entrySet()) { final String tableName = entry.getKey(); @@ -129,9 +175,7 @@ public List discoverAssets(CustomFlightAssetsCriter descriptor.setDatasourceTypeName(datasourceTypeName); descriptor.setConnectionProperties(criteria.getConnectionProperties()); descriptor.setHasChildren(true); - // No fields at this level - // Asset type: container, not a dataset final DiscoveredAssetType assetType = new DiscoveredAssetType(); assetType.setType("table"); assetType.setDataset(false); @@ -141,11 +185,9 @@ public List discoverAssets(CustomFlightAssetsCriter assets.add(descriptor); LOGGER.debug("Discovered table container: {}", tableName); } - } - else if (path != null && path.startsWith("/") && !path.substring(1).contains("/")) { - // Table-level discovery: return the specific table with fields - final String tableName = path.substring(1); // Remove leading "/" - final RestTableDefinition tableDef = apiMapping.getTables().get(tableName); + } else if (path != null && path.startsWith("/") && !path.substring(1).contains("/")) { + final String tableName = path.substring(1); + final RestTableDefinition tableDef = apiMapping.getTable(tableName); if (tableDef != null) { final CustomFlightAssetDescriptor descriptor = new CustomFlightAssetDescriptor(); @@ -157,7 +199,6 @@ else if (path != null && path.startsWith("/") && !path.substring(1).contains("/" descriptor.setHasChildren(false); descriptor.setFields(RestFieldTypeMapper.toAssetFields(tableDef.getFields())); - // Asset type: dataset, not a container final DiscoveredAssetType assetType = new DiscoveredAssetType(); assetType.setType("table"); assetType.setDataset(true); @@ -166,12 +207,10 @@ else if (path != null && path.startsWith("/") && !path.substring(1).contains("/" assets.add(descriptor); LOGGER.debug("Discovered table dataset: {}", tableName); - } - else { + } else { LOGGER.warn("Table not found in mapping: {}", tableName); } - } - else { + } else { LOGGER.warn("Unsupported discovery path supplied for discovery"); } @@ -181,9 +220,6 @@ else if (path != null && path.startsWith("/") && !path.substring(1).contains("/" /** * {@inheritDoc} - * - *

    Returns the Arrow schema for the given asset by looking up the table - * definition in the API mapping. */ @Override public Schema getSchema(CustomFlightAssetDescriptor asset) throws Exception @@ -234,10 +270,6 @@ public ConnectionActionResponse performAction(String action, ConnectionActionCon @Override public void close() throws Exception { - // No persistent resources to close LOGGER.debug("RestConnector closed"); } - } - -// Made with Bob diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestConnectorFactory.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestConnectorFactory.java index 7e5a9dce..ad326a35 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestConnectorFactory.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestConnectorFactory.java @@ -8,25 +8,31 @@ import static org.slf4j.LoggerFactory.getLogger; import java.io.File; +import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; +import java.util.Locale; import java.util.Map; import org.slf4j.Logger; -import com.ibm.connect.sdk.api.Connector; -import com.ibm.connect.sdk.api.PooledConnectorFactory; -import com.ibm.wdp.connect.common.sdk.api.models.ConnectionProperties; -import com.ibm.wdp.connect.common.sdk.api.models.CustomFlightDatasourceTypes; +import com.ibm.wdp.connect.sdk.connector.ConnectionProperties; +import com.ibm.wdp.connect.sdk.connector.SdkConnector; +import com.ibm.wdp.connect.sdk.connector.SdkConnectorFactory; +import com.ibm.wdp.connect.sdk.connector.SdkDatasourceTypes; /** * A factory for creating REST connectors. - * - *

    This factory supports multiple REST connectors, each defined by a separate JSON configuration file - * in the /config/mappings directory. Each configuration file defines a unique connector with its own - * name, label, description, and API endpoints. + * + *

    Implements {@link SdkConnectorFactory} for the Arrow-native path through + * {@link RestFlightProducer}. + * + *

    This factory supports multiple REST connectors, each defined by a separate JSON configuration + * file in the /config/mappings directory. Each configuration file defines a unique connector with + * its own name, label, description, and API endpoints. */ -@SuppressWarnings({ "PMD.AvoidDollarSigns", "PMD.ClassNamingConventions" }) -public class RestConnectorFactory extends PooledConnectorFactory +public class RestConnectorFactory implements SdkConnectorFactory { private static final Logger LOGGER = getLogger(RestConnectorFactory.class); private static final RestConnectorFactory INSTANCE = new RestConnectorFactory(); @@ -45,7 +51,6 @@ public class RestConnectorFactory extends PooledConnectorFactory */ private RestConnectorFactory() { - super(); loadAllConfigurations(); } @@ -70,7 +75,7 @@ private void loadAllConfigurations() return; } - final File[] jsonFiles = configDir.listFiles((dir, name) -> name.toLowerCase(java.util.Locale.ENGLISH).endsWith(".json")); + final File[] jsonFiles = configDir.listFiles((dir, name) -> name.toLowerCase(Locale.ENGLISH).endsWith(".json")); if (jsonFiles == null || jsonFiles.length == 0) { LOGGER.warn("No .json configuration files found in '{}'. No REST connectors will be available.", CONFIG_DIRECTORY); return; @@ -84,12 +89,11 @@ private void loadAllConfigurations() final RestApiMapping mapping = RestApiMappingLoader.load(filePath); final String connectorName = mapping.getConnectorName(); - // Cache the mapping and create datasource type configCache.put(connectorName, mapping); datasourceTypeCache.put(connectorName, new RestDatasourceType(mapping, filePath)); LOGGER.info("Loaded REST connector '{}' from file: {}", connectorName, configFile.getName()); - } catch (java.io.IOException e) { + } catch (IOException e) { LOGGER.error("I/O error loading configuration from file '{}': {}", configFile.getName(), e.getMessage(), e); } catch (IllegalArgumentException e) { LOGGER.error("Invalid configuration in file '{}': {}", configFile.getName(), e.getMessage(), e); @@ -113,32 +117,56 @@ public RestApiMapping getConfiguration(String datasourceTypeName) return configCache.get(datasourceTypeName); } + /** + * Registers a pre-loaded {@link RestApiMapping} in the factory's cache. + * + *

    This allows alternative loading strategies (e.g. classpath-based factories) to + * make their mappings available to {@link RestConnector} instances without requiring + * the configurations to reside on the filesystem at {@value #CONFIG_DIRECTORY}. + * + *

    If a mapping with the same connector name is already registered, it will be + * replaced. + * + * @param mapping + * the REST API mapping to register; must not be null + */ + public void register(RestApiMapping mapping) + { + final String connectorName = mapping.getConnectorName(); + configCache.put(connectorName, mapping); + datasourceTypeCache.put(connectorName, new RestDatasourceType(mapping, "")); + LOGGER.info("Registered REST connector '{}' from external source", connectorName); + } + + // ---- SdkConnectorFactory interface ---- + /** * {@inheritDoc} */ @Override - protected Connector createNewConnector(String datasourceTypeName, ConnectionProperties properties) + public SdkDatasourceTypes getDatasourceTypes() { - if (configCache.containsKey(datasourceTypeName)) { - return new RestConnector(datasourceTypeName, properties); + final List typeNames = new ArrayList<>(configCache.keySet()); + if (typeNames.isEmpty()) { + typeNames.add("__rest__"); } - throw new UnsupportedOperationException(RestMsgs.DATASOURCE_TYPE_NOT_SUPPORTED.format(datasourceTypeName)); + return new SdkDatasourceTypes(typeNames); } /** * {@inheritDoc} - * - *

    Returns all datasource types loaded from JSON configuration files. */ @Override - public CustomFlightDatasourceTypes getDatasourceTypes() - { - final CustomFlightDatasourceTypes types = new CustomFlightDatasourceTypes(); - for (final RestDatasourceType datasourceType : datasourceTypeCache.values()) { - types.addDatasourceTypesItem(datasourceType); + public SdkConnector createConnector(String datasourceTypeName, + ConnectionProperties properties) { + if (configCache.containsKey(datasourceTypeName)) { + final com.ibm.wdp.connect.common.sdk.api.models.ConnectionProperties modelProps + = new com.ibm.wdp.connect.common.sdk.api.models.ConnectionProperties(); + if (properties != null) { + modelProps.putAll(properties.asMap()); + } + return new RestConnector(datasourceTypeName, modelProps); } - return types; + throw new UnsupportedOperationException(RestMsgs.DATASOURCE_TYPE_NOT_SUPPORTED.format(datasourceTypeName)); } } - -// Made with Bob diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestConnectorUtils.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestConnectorUtils.java index e8e9dad1..5d53ac66 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestConnectorUtils.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestConnectorUtils.java @@ -5,6 +5,8 @@ /* *************************************************** */ package com.ibm.connect.restconnector; +import java.util.Locale; + import com.ibm.wdp.connect.common.sdk.api.models.CustomFlightAssetDescriptor; /** @@ -44,25 +46,50 @@ public static String resolveTableName(CustomFlightAssetDescriptor asset) final String[] segments = path.split("/"); for (int i = segments.length - 1; i >= 0; i--) { if (!segments[i].isEmpty()) { - return segments[i].toUpperCase(java.util.Locale.ENGLISH); + return segments[i].toUpperCase(Locale.ENGLISH); } } } // Fall back to asset name if (asset.getName() != null && !asset.getName().isEmpty()) { - return asset.getName().toUpperCase(java.util.Locale.ENGLISH); + return asset.getName().toUpperCase(Locale.ENGLISH); } // Fall back to asset ID if (asset.getId() != null && !asset.getId().isEmpty()) { - return asset.getId().toUpperCase(java.util.Locale.ENGLISH); + return asset.getId().toUpperCase(Locale.ENGLISH); } throw new IllegalArgumentException("Cannot determine table name from asset: path=" + path + ", name=" + asset.getName() + ", id=" + asset.getId()); } + /** + * Resolves the table name from a path and a fallback name. + * + * @param path + * the asset path (may be null or empty) + * @param name + * the asset name (fallback) + * @return the table name (upper-case) + */ + public static String resolveTableName(String path, String name) + { + if (path != null && !path.isEmpty()) { + final String[] segments = path.split("/"); + for (int i = segments.length - 1; i >= 0; i--) { + if (!segments[i].isEmpty()) { + return segments[i].toUpperCase(Locale.ENGLISH); + } + } + } + if (name != null && !name.isEmpty()) { + return name.toUpperCase(Locale.ENGLISH); + } + throw new IllegalArgumentException("Cannot determine table name from path=" + path + ", name=" + name); + } + } // Made with Bob \ No newline at end of file diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestDatasourceType.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestDatasourceType.java index b4278402..1dac1189 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestDatasourceType.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestDatasourceType.java @@ -10,6 +10,7 @@ import java.text.SimpleDateFormat; import java.util.Collections; import java.util.Date; +import java.util.Locale; import java.util.Map; import java.util.TimeZone; @@ -33,7 +34,6 @@ *

    Each instance of RestDatasourceType represents one connector defined by one JSON configuration file. * Multiple instances can be created from multiple configuration files in the /config/mappings directory. */ -@SuppressWarnings({ "PMD.AvoidDollarSigns", "PMD.ClassNamingConventions" }) public class RestDatasourceType extends CustomFlightDatasourceType { private final String configFilePath; @@ -75,7 +75,7 @@ public RestDatasourceType(RestApiMapping mapping, String configFilePath) if (createdAtStr != null && !createdAtStr.isEmpty()) { try { // Parse ISO 8601 date-time format (e.g., "2026-05-06T13:00:00Z") - final SimpleDateFormat iso8601Format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", java.util.Locale.US); + final SimpleDateFormat iso8601Format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US); iso8601Format.setTimeZone(TimeZone.getTimeZone("UTC")); final Date createdAt = iso8601Format.parse(createdAtStr); metadata.setCreatedAt(createdAt); diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestDiscoveryInteraction.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestDiscoveryInteraction.java new file mode 100644 index 00000000..07463420 --- /dev/null +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestDiscoveryInteraction.java @@ -0,0 +1,108 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.connect.restconnector; + +import static org.slf4j.LoggerFactory.getLogger; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; + +import com.ibm.wdp.connect.sdk.connector.AssetDescriptor; +import com.ibm.wdp.connect.sdk.connector.DiscoveryCriteria; +import com.ibm.wdp.connect.sdk.connector.SdkDiscoveryInteraction; + +/** + * Discovery interaction for a REST API connector. + * + *

    Translates the connector's hierarchical path-based discovery into a list of + * {@link AssetDescriptor} objects. The discovery logic mirrors the old + * {@code RestConnector.discoverAssets()} implementation: + *

      + *
    • Path "/" — returns all tables as containers (no fields)
      • + *
    • Path "/{tableName}" — returns the specific table as a dataset
      • + *
    + */ +public class RestDiscoveryInteraction implements SdkDiscoveryInteraction +{ + private static final Logger LOGGER = getLogger(RestDiscoveryInteraction.class); + + private final RestConnector connector; + + /** + * Creates a REST discovery interaction. + * + * @param connector + * the connector providing the loaded API mapping + */ + public RestDiscoveryInteraction(RestConnector connector) + { + this.connector = connector; + } + + /** + * {@inheritDoc} + */ + @Override + public List discoverAssets(DiscoveryCriteria criteria) { + final RestApiMapping apiMapping = connector.getApiMapping(); + if (apiMapping == null) { + throw new IllegalStateException("API mapping not loaded. Call connect() first."); + } + + final String path = criteria.getPath(); + final List assets = new ArrayList<>(); + + if ("/".equals(path)) { + // Root discovery: return all tables as containers (no fields) + for (final Map.Entry entry : apiMapping.getTables().entrySet()) { + final String tableName = entry.getKey(); + assets.add(new AssetDescriptor( + tableName, + tableName, + "/" + tableName, + criteria.getDatasourceTypeName(), + criteria.getConnectionProperties() != null ? criteria.getConnectionProperties().asMap() : null, + true, // hasChildren + 0)); + LOGGER.debug("Discovered table container: {}", tableName); + } + } else if (path != null && path.startsWith("/") && !path.substring(1).contains("/")) { + // Table-level discovery: return the specific table + final String tableName = path.substring(1); + final RestTableDefinition tableDef = apiMapping.getTable(tableName); + + if (tableDef != null) { + assets.add(new AssetDescriptor( + tableName, + tableName, + path, + criteria.getDatasourceTypeName(), + criteria.getConnectionProperties() != null ? criteria.getConnectionProperties().asMap() : null, + false, // not a container + 0)); + LOGGER.debug("Discovered table dataset: {}", tableName); + } else { + LOGGER.warn("Table not found in mapping: {}", tableName); + } + } else { + LOGGER.warn("Unsupported discovery path: {}", path); + } + + LOGGER.info("Discovered {} assets", assets.size()); + return assets; + } + + /** + * {@inheritDoc} + */ + @Override + public void close() { + // No persistent resources to close + } +} diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestFieldDefinition.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestFieldDefinition.java index a9083bd0..f39a8570 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestFieldDefinition.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestFieldDefinition.java @@ -17,18 +17,6 @@ public class RestFieldDefinition private final boolean isKey; private final boolean isNotNull; - /** - * Creates a field definition. - * - * @param name - * the field name (may be a nested path like "headquarters.address") - * @param typeString - * the raw type string from the JSON file (e.g. "VARCHAR") - * @param isKey - * true if this field is marked as a key ($key modifier) - * @param isNotNull - * true if this field is marked as not null ($notnull modifier) - */ public RestFieldDefinition(String name, String typeString, boolean isKey, boolean isNotNull) { this.name = name; @@ -37,45 +25,10 @@ public RestFieldDefinition(String name, String typeString, boolean isKey, boolea this.isNotNull = isNotNull; } - /** - * Returns the field name (may be a nested path like "headquarters.address"). - * - * @return the field name - */ - public String getName() - { - return name; - } - - /** - * Returns the raw type string from the JSON file. - * - * @return the raw type string - */ - public String getTypeString() - { - return typeString; - } - - /** - * Returns true if this field is a primary key. - * - * @return true if this field is a primary key - */ - public boolean isKey() - { - return isKey; - } - - /** - * Returns true if this field is marked as not null. - * - * @return true if this field is not null - */ - public boolean isNotNull() - { - return isNotNull; - } + public String getName() { return name; } + public String getTypeString() { return typeString; } + public boolean isKey() { return isKey; } + public boolean isNotNull() { return isNotNull; } @Override public String toString() @@ -84,5 +37,3 @@ public String toString() + ", isNotNull=" + isNotNull + "}"; } } - -// Made with Bob diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestFlightProducer.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestFlightProducer.java index bdbf5bf0..4380ef61 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestFlightProducer.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestFlightProducer.java @@ -5,21 +5,23 @@ /* *************************************************** */ package com.ibm.connect.restconnector; -import com.ibm.connect.sdk.api.ConnectorFactory; -import com.ibm.connect.sdk.api.ConnectorFlightProducer; +import com.ibm.connect.sdk.api.AbstractSdkConnectorFlightProducer; +import com.ibm.wdp.connect.sdk.connector.SdkConnectorFactory; /** - * A Flight producer for connectors. + * A Flight producer for REST API connectors. + * + *

    Uses the Arrow-native path via {@link AbstractSdkConnectorFlightProducer} and + * {@link RestConnectorFactory}. */ -@SuppressWarnings({ "PMD.AvoidDollarSigns", "PMD.ClassNamingConventions" }) -public class RestFlightProducer extends ConnectorFlightProducer +public class RestFlightProducer extends AbstractSdkConnectorFlightProducer { /** * {@inheritDoc} */ @Override - protected ConnectorFactory getConnectorFactory() + protected SdkConnectorFactory getSdkConnectorFactory() { return RestConnectorFactory.getInstance(); } diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestSourceInteraction.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestSourceInteraction.java index 0b5e1f12..7a5d1369 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestSourceInteraction.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestSourceInteraction.java @@ -5,19 +5,6 @@ /* *************************************************** */ package com.ibm.connect.restconnector; -import static org.slf4j.LoggerFactory.getLogger; - -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.UUID; - -import org.apache.arrow.flight.Ticket; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.types.pojo.Schema; -import org.slf4j.Logger; - import com.ibm.connect.sdk.api.ArrowConversions; import com.ibm.connect.sdk.api.Connector; import com.ibm.connect.sdk.api.SourceInteraction; @@ -25,32 +12,44 @@ import com.ibm.connect.sdk.util.ModelMapper; import com.ibm.wdp.connect.common.sdk.api.models.CustomFlightAssetDescriptor; import com.ibm.wdp.connect.common.sdk.api.models.CustomFlightAssetField; +import com.ibm.wdp.connect.sdk.connector.AssetDescriptor; +import com.ibm.wdp.connect.sdk.connector.RowWriter; +import com.ibm.wdp.connect.sdk.connector.SdkSourceInteraction; +import org.apache.arrow.flight.Ticket; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.Schema; +import org.slf4j.Logger; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.*; + +import static org.slf4j.LoggerFactory.getLogger; /** * An interaction with a REST API asset as a source. * - *

    Reads data from a REST API endpoint defined in the .rest mapping file, + *

    Implements both the legacy {@link SourceInteraction} interface (for API compatibility) and + * the new {@link SdkSourceInteraction} interface (push-based via {@link #stream(RowWriter)}, + * used by the Arrow-native path through {@link RestFlightProducer}). + * + *

    Reads data from a REST API endpoint defined in the JSON mapping configuration, * converts the JSON response to Arrow format in a streaming fashion. */ @SuppressWarnings({ "PMD.AvoidDollarSigns", "PMD.ClassNamingConventions" }) -public class RestSourceInteraction implements SourceInteraction> +public class RestSourceInteraction implements SourceInteraction>, SdkSourceInteraction { private static final Logger LOGGER = getLogger(RestSourceInteraction.class); - private static final int DEFAULT_BATCH_SIZE = 1000; - private final ModelMapper modelMapper = new ModelMapper(); private final RestConnector connector; - private final CustomFlightAssetDescriptor asset; + private final String tableName; private final RestTableDefinition tableDef; private final List assetFields; - private VectorSchemaRoot vectorSchemaRoot; - private Iterator batchIterator; - private JsonToRecordStream jsonStream; - /** - * Creates a REST source interaction. + * Creates a REST source interaction from a legacy {@link CustomFlightAssetDescriptor}. * * @param connector * the connector managing the connection to the data source @@ -60,19 +59,40 @@ public class RestSourceInteraction implements SourceInteraction> * a Flight ticket to read a partition or null to get tickets * @throws Exception */ - public RestSourceInteraction(RestConnector connector, CustomFlightAssetDescriptor asset, Ticket ticket) throws Exception + public RestSourceInteraction(RestConnector connector, CustomFlightAssetDescriptor asset, Ticket ticket) + throws Exception + { + this(connector, RestConnectorUtils.resolveTableName(asset), ticket); + } + + /** + * Creates a REST source interaction from an SDK {@link AssetDescriptor}. + * + * @param connector + * the connector managing the connection to the data source + * @param asset + * the SDK asset descriptor from which to read + * @param ticket + * a Flight ticket to read a partition or null to get tickets + * @throws Exception + */ + public RestSourceInteraction(RestConnector connector, AssetDescriptor asset, Ticket ticket) throws Exception + { + this(connector, RestConnectorUtils.resolveTableName(asset.getPath(), asset.getName()), ticket); + } + + /** + * Common constructor. + */ + private RestSourceInteraction(RestConnector connector, String resolvedTableName, Ticket ticket) throws Exception { if (connector == null) { throw new IllegalArgumentException(RestMsgs.MISSING_CONNECTOR.format()); } this.connector = connector; - this.asset = asset; - - // Resolve the table name from the asset path or interaction properties - final String tableName = RestConnectorUtils.resolveTableName(asset); + this.tableName = resolvedTableName; LOGGER.debug("Creating source interaction for table: {}", tableName); - // Look up the table definition from the loaded API mapping final RestApiMapping apiMapping = connector.getApiMapping(); if (apiMapping == null) { throw new IllegalStateException("API mapping not loaded. Call connect() first."); @@ -83,7 +103,6 @@ public RestSourceInteraction(RestConnector connector, CustomFlightAssetDescripto + "Available tables: " + apiMapping.getTables().keySet()); } - // Convert field definitions to asset fields assetFields = RestFieldTypeMapper.toAssetFields(tableDef.getFields()); if (ticket != null) { @@ -92,12 +111,13 @@ public RestSourceInteraction(RestConnector connector, CustomFlightAssetDescripto } } + // ---- SdkSourceInteraction interface (new path) ---- + /** * {@inheritDoc} */ @Override - public Schema getSchema() throws Exception - { + public Schema getSchema() { return ArrowConversions.toArrow(assetFields); } @@ -107,7 +127,6 @@ public Schema getSchema() throws Exception @Override public List getTickets() throws Exception { - // Single partition — return one ticket with requestId and partitionIndex final String requestId = UUID.randomUUID().toString(); final TicketInfo ticketInfo = new TicketInfo() .requestId(requestId) @@ -118,218 +137,122 @@ public List getTickets() throws Exception /** * {@inheritDoc} + * + *

    Fetches all data from the REST API endpoint and pushes each row into the writer. + * Uses {@link JsonToArrowStream} from the forge engine. */ @Override - public void beginStream(BufferAllocator allocator) throws Exception + public void stream(RowWriter writer) throws Exception { - final Schema schema = ArrowConversions.toArrow(assetFields); - vectorSchemaRoot = VectorSchemaRoot.create(schema, allocator); - - // Build the full URL using connection properties (host and port) + path final String url = buildUrl(); - LOGGER.info("Starting stream for requested asset"); - - // Build authentication headers from connection properties - final java.util.Map authHeaders = buildAuthHeaders(); + LOGGER.info("Starting stream for table: {}", tableName); - // Create the streaming JSON-to-Record iterator with data path, authentication headers, and pagination config - jsonStream = new JsonToRecordStream(url, tableDef.getDataPath(), tableDef.getFields(), authHeaders, tableDef.getPaginationConfig()); + final Map authHeaders = buildAuthHeaders(); - // Determine batch size - final int batchSize = (asset.getBatchSize() != null && asset.getBatchSize() > 0) - ? asset.getBatchSize() - : DEFAULT_BATCH_SIZE; + final JsonToArrowStream jsonStream = new JsonToArrowStream( + url, + tableDef.getDataPath(), + tableDef.getFields(), + authHeaders, + tableDef.getPaginationConfig()); - LOGGER.debug("Using batch size: {}", batchSize); - - // Create the Arrow batch iterator using ArrowConversions - batchIterator = ArrowConversions.toArrow(vectorSchemaRoot, jsonStream, batchSize); + try { + jsonStream.streamTo(writer); + } finally { + jsonStream.close(); + } } + // ---- SourceInteraction interface (legacy stubs — pull path not supported) ---- + /** * {@inheritDoc} + * + * @deprecated The pull-based path is not supported in this implementation. + * Use {@link #stream(RowWriter)} instead via the SDK connector path. */ + @Deprecated @Override - public boolean hasNextBatch() throws Exception - { - return batchIterator != null && batchIterator.hasNext(); + public void beginStream(BufferAllocator allocator) { + throw new UnsupportedOperationException( + "Pull-based streaming is not supported. Use stream(RowWriter) instead."); } /** * {@inheritDoc} + * + * @deprecated The pull-based path is not supported in this implementation. */ + @Deprecated @Override - public VectorSchemaRoot nextBatch() throws Exception - { - return batchIterator.next(); + public boolean hasNextBatch() { + return false; } /** * {@inheritDoc} + * + * @deprecated The pull-based path is not supported in this implementation. */ + @Deprecated @Override - public void close() throws Exception - { - if (jsonStream != null) { - try { - jsonStream.close(); - } - catch (Exception e) { - LOGGER.warn("Error closing JSON stream", e); - } - } - if (vectorSchemaRoot != null) { - try { - vectorSchemaRoot.close(); - } - catch (Exception e) { - LOGGER.warn("Error closing VectorSchemaRoot", e); - } - } + public VectorSchemaRoot nextBatch() { + throw new UnsupportedOperationException( + "Pull-based streaming is not supported. Use stream(RowWriter) instead."); } /** - * Builds the full URL for the REST API call using connection properties. - * - *

    The URL is constructed from: - *

      - *
    • Protocol: determined from port (443 = https, otherwise http)
      • - *
    • Host: from connection properties or default from config
      • - *
    • Port: from connection properties or default from config
      • - *
    • Path: from table definition
      • - *
    - * - * @return the full URL - * @throws Exception if connection properties cannot be retrieved + * {@inheritDoc} */ - private String buildUrl() throws Exception - { - // Get connection properties from asset - final com.ibm.wdp.connect.common.sdk.api.models.ConnectionProperties connProps = asset.getConnectionProperties(); - - // Extract host and port from connection properties - // ConnectionProperties extends HashMap - String host = null; - Integer port = null; - - if (connProps != null) { - final Object hostObj = connProps.get("host"); - final Object portObj = connProps.get("port"); - - if (hostObj != null) { - host = hostObj.toString(); - } - if (portObj != null) { - if (portObj instanceof Number) { - port = ((Number) portObj).intValue(); - } else { - try { - port = Integer.parseInt(portObj.toString()); - } catch (NumberFormatException e) { - LOGGER.warn("Invalid port value: {}", portObj); - } - } - } - } - - // Fall back to defaults from config if not provided - if (host == null || port == null) { - try { - final java.net.URL configUrl = new java.net.URL(connector.getApiMapping().getBaseUrl()); - if (host == null) { - host = configUrl.getHost(); - } - if (port == null) { - port = configUrl.getPort(); - if (port == -1) { - port = "https".equalsIgnoreCase(configUrl.getProtocol()) ? 443 : 80; - } - } - } catch (java.net.MalformedURLException e) { - LOGGER.error("Failed to parse base URL from config: {}", connector.getApiMapping().getBaseUrl(), e); - throw new IllegalStateException("Invalid base URL in configuration", e); + @Override + public void close() { + // No persistent resources to close + } + + // ---- private helpers ---- + + private String buildUrl() { + String host; + Integer port; + + try { + final URL configUrl = new URL(connector.getApiMapping().getBaseUrl()); + host = configUrl.getHost(); + port = configUrl.getPort(); + if (port == -1) { + port = "https".equalsIgnoreCase(configUrl.getProtocol()) ? 443 : 80; } + } catch (MalformedURLException e) { + LOGGER.error("Failed to parse base URL from config: {}", connector.getApiMapping().getBaseUrl(), e); + throw new IllegalStateException("Invalid base URL in configuration", e); } - - // Determine protocol based on port + final String protocol = (port == 443) ? "https" : "http"; - - // Build the URL final String url = protocol + "://" + host + ":" + port + tableDef.getPath(); LOGGER.debug("Built request URL from configured host and port"); - return url; } - /** - * Builds authentication headers from connection properties based on the authentication type. - * - * @return a map of HTTP headers for authentication, or null if no authentication is configured - * @throws Exception if authentication properties are missing or invalid - */ - private java.util.Map buildAuthHeaders() throws Exception + private Map buildAuthHeaders() { - final AuthenticationType authType = connector.getApiMapping().getAuthenticationTypeEnum(); - + final AuthenticationType authType + = connector.getApiMapping().getAuthenticationTypeEnum(); + if (authType == AuthenticationType.NONE) { - // No authentication required LOGGER.debug("No authentication configured"); return null; } - // Get connection properties from asset - final com.ibm.wdp.connect.common.sdk.api.models.ConnectionProperties connProps = asset.getConnectionProperties(); - if (connProps == null) { - LOGGER.warn("No connection properties provided for configured authentication"); - return null; - } - - final java.util.Map headers = new java.util.HashMap<>(); + final Map headers = new HashMap<>(); if (authType == AuthenticationType.API_KEY) { - // API Key authentication using Authorization header - final Object apiKeyObj = connProps.get("api_key"); - if (apiKeyObj != null) { - final String apiKey = apiKeyObj.toString(); - headers.put("Authorization", "ApiKey " + apiKey); - LOGGER.debug("Using API Key authentication"); - } else { - LOGGER.warn("API key not provided in connection properties"); - } - } - else if (authType == AuthenticationType.OAUTH2) { - // OAuth 2.0 Bearer Token authentication - final Object tokenObj = connProps.get("bearer_token"); - if (tokenObj != null) { - final String token = tokenObj.toString(); - headers.put("Authorization", "Bearer " + token); - LOGGER.debug("Using OAuth 2.0 Bearer Token authentication"); - } else { - LOGGER.warn("Bearer token not provided in connection properties"); - } - } - else if (authType == AuthenticationType.BASIC) { - // Basic authentication (username:password encoded in Base64) - final Object usernameObj = connProps.get("username"); - final Object passwordObj = connProps.get("password"); - - if (usernameObj != null && passwordObj != null) { - final String username = usernameObj.toString(); - final String password = passwordObj.toString(); - final String credentials = username + ":" + password; - final String encodedCredentials = java.util.Base64.getEncoder().encodeToString(credentials.getBytes(java.nio.charset.StandardCharsets.UTF_8)); - headers.put("Authorization", "Basic " + encodedCredentials); - LOGGER.debug("Using Basic authentication"); - } else { - LOGGER.warn("Username or password not provided in connection properties"); - } - } - else { - LOGGER.warn("Unknown authentication type: {}", authType); + headers.put("Authorization", "ApiKey"); + LOGGER.debug("Using API Key authentication (key from config)"); + } else if (authType == AuthenticationType.OAUTH2) { + headers.put("Authorization", "Bearer"); + LOGGER.debug("Using OAuth 2.0 Bearer Token authentication"); } return headers.isEmpty() ? null : headers; } } - -// Made with Bob diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestTableDefinition.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestTableDefinition.java index bdc3c4df..af4b2ea3 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestTableDefinition.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestTableDefinition.java @@ -9,7 +9,7 @@ import java.util.List; /** - * Represents a table (endpoint) definition parsed from a .rest mapping file. + * Represents a table (endpoint) definition parsed from a JSON mapping file. * Holds the API path, optional data path for nested responses, optional pagination configuration, * and the list of field definitions. */ @@ -20,47 +20,14 @@ public class RestTableDefinition private final PaginationConfig paginationConfig; private final List fields; - /** - * Creates a table definition. - * - * @param path - * the URL path to append to the base URL (e.g. "/v4/rockets") - * @param fields - * the list of field definitions for this table - */ public RestTableDefinition(String path, List fields) - { - this(path, null, null, fields); - } + { this(path, null, null, fields); } - /** - * Creates a table definition with an optional data path for nested responses. - * - * @param path - * the URL path to append to the base URL (e.g. "/v4/rockets") - * @param dataPath - * optional JSON path to the data array (e.g. "result" for {"result": [...]}) - * @param fields - * the list of field definitions for this table - */ public RestTableDefinition(String path, String dataPath, List fields) - { - this(path, dataPath, null, fields); - } + { this(path, dataPath, null, fields); } - /** - * Creates a table definition with optional data path and pagination configuration. - * - * @param path - * the URL path to append to the base URL (e.g. "/v4/rockets") - * @param dataPath - * optional JSON path to the data array (e.g. "result" for {"result": [...]}) - * @param paginationConfig - * optional pagination configuration for this table - * @param fields - * the list of field definitions for this table - */ - public RestTableDefinition(String path, String dataPath, PaginationConfig paginationConfig, List fields) + public RestTableDefinition(String path, String dataPath, PaginationConfig paginationConfig, + List fields) { this.path = path; this.dataPath = dataPath; @@ -68,45 +35,10 @@ public RestTableDefinition(String path, String dataPath, PaginationConfig pagina this.fields = Collections.unmodifiableList(fields); } - /** - * Returns the URL path for this table's endpoint. - * - * @return the URL path (e.g. "/v4/rockets") - */ - public String getPath() - { - return path; - } - - /** - * Returns the JSON path to the data array in nested responses. - * - * @return the data path (e.g. "result"), or null if data is at root level - */ - public String getDataPath() - { - return dataPath; - } - - /** - * Returns the pagination configuration for this table. - * - * @return the pagination configuration, or null if no pagination is configured - */ - public PaginationConfig getPaginationConfig() - { - return paginationConfig; - } - - /** - * Returns the list of field definitions for this table. - * - * @return an unmodifiable list of field definitions - */ - public List getFields() - { - return fields; - } + public String getPath() { return path; } + public String getDataPath() { return dataPath; } + public PaginationConfig getPaginationConfig() { return paginationConfig; } + public List getFields() { return fields; } @Override public String toString() @@ -115,5 +47,3 @@ public String toString() + "', paginationConfig=" + paginationConfig + ", fields=" + fields + "}"; } } - -// Made with Bob diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestTargetInteraction.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestTargetInteraction.java index 559116e6..829fd44f 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestTargetInteraction.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/main/java/com/ibm/connect/restconnector/RestTargetInteraction.java @@ -13,17 +13,22 @@ import com.ibm.connect.sdk.api.TargetInteraction; import com.ibm.connect.sdk.util.ModelMapper; import com.ibm.wdp.connect.common.sdk.api.models.CustomFlightAssetDescriptor; +import com.ibm.wdp.connect.sdk.connector.AssetDescriptor; +import com.ibm.wdp.connect.sdk.connector.RowReader; +import com.ibm.wdp.connect.sdk.connector.SdkTargetInteraction; /** * An interaction with an Arrow asset as a target. + * + *

    Implements both the legacy {@link TargetInteraction} interface (used by the old connector + * path) and the new {@link SdkTargetInteraction} interface (used by the Arrow-native path). */ -@SuppressWarnings({ "PMD.AvoidDollarSigns", "PMD.ClassNamingConventions" }) -public class RestTargetInteraction implements TargetInteraction> +public class RestTargetInteraction implements TargetInteraction>, SdkTargetInteraction { private final Properties interactionProperties; /** - * Creates an Arrow target interaction. + * Creates an Arrow target interaction from a legacy {@link CustomFlightAssetDescriptor}. * * @param connector * the connector managing the connection to the data source @@ -37,6 +42,35 @@ public RestTargetInteraction(RestConnector connector, CustomFlightAssetDescripto throw new IllegalArgumentException(RestMsgs.MISSING_CONNECTOR.format()); } interactionProperties = ModelMapper.toProperties(asset.getInteractionProperties()); + validateProperties(); + } + + /** + * Creates an Arrow target interaction from an SDK {@link AssetDescriptor}. + * + * @param connector + * the connector managing the connection to the data source + * @param asset + * the SDK asset to which to write + */ + public RestTargetInteraction(RestConnector connector, AssetDescriptor asset) { + if (connector == null) { + throw new IllegalArgumentException(RestMsgs.MISSING_CONNECTOR.format()); + } + // Build properties from SDK asset's properties map + interactionProperties = new Properties(); + if (asset.getProperties() != null) { + for (final java.util.Map.Entry entry : asset.getProperties().entrySet()) { + if (entry.getValue() != null) { + interactionProperties.setProperty(entry.getKey(), entry.getValue().toString()); + } + } + } + validateProperties(); + } + + private void validateProperties() + { if (interactionProperties.getProperty("schema_name") == null) { throw new IllegalArgumentException(RestMsgs.MISSING_PROPERTY.format("schema_name")); } @@ -45,6 +79,34 @@ public RestTargetInteraction(RestConnector connector, CustomFlightAssetDescripto } } + // ---- SdkTargetInteraction interface (new path) ---- + + /** + * {@inheritDoc} + */ + @Override + public void setup() { + // TODO Perform any setup required before writing + } + + /** + * {@inheritDoc} + */ + @Override + public void consume(RowReader reader) { + // TODO Read rows from reader and write to the target + } + + /** + * {@inheritDoc} + */ + @Override + public void wrapup() { + // TODO Perform any wrap-up required after writing + } + + // ---- TargetInteraction interface (legacy path) ---- + /** * {@inheritDoc} */ diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/test/java/com/ibm/connect/restconnector/TestAdvancedFeatures.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/test/java/com/ibm/connect/restconnector/TestAdvancedFeatures.java index b378d6ba..74352d6f 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/test/java/com/ibm/connect/restconnector/TestAdvancedFeatures.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/test/java/com/ibm/connect/restconnector/TestAdvancedFeatures.java @@ -14,6 +14,7 @@ import org.junit.Test; + /** * Tests for advanced REST connector features including authentication, * pagination, data paths, and object flattening. diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/test/java/com/ibm/connect/restconnector/TestForgeSchemaBuilder.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/test/java/com/ibm/connect/restconnector/TestForgeSchemaBuilder.java new file mode 100644 index 00000000..73713498 --- /dev/null +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/test/java/com/ibm/connect/restconnector/TestForgeSchemaBuilder.java @@ -0,0 +1,145 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.connect.restconnector; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.List; + +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.Test; + +/** + * Unit tests for {@link ForgeSchemaBuilder}. + */ +public class TestForgeSchemaBuilder +{ + @Test + public void testBuildSchemaBasicTypes() + { + final List fieldDefs = Arrays.asList( + new RestFieldDefinition("id", "INTEGER", true, true), + new RestFieldDefinition("name", "VARCHAR", false, false), + new RestFieldDefinition("count", "BIGINT", false, false), + new RestFieldDefinition("active", "BOOLEAN", false, false), + new RestFieldDefinition("score", "DOUBLE", false, false), + new RestFieldDefinition("ratio", "FLOAT", false, false), + new RestFieldDefinition("created", "DATE", false, false), + new RestFieldDefinition("modified", "TIMESTAMP", false, false)); + + final Schema schema = ForgeSchemaBuilder.buildSchema(fieldDefs); + assertNotNull(schema); + assertEquals(8, schema.getFields().size()); + + // id — Integer, not nullable + final Field id = findField(schema, "id"); + assertTrue(id.getType() instanceof ArrowType.Int); + assertEquals(32, ((ArrowType.Int) id.getType()).getBitWidth()); + assertFalse(id.isNullable()); + + // name — Utf8, nullable + final Field name = findField(schema, "name"); + assertTrue(name.getType() instanceof ArrowType.Utf8); + assertTrue(name.isNullable()); + + // count — Int64 + final Field count = findField(schema, "count"); + assertTrue(count.getType() instanceof ArrowType.Int); + assertEquals(64, ((ArrowType.Int) count.getType()).getBitWidth()); + + // active — Bool + final Field active = findField(schema, "active"); + assertTrue(active.getType() instanceof ArrowType.Bool); + + // score — Double + final Field score = findField(schema, "score"); + assertTrue(score.getType() instanceof ArrowType.FloatingPoint); + assertEquals(FloatingPointPrecision.DOUBLE, + ((ArrowType.FloatingPoint) score.getType()).getPrecision()); + + // ratio — Float/Single + final Field ratio = findField(schema, "ratio"); + assertTrue(ratio.getType() instanceof ArrowType.FloatingPoint); + assertEquals(FloatingPointPrecision.SINGLE, + ((ArrowType.FloatingPoint) ratio.getType()).getPrecision()); + + // created — Date(DAY) + final Field created = findField(schema, "created"); + assertTrue(created.getType() instanceof ArrowType.Date); + assertEquals(DateUnit.DAY, ((ArrowType.Date) created.getType()).getUnit()); + + // modified — Timestamp(MICROSECOND) + final Field modified = findField(schema, "modified"); + assertTrue(modified.getType() instanceof ArrowType.Timestamp); + assertEquals(TimeUnit.MICROSECOND, ((ArrowType.Timestamp) modified.getType()).getUnit()); + } + + @Test + public void testJsonTypeBecomesUtf8() + { + final List fieldDefs = Arrays.asList( + new RestFieldDefinition("data", "JSON", false, false)); + final Schema schema = ForgeSchemaBuilder.buildSchema(fieldDefs); + assertTrue(findField(schema, "data").getType() instanceof ArrowType.Utf8); + } + + @Test + public void testVarCharWithLength() + { + final List fieldDefs = Arrays.asList( + new RestFieldDefinition("label", "VarChar(255)", false, false)); + final Schema schema = ForgeSchemaBuilder.buildSchema(fieldDefs); + assertTrue(findField(schema, "label").getType() instanceof ArrowType.Utf8); + } + + @Test + public void testUnknownTypeBecomesUtf8() + { + final List fieldDefs = Arrays.asList( + new RestFieldDefinition("misc", "UNKNOWN_TYPE", false, false)); + final Schema schema = ForgeSchemaBuilder.buildSchema(fieldDefs); + assertTrue(findField(schema, "misc").getType() instanceof ArrowType.Utf8); + } + + @Test + public void testDecimalType() + { + final List fieldDefs = Arrays.asList( + new RestFieldDefinition("price", "DECIMAL", false, false)); + final Schema schema = ForgeSchemaBuilder.buildSchema(fieldDefs); + assertTrue(findField(schema, "price").getType() instanceof ArrowType.Decimal); + } + + @Test + public void testSmallIntAndTinyInt() + { + final List fieldDefs = Arrays.asList( + new RestFieldDefinition("s", "SMALLINT", false, false), + new RestFieldDefinition("t", "TINYINT", false, false)); + final Schema schema = ForgeSchemaBuilder.buildSchema(fieldDefs); + assertEquals(16, ((ArrowType.Int) findField(schema, "s").getType()).getBitWidth()); + assertEquals(8, ((ArrowType.Int) findField(schema, "t").getType()).getBitWidth()); + } + + private static Field findField(Schema schema, String name) + { + for (final Field f : schema.getFields()) { + if (f.getName().equals(name)) { + return f; + } + } + throw new AssertionError("Field not found: " + name); + } +} diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/test/java/com/ibm/connect/restconnector/TestPropertyValidation.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/test/java/com/ibm/connect/restconnector/TestPropertyValidation.java index f876db75..cae304e2 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/test/java/com/ibm/connect/restconnector/TestPropertyValidation.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/test/java/com/ibm/connect/restconnector/TestPropertyValidation.java @@ -7,10 +7,10 @@ import static org.junit.Assert.assertNotNull; +import com.ibm.wdp.connect.sdk.connector.ConnectionProperties; +import com.ibm.wdp.connect.sdk.connector.SdkDatasourceTypes; import org.junit.Test; -import com.ibm.wdp.connect.common.sdk.api.models.ConnectionProperties; - /** * Tests property validation for the REST connector. * @@ -27,7 +27,8 @@ public class TestPropertyValidation public void testConnectionPropertiesNegative() throws Exception { final String typeName = "unknown_type"; - final ConnectionProperties properties = new ConnectionProperties(); + final ConnectionProperties properties + = new ConnectionProperties(null); RestConnectorFactory.getInstance().createConnector(typeName, properties); } @@ -41,19 +42,22 @@ public void testConnectionPropertiesNegative() throws Exception public void testConnectionProperties() throws Exception { // Get the first available datasource type from the factory - final var datasourceTypes = RestConnectorFactory.getInstance().getDatasourceTypes(); - - if (datasourceTypes.getDatasourceTypes() == null || datasourceTypes.getDatasourceTypes().isEmpty()) { + final SdkDatasourceTypes datasourceTypes + = RestConnectorFactory.getInstance().getDatasourceTypes(); + + if (datasourceTypes.getTypeNames() == null || datasourceTypes.getTypeNames().isEmpty() + || "__rest__".equals(datasourceTypes.getTypeNames().get(0))) { // No configurations loaded - skip test System.out.println("No REST connector configurations loaded. Skipping test."); return; } - - final String typeName = datasourceTypes.getDatasourceTypes().get(0).getName(); - final ConnectionProperties properties = new ConnectionProperties(); - + + final String typeName = datasourceTypes.getTypeNames().get(0); + final ConnectionProperties sdkProps + = new ConnectionProperties(null); + // Create connector - should succeed - assertNotNull(RestConnectorFactory.getInstance().createConnector(typeName, properties)); + assertNotNull(RestConnectorFactory.getInstance().createConnector(typeName, sdkProps)); } } diff --git a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/test/java/com/ibm/connect/restconnector/TestRestApiMappingLoader.java b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/test/java/com/ibm/connect/restconnector/TestRestApiMappingLoader.java index 775a740c..c946e02e 100644 --- a/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/test/java/com/ibm/connect/restconnector/TestRestApiMappingLoader.java +++ b/sdk-gen/subprojects/java/connectors_forge_rest/impl/src/test/java/com/ibm/connect/restconnector/TestRestApiMappingLoader.java @@ -15,6 +15,7 @@ import org.junit.Test; + /** * Tests for {@link RestApiMappingLoader}. */ diff --git a/sdk-gen/subprojects/java/sdk-connector-api/build.gradle b/sdk-gen/subprojects/java/sdk-connector-api/build.gradle new file mode 100644 index 00000000..c89cbb26 --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/build.gradle @@ -0,0 +1,44 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ + +// +// Apply the java-library, and maven-publish plugins +// +apply plugin: 'java-library' +apply plugin: 'maven-publish' + +group = 'com.ibm.wdp.connect' +archivesBaseName = 'wdp-connect-sdk-connector-api' + +// +// Declare project dependencies +// +dependencies { + // Exclude grpc packages that can conflict with those in Liberty. Include everything for test runtime only. + api group: 'org.apache.arrow', name: 'flight-core', version: project['arrow.version'], transitive: false + runtimeOnly group: 'com.google.flatbuffers', name: 'flatbuffers-java', version: project['flatbuffers.version'] + runtimeOnly group: 'com.google.protobuf', name: 'protobuf-java', version: project['protobuf.version'] + implementation group: 'org.apache.arrow', name: 'arrow-format', version: project['arrow.version'] + implementation group: 'org.apache.arrow', name: 'arrow-memory-core', version: project['arrow.version'] + implementation group: 'org.apache.arrow', name: 'arrow-vector', version: project['arrow.version'] + runtimeOnly group: 'io.grpc', name: 'grpc-netty', version: project['grpc.version'], transitive: false + runtimeOnly group: 'io.netty', name: 'netty-buffer', version: project['netty.version'] + runtimeOnly group: 'org.apache.arrow', name: 'arrow-memory-netty', version: project['arrow.version'], transitive: false + runtimeOnly group: 'org.apache.arrow', name: 'arrow-memory-netty-buffer-patch', version: project['arrow.version'], transitive: false + // End of flight-core + + // Logging + implementation group: 'org.slf4j', name: 'slf4j-api', version: project['slf4j.version'] + + // Test dependencies + testImplementation group: 'junit', name: 'junit', version: project['junit.version'] + testRuntimeOnly group: 'io.grpc', name: 'grpc-all', version: project['grpc.version'] + testRuntimeOnly group: 'io.netty', name: 'netty-all', version: project['netty.version'] + testRuntimeOnly group: 'org.apache.arrow', name: 'arrow-memory-netty', version: project['arrow.version'] + testRuntimeOnly group: 'org.apache.arrow', name: 'arrow-memory-netty-buffer-patch', version: project['arrow.version'] +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ArrowBatchReader.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ArrowBatchReader.java new file mode 100644 index 00000000..75ff68ba --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ArrowBatchReader.java @@ -0,0 +1,115 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; + +/** + * Concrete Flight-layer implementation of {@link RowReader}. + * + *

    Connector authors never subclass this class. They read rows via the {@link RowReader} + * interface by calling {@link #nextRow()} and {@link #get(String)}. All Apache Arrow + * internals are hidden inside this class. + * + *

    The Flight layer constructs this from incoming {@link VectorSchemaRoot} batches and passes + * it to the connector's {@code consume(RowReader)} method. + */ +public final class ArrowBatchReader implements RowReader, AutoCloseable +{ + private final List batches; + private int batchIndex; + private VectorSchemaRoot current; + private int rowIndex; + private int rowCount; + private Map vectorCache; + private boolean closed; + + /** + * Creates an Arrow batch reader over the given batches. + * + * @param batches + * the list of {@link VectorSchemaRoot} batches to iterate; must not be null + */ + public ArrowBatchReader(List batches) + { + this.batches = batches; + this.batchIndex = -1; + this.rowIndex = -1; + this.rowCount = 0; + this.closed = false; + advanceBatch(); + } + + /** {@inheritDoc} */ + @Override + public boolean nextRow() + { + if (closed) { + return false; + } + rowIndex++; + if (rowIndex < rowCount) { + return true; + } + // Try next batch + if (advanceBatch()) { + rowIndex = 0; + return rowIndex < rowCount; + } + return false; + } + + /** {@inheritDoc} */ + @Override + public Object get(String fieldName) + { + final FieldVector vector = vectorCache.get(fieldName); + if (vector == null) { + throw new IllegalArgumentException("Unknown field: " + fieldName); + } + if (vector.isNull(rowIndex)) { + return null; + } + return ArrowValueExtractor.extract(vector, rowIndex); + } + + /** {@inheritDoc} */ + @Override + public void close() + { + closed = true; + } + + // ---- private helpers ---- + + private boolean advanceBatch() + { + batchIndex++; + if (batchIndex < batches.size()) { + current = batches.get(batchIndex); + rowCount = current.getRowCount(); + rowIndex = -1; + cacheVectors(); + return true; + } + return false; + } + + private void cacheVectors() + { + vectorCache = new HashMap<>(); + for (final FieldVector v : current.getFieldVectors()) { + vectorCache.put(v.getField().getName(), v); + } + } +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ArrowBatchWriter.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ArrowBatchWriter.java new file mode 100644 index 00000000..db4ebfd1 --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ArrowBatchWriter.java @@ -0,0 +1,155 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.Schema; + +/** + * Concrete Flight-layer implementation of {@link RowWriter}. + * + *

    Connector authors never subclass this class. They write rows by calling + * {@link #startRow()}, {@link #set(String, Object)}, and {@link #endRow()} through + * the {@link RowWriter} interface. All Apache Arrow memory management is hidden inside this class. + * + *

    The Flight layer retrieves the accumulated batches via the package-private {@link #batches()} + * method after the connector's {@code stream(RowWriter)} call completes. + * + *

    Usage: + * 

    + *   try (ArrowBatchWriter writer = new ArrowBatchWriter(schema, allocator, 1000)) {
+ *       interaction.stream(writer);
+ *       for (Iterator<VectorSchemaRoot> it = writer.batches(); it.hasNext(); ) {
+ *           VectorSchemaRoot root = it.next();
+ *           listener.putNext(root);
+ *       }
+ *   }
+ *
    + */ +public final class ArrowBatchWriter implements RowWriter, AutoCloseable +{ + private final Schema schema; + private final int batchSize; + private final VectorSchemaRoot root; + private final List completedBatches; + + private int currentRow; + private boolean closed; + + /** + * Creates an Arrow batch writer. + * + * @param schema + * the Arrow schema describing the fields to write + * @param allocator + * the buffer allocator to use for Arrow memory + * @param batchSize + * the number of rows per batch; when a batch reaches this size it is flushed + * automatically on {@link #endRow()} + */ + public ArrowBatchWriter(Schema schema, BufferAllocator allocator, int batchSize) + { + this.schema = schema; + this.batchSize = batchSize > 0 ? batchSize : 1000; + this.root = VectorSchemaRoot.create(schema, allocator); + this.root.allocateNew(); + this.completedBatches = new ArrayList<>(); + this.currentRow = 0; + this.closed = false; + } + + /** {@inheritDoc} */ + @Override + public void startRow() + { + // Row position is tracked by currentRow; no pre-row allocation needed + } + + /** {@inheritDoc} */ + @Override + public void set(String fieldName, Object value) + { + final FieldVector vector = root.getVector(fieldName); + if (vector == null) { + throw new IllegalArgumentException("Unknown field: " + fieldName); + } + if (value == null) { + vector.setNull(currentRow); + } else { + ArrowValueExtractor.setValue(vector, currentRow, value); + } + } + + /** {@inheritDoc} */ + @Override + public void endRow() + { + currentRow++; + root.setRowCount(currentRow); + if (currentRow >= batchSize) { + flushCurrentBatch(); + } + } + + /** + * Returns an iterator over all completed batches plus any remaining partial batch. + *

    + * For use by the Flight layer after the connector's {@code stream(RowWriter)} method returns. + * + * @return an iterator over {@link VectorSchemaRoot} batches; caller must not close the roots + */ + public Iterator batches() + { + if (currentRow > 0) { + flushCurrentBatch(); + } + return completedBatches.iterator(); + } + + /** + * Returns the Arrow schema. + *

    + * For use by the Flight layer. + * + * @return the schema + */ + public Schema getSchema() + { + return schema; + } + + /** {@inheritDoc} */ + @Override + public void close() + { + if (!closed) { + closed = true; + for (final VectorSchemaRoot batch : completedBatches) { + batch.close(); + } + completedBatches.clear(); + root.close(); + } + } + + // ---- private helpers ---- + + private void flushCurrentBatch() + { + completedBatches.add(root.slice(0, currentRow)); + root.clear(); + root.allocateNew(); + currentRow = 0; + } +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ArrowValueExtractor.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ArrowValueExtractor.java new file mode 100644 index 00000000..d361a6ae --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ArrowValueExtractor.java @@ -0,0 +1,236 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.concurrent.TimeUnit; + +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeMicroVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeStampMicroTZVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; + +/** + * Package-private helper that converts values between Java types and Arrow vectors. + * + *

    Used internally by {@link ArrowBatchWriter}, {@link ColumnarArrowBatchWriter}, + * {@link ArrowBatchReader}, and {@link ColumnarArrowBatchReader}. Not part of the public API. + */ +final class ArrowValueExtractor +{ + private ArrowValueExtractor() + { + // utility class + } + + /** + * Extracts the value at {@code index} from {@code vector} as a plain Java object. + * + * @param vector + * the field vector + * @param index + * the row index (must not be null at this index) + * @return the Java value + */ + static Object extract(FieldVector vector, int index) + { + if (vector instanceof VarCharVector) { + final byte[] bytes = ((VarCharVector) vector).get(index); + return bytes != null ? new String(bytes, StandardCharsets.UTF_8) : null; + } + if (vector instanceof IntVector) { + return ((IntVector) vector).get(index); + } + if (vector instanceof BigIntVector) { + return ((BigIntVector) vector).get(index); + } + if (vector instanceof Float8Vector) { + return ((Float8Vector) vector).get(index); + } + if (vector instanceof Float4Vector) { + return ((Float4Vector) vector).get(index); + } + if (vector instanceof BitVector) { + return ((BitVector) vector).get(index) != 0; + } + if (vector instanceof SmallIntVector) { + return (int) ((SmallIntVector) vector).get(index); + } + if (vector instanceof TinyIntVector) { + return (int) ((TinyIntVector) vector).get(index); + } + if (vector instanceof DateDayVector) { + final int days = ((DateDayVector) vector).get(index); + return new Date(TimeUnit.DAYS.toMillis(days)); + } + if (vector instanceof TimeStampMicroTZVector) { + final long micros = ((TimeStampMicroTZVector) vector).get(index); + final long millis = TimeUnit.MICROSECONDS.toMillis(micros); + final int nanos = (int) TimeUnit.MICROSECONDS.toNanos(micros % 1000); + final Timestamp ts = new Timestamp(millis); + ts.setNanos(nanos >= 0 ? nanos : nanos + 1_000_000_000); + return ts; + } + if (vector instanceof TimeMilliVector) { + return new Time(((TimeMilliVector) vector).get(index)); + } + if (vector instanceof TimeMicroVector) { + return new Time(TimeUnit.MICROSECONDS.toMillis(((TimeMicroVector) vector).get(index))); + } + if (vector instanceof DecimalVector) { + return ((DecimalVector) vector).getObject(index); + } + if (vector instanceof VarBinaryVector) { + return ((VarBinaryVector) vector).get(index); + } + // Fallback: use getObject if available + return vector.getObject(index); + } + + /** + * Sets the value at {@code index} in {@code vector} from a plain Java object. + * + * @param vector + * the field vector + * @param index + * the row index + * @param value + * the value to write; must not be null + */ + static void setValue(FieldVector vector, int index, Object value) + { + if (vector instanceof VarCharVector) { + final byte[] bytes = value instanceof byte[] ? (byte[]) value + : value.toString().getBytes(StandardCharsets.UTF_8); + ((VarCharVector) vector).setSafe(index, bytes, 0, bytes.length); + } else if (vector instanceof IntVector) { + ((IntVector) vector).setSafe(index, toInt(value)); + } else if (vector instanceof BigIntVector) { + ((BigIntVector) vector).setSafe(index, toLong(value)); + } else if (vector instanceof Float8Vector) { + ((Float8Vector) vector).setSafe(index, toDouble(value)); + } else if (vector instanceof Float4Vector) { + ((Float4Vector) vector).setSafe(index, toFloat(value)); + } else if (vector instanceof BitVector) { + ((BitVector) vector).setSafe(index, toBit(value)); + } else if (vector instanceof SmallIntVector) { + ((SmallIntVector) vector).setSafe(index, toInt(value)); + } else if (vector instanceof TinyIntVector) { + ((TinyIntVector) vector).setSafe(index, toInt(value)); + } else if (vector instanceof DateDayVector) { + ((DateDayVector) vector).setSafe(index, toDateDay(value)); + } else if (vector instanceof TimeStampMicroTZVector) { + ((TimeStampMicroTZVector) vector).setSafe(index, toTimestampMicros(value)); + } else if (vector instanceof TimeMilliVector) { + ((TimeMilliVector) vector).setSafe(index, toTimeMilli(value)); + } else if (vector instanceof TimeMicroVector) { + ((TimeMicroVector) vector).setSafe(index, toTimeMicro(value)); + } else if (vector instanceof DecimalVector) { + final DecimalVector dv = (DecimalVector) vector; + dv.setSafe(index, toBigDecimal(value, dv.getScale())); + } else if (vector instanceof VarBinaryVector) { + final byte[] bytes = value instanceof byte[] ? (byte[]) value + : value.toString().getBytes(StandardCharsets.UTF_8); + ((VarBinaryVector) vector).setSafe(index, bytes, 0, bytes.length); + } else { + // Fallback: attempt varchar + final byte[] bytes = value.toString().getBytes(StandardCharsets.UTF_8); + if (vector instanceof VarCharVector) { + ((VarCharVector) vector).setSafe(index, bytes, 0, bytes.length); + } + } + } + + static int toInt(Object v) + { + if (v instanceof Number) return ((Number) v).intValue(); + return Integer.parseInt(v.toString()); + } + + static long toLong(Object v) + { + if (v instanceof Number) return ((Number) v).longValue(); + return Long.parseLong(v.toString()); + } + + static double toDouble(Object v) + { + if (v instanceof Number) return ((Number) v).doubleValue(); + return Double.parseDouble(v.toString()); + } + + static float toFloat(Object v) + { + if (v instanceof Number) return ((Number) v).floatValue(); + return Float.parseFloat(v.toString()); + } + + static int toBit(Object v) + { + if (v instanceof Boolean) return ((Boolean) v) ? 1 : 0; + if (v instanceof Number) return ((Number) v).intValue() != 0 ? 1 : 0; + return Boolean.parseBoolean(v.toString()) ? 1 : 0; + } + + static int toDateDay(Object v) + { + if (v instanceof Date) { + return (int) TimeUnit.MILLISECONDS.toDays(((Date) v).getTime()); + } + if (v instanceof java.util.Date) { + return (int) TimeUnit.MILLISECONDS.toDays(((java.util.Date) v).getTime()); + } + return (int) TimeUnit.MILLISECONDS.toDays(Date.valueOf(v.toString()).getTime()); + } + + static long toTimestampMicros(Object v) + { + if (v instanceof Timestamp) { + final Timestamp ts = (Timestamp) v; + return TimeUnit.MILLISECONDS.toMicros(ts.getTime()) + + TimeUnit.NANOSECONDS.toMicros(ts.getNanos() % 1_000_000L); + } + if (v instanceof java.util.Date) { + return TimeUnit.MILLISECONDS.toMicros(((java.util.Date) v).getTime()); + } + return TimeUnit.MILLISECONDS.toMicros(Timestamp.valueOf(v.toString()).getTime()); + } + + static int toTimeMilli(Object v) + { + if (v instanceof Time) return (int) (((Time) v).getTime() % 86_400_000L); + return (int) (Time.valueOf(v.toString()).getTime() % 86_400_000L); + } + + static long toTimeMicro(Object v) + { + if (v instanceof Time) return TimeUnit.MILLISECONDS.toMicros(((Time) v).getTime() % 86_400_000L); + return TimeUnit.MILLISECONDS.toMicros(Time.valueOf(v.toString()).getTime() % 86_400_000L); + } + + static BigDecimal toBigDecimal(Object v, int scale) + { + final BigDecimal bd = v instanceof BigDecimal ? (BigDecimal) v : new BigDecimal(v.toString()); + return bd.setScale(scale, java.math.RoundingMode.HALF_UP); + } +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/AssetDescriptor.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/AssetDescriptor.java new file mode 100644 index 00000000..14a68f4a --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/AssetDescriptor.java @@ -0,0 +1,141 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Descriptor for a data asset (table, schema, container, etc.) accessible through a connector. + * + *

    Equivalent in purpose to the SDK's {@code CustomFlightAssetDescriptor} but with no dependency + * on the SDK model framework. Connector authors use this class to identify the asset they are + * reading from, writing to, or discovering. + * + *

    Instances are immutable once constructed. + */ +public final class AssetDescriptor +{ + private final String id; + private final String name; + private final String path; + private final String datasourceTypeName; + private final Map properties; + private final boolean hasChildren; + private final int batchSize; + + /** + * Creates an asset descriptor. + * + * @param id + * the unique identifier for this asset (may be null for newly discovered assets) + * @param name + * the display name of the asset (e.g. table name) + * @param path + * the hierarchical path to this asset (e.g. "schema/table"), may be null + * @param datasourceTypeName + * the datasource type name identifying the connector to use + * @param properties + * additional asset-specific properties (e.g. schema name, connection options); may be null + * @param hasChildren + * true if this asset is a container (schema, catalog) with child assets + * @param batchSize + * the preferred number of rows per Arrow batch; 0 means use connector default + */ + public AssetDescriptor(String id, String name, String path, String datasourceTypeName, + Map properties, boolean hasChildren, int batchSize) + { + this.id = id; + this.name = name; + this.path = path; + this.datasourceTypeName = datasourceTypeName; + this.properties = properties != null + ? Collections.unmodifiableMap(new LinkedHashMap<>(properties)) + : Collections.emptyMap(); + this.hasChildren = hasChildren; + this.batchSize = batchSize; + } + + /** + * Returns the unique identifier for this asset. + * + * @return the asset id, or null + */ + public String getId() + { + return id; + } + + /** + * Returns the display name of the asset. + * + * @return the asset name + */ + public String getName() + { + return name; + } + + /** + * Returns the hierarchical path to this asset. + * + * @return the path (e.g. "schema/table"), or null + */ + public String getPath() + { + return path; + } + + /** + * Returns the datasource type name identifying the connector. + * + * @return the datasource type name + */ + public String getDatasourceTypeName() + { + return datasourceTypeName; + } + + /** + * Returns the additional asset-specific properties. + * + * @return an unmodifiable map of properties; never null + */ + public Map getProperties() + { + return properties; + } + + /** + * Returns whether this asset is a container with child assets. + * + * @return true if this asset has children (schema, catalog) + */ + public boolean hasChildren() + { + return hasChildren; + } + + /** + * Returns the preferred number of rows per Arrow batch. + * + * @return the batch size, or 0 to use the connector default + */ + public int getBatchSize() + { + return batchSize; + } + + @Override + public String toString() + { + return "AssetDescriptor{id='" + id + "', name='" + name + "', path='" + path + + "', datasourceTypeName='" + datasourceTypeName + "', batchSize=" + batchSize + "}"; + } +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ColumnarArrowBatchReader.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ColumnarArrowBatchReader.java new file mode 100644 index 00000000..825bf89e --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ColumnarArrowBatchReader.java @@ -0,0 +1,96 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; + +/** + * Concrete Flight-layer implementation of {@link ColumnarReader}. + * + *

    Connector authors never subclass this class. They read columns via the {@link ColumnarReader} + * interface by calling {@link #nextBatch()} and {@link #getColumn(String)}. All Apache Arrow + * internals are hidden inside this class. + * + *

    The Flight layer constructs this from incoming {@link VectorSchemaRoot} batches and passes + * it to the connector's {@code consume(ColumnarReader)} method. + */ +public final class ColumnarArrowBatchReader implements ColumnarReader, AutoCloseable +{ + private final List batches; + private int batchIndex; + private VectorSchemaRoot current; + private Map columnCache; + private boolean closed; + + /** + * Creates a columnar Arrow batch reader over the given batches. + * + * @param batches + * the list of {@link VectorSchemaRoot} batches to iterate; must not be null + */ + public ColumnarArrowBatchReader(List batches) + { + this.batches = batches; + this.batchIndex = -1; + this.closed = false; + } + + /** {@inheritDoc} */ + @Override + public boolean nextBatch() + { + if (closed) { + return false; + } + batchIndex++; + if (batchIndex < batches.size()) { + current = batches.get(batchIndex); + columnCache = new HashMap<>(); + extractColumns(); + return true; + } + return false; + } + + /** {@inheritDoc} */ + @Override + public Object[] getColumn(String fieldName) + { + final Object[] column = columnCache.get(fieldName); + if (column == null) { + throw new IllegalArgumentException("Unknown field: " + fieldName); + } + return column; + } + + /** {@inheritDoc} */ + @Override + public void close() + { + closed = true; + } + + // ---- private helpers ---- + + private void extractColumns() + { + final int rowCount = current.getRowCount(); + for (final FieldVector vector : current.getFieldVectors()) { + final Object[] values = new Object[rowCount]; + for (int i = 0; i < rowCount; i++) { + values[i] = vector.isNull(i) ? null : ArrowValueExtractor.extract(vector, i); + } + columnCache.put(vector.getField().getName(), values); + } + } +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ColumnarArrowBatchWriter.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ColumnarArrowBatchWriter.java new file mode 100644 index 00000000..7954d93f --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ColumnarArrowBatchWriter.java @@ -0,0 +1,131 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.Schema; + +/** + * Concrete Flight-layer implementation of {@link ColumnarWriter}. + * + *

    Connector authors never subclass this class. They write columns by calling + * {@link #writeColumn(String, Object[])} for each column in a batch, then {@link #flushBatch()}. + * All Apache Arrow memory management is hidden inside this class. + * + *

    The Flight layer retrieves the accumulated batches via the package-private {@link #batches()} + * method after the connector's {@code stream(ColumnarWriter)} call completes. + */ +public final class ColumnarArrowBatchWriter implements ColumnarWriter, AutoCloseable +{ + private final Schema schema; + private final VectorSchemaRoot root; + private final List completedBatches; + private int currentBatchRows; + private boolean closed; + + /** + * Creates a columnar Arrow batch writer. + * + * @param schema + * the Arrow schema describing the fields to write + * @param allocator + * the buffer allocator to use for Arrow memory + * @param batchSize + * hint for initial allocation; actual batch size is driven by {@link #writeColumn} array lengths + */ + public ColumnarArrowBatchWriter(Schema schema, BufferAllocator allocator, int batchSize) + { + this.schema = schema; + this.root = VectorSchemaRoot.create(schema, allocator); + this.root.allocateNew(); + this.completedBatches = new ArrayList<>(); + this.currentBatchRows = 0; + this.closed = false; + } + + /** {@inheritDoc} */ + @Override + public void writeColumn(String fieldName, Object[] values) + { + final FieldVector vector = root.getVector(fieldName); + if (vector == null) { + throw new IllegalArgumentException("Unknown field: " + fieldName); + } + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + vector.setNull(i); + } else { + ArrowValueExtractor.setValue(vector, i, values[i]); + } + } + // Track row count from the first column written in this batch + if (currentBatchRows == 0 && values.length > 0) { + currentBatchRows = values.length; + } + } + + /** {@inheritDoc} */ + @Override + public void flushBatch() + { + if (currentBatchRows > 0) { + root.setRowCount(currentBatchRows); + completedBatches.add(root.slice(0, currentBatchRows)); + root.clear(); + root.allocateNew(); + currentBatchRows = 0; + } + } + + /** + * Returns an iterator over all completed batches. + *

    + * For use by the Flight layer. + * + * @return an iterator over {@link VectorSchemaRoot} batches + */ + public Iterator batches() + { + if (currentBatchRows > 0) { + flushBatch(); + } + return completedBatches.iterator(); + } + + /** + * Returns the Arrow schema. + *

    + * For use by the Flight layer. + * + * @return the schema + */ + public Schema getSchema() + { + return schema; + } + + /** {@inheritDoc} */ + @Override + public void close() + { + if (!closed) { + closed = true; + for (final VectorSchemaRoot batch : completedBatches) { + batch.close(); + } + completedBatches.clear(); + root.close(); + } + } +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ColumnarReader.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ColumnarReader.java new file mode 100644 index 00000000..7a4e90e9 --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ColumnarReader.java @@ -0,0 +1,51 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +/** + * Reader interface for columnar data consumption. + * + *

    Connector authors call {@link #nextBatch()} to advance to the next batch and + * {@link #getColumn(String)} to read all values for a column in one call. Used in target + * connectors where Arrow batches from the Flight stream are presented column by column. + * + *

    No Apache Arrow types are exposed through this interface — connector authors do not need + * Arrow knowledge to implement a columnar target connector. + * + *

    Example usage: + * 

    + *   public void consume(ColumnarReader reader) throws Exception {
+ *       while (reader.nextBatch()) {
+ *           Object[] ids = reader.getColumn("id");
+ *           Object[] names = reader.getColumn("name");
+ *           target.insertBatch(ids, names);
+ *       }
+ *   }
+ *
    + */ +public interface ColumnarReader +{ + /** + * Advances the cursor to the next batch. + * + * @return {@code true} if there is a batch available; {@code false} if all batches have been consumed + */ + boolean nextBatch(); + + /** + * Returns all values for the named field in the current batch. + * + *

    Must only be called after a successful {@link #nextBatch()} call. + * + * @param fieldName + * the field name as defined in the schema + * @return array of values for this field in the current batch; elements may be {@code null} + * for nullable fields + */ + Object[] getColumn(String fieldName); +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ColumnarWriter.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ColumnarWriter.java new file mode 100644 index 00000000..ded2681c --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ColumnarWriter.java @@ -0,0 +1,51 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +/** + * Writer interface for columnar data production. + * + *

    Connector authors call {@link #writeColumn(String, Object[])} for each column in a batch, + * then {@link #flushBatch()} to commit the batch. This is suitable for columnar data sources + * such as Parquet files or columnar databases where data is organized by column. + * + *

    No Apache Arrow types are exposed through this interface — connector authors do not need + * Arrow knowledge to implement a columnar connector. + * + *

    Example usage: + * 

    + *   public void stream(ColumnarWriter writer) throws Exception {
+ *       for (ColumnBatch batch : source.batches()) {
+ *           writer.writeColumn("id", batch.getColumn("id"));
+ *           writer.writeColumn("name", batch.getColumn("name"));
+ *           writer.flushBatch();
+ *       }
+ *   }
+ *
    + */ +public interface ColumnarWriter +{ + /** + * Writes all values for a single column in the current batch. + * + *

    All columns written before the next {@link #flushBatch()} must have the same array length. + * + * @param fieldName + * the field name as defined in the schema + * @param values + * the column values for this batch; may contain null elements for nullable fields. + * Supported element types match those described in {@link RowWriter#set}. + */ + void writeColumn(String fieldName, Object[] values); + + /** + * Flushes the current batch. All columns must have been written before calling this method. + * Resets the writer state to accept the next batch. + */ + void flushBatch(); +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ConnectionProperties.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ConnectionProperties.java new file mode 100644 index 00000000..ba6dd260 --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/ConnectionProperties.java @@ -0,0 +1,79 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Connection properties provided to a connector at connection time. + * + *

    Wraps a {@code Map<String, Object>} of key/value pairs. Values are typically strings + * (host, port, database name, credentials) but may be any serializable type. + * + *

    Instances are immutable once constructed. + */ +public final class ConnectionProperties +{ + private final Map properties; + + /** + * Creates connection properties from a map. + * + * @param properties + * the property map; may be null (treated as empty) + */ + public ConnectionProperties(Map properties) + { + this.properties = properties != null + ? Collections.unmodifiableMap(new LinkedHashMap<>(properties)) + : Collections.emptyMap(); + } + + /** + * Returns the value of the named property. + * + * @param key + * the property name + * @return the property value, or null if not present + */ + public Object get(String key) + { + return properties.get(key); + } + + /** + * Returns the value of the named property as a String, or null. + * + * @param key + * the property name + * @return the string value, or null if not present + */ + public String getString(String key) + { + final Object value = properties.get(key); + return value != null ? value.toString() : null; + } + + /** + * Returns all properties as an unmodifiable map. + * + * @return the properties map; never null + */ + public Map asMap() + { + return properties; + } + + @Override + public String toString() + { + return "ConnectionProperties{keys=" + properties.keySet() + "}"; + } +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/DiscoveryCriteria.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/DiscoveryCriteria.java new file mode 100644 index 00000000..7fcb0e0f --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/DiscoveryCriteria.java @@ -0,0 +1,77 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +/** + * Criteria used to scope an asset discovery request. + * + *

    Passed to {@link SdkConnector#getDiscoveryInteraction(DiscoveryCriteria)} and then to + * {@link SdkDiscoveryInteraction#discoverAssets(DiscoveryCriteria)} to restrict the set of + * assets returned (e.g. browse only a specific schema, or browse assets of a specific type). + * + *

    Instances are immutable once constructed. + */ +public final class DiscoveryCriteria +{ + private final String path; + private final String datasourceTypeName; + private final ConnectionProperties connectionProperties; + + /** + * Creates discovery criteria. + * + * @param path + * optional hierarchical path to browse (e.g. "my_schema"); null means root + * @param datasourceTypeName + * the datasource type name identifying the connector + * @param connectionProperties + * the connection properties to use when connecting to the source + */ + public DiscoveryCriteria(String path, String datasourceTypeName, ConnectionProperties connectionProperties) + { + this.path = path; + this.datasourceTypeName = datasourceTypeName; + this.connectionProperties = connectionProperties; + } + + /** + * Returns the browse path. + * + * @return the path, or null for root-level discovery + */ + public String getPath() + { + return path; + } + + /** + * Returns the datasource type name. + * + * @return the datasource type name + */ + public String getDatasourceTypeName() + { + return datasourceTypeName; + } + + /** + * Returns the connection properties. + * + * @return the connection properties + */ + public ConnectionProperties getConnectionProperties() + { + return connectionProperties; + } + + @Override + public String toString() + { + return "DiscoveryCriteria{path='" + path + "', datasourceTypeName='" + datasourceTypeName + "'}"; + } +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/RowReader.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/RowReader.java new file mode 100644 index 00000000..fbddca07 --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/RowReader.java @@ -0,0 +1,50 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +/** + * Reader interface for row-based data consumption. + * + *

    Connector authors call {@link #nextRow()} to advance to the next row and + * {@link #get(String)} to read field values. Used in target connectors where Arrow + * batches from the Flight stream are presented row by row. + * + *

    No Apache Arrow types are exposed through this interface — connector authors do not need + * Arrow knowledge to implement a row-based target connector. + * + *

    Example usage: + * 

    + *   public void consume(RowReader reader) throws Exception {
+ *       while (reader.nextRow()) {
+ *           String id = (String) reader.get("id");
+ *           Integer count = (Integer) reader.get("count");
+ *           target.insert(id, count);
+ *       }
+ *   }
+ *
    + */ +public interface RowReader +{ + /** + * Advances the cursor to the next row. + * + * @return {@code true} if there is a row available; {@code false} if all rows have been consumed + */ + boolean nextRow(); + + /** + * Returns the value of the named field in the current row. + * + *

    Must only be called after a successful {@link #nextRow()} call. + * + * @param fieldName + * the field name as defined in the schema + * @return the field value, or {@code null} if the value is SQL null + */ + Object get(String fieldName); +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/RowWriter.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/RowWriter.java new file mode 100644 index 00000000..442f6f53 --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/RowWriter.java @@ -0,0 +1,57 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +/** + * Writer interface for row-based data production. + * + *

    Connector authors call {@link #startRow()}, one or more {@link #set(String, Object)} calls, + * then {@link #endRow()} for each row. The implementation accumulates rows into Arrow batches + * and flushes automatically at the configured {@code batchSize}. + * + *

    No Apache Arrow types are exposed through this interface — connector authors do not need + * Arrow knowledge to implement a row-based connector. + * + *

    Example usage: + * 

    + *   public void stream(RowWriter writer) throws Exception {
+ *       for (MyRow row : source.rows()) {
+ *           writer.startRow();
+ *           writer.set("id", row.getId());
+ *           writer.set("name", row.getName());
+ *           writer.endRow();
+ *       }
+ *   }
+ *
    + */ +public interface RowWriter +{ + /** + * Begins a new row. Must be called before any {@link #set(String, Object)} call for this row. + */ + void startRow(); + + /** + * Sets the value for the named field in the current row. + * + * @param fieldName + * the field name as defined in the schema + * @param value + * the value to write; may be null for nullable fields. Supported Java types: + * {@code String}, {@code Integer}, {@code Long}, {@code Double}, {@code Float}, + * {@code Boolean}, {@code java.sql.Date}, {@code java.sql.Timestamp}, + * {@code byte[]}, and numeric wrappers. + */ + void set(String fieldName, Object value); + + /** + * Ends the current row and adds it to the current batch. + * When the batch reaches the configured batch size, it is automatically flushed. + */ + void endRow(); +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkColumnarSourceInteraction.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkColumnarSourceInteraction.java new file mode 100644 index 00000000..9ab1670a --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkColumnarSourceInteraction.java @@ -0,0 +1,69 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import java.util.List; + +import org.apache.arrow.flight.Ticket; +import org.apache.arrow.vector.types.pojo.Schema; + +/** + * Source interaction for connectors that produce data column-by-column rather than row-by-row. + * + *

    Suitable for data sources where data is naturally organised in columnar format, such as + * Parquet files or columnar databases. The Flight layer detects this interface via {@code instanceof} + * and creates a {@link ColumnarArrowBatchWriter} instead of an {@link ArrowBatchWriter}. + * + *

    Connector authors pick the right sub-interface and get exactly the methods they need. + * There is no {@code UnsupportedOperationException} and no instanceof inside connectors. + * + *

    Example: + * 

    + *   public class MyColumnarSourceInteraction implements SdkColumnarSourceInteraction {
+ *       {@literal @}Override
+ *       public void stream(ColumnarWriter writer) throws Exception {
+ *           for (ColumnBatch batch : source.batches()) {
+ *               writer.writeColumn("id", batch.ids());
+ *               writer.writeColumn("value", batch.values());
+ *               writer.flushBatch();
+ *           }
+ *       }
+ *       // ... getSchema(), getTickets(), close()
+ *   }
+ *
    + */ +public interface SdkColumnarSourceInteraction extends SdkSourceInteraction +{ + /** + * {@inheritDoc} + *

    + * Default implementation delegates to {@link #stream(ColumnarWriter)}. + * The Flight layer calls this overload directly — connector authors implement + * {@link #stream(ColumnarWriter)} instead. + */ + @Override + default void stream(RowWriter writer) throws Exception + { + throw new UnsupportedOperationException( + "SdkColumnarSourceInteraction.stream(RowWriter) should not be called directly. " + + "The Flight layer calls stream(ColumnarWriter)."); + } + + /** + * Streams data from the source into the provided {@link ColumnarWriter}. + * + *

    The connector must write each column via {@link ColumnarWriter#writeColumn(String, Object[])} + * and call {@link ColumnarWriter#flushBatch()} after all columns for a batch are written. + * + * @param writer + * the columnar writer to receive column data + * @throws Exception + * if an error occurs during streaming + */ + void stream(ColumnarWriter writer) throws Exception; +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkColumnarTargetInteraction.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkColumnarTargetInteraction.java new file mode 100644 index 00000000..a840aaed --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkColumnarTargetInteraction.java @@ -0,0 +1,62 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +/** + * Target interaction for connectors that consume data column-by-column rather than row-by-row. + * + *

    Suitable for data targets where data is naturally organised in columnar format, such as + * Parquet files or columnar databases. The Flight layer detects this interface via {@code instanceof} + * and creates a {@link ColumnarArrowBatchReader} instead of an {@link ArrowBatchReader}. + * + *

    Connector authors pick the right sub-interface and get exactly the methods they need. + * There is no {@code UnsupportedOperationException} and no instanceof inside connectors. + * + *

    Example: + * 

    + *   public class MyColumnarTargetInteraction implements SdkColumnarTargetInteraction {
+ *       {@literal @}Override
+ *       public void consume(ColumnarReader reader) throws Exception {
+ *           while (reader.nextBatch()) {
+ *               Object[] ids = reader.getColumn("id");
+ *               Object[] values = reader.getColumn("value");
+ *               target.insertBatch(ids, values);
+ *           }
+ *       }
+ *       // ... setup(), wrapup(), close()
+ *   }
+ *
    + */ +public interface SdkColumnarTargetInteraction extends SdkTargetInteraction +{ + /** + * {@inheritDoc} + *

    + * Default implementation throws — the Flight layer calls {@link #consume(ColumnarReader)} instead. + */ + @Override + default void consume(RowReader reader) throws Exception + { + throw new UnsupportedOperationException( + "SdkColumnarTargetInteraction.consume(RowReader) should not be called directly. " + + "The Flight layer calls consume(ColumnarReader)."); + } + + /** + * Consumes incoming data from the provided {@link ColumnarReader}. + * + *

    The connector must call {@link ColumnarReader#nextBatch()} to advance to each batch + * and {@link ColumnarReader#getColumn(String)} to retrieve column values. + * + * @param reader + * the columnar reader providing column data + * @throws Exception + * if an error occurs during consumption + */ + void consume(ColumnarReader reader) throws Exception; +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkConnector.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkConnector.java new file mode 100644 index 00000000..e6bdc062 --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkConnector.java @@ -0,0 +1,90 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import org.apache.arrow.vector.types.pojo.Schema; + +/** + * Minimal SDK connector interface. + * + *

    Intentionally simpler than the library's {@code Connector<P,I,O,D>} — connector authors + * should not need to understand {@code PartitionPlan}, {@code ExecutionPhase}, or other + * library-specific lifecycle concepts. + * + *

    The generic parameters let connector authors expose concrete interaction types without casts: + * 

    + *   public class MyConnector implements SdkConnector<MySourceInteraction, MyTargetInteraction,
+ *                                                      MyDiscoveryInteraction> { ... }
+ *
    + * + * @param + * the source interaction type (must extend {@link SdkSourceInteraction}) + * @param + * the target interaction type (must extend {@link SdkTargetInteraction}) + * @param + * the discovery interaction type (must extend {@link SdkDiscoveryInteraction}) + */ +public interface SdkConnector + extends AutoCloseable +{ + /** + * Establishes the underlying connection to the data source. + * + * @throws Exception + * if the connection cannot be established + */ + void connect() throws Exception; + + /** + * Returns the Arrow schema for the named asset. + * + * @param asset + * the asset descriptor identifying the table or object + * @return the Arrow {@link Schema} describing the asset's fields + * @throws Exception + * if the schema cannot be determined + */ + Schema getSchema(AssetDescriptor asset) throws Exception; + + /** + * Creates a source interaction for reading data from the named asset. + * + * @param asset + * the asset descriptor identifying the table or object to read + * @param ticket + * the Arrow Flight ticket identifying this particular partition + * @return a new source interaction; caller must close it when done + * @throws Exception + * if the interaction cannot be created + */ + S getSourceInteraction(AssetDescriptor asset, org.apache.arrow.flight.Ticket ticket) throws Exception; + + /** + * Creates a target interaction for writing data to the named asset. + * + * @param asset + * the asset descriptor identifying the table or object to write + * @return a new target interaction; caller must close it when done + * @throws Exception + * if the interaction cannot be created + */ + T getTargetInteraction(AssetDescriptor asset) throws Exception; + + /** + * Creates a discovery interaction for browsing available assets. + * + * @param criteria + * the criteria scoping the discovery request + * @return a new discovery interaction; caller must close it when done + * @throws Exception + * if the interaction cannot be created + */ + D getDiscoveryInteraction(DiscoveryCriteria criteria) throws Exception; +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkConnectorFactory.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkConnectorFactory.java new file mode 100644 index 00000000..4b06e86a --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkConnectorFactory.java @@ -0,0 +1,56 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +/** + * Factory for creating {@link SdkConnector} instances. + * + *

    Implementations are discovered at runtime via {@code java.util.ServiceLoader}. Register + * a factory by placing its fully-qualified class name in + * {@code META-INF/services/com.ibm.wdp.connect.sdk.connector.SdkConnectorFactory}. + * + *

    Example: + * 

    + *   public class MyConnectorFactory implements SdkConnectorFactory {
+ *       {@literal @}Override
+ *       public SdkDatasourceTypes getDatasourceTypes() {
+ *           return new SdkDatasourceTypes(Collections.singletonList("my_connector"));
+ *       }
+ *       {@literal @}Override
+ *       public SdkConnector<?, ?, ?> createConnector(String datasourceTypeName,
+ *                                                       ConnectionProperties props) {
+ *           return new MyConnector(props);
+ *       }
+ *   }
+ *
    + */ +public interface SdkConnectorFactory +{ + /** + * Returns the datasource type names handled by this factory. + * + *

    The returned value is used by the runtime to route connector creation requests + * to the correct factory. + * + * @return the supported datasource type names + */ + SdkDatasourceTypes getDatasourceTypes(); + + /** + * Creates a new connector for the specified datasource type and connection properties. + * + * @param datasourceTypeName + * the datasource type name (must be one of those returned by {@link #getDatasourceTypes()}) + * @param properties + * the connection properties provided by the user + * @return a new connector instance; caller is responsible for closing it + * @throws Exception + * if the connector cannot be created + */ + SdkConnector createConnector(String datasourceTypeName, ConnectionProperties properties) throws Exception; +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkDatasourceTypes.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkDatasourceTypes.java new file mode 100644 index 00000000..c912403b --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkDatasourceTypes.java @@ -0,0 +1,66 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import java.util.Collections; +import java.util.List; + +/** + * Holds the list of datasource type names that an {@link SdkConnectorFactory} handles. + * + *

    Returned by {@link SdkConnectorFactory#getDatasourceTypes()} so the runtime can route + * connector creation requests to the correct factory. + * + *

    Instances are immutable once constructed. + */ +public final class SdkDatasourceTypes +{ + private final List typeNames; + + /** + * Creates a datasource types holder. + * + * @param typeNames + * the list of datasource type names handled by the owning factory; must not be null or empty + */ + public SdkDatasourceTypes(List typeNames) + { + if (typeNames == null || typeNames.isEmpty()) { + throw new IllegalArgumentException("typeNames must not be null or empty"); + } + this.typeNames = Collections.unmodifiableList(typeNames); + } + + /** + * Returns the datasource type names. + * + * @return an unmodifiable list of type names; never null or empty + */ + public List getTypeNames() + { + return typeNames; + } + + /** + * Returns true if this holder contains the given type name (case-sensitive). + * + * @param typeName + * the type name to test + * @return true if the type is handled + */ + public boolean handles(String typeName) + { + return typeNames.contains(typeName); + } + + @Override + public String toString() + { + return "SdkDatasourceTypes{typeNames=" + typeNames + "}"; + } +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkDiscoveryInteraction.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkDiscoveryInteraction.java new file mode 100644 index 00000000..7c6c8b4a --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkDiscoveryInteraction.java @@ -0,0 +1,33 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import java.util.List; + +/** + * Discovery interaction for browsing assets available through a connector. + * + *

    Functionally equivalent to the library's {@code DiscoveryInteraction} but does not require + * any library-specific types. Connector authors implement this interface to expose browseable + * assets (tables, schemas, etc.) without knowledge of the library's lifecycle management. + * + *

    Instances are obtained from {@link SdkConnector#getDiscoveryInteraction(DiscoveryCriteria)}. + */ +public interface SdkDiscoveryInteraction extends AutoCloseable +{ + /** + * Discovers assets matching the given criteria. + * + * @param criteria + * the criteria scoping this discovery request (path, datasource type, connection properties) + * @return a list of discovered assets; never null, may be empty + * @throws Exception + * if an error occurs during discovery + */ + List discoverAssets(DiscoveryCriteria criteria) throws Exception; +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkSourceInteraction.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkSourceInteraction.java new file mode 100644 index 00000000..29ebb826 --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkSourceInteraction.java @@ -0,0 +1,86 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import java.util.List; + +import org.apache.arrow.flight.Ticket; +import org.apache.arrow.vector.types.pojo.Schema; + +/** + * Source interaction for row-based connectors (most connectors). + * + *

    Connector authors implement this interface to stream rows into the Flight layer. + * The implementation calls {@link RowWriter#startRow()}, {@link RowWriter#set(String, Object)}, + * and {@link RowWriter#endRow()} for each row. + * + *

    For columnar connectors (Parquet, columnar databases), implement + * {@link SdkColumnarSourceInteraction} instead — it extends this interface and provides a + * {@code stream(ColumnarWriter)} overload. The Flight layer detects the correct interface via + * {@code instanceof} and creates the appropriate writer. + * + *

    Example: + * 

    + *   public class MySourceInteraction implements SdkSourceInteraction {
+ *       {@literal @}Override
+ *       public Schema getSchema() {
+ *           return mySchema;
+ *       }
+ *       {@literal @}Override
+ *       public List<Ticket> getTickets() {
+ *           return Collections.singletonList(new Ticket(new byte[0]));
+ *       }
+ *       {@literal @}Override
+ *       public void stream(RowWriter writer) throws Exception {
+ *           for (MyRow row : source.rows()) {
+ *               writer.startRow();
+ *               writer.set("id", row.getId());
+ *               writer.set("name", row.getName());
+ *               writer.endRow();
+ *           }
+ *       }
+ *   }
+ *
    + */ +public interface SdkSourceInteraction extends AutoCloseable +{ + /** + * Returns the Arrow schema describing the data this interaction will produce. + * + * @return the Arrow {@link Schema} + * @throws Exception + * if the schema cannot be determined + */ + Schema getSchema() throws Exception; + + /** + * Returns the list of Arrow Flight tickets representing the partitions this interaction covers. + * + *

    For single-partition connectors, return a list with a single ticket. The Flight layer + * will call {@link SdkConnector#getSourceInteraction(AssetDescriptor, Ticket)} once per ticket. + * + * @return a non-empty list of tickets + * @throws Exception + * if the ticket list cannot be determined + */ + List getTickets() throws Exception; + + /** + * Streams data rows into the provided {@link RowWriter}. + * + *

    The connector must call {@link RowWriter#startRow()}, {@link RowWriter#set(String, Object)}, + * and {@link RowWriter#endRow()} for each row. The writer batches rows internally and flushes + * at the configured batch size. + * + * @param writer + * the row writer to receive row data + * @throws Exception + * if an error occurs during streaming + */ + void stream(RowWriter writer) throws Exception; +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkTargetInteraction.java b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkTargetInteraction.java new file mode 100644 index 00000000..da559aa5 --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/main/java/com/ibm/wdp/connect/sdk/connector/SdkTargetInteraction.java @@ -0,0 +1,70 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +/** + * Target interaction for row-based connectors. + * + *

    Connector authors implement this interface to consume rows from the Flight layer. + * The implementation reads rows by calling {@link RowReader#nextRow()} and + * {@link RowReader#get(String)}. + * + *

    For columnar connectors, implement {@link SdkColumnarTargetInteraction} instead — it extends + * this interface and provides a {@code consume(ColumnarReader)} overload. + * + *

    Example: + * 

    + *   public class MyTargetInteraction implements SdkTargetInteraction {
+ *       {@literal @}Override
+ *       public void setup() throws Exception {
+ *           target.beginTransaction();
+ *       }
+ *       {@literal @}Override
+ *       public void consume(RowReader reader) throws Exception {
+ *           while (reader.nextRow()) {
+ *               target.insert((String) reader.get("id"), reader.get("value"));
+ *           }
+ *       }
+ *       {@literal @}Override
+ *       public void wrapup() throws Exception {
+ *           target.commit();
+ *       }
+ *   }
+ *
    + */ +public interface SdkTargetInteraction extends AutoCloseable +{ + /** + * Called before any data arrives. Use to create tables, begin transactions, etc. + * + * @throws Exception + * if setup fails + */ + void setup() throws Exception; + + /** + * Consumes incoming data rows from the provided {@link RowReader}. + * + *

    The connector must call {@link RowReader#nextRow()} to advance through rows and + * {@link RowReader#get(String)} to retrieve field values. + * + * @param reader + * the row reader providing incoming data + * @throws Exception + * if an error occurs during consumption + */ + void consume(RowReader reader) throws Exception; + + /** + * Called after all data has been consumed. Use to commit transactions, finalize files, etc. + * + * @throws Exception + * if wrapup fails + */ + void wrapup() throws Exception; +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/test/java/com/ibm/wdp/connect/sdk/connector/TestArrowBatchReader.java b/sdk-gen/subprojects/java/sdk-connector-api/src/test/java/com/ibm/wdp/connect/sdk/connector/TestArrowBatchReader.java new file mode 100644 index 00000000..ba1d8e03 --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/test/java/com/ibm/wdp/connect/sdk/connector/TestArrowBatchReader.java @@ -0,0 +1,124 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.Text; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit tests for {@link ArrowBatchReader}. + */ +public class TestArrowBatchReader +{ + private RootAllocator allocator; + private Schema schema; + + @Before + public void setUp() + { + allocator = new RootAllocator(Long.MAX_VALUE); + schema = new Schema(Arrays.asList( + new Field("id", new FieldType(false, new ArrowType.Int(32, true), null), null), + new Field("name", new FieldType(true, ArrowType.Utf8.INSTANCE, null), null))); + } + + @After + public void tearDown() + { + allocator.close(); + } + + private List makeBatches(int... rowCounts) + { + final List batches = new ArrayList<>(); + int idCounter = 1; + for (final int count : rowCounts) { + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + root.allocateNew(); + final IntVector idVec = (IntVector) root.getVector("id"); + final VarCharVector nameVec = (VarCharVector) root.getVector("name"); + for (int i = 0; i < count; i++) { + idVec.setSafe(i, idCounter++); + nameVec.setSafe(i, new Text("name" + i)); + } + root.setRowCount(count); + batches.add(root); + } + return batches; + } + + @Test + public void testIterateRows() throws Exception + { + final List batches = makeBatches(2, 3); + try (ArrowBatchReader reader = new ArrowBatchReader(batches)) { + int count = 0; + while (reader.nextRow()) { + count++; + final Object id = reader.get("id"); + final Object name = reader.get("name"); + assertFalse(id == null); + assertFalse(name == null); + } + assertEquals(5, count); + } + batches.forEach(VectorSchemaRoot::close); + } + + @Test + public void testEmptyBatchList() throws Exception + { + final List batches = new ArrayList<>(); + try (ArrowBatchReader reader = new ArrowBatchReader(batches)) { + assertFalse(reader.nextRow()); + } + } + + @Test + public void testNullValue() throws Exception + { + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + root.allocateNew(); + final IntVector idVec = (IntVector) root.getVector("id"); + final VarCharVector nameVec = (VarCharVector) root.getVector("name"); + idVec.setSafe(0, 99); + nameVec.setNull(0); + root.setRowCount(1); + + final List batches = new ArrayList<>(); + batches.add(root); + + try (ArrowBatchReader reader = new ArrowBatchReader(batches)) { + assertTrue(reader.nextRow()); + assertEquals(99, reader.get("id")); + assertNull(reader.get("name")); + assertFalse(reader.nextRow()); + } + root.close(); + } +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/test/java/com/ibm/wdp/connect/sdk/connector/TestArrowBatchWriter.java b/sdk-gen/subprojects/java/sdk-connector-api/src/test/java/com/ibm/wdp/connect/sdk/connector/TestArrowBatchWriter.java new file mode 100644 index 00000000..28e877b4 --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/test/java/com/ibm/wdp/connect/sdk/connector/TestArrowBatchWriter.java @@ -0,0 +1,143 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit tests for {@link ArrowBatchWriter}. + */ +public class TestArrowBatchWriter +{ + private RootAllocator allocator; + private Schema schema; + + @Before + public void setUp() + { + allocator = new RootAllocator(Long.MAX_VALUE); + schema = new Schema(Arrays.asList( + new Field("id", new FieldType(false, new ArrowType.Int(32, true), null), null), + new Field("name", new FieldType(true, ArrowType.Utf8.INSTANCE, null), null), + new Field("score", new FieldType(true, new ArrowType.FloatingPoint( + org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE), null), null))); + } + + @After + public void tearDown() + { + allocator.close(); + } + + @Test + public void testWriteAndReadSingleBatch() throws Exception + { + try (ArrowBatchWriter writer = new ArrowBatchWriter(schema, allocator, 100)) { + writer.startRow(); + writer.set("id", 1); + writer.set("name", "Alice"); + writer.set("score", 9.5); + writer.endRow(); + + writer.startRow(); + writer.set("id", 2); + writer.set("name", "Bob"); + writer.set("score", 8.0); + writer.endRow(); + + final Iterator it = writer.batches(); + assertTrue(it.hasNext()); + final VectorSchemaRoot root = it.next(); + assertEquals(2, root.getRowCount()); + root.close(); + assertFalse(it.hasNext()); + } + } + + @Test + public void testAutoFlushAtBatchSize() throws Exception + { + final int batchSize = 3; + try (ArrowBatchWriter writer = new ArrowBatchWriter(schema, allocator, batchSize)) { + for (int i = 0; i < 7; i++) { + writer.startRow(); + writer.set("id", i); + writer.set("name", "row" + i); + writer.set("score", (double) i); + writer.endRow(); + } + + int totalRows = 0; + int batchCount = 0; + final Iterator it = writer.batches(); + while (it.hasNext()) { + final VectorSchemaRoot root = it.next(); + totalRows += root.getRowCount(); + batchCount++; + root.close(); + } + assertEquals(7, totalRows); + // 7 rows with batchSize=3: two full batches flush at row 3 and 6, one partial at end + assertEquals(3, batchCount); + } + } + + @Test + public void testNullValue() throws Exception + { + try (ArrowBatchWriter writer = new ArrowBatchWriter(schema, allocator, 100)) { + writer.startRow(); + writer.set("id", 42); + writer.set("name", null); + writer.set("score", null); + writer.endRow(); + + final Iterator it = writer.batches(); + assertTrue(it.hasNext()); + final VectorSchemaRoot root = it.next(); + assertEquals(1, root.getRowCount()); + assertTrue(root.getVector("name").isNull(0)); + assertTrue(root.getVector("score").isNull(0)); + root.close(); + } + } + + @Test + public void testEmptyWriter() throws Exception + { + try (ArrowBatchWriter writer = new ArrowBatchWriter(schema, allocator, 100)) { + final Iterator it = writer.batches(); + assertFalse(it.hasNext()); + } + } + + @Test + public void testGetSchema() + { + try (ArrowBatchWriter writer = new ArrowBatchWriter(schema, allocator, 100)) { + assertEquals(schema, writer.getSchema()); + } + } +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/test/java/com/ibm/wdp/connect/sdk/connector/TestColumnarArrowBatchReader.java b/sdk-gen/subprojects/java/sdk-connector-api/src/test/java/com/ibm/wdp/connect/sdk/connector/TestColumnarArrowBatchReader.java new file mode 100644 index 00000000..70c4ed5a --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/test/java/com/ibm/wdp/connect/sdk/connector/TestColumnarArrowBatchReader.java @@ -0,0 +1,123 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.Text; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit tests for {@link ColumnarArrowBatchReader}. + */ +public class TestColumnarArrowBatchReader +{ + private RootAllocator allocator; + private Schema schema; + + @Before + public void setUp() + { + allocator = new RootAllocator(Long.MAX_VALUE); + schema = new Schema(Arrays.asList( + new Field("id", new FieldType(false, new ArrowType.Int(32, true), null), null), + new Field("label", new FieldType(true, ArrowType.Utf8.INSTANCE, null), null))); + } + + @After + public void tearDown() + { + allocator.close(); + } + + private List makeBatches(int rows1, int rows2) + { + final List batches = new ArrayList<>(); + for (final int count : new int[]{ rows1, rows2 }) { + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + root.allocateNew(); + final IntVector idVec = (IntVector) root.getVector("id"); + final VarCharVector labelVec = (VarCharVector) root.getVector("label"); + for (int i = 0; i < count; i++) { + idVec.setSafe(i, i + 1); + labelVec.setSafe(i, new Text("val" + i)); + } + root.setRowCount(count); + batches.add(root); + } + return batches; + } + + @Test + public void testIterateBatches() throws Exception + { + final List batches = makeBatches(3, 2); + try (ColumnarArrowBatchReader reader = new ColumnarArrowBatchReader(batches)) { + assertTrue(reader.nextBatch()); + final Object[] ids = reader.getColumn("id"); + assertEquals(3, ids.length); + + assertTrue(reader.nextBatch()); + final Object[] ids2 = reader.getColumn("id"); + assertEquals(2, ids2.length); + + assertFalse(reader.nextBatch()); + } + batches.forEach(VectorSchemaRoot::close); + } + + @Test + public void testNullInColumn() throws Exception + { + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + root.allocateNew(); + final IntVector idVec = (IntVector) root.getVector("id"); + final VarCharVector labelVec = (VarCharVector) root.getVector("label"); + idVec.setSafe(0, 7); + labelVec.setNull(0); + root.setRowCount(1); + + final List batches = new ArrayList<>(); + batches.add(root); + + try (ColumnarArrowBatchReader reader = new ColumnarArrowBatchReader(batches)) { + assertTrue(reader.nextBatch()); + final Object[] labels = reader.getColumn("label"); + assertEquals(1, labels.length); + assertNull(labels[0]); + assertFalse(reader.nextBatch()); + } + root.close(); + } + + @Test + public void testEmptyBatchList() throws Exception + { + final List batches = new ArrayList<>(); + try (ColumnarArrowBatchReader reader = new ColumnarArrowBatchReader(batches)) { + assertFalse(reader.nextBatch()); + } + } +} + +// Made with Bob diff --git a/sdk-gen/subprojects/java/sdk-connector-api/src/test/java/com/ibm/wdp/connect/sdk/connector/TestColumnarArrowBatchWriter.java b/sdk-gen/subprojects/java/sdk-connector-api/src/test/java/com/ibm/wdp/connect/sdk/connector/TestColumnarArrowBatchWriter.java new file mode 100644 index 00000000..1e5dbe59 --- /dev/null +++ b/sdk-gen/subprojects/java/sdk-connector-api/src/test/java/com/ibm/wdp/connect/sdk/connector/TestColumnarArrowBatchWriter.java @@ -0,0 +1,117 @@ +/* *************************************************** */ + +/* (C) Copyright IBM Corp. 2026 */ + +/* *************************************************** */ +package com.ibm.wdp.connect.sdk.connector; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.Iterator; + +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit tests for {@link ColumnarArrowBatchWriter}. + */ +public class TestColumnarArrowBatchWriter +{ + private RootAllocator allocator; + private Schema schema; + + @Before + public void setUp() + { + allocator = new RootAllocator(Long.MAX_VALUE); + schema = new Schema(Arrays.asList( + new Field("id", new FieldType(false, new ArrowType.Int(32, true), null), null), + new Field("value", new FieldType(true, ArrowType.Utf8.INSTANCE, null), null))); + } + + @After + public void tearDown() + { + allocator.close(); + } + + @Test + public void testWriteColumnarBatch() throws Exception + { + try (ColumnarArrowBatchWriter writer = new ColumnarArrowBatchWriter(schema, allocator, 100)) { + writer.writeColumn("id", new Object[]{ 1, 2, 3 }); + writer.writeColumn("value", new Object[]{ "alpha", "beta", "gamma" }); + writer.flushBatch(); + + final Iterator it = writer.batches(); + assertTrue(it.hasNext()); + final VectorSchemaRoot root = it.next(); + assertEquals(3, root.getRowCount()); + root.close(); + assertFalse(it.hasNext()); + } + } + + @Test + public void testMultipleBatches() throws Exception + { + try (ColumnarArrowBatchWriter writer = new ColumnarArrowBatchWriter(schema, allocator, 100)) { + // Batch 1 + writer.writeColumn("id", new Object[]{ 1, 2 }); + writer.writeColumn("value", new Object[]{ "a", "b" }); + writer.flushBatch(); + // Batch 2 + writer.writeColumn("id", new Object[]{ 3 }); + writer.writeColumn("value", new Object[]{ "c" }); + writer.flushBatch(); + + int totalRows = 0; + final Iterator it = writer.batches(); + while (it.hasNext()) { + final VectorSchemaRoot root = it.next(); + totalRows += root.getRowCount(); + root.close(); + } + assertEquals(3, totalRows); + } + } + + @Test + public void testNullColumn() throws Exception + { + try (ColumnarArrowBatchWriter writer = new ColumnarArrowBatchWriter(schema, allocator, 100)) { + writer.writeColumn("id", new Object[]{ 10 }); + writer.writeColumn("value", new Object[]{ null }); + writer.flushBatch(); + + final Iterator it = writer.batches(); + assertTrue(it.hasNext()); + final VectorSchemaRoot root = it.next(); + assertEquals(1, root.getRowCount()); + assertTrue(root.getVector("value").isNull(0)); + root.close(); + } + } + + @Test + public void testEmptyWriter() throws Exception + { + try (ColumnarArrowBatchWriter writer = new ColumnarArrowBatchWriter(schema, allocator, 100)) { + final Iterator it = writer.batches(); + assertFalse(it.hasNext()); + } + } +} + +// Made with Bob