Skip to content

Commit

Permalink
Fix typo in ROW_INDEX_STRUCT_FIELD
Browse files Browse the repository at this point in the history
  • Loading branch information
johanl-db committed Feb 9, 2024
1 parent 49f2625 commit d9ce381
Show file tree
Hide file tree
Showing 3 changed files with 5 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,7 @@ object DeltaParquetFileFormat {

/** Row index for each column */
val ROW_INDEX_COLUMN_NAME = "__delta_internal_row_index"
val ROW_INDEX_STRUCT_FILED = StructField(ROW_INDEX_COLUMN_NAME, LongType)
val ROW_INDEX_STRUCT_FIELD = StructField(ROW_INDEX_COLUMN_NAME, LongType)

/** Utility method to create a new writable vector */
private def newVector(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,19 +77,19 @@ object DMLWithDeletionVectorsHelper extends DeltaCommand {
* @param fileIndex the new file index
*/
private def replaceFileIndex(target: LogicalPlan, fileIndex: TahoeFileIndex): LogicalPlan = {
val rowIndexCol = AttributeReference(ROW_INDEX_COLUMN_NAME, ROW_INDEX_STRUCT_FILED.dataType)();
val rowIndexCol = AttributeReference(ROW_INDEX_COLUMN_NAME, ROW_INDEX_STRUCT_FIELD.dataType)();
var fileMetadataCol: AttributeReference = null

val newTarget = target.transformUp {
case l @ LogicalRelation(
hfsr @ HadoopFsRelation(_, _, _, _, format: DeltaParquetFileFormat, _), _, _, _) =>
fileMetadataCol = format.createFileMetadataCol()
// Take the existing schema and add additional metadata columns
val newDataSchema = StructType(hfsr.dataSchema).add(ROW_INDEX_STRUCT_FILED)
val newDataSchema =
StructType(hfsr.dataSchema).add(ROW_INDEX_STRUCT_FIELD)
val finalOutput = l.output ++ Seq(rowIndexCol, fileMetadataCol)
// Disable splitting and filter pushdown in order to generate the row-indexes
val newFormat = format.copy(isSplittable = false, disablePushDowns = true)

val newBaseRelation = hfsr.copy(
location = fileIndex,
dataSchema = newDataSchema,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ class DeltaParquetFileFormatSuite extends QueryTest
readingSchema = readingSchema.add(DeltaParquetFileFormat.IS_ROW_DELETED_STRUCT_FIELD)
}
if (readRowIndexCol) {
readingSchema = readingSchema.add(DeltaParquetFileFormat.ROW_INDEX_STRUCT_FILED)
readingSchema = readingSchema.add(DeltaParquetFileFormat.ROW_INDEX_STRUCT_FIELD)
}

// Fetch the only file in the DeltaLog snapshot
Expand Down

0 comments on commit d9ce381

Please sign in to comment.