Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Spark][Refactor] Create helpers for DeletionVectorDescriptor #4123

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+22 −6
Open

[Spark][Refactor] Create helpers for DeletionVectorDescriptor #4123

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 22 additions & 6 deletions spark/src/main/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptor.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,11 +112,7 @@ case class DeletionVectorDescriptor(
require(isOnDisk, "Can't get a path for an inline deletion vector")
storageType match {
case UUID_DV_MARKER =>
// If the file was written with a random prefix, we have to extract that,
// before decoding the UUID.
val randomPrefixLength = pathOrInlineDv.length - Codec.Base85Codec.ENCODED_UUID_LENGTH
val (randomPrefix, encodedUuid) = pathOrInlineDv.splitAt(randomPrefixLength)
val uuid = Codec.Base85Codec.decodeUUID(encodedUuid)
val (randomPrefix, uuid) = getRandomPrefixAndUuid.get
assembleDeletionVectorPath(tableLocation, uuid, randomPrefix)
case PATH_DV_MARKER =>
// Since there is no need for legacy support for relative paths for DVs,
Expand All @@ -132,6 +128,20 @@ case class DeletionVectorDescriptor(
def urlEncodedPath(tablePath: Path): String =
SparkPath.fromPath(absolutePath(tablePath)).urlEncoded

/**
* Parse the prefix and UUID of a relative DV. Returns None if the DV is not relative.
*/
@JsonIgnore
def getRandomPrefixAndUuid: Option[(String, UUID)] = storageType match {
case UUID_DV_MARKER =>
// If the file was written with a random prefix, we have to extract that,
// before decoding the UUID.
val randomPrefixLength = pathOrInlineDv.length - Codec.Base85Codec.ENCODED_UUID_LENGTH
val (randomPrefix, encodedUuid) = pathOrInlineDv.splitAt(randomPrefixLength)
Some((randomPrefix, Codec.Base85Codec.decodeUUID(encodedUuid)))
case _ =>
None
}
/**
* Produce a copy of this DV, but using an absolute path.
*
Expand Down Expand Up @@ -318,14 +328,20 @@ object DeletionVectorDescriptor {
* Optionally, prepend a `prefix` to the name.
*/
def assembleDeletionVectorPath(targetParentPath: Path, id: UUID, prefix: String = ""): Path = {
val fileName = s"${DELETION_VECTOR_FILE_NAME_CORE}_${id}.bin"
val fileName = assembleDeletionVectorFileName(id)
if (prefix.nonEmpty) {
new Path(new Path(targetParentPath, prefix), fileName)
} else {
new Path(targetParentPath, fileName)
}
}

/**
* Return the unique file name for a deletion vector based on `id`.
*/
def assembleDeletionVectorFileName(id: UUID): String =
s"${DELETION_VECTOR_FILE_NAME_CORE}_${id}.bin"

/** Descriptor for an empty stored bitmap. */
val EMPTY: DeletionVectorDescriptor = DeletionVectorDescriptor(
storageType = INLINE_DV_MARKER,
Expand Down
Loading