The identifier is computed in the way described by the - * SWHID contents identifier, but it can use any hash - * algorithm.
- * - *When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier.
- * - * @param messageDigest The MessageDigest to use (for example SHA-1). - * @param data Data to digest. - * @return A generalized Git blob identifier. - * @since 1.22.0 - */ - public static byte[] gitBlob(final MessageDigest messageDigest, final byte[] data) { - messageDigest.reset(); - updateDigest(messageDigest, gitBlobPrefix(data.length)); - return digest(messageDigest, data); - } - - /** - * Reads through a byte array and return a generalized Git blob identifier. - * - *The identifier is computed in the way described by the - * SWHID contents identifier, but it can use any hash - * algorithm.
- * - *When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier.
- * - * @param messageDigest The MessageDigest to use (for example SHA-1). - * @param data Data to digest. - * @param options Options how to open the file. - * @return A generalized Git blob identifier. - * @throws IOException On error accessing the file. - * @since 1.22.0 - */ - public static byte[] gitBlob(final MessageDigest messageDigest, final Path data, final OpenOption... options) throws IOException { - messageDigest.reset(); - if (Files.isSymbolicLink(data)) { - final byte[] linkTarget = Files.readSymbolicLink(data).toString().getBytes(StandardCharsets.UTF_8); - updateDigest(messageDigest, gitBlobPrefix(linkTarget.length)); - return digest(messageDigest, linkTarget); - } - updateDigest(messageDigest, gitBlobPrefix(Files.size(data))); - return updateDigest(messageDigest, data, options).digest(); - } - - private static byte[] gitBlobPrefix(final long dataSize) { - return gitPrefix("blob ", dataSize); - } - - private static byte[] gitPrefix(final String prefix, final long dataSize) { - return (prefix + dataSize + "\0").getBytes(StandardCharsets.UTF_8); - } - - /** - * Returns a generalized Git tree identifier. - * - *The identifier is computed in the way described by the - * SWHID directory identifier, but it can use any hash - * algorithm.
- * - *When the hash algorithm is SHA-1, the identifier is identical to Git tree identifier and SWHID directory identifier.
- * - * @param messageDigest The MessageDigest to use (for example SHA-1). - * @param entries The directory entries. - * @return A generalized Git tree identifier. - */ - static byte[] gitTree(final MessageDigest messageDigest, final CollectionThe identifier is computed in the way described by the - * SWHID directory identifier, but it can use any hash - * algorithm.
- * - *When the hash algorithm is SHA-1, the identifier is identical to Git tree identifier and SWHID directory identifier.
- * - * @param messageDigest The MessageDigest to use (for example SHA-1). - * @param data Data to digest. - * @param options Options how to open the file. - * @return A generalized Git tree identifier. - * @throws IOException On error accessing the file. - * @since 1.22.0 - */ - public static byte[] gitTree(final MessageDigest messageDigest, final Path data, final OpenOption... options) throws IOException { - final ListA Git tree object encodes a directory snapshot. Each entry holds:
- *Entries are ordered by {@link #compareTo} using Git's tree-sort rule: directory names are compared as if they ended with {@code '/'}, so that {@code foo/} - * sorts after {@code foobar}.
- * - *Call {@link #toTreeEntryBytes()} to obtain the binary encoding that Git feeds to its hash function when computing the tree object identifier.
- * - * @see Git Internals – Git Objects - * @see SWHID Directory Identifier - */ -class GitDirectoryEntry implements ComparableGit encodes the file type and permission bits as an ASCII octal string that precedes the entry name in the binary tree format. The values defined here - * cover the four entry types that Git itself produces.
- * - *This enum is package-private. If it were made public, {@link #mode} would need to be wrapped in an immutable copy to prevent external mutation.
- */ - enum Type { - - /** - * A sub-directory (Git sub-tree). - */ - DIRECTORY("40000"), - - /** - * An executable file. - */ - EXECUTABLE("100755"), - - /** - * A regular (non-executable) file. - */ - REGULAR("100644"), - - /** - * A symbolic link. - */ - SYMBOLIC_LINK("120000"); - - /** - * The ASCII-encoded octal mode string as it appears in the binary tree entry. - */ - private final byte[] mode; - - Type(final String mode) { - this.mode = mode.getBytes(StandardCharsets.US_ASCII); - } - } - - private static String getFileName(final Path path) { - final Path fileName = path.getFileName(); - if (fileName == null) { - throw new IllegalArgumentException(path.toString()); - } - return fileName.toString(); - } - - /** - * The entry name (file or directory name, no path separator). - */ - private final String name; - - /** - * The key used for ordering entries within a tree object. - * - *>Git appends {@code '/'} to directory names before comparing.
- */ - private final String sortKey; - - /** - * The Git object type, which determines the Unix file-mode prefix. - */ - private final Type type; - - /** - * The raw object id of the referenced blob or sub-tree. - */ - private final byte[] rawObjectId; - - /** - * Creates an entry. - * - * @param path The path of the entry; must not be an empty path. - * @param type The type of the entry. - * @param rawObjectId The id of the entry. - * @throws IllegalArgumentException If the path is empty. - * @throws NullPointerException If any argument is {@code null}. - */ - GitDirectoryEntry(final Path path, final Type type, final byte[] rawObjectId) { - this(getFileName(path), type, rawObjectId); - } - - /** - * Creates an entry. - * - * @param name The name of the entry - * @param type The type of the entry - * @param rawObjectId The id of the entry - */ - private GitDirectoryEntry(final String name, final Type type, final byte[] rawObjectId) { - this.name = name; - this.type = Objects.requireNonNull(type); - this.sortKey = type == Type.DIRECTORY ? name + "/" : name; - this.rawObjectId = Objects.requireNonNull(rawObjectId); - } - - @Override - public int compareTo(final GitDirectoryEntry o) { - return sortKey.compareTo(o.sortKey); - } - - @Override - public boolean equals(final Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof GitDirectoryEntry)) { - return false; - } - final GitDirectoryEntry other = (GitDirectoryEntry) obj; - return name.equals(other.name); - } - - @Override - public int hashCode() { - return name.hashCode(); - } - - /** - * Returns the binary encoding of this entry as it appears inside a Git tree object. - * - *The format follows the Git tree entry layout:
- *- * <mode> SP <name> NUL <20-byte-object-id> - *- * - * @return the binary tree-entry encoding; never {@code null}. - */ - byte[] toTreeEntryBytes() { - final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); - final byte[] result = new byte[type.mode.length + nameBytes.length + rawObjectId.length + 2]; - System.arraycopy(type.mode, 0, result, 0, type.mode.length); - result[type.mode.length] = ' '; - System.arraycopy(nameBytes, 0, result, type.mode.length + 1, nameBytes.length); - result[type.mode.length + nameBytes.length + 1] = '\0'; - System.arraycopy(rawObjectId, 0, result, type.mode.length + nameBytes.length + 2, rawObjectId.length); - return result; - } -} diff --git a/src/main/java/org/apache/commons/codec/digest/GitIdentifiers.java b/src/main/java/org/apache/commons/codec/digest/GitIdentifiers.java new file mode 100644 index 0000000000..6b4c0ccf2f --- /dev/null +++ b/src/main/java/org/apache/commons/codec/digest/GitIdentifiers.java @@ -0,0 +1,452 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.commons.codec.digest; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +/** + * Operations for computing Git object identifiers and their generalizations described by the + * SWHID specification. + * + *
When the hash algorithm is SHA-1, the identifiers produced by this class are identical to those used by Git. + * Other hash algorithms produce generalized identifiers as described by the SWHID specification.
+ * + *This class is immutable and thread-safe. However, the {@link MessageDigest} instances passed to it generally won't be.
+ * + * @see Git Internals – Git Objects + * @see SWHID Specification + * @since 1.22.0 + */ +public class GitIdentifiers { + + /** + * The type of a Git tree entry, which maps to a Unix file-mode string. + * + *Git encodes the file type and permission bits as an ASCII octal string that precedes the entry name in the binary tree format. The values defined here + * cover the four entry types that Git itself produces.
+ */ + public enum FileMode { + + /** + * A sub-directory (Git sub-tree). + */ + DIRECTORY("40000"), + + /** + * An executable file. + */ + EXECUTABLE("100755"), + + /** + * A regular (non-executable) file. + */ + REGULAR("100644"), + + /** + * A symbolic link. + */ + SYMBOLIC_LINK("120000"); + + /** + * The octal mode as used by Git. + */ + private final String mode; + + /** + * Serialized {@code mode}: since this is mutable, it must remain private. + */ + private final byte[] modeBytes; + + FileMode(final String mode) { + this.mode = mode; + this.modeBytes = mode.getBytes(StandardCharsets.US_ASCII); + } + + /** + * Gets the octal mode as used by Git. + * + * @return The octal mode + */ + public String getMode() { + return mode; + } + } + + /** + * Represents a single entry in a Git tree object. + * + *A Git tree object encodes a directory snapshot. Each entry holds:
+ *Entries are ordered by {@link #compareTo} using Git's tree-sort rule: directory names are compared as if they ended with {@code '/'}, so that {@code foo/} + * sorts after {@code foobar}.
+ * + * @see Git Internals – Git Objects + * @see SWHID Directory Identifier + */ + static class DirectoryEntry implements Comparable>Git appends {@code '/'} to directory names before comparing.
+ */ + private final String sortKey; + /** + * The Git object type, which determines the Unix file-mode prefix. + */ + private final FileMode type; + + /** + * Creates an entry. + * + * @param name The name of the entry + * @param type The type of the entry + * @param rawObjectId The id of the entry + */ + DirectoryEntry(final String name, final FileMode type, final byte[] rawObjectId) { + if (Objects.requireNonNull(name).indexOf('/') >= 0) { + throw new IllegalArgumentException("Entry name must not contain '/': " + name); + } + this.name = name; + this.type = Objects.requireNonNull(type); + this.sortKey = type == FileMode.DIRECTORY ? name + "/" : name; + this.rawObjectId = Objects.requireNonNull(rawObjectId); + } + + @Override + public int compareTo(final DirectoryEntry o) { + return sortKey.compareTo(o.sortKey); + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof DirectoryEntry)) { + return false; + } + final DirectoryEntry other = (DirectoryEntry) obj; + return name.equals(other.name); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + } + + /** + * Builds a Git tree identifier for a virtual directory structure, such as the contents of + * an archive. + */ + public static final class TreeIdBuilder { + + /** + * A supplier of a blob identifier that may throw {@link IOException}. + */ + @FunctionalInterface + private interface BlobIdSupplier { + byte[] get() throws IOException; + } + + private static String requireNoParentTraversal(String name) { + if ("..".equals(name)) { + throw new IllegalArgumentException("Path component not allowed: " + name); + } + return name; + } + + private final MapIf {@code name} contains {@code '/'}, intermediate subdirectories are created automatically.
+ * + *The stream is eagerly drained.
+ * + * @param mode The file mode (e.g. {@link FileMode#REGULAR}). + * @param name The relative path of the entry in normalized form(may contain {@code '/'}). + * @param dataSize The exact number of bytes in {@code data}. + * @param data The file content. + * @throws IOException If the stream cannot be read. + * @throws IllegalArgumentException If any path component is {@code ".."}. + */ + public void addFile(final FileMode mode, final String name, final long dataSize, final InputStream data) throws IOException { + addFile(mode, name, () -> blobId(messageDigest, dataSize, data)); + } + + private void addFile(final FileMode mode, final String name, final BlobIdSupplier blobId) throws IOException { + final int slash = name.lastIndexOf('/'); + if (slash < 0) { + fileEntries.put(name, new DirectoryEntry(requireNoParentTraversal(name), mode, blobId.get())); + } else { + addDirectory(name.substring(0, slash)).addFile(mode, name.substring(slash + 1), blobId); + } + } + + /** + * Adds a file entry at the given path within this tree. + * + *If {@code name} contains {@code '/'}, intermediate subdirectories are created automatically.
+ * + * @param mode The file mode (e.g. {@link FileMode#REGULAR}). + * @param name The relative path of the entry in normalized form(may contain {@code '/'}). + * @param data The file content. + * @throws IOException If an I/O error occurs. + * @throws IllegalArgumentException If any path component is {@code ".."}. + */ + public void addFile(final FileMode mode, final String name, final byte[] data) throws IOException { + addFile(mode, name, () -> blobId(messageDigest, data)); + } + + /** + * Adds a symbolic link entry at the give path within this tree. + * + *If {@code name} contains {@code '/'}, intermediate subdirectories are created automatically.
+ * + * @param name The relative path of the entry in normalized form(may contain {@code '/'}). + * @param target The target of the symbolic link. + * @throws IOException If an I/O error occurs. + * @throws IllegalArgumentException If any path component is {@code ".."}. + */ + public void addSymbolicLink(final String name, final String target) throws IOException { + addFile(FileMode.SYMBOLIC_LINK, name, target.getBytes(StandardCharsets.UTF_8)); + } + + /** + * Computes the Git tree identifier for this directory and all its descendants. + * + * @return The raw tree identifier bytes. + * @throws IOException If a digest operation fails. + */ + public byte[] build() throws IOException { + final SetThe identifier is computed in the way described by the + * SWHID contents identifier, but it can use any hash + * algorithm.
+ * + *When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier.
+ * + * @param messageDigest The MessageDigest to use (for example SHA-1). + * @param data Data to digest. + * @return A generalized Git blob identifier. + */ + public static byte[] blobId(final MessageDigest messageDigest, final byte[] data) { + messageDigest.reset(); + DigestUtils.updateDigest(messageDigest, getGitBlobPrefix(data.length)); + return DigestUtils.digest(messageDigest, data); + } + + /** + * Reads through a stream of known size and returns a generalized Git blob identifier, without buffering. + * + *When the size of the content is known in advance, this overload streams {@code data} directly through + * the digest without buffering the full content in memory.
+ * + *When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier.
+ * + * @param messageDigest The MessageDigest to use (for example SHA-1). + * @param dataSize The exact number of bytes in {@code data}. + * @param data Stream to digest. + * @return A generalized Git blob identifier. + * @throws IOException On error reading the stream. + */ + public static byte[] blobId(final MessageDigest messageDigest, final long dataSize, final InputStream data) throws IOException { + messageDigest.reset(); + DigestUtils.updateDigest(messageDigest, getGitBlobPrefix(dataSize)); + return DigestUtils.updateDigest(messageDigest, data).digest(); + } + + /** + * Reads through a file and returns a generalized Git blob identifier. + * + *The identifier is computed in the way described by the + * SWHID contents identifier, but it can use any hash + * algorithm.
+ * + *When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier.
+ * + * @param messageDigest The MessageDigest to use (for example SHA-1). + * @param data Path to the file to digest. + * @return A generalized Git blob identifier. + * @throws IOException On error accessing the file. + */ + public static byte[] blobId(final MessageDigest messageDigest, final Path data) throws IOException { + if (Files.isSymbolicLink(data)) { + final byte[] linkTarget = Files.readSymbolicLink(data).toString().getBytes(StandardCharsets.UTF_8); + return blobId(messageDigest, linkTarget); + } + messageDigest.reset(); + DigestUtils.updateDigest(messageDigest, getGitBlobPrefix(Files.size(data))); + return DigestUtils.updateDigest(messageDigest, data).digest(); + } + + private static FileMode getGitDirectoryEntryType(final Path path) { + // Symbolic links first + if (Files.isSymbolicLink(path)) { + return FileMode.SYMBOLIC_LINK; + } + if (Files.isDirectory(path)) { + return FileMode.DIRECTORY; + } + if (Files.isExecutable(path)) { + return FileMode.EXECUTABLE; + } + return FileMode.REGULAR; + } + + private static byte[] getGitBlobPrefix(final long dataSize) { + return getGitPrefix("blob", dataSize); + } + + private static byte[] getGitPrefix(final String type, final long dataSize) { + return (type + " " + dataSize + "\0").getBytes(StandardCharsets.UTF_8); + } + + private static byte[] getGitTreePrefix(final long dataSize) { + return getGitPrefix("tree", dataSize); + } + + private static void populateFromPath(final TreeIdBuilder builder, final Path directory) throws IOException { + try (DirectoryStreamThe identifier is computed in the way described by the + * SWHID directory identifier, but it can use any hash + * algorithm.
+ * + *When the hash algorithm is SHA-1, the identifier is identical to Git tree identifier and SWHID directory identifier.
+ * + * @param messageDigest The MessageDigest to use (for example SHA-1). + * @param data Path to the directory to digest. + * @return A generalized Git tree identifier. + * @throws IOException On error accessing the directory or its contents. + */ + public static byte[] treeId(final MessageDigest messageDigest, final Path data) throws IOException { + final TreeIdBuilder builder = treeIdBuilder(messageDigest); + populateFromPath(builder, data); + return builder.build(); + } + + /** + * Returns a new {@link TreeIdBuilder} for constructing a generalized Git tree identifier from a virtual directory + * structure, such as the contents of an archive. + * + *The identifier is computed in the way described by the + * SWHID directory identifier, but it can use any hash + * algorithm.
+ * + *When the hash algorithm is SHA-1, the identifier is identical to Git tree identifier and SWHID directory identifier.
+ * + * @param messageDigest The MessageDigest to use (for example SHA-1). + * @return A new {@link TreeIdBuilder}. + */ + public static TreeIdBuilder treeIdBuilder(final MessageDigest messageDigest) { + return new TreeIdBuilder(messageDigest); + } + + private GitIdentifiers() { + // utility class + } +} diff --git a/src/test/java/org/apache/commons/codec/digest/DigestUtilsTest.java b/src/test/java/org/apache/commons/codec/digest/DigestUtilsTest.java index 7d1e72b0b8..6f7160baa7 100644 --- a/src/test/java/org/apache/commons/codec/digest/DigestUtilsTest.java +++ b/src/test/java/org/apache/commons/codec/digest/DigestUtilsTest.java @@ -32,14 +32,11 @@ import java.io.OutputStream; import java.io.RandomAccessFile; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.security.MessageDigest; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; import java.util.Locale; import java.util.Random; import java.util.stream.Stream; @@ -50,14 +47,11 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.ValueSource; /** * Tests {@link DigestUtils}. @@ -244,31 +238,6 @@ class DigestUtilsTest { "CA 92 BF 0B E5 61 5E 96 95 9D 76 71 97 A0 BE EB"; // @formatter:on - /** - * Binary body of the test tree object used in {@link #testGitTreeCollection}. - * - *Each entry has the format {@code
Git compares the names of the entries, but adds a {@code /} at the end of directory entries.
- */ - @Test - void testSortOrder() { - final GitDirectoryEntry alpha = new GitDirectoryEntry(Paths.get("alpha.txt"), GitDirectoryEntry.Type.REGULAR, ZERO_ID); - final GitDirectoryEntry fooTxt = new GitDirectoryEntry(Paths.get("foo.txt"), GitDirectoryEntry.Type.REGULAR, ZERO_ID); - final GitDirectoryEntry fooDir = new GitDirectoryEntry(Paths.get("foo"), GitDirectoryEntry.Type.DIRECTORY, ZERO_ID); - final GitDirectoryEntry foobar = new GitDirectoryEntry(Paths.get("foobar"), GitDirectoryEntry.Type.REGULAR, ZERO_ID); - final GitDirectoryEntry zeta = new GitDirectoryEntry(Paths.get("zeta.txt"), GitDirectoryEntry.Type.REGULAR, ZERO_ID); - final ListGit compares the names of the entries, but adds a {@code /} at the end of directory entries.
+ */ + @Test + void testDirectoryEntrySortOrder() { + final DirectoryEntry alpha = new DirectoryEntry("alpha.txt", GitIdentifiers.FileMode.REGULAR, ZERO_ID); + final DirectoryEntry fooTxt = new DirectoryEntry("foo.txt", GitIdentifiers.FileMode.REGULAR, ZERO_ID); + final DirectoryEntry fooDir = new DirectoryEntry("foo", GitIdentifiers.FileMode.DIRECTORY, ZERO_ID); + final DirectoryEntry foobar = new DirectoryEntry("foobar", GitIdentifiers.FileMode.REGULAR, ZERO_ID); + final DirectoryEntry zeta = new DirectoryEntry("zeta.txt", GitIdentifiers.FileMode.REGULAR, ZERO_ID); + final List