HADOOP-19256. filesystem options

wired up to s3a create file builder, and nowhere else.

Change-Id: Ieaafa9ecdfd68306deb41ecf40c416374a310859

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java

@@ -710,4 +710,55 @@ private OpenFileOptions() {
     public static final String FS_OPTION_OPENFILE_EC_POLICY =
         FS_OPTION_OPENFILE + "ec.policy";
   }
+
+  /**
+   * The standard {@code createFile()} options.
+   * <p>
+   * If an option is not supported during file creation and it is considered
+   * part of a commit protocol, then, when supplied in a must() option,
+   * it MUST be rejected.
+   */
+  @InterfaceAudience.Public
+  @InterfaceStability.Evolving
+  public interface CreateFileOptionKeys {
+
+    /**
+     * {code createFile()} option to write a file iff there is nothing at the destination.
+     * This may happen during create() or in the close.
+     * <p>
+     * Explicitly set {@link #FS_OPTION_CREATE_IN_CLOSE} if you want to force the end of file
+     * creation.
+     *
+     * Value {@value}.
+     * <p>
+     * This can be set in the builder.
+     * <p>
+     * It should be exported as a path capability for all stores where
+     * the feature is available *and* enabled.
+     */
+    String FS_OPTION_CREATE_CONDITIONAL_OVERWRITE = "fs.option.create.conditional.overwrite";
+
+    /**
+     * Overwrite a file only if there is an Etag match. This option takes a string.
+     * Value {@value}.
+     */
+    String FS_OPTION_CREATE_CONDITIONAL_OVERWRITE_ETAG =
+        "fs.option.create.conditional.overwrite.etag";
+
+    /**
+     * String to define the content filetype.
+     * Value {@value}.
+     */
+    String FS_OPTION_CREATE_CONTENT_TYPE = "fs.option.create.content.type";
+
+    /**
+     * A flag which requires the filesystem to create files/objects in close(),
+     * rather than create/createFile.
+     * <p>
+     * Object stores with this behavior should also export it as a path capability.
+     *
+     * Value {@value}.
+     */
+    String FS_OPTION_CREATE_IN_CLOSE = "fs.option.create.in.close";
+  }
 }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java

@@ -1523,26 +1523,21 @@ private Constants() {
       "fs.s3a.performance.flags";
 
 
-  /**
-   * Flag for commit if none match.
-   * This can be set in the {code createFile()} builder.
-   * Value {@value}.
-   */
-  public static final String FS_OPTION_CREATEFILE_OVERWRITE_NONE = "fs.option.create.overwrite.none";
-
-  /**
-   * Etag matching takes a string.
-   * Value {@value}.
-   */
-  public static final String FS_OPTION_CREATEFILE_OVERWRITE_ETAG = "fs.option.create.overwrite.etag";
-
   /**
    * Is the create overwrite feature enabled or not?
    * A configuration option and a path status probe.
    * Value {@value}.
    */
   public static final String FS_S3A_CREATE_OVERWRITE_SUPPORTED = "fs.s3a.create.overwrite.supported";
 
+  /**
+   * Create a multipart file, always: {@value}.
+   * <p>
+   * This is inefficient and will not work on a store which doesn't support that feature,
+   * so is primarily for testing.
+   */
+  public static final String FS_S3A_CREATE_MULTIPART = "fs.s3a.create.multipart";
+
   /**
    * Prefix for adding a header to the object when created.
    * The actual value must have a "." suffix and then the actual header.

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java

@@ -223,6 +223,10 @@
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
 import static org.apache.hadoop.fs.CommonPathCapabilities.DIRECTORY_LISTING_INCONSISTENT;
+import static org.apache.hadoop.fs.Options.CreateFileOptionKeys.FS_OPTION_CREATE_CONDITIONAL_OVERWRITE;
+import static org.apache.hadoop.fs.Options.CreateFileOptionKeys.FS_OPTION_CREATE_CONDITIONAL_OVERWRITE_ETAG;
+import static org.apache.hadoop.fs.Options.CreateFileOptionKeys.FS_OPTION_CREATE_CONTENT_TYPE;
+import static org.apache.hadoop.fs.Options.CreateFileOptionKeys.FS_OPTION_CREATE_IN_CLOSE;
 import static org.apache.hadoop.fs.impl.FlagSet.buildFlagSet;
 import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
 import static org.apache.hadoop.fs.s3a.Constants.*;
@@ -2171,7 +2175,7 @@ private FSDataOutputStream innerCreateFile(
     String destKey = putTracker.getDestKey();
 
     // put options are derived from the option builder.
-    boolean conditionalCreate = options.isConditionalCreate();
+    boolean conditionalCreate = options.isConditionalOverwrite();
     final PutObjectOptions putOptions =
         new PutObjectOptions(null, options.getHeaders(), conditionalCreate, null);
 
@@ -5417,11 +5421,19 @@ public boolean hasPathCapability(final Path path, final String capability)
     case STORE_CAPABILITY_DIRECTORY_MARKER_MULTIPART_UPLOAD_ENABLED:
       return isMultipartUploadEnabled();
 
-    // create file options
+    // create file options which are always true
+
+    case FS_OPTION_CREATE_IN_CLOSE:
+    case FS_OPTION_CREATE_CONTENT_TYPE:
     case FS_S3A_CREATE_PERFORMANCE:
     case FS_S3A_CREATE_HEADER:
       return true;
 
+    case FS_OPTION_CREATE_CONDITIONAL_OVERWRITE:
+    case FS_OPTION_CREATE_CONDITIONAL_OVERWRITE_ETAG:
+      // TODO HADOOP-19256. conditional on enablement
+      return true;
+
     // is the FS configured for create file performance
     case FS_S3A_CREATE_PERFORMANCE_ENABLED:
       return performanceFlags.enabled(PerformanceFlagEnum.Create);