Add Files API endpoints

csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram.Files.cs

@@ -0,0 +1,70 @@
+namespace Microsoft.Azure.Databricks.Client.Sample;
+
+internal static partial class SampleProgram
+{
+    private static async Task TestFilesApi(DatabricksClient client)
+    {
+        Console.WriteLine("First specify a volume URI path where the tests will be executed...");
+        var basePath = Console.ReadLine();
+
+        Console.WriteLine($"Using '{basePath}' as base path for the next tests.");
+
+        // Create directory
+        var directoryPath = basePath + "/" + Guid.NewGuid();
+        Console.WriteLine($"Creating empty '{directoryPath}' directory.");
+
+        await client.Files.CreateDirectory(directoryPath);
+
+        // Get newly created directory metadata
+        Console.WriteLine("Fetching directory metadata.");
+        var contentHeaders = await client.Files.GetDirectoryMetadata(directoryPath);
+        foreach (var contentHeader in contentHeaders)
+        {
+            Console.WriteLine($"'{contentHeader.Key}': '{contentHeader.Value.First()}'");
+        }
+
+        // Populate the directory with a file
+        Console.WriteLine($"Uploading test file to '{directoryPath}' directory.'");
+        var uploadPath = directoryPath + "/" + Guid.NewGuid() + ".txt";
+
+        using var httpClient = new HttpClient();
+        var response = await httpClient.GetAsync("https://norvig.com/big.txt",
+            HttpCompletionOption.ResponseHeadersRead);
+
+        var fileContents = await response.Content.ReadAsStreamAsync();
+        await client.Files.Upload(uploadPath, fileContents, true);
+
+        // Read the file first 100 bytes
+        Console.WriteLine("Reading test file...");
+        using var msDownload = new MemoryStream();
+        await client.Files.Download(uploadPath, msDownload, "bytes=0-99");
+        msDownload.Position = 0;
+        var sr = new StreamReader(msDownload);
+        var content = await sr.ReadToEndAsync();
+        Console.WriteLine(content[..99]);
+
+        // Read the file metadata
+        Console.WriteLine("Getting file metadata...");
+        var metadataHeaders = await client.Files.GetFileMetadata(uploadPath);
+        foreach (var header in metadataHeaders)
+        {
+            Console.WriteLine($"'{header.Key}': '{header.Value.First()}'");
+        }
+
+        // List directory contents
+        Console.WriteLine($"Listing test directory contents...");
+
+        await foreach (var entry in client.Files.ListDirectoryContentsPageable(directoryPath))
+        {
+            Console.WriteLine(entry);
+        }
+
+        // Delete the file
+        Console.WriteLine("Deleting test file...");
+        await client.Files.Delete(uploadPath);
+
+        // Delete the directory
+        Console.WriteLine("Deleting test directory...");
+        await client.Files.DeleteDirectory(directoryPath);
+    }
+}

csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram.cs

@@ -2,12 +2,11 @@
 // Licensed under the MIT License.
 
 using Azure.Identity;
+
 using Microsoft.Azure.Databricks.Client.Converters;
-using System;
-using System.Net.Http;
+
 using System.Text.Json;
 using System.Text.Json.Serialization;
-using System.Threading.Tasks;
 
 namespace Microsoft.Azure.Databricks.Client.Sample;
 
@@ -67,6 +66,7 @@ public static async Task Main(string[] args)
             await TestClustersApi(client);
             await TestGroupsApi(client);
             await TestDbfsApi(client);
+            await TestFilesApi(client);
             await TestJobsApi(client);
             await TestPermissionsApi(client);
             await TestWarehouseApi(client);

csharp/Microsoft.Azure.Databricks.Client.Test/FilesApiClientTest.cs

@@ -0,0 +1,231 @@
+using System.Net;
+using System.Text;
+using System.Text.Json;
+
+using Moq.Contrib.HttpClient;
+
+namespace Microsoft.Azure.Databricks.Client.Test;
+
+[TestClass]
+public class FilesApiClientTest : ApiClientTest
+{
+    private static readonly Uri DirectoriesApiUri = new(BaseApiUri, "2.0/fs/directories");
+    private static readonly Uri FilesApiUri = new(BaseApiUri, "2.0/fs/files");
+
+    private static readonly string DirectoryRelativePath = "/" + Guid.NewGuid();
+    private static readonly string FileRelativePath = DirectoryRelativePath + "/" + Guid.NewGuid() + ".txt";
+
+    [TestMethod]
+    public async Task TestCreateDirectory()
+    {
+        var requestUri = $"{DirectoriesApiUri}{DirectoryRelativePath}";
+
+        var handler = CreateMockHandler();
+        handler
+            .SetupRequest(HttpMethod.Put, requestUri)
+            .ReturnsResponse(HttpStatusCode.NoContent);
+
+        var mockClient = handler.CreateClient();
+        mockClient.BaseAddress = BaseApiUri;
+
+        using var client = new FilesApiClient(mockClient);
+        await client.CreateDirectory(DirectoryRelativePath);
+
+        handler.VerifyRequest(
+            HttpMethod.Put,
+            requestUri);
+    }
+
+    [TestMethod]
+    public async Task TestDeleteDirectory()
+    {
+        var requestUri = $"{DirectoriesApiUri}{DirectoryRelativePath}";
+
+        var handler = CreateMockHandler();
+        handler
+            .SetupRequest(HttpMethod.Delete, requestUri)
+            .ReturnsResponse(HttpStatusCode.NoContent);
+
+        var mockClient = handler.CreateClient();
+        mockClient.BaseAddress = BaseApiUri;
+
+        using var client = new FilesApiClient(mockClient);
+        await client.DeleteDirectory(DirectoryRelativePath);
+
+        handler.VerifyRequest(
+            HttpMethod.Delete,
+            requestUri);
+    }
+
+    [TestMethod]
+    public async Task TestGetDirectoryMetadata()
+    {
+        const string expectedHeaderName = "X-Test-Header";
+        const string expectedHeaderValue = "Value";
+        var requestUri = $"{DirectoriesApiUri}{DirectoryRelativePath}";
+
+        var handler = CreateMockHandler();
+        handler
+            .SetupRequest(HttpMethod.Head, requestUri)
+            .ReturnsResponse(HttpStatusCode.OK, message =>
+            {
+                message.Content.Headers.Add(expectedHeaderName, expectedHeaderValue);
+            });
+
+        var mockClient = handler.CreateClient();
+        mockClient.BaseAddress = BaseApiUri;
+
+        using var client = new FilesApiClient(mockClient);
+        var response = await client.GetDirectoryMetadata(DirectoryRelativePath);
+
+        Assert.IsTrue(response.Contains(expectedHeaderName));
+
+        handler.VerifyRequest(
+            HttpMethod.Head,
+            requestUri);
+    }
+
+    [TestMethod]
+    public async Task TestListDirectoryContents_ThrowsWithInvalidPageSize()
+    {
+        var handler = CreateMockHandler();
+        var mockClient = handler.CreateClient();
+        mockClient.BaseAddress = BaseApiUri;
+        using var client = new FilesApiClient(mockClient);
+        await Assert.ThrowsExceptionAsync<ArgumentOutOfRangeException>(async () => await client.ListDirectoryContents(DirectoryRelativePath, 15000));
+    }
+
+    [TestMethod]
+    public async Task TestListDirectoryContents()
+    {
+        var expectedResponse = @"
+        {
+          ""contents"": [
+            {
+              ""file_size"": 114864646,
+              ""is_directory"": true,
+              ""last_modified"": 646859599,
+              ""name"": ""test"",
+              ""path"": ""/Volumes/test-catalog/test-schema/test-volume/test/test.txt""
+            }
+          ],
+          ""next_page_token"": ""test-token""
+        }";
+
+        var requestUri = $"{DirectoriesApiUri}{DirectoryRelativePath}?";
+
+        var handler = CreateMockHandler();
+        handler
+            .SetupRequest(HttpMethod.Get, requestUri)
+            .ReturnsResponse(HttpStatusCode.OK, expectedResponse, "application/json");
+
+        var mockClient = handler.CreateClient();
+        mockClient.BaseAddress = BaseApiUri;
+
+        using var client = new FilesApiClient(mockClient);
+        var actual = await client.ListDirectoryContents(DirectoryRelativePath);
+
+        var actualJson = JsonSerializer.Serialize(new { contents = actual.Item1, next_page_token = actual.Item2 }, Options);
+        AssertJsonDeepEquals(expectedResponse, actualJson);
+    }
+
+    [TestMethod]
+    public async Task TestUploadFile()
+    {
+        var requestUri = $"{FilesApiUri}{FileRelativePath}?overwrite=true";
+
+        var fileContents = new MemoryStream(Guid.NewGuid().ToByteArray());
+
+        var handler = CreateMockHandler();
+        handler
+            .SetupRequest(HttpMethod.Put, requestUri)
+            .ReturnsResponse(HttpStatusCode.NoContent);
+
+        var mockClient = handler.CreateClient();
+        mockClient.BaseAddress = BaseApiUri;
+
+        using var client = new FilesApiClient(mockClient);
+        await client.Upload(FileRelativePath, fileContents, true);
+
+        handler.VerifyRequest(
+            HttpMethod.Put,
+            requestUri);
+    }
+
+    [TestMethod]
+    public async Task TestGetFileMetadata()
+    {
+        const string expectedHeaderName = "X-Test-Header";
+        const string expectedHeaderValue = "Value";
+        var requestUri = $"{FilesApiUri}{FileRelativePath}";
+
+        var handler = CreateMockHandler();
+        handler
+            .SetupRequest(HttpMethod.Head, requestUri)
+            .ReturnsResponse(HttpStatusCode.OK, message =>
+            {
+                message.Content.Headers.Add(expectedHeaderName, expectedHeaderValue);
+            });
+
+        var mockClient = handler.CreateClient();
+        mockClient.BaseAddress = BaseApiUri;
+
+        using var client = new FilesApiClient(mockClient);
+        var response = await client.GetFileMetadata(FileRelativePath);
+
+        Assert.IsTrue(response.Contains(expectedHeaderName));
+
+        handler.VerifyRequest(
+            HttpMethod.Head,
+            requestUri);
+    }
+
+    [TestMethod]
+    public async Task TestDownloadFile()
+    {
+        var requestUri = $"{FilesApiUri}{FileRelativePath}";
+        var expectedResponse = "Hello World!"u8.ToArray();
+
+        var handler = CreateMockHandler();
+        handler
+            .SetupRequest(HttpMethod.Get, requestUri)
+            .ReturnsResponse(HttpStatusCode.OK, expectedResponse, "application/octet-stream");
+
+        var mockClient = handler.CreateClient();
+        mockClient.BaseAddress = BaseApiUri;
+
+        using var msDownload = new MemoryStream();
+        using var client = new FilesApiClient(mockClient);
+        await client.Download(FileRelativePath, msDownload);
+        msDownload.Position = 0;
+        var sr = new StreamReader(msDownload, Encoding.UTF8);
+        var content = await sr.ReadToEndAsync();
+
+        Assert.AreEqual(Encoding.UTF8.GetString(expectedResponse), content);
+
+        handler.VerifyRequest(
+            HttpMethod.Get,
+            requestUri);
+    }
+
+    [TestMethod]
+    public async Task TestDeleteFile()
+    {
+        var requestUri = $"{FilesApiUri}{FileRelativePath}";
+
+        var handler = CreateMockHandler();
+        handler
+            .SetupRequest(HttpMethod.Delete, requestUri)
+            .ReturnsResponse(HttpStatusCode.NoContent);
+
+        var mockClient = handler.CreateClient();
+        mockClient.BaseAddress = BaseApiUri;
+
+        using var client = new FilesApiClient(mockClient);
+        await client.Delete(FileRelativePath);
+
+        handler.VerifyRequest(
+            HttpMethod.Delete,
+            requestUri);
+    }
+}