diff --git a/csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram.Files.cs b/csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram.Files.cs new file mode 100644 index 0000000..bfddd5c --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram.Files.cs @@ -0,0 +1,70 @@ +namespace Microsoft.Azure.Databricks.Client.Sample; + +internal static partial class SampleProgram +{ + private static async Task TestFilesApi(DatabricksClient client) + { + Console.WriteLine("First specify a volume URI path where the tests will be executed..."); + var basePath = Console.ReadLine(); + + Console.WriteLine($"Using '{basePath}' as base path for the next tests."); + + // Create directory + var directoryPath = basePath + "/" + Guid.NewGuid(); + Console.WriteLine($"Creating empty '{directoryPath}' directory."); + + await client.Files.CreateDirectory(directoryPath); + + // Get newly created directory metadata + Console.WriteLine("Fetching directory metadata."); + var contentHeaders = await client.Files.GetDirectoryMetadata(directoryPath); + foreach (var contentHeader in contentHeaders) + { + Console.WriteLine($"'{contentHeader.Key}': '{contentHeader.Value.First()}'"); + } + + // Populate the directory with a file + Console.WriteLine($"Uploading test file to '{directoryPath}' directory.'"); + var uploadPath = directoryPath + "/" + Guid.NewGuid() + ".txt"; + + using var httpClient = new HttpClient(); + var response = await httpClient.GetAsync("https://norvig.com/big.txt", + HttpCompletionOption.ResponseHeadersRead); + + var fileContents = await response.Content.ReadAsStreamAsync(); + await client.Files.Upload(uploadPath, fileContents, true); + + // Read the file first 100 bytes + Console.WriteLine("Reading test file..."); + using var msDownload = new MemoryStream(); + await client.Files.Download(uploadPath, msDownload, "bytes=0-99"); + msDownload.Position = 0; + var sr = new StreamReader(msDownload); + var content = await sr.ReadToEndAsync(); + Console.WriteLine(content[..99]); + + // Read the file metadata + Console.WriteLine("Getting file metadata..."); + var metadataHeaders = await client.Files.GetFileMetadata(uploadPath); + foreach (var header in metadataHeaders) + { + Console.WriteLine($"'{header.Key}': '{header.Value.First()}'"); + } + + // List directory contents + Console.WriteLine($"Listing test directory contents..."); + + await foreach (var entry in client.Files.ListDirectoryContentsPageable(directoryPath)) + { + Console.WriteLine(entry); + } + + // Delete the file + Console.WriteLine("Deleting test file..."); + await client.Files.Delete(uploadPath); + + // Delete the directory + Console.WriteLine("Deleting test directory..."); + await client.Files.DeleteDirectory(directoryPath); + } +} diff --git a/csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram.cs b/csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram.cs index 8e901d1..6911b5e 100644 --- a/csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram.cs +++ b/csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram.cs @@ -2,12 +2,11 @@ // Licensed under the MIT License. using Azure.Identity; + using Microsoft.Azure.Databricks.Client.Converters; -using System; -using System.Net.Http; + using System.Text.Json; using System.Text.Json.Serialization; -using System.Threading.Tasks; namespace Microsoft.Azure.Databricks.Client.Sample; @@ -67,6 +66,7 @@ public static async Task Main(string[] args) await TestClustersApi(client); await TestGroupsApi(client); await TestDbfsApi(client); + await TestFilesApi(client); await TestJobsApi(client); await TestPermissionsApi(client); await TestWarehouseApi(client); diff --git a/csharp/Microsoft.Azure.Databricks.Client.Test/FilesApiClientTest.cs b/csharp/Microsoft.Azure.Databricks.Client.Test/FilesApiClientTest.cs new file mode 100644 index 0000000..a7026d1 --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client.Test/FilesApiClientTest.cs @@ -0,0 +1,231 @@ +using System.Net; +using System.Text; +using System.Text.Json; + +using Moq.Contrib.HttpClient; + +namespace Microsoft.Azure.Databricks.Client.Test; + +[TestClass] +public class FilesApiClientTest : ApiClientTest +{ + private static readonly Uri DirectoriesApiUri = new(BaseApiUri, "2.0/fs/directories"); + private static readonly Uri FilesApiUri = new(BaseApiUri, "2.0/fs/files"); + + private static readonly string DirectoryRelativePath = "/" + Guid.NewGuid(); + private static readonly string FileRelativePath = DirectoryRelativePath + "/" + Guid.NewGuid() + ".txt"; + + [TestMethod] + public async Task TestCreateDirectory() + { + var requestUri = $"{DirectoriesApiUri}{DirectoryRelativePath}"; + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Put, requestUri) + .ReturnsResponse(HttpStatusCode.NoContent); + + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = BaseApiUri; + + using var client = new FilesApiClient(mockClient); + await client.CreateDirectory(DirectoryRelativePath); + + handler.VerifyRequest( + HttpMethod.Put, + requestUri); + } + + [TestMethod] + public async Task TestDeleteDirectory() + { + var requestUri = $"{DirectoriesApiUri}{DirectoryRelativePath}"; + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Delete, requestUri) + .ReturnsResponse(HttpStatusCode.NoContent); + + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = BaseApiUri; + + using var client = new FilesApiClient(mockClient); + await client.DeleteDirectory(DirectoryRelativePath); + + handler.VerifyRequest( + HttpMethod.Delete, + requestUri); + } + + [TestMethod] + public async Task TestGetDirectoryMetadata() + { + const string expectedHeaderName = "X-Test-Header"; + const string expectedHeaderValue = "Value"; + var requestUri = $"{DirectoriesApiUri}{DirectoryRelativePath}"; + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Head, requestUri) + .ReturnsResponse(HttpStatusCode.OK, message => + { + message.Content.Headers.Add(expectedHeaderName, expectedHeaderValue); + }); + + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = BaseApiUri; + + using var client = new FilesApiClient(mockClient); + var response = await client.GetDirectoryMetadata(DirectoryRelativePath); + + Assert.IsTrue(response.Contains(expectedHeaderName)); + + handler.VerifyRequest( + HttpMethod.Head, + requestUri); + } + + [TestMethod] + public async Task TestListDirectoryContents_ThrowsWithInvalidPageSize() + { + var handler = CreateMockHandler(); + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = BaseApiUri; + using var client = new FilesApiClient(mockClient); + await Assert.ThrowsExceptionAsync(async () => await client.ListDirectoryContents(DirectoryRelativePath, 15000)); + } + + [TestMethod] + public async Task TestListDirectoryContents() + { + var expectedResponse = @" + { + ""contents"": [ + { + ""file_size"": 114864646, + ""is_directory"": true, + ""last_modified"": 646859599, + ""name"": ""test"", + ""path"": ""/Volumes/test-catalog/test-schema/test-volume/test/test.txt"" + } + ], + ""next_page_token"": ""test-token"" + }"; + + var requestUri = $"{DirectoriesApiUri}{DirectoryRelativePath}?"; + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Get, requestUri) + .ReturnsResponse(HttpStatusCode.OK, expectedResponse, "application/json"); + + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = BaseApiUri; + + using var client = new FilesApiClient(mockClient); + var actual = await client.ListDirectoryContents(DirectoryRelativePath); + + var actualJson = JsonSerializer.Serialize(new { contents = actual.Item1, next_page_token = actual.Item2 }, Options); + AssertJsonDeepEquals(expectedResponse, actualJson); + } + + [TestMethod] + public async Task TestUploadFile() + { + var requestUri = $"{FilesApiUri}{FileRelativePath}?overwrite=true"; + + var fileContents = new MemoryStream(Guid.NewGuid().ToByteArray()); + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Put, requestUri) + .ReturnsResponse(HttpStatusCode.NoContent); + + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = BaseApiUri; + + using var client = new FilesApiClient(mockClient); + await client.Upload(FileRelativePath, fileContents, true); + + handler.VerifyRequest( + HttpMethod.Put, + requestUri); + } + + [TestMethod] + public async Task TestGetFileMetadata() + { + const string expectedHeaderName = "X-Test-Header"; + const string expectedHeaderValue = "Value"; + var requestUri = $"{FilesApiUri}{FileRelativePath}"; + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Head, requestUri) + .ReturnsResponse(HttpStatusCode.OK, message => + { + message.Content.Headers.Add(expectedHeaderName, expectedHeaderValue); + }); + + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = BaseApiUri; + + using var client = new FilesApiClient(mockClient); + var response = await client.GetFileMetadata(FileRelativePath); + + Assert.IsTrue(response.Contains(expectedHeaderName)); + + handler.VerifyRequest( + HttpMethod.Head, + requestUri); + } + + [TestMethod] + public async Task TestDownloadFile() + { + var requestUri = $"{FilesApiUri}{FileRelativePath}"; + var expectedResponse = "Hello World!"u8.ToArray(); + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Get, requestUri) + .ReturnsResponse(HttpStatusCode.OK, expectedResponse, "application/octet-stream"); + + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = BaseApiUri; + + using var msDownload = new MemoryStream(); + using var client = new FilesApiClient(mockClient); + await client.Download(FileRelativePath, msDownload); + msDownload.Position = 0; + var sr = new StreamReader(msDownload, Encoding.UTF8); + var content = await sr.ReadToEndAsync(); + + Assert.AreEqual(Encoding.UTF8.GetString(expectedResponse), content); + + handler.VerifyRequest( + HttpMethod.Get, + requestUri); + } + + [TestMethod] + public async Task TestDeleteFile() + { + var requestUri = $"{FilesApiUri}{FileRelativePath}"; + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Delete, requestUri) + .ReturnsResponse(HttpStatusCode.NoContent); + + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = BaseApiUri; + + using var client = new FilesApiClient(mockClient); + await client.Delete(FileRelativePath); + + handler.VerifyRequest( + HttpMethod.Delete, + requestUri); + } +} diff --git a/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs b/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs index f1ed964..3743fe5 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs +++ b/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs @@ -2,8 +2,10 @@ // Licensed under the MIT License. using Microsoft.Azure.Databricks.Client.Converters; + using System; using System.Net.Http; +using System.Net.Http.Headers; using System.Net.Mime; using System.Text; using System.Text.Json; @@ -52,35 +54,40 @@ protected static ClientApiException CreateApiException(HttpResponseMessage respo private static async Task SendRequest(HttpClient httpClient, HttpMethod method, string requestUri, TBody body, CancellationToken cancellationToken = default) { - var request = new HttpRequestMessage(method, requestUri) - { - Content = body == null ? null : new StringContent(JsonSerializer.Serialize(body, Options), Encoding.UTF8, MediaTypeNames.Application.Json) - }; - - using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + using var response = await FetchResponse(httpClient, method, requestUri, body, cancellationToken).ConfigureAwait(false); + await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); - if (!response.IsSuccessStatusCode) - { - throw CreateApiException(response); - } - - using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); return await JsonSerializer.DeserializeAsync(responseStream, Options, cancellationToken).ConfigureAwait(false); } private static async Task SendRequest(HttpClient httpClient, HttpMethod method, string requestUri, TBody body, CancellationToken cancellationToken = default) + { + await FetchResponse(httpClient, method, requestUri, body, cancellationToken).ConfigureAwait(false); + } + + private static async Task SendHeadRequest(HttpClient httpClient, HttpMethod method, + string requestUri, TBody body = default, CancellationToken cancellationToken = default) + { + using var response = await FetchResponse(httpClient, method, requestUri, body, cancellationToken).ConfigureAwait(false); + return response.Content.Headers; + } + + private static async Task FetchResponse(HttpClient httpClient, HttpMethod method, + string requestUri, TBody body, CancellationToken cancellationToken = default) { var request = new HttpRequestMessage(method, requestUri) { Content = body == null ? null : new StringContent(JsonSerializer.Serialize(body, Options), Encoding.UTF8, MediaTypeNames.Application.Json) }; - using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); if (!response.IsSuccessStatusCode) { throw CreateApiException(response); } + + return response; } protected static async Task HttpGet(HttpClient httpClient, string requestUri, CancellationToken cancellationToken = default) @@ -125,6 +132,17 @@ protected static async Task HttpDelete(HttpClient httpClient, string requestUri, await SendRequest(httpClient, HttpMethod.Delete, requestUri, null, cancellationToken).ConfigureAwait(false); } + protected static async Task HttpHead(HttpClient httpClient, string requestUri, TBody body = default, + CancellationToken cancellationToken = default) + { + return await SendHeadRequest(httpClient, HttpMethod.Head, requestUri, body, cancellationToken).ConfigureAwait(false); + } + + protected static async Task HttpHead(HttpClient httpClient, string requestUri, CancellationToken cancellationToken = default) + { + return await SendHeadRequest(httpClient, HttpMethod.Head, requestUri, null, cancellationToken).ConfigureAwait(false); + } + protected virtual void Dispose(bool disposing) { if (disposing) @@ -138,4 +156,4 @@ public void Dispose() Dispose(true); GC.SuppressFinalize(this); } -} \ No newline at end of file +} diff --git a/csharp/Microsoft.Azure.Databricks.Client/DatabricksClient.cs b/csharp/Microsoft.Azure.Databricks.Client/DatabricksClient.cs index 016a9ed..5b38d5c 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/DatabricksClient.cs +++ b/csharp/Microsoft.Azure.Databricks.Client/DatabricksClient.cs @@ -22,6 +22,7 @@ private DatabricksClient(HttpClient httpClient) this.Clusters = new ClustersApiClient(httpClient); this.Jobs = new JobsApiClient(httpClient); this.Dbfs = new DbfsApiClient(httpClient); + this.Files = new FilesApiClient(httpClient); this.Secrets = new SecretsApiClient(httpClient); this.Groups = new GroupsApiClient(httpClient); this.Libraries = new LibrariesApiClient(httpClient); @@ -193,6 +194,8 @@ public static DatabricksClient CreateClient(string baseUrl, TokenCredential cred public virtual IDbfsApi Dbfs { get; } + public virtual IFilesApi Files { get; } + public virtual ISecretsApi Secrets { get; } public virtual IGroupsApi Groups { get; } @@ -226,6 +229,7 @@ public void Dispose() Clusters?.Dispose(); Jobs?.Dispose(); Dbfs?.Dispose(); + Files?.Dispose(); Secrets?.Dispose(); Groups?.Dispose(); Libraries?.Dispose(); diff --git a/csharp/Microsoft.Azure.Databricks.Client/FilesApiClient.cs b/csharp/Microsoft.Azure.Databricks.Client/FilesApiClient.cs new file mode 100644 index 0000000..bb89645 --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client/FilesApiClient.cs @@ -0,0 +1,154 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Azure.Databricks.Client.Models; + +namespace Microsoft.Azure.Databricks.Client; + +public sealed class FilesApiClient : ApiClient, IFilesApi +{ + private readonly string _apiBaseUrl; + + public FilesApiClient(HttpClient httpClient) : base(httpClient) + { + _apiBaseUrl = $"{ApiVersion}/fs"; + } + + public async Task<(IEnumerable, string)> ListDirectoryContents(string directoryPath, long? pageSize = default, + string pageToken = default, CancellationToken cancellationToken = default) + { + if (pageSize < 0 || pageSize > 1000) + { + throw new ArgumentOutOfRangeException(nameof(pageSize), "Page size must be between 0 and 1000."); + } + + StringBuilder requestUriSb = new($"{_apiBaseUrl}/directories{directoryPath}?"); + if (pageSize != null) + { + requestUriSb.Append($"&page_size={pageSize}"); + } + + if (!string.IsNullOrEmpty(pageToken)) + { + requestUriSb.Append($"&page_token={pageToken}"); + } + + var requestUri = requestUriSb.ToString(); + var response = await HttpGet(this.HttpClient, requestUri, cancellationToken).ConfigureAwait(false); + + response.TryGetPropertyValue("contents", out var contentNode); + response.TryGetPropertyValue("next_page_token", out var nextPageTokenNode); + + var entries = contentNode?.Deserialize>(Options) ?? Enumerable.Empty(); + var nextPageToken = nextPageTokenNode?.Deserialize(Options) ?? string.Empty; + + return (entries, nextPageToken); + } + + public global::Azure.AsyncPageable ListDirectoryContentsPageable(string directoryPath, long? pageSize = null, CancellationToken cancellationToken = default) + { + return new AsyncPageable(async (pageToken) => + { + var (entries, nextPageToken) = await ListDirectoryContents( + directoryPath, + pageSize, + pageToken, + cancellationToken).ConfigureAwait(false); + + return (entries.ToList(), !string.IsNullOrEmpty(nextPageToken), nextPageToken); + }); + } + + public async Task GetDirectoryMetadata(string directoryPath, CancellationToken cancellationToken = default) + { + return await HttpHead(this.HttpClient, $"{_apiBaseUrl}/directories{directoryPath}", cancellationToken); + } + + public async Task CreateDirectory(string directoryPath, CancellationToken cancellationToken = default) + { + await HttpPut(this.HttpClient, $"{_apiBaseUrl}/directories{directoryPath}", null, cancellationToken); + } + + public async Task DeleteDirectory(string directoryPath, CancellationToken cancellationToken = default) + { + await HttpDelete(this.HttpClient, $"{_apiBaseUrl}/directories{directoryPath}", cancellationToken); + } + + public async Task Download(string filePath, Stream stream, string range = default, string ifUnmodifiedSince = default, CancellationToken cancellationToken = default) + { + using var request = new HttpRequestMessage(HttpMethod.Get, $"{_apiBaseUrl}/files{filePath}"); + if (!string.IsNullOrEmpty(range)) + { + request.Headers.Add("Range", range); + } + + if (!string.IsNullOrEmpty(ifUnmodifiedSince)) + { + request.Headers.Add("If-Unmodified-Since", ifUnmodifiedSince); + } + + var response = await this.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + throw CreateApiException(response); + } + + using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + await contentStream.CopyToAsync(stream, cancellationToken).ConfigureAwait(false); + } + + public async Task GetFileMetadata(string filePath, string range = default, string ifUnmodifiedSince = default, CancellationToken cancellationToken = default) + { + using var request = new HttpRequestMessage(HttpMethod.Head, $"{_apiBaseUrl}/files{filePath}"); + if (!string.IsNullOrEmpty(range)) + { + request.Headers.Add("Range", range); + } + + if (!string.IsNullOrEmpty(ifUnmodifiedSince)) + { + request.Headers.Add("If-Unmodified-Since", ifUnmodifiedSince); + } + + var response = await this.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + throw CreateApiException(response); + } + + return response.Content.Headers; + } + + public async Task Upload(string filePath, Stream stream, bool? overwrite = default, + CancellationToken cancellationToken = default) + { + var requestUri = overwrite == null ? $"{_apiBaseUrl}/files{filePath}" : $"{_apiBaseUrl}/files{filePath}?overwrite={overwrite.ToString().ToLowerInvariant()}"; + + using var content = new StreamContent(stream); + content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + var response = await this.HttpClient.PutAsync(requestUri, content, cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + throw CreateApiException(response); + } + } + + public async Task Delete(string filePath, CancellationToken cancellationToken = default) + { + await HttpDelete(this.HttpClient, $"{_apiBaseUrl}/files{filePath}", cancellationToken).ConfigureAwait(false); + } + + +} diff --git a/csharp/Microsoft.Azure.Databricks.Client/IFilesApi.cs b/csharp/Microsoft.Azure.Databricks.Client/IFilesApi.cs new file mode 100644 index 0000000..b8d8c1d --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client/IFilesApi.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Azure.Databricks.Client.Models; + +namespace Microsoft.Azure.Databricks.Client; + +public interface IFilesApi : IDisposable +{ + /// + /// Returns the contents of a directory. + /// If there is no directory at the specified path, the API returns an HTTP 404 error. + /// + /// The absolute path of a directory. + /// + /// The maximum number of directory entries to return. The response may contain fewer entries. If the response contains a next_page_token, there may be more entries, even if fewer than page_size entries are in the response. + /// We recommend not to set this value unless you are intentionally listing less than the complete directory contents. + /// If unspecified, at most 1000 directory entries will be returned. The maximum value is 1000. Values above 1000 will be coerced to 1000. + /// + /// + /// An opaque page token which was the next_page_token in the response of the previous request to list the contents of this directory. + /// Provide this token to retrieve the next page of directory entries. + /// When providing a page_token, all other parameters provided to the request must match the previous request. + /// To list all the entries in a directory, it is necessary to continue requesting pages of entries until the response contains no next_page_token. + /// Note that the number of entries returned must not be used to determine when the listing is complete. + /// + Task<(IEnumerable, string)> ListDirectoryContents(string directoryPath, long? pageSize = default, string pageToken = default, CancellationToken cancellationToken = default); + + /// + /// Returns the contents of a directory. + /// If there is no directory at the specified path, the API returns an HTTP 404 error. + /// + /// The absolute path of a directory. + /// + /// The maximum number of directory entries to return. The response may contain fewer entries. If the response contains a next_page_token, there may be more entries, even if fewer than page_size entries are in the response. + /// We recommend not to set this value unless you are intentionally listing less than the complete directory contents. + /// If unspecified, at most 1000 directory entries will be returned. The maximum value is 1000. Values above 1000 will be coerced to 1000. + /// + global::Azure.AsyncPageable ListDirectoryContentsPageable(string directoryPath, long? pageSize = default, CancellationToken cancellationToken = default); + + /// + /// Get the metadata of a directory. The response HTTP headers contain the metadata. There is no response body. + /// This method is useful to check if a directory exists and the caller has access to it. + /// If you wish to ensure the directory exists, you can instead use PUT, which will create the directory if it does not exist, and is idempotent (it will succeed if the directory already exists). + /// + /// The absolute path of a directory. + Task GetDirectoryMetadata(string directoryPath, CancellationToken cancellationToken = default); + + /// + /// Creates an empty directory. + /// If necessary, also creates any parent directories of the new, empty directory (like the shell command mkdir -p). + /// If called on an existing directory, returns a success response; this method is idempotent (it will succeed if the directory already exists). + /// + /// The absolute path of a directory. + Task CreateDirectory(string directoryPath, CancellationToken cancellationToken = default); + + /// + /// Deletes an empty directory. + /// To delete a non-empty directory, first delete all of its contents. + /// This can be done by listing the directory contents and deleting each file and subdirectory recursively. + /// + /// The absolute path of a directory. + Task DeleteDirectory(string directoryPath, CancellationToken cancellationToken = default); + + /// + /// Downloads a file. + /// The file contents are the response body. + /// This is a standard HTTP file download, not a JSON RPC. It supports the `Range` and `If-Unmodified-Since` HTTP headers. + /// + /// The file contents will be written asynchronously to the stream passed as argument. + /// + /// The absolute path of the file. + /// The data stream to write to. + /// The range of bytes to retrieve. The range is inclusive and zero-based, see RFC 9110 for further details. + /// Download the file only if it has not been modified since the specified timestamp. If it has, a 412 Precondition Failed error will be returned. See RFC 9110 for further details. + Task Download(string filePath, Stream stream, string range = default, string ifUnmodifiedSince = default, CancellationToken cancellationToken = default); + + /// + /// Get the metadata of a file. + /// The response HTTP headers contain the metadata. + /// There is no response body. + /// + /// The absolute path of the file. + /// The range of bytes to retrieve. The range is inclusive and zero-based, see RFC 9110 for further details. + /// Download the file only if it has not been modified since the specified timestamp. If it has, a 412 Precondition Failed error will be returned. See RFC 9110 for further details. + Task GetFileMetadata(string filePath, string range = default, string ifUnmodifiedSince = default, CancellationToken cancellationToken = default); + + /// + /// Uploads a file up to 5 GiB. + /// The file contents should be sent as the request body as raw bytes (an octet stream); do not encode or otherwise modify the bytes before sending. + /// The contents of the resulting file will be exactly the bytes sent in the request body. + /// If the request is successful, there is no response body. + /// + /// The absolute path of the file. + /// The data stream to read from. + /// If true or unspecified, an existing file will be overwritten. If false, an error will be returned if the path points to an existing file. + Task Upload(string filePath, Stream stream, bool? overwrite = default, CancellationToken cancellationToken = default); + + /// + /// Deletes a file. + /// If the request is successful, there is no response body. + /// + /// The absolute path of the file. + Task Delete(string filePath, CancellationToken cancellationToken = default); +} diff --git a/csharp/Microsoft.Azure.Databricks.Client/Models/DirectoryEntry.cs b/csharp/Microsoft.Azure.Databricks.Client/Models/DirectoryEntry.cs new file mode 100644 index 0000000..db473ef --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client/Models/DirectoryEntry.cs @@ -0,0 +1,37 @@ +using System; +using System.Text.Json.Serialization; + +namespace Microsoft.Azure.Databricks.Client.Models; + +public record DirectoryEntry +{ + /// + /// The absolute path of the file or directory. + /// + [JsonPropertyName("path")] + public string Path { get; set; } + + /// + /// True if the path is a directory. + /// + [JsonPropertyName("is_directory")] + public bool IsDirectory { get; set; } + + /// + /// The length of the file in bytes. This field is omitted for directories. + /// + [JsonPropertyName("file_size")] + public long FileSize { get; set; } + + /// + /// Last modification time of given file in milliseconds since unix epoch. + /// + [JsonPropertyName("last_modified")] + public DateTimeOffset? LastModified { get; set; } + + /// + /// The name of the file or directory. This is the last component of the path. + /// + [JsonPropertyName("name")] + public string Name { get; set; } +}