diff --git a/src/ApiService/ApiService/Functions/Jobs.cs b/src/ApiService/ApiService/Functions/Jobs.cs index 3f8746df1f..f4b5f04fa9 100644 --- a/src/ApiService/ApiService/Functions/Jobs.cs +++ b/src/ApiService/ApiService/Functions/Jobs.cs @@ -136,13 +136,11 @@ private async Task Get(HttpRequestData req) { static JobTaskInfo TaskToJobTaskInfo(Task t) => new(t.TaskId, t.Config.Task.Type, t.State); var tasks = _context.TaskOperations.SearchStates(jobId); - if (search.WithTasks ?? false) { - var ts = await tasks.ToListAsync(); - return await RequestHandling.Ok(req, JobResponse.ForJob(job, ts)); - } else { - var taskInfo = await tasks.Select(TaskToJobTaskInfo).ToListAsync(); - return await RequestHandling.Ok(req, JobResponse.ForJob(job, taskInfo)); - } + + IAsyncEnumerable taskInfo = search.WithTasks ?? false ? tasks : tasks.Select(TaskToJobTaskInfo); + + var crashReported = await _context.JobCrashReportedOperations.CrashReported(jobId); + return await RequestHandling.Ok(req, JobResponse.ForJob(job, taskInfo.ToEnumerable(), crashReported)); } var jobs = await _context.JobOperations.SearchState(states: search.State ?? Enumerable.Empty()).ToListAsync(); diff --git a/src/ApiService/ApiService/Functions/QueueJobResult.cs b/src/ApiService/ApiService/Functions/QueueJobResult.cs index 31b39802d6..3f863cb2f2 100644 --- a/src/ApiService/ApiService/Functions/QueueJobResult.cs +++ b/src/ApiService/ApiService/Functions/QueueJobResult.cs @@ -49,6 +49,10 @@ public async Async.Task Run([QueueTrigger("job-result", Connection = "AzureWebJo var jobResultType = data.Type; _log.LogInformation($"job result data type: {jobResultType}"); + if (jobResultType == "CrashReported") { + var _result = await _context.JobCrashReportedOperations.ReportCrash(job.JobId, jr.TaskId); + } + Dictionary value; if (jr.Value.Count > 0) { value = jr.Value; diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index 4dd4000283..a5e2db1e26 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -968,9 +968,12 @@ public record Job( StoredUserInfo? UserInfo, string? Error = null, DateTimeOffset? EndTime = null -) : StatefulEntityBase(State) { +) : StatefulEntityBase(State); -} +public record JobCrashReported( + [PartitionKey] Guid JobId, + [RowKey] Guid TaskId +) : EntityBase; // This is like UserInfo but lacks the UPN: public record StoredUserInfo(Guid? ApplicationId, Guid? ObjectId); @@ -1178,3 +1181,11 @@ string ReproCmd public interface ITruncatable { public T Truncate(int maxLength); } + +public record AdoNotificationEntry( + [PartitionKey] Guid JobId, + [RowKey] int Id, + string Title +) : EntityBase { + +} diff --git a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs index f3a4c32965..b75fa37af0 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs @@ -171,6 +171,7 @@ public record JobSearch( List? State = null, List? TaskState = null, bool? WithTasks = null + ) : BaseRequest; public record NodeAddSshKeyPost( diff --git a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs index c1067305ad..2760837a66 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs @@ -92,6 +92,7 @@ public record ContainerInfo( Uri SasUrl ) : BaseResponse(); + public record JobResponse( Guid JobId, JobState State, @@ -101,10 +102,11 @@ public record JobResponse( IEnumerable? TaskInfo, StoredUserInfo? UserInfo, [property: JsonPropertyName("Timestamp")] // must retain capital T for backcompat - DateTimeOffset? Timestamp + DateTimeOffset? Timestamp, + bool CrashReported // not including UserInfo from Job model ) : BaseResponse() { - public static JobResponse ForJob(Job j, IEnumerable? taskInfo) + public static JobResponse ForJob(Job j, IEnumerable? taskInfo, bool crashReported = false) => new( JobId: j.JobId, State: j.State, @@ -113,7 +115,8 @@ public static JobResponse ForJob(Job j, IEnumerable? taskInfo) EndTime: j.EndTime, TaskInfo: taskInfo, UserInfo: j.UserInfo, - Timestamp: j.Timestamp + Timestamp: j.Timestamp, + CrashReported: crashReported ); public DateTimeOffset? StartTime => EndTime is DateTimeOffset endTime ? endTime.Subtract(TimeSpan.FromHours(Config.Duration)) : null; } diff --git a/src/ApiService/ApiService/Program.cs b/src/ApiService/ApiService/Program.cs index f26463883b..8e491b83e2 100644 --- a/src/ApiService/ApiService/Program.cs +++ b/src/ApiService/ApiService/Program.cs @@ -24,11 +24,11 @@ namespace Microsoft.OneFuzz.Service; public class Program { /// - /// + /// /// public class LoggingMiddleware : IFunctionsWorkerMiddleware { /// - /// + /// /// /// /// @@ -198,6 +198,8 @@ public static async Async.Task Main() { .AddScoped() .AddScoped() .AddScoped() + .AddScoped() + .AddScoped() .AddSingleton(new GraphServiceClient(new DefaultAzureCredential())) .AddSingleton() .AddSingleton() diff --git a/src/ApiService/ApiService/onefuzzlib/AdoNotificationEntryOperation.cs b/src/ApiService/ApiService/onefuzzlib/AdoNotificationEntryOperation.cs new file mode 100644 index 0000000000..9a21428b23 --- /dev/null +++ b/src/ApiService/ApiService/onefuzzlib/AdoNotificationEntryOperation.cs @@ -0,0 +1,26 @@ +using ApiService.OneFuzzLib.Orm; +using Microsoft.Extensions.Logging; +namespace Microsoft.OneFuzz.Service; + +public interface IAdoNotificationEntryOperations : IOrm { + + public IAsyncEnumerable GetByJobId(Guid jobId); + + public Async.Task WasNotfied(Guid jobId); + +} +public class AdoNotificationEntryOperations : Orm, IAdoNotificationEntryOperations { + + public AdoNotificationEntryOperations(ILogger log, IOnefuzzContext context) + : base(log, context) { + + } + + public IAsyncEnumerable GetByJobId(Guid jobId) { + return QueryAsync(filter: Query.PartitionKey(jobId.ToString())); + } + + public async Async.Task WasNotfied(Guid jobId) { + return await QueryAsync(filter: Query.PartitionKey(jobId.ToString()), maxPerPage: 1).AnyAsync(); + } +} diff --git a/src/ApiService/ApiService/onefuzzlib/JobCrashReported.cs b/src/ApiService/ApiService/onefuzzlib/JobCrashReported.cs new file mode 100644 index 0000000000..f5d3227d30 --- /dev/null +++ b/src/ApiService/ApiService/onefuzzlib/JobCrashReported.cs @@ -0,0 +1,28 @@ +using System.Threading.Tasks; +using ApiService.OneFuzzLib.Orm; +using Microsoft.Extensions.Logging; +namespace Microsoft.OneFuzz.Service; + +public interface IJobCrashReportedOperations : IOrm { + public Task CrashReported(Guid jobId); + public Task ReportCrash(Guid jobId, Guid taskId); +} + +public class JobCrashReportedOperations : Orm, IJobCrashReportedOperations { + public JobCrashReportedOperations(ILogger logTracer, IOnefuzzContext context) : base(logTracer, context) { + } + + public async Task CrashReported(Guid jobId) { + return await QueryAsync(Query.RowKey(jobId.ToString())).AnyAsync(); + } + + public async Task ReportCrash(Guid jobId, Guid taskId) { + + var result = await Update(new JobCrashReported(jobId, taskId)); + if (!result.IsOk) { + return OneFuzzResultVoid.Error(ErrorCode.UNABLE_TO_UPDATE, "Failed to update job crash reported"); + } + + return OneFuzzResultVoid.Ok; + } +} diff --git a/src/ApiService/ApiService/onefuzzlib/OnefuzzContext.cs b/src/ApiService/ApiService/onefuzzlib/OnefuzzContext.cs index 03c6322663..5001812287 100644 --- a/src/ApiService/ApiService/onefuzzlib/OnefuzzContext.cs +++ b/src/ApiService/ApiService/onefuzzlib/OnefuzzContext.cs @@ -49,6 +49,8 @@ public interface IOnefuzzContext { ITeams Teams { get; } IGithubIssues GithubIssues { get; } IAdo Ado { get; } + IAdoNotificationEntryOperations AdoNotificationEntryOperations { get; } + IJobCrashReportedOperations JobCrashReportedOperations =>; IFeatureManagerSnapshot FeatureManagerSnapshot { get; } IConfigurationRefresher ConfigurationRefresher { get; } @@ -101,8 +103,11 @@ public OnefuzzContext(IServiceProvider serviceProvider) { public ITeams Teams => _serviceProvider.GetRequiredService(); public IGithubIssues GithubIssues => _serviceProvider.GetRequiredService(); public IAdo Ado => _serviceProvider.GetRequiredService(); + public IJobCrashReportedOperations JobCrashReportedOperations => _serviceProvider.GetRequiredService(); public IFeatureManagerSnapshot FeatureManagerSnapshot => _serviceProvider.GetRequiredService(); public IConfigurationRefresher ConfigurationRefresher => _serviceProvider.GetRequiredService().Refreshers.First(); + + public IAdoNotificationEntryOperations AdoNotificationEntryOperations => _serviceProvider.GetRequiredService(); } diff --git a/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs b/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs index b3633f12d4..d691fdc12d 100644 --- a/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs +++ b/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs @@ -21,6 +21,14 @@ public class Ado : NotificationsBase, IAdo { private const string TITLE_FIELD = "System.Title"; private static List DEFAULT_REGRESSION_IGNORE_STATES = new() { "New", "Commited", "Active" }; + public enum AdoNotificationPublishignState { + Added, + Updated, + Skipped, + } + + public record ProcessResult(AdoNotificationPublishignState State, WorkItem WorkItem); + public Ado(ILogger logTracer, IOnefuzzContext context) : base(logTracer, context) { } @@ -319,7 +327,14 @@ private static async Async.Task ProcessNotification(IOnefuzzContext context, Con var renderedConfig = RenderAdoTemplate(logTracer, renderer, config, instanceUrl); var ado = new AdoConnector(renderedConfig, project!, client, instanceUrl, logTracer, await GetValidFields(client, project)); - await ado.Process(notificationInfo, isRegression); + await foreach (var processState in ado.Process(notificationInfo, isRegression)) { + if (processState.State == AdoNotificationPublishignState.Added) { + if (processState.WorkItem.Id == null) { + continue; + } + _ = await context.AdoNotificationEntryOperations.Update(new AdoNotificationEntry(report.JobId, (int)processState.WorkItem.Id, (string)processState.WorkItem.Fields[TITLE_FIELD])); + } + } } public static RenderedAdoTemplate RenderAdoTemplate(ILogger logTracer, Renderer renderer, AdoTemplate original, Uri instanceUrl) { @@ -529,28 +544,25 @@ public async Async.Task UpdateExisting(WorkItem item, IList<(string, strin // the below was causing on_duplicate not to work // var systemState = JsonSerializer.Serialize(item.Fields["System.State"]); var systemState = (string)item.Fields["System.State"]; - var stateUpdated = false; if (_config.OnDuplicate.SetState.TryGetValue(systemState, out var v)) { document.Add(new JsonPatchOperation() { Operation = VisualStudio.Services.WebApi.Patch.Operation.Replace, Path = "/fields/System.State", Value = v }); - - stateUpdated = true; } if (document.Any()) { _ = await _client.UpdateWorkItemAsync(document, _project, (int)item.Id!); var adoEventType = "AdoUpdate"; _logTracer.LogEvent(adoEventType); + return true; } else { var adoEventType = "AdoNoUpdate"; _logTracer.LogEvent(adoEventType); + return false; } - - return stateUpdated; } private bool MatchesUnlessCase(WorkItem workItem) => @@ -607,7 +619,8 @@ private async Async.Task CreateNew() { return (taskType, document); } - public async Async.Task Process(IList<(string, string)> notificationInfo, bool isRegression) { + + public async IAsyncEnumerable Process(IList<(string, string)> notificationInfo, bool isRegression) { var updated = false; WorkItem? oldestWorkItem = null; await foreach (var workItem in ExistingWorkItems(notificationInfo)) { @@ -617,7 +630,9 @@ public async Async.Task Process(IList<(string, string)> notificationInfo, bool i _logTracer.AddTags(new List<(string, string)> { ("MatchingWorkItemIds", $"{workItem.Id}") }); _logTracer.LogInformation("Found matching work item"); } + if (IsADODuplicateWorkItem(workItem, _config.AdoDuplicateFields)) { + yield return new ProcessResult(AdoNotificationPublishignState.Skipped, workItem); continue; } @@ -634,11 +649,12 @@ public async Async.Task Process(IList<(string, string)> notificationInfo, bool i } _ = await UpdateExisting(workItem, notificationInfo); + yield return new ProcessResult(AdoNotificationPublishignState.Updated, workItem); updated = true; } if (updated || isRegression) { - return; + yield break; } if (oldestWorkItem != null) { @@ -656,6 +672,7 @@ public async Async.Task Process(IList<(string, string)> notificationInfo, bool i _project, (int)oldestWorkItem.Id!); } + yield return new ProcessResult(AdoNotificationPublishignState.Updated, oldestWorkItem); } else { // We never saw a work item like this before, it must be new var entry = await CreateNew(); @@ -663,6 +680,7 @@ public async Async.Task Process(IList<(string, string)> notificationInfo, bool i _logTracer.AddTags(notificationInfo); _logTracer.AddTag("WorkItemId", entry.Id.HasValue ? entry.Id.Value.ToString() : ""); _logTracer.LogEvent(adoEventType); + yield return new ProcessResult(AdoNotificationPublishignState.Added, entry); } } diff --git a/src/ApiService/FunctionalTests/1f-api/Jobs.cs b/src/ApiService/FunctionalTests/1f-api/Jobs.cs index 7e14f0ec8e..873be12333 100644 --- a/src/ApiService/FunctionalTests/1f-api/Jobs.cs +++ b/src/ApiService/FunctionalTests/1f-api/Jobs.cs @@ -50,6 +50,7 @@ public async Task, Error>> Get(Guid? jobId = null, List< .AddIfNotNullV("task_state", taskState) .AddIfNotNullV("with_tasks", withTasks); + var r = await Get(n); return IEnumerableResult(r); } diff --git a/src/ApiService/IntegrationTests/Fakes/TestAdoNotificationEntryOperations.cs b/src/ApiService/IntegrationTests/Fakes/TestAdoNotificationEntryOperations.cs new file mode 100644 index 0000000000..ec8f8dc16d --- /dev/null +++ b/src/ApiService/IntegrationTests/Fakes/TestAdoNotificationEntryOperations.cs @@ -0,0 +1,8 @@ +using Microsoft.Extensions.Logging; +using Microsoft.OneFuzz.Service; +namespace IntegrationTests.Fakes; + +public sealed class TestAdoNotificationEntryOperations : AdoNotificationEntryOperations { + public TestAdoNotificationEntryOperations(ILogger log, IOnefuzzContext context) + : base(log, context) { } +} diff --git a/src/ApiService/IntegrationTests/Fakes/TestContext.cs b/src/ApiService/IntegrationTests/Fakes/TestContext.cs index 66d121e746..5c1136263d 100644 --- a/src/ApiService/IntegrationTests/Fakes/TestContext.cs +++ b/src/ApiService/IntegrationTests/Fakes/TestContext.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Net.Http; using Microsoft.Extensions.Caching.Memory; @@ -42,6 +43,7 @@ public TestContext(IHttpClientFactory httpClientFactory, OneFuzzLoggerProvider p ReproOperations = new ReproOperations(provider.CreateLogger(), this); Reports = new Reports(provider.CreateLogger(), Containers); NotificationOperations = new NotificationOperations(provider.CreateLogger(), this); + AdoNotificationEntryOperations = new TestAdoNotificationEntryOperations(provider.CreateLogger(), this); FeatureManagerSnapshot = new TestFeatureManagerSnapshot(); WebhookOperations = new TestWebhookOperations(httpClientFactory, provider.CreateLogger(), this); @@ -65,9 +67,28 @@ public Async.Task InsertAll(params EntityBase[] objs) InstanceConfig ic => ConfigOperations.Insert(ic), Notification n => NotificationOperations.Insert(n), Webhook w => WebhookOperations.Insert(w), + AdoNotificationEntry ado => AdoNotificationEntryOperations.Insert(ado), _ => throw new NotSupportedException($"You will need to add an TestContext.InsertAll case for {x.GetType()} entities"), })); + public Async.Task InsertAll(IEnumerable objs) + => Async.Task.WhenAll( + objs.Select(x => x switch { + Task t => TaskOperations.Insert(t), + Node n => NodeOperations.Insert(n), + Pool p => PoolOperations.Insert(p), + Job j => JobOperations.Insert(j), + JobResult jr => JobResultOperations.Insert(jr), + Repro r => ReproOperations.Insert(r), + Scaleset ss => ScalesetOperations.Insert(ss), + NodeTasks nt => NodeTasksOperations.Insert(nt), + InstanceConfig ic => ConfigOperations.Insert(ic), + Notification n => NotificationOperations.Insert(n), + Webhook w => WebhookOperations.Insert(w), + AdoNotificationEntry ado => AdoNotificationEntryOperations.Insert(ado), + _ => throw new NotSupportedException($"You will need to add an TestContext.InsertAll case for {x.GetType()} entities"), + })); + // Implementations: public IMemoryCache Cache { get; } @@ -109,6 +130,8 @@ public Async.Task InsertAll(params EntityBase[] objs) public IWebhookMessageLogOperations WebhookMessageLogOperations { get; } + public IAdoNotificationEntryOperations AdoNotificationEntryOperations { get; } + // -- Remainder not implemented -- public IConfig Config => throw new System.NotImplementedException(); @@ -143,4 +166,6 @@ public Async.Task InsertAll(params EntityBase[] objs) public IAdo Ado => throw new NotImplementedException(); public IConfigurationRefresher ConfigurationRefresher => throw new NotImplementedException(); + + } diff --git a/src/ApiService/IntegrationTests/JobsTests.cs b/src/ApiService/IntegrationTests/JobsTests.cs index 28dbe8457f..563e92cbdd 100644 --- a/src/ApiService/IntegrationTests/JobsTests.cs +++ b/src/ApiService/IntegrationTests/JobsTests.cs @@ -226,4 +226,28 @@ await Context.InsertAll( Assert.Equal(task.Config.Task.Type, returnedTasks[0].Type); } + + [Fact] + public async Async.Task Get_CanFindSpecificJobWithBugs() { + var taskConfig = new TaskConfig(_jobId, new List(), new TaskDetails(TaskType.Coverage, 60)); + + var random = new Random(); + var bugs = Enumerable.Range(1, 100).Select(i => random.Next(0, 100)).Distinct().Select(i => new AdoNotificationEntry(_jobId, i, $"test_i")).ToList(); + await Context.InsertAll(bugs); + await Context.InsertAll( + new Job(_jobId, JobState.Stopped, _config, null), + new Task(_jobId, Guid.NewGuid(), TaskState.Running, Os.Windows, taskConfig) + ); + + var func = new Jobs(Context, LoggerProvider.CreateLogger()); + + var ctx = new TestFunctionContext(); + var result = await func.Run(TestHttpRequestData.FromJson("GET", new JobSearch(JobId: _jobId)), ctx); + Assert.Equal(HttpStatusCode.OK, result.StatusCode); + + var response = BodyAs(result); + Assert.Equal(_jobId, response.JobId); + Assert.NotNull(response.TaskInfo); + Assert.True(response.CrashReported); + } } diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index 746c528c1c..8adae9c9ee 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -761,6 +761,7 @@ class Job(BaseModel): task_info: Optional[List[Union[Task, JobTaskInfo]]] user_info: Optional[UserInfo] start_time: Optional[datetime] = None + crash_reported: Optional[bool] = None class NetworkConfig(BaseModel):