From 88b2cb5c58638a310e446fece7d5fa2432c3d10c Mon Sep 17 00:00:00 2001
From: Cheick Keita <chkeita@microsoft.com>
Date: Mon, 30 Oct 2023 17:39:50 -0700
Subject: [PATCH 1/3] Add a field to the job summary to indicate if at least
one bug was created
---
src/ApiService/ApiService/Functions/Jobs.cs | 12 ++++----
.../ApiService/Functions/QueueJobResult.cs | 4 +++
.../ApiService/OneFuzzTypes/Model.cs | 7 +++--
.../ApiService/OneFuzzTypes/Responses.cs | 9 ++++--
src/ApiService/ApiService/Program.cs | 5 ++--
.../ApiService/onefuzzlib/JobCrashReported.cs | 28 +++++++++++++++++++
.../ApiService/onefuzzlib/OnefuzzContext.cs | 3 +-
.../onefuzzlib/notifications/Ado.cs | 2 +-
.../IntegrationTests/Fakes/TestContext.cs | 25 +++++++++++++++++
src/ApiService/IntegrationTests/JobsTests.cs | 21 ++++++++++++++
src/pytypes/onefuzztypes/models.py | 1 +
11 files changed, 101 insertions(+), 16 deletions(-)
create mode 100644 src/ApiService/ApiService/onefuzzlib/JobCrashReported.cs
diff --git a/src/ApiService/ApiService/Functions/Jobs.cs b/src/ApiService/ApiService/Functions/Jobs.cs
index 3f8746df1f..f4b5f04fa9 100644
--- a/src/ApiService/ApiService/Functions/Jobs.cs
+++ b/src/ApiService/ApiService/Functions/Jobs.cs
@@ -136,13 +136,11 @@ private async Task<HttpResponseData> Get(HttpRequestData req) {
static JobTaskInfo TaskToJobTaskInfo(Task t) => new(t.TaskId, t.Config.Task.Type, t.State);
var tasks = _context.TaskOperations.SearchStates(jobId);
- if (search.WithTasks ?? false) {
- var ts = await tasks.ToListAsync();
- return await RequestHandling.Ok(req, JobResponse.ForJob(job, ts));
- } else {
- var taskInfo = await tasks.Select(TaskToJobTaskInfo).ToListAsync();
- return await RequestHandling.Ok(req, JobResponse.ForJob(job, taskInfo));
- }
+
+ IAsyncEnumerable<IJobTaskInfo> taskInfo = search.WithTasks ?? false ? tasks : tasks.Select(TaskToJobTaskInfo);
+
+ var crashReported = await _context.JobCrashReportedOperations.CrashReported(jobId);
+ return await RequestHandling.Ok(req, JobResponse.ForJob(job, taskInfo.ToEnumerable(), crashReported));
}
var jobs = await _context.JobOperations.SearchState(states: search.State ?? Enumerable.Empty<JobState>()).ToListAsync();
diff --git a/src/ApiService/ApiService/Functions/QueueJobResult.cs b/src/ApiService/ApiService/Functions/QueueJobResult.cs
index 31b39802d6..3f863cb2f2 100644
--- a/src/ApiService/ApiService/Functions/QueueJobResult.cs
+++ b/src/ApiService/ApiService/Functions/QueueJobResult.cs
@@ -49,6 +49,10 @@ public async Async.Task Run([QueueTrigger("job-result", Connection = "AzureWebJo
var jobResultType = data.Type;
_log.LogInformation($"job result data type: {jobResultType}");
+ if (jobResultType == "CrashReported") {
+ var _result = await _context.JobCrashReportedOperations.ReportCrash(job.JobId, jr.TaskId);
+ }
+
Dictionary<string, double> value;
if (jr.Value.Count > 0) {
value = jr.Value;
diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs
index 4dd4000283..38b4c19938 100644
--- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs
+++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs
@@ -968,9 +968,12 @@ public record Job(
StoredUserInfo? UserInfo,
string? Error = null,
DateTimeOffset? EndTime = null
-) : StatefulEntityBase<JobState>(State) {
+) : StatefulEntityBase<JobState>(State);
-}
+public record JobCrashReported(
+ [PartitionKey] Guid JobId,
+ [RowKey] Guid TaskId
+) : EntityBase;
// This is like UserInfo but lacks the UPN:
public record StoredUserInfo(Guid? ApplicationId, Guid? ObjectId);
diff --git a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs
index c1067305ad..2760837a66 100644
--- a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs
+++ b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs
@@ -92,6 +92,7 @@ public record ContainerInfo(
Uri SasUrl
) : BaseResponse();
+
public record JobResponse(
Guid JobId,
JobState State,
@@ -101,10 +102,11 @@ public record JobResponse(
IEnumerable<IJobTaskInfo>? TaskInfo,
StoredUserInfo? UserInfo,
[property: JsonPropertyName("Timestamp")] // must retain capital T for backcompat
- DateTimeOffset? Timestamp
+ DateTimeOffset? Timestamp,
+ bool CrashReported
// not including UserInfo from Job model
) : BaseResponse() {
- public static JobResponse ForJob(Job j, IEnumerable<IJobTaskInfo>? taskInfo)
+ public static JobResponse ForJob(Job j, IEnumerable<IJobTaskInfo>? taskInfo, bool crashReported = false)
=> new(
JobId: j.JobId,
State: j.State,
@@ -113,7 +115,8 @@ public static JobResponse ForJob(Job j, IEnumerable<IJobTaskInfo>? taskInfo)
EndTime: j.EndTime,
TaskInfo: taskInfo,
UserInfo: j.UserInfo,
- Timestamp: j.Timestamp
+ Timestamp: j.Timestamp,
+ CrashReported: crashReported
);
public DateTimeOffset? StartTime => EndTime is DateTimeOffset endTime ? endTime.Subtract(TimeSpan.FromHours(Config.Duration)) : null;
}
diff --git a/src/ApiService/ApiService/Program.cs b/src/ApiService/ApiService/Program.cs
index f26463883b..112b2d358c 100644
--- a/src/ApiService/ApiService/Program.cs
+++ b/src/ApiService/ApiService/Program.cs
@@ -24,11 +24,11 @@ namespace Microsoft.OneFuzz.Service;
public class Program {
/// <summary>
- ///
+ ///
/// </summary>
public class LoggingMiddleware : IFunctionsWorkerMiddleware {
/// <summary>
- ///
+ ///
/// </summary>
/// <param name="context"></param>
/// <param name="next"></param>
@@ -198,6 +198,7 @@ public static async Async.Task Main() {
.AddScoped<INodeMessageOperations, NodeMessageOperations>()
.AddScoped<ISubnet, Subnet>()
.AddScoped<IAutoScaleOperations, AutoScaleOperations>()
+ .AddScoped<IJobCrashReportedOperations, JobCrashReportedOperations>()
.AddSingleton<GraphServiceClient>(new GraphServiceClient(new DefaultAzureCredential()))
.AddSingleton<DependencyTrackingTelemetryModule>()
.AddSingleton<ICreds, Creds>()
diff --git a/src/ApiService/ApiService/onefuzzlib/JobCrashReported.cs b/src/ApiService/ApiService/onefuzzlib/JobCrashReported.cs
new file mode 100644
index 0000000000..e0b903979b
--- /dev/null
+++ b/src/ApiService/ApiService/onefuzzlib/JobCrashReported.cs
@@ -0,0 +1,28 @@
+using System.Threading.Tasks;
+using ApiService.OneFuzzLib.Orm;
+using Microsoft.Extensions.Logging;
+namespace Microsoft.OneFuzz.Service;
+
+public interface IJobCrashReportedOperations : IOrm<JobCrashReported> {
+ public Task<bool> CrashReported(Guid jobId);
+ public Task<OneFuzzResultVoid> ReportCrash(Guid jobId, Guid taskId);
+}
+
+public class JobCrashReportedOperations : Orm<JobCrashReported>, IJobCrashReportedOperations {
+ public JobCrashReportedOperations(ILogger<JobCrashReportedOperations> logTracer, IOnefuzzContext context) : base(logTracer, context) {
+ }
+
+ public async Task<bool> CrashReported(Guid jobId) {
+ return await QueryAsync(Query.RowKey(jobId.ToString())).AnyAsync();
+ }
+
+ public async Task<OneFuzzResultVoid> ReportCrash(Guid jobId, Guid taskId) {
+
+ var result = await Update(new JobCrashReported(jobId, taskId));
+ if (!result.IsOk) {
+ return OneFuzzResultVoid.Error(ErrorCode.UNABLE_TO_UPDATE, "Failed to update job crash reported");
+ }
+
+ return OneFuzzResultVoid.Ok;
+ }
+}
diff --git a/src/ApiService/ApiService/onefuzzlib/OnefuzzContext.cs b/src/ApiService/ApiService/onefuzzlib/OnefuzzContext.cs
index 03c6322663..3da5f6522e 100644
--- a/src/ApiService/ApiService/onefuzzlib/OnefuzzContext.cs
+++ b/src/ApiService/ApiService/onefuzzlib/OnefuzzContext.cs
@@ -49,7 +49,7 @@ public interface IOnefuzzContext {
ITeams Teams { get; }
IGithubIssues GithubIssues { get; }
IAdo Ado { get; }
-
+ IJobCrashReportedOperations JobCrashReportedOperations { get; }
IFeatureManagerSnapshot FeatureManagerSnapshot { get; }
IConfigurationRefresher ConfigurationRefresher { get; }
}
@@ -101,6 +101,7 @@ public OnefuzzContext(IServiceProvider serviceProvider) {
public ITeams Teams => _serviceProvider.GetRequiredService<ITeams>();
public IGithubIssues GithubIssues => _serviceProvider.GetRequiredService<IGithubIssues>();
public IAdo Ado => _serviceProvider.GetRequiredService<IAdo>();
+ public IJobCrashReportedOperations JobCrashReportedOperations => _serviceProvider.GetRequiredService<IJobCrashReportedOperations>();
public IFeatureManagerSnapshot FeatureManagerSnapshot => _serviceProvider.GetRequiredService<IFeatureManagerSnapshot>();
diff --git a/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs b/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs
index b3633f12d4..46ff35f085 100644
--- a/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs
+++ b/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs
@@ -663,8 +663,8 @@ public async Async.Task Process(IList<(string, string)> notificationInfo, bool i
_logTracer.AddTags(notificationInfo);
_logTracer.AddTag("WorkItemId", entry.Id.HasValue ? entry.Id.Value.ToString() : "");
_logTracer.LogEvent(adoEventType);
- }
}
+ }
private static bool IsADODuplicateWorkItem(WorkItem wi, Dictionary<string, string>? duplicateFields) {
// A work item could have System.State == Resolve && System.Reason == Duplicate
diff --git a/src/ApiService/IntegrationTests/Fakes/TestContext.cs b/src/ApiService/IntegrationTests/Fakes/TestContext.cs
index 66d121e746..74997f8210 100644
--- a/src/ApiService/IntegrationTests/Fakes/TestContext.cs
+++ b/src/ApiService/IntegrationTests/Fakes/TestContext.cs
@@ -1,4 +1,5 @@
using System;
+using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using Microsoft.Extensions.Caching.Memory;
@@ -42,6 +43,7 @@ public TestContext(IHttpClientFactory httpClientFactory, OneFuzzLoggerProvider p
ReproOperations = new ReproOperations(provider.CreateLogger<ReproOperations>(), this);
Reports = new Reports(provider.CreateLogger<Reports>(), Containers);
NotificationOperations = new NotificationOperations(provider.CreateLogger<NotificationOperations>(), this);
+ JobCrashReportedOperations = new JobCrashReportedOperations(provider.CreateLogger<JobCrashReportedOperations>(), this);
FeatureManagerSnapshot = new TestFeatureManagerSnapshot();
WebhookOperations = new TestWebhookOperations(httpClientFactory, provider.CreateLogger<WebhookOperations>(), this);
@@ -65,9 +67,28 @@ public Async.Task InsertAll(params EntityBase[] objs)
InstanceConfig ic => ConfigOperations.Insert(ic),
Notification n => NotificationOperations.Insert(n),
Webhook w => WebhookOperations.Insert(w),
+ JobCrashReported crashReported => JobCrashReportedOperations.Insert(crashReported),
_ => throw new NotSupportedException($"You will need to add an TestContext.InsertAll case for {x.GetType()} entities"),
}));
+ public Async.Task InsertAll(IEnumerable<EntityBase> objs)
+ => Async.Task.WhenAll(
+ objs.Select(x => x switch {
+ Task t => TaskOperations.Insert(t),
+ Node n => NodeOperations.Insert(n),
+ Pool p => PoolOperations.Insert(p),
+ Job j => JobOperations.Insert(j),
+ JobResult jr => JobResultOperations.Insert(jr),
+ Repro r => ReproOperations.Insert(r),
+ Scaleset ss => ScalesetOperations.Insert(ss),
+ NodeTasks nt => NodeTasksOperations.Insert(nt),
+ InstanceConfig ic => ConfigOperations.Insert(ic),
+ Notification n => NotificationOperations.Insert(n),
+ Webhook w => WebhookOperations.Insert(w),
+ JobCrashReported crashReported => JobCrashReportedOperations.Insert(crashReported),
+ _ => throw new NotSupportedException($"You will need to add an TestContext.InsertAll case for {x.GetType()} entities"),
+ }));
+
// Implementations:
public IMemoryCache Cache { get; }
@@ -109,6 +130,8 @@ public Async.Task InsertAll(params EntityBase[] objs)
public IWebhookMessageLogOperations WebhookMessageLogOperations { get; }
+ public IJobCrashReportedOperations JobCrashReportedOperations {get;}
+
// -- Remainder not implemented --
public IConfig Config => throw new System.NotImplementedException();
@@ -143,4 +166,6 @@ public Async.Task InsertAll(params EntityBase[] objs)
public IAdo Ado => throw new NotImplementedException();
public IConfigurationRefresher ConfigurationRefresher => throw new NotImplementedException();
+
+
}
diff --git a/src/ApiService/IntegrationTests/JobsTests.cs b/src/ApiService/IntegrationTests/JobsTests.cs
index 28dbe8457f..857b54bd64 100644
--- a/src/ApiService/IntegrationTests/JobsTests.cs
+++ b/src/ApiService/IntegrationTests/JobsTests.cs
@@ -226,4 +226,25 @@ await Context.InsertAll(
Assert.Equal(task.Config.Task.Type, returnedTasks[0].Type);
}
+
+ [Fact]
+ public async Async.Task Get_CanFindSpecificJobWithBugs() {
+ var taskConfig = new TaskConfig(_jobId, new List<Guid>(), new TaskDetails(TaskType.Coverage, 60));
+ await Context.InsertAll(
+ new Job(_jobId, JobState.Stopped, _config, null),
+ new Task(_jobId, Guid.NewGuid(), TaskState.Running, Os.Windows, taskConfig),
+ new JobCrashReported(_jobId, Guid.NewGuid())
+ );
+
+ var func = new Jobs(Context, LoggerProvider.CreateLogger<Jobs>());
+
+ var ctx = new TestFunctionContext();
+ var result = await func.Run(TestHttpRequestData.FromJson("GET", new JobSearch(JobId: _jobId)), ctx);
+ Assert.Equal(HttpStatusCode.OK, result.StatusCode);
+
+ var response = BodyAs<JobResponse>(result);
+ Assert.Equal(_jobId, response.JobId);
+ Assert.NotNull(response.TaskInfo);
+ Assert.True(response.CrashReported);
+ }
}
diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py
index 746c528c1c..8adae9c9ee 100644
--- a/src/pytypes/onefuzztypes/models.py
+++ b/src/pytypes/onefuzztypes/models.py
@@ -761,6 +761,7 @@ class Job(BaseModel):
task_info: Optional[List[Union[Task, JobTaskInfo]]]
user_info: Optional[UserInfo]
start_time: Optional[datetime] = None
+ crash_reported: Optional[bool] = None
class NetworkConfig(BaseModel):
From 4c8a30066b0c4b06db69b088250f3de4895e3e0f Mon Sep 17 00:00:00 2001
From: Cheick Keita <chkeita@microsoft.com>
Date: Mon, 30 Oct 2023 17:45:30 -0700
Subject: [PATCH 2/3] format
---
src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs | 2 +-
src/ApiService/IntegrationTests/Fakes/TestContext.cs | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs b/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs
index 46ff35f085..b3633f12d4 100644
--- a/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs
+++ b/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs
@@ -663,8 +663,8 @@ public async Async.Task Process(IList<(string, string)> notificationInfo, bool i
_logTracer.AddTags(notificationInfo);
_logTracer.AddTag("WorkItemId", entry.Id.HasValue ? entry.Id.Value.ToString() : "");
_logTracer.LogEvent(adoEventType);
- }
}
+ }
private static bool IsADODuplicateWorkItem(WorkItem wi, Dictionary<string, string>? duplicateFields) {
// A work item could have System.State == Resolve && System.Reason == Duplicate
diff --git a/src/ApiService/IntegrationTests/Fakes/TestContext.cs b/src/ApiService/IntegrationTests/Fakes/TestContext.cs
index 74997f8210..ada13f0d44 100644
--- a/src/ApiService/IntegrationTests/Fakes/TestContext.cs
+++ b/src/ApiService/IntegrationTests/Fakes/TestContext.cs
@@ -130,7 +130,7 @@ public Async.Task InsertAll(IEnumerable<EntityBase> objs)
public IWebhookMessageLogOperations WebhookMessageLogOperations { get; }
- public IJobCrashReportedOperations JobCrashReportedOperations {get;}
+ public IJobCrashReportedOperations JobCrashReportedOperations { get; }
// -- Remainder not implemented --
From ee4b7123c03a81379e3779b04bc9dc733906a18f Mon Sep 17 00:00:00 2001
From: Cheick Keita <chkeita@microsoft.com>
Date: Mon, 30 Oct 2023 18:39:06 -0700
Subject: [PATCH 3/3] fix test
---
src/ApiService/ApiService/onefuzzlib/JobCrashReported.cs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/ApiService/ApiService/onefuzzlib/JobCrashReported.cs b/src/ApiService/ApiService/onefuzzlib/JobCrashReported.cs
index e0b903979b..2739c5945e 100644
--- a/src/ApiService/ApiService/onefuzzlib/JobCrashReported.cs
+++ b/src/ApiService/ApiService/onefuzzlib/JobCrashReported.cs
@@ -13,7 +13,7 @@ public JobCrashReportedOperations(ILogger<JobCrashReportedOperations> logTracer,
}
public async Task<bool> CrashReported(Guid jobId) {
- return await QueryAsync(Query.RowKey(jobId.ToString())).AnyAsync();
+ return await QueryAsync(Query.PartitionKey(jobId.ToString())).AnyAsync();
}
public async Task<OneFuzzResultVoid> ReportCrash(Guid jobId, Guid taskId) {