From cba3826bdebe9c94c8e589b22d6eec111945e9e1 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Fri, 11 Jul 2025 16:41:23 -0700 Subject: [PATCH 1/3] Improve 'Resolve-Error' command and allow default system prompt for openai-gpt --- .../Commands/ResolveErrorCommand.cs | 135 +++++++++--------- shell/AIShell.Integration/ErrorFeedback.cs | 28 ++-- shell/agents/AIShell.OpenAI.Agent/GPT.cs | 1 - shell/agents/AIShell.OpenAI.Agent/Helpers.cs | 63 ++++++++ shell/agents/AIShell.OpenAI.Agent/Service.cs | 24 ++-- 5 files changed, 159 insertions(+), 92 deletions(-) diff --git a/shell/AIShell.Integration/Commands/ResolveErrorCommand.cs b/shell/AIShell.Integration/Commands/ResolveErrorCommand.cs index 8a50965d..ca010a53 100644 --- a/shell/AIShell.Integration/Commands/ResolveErrorCommand.cs +++ b/shell/AIShell.Integration/Commands/ResolveErrorCommand.cs @@ -20,16 +20,6 @@ public class ResolveErrorCommand : PSCmdlet protected override void EndProcessing() { - bool questionMarkValue = (bool)GetVariableValue("?"); - if (questionMarkValue) - { - WriteWarning("No error to resolve. The last command execution was successful."); - return; - } - - object value = GetVariableValue("LASTEXITCODE"); - int lastExitCode = value is null ? 0 : (int)value; - using var pwsh = PowerShell.Create(RunspaceMode.CurrentRunspace); var results = pwsh .AddCommand("Get-History") @@ -42,41 +32,64 @@ protected override void EndProcessing() return; } - string query = null, context = null; HistoryInfo lastHistory = results[0]; + ArrayList errors = (ArrayList)GetVariableValue("Error"); + bool questionMarkValue = (bool)GetVariableValue("?"); + object value = GetVariableValue("LASTEXITCODE"); + int lastExitCode = value is null ? 0 : (int)value; + + ErrorRecord lastError = null; + bool useLastError = false; + + if (errors.Count > 0) + { + object last = errors[0]; + lastError = last switch + { + ParseException pe => pe.ErrorRecord, + ErrorRecord er => er, + _ => throw new NotSupportedException($"Unexpected type of object '{last.GetType().FullName}' is found in '$Error'.") + }; + + // Use the last error for troubleshooting when + // - last error maps to the last history command, OR + // - they don't map but 'LastExitCode' is 0 (meaning no native command failure). + useLastError = lastError.InvocationInfo.HistoryId == lastHistory.Id || lastExitCode is 0; + } + + string query = null; Channel channel = Channel.Singleton; string commandLine = lastHistory.CommandLine; - if (TryGetLastError(lastHistory, out ErrorRecord lastError)) + if (useLastError) { - query = ErrorFeedback.CreateQueryForError(commandLine, lastError, channel); + query = ErrorFeedback.CreateQueryForError(lastError); } else if (lastExitCode is 0) { - // Cannot find the ErrorRecord associated with the last command, and no native command failure, so we don't know why '$?' was set to False. - ErrorRecord error = new( - new NotSupportedException($"Failed to detect the actual error even though '$?' is 'False'. No 'ErrorRecord' can be found that is associated with the last command line '{commandLine}' and no executable failure was found."), - errorId: "FailedToDetectActualError", - ErrorCategory.ObjectNotFound, - targetObject: null); - ThrowTerminatingError(error); + // $Error is empty and no failure from native command execution. + WriteWarning("Cannot find an error to resolve."); + return; } - else + else if (!questionMarkValue) { - // '$? == False' but no 'ErrorRecord' can be found that is associated with the last command line, - // and '$LASTEXITCODE' is non-zero, which indicates the last failed command is a native command. - query = $""" - Running the command line `{commandLine}` in PowerShell v{channel.PSVersion} failed. - Please try to explain the failure and suggest the right fix. - Output of the command line can be found in the context information below. - """; - - context = ScrapeScreenForNativeCommandOutput(commandLine); - if (context is null) + // In this scenario, we have: + // - Last error doesn't map to the last history command; + // - $LastExitCode is non-zero; + // - $? is false. + // It indicates the last command is a native command and it failed. + string output = ScrapeScreenForNativeCommandOutput(commandLine); + + if (output is null) { if (UseClipboardForCommandOutput()) { - IncludeOutputFromClipboard = true; + pwsh.Commands.Clear(); + output = pwsh + .AddCommand("Get-Clipboard") + .AddParameter("Raw") + .Invoke() + .FirstOrDefault(); } else { @@ -88,20 +101,35 @@ Output of the command line can be found in the context information below. )); } } - } - if (context is null && IncludeOutputFromClipboard) - { - pwsh.Commands.Clear(); - var r = pwsh - .AddCommand("Get-Clipboard") - .AddParameter("Raw") - .Invoke(); + query = $""" + Please troubleshoot the command-line error that happend in the connected PowerShell session, and suggest the fix. The error details are given below: + + --- - context = r?.Count > 0 ? r[0] : null; + ## Command Line + ``` + {commandLine} + ``` + + ## Error Output + ``` + {output} + ``` + """; + } + else + { + // When reaching here, we have + // - Last error doesn't map to the last history command; + // - $LastExitCode is non-zero; + // - $? is true. + // The user may want to fix a command that failed previously, but it's unknown whether + // that was a native command failure or PowerShell command failure. + query = "Take a look at the terminal output of the connected PowerShell session and try resolving the last error you see."; } - channel.PostQuery(new PostQueryMessage(query, context, Agent)); + channel.PostQuery(new PostQueryMessage(query, context: null, Agent)); } private bool UseClipboardForCommandOutput() @@ -115,29 +143,6 @@ private bool UseClipboardForCommandOutput() return ShouldContinue(query, caption: "Include output from the clipboard"); } - private bool TryGetLastError(HistoryInfo lastHistory, out ErrorRecord lastError) - { - lastError = null; - ArrayList errors = (ArrayList)GetVariableValue("Error"); - if (errors.Count == 0) - { - return false; - } - - lastError = errors[0] as ErrorRecord; - if (lastError is null && errors[0] is RuntimeException rtEx) - { - lastError = rtEx.ErrorRecord; - } - - if (lastError?.InvocationInfo is null || lastError.InvocationInfo.HistoryId != lastHistory.Id) - { - return false; - } - - return true; - } - private string ScrapeScreenForNativeCommandOutput(string lastCommandLine) { if (!OperatingSystem.IsWindows()) diff --git a/shell/AIShell.Integration/ErrorFeedback.cs b/shell/AIShell.Integration/ErrorFeedback.cs index e95fbf7f..4957b964 100644 --- a/shell/AIShell.Integration/ErrorFeedback.cs +++ b/shell/AIShell.Integration/ErrorFeedback.cs @@ -34,7 +34,7 @@ public FeedbackItem GetFeedback(FeedbackContext context, CancellationToken token Channel channel = Channel.Singleton; if (channel.CheckConnection(blocking: false, out _)) { - string query = CreateQueryForError(context.CommandLine, context.LastError, channel); + string query = CreateQueryForError(context.LastError); PostQueryMessage message = new(query, context: null, agent: null); channel.PostQuery(message); @@ -44,37 +44,37 @@ public FeedbackItem GetFeedback(FeedbackContext context, CancellationToken token return null; } - internal static string CreateQueryForError(string commandLine, ErrorRecord lastError, Channel channel) + internal static string CreateQueryForError(ErrorRecord lastError) { Exception exception = lastError.Exception; StringBuilder sb = new StringBuilder(capacity: 100) - .Append( - $""" - Running the command line `{commandLine}` in PowerShell v{channel.PSVersion} failed. - Please try to explain the failure and suggest the right fix. - The error details can be found below in the markdown format. - """) - .Append("\n\n") - .Append("# Error Details\n") + .Append("Please troubleshoot the command-line error that happend in the connected PowerShell session, and suggest the fix. The error details are given below:\n\n---\n\n") + .Append("## Command Line\n") + .Append("```\n") + .Append(lastError.InvocationInfo.Line).Append('\n') + .Append("```\n\n") .Append("## Exception Messages\n") .Append($"{exception.GetType().FullName}: {exception.Message}\n"); exception = exception.InnerException; if (exception is not null) { - sb.Append("Inner Exceptions:\n"); + sb.Append("\nInner Exceptions:\n"); do { - sb.Append($" - {exception.GetType().FullName}: {exception.Message}\n"); + sb.Append($"- {exception.GetType().FullName}: {exception.Message}\n"); exception = exception.InnerException; } while (exception is not null); } - string positionMessage = lastError.InvocationInfo?.PositionMessage; + string positionMessage = lastError.InvocationInfo.PositionMessage; if (!string.IsNullOrEmpty(positionMessage)) { - sb.Append("## Error Position\n").Append(positionMessage).Append('\n'); + sb.Append("\n## Error Position\n") + .Append("```\n") + .Append(positionMessage).Append('\n') + .Append("```\n"); } return sb.ToString(); diff --git a/shell/agents/AIShell.OpenAI.Agent/GPT.cs b/shell/agents/AIShell.OpenAI.Agent/GPT.cs index d5db24b3..42716f89 100644 --- a/shell/agents/AIShell.OpenAI.Agent/GPT.cs +++ b/shell/agents/AIShell.OpenAI.Agent/GPT.cs @@ -50,7 +50,6 @@ public GPT( ArgumentException.ThrowIfNullOrEmpty(name); ArgumentException.ThrowIfNullOrEmpty(description); ArgumentException.ThrowIfNullOrEmpty(modelName); - ArgumentException.ThrowIfNullOrEmpty(systemPrompt); Name = name; Description = description; diff --git a/shell/agents/AIShell.OpenAI.Agent/Helpers.cs b/shell/agents/AIShell.OpenAI.Agent/Helpers.cs index cbfccd87..d2e83fc8 100644 --- a/shell/agents/AIShell.OpenAI.Agent/Helpers.cs +++ b/shell/agents/AIShell.OpenAI.Agent/Helpers.cs @@ -189,3 +189,66 @@ private bool ShouldRetryImpl(PipelineMessage message, Exception exception) return default; } } + +internal static class Prompt +{ + internal static string SystemPromptWithConnectedPSSession = $""" + You are a virtual assistant in **AIShell**, specializing in PowerShell and other command-line tools. + + You are connected to an interactive PowerShell session and can access contextual information and interact with the session using built-in tools. + + Your primary function is to assist users with accomplishing tasks and troubleshooting errors in the command line. Autonomously resolve the user's query to the best of your ability before returning with a response. + + --- + + ## General Behavior + + - Respond clearly, concisely, and with empathy. + - Use markdown **code block syntax** for formatting: + - Use ` ```powershell ` for PowerShell commands and scripts. + - Use ` ```sh ` for non-PowerShell CLI commands (e.g., bash, CMD). + - **Do not** use code blocks for tables. + - When generating CLI commands, keep each command **on a single line**. Always include all parameters and arguments on that line. + + ## Tool Calling + + - **Strictly** follow the tool call schema and ensure all required parameters are included. + - Tools prefixed with `AIShell__` are built-in for interacting with the PowerShell session. External tools may also be configured by the user. + - Prefer using available tools to gather needed information instead of prompting the user for it. + - Explain why a tool is being used **before** calling it, unless the reason is already obvious from the ongoing context. + + ## Runtime Environment + + - Operating System: **{Utils.OS}** + - PowerShell version: **v7.4 or above** + """; + + internal static string SystemPrompForStandaloneApp = $""" + You are a virtual assistant in **AIShell**, specializing in PowerShell and other command-line tools. + + Your primary function is to assist users with accomplishing tasks in the command line. Autonomously resolve the user's query to the best of your ability before returning with a response. + + --- + + ## General Behavior + + - Respond clearly, concisely, and with empathy. + - Use markdown **code block syntax** for formatting: + - Use ` ```powershell ` for PowerShell commands and scripts. + - Use ` ```sh ` for non-PowerShell CLI commands (e.g., bash, CMD). + - **Do not** use code blocks for tables. + - When generating CLI commands, keep each command **on a single line**. Always include all parameters and arguments on that line. + + ## Tool Calling + + - You may have access to external tools provided to help resolve the user's query. + - **Strictly** follow the tool call schema and ensure all required parameters are included. + - Prefer using available tools to gather needed information instead of prompting the user for it. + - Explain why a tool is being used **before** calling it, unless the reason is already obvious from the ongoing context. + + ## Runtime Environment + + - Operating System: **{Utils.OS}** + - PowerShell version: **v7.4 or above** + """; +} diff --git a/shell/agents/AIShell.OpenAI.Agent/Service.cs b/shell/agents/AIShell.OpenAI.Agent/Service.cs index b366b915..1c2d0467 100644 --- a/shell/agents/AIShell.OpenAI.Agent/Service.cs +++ b/shell/agents/AIShell.OpenAI.Agent/Service.cs @@ -69,7 +69,7 @@ private void RefreshOpenAIClient() OpenAIChatClient client; EndpointType type = _gptToUse.Type; // Reasoning models do not support the temperature setting. - _chatOptions.Temperature = _gptToUse.ModelInfo.Reasoning ? null : 0; + _chatOptions.Temperature = _gptToUse.ModelInfo.Reasoning ? null : 0.5f; if (type is EndpointType.AzureOpenAI) { @@ -131,22 +131,21 @@ private void RefreshOpenAIClient() .Build(); } - private void PrepareForChat(string input) + private void PrepareForChat(string input, IShell shell) { - const string Guidelines = """ - ## Tool Use Guidelines - You may have access to external tools. - Before making any tool call, you must first explain the reason for using the tool. Only issue the tool call after providing this explanation. - - ## Other Guidelines - """; - // Refresh the client in case the active model was changed. RefreshOpenAIClient(); if (_chatHistory.Count is 0) { - string system = $"{Guidelines}\n{_gptToUse.SystemPrompt}"; + string system = _gptToUse.SystemPrompt; + if (string.IsNullOrEmpty(system)) + { + system = shell.ChannelEstablished + ? Prompt.SystemPromptWithConnectedPSSession + : Prompt.SystemPrompForStandaloneApp; + } + _chatHistory.Add(new(ChatRole.System, system)); } @@ -157,7 +156,8 @@ public async Task> GetStreamingChatResponse { try { - PrepareForChat(input); + PrepareForChat(input, shell); + var tools = await shell.GetAIFunctions(); if (tools is { Count: > 0 }) { From 027854fa43a27162071f53a79e08aa2f427e4db9 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Fri, 11 Jul 2025 23:14:52 -0700 Subject: [PATCH 2/3] Update a comment --- shell/AIShell.Integration/Commands/ResolveErrorCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/AIShell.Integration/Commands/ResolveErrorCommand.cs b/shell/AIShell.Integration/Commands/ResolveErrorCommand.cs index ca010a53..9cd62cc5 100644 --- a/shell/AIShell.Integration/Commands/ResolveErrorCommand.cs +++ b/shell/AIShell.Integration/Commands/ResolveErrorCommand.cs @@ -121,7 +121,7 @@ protected override void EndProcessing() else { // When reaching here, we have - // - Last error doesn't map to the last history command; + // - Last error doesn't map to the last history command, or $Error is empty; // - $LastExitCode is non-zero; // - $? is true. // The user may want to fix a command that failed previously, but it's unknown whether From 18f03835b2ec383dd306552fd2e7a5eb41af7013 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Mon, 14 Jul 2025 10:17:32 -0700 Subject: [PATCH 3/3] Update the system prompt a bit --- shell/agents/AIShell.OpenAI.Agent/Helpers.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/agents/AIShell.OpenAI.Agent/Helpers.cs b/shell/agents/AIShell.OpenAI.Agent/Helpers.cs index d2e83fc8..c4df2e5e 100644 --- a/shell/agents/AIShell.OpenAI.Agent/Helpers.cs +++ b/shell/agents/AIShell.OpenAI.Agent/Helpers.cs @@ -195,7 +195,7 @@ internal static class Prompt internal static string SystemPromptWithConnectedPSSession = $""" You are a virtual assistant in **AIShell**, specializing in PowerShell and other command-line tools. - You are connected to an interactive PowerShell session and can access contextual information and interact with the session using built-in tools. + You are connected to an interactive PowerShell session and can retrieve session context and interact with the session using built-in tools. When user queries are ambiguous or minimal, rely on session context to better understand intent and deliver accurate, helpful responses.. Your primary function is to assist users with accomplishing tasks and troubleshooting errors in the command line. Autonomously resolve the user's query to the best of your ability before returning with a response.