Play sound effects for notifications and events (RooVetGit#38)

Co-authored-by: HeavenOSK <[email protected]>

Author: mrubens

audio/celebration.wav

@@ -195,6 +195,7 @@
     "os-name": "^6.0.0",
     "p-wait-for": "^5.0.2",
     "pdf-parse": "^1.1.1",
+    "play-sound": "^1.1.6",
     "puppeteer-chromium-resolver": "^23.0.0",
     "puppeteer-core": "^23.4.0",
     "serialize-error": "^11.0.3",

audio/notification.wav

@@ -20,6 +20,7 @@ import { Cline } from "../Cline"
 import { openMention } from "../mentions"
 import { getNonce } from "./getNonce"
 import { getUri } from "./getUri"
+import { playSound, setSoundEnabled } from "../../utils/sound"
 
 /*
 https://github.com/microsoft/vscode-webview-ui-toolkit-samples/blob/main/default/weather-webview/src/providers/WeatherViewProvider.ts
@@ -61,6 +62,7 @@ type GlobalStateKey =
 	| "openRouterModelId"
 	| "openRouterModelInfo"
 	| "allowedCommands"
+	| "soundEnabled"
 
 export const GlobalFileNames = {
 	apiConversationHistory: "api_conversation_history.json",
@@ -520,6 +522,18 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 						break;
 					// Add more switch case statements here as more webview message commands
 					// are created within the webview context (i.e. inside media/main.js)
+					case "playSound":
+						if (message.audioType) {
+							const soundPath = path.join(this.context.extensionPath, "audio", `${message.audioType}.wav`)
+							playSound(soundPath)
+						}
+						break
+					case "soundEnabled":
+						const enabled = message.bool ?? true
+						await this.updateGlobalState("soundEnabled", enabled)
+						setSoundEnabled(enabled)
+						await this.postStateToWebview()
+						break
 				}
 			},
 			null,
@@ -825,6 +839,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 			alwaysAllowWrite,
 			alwaysAllowExecute,
 			alwaysAllowBrowser,
+			soundEnabled,
 			taskHistory,
 		} = await this.getState()
 
@@ -845,6 +860,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 			taskHistory: (taskHistory || [])
 				.filter((item) => item.ts && item.task)
 				.sort((a, b) => b.ts - a.ts),
+			soundEnabled: soundEnabled ?? true,
 			shouldShowAnnouncement: lastShownAnnouncementId !== this.latestAnnouncementId,
 			allowedCommands,
 		}
@@ -935,6 +951,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 			alwaysAllowBrowser,
 			taskHistory,
 			allowedCommands,
+			soundEnabled,
 		] = await Promise.all([
 			this.getGlobalState("apiProvider") as Promise<ApiProvider | undefined>,
 			this.getGlobalState("apiModelId") as Promise<string | undefined>,
@@ -968,6 +985,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 			this.getGlobalState("alwaysAllowBrowser") as Promise<boolean | undefined>,
 			this.getGlobalState("taskHistory") as Promise<HistoryItem[] | undefined>,
 			this.getGlobalState("allowedCommands") as Promise<string[] | undefined>,
+			this.getGlobalState("soundEnabled") as Promise<boolean | undefined>,
 		])
 
 		let apiProvider: ApiProvider
@@ -1019,6 +1037,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 			alwaysAllowBrowser: alwaysAllowBrowser ?? false,
 			taskHistory,
 			allowedCommands,
+			soundEnabled,
 		}
 	}
 

audio/progress_loop.wav

@@ -41,6 +41,7 @@ export interface ExtensionState {
 	alwaysAllowBrowser?: boolean
 	uriScheme?: string
 	allowedCommands?: string[]
+	soundEnabled?: boolean
 }
 
 export interface ClineMessage {

package-lock.json

@@ -1,5 +1,7 @@
 import { ApiConfiguration, ApiProvider } from "./api"
 
+export type AudioType = "notification" | "celebration" | "progress_loop"
+
 export interface WebviewMessage {
 	type:
 		| "apiConfiguration"
@@ -27,12 +29,15 @@ export interface WebviewMessage {
 		| "cancelTask"
 		| "refreshOpenRouterModels"
 		| "alwaysAllowBrowser"
+		| "playSound"
+		| "soundEnabled"
 	text?: string
 	askResponse?: ClineAskResponse
 	apiConfiguration?: ApiConfiguration
 	images?: string[]
 	bool?: boolean
 	commands?: string[]
+	audioType?: AudioType
 }
 
 export type ClineAskResponse = "yesButtonClicked" | "noButtonClicked" | "messageResponse"

package.json

@@ -0,0 +1,68 @@
+import * as vscode from "vscode"
+import * as path from "path"
+
+/**
+ * Minimum interval (in milliseconds) to prevent continuous playback
+ */
+const MIN_PLAY_INTERVAL = 500
+
+/**
+ * Timestamp of when sound was last played
+ */
+let lastPlayedTime = 0
+
+/**
+ * Determine if a file is a WAV file
+ * @param filepath string
+ * @returns boolean
+ */
+export const isWAV = (filepath: string): boolean => {
+	return path.extname(filepath).toLowerCase() === ".wav"
+}
+
+let isSoundEnabled = true
+
+/**
+ * Set sound configuration
+ * @param enabled boolean
+ */
+export const setSoundEnabled = (enabled: boolean): void => {
+	isSoundEnabled = enabled
+}
+
+/**
+ * Play a sound file
+ * @param filepath string
+ * @return void
+ */
+export const playSound = (filepath: string): void => {
+	try {
+		if (!isSoundEnabled) {
+			return
+		}
+
+		if (!filepath) {
+			return
+		}
+
+		if (!isWAV(filepath)) {
+			throw new Error("Only wav files are supported.")
+		}
+
+		const currentTime = Date.now()
+		if (currentTime - lastPlayedTime < MIN_PLAY_INTERVAL) {
+			return // Skip playback within minimum interval to prevent continuous playback
+		}
+
+		const player = require("play-sound")()
+		player.play(filepath, function (err: any) {
+			if (err) {
+				throw new Error("Failed to play sound effect")
+			}
+		})
+
+		lastPlayedTime = currentTime
+	} catch (error: any) {
+		vscode.window.showErrorMessage(error.message)
+	}
+}

src/core/webview/ClineProvider.ts

@@ -24,6 +24,7 @@ import BrowserSessionRow from "./BrowserSessionRow"
 import ChatRow from "./ChatRow"
 import ChatTextArea from "./ChatTextArea"
 import TaskHeader from "./TaskHeader"
+import { AudioType } from "../../../../src/shared/WebviewMessage"
 
 interface ChatViewProps {
 	isHidden: boolean
@@ -61,10 +62,24 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 	const [showScrollToBottom, setShowScrollToBottom] = useState(false)
 	const [isAtBottom, setIsAtBottom] = useState(false)
 
+	const [wasStreaming, setWasStreaming] = useState<boolean>(false)
+	const [hasStarted, setHasStarted] = useState(false)
+
 	// UI layout depends on the last 2 messages
 	// (since it relies on the content of these messages, we are deep comparing. i.e. the button state after hitting button sets enableButtons to false, and this effect otherwise would have to true again even if messages didn't change
 	const lastMessage = useMemo(() => messages.at(-1), [messages])
 	const secondLastMessage = useMemo(() => messages.at(-2), [messages])
+
+	function playSound(audioType: AudioType) {
+		vscode.postMessage({ type: "playSound", audioType })
+	}
+
+	function playSoundOnMessage(audioType: AudioType) {
+		if (hasStarted && !isStreaming) {
+			playSound(audioType)
+		}
+	}
+
 	useDeepCompareEffect(() => {
 		// if last message is an ask, show user ask UI
 		// if user finished a task, then start a new task with a new conversation history since in this moment that the extension is waiting for user response, the user could close the extension and the conversation history would be lost.
@@ -75,27 +90,31 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 					const isPartial = lastMessage.partial === true
 					switch (lastMessage.ask) {
 						case "api_req_failed":
+							playSoundOnMessage("progress_loop")
 							setTextAreaDisabled(true)
 							setClineAsk("api_req_failed")
 							setEnableButtons(true)
 							setPrimaryButtonText("Retry")
 							setSecondaryButtonText("Start New Task")
 							break
 						case "mistake_limit_reached":
+							playSoundOnMessage("progress_loop")
 							setTextAreaDisabled(false)
 							setClineAsk("mistake_limit_reached")
 							setEnableButtons(true)
 							setPrimaryButtonText("Proceed Anyways")
 							setSecondaryButtonText("Start New Task")
 							break
 						case "followup":
+							playSoundOnMessage("notification")
 							setTextAreaDisabled(isPartial)
 							setClineAsk("followup")
 							setEnableButtons(isPartial)
 							// setPrimaryButtonText(undefined)
 							// setSecondaryButtonText(undefined)
 							break
 						case "tool":
+							playSoundOnMessage("notification")
 							setTextAreaDisabled(isPartial)
 							setClineAsk("tool")
 							setEnableButtons(!isPartial)
@@ -113,20 +132,23 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 							}
 							break
 						case "browser_action_launch":
+							playSoundOnMessage("notification")
 							setTextAreaDisabled(isPartial)
 							setClineAsk("browser_action_launch")
 							setEnableButtons(!isPartial)
 							setPrimaryButtonText("Approve")
 							setSecondaryButtonText("Reject")
 							break
 						case "command":
+							playSoundOnMessage("notification")
 							setTextAreaDisabled(isPartial)
 							setClineAsk("command")
 							setEnableButtons(!isPartial)
 							setPrimaryButtonText("Run Command")
 							setSecondaryButtonText("Reject")
 							break
 						case "command_output":
+							playSoundOnMessage("notification")
 							setTextAreaDisabled(false)
 							setClineAsk("command_output")
 							setEnableButtons(true)
@@ -135,13 +157,15 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 							break
 						case "completion_result":
 							// extension waiting for feedback. but we can just present a new task button
+							playSoundOnMessage("celebration")
 							setTextAreaDisabled(isPartial)
 							setClineAsk("completion_result")
 							setEnableButtons(!isPartial)
 							setPrimaryButtonText("Start New Task")
 							setSecondaryButtonText(undefined)
 							break
 						case "resume_task":
+							playSoundOnMessage("notification")
 							setTextAreaDisabled(false)
 							setClineAsk("resume_task")
 							setEnableButtons(true)
@@ -150,6 +174,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 							setDidClickCancel(false) // special case where we reset the cancel button state							
 							break
 						case "resume_completed_task":
+							playSoundOnMessage("celebration")
 							setTextAreaDisabled(false)
 							setClineAsk("resume_completed_task")
 							setEnableButtons(true)
@@ -441,6 +466,36 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 			return true
 		})
 	}, [modifiedMessages])
+	useEffect(() => {
+		if (isStreaming) {
+			// Set to true once any request has started
+			setHasStarted(true)
+		}
+		// Only execute when isStreaming changes from true to false
+		if (wasStreaming && !isStreaming && lastMessage) {
+			// Play appropriate sound based on lastMessage content
+			if (lastMessage.type === "ask") {
+				switch (lastMessage.ask) {
+					case "api_req_failed":
+					case "mistake_limit_reached":
+						playSound("progress_loop")
+						break
+					case "tool":
+					case "followup":
+					case "browser_action_launch":
+					case "resume_task":
+						playSound("notification")
+						break
+					case "completion_result":
+					case "resume_completed_task":
+						playSound("celebration")
+						break
+				}
+			}
+		}
+		// Update previous value
+		setWasStreaming(isStreaming)
+	}, [isStreaming, lastMessage])
 
 	const isBrowserSessionMessage = (message: ClineMessage): boolean => {
 		// which of visible messages are browser session messages, see above