Skip to content

Commit 43079c7

Browse files
authored
[Obs AI Assistant] Remove prompts from recall ranking Telemetry event (elastic#214573)
Closes elastic/obs-ai-assistant-team#224 ## Summary Relates to elastic/obs-ai-assistant-team#186 Removes the prompts from the recall ranking telemetry event. ### Checklist - [x] The PR description includes the appropriate Release Notes section, and the correct `release_note:*` label is applied per the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
1 parent 508c89f commit 43079c7

File tree

3 files changed

+17
-21
lines changed

3 files changed

+17
-21
lines changed

x-pack/platform/packages/shared/kbn-ai-assistant/src/chat/chat_body.tsx

+17-5
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import {
3838
import type { AuthenticatedUser } from '@kbn/security-plugin/common';
3939
import { findLastIndex } from 'lodash';
4040
import React, { useCallback, useEffect, useRef, useState } from 'react';
41+
import { ChatFeedback } from '@kbn/observability-ai-assistant-plugin/public/analytics/schemas/chat_feedback';
4142
import type { UseKnowledgeBaseResult } from '../hooks/use_knowledge_base';
4243
import { ASSISTANT_SETUP_TITLE, EMPTY_CONVERSATION_TITLE, UPGRADE_LICENSE_TITLE } from '../i18n';
4344
import { useAIAssistantChatService } from '../hooks/use_ai_assistant_chat_service';
@@ -220,13 +221,24 @@ export function ChatBody({
220221
if (conversation.value?.conversation && 'user' in conversation.value) {
221222
const {
222223
messages: _removedMessages, // Exclude messages
223-
conversation: { title: _removedTitle, ...conversationRest }, // Exclude title
224-
...rest
224+
systemMessage: _removedSystemMessage, // Exclude systemMessage
225+
conversation: { title: _removedTitle, id, last_updated: lastUpdated }, // Exclude title
226+
user,
227+
labels,
228+
numeric_labels: numericLabels,
229+
namespace,
230+
public: isPublic,
231+
'@timestamp': timestamp,
225232
} = conversation.value;
226233

227-
const conversationWithoutMessagesAndTitle = {
228-
...rest,
229-
conversation: conversationRest,
234+
const conversationWithoutMessagesAndTitle: ChatFeedback['conversation'] = {
235+
'@timestamp': timestamp,
236+
user,
237+
labels,
238+
numeric_labels: numericLabels,
239+
namespace,
240+
public: isPublic,
241+
conversation: { id, last_updated: lastUpdated },
230242
};
231243

232244
chatService.sendAnalyticsEvent({

x-pack/platform/plugins/shared/observability_ai_assistant/server/analytics/recall_ranking.ts

-14
Original file line numberDiff line numberDiff line change
@@ -8,33 +8,19 @@
88
import { RootSchema, EventTypeOpts } from '@kbn/core/server';
99

1010
interface ScoredDocument {
11-
content: string;
1211
elserScore: number;
1312
llmScore: number;
1413
}
1514

1615
export interface RecallRanking {
17-
prompt: string;
1816
scoredDocuments: ScoredDocument[];
1917
}
2018

2119
const schema: RootSchema<RecallRanking> = {
22-
prompt: {
23-
type: 'text',
24-
_meta: {
25-
description: 'The user prompt that was used for the ELSER text_expansion',
26-
},
27-
},
2820
scoredDocuments: {
2921
type: 'array',
3022
items: {
3123
properties: {
32-
content: {
33-
type: 'text',
34-
_meta: {
35-
description: 'The raw content of the recalled document',
36-
},
37-
},
3824
elserScore: {
3925
type: 'float',
4026
_meta: {

x-pack/platform/plugins/shared/observability_ai_assistant/server/utils/recall/recall_and_score.ts

-2
Original file line numberDiff line numberDiff line change
@@ -68,11 +68,9 @@ export async function recallAndScore({
6868
});
6969

7070
analytics.reportEvent<RecallRanking>(recallRankingEventType, {
71-
prompt: queries.map((query) => query.text).join('\n\n'),
7271
scoredDocuments: suggestions.map((suggestion) => {
7372
const llmScore = scores.find((score) => score.id === suggestion.id);
7473
return {
75-
content: suggestion.text,
7674
elserScore: suggestion.score ?? -1,
7775
llmScore: llmScore ? llmScore.score : -1,
7876
};

0 commit comments

Comments
 (0)