Enhance LLM configuration handling and settings UI (#3366)

* Improve LLM settings & vertexai bugfix
* Update test_system_settings_resource test
* Update ollama.py to check for required settings, update ollama default config
* Simplify SettingsDialog.vue

---------

Co-authored-by: Janosch <[email protected]>

Author: itsmvd

data/timesketch.conf

@@ -389,8 +389,8 @@ LLM_PROVIDER_CONFIGS = {
     },
     'default': {
         'ollama': {
-            'server_url': 'http://ollama:11434',
-            'model': 'gemma:7b',
+            'server_url': '',
+            'model': '',
         },
     }
 }

timesketch/api/v1/resources/settings.py

@@ -12,11 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """System settings."""
-
 import logging
+from typing import Any
+
 from flask import current_app, jsonify
 from flask_restful import Resource
 from flask_login import login_required
+from timesketch.lib.llms.providers import manager as llm_manager
 
 logger = logging.getLogger("timesketch.system_settings")
 
@@ -31,42 +33,89 @@ def get(self):
         Returns:
             JSON object with system settings.
         """
-        # Settings from timesketch.conf to expose to the frontend clients.
-        settings_to_return = ["DFIQ_ENABLED"]
-        result = {}
-
-        for setting in settings_to_return:
-            result[setting] = current_app.config.get(setting)
-
-        # Derive the default LLM provider from the new configuration.
-        # Expecting the "default" config to be a dict with exactly one key:
-        # the provider name.
-        llm_configs = current_app.config.get("LLM_PROVIDER_CONFIGS", {})
-        default_provider = None
-        default_conf = llm_configs.get("default")
-        if default_conf and isinstance(default_conf, dict) and len(default_conf) == 1:
-            default_provider = next(iter(default_conf))
-        result["LLM_PROVIDER"] = default_provider
-
-        # TODO(mvd): Remove by 2025/06/01 once all users have updated their config.
-        old_llm_provider = current_app.config.get("LLM_PROVIDER")
+        result = {
+            "DFIQ_ENABLED": current_app.config.get("DFIQ_ENABLED", False),
+            "SEARCH_PROCESSING_TIMELINES": current_app.config.get(
+                "SEARCH_PROCESSING_TIMELINES", False
+            ),
+            "LLM_FEATURES_AVAILABLE": self._get_llm_features_availability(
+                current_app.config.get("LLM_PROVIDER_CONFIGS", {})
+            ),
+        }
+
+        return jsonify(result)
+
+    def _get_llm_features_availability(
+        self, llm_configs: dict[str, Any]
+    ) -> dict[str, bool]:
+        """Get availability status for all LLM features.
+
+        Args:
+            llm_configs: LLM provider configuration dictionary
+
+        Returns:
+            dict mapping feature names to availability status (bool)
+        """
+        default_provider_working = False
+
+        if not isinstance(llm_configs, dict):
+            logger.debug(
+                "LLM_PROVIDER_CONFIGS is not a dictionary: %s",
+                type(llm_configs).__name__,
+            )
+            return {"default": default_provider_working}
+
         if (
-            old_llm_provider and "default" not in llm_configs
-        ):  # Basic check for old config
-            warning_message = (
-                "Your LLM configuration in timesketch.conf is outdated and may cause "
-                "issues with LLM features. "
-                "Please update your LLM_PROVIDER_CONFIGS section to the new format. "
-                "Refer to the documentation for the updated configuration structure."
+            "default" in llm_configs
+            and isinstance(llm_configs["default"], dict)
+            and len(llm_configs["default"]) == 1
+        ):
+            default_provider = next(iter(llm_configs["default"]))
+            default_provider_config = llm_configs["default"][default_provider]
+            default_provider_working = self._check_provider_working(
+                default_provider, default_provider_config
             )
-            result["llm_config_warning"] = warning_message
-            logger.warning(warning_message)
 
-        # Get the search processing timelines setting, default is False if not set.
-        # if set to True, the search processing timelines will be displayed in the UI.
-        search_processing_timelines = current_app.config.get(
-            "SEARCH_PROCESSING_TIMELINES", False
-        )
-        result["SEARCH_PROCESSING_TIMELINES"] = search_processing_timelines
+        llm_feature_availability = {"default": default_provider_working}
 
-        return jsonify(result)
+        for feature_name, feature_conf in llm_configs.items():
+            if feature_name == "default":
+                continue
+
+            feature_provider_working = False
+            if isinstance(feature_conf, dict) and len(feature_conf) == 1:
+                feature_provider = next(iter(feature_conf))
+                feature_provider_config = feature_conf[feature_provider]
+                feature_provider_working = self._check_provider_working(
+                    feature_provider, feature_provider_config
+                )
+
+            # Feature is available if either specific provider works
+            # or default provider works
+            llm_feature_availability[feature_name] = (
+                feature_provider_working or default_provider_working
+            )
+
+        return llm_feature_availability
+
+    def _check_provider_working(self, provider_name: str, config: dict) -> bool:
+        """Check if a specific LLM provider works with given configuration.
+
+        Args:
+            provider_name: Name of the provider to check
+            config: Configuration dict for the provider
+
+        Returns:
+            bool: Whether the provider is working
+        """
+        try:
+            provider_class = llm_manager.LLMManager.get_provider(provider_name)
+            provider_class(config=config)
+            return True
+        except Exception as e:  # pylint: disable=broad-except
+            logger.debug(
+                "LLM provider '%s' failed to initialize: %s",
+                provider_name,
+                str(e),
+            )
+            return False

timesketch/api/v1/resources_test.py

@@ -1368,15 +1368,28 @@ def test_system_settings_resource(self):
         """Authenticated request to get system settings."""
         self.app.config["LLM_PROVIDER_CONFIGS"] = {"default": {"test": {}}}
         self.app.config["DFIQ_ENABLED"] = False
+        self.login()
+        response = self.client.get(self.resource_url)
+
+        self.assertEqual(response.json["DFIQ_ENABLED"], False)
+        self.assertEqual(response.json["SEARCH_PROCESSING_TIMELINES"], False)
 
+        self.assertIn("LLM_FEATURES_AVAILABLE", response.json)
+        self.assertIn("default", response.json["LLM_FEATURES_AVAILABLE"])
+
+    def test_system_settings_invalid_llm_config(self):
+        """Test with invalid LLM configuration."""
+        self.app.config["LLM_PROVIDER_CONFIGS"] = "invalid_config"
         self.login()
         response = self.client.get(self.resource_url)
+
         expected_response = {
             "DFIQ_ENABLED": False,
-            "LLM_PROVIDER": "test",
             "SEARCH_PROCESSING_TIMELINES": False,
+            "LLM_FEATURES_AVAILABLE": {"default": False},
         }
-        self.assertEqual(response.json, expected_response)
+
+        self.assertDictEqual(response.json, expected_response)
 
 
 class ScenariosResourceTest(BaseTest):

timesketch/frontend-ng/src/components/SettingsDialog.vue

@@ -1,12 +1,9 @@
 <!--
 Copyright 2025 Google Inc. All rights reserved.
-
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
     http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,7 +17,11 @@ limitations under the License.
       <v-subheader>Layout</v-subheader>
       <v-list-item>
         <v-list-item-action>
-          <v-switch v-model="settings.showLeftPanel" color="primary" @change="saveSettings()"></v-switch>
+          <v-switch
+            v-model="settings.showLeftPanel"
+            color="primary"
+            @change="saveSettings()"
+          ></v-switch>
         </v-list-item-action>
         <v-list-item-content>
           <v-list-item-title>Show side panel</v-list-item-title>
@@ -31,9 +32,21 @@ limitations under the License.
       </v-list-item>
 
       <!-- AI Powered Features Main Setting -->
-      <v-list-item v-if="systemSettings.LLM_PROVIDER">
+      <v-list-item>
         <v-list-item-action>
-          <v-switch v-model="settings.aiPoweredFeaturesMain" color="primary" @change="updateAiFeatures" ></v-switch>
+          <v-tooltip bottom>
+            <template v-slot:activator="{ on, attrs }">
+              <div v-on="isAnyFeatureAvailable ? {} : on" v-bind="attrs">
+                <v-switch
+                  v-model="settings.aiPoweredFeaturesMain"
+                  color="primary"
+                  @change="updateAiFeatures"
+                  :disabled="!isAnyFeatureAvailable"
+                ></v-switch>
+              </div>
+            </template>
+            <span>This feature requires an LLM provider to be configured. Please contact your administrator.</span>
+          </v-tooltip>
         </v-list-item-action>
         <v-list-item-content>
           <v-list-item-title>AI powered features (experimental)</v-list-item-title>
@@ -42,14 +55,21 @@ limitations under the License.
       </v-list-item>
 
       <!-- Child Setting: Event Summarization -->
-      <v-list-item v-if="systemSettings.LLM_PROVIDER">
+      <v-list-item>
         <v-list-item-action class="ml-8">
-          <v-switch
-            v-model="settings.eventSummarization"
-            color="primary"
-            @change="saveSettings()"
-            :disabled="!settings.aiPoweredFeaturesMain"
-          ></v-switch>
+          <v-tooltip bottom>
+            <template v-slot:activator="{ on, attrs }">
+              <div v-on="isFeatureAvailable('llm_summarize') ? {} : on" v-bind="attrs">
+                <v-switch
+                  v-model="settings.eventSummarization"
+                  color="primary"
+                  @change="saveSettings()"
+                  :disabled="!settings.aiPoweredFeaturesMain || !isFeatureAvailable('llm_summarize')"
+                ></v-switch>
+              </div>
+            </template>
+            <span>Event summarization requires an LLM provider to be configured. Please contact your administrator.</span>
+          </v-tooltip>
         </v-list-item-action>
         <v-list-item-content class="ml-8">
           <v-list-item-title>Event summarization</v-list-item-title>
@@ -60,14 +80,21 @@ limitations under the License.
       </v-list-item>
 
       <!-- Child Setting: AI Generated Queries -->
-      <v-list-item v-if="systemSettings.LLM_PROVIDER">
+      <v-list-item>
         <v-list-item-action class="ml-8">
-          <v-switch
-            v-model="settings.generateQuery"
-            color="primary"
-            @change="saveSettings()"
-            :disabled="!settings.aiPoweredFeaturesMain"
-          ></v-switch>
+          <v-tooltip bottom>
+            <template v-slot:activator="{ on, attrs }">
+              <div v-on="isFeatureAvailable('nl2q') ? {} : on" v-bind="attrs">
+                <v-switch
+                  v-model="settings.generateQuery"
+                  color="primary"
+                  @change="saveSettings()"
+                  :disabled="!settings.aiPoweredFeaturesMain || !isFeatureAvailable('nl2q')"
+                ></v-switch>
+              </div>
+            </template>
+            <span>AI query generation requires an LLM provider to be configured. Please contact your administrator.</span>
+          </v-tooltip>
         </v-list-item-action>
         <v-list-item-content class="ml-8">
           <v-list-item-title>AI generated queries</v-list-item-title>
@@ -76,9 +103,15 @@ limitations under the License.
           >
         </v-list-item-content>
       </v-list-item>
+
+      <!-- Setting: Searching processing timelines -->
       <v-list-item v-if="systemSettings.SEARCH_PROCESSING_TIMELINES">
         <v-list-item-action>
-          <v-switch v-model="settings.showProcessingTimelineEvents" color="primary" @change="saveSettings()"></v-switch>
+          <v-switch
+            v-model="settings.showProcessingTimelineEvents"
+            color="primary"
+            @change="saveSettings()"
+          ></v-switch>
         </v-list-item-action>
         <v-list-item-content>
           <v-list-item-title>Include Processing Events</v-list-item-title>
@@ -90,18 +123,15 @@ limitations under the License.
     </v-list>
   </v-card>
 </template>
-
 <script>
 import ApiClient from '../utils/RestApiClient'
-
 const DEFAULT_SETTINGS = {
   showLeftPanel: true,
   aiPoweredFeaturesMain: false,
   eventSummarization: false,
   generateQuery: false,
   showProcessingTimelineEvents: false,
 }
-
 export default {
   data() {
     return {
@@ -124,37 +154,41 @@ export default {
     userSettings() {
       return this.$store.state.settings
     },
+    llmFeatures() {
+      return this.systemSettings.LLM_FEATURES_AVAILABLE || {}
+    },
+    isAnyFeatureAvailable() {
+      return Object.values(this.llmFeatures).some(available => available === true)
+    },
   },
   methods: {
     saveSettings() {
       ApiClient.saveUserSettings(this.settings)
-        .then(() => this.$store.dispatch('updateUserSettings'))
+        .then(() => {
+          return this.$store.dispatch('updateUserSettings')
+        })
+        .then(() => {
+          this.settings = { ...this.userSettings }
+        })
         .catch((error) => {
           console.log(error)
         })
     },
     updateAiFeatures() {
       if (!this.settings.aiPoweredFeaturesMain) {
-        this.settings.eventSummarization = false;
-        this.settings.generateQuery = false;
+        this.settings.eventSummarization = false
+        this.settings.generateQuery = false
       }
-      this.saveSettings();
+      this.saveSettings()
+    },
+    isFeatureAvailable(featureName) {
+      return this.llmFeatures[featureName] === true
     },
   },
   mounted() {
-    this.settings = { ...this.userSettings }
-
-    // Set default values when a user don't have any settings saved.
-    if (!this.settings || !Object.keys(this.settings).length) {
-      this.settings = { ...DEFAULT_SETTINGS }
-      this.saveSettings()
-    } else {
-      // Ensure default values for new settings are applied if user settings are older
-      this.settings.aiPoweredFeaturesMain = this.settings.aiPoweredFeaturesMain !== undefined ? this.settings.aiPoweredFeaturesMain : DEFAULT_SETTINGS.aiPoweredFeaturesMain;
-      this.settings.eventSummarization = this.settings.eventSummarization !== undefined ? this.settings.eventSummarization : DEFAULT_SETTINGS.eventSummarization;
-      this.settings.generateQuery = this.settings.generateQuery !== undefined ? this.settings.generateQuery : DEFAULT_SETTINGS.generateQuery;
-      this.settings.showProcessingTimelineEvents = this.settings.showProcessingTimelineEvents !== undefined ? this.settings.showProcessingTimelineEvents : DEFAULT_SETTINGS.showProcessingTimelineEvents;
-    }
+    // Set default settings if no user settings are defined.
+    this.settings = { ...DEFAULT_SETTINGS, ...this.userSettings };
+    this.saveSettings();
   },
 }
 </script>