diff --git a/core/http/app.go b/core/http/app.go index dcd9a2219958..4ad554f70875 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -128,6 +128,7 @@ func API(application *application.Application) (*fiber.App, error) { router.Use(recover.New()) } + // OpenTelemetry metrics for Prometheus export if !application.ApplicationConfig().DisableMetrics { metricsService, err := services.NewLocalAIMetricsService() if err != nil { @@ -141,6 +142,7 @@ func API(application *application.Application) (*fiber.App, error) { }) } } + // Health Checks should always be exempt from auth, so register these first routes.HealthRoutes(router) @@ -202,12 +204,28 @@ func API(application *application.Application) (*fiber.App, error) { routes.RegisterElevenLabsRoutes(router, requestExtractor, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig()) routes.RegisterLocalAIRoutes(router, requestExtractor, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.GalleryService()) routes.RegisterOpenAIRoutes(router, requestExtractor, application) + if !application.ApplicationConfig().DisableWebUI { + + // Create metrics store for tracking usage (before API routes registration) + metricsStore := services.NewInMemoryMetricsStore() + + // Add metrics middleware BEFORE API routes so it can intercept them + router.Use(middleware.MetricsMiddleware(metricsStore)) + + // Register cleanup on shutdown + router.Hooks().OnShutdown(func() error { + metricsStore.Stop() + log.Info().Msg("Metrics store stopped") + return nil + }) + // Create opcache for tracking UI operations opcache := services.NewOpCache(application.GalleryService()) - routes.RegisterUIAPIRoutes(router, application.ModelConfigLoader(), application.ApplicationConfig(), application.GalleryService(), opcache) + routes.RegisterUIAPIRoutes(router, application.ModelConfigLoader(), application.ApplicationConfig(), application.GalleryService(), opcache, metricsStore) routes.RegisterUIRoutes(router, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.GalleryService()) } + routes.RegisterJINARoutes(router, requestExtractor, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig()) // Define a custom 404 handler diff --git a/core/http/endpoints/localai/settings.go b/core/http/endpoints/localai/settings.go new file mode 100644 index 000000000000..50291f3db20c --- /dev/null +++ b/core/http/endpoints/localai/settings.go @@ -0,0 +1,61 @@ +package localai + +import ( + "github.com/gofiber/fiber/v2" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/http/utils" + "github.com/mudler/LocalAI/core/services" + "github.com/mudler/LocalAI/internal" + "github.com/mudler/LocalAI/pkg/model" +) + +// SettingsEndpoint handles the settings page which shows detailed model/backend management +func SettingsEndpoint(appConfig *config.ApplicationConfig, + cl *config.ModelConfigLoader, ml *model.ModelLoader, opcache *services.OpCache) func(*fiber.Ctx) error { + return func(c *fiber.Ctx) error { + modelConfigs := cl.GetAllModelsConfigs() + galleryConfigs := map[string]*gallery.ModelConfig{} + + installedBackends, err := gallery.ListSystemBackends(appConfig.SystemState) + if err != nil { + return err + } + + for _, m := range modelConfigs { + cfg, err := gallery.GetLocalModelConfiguration(ml.ModelPath, m.Name) + if err != nil { + continue + } + galleryConfigs[m.Name] = cfg + } + + loadedModels := ml.ListLoadedModels() + loadedModelsMap := map[string]bool{} + for _, m := range loadedModels { + loadedModelsMap[m.ID] = true + } + + modelsWithoutConfig, _ := services.ListModels(cl, ml, config.NoFilterFn, services.LOOSE_ONLY) + + // Get model statuses to display in the UI the operation in progress + processingModels, taskTypes := opcache.GetStatus() + + summary := fiber.Map{ + "Title": "LocalAI - Settings & Management", + "Version": internal.PrintableVersion(), + "BaseURL": utils.BaseURL(c), + "Models": modelsWithoutConfig, + "ModelsConfig": modelConfigs, + "GalleryConfig": galleryConfigs, + "ApplicationConfig": appConfig, + "ProcessingModels": processingModels, + "TaskTypes": taskTypes, + "LoadedModels": loadedModelsMap, + "InstalledBackends": installedBackends, + } + + // Render settings page + return c.Render("views/settings", summary) + } +} diff --git a/core/http/middleware/metrics.go b/core/http/middleware/metrics.go new file mode 100644 index 000000000000..6a2f474c29fb --- /dev/null +++ b/core/http/middleware/metrics.go @@ -0,0 +1,174 @@ +package middleware + +import ( + "encoding/json" + "strings" + "time" + + "github.com/gofiber/fiber/v2" + "github.com/mudler/LocalAI/core/services" + "github.com/rs/zerolog/log" +) + +// MetricsMiddleware creates a middleware that tracks API usage metrics +// Note: Uses CONTEXT_LOCALS_KEY_MODEL_NAME constant defined in request.go +func MetricsMiddleware(metricsStore services.MetricsStore) fiber.Handler { + return func(c *fiber.Ctx) error { + path := c.Path() + + // Skip tracking for UI routes, static files, and non-API endpoints + if shouldSkipMetrics(path) { + return c.Next() + } + + // Record start time + start := time.Now() + + // Get endpoint category + endpoint := categorizeEndpoint(path) + + // Continue with the request + err := c.Next() + + // Record metrics after request completes + duration := time.Since(start) + success := err == nil && c.Response().StatusCode() < 400 + + // Extract model name from context (set by RequestExtractor middleware) + // Use the same constant as RequestExtractor + model := "unknown" + if modelVal, ok := c.Locals(CONTEXT_LOCALS_KEY_MODEL_NAME).(string); ok && modelVal != "" { + model = modelVal + log.Debug().Str("model", model).Str("endpoint", endpoint).Msg("Recording metrics for request") + } else { + // Fallback: try to extract from path params or query + model = extractModelFromRequest(c) + log.Debug().Str("model", model).Str("endpoint", endpoint).Msg("Recording metrics for request (fallback)") + } + + // Extract backend from response headers if available + backend := string(c.Response().Header.Peek("X-LocalAI-Backend")) + + // Record the request + metricsStore.RecordRequest(endpoint, model, backend, success, duration) + + return err + } +} + +// shouldSkipMetrics determines if a request should be excluded from metrics +func shouldSkipMetrics(path string) bool { + // Skip UI routes + skipPrefixes := []string{ + "/views/", + "/static/", + "/browse/", + "/chat/", + "/text2image/", + "/tts/", + "/talk/", + "/models/edit/", + "/import-model", + "/settings", + "/api/models", // UI API endpoints + "/api/backends", // UI API endpoints + "/api/operations", // UI API endpoints + "/api/p2p", // UI API endpoints + "/api/metrics", // Metrics API itself + } + + for _, prefix := range skipPrefixes { + if strings.HasPrefix(path, prefix) { + return true + } + } + + // Also skip root path and other UI pages + if path == "/" || path == "/index" { + return true + } + + return false +} + +// categorizeEndpoint maps request paths to friendly endpoint categories +func categorizeEndpoint(path string) string { + // OpenAI-compatible endpoints + if strings.HasPrefix(path, "/v1/chat/completions") || strings.HasPrefix(path, "/chat/completions") { + return "chat" + } + if strings.HasPrefix(path, "/v1/completions") || strings.HasPrefix(path, "/completions") { + return "completions" + } + if strings.HasPrefix(path, "/v1/embeddings") || strings.HasPrefix(path, "/embeddings") { + return "embeddings" + } + if strings.HasPrefix(path, "/v1/images/generations") || strings.HasPrefix(path, "/images/generations") { + return "image-generation" + } + if strings.HasPrefix(path, "/v1/audio/transcriptions") || strings.HasPrefix(path, "/audio/transcriptions") { + return "transcriptions" + } + if strings.HasPrefix(path, "/v1/audio/speech") || strings.HasPrefix(path, "/audio/speech") { + return "text-to-speech" + } + if strings.HasPrefix(path, "/v1/models") || strings.HasPrefix(path, "/models") { + return "models" + } + + // LocalAI-specific endpoints + if strings.HasPrefix(path, "/v1/internal") { + return "internal" + } + if strings.Contains(path, "/tts") { + return "text-to-speech" + } + if strings.Contains(path, "/stt") || strings.Contains(path, "/whisper") { + return "speech-to-text" + } + if strings.Contains(path, "/sound-generation") { + return "sound-generation" + } + + // Default to the first path segment + parts := strings.Split(strings.Trim(path, "/"), "/") + if len(parts) > 0 { + return parts[0] + } + + return "unknown" +} + +// extractModelFromRequest attempts to extract the model name from the request +func extractModelFromRequest(c *fiber.Ctx) string { + // Try query parameter first + model := c.Query("model") + if model != "" { + return model + } + + // Try to extract from JSON body for POST requests + if c.Method() == fiber.MethodPost { + // Read body + bodyBytes := c.Body() + if len(bodyBytes) > 0 { + // Parse JSON + var reqBody map[string]interface{} + if err := json.Unmarshal(bodyBytes, &reqBody); err == nil { + if modelVal, ok := reqBody["model"]; ok { + if modelStr, ok := modelVal.(string); ok { + return modelStr + } + } + } + } + } + + // Try path parameter for endpoints like /models/:model + model = c.Params("model") + if model != "" { + return model + } + + return "unknown" +} diff --git a/core/http/middleware/request.go b/core/http/middleware/request.go index 35f39f7f37f9..1147c11dfdd3 100644 --- a/core/http/middleware/request.go +++ b/core/http/middleware/request.go @@ -127,6 +127,10 @@ func (re *RequestExtractor) SetModelAndConfig(initializer func() schema.LocalAIR log.Debug().Str("context localModelName", localModelName).Msg("overriding empty model name in request body with value found earlier in middleware chain") input.ModelName(&localModelName) } + } else { + // Update context locals with the model name from the request body + // This ensures downstream middleware (like metrics) can access it + ctx.Locals(CONTEXT_LOCALS_KEY_MODEL_NAME, input.ModelName(nil)) } cfg, err := re.modelConfigLoader.LoadModelConfigFileByNameDefaultOptions(input.ModelName(nil), re.applicationConfig) diff --git a/core/http/routes/ui.go b/core/http/routes/ui.go index c781bd88b021..1816a86fcb9c 100644 --- a/core/http/routes/ui.go +++ b/core/http/routes/ui.go @@ -23,6 +23,9 @@ func RegisterUIRoutes(app *fiber.App, app.Get("/", localai.WelcomeEndpoint(appConfig, cl, ml, processingOps)) + // Settings page - detailed model/backend management + app.Get("/settings", localai.SettingsEndpoint(appConfig, cl, ml, processingOps)) + // P2P app.Get("/p2p", func(c *fiber.Ctx) error { summary := fiber.Map{ diff --git a/core/http/routes/ui_api.go b/core/http/routes/ui_api.go index 1fb016825c2b..0ea8f6ca42d4 100644 --- a/core/http/routes/ui_api.go +++ b/core/http/routes/ui_api.go @@ -18,7 +18,7 @@ import ( ) // RegisterUIAPIRoutes registers JSON API routes for the web UI -func RegisterUIAPIRoutes(app *fiber.App, cl *config.ModelConfigLoader, appConfig *config.ApplicationConfig, galleryService *services.GalleryService, opcache *services.OpCache) { +func RegisterUIAPIRoutes(app *fiber.App, cl *config.ModelConfigLoader, appConfig *config.ApplicationConfig, galleryService *services.GalleryService, opcache *services.OpCache, metricsStore services.MetricsStore) { // Operations API - Get all current operations (models + backends) app.Get("/api/operations", func(c *fiber.Ctx) error { @@ -716,4 +716,104 @@ func RegisterUIAPIRoutes(app *fiber.App, cl *config.ModelConfigLoader, appConfig }, }) }) + + // Metrics API endpoints + if metricsStore != nil { + // Get metrics summary + app.Get("/api/metrics/summary", func(c *fiber.Ctx) error { + endpointStats := metricsStore.GetEndpointStats() + modelStats := metricsStore.GetModelStats() + backendStats := metricsStore.GetBackendStats() + + // Get top 5 models + type modelStat struct { + Name string `json:"name"` + Count int64 `json:"count"` + } + topModels := make([]modelStat, 0) + for model, count := range modelStats { + topModels = append(topModels, modelStat{Name: model, Count: count}) + } + sort.Slice(topModels, func(i, j int) bool { + return topModels[i].Count > topModels[j].Count + }) + if len(topModels) > 5 { + topModels = topModels[:5] + } + + // Get top 5 endpoints + type endpointStat struct { + Name string `json:"name"` + Count int64 `json:"count"` + } + topEndpoints := make([]endpointStat, 0) + for endpoint, count := range endpointStats { + topEndpoints = append(topEndpoints, endpointStat{Name: endpoint, Count: count}) + } + sort.Slice(topEndpoints, func(i, j int) bool { + return topEndpoints[i].Count > topEndpoints[j].Count + }) + if len(topEndpoints) > 5 { + topEndpoints = topEndpoints[:5] + } + + return c.JSON(fiber.Map{ + "totalRequests": metricsStore.GetTotalRequests(), + "successRate": metricsStore.GetSuccessRate(), + "topModels": topModels, + "topEndpoints": topEndpoints, + "topBackends": backendStats, + }) + }) + + // Get endpoint statistics + app.Get("/api/metrics/endpoints", func(c *fiber.Ctx) error { + stats := metricsStore.GetEndpointStats() + return c.JSON(fiber.Map{ + "endpoints": stats, + }) + }) + + // Get model statistics + app.Get("/api/metrics/models", func(c *fiber.Ctx) error { + stats := metricsStore.GetModelStats() + return c.JSON(fiber.Map{ + "models": stats, + }) + }) + + // Get backend statistics + app.Get("/api/metrics/backends", func(c *fiber.Ctx) error { + stats := metricsStore.GetBackendStats() + return c.JSON(fiber.Map{ + "backends": stats, + }) + }) + + // Get time series data + app.Get("/api/metrics/timeseries", func(c *fiber.Ctx) error { + // Default to last 24 hours + hours := 24 + if hoursParam := c.Query("hours"); hoursParam != "" { + if h, err := strconv.Atoi(hoursParam); err == nil && h > 0 { + hours = h + } + } + + timeSeries := metricsStore.GetRequestsOverTime(hours) + return c.JSON(fiber.Map{ + "timeseries": timeSeries, + "hours": hours, + }) + }) + + // Reset metrics (optional - for testing/admin purposes) + app.Post("/api/metrics/reset", func(c *fiber.Ctx) error { + metricsStore.Reset() + return c.JSON(fiber.Map{ + "success": true, + "message": "Metrics reset successfully", + }) + }) + } } diff --git a/core/http/views/index.html b/core/http/views/index.html index b92c6f2e90e2..17db109e356a 100644 --- a/core/http/views/index.html +++ b/core/http/views/index.html @@ -3,599 +3,506 @@ {{template "views/partials/head" .}} -
+
{{template "views/partials/navbar" .}} - -
- -
-
- -
+ {{ if eq (len .ModelsConfig) 0 }} + +
-
+
+
+ +

Welcome to your LocalAI

-

The powerful FOSS alternative to OpenAI, Claude, and more

+

Get started by installing your first AI model

-
- - - Documentation - - - - - - Model Gallery +
+ + + Browse Model Gallery - - - - Import Model - + + + Documentation + +
- +
+

Quick Start Guide

+
+
+
+ 1 +
+
+

Browse Models

+

Explore our curated gallery of AI models

+
+
+
+
+ 2 +
+
+

Install Model

+

One-click installation from the gallery

+
+
+
+
+ 3 +
+
+

Start Using

+

Begin chatting or generating content

+
+
+
- - -
- {{template "views/partials/inprogress" .}} + {{ else }} + + + +
+
+
+
+
- {{ if eq (len .ModelsConfig) 0 }} - -
-
-
-
- -
-

No models installed yet

-

Get started by installing models from the gallery or check our documentation for guidance

- -
- - - Browse Gallery - - - - Documentation - +
+
+
+

+ + Dashboard + +

+

Monitor your LocalAI instance

+ + + Settings + +
+
+
- {{ if ne (len .Models) 0 }} -
-

Detected Model Files

-

These models were found but don't have configuration files yet

-
- {{ range .Models }} -
-
- + +
+ + +
+
+ +
+
-
-

{{.}}

-

No configuration

+

{{len .ModelsConfig}}

+

Installed Models

+ + + + +
+
+
+
- {{end}} -
+

{{len .InstalledBackends}}

+

Active Backends

+ + + +
+
+
+
- {{end}}
-
- {{ else }} - - {{ $modelsN := len .ModelsConfig}} - {{ $modelsN = add $modelsN (len .Models)}} -
-
-

- Installed Models -

-

- {{$modelsN}} model{{if gt $modelsN 1}}s{{end}} ready to use -

+

0

+

Total API Requests

+
+

0% success

-
- {{$galleryConfig:=.GalleryConfig}} - {{ $loadedModels := .LoadedModels }} - {{$noicon:="https://upload.wikimedia.org/wikipedia/commons/6/65/No-Image-Placeholder.svg"}} - - {{ range .ModelsConfig }} - {{ $backendCfg := . }} - {{ $cfg:= index $galleryConfig .Name}} -
- -
-
-
- {{.Name}} icon - {{ if index $loadedModels .Name }} -
- {{ end }} -
- -
-
-

{{.Name}}

-
- -
- {{ if .Backend }} - - {{.Backend}} - - {{ else }} - - Auto - - {{ end }} - - {{ if and $backendCfg (or (ne $backendCfg.MCP.Servers "") (ne $backendCfg.MCP.Stdio "")) }} - - MCP - - {{ end }} - - {{ if index $loadedModels .Name }} - - Running - - {{ end }} + +
+
+
+
+

{{ len .LoadedModels }}

+

Currently Loaded

- -
-
- {{ range .KnownUsecaseStrings }} - {{ if eq . "FLAG_CHAT" }} - - - Chat - - {{ end }} - {{ if eq . "FLAG_IMAGE" }} - - - Image - - {{ end }} - {{ if eq . "FLAG_TTS" }} - - - TTS - - {{ end }} - {{ end }} -
- - -
-
- {{ if index $loadedModels .Name }} - - {{ end }} -
- -
- - Edit - - -
+ +
+ +
+

+ + Endpoint Usage +

+
+ +
+
+ +

No API requests yet

- {{ end }} - - - {{ range .Models }} -
-
-
-
- -
-
-

{{.}}

- -
- - Auto Backend - - - No Config -
-
- - - Configuration required for full functionality - -
-
+ +
+

+ + Model Usage +

+
+ +
+
+ +

No model requests yet

- {{end}}
- {{ end }}
- -
-
-

- Installed Backends -

-

- {{len .InstalledBackends}} backend{{if gt (len .InstalledBackends) 1}}s{{end}} ready to use -

+ +
+

+ + Requests Over Time (Last 24 Hours) +

+
+ +
+
+ +

No timeline data yet

+
+
+
- {{ if eq (len .InstalledBackends) 0 }} - -
-
-
-
- + +
+ +
+
+
-

No backends installed yet

-

Backends power your AI models. Install them from the backend gallery to get started

- -
- - - Browse Backend Gallery - - - - Documentation - +
+

Manage

+

Models & Backends

+ + + +
+
+ +
+
+

Gallery

+

Browse Models

- {{ else }} - -
- {{ range .InstalledBackends }} -
- -
-
-
- -
-
-

{{.Name}}

- -
- {{ if .IsSystem }} - - System - - {{ else }} - - User Installed - - {{ end }} - - {{ if .IsMeta }} - - Meta - - {{ end }} -
-
-
+
+ + + +
+
+
- - -
-
- {{ if and .Metadata .Metadata.Alias }} -
- -
- Alias: - {{.Metadata.Alias}} -
-
- {{ end }} - - {{ if and .Metadata .Metadata.InstalledAt }} -
- -
- Installed: - {{.Metadata.InstalledAt}} -
-
- {{ end }} - - {{ if and .Metadata .Metadata.MetaBackendFor }} -
- -
- Meta backend for: - {{.Metadata.MetaBackendFor}} -
-
- {{ end }} - - {{ if and .Metadata .Metadata.GalleryURL }} -
- -
- Gallery: - - {{.Metadata.GalleryURL}} - - -
-
- {{ end }} - -
- -
- Path: - {{.RunFile}} -
-
-
- - - {{ if not .IsSystem }} -
- -
- {{ end }} +
+

Docs

+

Documentation

+
+
+ + + +
+
+
+
+

API

+

Reference

- {{end}}
- {{ end }} +
+ + {{ end }}
{{template "views/partials/footer" .}}
+ + + diff --git a/core/http/views/partials/navbar.html b/core/http/views/partials/navbar.html index 91f754886198..45c3346c1d3b 100644 --- a/core/http/views/partials/navbar.html +++ b/core/http/views/partials/navbar.html @@ -22,6 +22,9 @@ Home + + Settings + Models @@ -55,6 +58,9 @@ Home + + Settings + Models diff --git a/core/http/views/settings.html b/core/http/views/settings.html new file mode 100644 index 000000000000..f6ee793560e3 --- /dev/null +++ b/core/http/views/settings.html @@ -0,0 +1,609 @@ + + +{{template "views/partials/head" .}} + + +
+ + {{template "views/partials/navbar" .}} + + +
+ +
+ +
+ +
+ +
+
+
+
+ +
+

+ + Settings & Management + +

+

Manage your models, backends, and system configuration

+ +
+ + + Back to Dashboard + + + + + + Documentation + + + + + + Model Gallery + + + + + + Import Model + + + + +
+
+
+ + +
+ {{template "views/partials/inprogress" .}} + + {{ if eq (len .ModelsConfig) 0 }} + +
+
+
+
+ +
+

No models installed yet

+

Get started by installing models from the gallery or check our documentation for guidance

+ +
+ + + Browse Gallery + + + + Documentation + +
+ + {{ if ne (len .Models) 0 }} +
+

Detected Model Files

+

These models were found but don't have configuration files yet

+
+ {{ range .Models }} +
+
+ +
+
+

{{.}}

+

No configuration

+
+
+ {{end}} +
+
+ {{end}} +
+
+ {{ else }} + + {{ $modelsN := len .ModelsConfig}} + {{ $modelsN = add $modelsN (len .Models)}} +
+
+

+ Installed Models +

+

+ {{$modelsN}} model{{if gt $modelsN 1}}s{{end}} ready to use +

+
+
+ +
+ {{$galleryConfig:=.GalleryConfig}} + {{ $loadedModels := .LoadedModels }} + {{$noicon:="https://upload.wikimedia.org/wikipedia/commons/6/65/No-Image-Placeholder.svg"}} + + {{ range .ModelsConfig }} + {{ $backendCfg := . }} + {{ $cfg:= index $galleryConfig .Name}} +
+ +
+
+
+ {{.Name}} icon + {{ if index $loadedModels .Name }} +
+ {{ end }} +
+ +
+
+

{{.Name}}

+
+ +
+ {{ if .Backend }} + + {{.Backend}} + + {{ else }} + + Auto + + {{ end }} + + {{ if and $backendCfg (or (ne $backendCfg.MCP.Servers "") (ne $backendCfg.MCP.Stdio "")) }} + + MCP + + {{ end }} + + {{ if index $loadedModels .Name }} + + Running + + {{ end }} +
+
+
+
+ + +
+
+ {{ range .KnownUsecaseStrings }} + {{ if eq . "FLAG_CHAT" }} + + + Chat + + {{ end }} + {{ if eq . "FLAG_IMAGE" }} + + + Image + + {{ end }} + {{ if eq . "FLAG_TTS" }} + + + TTS + + {{ end }} + {{ end }} +
+ + +
+
+ {{ if index $loadedModels .Name }} + + {{ end }} +
+ +
+ + Edit + + +
+
+
+
+ {{ end }} + + + {{ range .Models }} +
+
+
+
+ +
+
+

{{.}}

+ +
+ + Auto Backend + + + No Config + +
+ +
+ + + Configuration required for full functionality + +
+
+
+
+
+ {{end}} +
+ {{ end }} +
+ + +
+
+

+ Installed Backends +

+

+ {{len .InstalledBackends}} backend{{if gt (len .InstalledBackends) 1}}s{{end}} ready to use +

+
+ + {{ if eq (len .InstalledBackends) 0 }} + +
+
+
+
+ +
+

No backends installed yet

+

Backends power your AI models. Install them from the backend gallery to get started

+ +
+ + + Browse Backend Gallery + + + + Documentation + +
+
+
+ {{ else }} + +
+ {{ range .InstalledBackends }} +
+ +
+
+
+ +
+
+

{{.Name}}

+ +
+ {{ if .IsSystem }} + + System + + {{ else }} + + User Installed + + {{ end }} + + {{ if .IsMeta }} + + Meta + + {{ end }} +
+
+
+
+ + +
+
+ {{ if and .Metadata .Metadata.Alias }} +
+ +
+ Alias: + {{.Metadata.Alias}} +
+
+ {{ end }} + + {{ if and .Metadata .Metadata.InstalledAt }} +
+ +
+ Installed: + {{.Metadata.InstalledAt}} +
+
+ {{ end }} + + {{ if and .Metadata .Metadata.MetaBackendFor }} +
+ +
+ Meta backend for: + {{.Metadata.MetaBackendFor}} +
+
+ {{ end }} + + {{ if and .Metadata .Metadata.GalleryURL }} +
+ +
+ Gallery: + + {{.Metadata.GalleryURL}} + + +
+
+ {{ end }} + +
+ +
+ Path: + {{.RunFile}} +
+
+
+ + + {{ if not .IsSystem }} +
+ +
+ {{ end }} +
+
+ {{end}} +
+ {{ end }} +
+
+ + {{template "views/partials/footer" .}} +
+ + + + + diff --git a/core/services/metrics.go b/core/services/metrics.go index 68ecf4776b67..51d9e1c1e4f5 100644 --- a/core/services/metrics.go +++ b/core/services/metrics.go @@ -2,6 +2,8 @@ package services import ( "context" + "sync" + "time" "github.com/rs/zerolog/log" "go.opentelemetry.io/otel/attribute" @@ -10,6 +12,315 @@ import ( metricApi "go.opentelemetry.io/otel/sdk/metric" ) +// MetricsStore is the interface for storing and retrieving metrics +// This allows for future implementations with persistence (JSON files, databases, etc.) +type MetricsStore interface { + RecordRequest(endpoint, model, backend string, success bool, duration time.Duration) + GetEndpointStats() map[string]int64 + GetModelStats() map[string]int64 + GetBackendStats() map[string]int64 + GetRequestsOverTime(hours int) []TimeSeriesPoint + GetTotalRequests() int64 + GetSuccessRate() float64 + Reset() +} + +// TimeSeriesPoint represents a single point in the time series +type TimeSeriesPoint struct { + Timestamp time.Time `json:"timestamp"` + Count int64 `json:"count"` +} + +// RequestRecord stores individual request information +type RequestRecord struct { + Timestamp time.Time + Endpoint string + Model string + Backend string + Success bool + Duration time.Duration +} + +// InMemoryMetricsStore implements MetricsStore with in-memory storage +type InMemoryMetricsStore struct { + endpoints map[string]int64 + models map[string]int64 + backends map[string]int64 + timeSeries []RequestRecord + successCount int64 + failureCount int64 + mu sync.RWMutex + stopChan chan struct{} + maxRecords int // Maximum number of time series records to keep + maxMapKeys int // Maximum number of unique keys per map + pruneEvery time.Duration // How often to prune old data +} + +// NewInMemoryMetricsStore creates a new in-memory metrics store +func NewInMemoryMetricsStore() *InMemoryMetricsStore { + store := &InMemoryMetricsStore{ + endpoints: make(map[string]int64), + models: make(map[string]int64), + backends: make(map[string]int64), + timeSeries: make([]RequestRecord, 0), + stopChan: make(chan struct{}), + maxRecords: 10000, // Limit to 10k records (~1-2MB of memory) + maxMapKeys: 1000, // Limit to 1000 unique keys per map (~50KB per map) + pruneEvery: 5 * time.Minute, // Prune every 5 minutes instead of every request + } + + // Start background pruning goroutine + go store.pruneLoop() + + return store +} + +// pruneLoop runs periodically to clean up old data +func (m *InMemoryMetricsStore) pruneLoop() { + ticker := time.NewTicker(m.pruneEvery) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + m.pruneOldData() + case <-m.stopChan: + return + } + } +} + +// pruneOldData removes data older than 24 hours and enforces max record limit +func (m *InMemoryMetricsStore) pruneOldData() { + m.mu.Lock() + defer m.mu.Unlock() + + cutoff := time.Now().Add(-24 * time.Hour) + newTimeSeries := make([]RequestRecord, 0, len(m.timeSeries)) + + for _, r := range m.timeSeries { + if r.Timestamp.After(cutoff) { + newTimeSeries = append(newTimeSeries, r) + } + } + + // If still over the limit, keep only the most recent records + if len(newTimeSeries) > m.maxRecords { + // Keep the most recent maxRecords entries + newTimeSeries = newTimeSeries[len(newTimeSeries)-m.maxRecords:] + log.Warn(). + Int("dropped", len(m.timeSeries)-len(newTimeSeries)). + Int("kept", len(newTimeSeries)). + Msg("Metrics store exceeded maximum records, dropping oldest entries") + } + + m.timeSeries = newTimeSeries + + // Also check if maps have grown too large + m.pruneMapIfNeeded("endpoints", m.endpoints, m.maxMapKeys) + m.pruneMapIfNeeded("models", m.models, m.maxMapKeys) + m.pruneMapIfNeeded("backends", m.backends, m.maxMapKeys) +} + +// pruneMapIfNeeded keeps only the top N entries in a map by count +func (m *InMemoryMetricsStore) pruneMapIfNeeded(name string, mapData map[string]int64, maxKeys int) { + if len(mapData) <= maxKeys { + return + } + + // Convert to slice for sorting + type kv struct { + key string + value int64 + } + + entries := make([]kv, 0, len(mapData)) + for k, v := range mapData { + entries = append(entries, kv{k, v}) + } + + // Sort by value descending (keep highest counts) + for i := 0; i < len(entries); i++ { + for j := i + 1; j < len(entries); j++ { + if entries[i].value < entries[j].value { + entries[i], entries[j] = entries[j], entries[i] + } + } + } + + // Keep only top maxKeys entries + for k := range mapData { + delete(mapData, k) + } + + for i := 0; i < maxKeys && i < len(entries); i++ { + mapData[entries[i].key] = entries[i].value + } + + log.Warn(). + Str("map", name). + Int("dropped", len(entries)-maxKeys). + Int("kept", maxKeys). + Msg("Metrics map exceeded maximum keys, keeping only top entries") +} + +// Stop gracefully shuts down the metrics store +func (m *InMemoryMetricsStore) Stop() { + close(m.stopChan) +} + +// RecordRequest records a new API request +func (m *InMemoryMetricsStore) RecordRequest(endpoint, model, backend string, success bool, duration time.Duration) { + m.mu.Lock() + defer m.mu.Unlock() + + // Record endpoint + if endpoint != "" { + m.endpoints[endpoint]++ + } + + // Record model + if model != "" { + m.models[model]++ + } + + // Record backend + if backend != "" { + m.backends[backend]++ + } + + // Record success/failure + if success { + m.successCount++ + } else { + m.failureCount++ + } + + // Add to time series + record := RequestRecord{ + Timestamp: time.Now(), + Endpoint: endpoint, + Model: model, + Backend: backend, + Success: success, + Duration: duration, + } + m.timeSeries = append(m.timeSeries, record) + + // Note: Pruning is done periodically by pruneLoop() to avoid overhead on every request +} + +// GetEndpointStats returns request counts per endpoint +func (m *InMemoryMetricsStore) GetEndpointStats() map[string]int64 { + m.mu.RLock() + defer m.mu.RUnlock() + + result := make(map[string]int64) + for k, v := range m.endpoints { + result[k] = v + } + return result +} + +// GetModelStats returns request counts per model +func (m *InMemoryMetricsStore) GetModelStats() map[string]int64 { + m.mu.RLock() + defer m.mu.RUnlock() + + result := make(map[string]int64) + for k, v := range m.models { + result[k] = v + } + return result +} + +// GetBackendStats returns request counts per backend +func (m *InMemoryMetricsStore) GetBackendStats() map[string]int64 { + m.mu.RLock() + defer m.mu.RUnlock() + + result := make(map[string]int64) + for k, v := range m.backends { + result[k] = v + } + return result +} + +// GetRequestsOverTime returns time series data for the specified number of hours +func (m *InMemoryMetricsStore) GetRequestsOverTime(hours int) []TimeSeriesPoint { + m.mu.RLock() + defer m.mu.RUnlock() + + cutoff := time.Now().Add(-time.Duration(hours) * time.Hour) + + // Group by hour + hourlyBuckets := make(map[int64]int64) + for _, record := range m.timeSeries { + if record.Timestamp.After(cutoff) { + // Round down to the hour + hourTimestamp := record.Timestamp.Truncate(time.Hour).Unix() + hourlyBuckets[hourTimestamp]++ + } + } + + // Convert to sorted time series + result := make([]TimeSeriesPoint, 0) + for ts, count := range hourlyBuckets { + result = append(result, TimeSeriesPoint{ + Timestamp: time.Unix(ts, 0), + Count: count, + }) + } + + // Sort by timestamp + for i := 0; i < len(result); i++ { + for j := i + 1; j < len(result); j++ { + if result[i].Timestamp.After(result[j].Timestamp) { + result[i], result[j] = result[j], result[i] + } + } + } + + return result +} + +// GetTotalRequests returns the total number of requests recorded +func (m *InMemoryMetricsStore) GetTotalRequests() int64 { + m.mu.RLock() + defer m.mu.RUnlock() + + return m.successCount + m.failureCount +} + +// GetSuccessRate returns the percentage of successful requests +func (m *InMemoryMetricsStore) GetSuccessRate() float64 { + m.mu.RLock() + defer m.mu.RUnlock() + + total := m.successCount + m.failureCount + if total == 0 { + return 0.0 + } + return float64(m.successCount) / float64(total) * 100.0 +} + +// Reset clears all metrics +func (m *InMemoryMetricsStore) Reset() { + m.mu.Lock() + defer m.mu.Unlock() + + m.endpoints = make(map[string]int64) + m.models = make(map[string]int64) + m.backends = make(map[string]int64) + m.timeSeries = make([]RequestRecord, 0) + m.successCount = 0 + m.failureCount = 0 +} + +// ============================================================================ +// OpenTelemetry Metrics Service (for Prometheus export) +// ============================================================================ + type LocalAIMetricsService struct { Meter metric.Meter ApiTimeMetric metric.Float64Histogram @@ -23,7 +334,7 @@ func (m *LocalAIMetricsService) ObserveAPICall(method string, path string, durat m.ApiTimeMetric.Record(context.Background(), duration, opts) } -// setupOTelSDK bootstraps the OpenTelemetry pipeline. +// NewLocalAIMetricsService bootstraps the OpenTelemetry pipeline for Prometheus export. // If it does not return an error, make sure to call shutdown for proper cleanup. func NewLocalAIMetricsService() (*LocalAIMetricsService, error) { exporter, err := prometheus.New()