From ad1197a37525e1667446f3e242d834097aabddbc Mon Sep 17 00:00:00 2001 From: scillidan Date: Tue, 16 Jun 2026 17:37:20 +0800 Subject: [PATCH 1/2] feat: add llama.cpp provider support --- README.md | 25 ++++++++- src/commands/config.ts | 10 +++- src/commands/setup.ts | 48 ++++++++++++++++- src/engine/llamacpp.ts | 52 +++++++++++++++++++ src/utils/engine.ts | 4 ++ src/utils/modelCache.ts | 21 ++++++++ test/unit/llamacpp.test.ts | 102 +++++++++++++++++++++++++++++++++++++ 7 files changed, 256 insertions(+), 6 deletions(-) create mode 100644 src/engine/llamacpp.ts create mode 100644 test/unit/llamacpp.test.ts diff --git a/README.md b/README.md index d2cbe375..19249584 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,25 @@ export OLLAMA_HOST=0.0.0.0 This will make Ollama listen on all interfaces, including IPv6 and IPv4, resolving the connection issue. You can add this line to your shell configuration file (like `.bashrc` or `.zshrc`) to make it persistent across sessions. +### Running locally with llama.cpp + +You can run OpenCommit with local models served by [llama.cpp](https://github.com/ggerganov/llama.cpp): + +- install and build llama.cpp +- start the server with a GGUF model: `./llama-server -m model.gguf --port 8080` +- run (in your project directory): + +```sh +git add +oco config set OCO_AI_PROVIDER='llamacpp' OCO_API_URL='http://localhost:8080' +``` + +If the server is running on a different machine, set the URL accordingly: + +```sh +oco config set OCO_API_URL='http://192.168.1.10:8080' +``` + ### Flags There are multiple optional flags that can be used with the `oco` command: @@ -122,7 +141,7 @@ Create a `.env` file and add OpenCommit config variables there like this: ```env ... -OCO_AI_PROVIDER= +OCO_AI_PROVIDER= OCO_API_KEY= // or other LLM provider API token OCO_API_URL= OCO_API_CUSTOM_HEADERS= @@ -227,7 +246,7 @@ oco models --provider anthropic By default OpenCommit uses [OpenAI](https://openai.com). -You could switch to [Azure OpenAI Service](https://learn.microsoft.com/azure/cognitive-services/openai/) or Flowise or Ollama. +You could switch to [Azure OpenAI Service](https://learn.microsoft.com/azure/cognitive-services/openai/) or Flowise or Ollama or llama.cpp. ```sh oco config set OCO_AI_PROVIDER=azure OCO_API_KEY= OCO_API_URL= @@ -235,6 +254,8 @@ oco config set OCO_AI_PROVIDER=azure OCO_API_KEY= OCO_API_UR oco config set OCO_AI_PROVIDER=flowise OCO_API_KEY= OCO_API_URL= oco config set OCO_AI_PROVIDER=ollama OCO_API_KEY= OCO_API_URL= + +oco config set OCO_AI_PROVIDER=llamacpp OCO_API_URL= ``` ### Use with Proxy diff --git a/src/commands/config.ts b/src/commands/config.ts index 9f490093..62739b70 100644 --- a/src/commands/config.ts +++ b/src/commands/config.ts @@ -579,6 +579,8 @@ const getDefaultModel = (provider: string | undefined): string => { switch (provider) { case 'ollama': return ''; + case 'llamacpp': + return ''; case 'mlx': return ''; case 'anthropic': @@ -797,8 +799,10 @@ export const configValidators = { 'deepseek', 'aimlapi', 'openrouter' - ].includes(value) || value.startsWith('ollama'), - `${value} is not supported yet, use 'ollama', 'mlx', 'anthropic', 'azure', 'gemini', 'flowise', 'mistral', 'deepseek', 'aimlapi' or 'openai' (default)` + ].includes(value) || + value.startsWith('ollama') || + value.startsWith('llamacpp'), + `${value} is not supported yet, use 'ollama', 'llamacpp', 'mlx', 'anthropic', 'azure', 'gemini', 'flowise', 'mistral', 'deepseek', 'aimlapi' or 'openai' (default)` ); return value; @@ -854,6 +858,7 @@ export const configValidators = { export enum OCO_AI_PROVIDER_ENUM { OLLAMA = 'ollama', + LLAMACPP = 'llamacpp', OPENAI = 'openai', ANTHROPIC = 'anthropic', GEMINI = 'gemini', @@ -880,6 +885,7 @@ export const PROVIDER_API_KEY_URLS: Record = { [OCO_AI_PROVIDER_ENUM.AIMLAPI]: 'https://aimlapi.com/app/keys', [OCO_AI_PROVIDER_ENUM.AZURE]: 'https://portal.azure.com/', [OCO_AI_PROVIDER_ENUM.OLLAMA]: null, + [OCO_AI_PROVIDER_ENUM.LLAMACPP]: null, [OCO_AI_PROVIDER_ENUM.MLX]: null, [OCO_AI_PROVIDER_ENUM.FLOWISE]: null, [OCO_AI_PROVIDER_ENUM.TEST]: null diff --git a/src/commands/setup.ts b/src/commands/setup.ts index 28d40295..dda5e6c1 100644 --- a/src/commands/setup.ts +++ b/src/commands/setup.ts @@ -24,6 +24,7 @@ const PROVIDER_DISPLAY_NAMES: Record = { [OCO_AI_PROVIDER_ENUM.OPENAI]: 'OpenAI (GPT-4o, GPT-4)', [OCO_AI_PROVIDER_ENUM.ANTHROPIC]: 'Anthropic (Claude Sonnet, Opus)', [OCO_AI_PROVIDER_ENUM.OLLAMA]: 'Ollama (Free, runs locally)', + [OCO_AI_PROVIDER_ENUM.LLAMACPP]: 'llama.cpp (Free, runs locally)', [OCO_AI_PROVIDER_ENUM.GEMINI]: 'Google Gemini', [OCO_AI_PROVIDER_ENUM.GROQ]: 'Groq (Fast inference, free tier)', [OCO_AI_PROVIDER_ENUM.MISTRAL]: 'Mistral AI', @@ -37,7 +38,8 @@ const PROVIDER_DISPLAY_NAMES: Record = { const PRIMARY_PROVIDERS = [ OCO_AI_PROVIDER_ENUM.OPENAI, OCO_AI_PROVIDER_ENUM.ANTHROPIC, - OCO_AI_PROVIDER_ENUM.OLLAMA + OCO_AI_PROVIDER_ENUM.OLLAMA, + OCO_AI_PROVIDER_ENUM.LLAMACPP ]; const OTHER_PROVIDERS = [ @@ -53,13 +55,15 @@ const OTHER_PROVIDERS = [ const NO_API_KEY_PROVIDERS = [ OCO_AI_PROVIDER_ENUM.OLLAMA, + OCO_AI_PROVIDER_ENUM.LLAMACPP, OCO_AI_PROVIDER_ENUM.MLX, OCO_AI_PROVIDER_ENUM.TEST ]; const MODEL_REQUIRED_PROVIDERS = [ OCO_AI_PROVIDER_ENUM.OLLAMA, - OCO_AI_PROVIDER_ENUM.MLX + OCO_AI_PROVIDER_ENUM.MLX, + OCO_AI_PROVIDER_ENUM.LLAMACPP ]; async function selectProvider(): Promise { @@ -335,6 +339,33 @@ async function setupOllama(): Promise<{ }; } +async function setupLlamaCpp(): Promise<{ + provider: string; + model: string; + apiUrl: string; +} | null> { + console.log(chalk.cyan('\n llama.cpp - Free Local AI\n')); + console.log(chalk.dim(' Setup steps:')); + console.log(chalk.dim(' 1. Install: https://github.com/ggerganov/llama.cpp')); + console.log(chalk.dim(' 2. Start server: llama-server -m --port 8080\n')); + + const defaultUrl = 'http://localhost:8080'; + + const apiUrl = await text({ + message: 'llama.cpp server URL (press Enter for default):', + placeholder: defaultUrl, + defaultValue: defaultUrl + }); + + if (isCancel(apiUrl)) return null; + + return { + provider: OCO_AI_PROVIDER_ENUM.LLAMACPP, + model: '', + apiUrl: (apiUrl as string) || defaultUrl + }; +} + export async function runSetup(): Promise { intro(chalk.bgCyan(' Welcome to OpenCommit! ')); @@ -382,6 +413,19 @@ export async function runSetup(): Promise { OCO_MODEL: model, OCO_API_KEY: 'mlx' // Placeholder }; + } else if (provider === OCO_AI_PROVIDER_ENUM.LLAMACPP) { + const llamacppConfig = await setupLlamaCpp(); + if (!llamacppConfig) { + outro('Setup cancelled'); + return false; + } + + config = { + OCO_AI_PROVIDER: llamacppConfig.provider, + OCO_MODEL: llamacppConfig.model, + OCO_API_URL: llamacppConfig.apiUrl, + OCO_API_KEY: 'llamacpp' // Placeholder + }; } else { // Standard provider flow: API key then model const apiKey = await getApiKey(provider as string); diff --git a/src/engine/llamacpp.ts b/src/engine/llamacpp.ts new file mode 100644 index 00000000..08745dca --- /dev/null +++ b/src/engine/llamacpp.ts @@ -0,0 +1,52 @@ +import axios, { AxiosInstance } from 'axios'; +import { OpenAI } from 'openai'; +import { normalizeEngineError } from '../utils/engineErrorHandler'; +import { removeContentTags } from '../utils/removeContentTags'; +import { AiEngine, AiEngineConfig } from './Engine'; + +interface LlamaCppConfig extends AiEngineConfig {} + +const DEFAULT_LLAMACPP_URL = 'http://localhost:8080'; +const LLAMACPP_CHAT_PATH = '/v1/chat/completions'; + +export class LlamaCppEngine implements AiEngine { + config: LlamaCppConfig; + client: AxiosInstance; + private chatUrl: string; + + constructor(config: LlamaCppConfig) { + this.config = config; + + const baseUrl = config.baseURL || DEFAULT_LLAMACPP_URL; + this.chatUrl = `${baseUrl}${LLAMACPP_CHAT_PATH}`; + + const headers = { + 'Content-Type': 'application/json', + ...config.customHeaders + }; + + this.client = axios.create({ headers }); + } + + async generateCommitMessage( + messages: Array + ): Promise { + const params: Record = { + model: this.config.model ?? '', + messages, + temperature: 0, + top_p: 0.1, + repeat_penalty: 1.1, + stream: false + }; + try { + const response = await this.client.post(this.chatUrl, params); + + const choices = response.data.choices; + const message = choices[0].message; + return removeContentTags(message?.content, 'think'); + } catch (error) { + throw normalizeEngineError(error, 'llamacpp', this.config.model); + } + } +} diff --git a/src/utils/engine.ts b/src/utils/engine.ts index 0a07fe4c..af358edb 100644 --- a/src/utils/engine.ts +++ b/src/utils/engine.ts @@ -4,6 +4,7 @@ import { AzureEngine } from '../engine/azure'; import { AiEngine } from '../engine/Engine'; import { FlowiseEngine } from '../engine/flowise'; import { GeminiEngine } from '../engine/gemini'; +import { LlamaCppEngine } from '../engine/llamacpp'; import { OllamaEngine } from '../engine/ollama'; import { OpenAiEngine } from '../engine/openAi'; import { MistralAiEngine } from '../engine/mistral'; @@ -40,6 +41,9 @@ export function getEngine(): AiEngine { ollamaThink: config.OCO_OLLAMA_THINK }); + case OCO_AI_PROVIDER_ENUM.LLAMACPP: + return new LlamaCppEngine(DEFAULT_CONFIG); + case OCO_AI_PROVIDER_ENUM.ANTHROPIC: return new AnthropicEngine(DEFAULT_CONFIG); diff --git a/src/utils/modelCache.ts b/src/utils/modelCache.ts index 76ffc214..afe0834e 100644 --- a/src/utils/modelCache.ts +++ b/src/utils/modelCache.ts @@ -87,6 +87,23 @@ export async function fetchOllamaModels( } } +export async function fetchLlamaCppModels( + baseUrl: string = 'http://localhost:8080' +): Promise { + try { + const response = await fetch(`${baseUrl}/v1/models`); + + if (!response.ok) { + return []; + } + + const data = await response.json(); + return data.data?.map((m: { id: string }) => m.id) || []; + } catch { + return []; + } +} + export async function fetchAnthropicModels(apiKey: string): Promise { try { const response = await fetch('https://api.anthropic.com/v1/models', { @@ -231,6 +248,10 @@ export async function fetchModelsForProvider( models = await fetchOllamaModels(baseUrl); break; + case OCO_AI_PROVIDER_ENUM.LLAMACPP: + models = await fetchLlamaCppModels(baseUrl); + break; + case OCO_AI_PROVIDER_ENUM.ANTHROPIC: if (apiKey) { models = await fetchAnthropicModels(apiKey); diff --git a/test/unit/llamacpp.test.ts b/test/unit/llamacpp.test.ts new file mode 100644 index 00000000..3f0df0aa --- /dev/null +++ b/test/unit/llamacpp.test.ts @@ -0,0 +1,102 @@ +import { LlamaCppEngine } from '../../src/engine/llamacpp'; + +describe('LlamaCppEngine', () => { + it('sends request to /v1/chat/completions', async () => { + const engine = new LlamaCppEngine({ + apiKey: 'llamacpp', + model: 'llama-3', + maxTokensOutput: 500, + maxTokensInput: 4096 + }); + + const post = jest.fn().mockResolvedValue({ + data: { + choices: [ + { + message: { + content: 'feat: add support for llama.cpp provider' + } + } + ] + } + }); + + engine.client = { post } as any; + + await engine.generateCommitMessage([ + { role: 'user', content: 'diff --git a/file b/file' } + ]); + + expect(post).toHaveBeenCalledWith( + 'http://localhost:8080/v1/chat/completions', + expect.objectContaining({ + temperature: 0, + top_p: 0.1, + repeat_penalty: 1.1, + stream: false + }) + ); + }); + + it('uses custom baseURL when provided', async () => { + const engine = new LlamaCppEngine({ + apiKey: 'llamacpp', + model: 'llama-3', + maxTokensOutput: 500, + maxTokensInput: 4096, + baseURL: 'http://192.168.1.10:8080' + }); + + const post = jest.fn().mockResolvedValue({ + data: { + choices: [ + { + message: { + content: 'fix: resolve connection issue' + } + } + ] + } + }); + + engine.client = { post } as any; + + await engine.generateCommitMessage([ + { role: 'user', content: 'diff --git a/file b/file' } + ]); + + expect(post).toHaveBeenCalledWith( + 'http://192.168.1.10:8080/v1/chat/completions', + expect.anything() + ); + }); + + it('strips tags from response content', async () => { + const engine = new LlamaCppEngine({ + apiKey: 'llamacpp', + model: 'llama-3', + maxTokensOutput: 500, + maxTokensInput: 4096 + }); + + const post = jest.fn().mockResolvedValue({ + data: { + choices: [ + { + message: { + content: 'reasoning herefeat: add support for llama.cpp provider' + } + } + ] + } + }); + + engine.client = { post } as any; + + const result = await engine.generateCommitMessage([ + { role: 'user', content: 'diff --git a/file b/file' } + ]); + + expect(result).toBe('feat: add support for llama.cpp provider'); + }); +}); From 95f256f66f3914b864cf6aeb93b7da878f17d2dc Mon Sep 17 00:00:00 2001 From: scillidan Date: Wed, 17 Jun 2026 09:55:33 +0800 Subject: [PATCH 2/2] llama.cpp: fix code formatting with Prettier --- src/commands/setup.ts | 8 ++++++-- test/unit/llamacpp.test.ts | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/commands/setup.ts b/src/commands/setup.ts index dda5e6c1..a4507193 100644 --- a/src/commands/setup.ts +++ b/src/commands/setup.ts @@ -346,8 +346,12 @@ async function setupLlamaCpp(): Promise<{ } | null> { console.log(chalk.cyan('\n llama.cpp - Free Local AI\n')); console.log(chalk.dim(' Setup steps:')); - console.log(chalk.dim(' 1. Install: https://github.com/ggerganov/llama.cpp')); - console.log(chalk.dim(' 2. Start server: llama-server -m --port 8080\n')); + console.log( + chalk.dim(' 1. Install: https://github.com/ggerganov/llama.cpp') + ); + console.log( + chalk.dim(' 2. Start server: llama-server -m --port 8080\n') + ); const defaultUrl = 'http://localhost:8080'; diff --git a/test/unit/llamacpp.test.ts b/test/unit/llamacpp.test.ts index 3f0df0aa..7a1ad338 100644 --- a/test/unit/llamacpp.test.ts +++ b/test/unit/llamacpp.test.ts @@ -84,7 +84,8 @@ describe('LlamaCppEngine', () => { choices: [ { message: { - content: 'reasoning herefeat: add support for llama.cpp provider' + content: + 'reasoning herefeat: add support for llama.cpp provider' } } ]