feat: switch default tokenizer to o200k_base for better Gemini compat…

…ibility

- Changed default tokenizer to o200k_base for better compatibility with Gemini models

- Added configurable tokenizer support through tokenCount.encoding in config file

- Updated documentation to reflect new tokenizer configuration options

Fixes #1

cursor-tools.config.json

@@ -4,5 +4,8 @@
   },
   "gemini": {
     "model": "gemini-2.0-flash-thinking-exp-01-21"
+  },
+  "tokenCount": {
+    "encoding": "o200k_base"
   }
 }

src/commands/doc.ts

@@ -98,6 +98,9 @@ export class DocCommand implements Command {
               includePatterns: includePatterns.join(','),
               ignorePatterns: ignorePatterns.join(','),
             },
+            tokenCount: {
+              encoding: this.config.tokenCount?.encoding || 'o200k_base',
+            },
             signal: {},
           }),
           method: 'POST',
@@ -293,7 +296,7 @@ Focus on:
             enableSecurityCheck: true,
           },
           tokenCount: {
-            encoding: 'cl100k_base',
+            encoding: this.config.tokenCount?.encoding || 'o200k_base',
           },
           cwd: process.cwd(),
         });

src/commands/repo.ts

@@ -81,7 +81,7 @@ export class RepoCommand implements Command {
           enableSecurityCheck: true,
         },
         tokenCount: {
-          encoding: 'cl100k_base',
+          encoding: this.config.tokenCount?.encoding || 'o200k_base',
         },
         cwd: process.cwd(),
       });

src/config.ts

@@ -12,6 +12,9 @@ export interface Config {
   doc?: {
     maxRepoSizeMB?: number; // Maximum repository size in MB for remote processing
   };
+  tokenCount?: {
+    encoding: 'o200k_base' | 'gpt2' | 'r50k_base' | 'p50k_base' | 'p50k_edit' | 'cl100k_base'; // The tokenizer encoding to use
+  };
 }
 
 export const defaultConfig: Config = {
@@ -26,6 +29,9 @@ export const defaultConfig: Config = {
   doc: {
     maxRepoSizeMB: 100, // Default to 100MB
   },
+  tokenCount: {
+    encoding: 'o200k_base', // Default to o200k_base as it's optimized for Gemini
+  },
 };
 
 import { existsSync, readFileSync } from 'node:fs';