merge

Author: r2d4

packages/model/src/model/llm.ts

@@ -196,7 +196,7 @@ export class LLMInstanceScope {
   }
 
   async generate(request: GenerateTextRequest, cb: GenerateTextCallback) {
-    const { conversation, maxTokens, assistantRoleName, stopTexts } = request;
+    const { conversation, maxTokens, assistantRoleName, stopTexts, temperature, top_p } = request;
     const tokens = await this.getTokens(conversation, maxTokens);
     tokens.push(...(await this.tokenizer.encodeIds(`${assistantRoleName}:`)));
     console.log("debug: ", await this.tokenizer.decodeIds(tokens));
@@ -222,7 +222,7 @@ export class LLMInstanceScope {
         this.forward(input, this.kvCacheLength + inputTokenLength + step)
       );
       this.tvm.endScope();
-      const nextToken = await this.sampleTokenFromLogits(logits);
+      const nextToken = await this.sampleTokenFromLogits(logits,temperature, top_p);
       logits.dispose();
 
       tokens.push(nextToken);

packages/model/src/types/modelApi.ts

@@ -47,6 +47,8 @@ export type GenerateTextRequest = {
     stopTexts: string[],
     maxTokens: number,
     assistantRoleName: string,
+    temperature: number,
+    top_p: number,
 }
 
 export type GenerateTextResponse = {