Merge pull request #2 from lxfater/feature/add_temperature_and_topP

Add temperature, topP to GenerateTextRequest

Author: r2d4

packages/headless/src/types/worker_message.ts

@@ -15,6 +15,9 @@ export type GenerateTextRequest = {
     stopTexts: string[],
     maxTokens: number,
     assistantRoleName: string,
+    temperature: number,
+    top_p: number,
+
 }
 
 export type GenerateTextResponse = {

packages/headless/src/worker/llm.ts

@@ -198,7 +198,7 @@ export class LLMInstanceScope {
   }
 
   async generate(request: GenerateTextRequest, cb: GenerateTextCallback) {
-    const { conversation, maxTokens, assistantRoleName, stopTexts } = request;
+    const { conversation, maxTokens, assistantRoleName, stopTexts, temperature, top_p } = request;
     const tokens = await this.getTokens(conversation, maxTokens);
     tokens.push(...(await this.tokenizer.encodeIds(`${assistantRoleName}:`)));
     console.log("debug: ", await this.tokenizer.decodeIds(tokens));
@@ -224,7 +224,7 @@ export class LLMInstanceScope {
         this.forward(input, this.kvCacheLength + inputTokenLength + step)
       );
       this.tvm.endScope();
-      const nextToken = await this.sampleTokenFromLogits(logits);
+      const nextToken = await this.sampleTokenFromLogits(logits,temperature, top_p);
       logits.dispose();
 
       tokens.push(nextToken);