GithubProxy/coai
1
0
Fork 0
mirror of https://github.com/coaidev/coai.git synced 2025-05-21 14:00:13 +09:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix tokenizer

Browse Source
This commit is contained in:
Zhang Minghan 2023-11-09 10:42:09 +08:00
parent 38dad633ee
commit b6968bb12b
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
utils/tokenizer.go
View File

@ -2,7 +2,6 @@ package utils
import ( import (
"chat/globals" "chat/globals"
"fmt"
"github.com/pkoukk/tiktoken-go" "github.com/pkoukk/tiktoken-go"
"strings" "strings"
) )
@ -23,7 +22,8 @@ func GetWeightByModel(model string) int {
globals.GPT432k, globals.GPT432k0613, globals.GPT432k0314, globals.GPT432k, globals.GPT432k0613, globals.GPT432k0314,
globals.SparkDesk, globals.SparkDeskV2, globals.SparkDeskV3, globals.SparkDesk, globals.SparkDeskV2, globals.SparkDeskV3,
globals.QwenTurbo, globals.QwenPlus, globals.QwenTurboNet, globals.QwenPlusNet: globals.QwenTurbo, globals.QwenPlus, globals.QwenTurboNet, globals.QwenPlusNet,
globals.BingPrecise, globals.BingCreative, globals.BingBalanced:
return 3 return 3
case globals.GPT3Turbo0301, globals.GPT3Turbo16k0301, case globals.GPT3Turbo0301, globals.GPT3Turbo16k0301,
globals.ZhiPuChatGLMLite, globals.ZhiPuChatGLMStd, globals.ZhiPuChatGLMPro: globals.ZhiPuChatGLMLite, globals.ZhiPuChatGLMStd, globals.ZhiPuChatGLMPro:
@ -43,7 +43,8 @@ func GetWeightByModel(model string) int {
return GetWeightByModel(globals.Claude2100k) return GetWeightByModel(globals.Claude2100k)
} else { } else {
// not implemented: See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens // not implemented: See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens
panic(fmt.Errorf("not implemented for model %s", model)) //panic(fmt.Errorf("not implemented for model %s", model))
return 3
} }
} }
} }