From baa4754cf498b3954726768e985b70f40c49b32c Mon Sep 17 00:00:00 2001 From: Zhang Minghan Date: Wed, 1 Nov 2023 15:34:08 +0800 Subject: [PATCH] update gpt-4-vison and 32k model --- README.md | 4 +- adapter/chatgpt/chat.go | 79 +++++++++------------- adapter/chatgpt/processor.go | 123 +++++++++++++++++++++++++++++++++++ adapter/chatgpt/struct.go | 3 +- adapter/chatgpt/types.go | 28 ++++++++ adapter/request.go | 15 +++++ addition/web/utils.go | 4 +- app/src/conf.ts | 14 ++-- auth/rule.go | 2 +- globals/variables.go | 58 +++++++++-------- utils/tokenizer.go | 5 +- 11 files changed, 245 insertions(+), 90 deletions(-) create mode 100644 adapter/chatgpt/processor.go diff --git a/README.md b/README.md index a12fc45..b032eec 100644 --- a/README.md +++ b/README.md @@ -61,9 +61,10 @@ - GPT-3.5-Turbo (_0613_, _0301_) - GPT-3.5-Turbo-16k (_0613_, _0301_) - GPT-3.5-Reverse (_text-davincci-002-render-sha_, _text-davincci-002-render-paid_) + - GPT-3.5-Turbo-Instruct - GPT-4 (_0314_, _0613_) - GPT-4-32k (_0314_, _0613_) - - GPT-4-Reverse (_gpt-4_) + - GPT-4-Reverse (_gpt-4_, _**gpt-4v**_) - DALL-E - Claude - Slack-Claude (unstable) @@ -72,6 +73,7 @@ - SparkDesk 讯飞星火 - v1.5 - v2.0 + - v3.0 - Google PaLM2 - Chat - Text diff --git a/adapter/chatgpt/chat.go b/adapter/chatgpt/chat.go index 0131077..bebb755 100644 --- a/adapter/chatgpt/chat.go +++ b/adapter/chatgpt/chat.go @@ -4,7 +4,6 @@ import "C" import ( "chat/globals" "chat/utils" - "errors" "fmt" "github.com/spf13/viper" "strings" @@ -16,11 +15,36 @@ type ChatProps struct { Token int } -func (c *ChatInstance) GetChatEndpoint() string { +func (c *ChatInstance) GetChatEndpoint(props *ChatProps) string { + if props.Model == globals.GPT3TurboInstruct { + return fmt.Sprintf("%s/v1/completions", c.GetEndpoint()) + } return fmt.Sprintf("%s/v1/chat/completions", c.GetEndpoint()) } +func (c *ChatInstance) GetCompletionPrompt(messages []globals.Message) string { + result := "" + for _, message := range messages { + result += fmt.Sprintf("%s: %s\n", message.Role, message.Content) + } + return result +} + func (c *ChatInstance) GetChatBody(props *ChatProps, stream bool) interface{} { + if props.Model == globals.GPT3TurboInstruct { + // for completions + return utils.Multi[interface{}](props.Token != -1, CompletionRequest{ + Model: props.Model, + Prompt: c.GetCompletionPrompt(props.Message), + MaxToken: props.Token, + Stream: stream, + }, CompletionWithInfinity{ + Model: props.Model, + Prompt: c.GetCompletionPrompt(props.Message), + Stream: stream, + }) + } + if props.Token != -1 { return ChatRequest{ Model: props.Model, @@ -37,54 +61,10 @@ func (c *ChatInstance) GetChatBody(props *ChatProps, stream bool) interface{} { } } -func (c *ChatInstance) ProcessLine(buf, data string) (string, error) { - rep := strings.NewReplacer( - "data: {", - "\"data\": {", - ) - item := rep.Replace(data) - if !strings.HasPrefix(item, "{") { - item = "{" + item - } - if !strings.HasSuffix(item, "}}") { - item = item + "}" - } - - if item == "{data: [DONE]}" || item == "{data: [DONE]}}" || item == "{[DONE]}" { - return "", nil - } else if item == "{data:}" || item == "{data:}}" { - return "", nil - } - - var form *ChatStreamResponse - if form = utils.UnmarshalForm[ChatStreamResponse](item); form == nil { - if form = utils.UnmarshalForm[ChatStreamResponse](item[:len(item)-1]); form == nil { - if len(buf) > 0 { - return c.ProcessLine("", buf+item) - } - - var err *ChatStreamErrorResponse - if err = utils.UnmarshalForm[ChatStreamErrorResponse](item); err == nil { - if err = utils.UnmarshalForm[ChatStreamErrorResponse](item + "}"); err == nil { - globals.Warn(fmt.Sprintf("chatgpt error: cannot parse response: %s", item)) - return data, errors.New("parser error: cannot parse response") - } - } - return "", fmt.Errorf("chatgpt error: %s (type: %s)", err.Data.Error.Message, err.Data.Error.Type) - } - } - - if len(form.Data.Choices) == 0 { - return "", nil - } - - return form.Data.Choices[0].Delta.Content, nil -} - // CreateChatRequest is the native http request body for chatgpt func (c *ChatInstance) CreateChatRequest(props *ChatProps) (string, error) { res, err := utils.Post( - c.GetChatEndpoint(), + c.GetChatEndpoint(props), c.GetHeader(), c.GetChatBody(props, false), ) @@ -105,14 +85,15 @@ func (c *ChatInstance) CreateChatRequest(props *ChatProps) (string, error) { // CreateStreamChatRequest is the stream response body for chatgpt func (c *ChatInstance) CreateStreamChatRequest(props *ChatProps, callback globals.Hook) error { buf := "" + instruct := props.Model == globals.GPT3TurboInstruct return utils.EventSource( "POST", - c.GetChatEndpoint(), + c.GetChatEndpoint(props), c.GetHeader(), c.GetChatBody(props, true), func(data string) error { - data, err := c.ProcessLine(buf, data) + data, err := c.ProcessLine(instruct, buf, data) if err != nil { if strings.HasPrefix(err.Error(), "chatgpt error") { diff --git a/adapter/chatgpt/processor.go b/adapter/chatgpt/processor.go new file mode 100644 index 0000000..370237f --- /dev/null +++ b/adapter/chatgpt/processor.go @@ -0,0 +1,123 @@ +package chatgpt + +import ( + "chat/globals" + "chat/utils" + "errors" + "fmt" + "strings" +) + +func processFormat(data string) string { + rep := strings.NewReplacer( + "data: {", + "\"data\": {", + ) + item := rep.Replace(data) + if !strings.HasPrefix(item, "{") { + item = "{" + item + } + if !strings.HasSuffix(item, "}}") { + item = item + "}" + } + + return item +} + +func processChatResponse(data string) *ChatStreamResponse { + if strings.HasPrefix(data, "{") { + var form *ChatStreamResponse + if form = utils.UnmarshalForm[ChatStreamResponse](data); form != nil { + return form + } + + if form = utils.UnmarshalForm[ChatStreamResponse](data[:len(data)-1]); form != nil { + return form + } + } + + return nil +} + +func processCompletionResponse(data string) *CompletionResponse { + if strings.HasPrefix(data, "{") { + var form *CompletionResponse + if form = utils.UnmarshalForm[CompletionResponse](data); form != nil { + return form + } + + if form = utils.UnmarshalForm[CompletionResponse](data[:len(data)-1]); form != nil { + return form + } + } + + return nil +} + +func processChatErrorResponse(data string) *ChatStreamErrorResponse { + if strings.HasPrefix(data, "{") { + var form *ChatStreamErrorResponse + if form = utils.UnmarshalForm[ChatStreamErrorResponse](data); form != nil { + return form + } + if form = utils.UnmarshalForm[ChatStreamErrorResponse](data + "}"); form != nil { + return form + } + } + + return nil +} + +func isDone(data string) bool { + return utils.Contains[string](data, []string{ + "{data: [DONE]}", "{data: [DONE]}}", + "{[DONE]}", "{data:}", "{data:}}", + }) +} + +func getChoices(form *ChatStreamResponse) string { + if len(form.Data.Choices) == 0 { + return "" + } + + return form.Data.Choices[0].Delta.Content +} + +func getCompletionChoices(form *CompletionResponse) string { + if len(form.Data.Choices) == 0 { + return "" + } + + return form.Data.Choices[0].Text +} + +func (c *ChatInstance) ProcessLine(instruct bool, buf, data string) (string, error) { + item := processFormat(buf + data) + if isDone(item) { + return "", nil + } + + if form := processChatResponse(item); form == nil { + if instruct { + // legacy support + if completion := processCompletionResponse(item); completion != nil { + return getCompletionChoices(completion), nil + } + } + + // recursive call + if len(buf) > 0 { + return c.ProcessLine(instruct, "", buf+item) + } + + if err := processChatErrorResponse(item); err == nil { + globals.Warn(fmt.Sprintf("chatgpt error: cannot parse response: %s", item)) + return data, errors.New("parser error: cannot parse response") + } else { + return "", fmt.Errorf("chatgpt error: %s (type: %s)", err.Data.Error.Message, err.Data.Error.Type) + } + + } else { + return getChoices(form), nil + } +} diff --git a/adapter/chatgpt/struct.go b/adapter/chatgpt/struct.go index 0d6f17d..f8bc704 100644 --- a/adapter/chatgpt/struct.go +++ b/adapter/chatgpt/struct.go @@ -57,9 +57,10 @@ func NewChatInstanceFromModel(props *InstanceProps) *ChatInstance { case globals.GPT432k, globals.GPT432k0613, globals.GPT432k0314: - return NewChatInstanceFromConfig("gpt4") + return NewChatInstanceFromConfig("32k") case globals.GPT3Turbo, + globals.GPT3TurboInstruct, globals.GPT3Turbo0613, globals.GPT3Turbo0301, globals.GPT3Turbo16k, diff --git a/adapter/chatgpt/types.go b/adapter/chatgpt/types.go index a7dda83..09af0c4 100644 --- a/adapter/chatgpt/types.go +++ b/adapter/chatgpt/types.go @@ -16,6 +16,20 @@ type ChatRequestWithInfinity struct { Stream bool `json:"stream"` } +// CompletionRequest ChatRequest is the request body for chatgpt completion +type CompletionRequest struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + MaxToken int `json:"max_tokens"` + Stream bool `json:"stream"` +} + +type CompletionWithInfinity struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + Stream bool `json:"stream"` +} + // ChatResponse is the native http request body for chatgpt type ChatResponse struct { ID string `json:"id"` @@ -48,6 +62,20 @@ type ChatStreamResponse struct { } `json:"data"` } +// CompletionResponse is the native http request body / stream response body for chatgpt completion +type CompletionResponse struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + Model string `json:"model"` + Data struct { + Choices []struct { + Text string `json:"text"` + Index int `json:"index"` + } `json:"choices"` + } `json:"data"` +} + type ChatStreamErrorResponse struct { Data struct { Error struct { diff --git a/adapter/request.go b/adapter/request.go index bbb7ab6..023465d 100644 --- a/adapter/request.go +++ b/adapter/request.go @@ -32,6 +32,21 @@ func retryChatGPTPool(props *ChatProps, hook globals.Hook, retry int) error { ), }, hook) + if globals.IsGPT4NativeModel(props.Model) && IsAvailableError(err) { + if !strings.Contains(err.Error(), "429") { + // not rate limited + return chatgpt.NewChatInstanceFromConfig("32k").CreateStreamChatRequest(&chatgpt.ChatProps{ + Model: props.Model, + Message: props.Message, + Token: utils.Multi( + props.Token == 0, + utils.Multi(globals.IsGPT4Model(props.Model) || props.Plan || props.Infinity, -1, 2500), + props.Token, + ), + }, hook) + } + } + if IsAvailableError(err) && retry < MaxRetries { fmt.Println(fmt.Sprintf("retrying chatgpt pool (times: %d, error: %s)", retry, err.Error())) return retryChatGPTPool(props, hook, retry+1) diff --git a/addition/web/utils.go b/addition/web/utils.go index 0350ca6..0fdc63c 100644 --- a/addition/web/utils.go +++ b/addition/web/utils.go @@ -13,7 +13,7 @@ func UsingWebSegment(instance *conversation.Conversation) (string, []globals.Mes if instance.IsEnableWeb() { keyword, segment = ChatWithWeb(func(message []globals.Message, token int) (string, error) { return chatgpt.NewChatInstanceFromConfig("gpt3").CreateChatRequest(&chatgpt.ChatProps{ - Model: globals.GPT3Turbo0613, + Model: globals.GPT3TurboInstruct, Message: message, Token: token, }) @@ -28,7 +28,7 @@ func UsingWebNativeSegment(enable bool, message []globals.Message) (string, []gl if enable { return ChatWithWeb(func(message []globals.Message, token int) (string, error) { return chatgpt.NewChatInstanceFromConfig("gpt3").CreateChatRequest(&chatgpt.ChatProps{ - Model: globals.GPT3Turbo0613, + Model: globals.GPT3TurboInstruct, Message: message, Token: token, }) diff --git a/app/src/conf.ts b/app/src/conf.ts index b8f772c..c42cb84 100644 --- a/app/src/conf.ts +++ b/app/src/conf.ts @@ -22,14 +22,14 @@ export const supportModels: Model[] = [ { id: "gpt-4", name: "GPT-4", free: false, auth: true }, { id: "gpt-4-32k", name: "GPT-4-32k", free: false, auth: true }, + // spark desk + { id: "spark-desk-v2", name: "讯飞星火 V2", free: false, auth: true }, + { id: "spark-desk-v3", name: "讯飞星火 V3", free: false, auth: true }, + // anthropic models { id: "claude-1", name: "Claude-2", free: true, auth: false }, { id: "claude-2", name: "Claude-2-100k", free: false, auth: true }, // not claude-2-100k - // spark desk - { id: "spark-desk-v2", name: "SparkDesk 讯飞星火", free: false, auth: true }, - { id: "spark-desk-v3", name: "SparkDesk 讯飞星火 V3", free: false, auth: true }, - // google palm2 { id: "chat-bison-001", name: "Palm2", free: true, auth: true }, @@ -39,19 +39,19 @@ export const supportModels: Model[] = [ // zhipu models { id: "zhipu-chatglm-pro", - name: "智谱 ChatGLM Pro", + name: "ChatGLM Pro", free: false, auth: true, }, { id: "zhipu-chatglm-std", - name: "智谱 ChatGLM Std", + name: "ChatGLM Std", free: false, auth: true, }, { id: "zhipu-chatglm-lite", - name: "智谱 ChatGLM Lite", + name: "ChatGLM Lite", free: true, auth: true, }, diff --git a/auth/rule.go b/auth/rule.go index 236506f..590256d 100644 --- a/auth/rule.go +++ b/auth/rule.go @@ -9,7 +9,7 @@ import ( // CanEnableModel returns whether the model can be enabled (without subscription) func CanEnableModel(db *sql.DB, user *User, model string) bool { switch model { - case globals.GPT3Turbo, globals.GPT3Turbo0301, globals.GPT3Turbo0613, + case globals.GPT3Turbo, globals.GPT3TurboInstruct, globals.GPT3Turbo0301, globals.GPT3Turbo0613, globals.Claude2: return true case globals.GPT4, globals.GPT4Vision, globals.GPT40613, globals.GPT40314: diff --git a/globals/variables.go b/globals/variables.go index c27eda3..9e0dbe4 100644 --- a/globals/variables.go +++ b/globals/variables.go @@ -38,37 +38,39 @@ func OriginIsOpen(c *gin.Context) bool { } const ( - GPT3Turbo = "gpt-3.5-turbo" - GPT3Turbo0613 = "gpt-3.5-turbo-0613" - GPT3Turbo0301 = "gpt-3.5-turbo-0301" - GPT3Turbo16k = "gpt-3.5-turbo-16k" - GPT3Turbo16k0613 = "gpt-3.5-turbo-16k-0613" - GPT3Turbo16k0301 = "gpt-3.5-turbo-16k-0301" - GPT4 = "gpt-4" - GPT4Vision = "gpt-4v" - GPT40314 = "gpt-4-0314" - GPT40613 = "gpt-4-0613" - GPT432k = "gpt-4-32k" - GPT432k0314 = "gpt-4-32k-0314" - GPT432k0613 = "gpt-4-32k-0613" - Dalle = "dalle" - Claude2 = "claude-1" // claude v1.3 - Claude2100k = "claude-2" - ClaudeSlack = "claude-slack" - SparkDesk = "spark-desk-v1.5" - SparkDeskV2 = "spark-desk-v2" - SparkDeskV3 = "spark-desk-v3" - ChatBison001 = "chat-bison-001" - BingCreative = "bing-creative" - BingBalanced = "bing-balanced" - BingPrecise = "bing-precise" - ZhiPuChatGLMPro = "zhipu-chatglm-pro" - ZhiPuChatGLMStd = "zhipu-chatglm-std" - ZhiPuChatGLMLite = "zhipu-chatglm-lite" + GPT3Turbo = "gpt-3.5-turbo" + GPT3TurboInstruct = "gpt-3.5-turbo-instruct" + GPT3Turbo0613 = "gpt-3.5-turbo-0613" + GPT3Turbo0301 = "gpt-3.5-turbo-0301" + GPT3Turbo16k = "gpt-3.5-turbo-16k" + GPT3Turbo16k0613 = "gpt-3.5-turbo-16k-0613" + GPT3Turbo16k0301 = "gpt-3.5-turbo-16k-0301" + GPT4 = "gpt-4" + GPT4Vision = "gpt-4v" + GPT40314 = "gpt-4-0314" + GPT40613 = "gpt-4-0613" + GPT432k = "gpt-4-32k" + GPT432k0314 = "gpt-4-32k-0314" + GPT432k0613 = "gpt-4-32k-0613" + Dalle = "dalle" + Claude2 = "claude-1" // claude v1.3 + Claude2100k = "claude-2" + ClaudeSlack = "claude-slack" + SparkDesk = "spark-desk-v1.5" + SparkDeskV2 = "spark-desk-v2" + SparkDeskV3 = "spark-desk-v3" + ChatBison001 = "chat-bison-001" + BingCreative = "bing-creative" + BingBalanced = "bing-balanced" + BingPrecise = "bing-precise" + ZhiPuChatGLMPro = "zhipu-chatglm-pro" + ZhiPuChatGLMStd = "zhipu-chatglm-std" + ZhiPuChatGLMLite = "zhipu-chatglm-lite" ) var GPT3TurboArray = []string{ GPT3Turbo, + GPT3TurboInstruct, GPT3Turbo0613, GPT3Turbo0301, } @@ -128,6 +130,7 @@ var LongContextModelArray = []string{ var FreeModelArray = []string{ GPT3Turbo, + GPT3TurboInstruct, GPT3Turbo0613, GPT3Turbo0301, GPT3Turbo16k, @@ -143,6 +146,7 @@ var FreeModelArray = []string{ var AllModels = []string{ GPT3Turbo, + GPT3TurboInstruct, GPT3Turbo0613, GPT3Turbo0301, GPT3Turbo16k, diff --git a/utils/tokenizer.go b/utils/tokenizer.go index c6dc932..4036adb 100644 --- a/utils/tokenizer.go +++ b/utils/tokenizer.go @@ -42,6 +42,7 @@ func GetWeightByModel(model string) int { globals.GPT432k0314: return 3 * 10 case globals.GPT3Turbo, + globals.GPT3TurboInstruct, globals.GPT3Turbo0613, globals.GPT3Turbo16k, @@ -103,7 +104,7 @@ func CountTokenPrice(messages []globals.Message, model string) int { func CountInputToken(model string, v []globals.Message) float32 { switch model { - case globals.GPT3Turbo, globals.GPT3Turbo0613, globals.GPT3Turbo0301, + case globals.GPT3Turbo, globals.GPT3Turbo0613, globals.GPT3Turbo0301, globals.GPT3TurboInstruct, globals.GPT3Turbo16k, globals.GPT3Turbo16k0613, globals.GPT3Turbo16k0301: return 0 case globals.GPT4, globals.GPT4Vision, globals.GPT40314, globals.GPT40613: @@ -129,7 +130,7 @@ func CountInputToken(model string, v []globals.Message) float32 { func CountOutputToken(model string, t int) float32 { switch model { - case globals.GPT3Turbo, globals.GPT3Turbo0613, globals.GPT3Turbo0301, + case globals.GPT3Turbo, globals.GPT3Turbo0613, globals.GPT3Turbo0301, globals.GPT3TurboInstruct, globals.GPT3Turbo16k, globals.GPT3Turbo16k0613, globals.GPT3Turbo16k0301: return 0 case globals.GPT4, globals.GPT4Vision, globals.GPT40314, globals.GPT40613: