GithubProxy/coai
1
0
Fork 0
mirror of https://github.com/coaidev/coai.git synced 2025-05-19 13:00:14 +09:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: suppport customize max_tokens, temperature, top_p, top_k, presenece_penalty, frequency_penalty, repetition_penalty (#56)

Browse Source
This commit is contained in:
Zhang Minghan 2024-01-30 12:51:46 +08:00
parent 5570814f89
commit 36dbeac62c
31 changed files with 740 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
adapter/adapter.go
View File

@ -32,7 +32,7 @@ type ChatProps struct {
Model string
Message []globals.Message
Token int
MaxTokens *int
PresencePenalty *float32
FrequencyPenalty *float32
RepetitionPenalty *float32
@ -50,13 +50,9 @@ func createChatRequest(conf globals.ChannelConfig, props *ChatProps, hook global
switch conf.GetType() {
case globals.OpenAIChannelType:
return chatgpt.NewChatInstanceFromConfig(conf).CreateStreamChatRequest(&chatgpt.ChatProps{
Model: model,
Message: props.Message,
Token: utils.Multi(
props.Token == 0,
utils.ToPtr(2500),
&props.Token,
),
Model: model,
Message: props.Message,
Token: props.MaxTokens,
PresencePenalty: props.PresencePenalty,
FrequencyPenalty: props.FrequencyPenalty,
Temperature: props.Temperature,
@ -68,13 +64,9 @@ func createChatRequest(conf globals.ChannelConfig, props *ChatProps, hook global
case globals.AzureOpenAIChannelType:
return azure.NewChatInstanceFromConfig(conf).CreateStreamChatRequest(&azure.ChatProps{
Model: model,
Message: props.Message,
Token: utils.Multi(
props.Token == 0,
utils.ToPtr(2500),
&props.Token,
),
Model: model,
Message: props.Message,
Token: props.MaxTokens,
PresencePenalty: props.PresencePenalty,
FrequencyPenalty: props.FrequencyPenalty,
Temperature: props.Temperature,
@ -88,7 +80,7 @@ func createChatRequest(conf globals.ChannelConfig, props *ChatProps, hook global
return claude.NewChatInstanceFromConfig(conf).CreateStreamChatRequest(&claude.ChatProps{
Model: model,
Message: props.Message,
Token: utils.Multi(props.Token == 0, 50000, props.Token),
Token: props.MaxTokens,
TopP: props.TopP,
TopK: props.TopK,
Temperature: props.Temperature,
@ -115,7 +107,7 @@ func createChatRequest(conf globals.ChannelConfig, props *ChatProps, hook global
return sparkdesk.NewChatInstance(conf, model).CreateStreamChatRequest(&sparkdesk.ChatProps{
Model: model,
Message: props.Message,
Token: utils.Multi(props.Token == 0, nil, utils.ToPtr(props.Token)),
Token: props.MaxTokens,
Temperature: props.Temperature,
TopK: props.TopK,
Tools: props.Tools,
@ -134,7 +126,7 @@ func createChatRequest(conf globals.ChannelConfig, props *ChatProps, hook global
return dashscope.NewChatInstanceFromConfig(conf).CreateStreamChatRequest(&dashscope.ChatProps{
Model: model,
Message: props.Message,
Token: props.Token,
Token: props.MaxTokens,
Temperature: props.Temperature,
TopP: props.TopP,
TopK: props.TopK,
@ -162,7 +154,7 @@ func createChatRequest(conf globals.ChannelConfig, props *ChatProps, hook global
return skylark.NewChatInstanceFromConfig(conf).CreateStreamChatRequest(&skylark.ChatProps{
Model: model,
Message: props.Message,
Token: props.Token,
Token: props.MaxTokens,
TopP: props.TopP,
TopK: props.TopK,
Temperature: props.Temperature,
@ -176,7 +168,7 @@ func createChatRequest(conf globals.ChannelConfig, props *ChatProps, hook global
return zhinao.NewChatInstanceFromConfig(conf).CreateStreamChatRequest(&zhinao.ChatProps{
Model: model,
Message: props.Message,
Token: &props.Token,
Token: props.MaxTokens,
TopP: props.TopP,
TopK: props.TopK,
Temperature: props.Temperature,
@ -193,7 +185,7 @@ func createChatRequest(conf globals.ChannelConfig, props *ChatProps, hook global
return oneapi.NewChatInstanceFromConfig(conf).CreateStreamChatRequest(&oneapi.ChatProps{
Model: model,
Message: props.Message,
Token: &props.Token,
Token: props.MaxTokens,
PresencePenalty: props.PresencePenalty,
FrequencyPenalty: props.FrequencyPenalty,
Temperature: props.Temperature,

8
adapter/bing/chat.go
View File

@ -30,8 +30,12 @@ func (c *ChatInstance) CreateStreamChatRequest(props *ChatProps, hook globals.Ho
}
for {
form := utils.ReadForm[ChatResponse](conn)
if form == nil {
form, err := utils.ReadForm[ChatResponse](conn)
if err != nil {
if strings.Contains(err.Error(), "websocket: close 1000") {
return nil
}
globals.Debug(fmt.Sprintf("bing error: %s", err.Error()))
return nil
}

14
adapter/claude/chat.go
View File

@ -7,10 +7,12 @@ import (
"strings"
)
const defaultTokens = 2500
type ChatProps struct {
Model string
Message []globals.Message
Token int
Token *int
Temperature *float32
TopP *float32
TopK *int
@ -50,10 +52,18 @@ func (c *ChatInstance) ConvertMessage(message []globals.Message) string {
return fmt.Sprintf("%s\n\nAssistant:", result)
}
func (c *ChatInstance) GetTokens(props *ChatProps) int {
if props.Token == nil || *props.Token <= 0 {
return defaultTokens
}
return *props.Token
}
func (c *ChatInstance) GetChatBody(props *ChatProps, stream bool) *ChatBody {
return &ChatBody{
Prompt: c.ConvertMessage(props.Message),
MaxTokensToSample: props.Token,
MaxTokensToSample: c.GetTokens(props),
Model: props.Model,
Stream: stream,
Temperature: props.Temperature,

55
adapter/dashscope/chat.go
View File

@ -7,9 +7,11 @@ import (
"strings"
)
const defaultMaxTokens = 1500
type ChatProps struct {
Model string
Token int
Token *int
Temperature *float32
TopP *float32
TopK *int
@ -41,22 +43,55 @@ func (c *ChatInstance) FormatMessages(message []globals.Message) []Message {
return messages
}
func (c *ChatInstance) GetChatBody(props *ChatProps) ChatRequest {
if props.Token <= 0 || props.Token > 1500 {
props.Token = 1500
func (c *ChatInstance) GetMaxTokens(props *ChatProps) int {
// dashscope has a restriction of 1500 tokens in completion
if props.Token == nil || *props.Token <= 0 || *props.Token > 1500 {
return defaultMaxTokens
}
return *props.Token
}
func (c *ChatInstance) GetTopP(props *ChatProps) *float32 {
// range of top_p should be (0.0, 1.0)
if props.TopP == nil {
return nil
}
if *props.TopP <= 0.0 {
return utils.ToPtr[float32](0.1)
} else if *props.TopP >= 1.0 {
return utils.ToPtr[float32](0.9)
}
return props.TopP
}
func (c *ChatInstance) GetRepeatPenalty(props *ChatProps) *float32 {
// range of repetition_penalty should greater than 0.0
if props.RepetitionPenalty == nil {
return nil
}
if *props.RepetitionPenalty <= 0.0 {
return utils.ToPtr[float32](0.1)
}
return props.RepetitionPenalty
}
func (c *ChatInstance) GetChatBody(props *ChatProps) ChatRequest {
return ChatRequest{
Model: strings.TrimSuffix(props.Model, "-net"),
Input: ChatInput{
Messages: c.FormatMessages(props.Message),
},
Parameters: ChatParam{
MaxTokens: props.Token,
MaxTokens: c.GetMaxTokens(props),
Temperature: props.Temperature,
TopP: props.TopP,
TopP: c.GetTopP(props),
TopK: props.TopK,
RepetitionPenalty: props.RepetitionPenalty,
RepetitionPenalty: c.GetRepeatPenalty(props),
EnableSearch: utils.ToPtr(strings.HasSuffix(props.Model, "-net")),
IncrementalOutput: true,
},
@ -74,6 +109,12 @@ func (c *ChatInstance) CreateStreamChatRequest(props *ChatProps, callback global
c.GetHeader(),
c.GetChatBody(props),
func(data string) error {
// example:
// id:1
// event:result
// :HTTP_STATUS/200
// data:{"output":{"finish_reason":"null","text":"hi"},"usage":{"total_tokens":15,"input_tokens":14,"output_tokens":1},"request_id":"08da1369-e009-9f8f-8363-54b966f80daf"}
data = strings.TrimSpace(data)
if !strings.HasPrefix(data, "data:") {
return nil

14
adapter/skylark/chat.go
View File

@ -8,10 +8,12 @@ import (
"github.com/volcengine/volc-sdk-golang/service/maas/models/api"
)
const defaultMaxTokens int64 = 1500
type ChatProps struct {
Model string
Message []globals.Message
Token int
Token *int
PresencePenalty *float32
FrequencyPenalty *float32
@ -37,6 +39,14 @@ func getMessages(messages []globals.Message) []*api.Message {
})
}
func (c *ChatInstance) GetMaxTokens(token *int) int64 {
if token == nil || *token < 0 {
return defaultMaxTokens
}
return int64(*token)
}
func (c *ChatInstance) CreateRequest(props *ChatProps) *api.ChatReq {
return &api.ChatReq{
Model: &api.Model{
@ -50,7 +60,7 @@ func (c *ChatInstance) CreateRequest(props *ChatProps) *api.ChatReq {
PresencePenalty: utils.GetPtrVal(props.PresencePenalty, 0.),
FrequencyPenalty: utils.GetPtrVal(props.FrequencyPenalty, 0.),
RepetitionPenalty: utils.GetPtrVal(props.RepeatPenalty, 0.),
MaxTokens: int64(props.Token),
MaxTokens: c.GetMaxTokens(props.Token),
},
Functions: getFunctions(props.Tools),
}

9
adapter/sparkdesk/chat.go
View File

@ -4,6 +4,7 @@ import (
"chat/globals"
"chat/utils"
"fmt"
"strings"
)
type ChatProps struct {
@ -116,8 +117,12 @@ func (c *ChatInstance) CreateStreamChatRequest(props *ChatProps, hook globals.Ho
}
for {
form := utils.ReadForm[ChatResponse](conn)
if form == nil {
form, err := utils.ReadForm[ChatResponse](conn)
if err != nil {
if strings.Contains(err.Error(), "websocket: close 1000") {
return nil
}
globals.Debug(fmt.Sprintf("sparkdesk error: %s", err.Error()))
return nil
}

4
addition/article/api.go
View File

@ -40,8 +40,8 @@ func GenerateAPI(c *gin.Context) {
}
defer conn.DeferClose()
var form *WebsocketArticleForm
if form = utils.ReadForm[WebsocketArticleForm](conn); form == nil {
form, err := utils.ReadForm[WebsocketArticleForm](conn)
if err != nil {
return
}

4
addition/generation/api.go
View File

@ -35,8 +35,8 @@ func GenerateAPI(c *gin.Context) {
}
defer conn.DeferClose()
var form *WebsocketGenerationForm
if form = utils.ReadForm[WebsocketGenerationForm](conn); form == nil {
form, err := utils.ReadForm[WebsocketGenerationForm](conn)
if err != nil {
return
}

1
app/package.json
View File

@ -24,6 +24,7 @@
"@radix-ui/react-scroll-area": "^1.0.5",
"@radix-ui/react-select": "^2.0.0",
"@radix-ui/react-separator": "^1.0.3",
"@radix-ui/react-slider": "^1.1.2",
"@radix-ui/react-slot": "^1.0.2",
"@radix-ui/react-switch": "^1.0.3",
"@radix-ui/react-tabs": "^1.0.4",

34
app/pnpm-lock.yaml generated
View File

@ -41,6 +41,9 @@ dependencies:
'@radix-ui/react-separator':
specifier: ^1.0.3
version: 1.0.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0)(react@18.2.0)
'@radix-ui/react-slider':
specifier: ^1.1.2
version: 1.1.2(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0)(react@18.2.0)
'@radix-ui/react-slot':
specifier: ^1.0.2
version: 1.0.2(@types/react@18.2.33)(react@18.2.0)
@ -1482,6 +1485,37 @@ packages:
react-dom: 18.2.0(react@18.2.0)
dev: false
/@radix-ui/react-slider@1.1.2(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0)(react@18.2.0):
resolution: {integrity: sha512-NKs15MJylfzVsCagVSWKhGGLNR1W9qWs+HtgbmjjVUB3B9+lb3PYoXxVju3kOrpf0VKyVCtZp+iTwVoqpa1Chw==}
peerDependencies:
'@types/react': '*'
'@types/react-dom': '*'
react: ^16.8 || ^17.0 || ^18.0
react-dom: ^16.8 || ^17.0 || ^18.0
peerDependenciesMeta:
'@types/react':
optional: true
'@types/react-dom':
optional: true
dependencies:
'@babel/runtime': 7.23.2
'@radix-ui/number': 1.0.1
'@radix-ui/primitive': 1.0.1
'@radix-ui/react-collection': 1.0.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0)(react@18.2.0)
'@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.33)(react@18.2.0)
'@radix-ui/react-context': 1.0.1(@types/react@18.2.33)(react@18.2.0)
'@radix-ui/react-direction': 1.0.1(@types/react@18.2.33)(react@18.2.0)
'@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.14)(@types/react@18.2.33)(react-dom@18.2.0)(react@18.2.0)
'@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.2.33)(react@18.2.0)
'@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.2.33)(react@18.2.0)
'@radix-ui/react-use-previous': 1.0.1(@types/react@18.2.33)(react@18.2.0)
'@radix-ui/react-use-size': 1.0.1(@types/react@18.2.33)(react@18.2.0)
'@types/react': 18.2.33
'@types/react-dom': 18.2.14
react: 18.2.0
react-dom: 18.2.0(react@18.2.0)
dev: false
/@radix-ui/react-slot@1.0.0(react@18.2.0):
resolution: {integrity: sha512-3mrKauI/tWXo1Ll+gN5dHcxDPdm/Df1ufcDLCecn+pnCIVcdWE7CujXo8QaXOWRJyZyQWWbpB8eFwHzWXlv5mQ==}
peerDependencies:

10
app/src/api/connection.ts
View File

@ -3,7 +3,7 @@ import { getMemory } from "@/utils/memory.ts";
import { getErrorMessage } from "@/utils/base.ts";
export const endpoint = `${websocketEndpoint}/chat`;
export const maxRetry = 30; // 15s max websocket retry
export const maxRetry = 60; // 30s max websocket retry
export type StreamMessage = {
conversation?: number;
@ -21,6 +21,14 @@ export type ChatProps = {
web?: boolean;
context?: number;
ignore_context?: boolean;
max_tokens?: number;
temperature?: number;
top_p?: number;
top_k?: number;
presence_penalty?: number;
frequency_penalty?: number;
repetition_penalty?: number;
};
type StreamCallback = (message: StreamMessage) => void;

17
app/src/assets/pages/settings.less
View File

@ -60,13 +60,21 @@
transition: .1s;
}
.slider-value {
min-width: 2rem;
}
input {
text-align: center;
max-width: 4rem;
max-height: 1.75rem;
&.large-value {
max-width: 6rem;
}
}
button {
button:not(.tips-trigger):not(.set-action) {
margin: 0.25rem 1rem;
}
@ -81,7 +89,14 @@
}
.name {
display: flex;
flex-direction: row;
align-items: center;
white-space: nowrap;
.tips-trigger {
transform: translateY(1px);
}
}
}

17
app/src/components/Tips.tsx
View File

@ -5,7 +5,7 @@ import {
TooltipTrigger,
} from "@/components/ui/tooltip.tsx";
import { HelpCircle } from "lucide-react";
import React, { useEffect, useMemo } from "react";
import React, { useEffect, useMemo, useRef } from "react";
import { cn } from "@/components/ui/lib/utils.ts";
import {
DropdownMenu,
@ -44,23 +44,30 @@ function Tips({
const [drop, setDrop] = React.useState(false);
const [tooltip, setTooltip] = React.useState(false);
const task = useRef<NodeJS.Timeout>();
useEffect(() => {
drop && setTimeout(() => setDrop(false), timeout);
drop
? (task.current = setTimeout(() => setDrop(false), timeout))
: clearTimeout(task.current);
}, [drop]);
useEffect(() => {
tooltip && drop && setTooltip(false);
if (!tooltip) return;
setTooltip(false);
!drop && setDrop(true);
}, [drop, tooltip]);
return (
<DropdownMenu open={drop} onOpenChange={setDrop}>
<DropdownMenuTrigger className={`select-none outline-none`}>
<DropdownMenuTrigger className={`tips-trigger select-none outline-none`}>
<TooltipProvider>
<Tooltip open={tooltip} onOpenChange={setTooltip}>
<TooltipTrigger asChild>
{trigger ?? <HelpCircle className={cn("tips-icon", className)} />}
</TooltipTrigger>
<TooltipContent className={classNamePopup}>{comp}</TooltipContent>
<TooltipContent className="hidden" />
</Tooltip>
</TooltipProvider>
</DropdownMenuTrigger>

22
app/src/components/home/ChatWrapper.tsx
View File

@ -21,7 +21,14 @@ import { ToastAction } from "@/components/ui/toast.tsx";
import {
alignSelector,
contextSelector,
frequencyPenaltySelector,
historySelector,
maxTokensSelector,
presencePenaltySelector,
repetitionPenaltySelector,
temperatureSelector,
topKSelector,
topPSelector,
} from "@/store/settings.ts";
import { FileArray } from "@/api/file.ts";
import {
@ -76,6 +83,14 @@ function ChatWrapper() {
const context = useSelector(contextSelector);
const align = useSelector(alignSelector);
const max_tokens = useSelector(maxTokensSelector);
const temperature = useSelector(temperatureSelector);
const top_p = useSelector(topPSelector);
const top_k = useSelector(topKSelector);
const presence_penalty = useSelector(presencePenaltySelector);
const frequency_penalty = useSelector(frequencyPenaltySelector);
const repetition_penalty = useSelector(repetitionPenaltySelector);
const requireAuth = useMemo(
(): boolean => !!getModelFromId(model)?.auth,
[model],
@ -113,6 +128,13 @@ function ChatWrapper() {
model,
context: history,
ignore_context: !context,
max_tokens,
temperature,
top_p,
top_k,
presence_penalty,
frequency_penalty,
repetition_penalty,
})
) {
forgetMemory("history");

4
app/src/components/ui/number-input.tsx
View File

@ -32,6 +32,8 @@ const NumberInput = React.forwardRef<HTMLInputElement, NumberInputProps>(
};
const formatValue = (v: string) => {
if (v.trim().length === 0) return v.trim();
if (!/^[-+]?(?:[0-9]*(?:\.[0-9]*)?)?$/.test(v)) {
const exp = /[-+]?[0-9]+(\.[0-9]+)?/g;
return v.match(exp)?.join("") || "";
@ -43,7 +45,7 @@ const NumberInput = React.forwardRef<HTMLInputElement, NumberInputProps>(
const raw = getNumber(v, props.acceptNegative);
let val = parseFloat(raw);
if (isNaN(val) && !props.acceptNaN) return "0";
if (isNaN(val) && !props.acceptNaN) return (props.min ?? 0).toString();
if (props.max !== undefined && val > props.max)
return props.max.toString();
else if (props.min !== undefined && val < props.min)

35
app/src/components/ui/slider.tsx Normal file
View File

@ -0,0 +1,35 @@
import * as React from "react";
import * as SliderPrimitive from "@radix-ui/react-slider";
import { cn } from "@/components/ui/lib/utils";
type SliderProps = {
classNameThumb?: string;
};
const Slider = React.forwardRef<
React.ElementRef<typeof SliderPrimitive.Root> & SliderProps,
React.ComponentPropsWithoutRef<typeof SliderPrimitive.Root> & SliderProps
>(({ className, classNameThumb, ...props }, ref) => (
<SliderPrimitive.Root
ref={ref}
className={cn(
"relative flex w-full touch-none select-none items-center",
className,
)}
{...props}
>
<SliderPrimitive.Track className="relative h-2 w-full cursor-pointer grow overflow-hidden rounded-full bg-accent">
<SliderPrimitive.Range className="absolute h-full bg-primary" />
</SliderPrimitive.Track>
<SliderPrimitive.Thumb
className={cn(
"block h-5 w-5 cursor-pointer rounded-full border-2 border-primary bg-background ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50",
classNameThumb,
)}
/>
</SliderPrimitive.Root>
));
Slider.displayName = SliderPrimitive.Root.displayName;
export { Slider };

248
app/src/dialogs/SettingsDialog.tsx
View File

@ -1,19 +1,7 @@
import "@/assets/pages/settings.less";
import { useTranslation } from "react-i18next";
import { useDispatch, useSelector } from "react-redux";
import {
alignSelector,
contextSelector,
dialogSelector,
historySelector,
senderSelector,
sendKeys,
setAlign,
setContext,
setDialog,
setHistory,
setSender,
} from "@/store/settings.ts";
import * as settings from "@/store/settings.ts";
import {
Dialog,
DialogContent,
@ -36,16 +24,39 @@ import {
import { langsProps, setLanguage } from "@/i18n.ts";
import { cn } from "@/components/ui/lib/utils.ts";
import Github from "@/components/ui/icons/Github.tsx";
import { Slider } from "@/components/ui/slider.tsx";
import Tips from "@/components/Tips.tsx";
import { Button } from "@/components/ui/button.tsx";
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
AlertDialogTrigger,
} from "@/components/ui/alert-dialog.tsx";
function SettingsDialog() {
const { t, i18n } = useTranslation();
const dispatch = useDispatch();
const open = useSelector(dialogSelector);
const open = useSelector(settings.dialogSelector);
const align = useSelector(settings.alignSelector);
const context = useSelector(settings.contextSelector);
const sender = useSelector(settings.senderSelector);
const history = useSelector(settings.historySelector);
const temperature = useSelector(settings.temperatureSelector);
const maxTokens = useSelector(settings.maxTokensSelector);
const topP = useSelector(settings.topPSelector);
const topK = useSelector(settings.topKSelector);
const presencePenalty = useSelector(settings.presencePenaltySelector);
const frequencyPenalty = useSelector(settings.frequencyPenaltySelector);
const repetitionPenalty = useSelector(settings.repetitionPenaltySelector);
const align = useSelector(alignSelector);
const context = useSelector(contextSelector);
const sender = useSelector(senderSelector);
const history = useSelector(historySelector);
const [memorySize, setMemorySize] = useState(getMemoryPerformance());
useEffect(() => {
@ -57,7 +68,10 @@ function SettingsDialog() {
}, []);
return (
<Dialog open={open} onOpenChange={(open) => dispatch(setDialog(open))}>
<Dialog
open={open}
onOpenChange={(open) => dispatch(settings.setDialog(open))}
>
<DialogContent className={`fixed-dialog settings-dialog`}>
<DialogHeader>
<DialogTitle>{t("settings.title")}</DialogTitle>
@ -106,15 +120,21 @@ function SettingsDialog() {
<Select
value={sender ? "true" : "false"}
onValueChange={(value: string) =>
dispatch(setSender(value === "true"))
dispatch(settings.setSender(value === "true"))
}
>
<SelectTrigger className={`select`}>
<SelectValue placeholder={sendKeys[sender ? 1 : 0]} />
<SelectValue
placeholder={settings.sendKeys[sender ? 1 : 0]}
/>
</SelectTrigger>
<SelectContent>
<SelectItem value={"false"}>{sendKeys[0]}</SelectItem>
<SelectItem value={"true"}>{sendKeys[1]}</SelectItem>
<SelectItem value={"false"}>
{settings.sendKeys[0]}
</SelectItem>
<SelectItem value={"true"}>
{settings.sendKeys[1]}
</SelectItem>
</SelectContent>
</Select>
</div>
@ -126,7 +146,7 @@ function SettingsDialog() {
className={`value`}
checked={align}
onCheckedChange={(state: boolean) => {
dispatch(setAlign(state));
dispatch(settings.setAlign(state));
}}
/>
</div>
@ -137,7 +157,7 @@ function SettingsDialog() {
className={`value`}
checked={context}
onCheckedChange={(state: boolean) => {
dispatch(setContext(state));
dispatch(settings.setContext(state));
}}
/>
</div>
@ -155,11 +175,187 @@ function SettingsDialog() {
min={0}
max={999}
onValueChange={(value: number) => {
dispatch(setHistory(value));
dispatch(settings.setHistory(value));
}}
/>
</div>
)}
<div className={`item`}>
<div className={`name`}>
{t("settings.max-tokens")}
<Tips content={t("settings.max-tokens-tip")} />
</div>
<div className={`grow`} />
<NumberInput
className={`value large-value`}
value={maxTokens}
acceptNaN={false}
min={1}
max={100000}
onValueChange={(value: number) => {
dispatch(settings.setMaxTokens(value));
}}
/>
</div>
</div>
<div className={`settings-segment`}>
<div className={`item`}>
<div className={`name`}>
{t("settings.temperature")}
<Tips content={t("settings.temperature-tip")} />
</div>
<div className={`grow`} />
<Slider
value={[temperature * 10]}
min={0}
max={10}
step={1}
className={`value ml-2 max-w-[10rem] mr-2`}
classNameThumb={`h-4 w-4`}
onValueChange={(value: number[]) => {
dispatch(settings.setTemperature(value[0] / 10));
}}
/>
<p className={`slider-value`}>{temperature.toFixed(1)}</p>
</div>
<div className={`item`}>
<div className={`name`}>
{t("settings.presence-penalty")}
<Tips content={t("settings.presence-penalty-tip")} />
</div>
<div className={`grow`} />
<Slider
value={[presencePenalty * 10]}
min={-20}
max={20}
step={1}
className={`value ml-2 max-w-[10rem] mr-2`}
classNameThumb={`h-4 w-4`}
onValueChange={(value: number[]) => {
dispatch(settings.setPresencePenalty(value[0] / 10));
}}
/>
<p className={`slider-value`}>
{presencePenalty.toFixed(1)}
</p>
</div>
<div className={`item`}>
<div className={`name`}>
{t("settings.frequency-penalty")}
<Tips content={t("settings.frequency-penalty-tip")} />
</div>
<div className={`grow`} />
<Slider
value={[frequencyPenalty * 10]}
min={-20}
max={20}
step={1}
className={`value ml-2 max-w-[10rem] mr-2`}
classNameThumb={`h-4 w-4`}
onValueChange={(value: number[]) => {
dispatch(settings.setFrequencyPenalty(value[0] / 10));
}}
/>
<p className={`slider-value`}>
{frequencyPenalty.toFixed(1)}
</p>
</div>
<div className={`item`}>
<div className={`name`}>
{t("settings.repetition-penalty")}
<Tips content={t("settings.repetition-penalty-tip")} />
</div>
<div className={`grow`} />
<Slider
value={[repetitionPenalty * 10]}
min={0}
max={20}
step={1}
className={`value ml-2 max-w-[10rem] mr-2`}
classNameThumb={`h-4 w-4`}
onValueChange={(value: number[]) => {
dispatch(settings.setRepetitionPenalty(value[0] / 10));
}}
/>
<p className={`slider-value`}>
{repetitionPenalty.toFixed(1)}
</p>
</div>
<div className={`item`}>
<div className={`name`}>
{t("settings.top-p")}
<Tips content={t("settings.top-p-tip")} />
</div>
<div className={`grow`} />
<Slider
value={[topP * 10]}
min={0}
max={10}
step={1}
className={`value ml-2 max-w-[10rem] mr-2`}
classNameThumb={`h-4 w-4`}
onValueChange={(value: number[]) => {
dispatch(settings.setTopP(value[0] / 10));
}}
/>
<p className={`slider-value`}>{topP.toFixed(1)}</p>
</div>
<div className={`item`}>
<div className={`name`}>
{t("settings.top-k")}
<Tips content={t("settings.top-k-tip")} />
</div>
<div className={`grow`} />
<Slider
value={[topK]}
min={0}
max={20}
step={1}
className={`value ml-2 max-w-[10rem] mr-2`}
classNameThumb={`h-4 w-4`}
onValueChange={(value: number[]) => {
dispatch(settings.setTopK(value[0]));
}}
/>
<p className={`slider-value`}>{topK.toFixed()}</p>
</div>
</div>
<div className={`settings-segment`}>
<div className={`item`}>
<div className={`name`}>{t("settings.reset-settings")}</div>
<div className={`grow`} />
<AlertDialog>
<AlertDialogTrigger asChild>
<Button
size={`sm`}
variant={`destructive`}
className={`set-action`}
>
{t("reset")}
</Button>
</AlertDialogTrigger>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>
{t("settings.reset-settings")}
</AlertDialogTitle>
<AlertDialogDescription>
{t("settings.reset-settings-description")}
</AlertDialogDescription>
<AlertDialogFooter>
<AlertDialogCancel>{t("cancel")}</AlertDialogCancel>
<AlertDialogAction
onClick={() => {
dispatch(settings.resetSettings());
}}
>
{t("confirm")}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogHeader>
</AlertDialogContent>
</AlertDialog>
</div>
</div>
</div>
<div className={`grow`} />

18
app/src/resources/i18n/cn.json
View File

@ -332,7 +332,23 @@
"context": "保留上下文",
"history": "最大历史会话数",
"align": "聊天框居中",
"memory": "内存占用"
"memory": "内存占用",
"max-tokens": "最大回复 Token 数",
"max-tokens-tip": "最大回复 Token 数，超过此数值将会被截断",
"temperature": "温度",
"temperature-tip": "随机采样的比例，高温度会产生更多的随机性，低温度会产生较集中和确定性的文本",
"top-p": "核采样概率阈值",
"top-p-tip": "(TopP) 概率取值越大，生成的随机性越高；取值越低，生成的确定性越高",
"top-k": "采样候选集大小",
"top-k-tip": "(TopK) 候选集大小，越大生成的随机性越高，越小生成的确定性越高",
"presence-penalty": "存在惩罚",
"presence-penalty-tip": "(PresencePenalty) 存在惩罚，控制模型生成的新话题的可能性，提高此值可以增加谈论新话题的可能性",
"frequency-penalty": "频率惩罚",
"frequency-penalty-tip": "(FrequencyPenalty) 频率惩罚，控制模型生成字词的重复程度，提高此值可以降低重复字词出现频率的可能性",
"repetition-penalty": "重复惩罚",
"repetition-penalty-tip": "(RepetitionPenalty) 控制模型生成的重复程度，提高此值可以减少重复，但是可能会导致模型生成不连贯的文本（与 FrequencyPenalty 相似）",
"reset-settings": "重置全部设置",
"reset-settings-description": "是否确定？此操作无法撤消。这将永久重置全部设置。"
},
"article": {
"title": "批量生成文章",

18
app/src/resources/i18n/en.json
View File

@ -281,7 +281,23 @@
"context": "Keep Context",
"history": "Max History Conversations",
"align": "Chatbox Centered",
"memory": "Memory Usage"
"memory": "Memory Usage",
"temperature": "temperature",
"temperature-tip": "Random sampling ratio, high temperature produces more randomness, low temperature produces more concentrated and deterministic text",
"max-tokens": "Maximum number of response tokens",
"max-tokens-tip": "Maximum number of reply tokens, exceeding this value will be truncated",
"top-p": "Kernel Sampling Probability Threshold",
"top-p-tip": "(TopP) The higher the probability value, the higher the randomness generated; the lower the value, the higher the certainty generated",
"top-k": "Sample Candidate Set Size",
"top-k-tip": "(TopK) Candidate set size, the larger the randomness of the generation, the smaller the generation, the higher the certainty",
"presence-penalty": "Existence of penalties",
"presence-penalty-tip": "(PresencePenalty) There is a penalty for controlling the likelihood of new topics generated by the model, increasing this value can increase the likelihood of talking about new topics",
"frequency-penalty": "Frequency penalty",
"frequency-penalty-tip": "(FrequencyPenalty) Frequency penalty, control the degree of repetition of words generated by the model, increasing this value can reduce the possibility of repetition of words",
"repetition-penalty": "Duplicate Punishment",
"repetition-penalty-tip": "(RepetitionPenalty) Controls the degree of repetition generated by the model. Increasing this value can reduce repetition, but may cause the model to generate incoherent text (similar to FrequencyPenalty)",
"reset-settings": "Reset all settings",
"reset-settings-description": "Are you sure? This action cannot be undone. This will permanently reset all settings."
},
"article": {
"title": "Batch Generate Articles",

18
app/src/resources/i18n/ja.json
View File

@ -281,7 +281,23 @@
"context": "コンテキストを保持",
"history": "最大履歴セッション数",
"align": "チャットボックスを中央に配置",
"memory": "メモリ使用量"
"memory": "メモリ使用量",
"temperature": "温度",
"temperature-tip": "ランダムサンプリング比、高温はよりランダム性を生み、低温はより集中的で決定論的なテキストを生成します",
"max-tokens": "レスポンストークンの最大数",
"max-tokens-tip": "この値を超える返信トークンの最大数は切り捨てられます",
"top-p": "カーネルサンプリング確率閾値",
"top-p-tip": "（ TopP ）確率値が高いほど生成されるランダム性が高く、値が低いほど生成される確実性が高くなります",
"top-k": "サンプル候補セットサイズ",
"top-k-tip": "(TopK)候補セットサイズ、生成のランダム性が大きいほど生成が小さいほど確実性が高い",
"presence-penalty": "ペナルティの存在",
"presence-penalty-tip": "(PresencePenalty)モデルによって生成された新しいトピックの可能性を制御するためのペナルティがあります。この値を増やすと、新しいトピックについて話す可能性が高くなります",
"frequency-penalty": "フリークエンシー・パニュメント",
"frequency-penalty-tip": "（ FrequencyPenalty ）周波数ペナルティ、モデルによって生成された単語の繰り返しの度合いを制御し、この値を増やすことで単語の繰り返しの可能性を減らすことができます",
"repetition-penalty": "重複した処罰",
"repetition-penalty-tip": "(RepetitionPenalty)モデルによって生成される繰り返しの度合いを制御します。この値を大きくすると、繰り返しを減らすことができますが、モデルが不整合のテキストを生成する可能性があります(FrequencyPenaltyと同様)",
"reset-settings": "すべての設定をリセット",
"reset-settings-description": "本当によろしいですか？この操作は元に戻せません。これにより、すべての設定が完全にリセットされます。"
},
"article": {
"title": "投稿の一括生成",

18
app/src/resources/i18n/ru.json
View File

@ -281,7 +281,23 @@
"context": "Сохранить контекст",
"history": "Максимальное количество исторических разговоров",
"align": "Выравнивание чата по центру",
"memory": "Использование памяти"
"memory": "Использование памяти",
"temperature": "Температура",
"temperature-tip": "Коэффициент случайной выборки, высокая температура создает больше случайности, низкая температура создает более концентрированный и детерминированный текст",
"max-tokens": "Максимальное количество маркеров ответа",
"max-tokens-tip": "Максимальное количество маркеров ответа, превышающее это значение, будет усечено",
"top-p": "Порог вероятности отбора проб ядра",
"top-p-tip": "(TopP) Чем выше значение вероятности, тем выше генерируемая случайность; чем ниже значение, тем выше генерируемая определенность",
"top-k": "Размер набора образцов-кандидатов",
"top-k-tip": "(TopK) Размер набора кандидатов, чем больше случайность генерации, чем меньше генерация, тем выше определенность",
"presence-penalty": "Наличие штрафных санкций",
"presence-penalty-tip": "(PresencePenalty) Существует штраф за контроль вероятности появления новых тем, генерируемых моделью, увеличение этого значения может увеличить вероятность разговора о новых темах",
"frequency-penalty": "Частотное наказание",
"frequency-penalty-tip": "(FrequencyPenalty) Штраф за частоту, контроль степени повторения слов, генерируемых моделью, увеличение этого значения может снизить возможность повторения слов",
"repetition-penalty": "Повторяющееся наказание",
"repetition-penalty-tip": "(RepetitionPenalty) Управляет степенью повторяемости, генерируемой моделью. Увеличение этого значения может уменьшить повторение, но может привести к тому, что модель будет генерировать некогерентный текст (аналогично FrequencyPenalty)",
"reset-settings": "Сбросить все настройки",
"reset-settings-description": "Вы уверены? Это действие нельзя отменить. Это приведет к окончательному сбросу всех настроек."
},
"article": {
"title": "Пакет генерации статей",

94
app/src/store/settings.ts
View File

@ -8,6 +8,20 @@ import {
import { RootState } from "@/store/index.ts";
export const sendKeys = ["Ctrl + Enter", "Enter"];
export const initialSettings = {
context: true,
align: false,
history: 8,
sender: false,
max_tokens: 2000,
temperature: 0.6,
top_p: 1,
top_k: 5,
presence_penalty: 0,
frequency_penalty: 0,
repetition_penalty: 1,
};
export const settingsSlice = createSlice({
name: "settings",
initialState: {
@ -16,6 +30,13 @@ export const settingsSlice = createSlice({
align: getBooleanMemory("align", false), // chat textarea align center
history: getNumberMemory("history_context", 8), // max history context length
sender: getBooleanMemory("sender", false), // sender (false: Ctrl + Enter, true: Enter)
max_tokens: getNumberMemory("max_tokens", 2000), // max tokens
temperature: getNumberMemory("temperature", 0.6), // temperature
top_p: getNumberMemory("top_p", 1), // top_p
top_k: getNumberMemory("top_k", 5), // top_k
presence_penalty: getNumberMemory("presence_penalty", 0), // presence_penalty
frequency_penalty: getNumberMemory("frequency_penalty", 0), // frequency_penalty
repetition_penalty: getNumberMemory("repetition_penalty", 1), // repetition_penalty
},
reducers: {
toggleDialog: (state) => {
@ -46,6 +67,59 @@ export const settingsSlice = createSlice({
state.sender = action.payload as boolean;
setBooleanMemory("sender", action.payload);
},
setMaxTokens: (state, action) => {
state.max_tokens = action.payload as number;
setNumberMemory("max_tokens", action.payload);
},
setTemperature: (state, action) => {
state.temperature = action.payload as number;
setNumberMemory("temperature", action.payload);
},
setTopP: (state, action) => {
state.top_p = action.payload as number;
setNumberMemory("top_p", action.payload);
},
setTopK: (state, action) => {
state.top_k = action.payload as number;
setNumberMemory("top_k", action.payload);
},
setPresencePenalty: (state, action) => {
state.presence_penalty = action.payload as number;
setNumberMemory("presence_penalty", action.payload);
},
setFrequencyPenalty: (state, action) => {
state.frequency_penalty = action.payload as number;
setNumberMemory("frequency_penalty", action.payload);
},
setRepetitionPenalty: (state, action) => {
state.repetition_penalty = action.payload as number;
setNumberMemory("repetition_penalty", action.payload);
},
resetSettings: (state) => {
state.context = initialSettings.context;
state.align = initialSettings.align;
state.history = initialSettings.history;
state.sender = initialSettings.sender;
state.max_tokens = initialSettings.max_tokens;
state.temperature = initialSettings.temperature;
state.top_p = initialSettings.top_p;
state.top_k = initialSettings.top_k;
state.presence_penalty = initialSettings.presence_penalty;
state.frequency_penalty = initialSettings.frequency_penalty;
state.repetition_penalty = initialSettings.repetition_penalty;
setBooleanMemory("context", initialSettings.context);
setBooleanMemory("align", initialSettings.align);
setNumberMemory("history_context", initialSettings.history);
setBooleanMemory("sender", initialSettings.sender);
setNumberMemory("max_tokens", initialSettings.max_tokens);
setNumberMemory("temperature", initialSettings.temperature);
setNumberMemory("top_p", initialSettings.top_p);
setNumberMemory("top_k", initialSettings.top_k);
setNumberMemory("presence_penalty", initialSettings.presence_penalty);
setNumberMemory("frequency_penalty", initialSettings.frequency_penalty);
setNumberMemory("repetition_penalty", initialSettings.repetition_penalty);
},
},
});
@ -58,6 +132,14 @@ export const {
setAlign,
setHistory,
setSender,
setMaxTokens,
setTemperature,
setTopP,
setTopK,
setPresencePenalty,
setFrequencyPenalty,
setRepetitionPenalty,
resetSettings,
} = settingsSlice.actions;
export default settingsSlice.reducer;
@ -71,3 +153,15 @@ export const historySelector = (state: RootState): number =>
state.settings.history;
export const senderSelector = (state: RootState): boolean =>
state.settings.sender;
export const maxTokensSelector = (state: RootState): number =>
state.settings.max_tokens;
export const temperatureSelector = (state: RootState): number =>
state.settings.temperature;
export const topPSelector = (state: RootState): number => state.settings.top_p;
export const topKSelector = (state: RootState): number => state.settings.top_k;
export const presencePenaltySelector = (state: RootState): number =>
state.settings.presence_penalty;
export const frequencyPenaltySelector = (state: RootState): number =>
state.settings.frequency_penalty;
export const repetitionPenaltySelector = (state: RootState): number =>
state.settings.repetition_penalty;

3
channel/channel.go
View File

@ -11,7 +11,8 @@ import (
var defaultMaxRetries = 1
var defaultReplacer = []string{
"openai_api", "anthropic_api",
"api2d", "closeai_api", "one_api", "new_api",
"api2d", "closeai_api",
"one_api", "new_api", "shell_api",
}
func (c *Channel) GetId() int {

13
manager/chat.go
View File

@ -95,9 +95,16 @@ func ChatHandler(conn *Connection, user *auth.User, instance *conversation.Conve
err := channel.NewChatRequest(
auth.GetGroup(db, user),
&adapter.ChatProps{
Model: model,
Message: segment,
Buffer: *buffer,
Model: model,
Message: segment,
Buffer: *buffer,
MaxTokens: instance.GetMaxTokens(),
Temperature: instance.GetTemperature(),
TopP: instance.GetTopP(),
TopK: instance.GetTopK(),
PresencePenalty: instance.GetPresencePenalty(),
FrequencyPenalty: instance.GetFrequencyPenalty(),
RepetitionPenalty: instance.GetRepetitionPenalty(),
},
func(data string) error {
if signal := conn.PeekWithType(StopType); signal != nil {

2
manager/chat_completions.go
View File

@ -71,7 +71,7 @@ func getChatProps(form RelayForm, messages []globals.Message, buffer *utils.Buff
return &adapter.ChatProps{
Model: form.Model,
Message: messages,
Token: utils.Multi(form.MaxTokens == 0, 2500, form.MaxTokens),
MaxTokens: form.MaxTokens,
PresencePenalty: form.PresencePenalty,
FrequencyPenalty: form.FrequencyPenalty,
RepetitionPenalty: form.RepetitionPenalty,

4
manager/connection.go
View File

@ -56,8 +56,8 @@ func (c *Connection) ReadWorker() {
break
}
form := utils.ReadForm[conversation.FormMessage](c.conn)
if form == nil {
form, err := utils.ReadForm[conversation.FormMessage](c.conn)
if err != nil {
break
}

114
manager/conversation/conversation.go
View File

@ -22,6 +22,14 @@ type Conversation struct {
EnableWeb bool `json:"enable_web"`
Shared bool `json:"shared"`
Context int `json:"context"`
MaxTokens *int `json:"max_tokens,omitempty"`
Temperature *float32 `json:"temperature,omitempty"`
TopP *float32 `json:"top_p,omitempty"`
TopK *int `json:"top_k,omitempty"`
PresencePenalty *float32 `json:"presence_penalty,omitempty"`
FrequencyPenalty *float32 `json:"frequency_penalty,omitempty"`
RepetitionPenalty *float32 `json:"repetition_penalty,omitempty"`
}
type FormMessage struct {
@ -31,6 +39,15 @@ type FormMessage struct {
Model string `json:"model"`
IgnoreContext bool `json:"ignore_context"`
Context int `json:"context"`
// request params
MaxTokens *int `json:"max_tokens,omitempty"`
Temperature *float32 `json:"temperature,omitempty"`
TopP *float32 `json:"top_p,omitempty"`
TopK *int `json:"top_k,omitempty"`
PresencePenalty *float32 `json:"presence_penalty,omitempty"`
FrequencyPenalty *float32 `json:"frequency_penalty,omitempty"`
RepetitionPenalty *float32 `json:"repetition_penalty,omitempty"`
}
func NewAnonymousConversation() *Conversation {
@ -106,7 +123,69 @@ func (c *Conversation) SetEnableWeb(enable bool) {
c.EnableWeb = enable
}
func (c *Conversation) SetContextLength(context int) {
func (c *Conversation) GetTemperature() *float32 {
return c.Temperature
}
func (c *Conversation) SetTemperature(temperature *float32) {
c.Temperature = temperature
}
func (c *Conversation) GetTopP() *float32 {
return c.TopP
}
func (c *Conversation) SetTopP(topP *float32) {
c.TopP = topP
}
func (c *Conversation) GetTopK() *int {
return c.TopK
}
func (c *Conversation) SetTopK(topK *int) {
c.TopK = topK
}
func (c *Conversation) GetPresencePenalty() *float32 {
return c.PresencePenalty
}
func (c *Conversation) SetPresencePenalty(presencePenalty *float32) {
c.PresencePenalty = presencePenalty
}
func (c *Conversation) GetFrequencyPenalty() *float32 {
return c.FrequencyPenalty
}
func (c *Conversation) SetFrequencyPenalty(frequencyPenalty *float32) {
c.FrequencyPenalty = frequencyPenalty
}
func (c *Conversation) GetRepetitionPenalty() *float32 {
return c.RepetitionPenalty
}
func (c *Conversation) SetRepetitionPenalty(repetitionPenalty *float32) {
c.RepetitionPenalty = repetitionPenalty
}
func (c *Conversation) GetMaxTokens() *int {
return c.MaxTokens
}
func (c *Conversation) SetMaxTokens(maxTokens *int) {
c.MaxTokens = maxTokens
}
func (c *Conversation) SetContextLength(context int, ignore bool) {
if ignore {
context = 1
} else if context <= 0 {
context = defaultConversationContext
}
c.Context = context
}
@ -211,6 +290,20 @@ func GetMessage(data []byte) (string, error) {
return form.Message, nil
}
func (c *Conversation) ApplyParam(form *FormMessage) {
c.SetModel(form.Model)
c.SetEnableWeb(form.Web)
c.SetContextLength(form.Context, form.IgnoreContext)
c.SetMaxTokens(form.MaxTokens)
c.SetTemperature(form.Temperature)
c.SetTopP(form.TopP)
c.SetTopK(form.TopK)
c.SetPresencePenalty(form.PresencePenalty)
c.SetFrequencyPenalty(form.FrequencyPenalty)
c.SetRepetitionPenalty(form.RepetitionPenalty)
}
func (c *Conversation) AddMessageFromByte(data []byte) (string, error) {
form, err := utils.Unmarshal[FormMessage](data)
if err != nil {
@ -220,16 +313,8 @@ func (c *Conversation) AddMessageFromByte(data []byte) (string, error) {
}
c.AddMessageFromUser(form.Message)
c.SetModel(form.Model)
c.SetEnableWeb(form.Web)
c.ApplyParam(&form)
if form.IgnoreContext {
form.Context = 1
} else if form.Context <= 0 {
form.Context = defaultConversationContext
}
c.SetContextLength(form.Context)
return form.Message, nil
}
@ -239,15 +324,8 @@ func (c *Conversation) AddMessageFromForm(form *FormMessage) error {
}
c.AddMessageFromUser(form.Message)
c.SetModel(form.Model)
c.SetEnableWeb(form.Web)
if form.IgnoreContext {
form.Context = 1
} else if form.Context <= 0 {
form.Context = defaultConversationContext
}
c.ApplyParam(form)
c.SetContextLength(form.Context)
return nil
}

8
manager/images.go
View File

@ -59,10 +59,10 @@ func ImagesRelayAPI(c *gin.Context) {
func getImageProps(form RelayImageForm, messages []globals.Message, buffer *utils.Buffer) *adapter.ChatProps {
return &adapter.ChatProps{
Model: form.Model,
Message: messages,
Token: 2500,
Buffer: *buffer,
Model: form.Model,
Message: messages,
MaxTokens: utils.ToPtr(-1),
Buffer: *buffer,
}
}

4
manager/manager.go
View File

@ -62,8 +62,8 @@ func ChatAPI(c *gin.Context) {
db := utils.GetDBFromContext(c)
var form *WebsocketAuthForm
if form = utils.ReadForm[WebsocketAuthForm](conn); form == nil {
form, err := utils.ReadForm[WebsocketAuthForm](conn)
if err != nil {
return
}

2
manager/types.go
View File

@ -30,7 +30,7 @@ type RelayForm struct {
Model string `json:"model" binding:"required"`
Messages []Message `json:"messages" binding:"required"`
Stream bool `json:"stream"`
MaxTokens int `json:"max_tokens"`
MaxTokens *int `json:"max_tokens"`
PresencePenalty *float32 `json:"presence_penalty"`
FrequencyPenalty *float32 `json:"frequency_penalty"`
RepetitionPenalty *float32 `json:"repetition_penalty"`

8
utils/websocket.go
View File

@ -168,19 +168,19 @@ func (w *WebSocket) IsClosed() bool {
return w.Closed
}
func ReadForm[T interface{}](w *WebSocket) *T {
func ReadForm[T interface{}](w *WebSocket) (*T, error) {
// golang cannot use generic type in class-like struct
// except ping
_, message, err := w.Read()
if err != nil {
return nil
return nil, err
} else if string(message) == "{\"type\":\"ping\"}" {
return ReadForm[T](w)
}
form, err := Unmarshal[T](message)
if err != nil {
return nil
return nil, err
}
return &form
return &form, nil
}