GithubProxy/coai
1
0
Fork 0
mirror of https://github.com/coaidev/coai.git synced 2025-06-03 12:20:15 +09:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat alpha: support searxng

Browse Source 
feat alpha: support searxng

Co-Authored-By: Minghan Zhang <112773885+zmh-program@users.noreply.github.com>
This commit is contained in:
Deng Junhai 2024-06-27 23:56:04 +08:00
parent 401de5ace7
commit 04ad6afa98
8 changed files with 122 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
addition/web/call.go
View File

@ -10,10 +10,8 @@ import (
type Hook func(message []globals.Message, token int) (string, error)
func ChatWithWeb(message []globals.Message) []globals.Message {
data := utils.GetSegmentString(
SearchWebResult(message[len(message)-1].Content), 2048,
)
func toWebSearchingMessage(message []globals.Message) []globals.Message {
data := GenerateSearchResult(message[len(message)-1].Content)
return utils.Insert(message, 0, globals.Message{
Role: globals.System,
@ -24,19 +22,19 @@ func ChatWithWeb(message []globals.Message) []globals.Message {
})
}
func UsingWebSegment(instance *conversation.Conversation, restart bool) []globals.Message {
func ToChatSearched(instance *conversation.Conversation, restart bool) []globals.Message {
segment := conversation.CopyMessage(instance.GetChatMessage(restart))
if instance.IsEnableWeb() {
segment = ChatWithWeb(segment)
segment = toWebSearchingMessage(segment)
}
return segment
}
func UsingWebNativeSegment(enable bool, message []globals.Message) []globals.Message {
func ToSearched(enable bool, message []globals.Message) []globals.Message {
if enable {
return ChatWithWeb(message)
return toWebSearchingMessage(message)
} else {
return message
}

76
addition/web/search.go
View File

@ -4,10 +4,77 @@ import (
"chat/globals"
"chat/utils"
"fmt"
"net/url"
"strconv"
"strings"
)
func SearchWebResult(q string) string {
res, err := CallDuckDuckGoAPI(q)
type SearXNGResponse struct {
Query string `json:"query"`
NumberOfResults int `json:"number_of_results"`
Results []struct {
Url string `json:"url"`
Title string `json:"title"`
Content string `json:"content"`
PublishedDate *string `json:"publishedDate,omitempty"`
Thumbnail *string `json:"thumbnail,omitempty"`
Engine string `json:"engine"`
ParsedUrl []string `json:"parsed_url"`
Template string `json:"template"`
Engines []string `json:"engines"`
Positions []int `json:"positions"`
Score float64 `json:"score"`
Category string `json:"category"`
IframeSrc string `json:"iframe_src,omitempty"`
} `json:"results"`
Answers []interface{} `json:"answers"`
Corrections []interface{} `json:"corrections"`
Infoboxes []interface{} `json:"infoboxes"`
Suggestions []interface{} `json:"suggestions"`
UnresponsiveEngines [][]string `json:"unresponsive_engines"`
}
func formatResponse(data *SearXNGResponse) string {
res := make([]string, 0)
for _, item := range data.Results {
if item.Content == "" || item.Url == "" || item.Title == "" {
continue
}
res = append(res, fmt.Sprintf("%s (%s): %s", item.Title, item.Url, item.Content))
}
return strings.Join(res, "\n")
}
func createURLParams(query string) string {
params := url.Values{}
params.Add("q", url.QueryEscape(query))
params.Add("format", "json")
params.Add("safesearch", strconv.Itoa(globals.SearchSafeSearch))
if len(globals.SearchEngines) > 0 {
params.Add("engines", globals.SearchEngines)
}
if len(globals.SearchImageProxy) > 0 {
params.Add("image_proxy", globals.SearchImageProxy)
}
return fmt.Sprintf("%s?%s", globals.SearchEndpoint, params.Encode())
}
func createSearXNGRequest(query string) (*SearXNGResponse, error) {
data, err := utils.Get(createURLParams(query), nil)
if err != nil {
return nil, err
}
return utils.MapToRawStruct[SearXNGResponse](data)
}
func GenerateSearchResult(q string) string {
res, err := createSearXNGRequest(q)
if err != nil {
globals.Warn(fmt.Sprintf("[web] failed to get search result: %s (query: %s)", err.Error(), q))
return ""
@ -15,5 +82,10 @@ func SearchWebResult(q string) string {
content := formatResponse(res)
globals.Debug(fmt.Sprintf("[web] search result: %s (query: %s)", utils.Extract(content, 50, "..."), q))
if globals.SearchCrop {
globals.Debug(fmt.Sprintf("[web] crop search result length %d to %d max", len(content), globals.SearchCropLength))
return utils.Extract(content, globals.SearchCropLength, "...")
}
return content
}

45
addition/web/searxng.go
View File

@ -1,45 +0,0 @@
package web
import (
"chat/channel"
"chat/utils"
"fmt"
"net/url"
"strings"
)
type DDGResponse struct {
Results []struct {
Body string `json:"body"`
Href string `json:"href"`
Title string `json:"title"`
} `json:"results"`
}
func formatResponse(data *DDGResponse) string {
res := make([]string, 0)
for _, item := range data.Results {
if item.Body == "" || item.Href == "" || item.Title == "" {
continue
}
res = append(res, fmt.Sprintf("%s (%s): %s", item.Title, item.Href, item.Body))
}
return strings.Join(res, "\n")
}
func CallDuckDuckGoAPI(query string) (*DDGResponse, error) {
data, err := utils.Get(fmt.Sprintf(
"%s/search?q=%s&max_results=%d",
channel.SystemInstance.GetSearchEndpoint(),
url.QueryEscape(query),
channel.SystemInstance.GetSearchQuery(),
), nil)
if err != nil {
return nil, err
}
return utils.MapToRawStruct[DDGResponse](data)
}

45
channel/system.go
View File

@ -63,9 +63,13 @@ type mailState struct {
WhiteList whiteList `json:"white_list" mapstructure:"whitelist"`
}
type searchState struct {
Endpoint string `json:"endpoint" mapstructure:"endpoint"`
Query int `json:"query" mapstructure:"query"`
type SearchState struct {
Endpoint string `json:"endpoint" mapstructure:"endpoint"`
Crop bool `json:"crop" mapstructure:"crop"`
CropLen int `json:"crop_len" mapstructure:"croplen"`
Engines []string `json:"engines" mapstructure:"engines"`
ImageProxy bool `json:"image_proxy" mapstructure:"imageproxy"`
SafeSearch int `json:"safe_search" mapstructure:"safesearch"`
}
type commonState struct {
@ -113,6 +117,13 @@ func (c *SystemConfig) Load() {
if c.General.PWAManifest == "" {
c.General.PWAManifest = utils.ReadPWAManifest()
}
globals.SearchEndpoint = c.Search.Endpoint
globals.SearchCrop = c.Search.Crop
globals.SearchCropLength = c.GetSearchCropLength()
globals.SearchEngines = c.GetSearchEngines()
globals.SearchImageProxy = c.GetImageProxy()
globals.SearchSafeSearch = c.Search.SafeSearch
}
func (c *SystemConfig) SaveConfig() error {
@ -220,27 +231,25 @@ func (c *SystemConfig) SendVerifyMail(email string, code string) error {
)
}
func (c *SystemConfig) GetSearchEndpoint() string {
if len(c.Search.Endpoint) == 0 {
return "https://duckduckgo-api.vercel.app"
func (c *SystemConfig) GetSearchCropLength() int {
if c.Search.CropLen <= 0 {
return 1000
}
endpoint := c.Search.Endpoint
if strings.HasSuffix(endpoint, "/search") {
return endpoint[:len(endpoint)-7]
} else if strings.HasSuffix(endpoint, "/") {
return endpoint[:len(endpoint)-1]
}
return endpoint
return c.Search.CropLen
}
func (c *SystemConfig) GetSearchQuery() int {
if c.Search.Query <= 0 {
return 5
func (c *SystemConfig) GetSearchEngines() string {
return strings.Join(c.Search.Engines, ",")
}
func (c *SystemConfig) GetImageProxy() string {
// return "True" or "False"
if c.Search.ImageProxy {
return "True"
}
return c.Search.Query
return "False"
}
func (c *SystemConfig) GetAppName() string {

10
globals/variables.go
View File

@ -1,9 +1,10 @@
package globals
import (
"github.com/gin-gonic/gin"
"net/url"
"strings"
"github.com/gin-gonic/gin"
)
const ChatMaxThread = 5
@ -22,6 +23,13 @@ var AcceptImageStore bool
var CloseRegistration bool
var CloseRelay bool
var SearchEndpoint string
var SearchCrop bool
var SearchCropLength int
var SearchEngines string // e.g. "google,bing"
var SearchImageProxy string // e.g. "True", "False"
var SearchSafeSearch int // e.g. 0: None, 1: Moderation, 2: Strict
func OriginIsAllowed(uri string) bool {
if len(AllowedOrigins) == 0 {
// if allowed origins is empty, allow all origins

2
manager/chat.go
View File

@ -197,7 +197,7 @@ func ChatHandler(conn *Connection, user *auth.User, instance *conversation.Conve
cache := conn.GetCache()
model := instance.GetModel()
segment := adapter.ClearMessages(model, web.UsingWebSegment(instance, restart))
segment := adapter.ClearMessages(model, web.ToChatSearched(instance, restart))
check, plan := auth.CanEnableModelWithSubscription(db, cache, user, model)
conn.Send(globals.ChatSegmentResponse{

2
manager/chat_completions.go
View File

@ -72,7 +72,7 @@ func ChatRelayAPI(c *gin.Context) {
suffix := strings.TrimPrefix(form.Model, "web-")
form.Model = suffix
messages = web.UsingWebNativeSegment(true, messages)
messages = web.ToSearched(true, messages)
}
if strings.HasSuffix(form.Model, "-official") {

7
manager/completions.go
View File

@ -1,7 +1,7 @@
package manager
import (
"chat/adapter/common"
adaptercommon "chat/adapter/common"
"chat/addition/web"
"chat/admin"
"chat/auth"
@ -9,8 +9,9 @@ import (
"chat/globals"
"chat/utils"
"fmt"
"github.com/gin-gonic/gin"
"runtime/debug"
"github.com/gin-gonic/gin"
)
func NativeChatHandler(c *gin.Context, user *auth.User, model string, message []globals.Message, enableWeb bool) (string, float32) {
@ -23,7 +24,7 @@ func NativeChatHandler(c *gin.Context, user *auth.User, model string, message []
}
}()
segment := web.UsingWebNativeSegment(enableWeb, message)
segment := web.ToSearched(enableWeb, message)
db := utils.GetDBFromContext(c)
cache := utils.GetCacheFromContext(c)