From 04ad6afa98a2f38cc6f9554ed26ee942088aea7a Mon Sep 17 00:00:00 2001 From: Deng Junhai Date: Thu, 27 Jun 2024 23:56:04 +0800 Subject: [PATCH] feat alpha: support searxng feat alpha: support searxng Co-Authored-By: Minghan Zhang <112773885+zmh-program@users.noreply.github.com> --- addition/web/call.go | 14 +++---- addition/web/search.go | 76 ++++++++++++++++++++++++++++++++++++- addition/web/searxng.go | 45 ---------------------- channel/system.go | 45 +++++++++++++--------- globals/variables.go | 10 ++++- manager/chat.go | 2 +- manager/chat_completions.go | 2 +- manager/completions.go | 7 ++-- 8 files changed, 122 insertions(+), 79 deletions(-) delete mode 100644 addition/web/searxng.go diff --git a/addition/web/call.go b/addition/web/call.go index d94373c..f477d3f 100644 --- a/addition/web/call.go +++ b/addition/web/call.go @@ -10,10 +10,8 @@ import ( type Hook func(message []globals.Message, token int) (string, error) -func ChatWithWeb(message []globals.Message) []globals.Message { - data := utils.GetSegmentString( - SearchWebResult(message[len(message)-1].Content), 2048, - ) +func toWebSearchingMessage(message []globals.Message) []globals.Message { + data := GenerateSearchResult(message[len(message)-1].Content) return utils.Insert(message, 0, globals.Message{ Role: globals.System, @@ -24,19 +22,19 @@ func ChatWithWeb(message []globals.Message) []globals.Message { }) } -func UsingWebSegment(instance *conversation.Conversation, restart bool) []globals.Message { +func ToChatSearched(instance *conversation.Conversation, restart bool) []globals.Message { segment := conversation.CopyMessage(instance.GetChatMessage(restart)) if instance.IsEnableWeb() { - segment = ChatWithWeb(segment) + segment = toWebSearchingMessage(segment) } return segment } -func UsingWebNativeSegment(enable bool, message []globals.Message) []globals.Message { +func ToSearched(enable bool, message []globals.Message) []globals.Message { if enable { - return ChatWithWeb(message) + return toWebSearchingMessage(message) } else { return message } diff --git a/addition/web/search.go b/addition/web/search.go index 95c1847..66223aa 100644 --- a/addition/web/search.go +++ b/addition/web/search.go @@ -4,10 +4,77 @@ import ( "chat/globals" "chat/utils" "fmt" + "net/url" + "strconv" + "strings" ) -func SearchWebResult(q string) string { - res, err := CallDuckDuckGoAPI(q) +type SearXNGResponse struct { + Query string `json:"query"` + NumberOfResults int `json:"number_of_results"` + Results []struct { + Url string `json:"url"` + Title string `json:"title"` + Content string `json:"content"` + PublishedDate *string `json:"publishedDate,omitempty"` + Thumbnail *string `json:"thumbnail,omitempty"` + Engine string `json:"engine"` + ParsedUrl []string `json:"parsed_url"` + Template string `json:"template"` + Engines []string `json:"engines"` + Positions []int `json:"positions"` + Score float64 `json:"score"` + Category string `json:"category"` + IframeSrc string `json:"iframe_src,omitempty"` + } `json:"results"` + Answers []interface{} `json:"answers"` + Corrections []interface{} `json:"corrections"` + Infoboxes []interface{} `json:"infoboxes"` + Suggestions []interface{} `json:"suggestions"` + UnresponsiveEngines [][]string `json:"unresponsive_engines"` +} + +func formatResponse(data *SearXNGResponse) string { + res := make([]string, 0) + for _, item := range data.Results { + if item.Content == "" || item.Url == "" || item.Title == "" { + continue + } + + res = append(res, fmt.Sprintf("%s (%s): %s", item.Title, item.Url, item.Content)) + } + + return strings.Join(res, "\n") +} + +func createURLParams(query string) string { + params := url.Values{} + + params.Add("q", url.QueryEscape(query)) + params.Add("format", "json") + params.Add("safesearch", strconv.Itoa(globals.SearchSafeSearch)) + if len(globals.SearchEngines) > 0 { + params.Add("engines", globals.SearchEngines) + } + if len(globals.SearchImageProxy) > 0 { + params.Add("image_proxy", globals.SearchImageProxy) + } + + return fmt.Sprintf("%s?%s", globals.SearchEndpoint, params.Encode()) +} + +func createSearXNGRequest(query string) (*SearXNGResponse, error) { + data, err := utils.Get(createURLParams(query), nil) + + if err != nil { + return nil, err + } + + return utils.MapToRawStruct[SearXNGResponse](data) +} + +func GenerateSearchResult(q string) string { + res, err := createSearXNGRequest(q) if err != nil { globals.Warn(fmt.Sprintf("[web] failed to get search result: %s (query: %s)", err.Error(), q)) return "" @@ -15,5 +82,10 @@ func SearchWebResult(q string) string { content := formatResponse(res) globals.Debug(fmt.Sprintf("[web] search result: %s (query: %s)", utils.Extract(content, 50, "..."), q)) + + if globals.SearchCrop { + globals.Debug(fmt.Sprintf("[web] crop search result length %d to %d max", len(content), globals.SearchCropLength)) + return utils.Extract(content, globals.SearchCropLength, "...") + } return content } diff --git a/addition/web/searxng.go b/addition/web/searxng.go deleted file mode 100644 index dfad6b6..0000000 --- a/addition/web/searxng.go +++ /dev/null @@ -1,45 +0,0 @@ -package web - -import ( - "chat/channel" - "chat/utils" - "fmt" - "net/url" - "strings" -) - -type DDGResponse struct { - Results []struct { - Body string `json:"body"` - Href string `json:"href"` - Title string `json:"title"` - } `json:"results"` -} - -func formatResponse(data *DDGResponse) string { - res := make([]string, 0) - for _, item := range data.Results { - if item.Body == "" || item.Href == "" || item.Title == "" { - continue - } - - res = append(res, fmt.Sprintf("%s (%s): %s", item.Title, item.Href, item.Body)) - } - - return strings.Join(res, "\n") -} - -func CallDuckDuckGoAPI(query string) (*DDGResponse, error) { - data, err := utils.Get(fmt.Sprintf( - "%s/search?q=%s&max_results=%d", - channel.SystemInstance.GetSearchEndpoint(), - url.QueryEscape(query), - channel.SystemInstance.GetSearchQuery(), - ), nil) - - if err != nil { - return nil, err - } - - return utils.MapToRawStruct[DDGResponse](data) -} diff --git a/channel/system.go b/channel/system.go index 51c413c..511eb35 100644 --- a/channel/system.go +++ b/channel/system.go @@ -63,9 +63,13 @@ type mailState struct { WhiteList whiteList `json:"white_list" mapstructure:"whitelist"` } -type searchState struct { - Endpoint string `json:"endpoint" mapstructure:"endpoint"` - Query int `json:"query" mapstructure:"query"` +type SearchState struct { + Endpoint string `json:"endpoint" mapstructure:"endpoint"` + Crop bool `json:"crop" mapstructure:"crop"` + CropLen int `json:"crop_len" mapstructure:"croplen"` + Engines []string `json:"engines" mapstructure:"engines"` + ImageProxy bool `json:"image_proxy" mapstructure:"imageproxy"` + SafeSearch int `json:"safe_search" mapstructure:"safesearch"` } type commonState struct { @@ -113,6 +117,13 @@ func (c *SystemConfig) Load() { if c.General.PWAManifest == "" { c.General.PWAManifest = utils.ReadPWAManifest() } + + globals.SearchEndpoint = c.Search.Endpoint + globals.SearchCrop = c.Search.Crop + globals.SearchCropLength = c.GetSearchCropLength() + globals.SearchEngines = c.GetSearchEngines() + globals.SearchImageProxy = c.GetImageProxy() + globals.SearchSafeSearch = c.Search.SafeSearch } func (c *SystemConfig) SaveConfig() error { @@ -220,27 +231,25 @@ func (c *SystemConfig) SendVerifyMail(email string, code string) error { ) } -func (c *SystemConfig) GetSearchEndpoint() string { - if len(c.Search.Endpoint) == 0 { - return "https://duckduckgo-api.vercel.app" +func (c *SystemConfig) GetSearchCropLength() int { + if c.Search.CropLen <= 0 { + return 1000 } - endpoint := c.Search.Endpoint - if strings.HasSuffix(endpoint, "/search") { - return endpoint[:len(endpoint)-7] - } else if strings.HasSuffix(endpoint, "/") { - return endpoint[:len(endpoint)-1] - } - - return endpoint + return c.Search.CropLen } -func (c *SystemConfig) GetSearchQuery() int { - if c.Search.Query <= 0 { - return 5 +func (c *SystemConfig) GetSearchEngines() string { + return strings.Join(c.Search.Engines, ",") +} + +func (c *SystemConfig) GetImageProxy() string { + // return "True" or "False" + if c.Search.ImageProxy { + return "True" } - return c.Search.Query + return "False" } func (c *SystemConfig) GetAppName() string { diff --git a/globals/variables.go b/globals/variables.go index b8ef9a3..68b9973 100644 --- a/globals/variables.go +++ b/globals/variables.go @@ -1,9 +1,10 @@ package globals import ( - "github.com/gin-gonic/gin" "net/url" "strings" + + "github.com/gin-gonic/gin" ) const ChatMaxThread = 5 @@ -22,6 +23,13 @@ var AcceptImageStore bool var CloseRegistration bool var CloseRelay bool +var SearchEndpoint string +var SearchCrop bool +var SearchCropLength int +var SearchEngines string // e.g. "google,bing" +var SearchImageProxy string // e.g. "True", "False" +var SearchSafeSearch int // e.g. 0: None, 1: Moderation, 2: Strict + func OriginIsAllowed(uri string) bool { if len(AllowedOrigins) == 0 { // if allowed origins is empty, allow all origins diff --git a/manager/chat.go b/manager/chat.go index e43af54..e8d6f6e 100644 --- a/manager/chat.go +++ b/manager/chat.go @@ -197,7 +197,7 @@ func ChatHandler(conn *Connection, user *auth.User, instance *conversation.Conve cache := conn.GetCache() model := instance.GetModel() - segment := adapter.ClearMessages(model, web.UsingWebSegment(instance, restart)) + segment := adapter.ClearMessages(model, web.ToChatSearched(instance, restart)) check, plan := auth.CanEnableModelWithSubscription(db, cache, user, model) conn.Send(globals.ChatSegmentResponse{ diff --git a/manager/chat_completions.go b/manager/chat_completions.go index c9b887c..2ac8a7a 100644 --- a/manager/chat_completions.go +++ b/manager/chat_completions.go @@ -72,7 +72,7 @@ func ChatRelayAPI(c *gin.Context) { suffix := strings.TrimPrefix(form.Model, "web-") form.Model = suffix - messages = web.UsingWebNativeSegment(true, messages) + messages = web.ToSearched(true, messages) } if strings.HasSuffix(form.Model, "-official") { diff --git a/manager/completions.go b/manager/completions.go index 47ad6b4..567038d 100644 --- a/manager/completions.go +++ b/manager/completions.go @@ -1,7 +1,7 @@ package manager import ( - "chat/adapter/common" + adaptercommon "chat/adapter/common" "chat/addition/web" "chat/admin" "chat/auth" @@ -9,8 +9,9 @@ import ( "chat/globals" "chat/utils" "fmt" - "github.com/gin-gonic/gin" "runtime/debug" + + "github.com/gin-gonic/gin" ) func NativeChatHandler(c *gin.Context, user *auth.User, model string, message []globals.Message, enableWeb bool) (string, float32) { @@ -23,7 +24,7 @@ func NativeChatHandler(c *gin.Context, user *auth.User, model string, message [] } }() - segment := web.UsingWebNativeSegment(enableWeb, message) + segment := web.ToSearched(enableWeb, message) db := utils.GetDBFromContext(c) cache := utils.GetCacheFromContext(c)