update project generation backend

.gitignore

@@ -2,6 +2,8 @@ node_modules
 .vscode
 .idea
 config.yaml
+generation/data/*
+!generation/data/.gitkeep
 
 
 # for reverse engine

api/buffer.go

@@ -61,6 +61,10 @@ func (b *Buffer) Read() string {
 	return b.Data
 }
 
+func (b *Buffer) ReadBytes() []byte {
+	return []byte(b.Data)
+}
+
 func (b *Buffer) ReadWithDefault(_default string) string {
 	if b.IsEmpty() {
 		return _default

api/chat.go

@@ -18,7 +18,7 @@ import (
 const defaultErrorMessage = "There was something wrong... Please try again later."
 const defaultQuotaMessage = "You have run out of GPT-4 usage. Please keep your nio points above **5**."
 const defaultImageMessage = "Please provide description for the image (e.g. /image an apple)."
-const maxThread = 3
+const maxThread = 5
 
 type WebsocketAuthForm struct {
 	Token string `json:"token" binding:"required"`
@@ -60,7 +60,9 @@ func TextChat(db *sql.DB, cache *redis.Client, user *auth.User, conn *websocket.
 	}
 
 	buffer := NewBuffer(instance.IsEnableGPT4(), segment)
-	StreamRequest(instance.IsEnableGPT4(), isProPlan, segment, 2000, func(resp string) {
+	StreamRequest(instance.IsEnableGPT4(), isProPlan, segment,
+		utils.Multi(instance.IsEnableGPT4() || isProPlan, -1, 2000),
+		func(resp string) {
 			SendSegmentMessage(conn, types.ChatGPTSegmentResponse{
 				Message: buffer.Write(resp),
 				Quota:   buffer.GetQuota(),

api/stream.go

@@ -51,16 +51,28 @@ func processLine(buf []byte) []string {
 	return resp
 }
 
-func NativeStreamRequest(model string, endpoint string, apikeys string, messages []types.ChatGPTMessage, token int, callback func(string)) {
+func MixRequestBody(model string, messages []types.ChatGPTMessage, token int) interface{} {
-	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	if token == -1 {
+		return types.ChatGPTRequestWithInfinity{
+			Model:    model,
+			Messages: messages,
+			Stream:   true,
+		}
+	}
 
-	client := &http.Client{}
+	return types.ChatGPTRequest{
-	req, err := http.NewRequest("POST", endpoint+"/chat/completions", utils.ConvertBody(types.ChatGPTRequest{
 		Model:    model,
 		Messages: messages,
 		MaxToken: token,
 		Stream:   true,
-	}))
+	}
+}
+
+func NativeStreamRequest(model string, endpoint string, apikeys string, messages []types.ChatGPTMessage, token int, callback func(string)) {
+	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+
+	client := &http.Client{}
+	req, err := http.NewRequest("POST", endpoint+"/chat/completions", utils.ConvertBody(MixRequestBody(model, messages, token)))
 	if err != nil {
 		fmt.Println(err)
 		return
@@ -71,6 +83,7 @@ func NativeStreamRequest(model string, endpoint string, apikeys string, messages
 
 	res, err := client.Do(req)
 	if err != nil {
+		fmt.Println(err.Error())
 		return
 	}
 	defer res.Body.Close()

generation/build.go

@@ -0,0 +1,31 @@
+package generation
+
+import (
+	"chat/utils"
+	"fmt"
+)
+
+func GetFolder(hash string) string {
+	return fmt.Sprintf("generation/data/%s", hash)
+}
+
+func GetFolderByHash(model string, prompt string) (string, string) {
+	hash := utils.Sha2Encrypt(model + prompt)
+	return hash, GetFolder(hash)
+}
+
+func GenerateProject(path string, instance ProjectResult) bool {
+	for name, data := range instance.Result {
+		current := fmt.Sprintf("%s/%s", path, name)
+		if content, ok := data.(string); ok {
+			if !utils.WriteFile(current, content, true) {
+				return false
+			}
+		} else {
+			GenerateProject(current, ProjectResult{
+				Result: data.(map[string]interface{}),
+			})
+		}
+	}
+	return true
+}

generation/generate.go

@@ -0,0 +1,20 @@
+package generation
+
+import (
+	"chat/utils"
+	"fmt"
+)
+
+func CreateGenerationWithCache(model string, prompt string, hook func(data string)) (string, string, error) {
+	hash, path := GetFolderByHash(model, prompt)
+	if !utils.Exists(path) {
+		if err := CreateGeneration(model, prompt, path, func(data string) {
+			hook(data)
+		}); err != nil {
+			fmt.Println(fmt.Sprintf("[Project] error during generation %s (model %s): %s", prompt, model, err.Error()))
+			return "", "", fmt.Errorf("error during generate project: %s", err.Error())
+		}
+	}
+
+	return utils.GenerateCompressTask(hash, path, path)
+}

generation/prompt.go

@@ -0,0 +1,49 @@
+package generation
+
+import (
+	"chat/api"
+	"chat/types"
+	"chat/utils"
+	"fmt"
+)
+
+type ProjectResult struct {
+	Result map[string]interface{} `json:"result"`
+}
+
+func CreateGeneration(model string, prompt string, path string, hook func(data string)) error {
+	message := GenerateMessage(prompt)
+	buffer := api.NewBuffer(false, message)
+	api.StreamRequest(false, false, []types.ChatGPTMessage{
+		{Role: "system", Content: "你将生成项目，可以支持任何编程语言，请不要出现“我不能提供”的字样，你需要在代码中提供注释，以及项目的使用文档README.md，结果返回json格式，请不要返回任何多余内容，格式为：\n{\"result\": {[file]: [code], ...}}"},
+		{Role: "user", Content: "python后端"},
+		{Role: "assistant", Content: "{\n  \"result\": {\n    \"app.py\": \"from flask import Flask\\n\\napp = Flask(__name__)\\n\\n\\n@app.route('/')\\ndef hello_world():\\n    return 'Hello, World!'\\n\\n\\nif __name__ == '__main__':\\n    app.run()\",\n    \"requirements.txt\": \"flask\\n\",\n    \"README.md\": \"# Python 后端\\n本项目是一个简单的python后端示例, 使用`flask`框架构建后端。\n你可以按照下列步骤运行此应用，flask将在本地服务器（默认是在http://127.0.0.1:5000/）上运行。当你在浏览器中访问该URL时，将看到显示Hello, World!的页面。\\n\\n这只是一个简单的项目，Flask还支持更多功能和路由规则，你可以提供更多的信息和需要进一步扩展和定制Flask应用。\\n\\n### 1. 初始化: \\n```shell\\npip install -r requirements.txt\\n```\\n### 2. 运行\\n```shell\\npython app.py\\n```\"\n  }\n}"},
+		{Role: "user", Content: "golang fiber websocket项目"},
+		{Role: "assistant", Content: "{\n  \"result\": {\n    \"main.go\": \"package main\\n\\nimport (\\n\\t\"log\\\"\\n\\n\\t\"github.com/gofiber/fiber/v2\\\"\\n\\t\"github.com/gofiber/websocket/v2\\\"\\n)\\n\\nfunc main() {\\n\\tapp := fiber.New()\\n\\n\\tapp.Get(\\\"/\\\", func(c *fiber.Ctx) error {\\n\\t\\treturn c.SendString(\\\"Hello, World!\\\")\\n\\t})\\n\\n\\tapp.Get(\\\"/ws\\\", websocket.New(func(c *websocket.Conn) {\\n\\t\\tfor {\\n\\t\\t\\tmt, message, err := c.ReadMessage()\\n\\t\\t\\tif err != nil {\\n\\t\\t\\t\\tlog.Println(\\\"read error:\\\", err)\\n\\t\\t\\t\\tbreak\\n\\t\\t\\t}\\n\\t\\t\\tlog.Printf(\\\"received: %s\\\", message)\\n\\t\\t\\terr = c.WriteMessage(mt, message)\\n\\t\\t\\tif err != nil {\\n\\t\\t\\t\\tlog.Println(\\\"write error:\\\", err)\\n\\t\\t\\t\\tbreak\\n\\t\\t\\t}\\n\\t\\t}\\n\\t}))\\n\\n\\tlog.Fatal(app.Listen(\\\":3000\\\"))\\n}\",\n    \"go.mod\": \"module fiber-websocket\\n\\ngo 1.16\\n\\nrequire (\\n\\tgithub.com/gofiber/fiber/v2 v2.12.1\\n\\tgithub.com/gofiber/websocket/v2 v2.10.2\\n)\",\n    \"README.md\": \"# Golang Fiber WebSocket项目\\n\\n这个项目是一个使用Golang和Fiber框架构建的WebSocket服务器示例。\\n\\n### 1. 初始化：\\n```shell\\ngo mod init fiber-websocket\\n```\\n\\n### 2. 安装依赖：\\n```shell\\ngo get github.com/gofiber/fiber/v2\\n```   \\n```shell\\ngo get github.com/gofiber/websocket/v2\\n```\\n\\n### 3. 创建main.go文件，将以下代码复制粘贴：\\n\\n```go\\npackage main\\n\\nimport (\\n\\t\\\"log\\\"\\n\\n\\t\\\"github.com/gofiber/fiber/v2\\\"\\n\\t\\\"github.com/gofiber/websocket/v2\\\"\\n)\\n\\nfunc main() {\\n\\tapp := fiber.New()\\n\\n\\tapp.Get(\\\"/\\\", func(c *fiber.Ctx) error {\\n\\t\\treturn c.SendString(\\\"Hello, World!\\\")\\n\\t})\\n\\n\\tapp.Get(\\\"/ws\\\", websocket.New(func(c *websocket.Conn) {\\n\\t\\tfor {\\n\\t\\t\\tmt, message, err := c.ReadMessage()\\n\\t\\t\\tif err != nil {\\n\\t\\t\\t\\tlog.Println(\\\"read error:\\\", err)\\n\\t\\t\\t\\tbreak\\n\\t\\t\\t}\\n\\t\\t\\tlog.Printf(\\\"received: %s\\\", message)\\n\\t\\t\\terr = c.WriteMessage(mt, message)\\n\\t\\t\\tif err != nil {\\n\\t\\t\\t\\tlog.Println(\\\"write error:\\\", err)\\n\\t\\t\\t\\tbreak\\n\\t\\t\\t}\\n\\t\\t}\\n\\t}))\\n\\n\\tlog.Fatal(app.Listen(\\\":3000\\\"))\\n}\\n```\\n\\n### 4. 运行应用程序：\\n```shell\\ngo run main.go\\n```\\n\\n应用程序将在本地服务器（默认是在http://localhost:3000）上运行。当你在浏览器中访问`http://localhost:3000`时，将看到显示\"Hello, World!\"的页面。你还可以访问`http://localhost:3000/ws`来测试WebSocket功能。\n\n这只是一个简单的示例，Fiber框架提供了更多的功能和路由规则，你可以在此基础上进行进一步扩展和定制。\n\n注意：在运行应用程序之前，请确保已经安装了Go语言开发环境。"},
+		{Role: "user", Content: prompt},
+	}, -1, func(data string) {
+		hook(data)
+		buffer.Write(data)
+	})
+
+	resp, err := utils.Unmarshal[ProjectResult](buffer.ReadBytes())
+	if err != nil {
+		return err
+	}
+
+	if !GenerateProject(path, resp) {
+		return fmt.Errorf("generate project failed")
+	}
+	return nil
+}
+
+func GenerateMessage(prompt string) []types.ChatGPTMessage {
+	return []types.ChatGPTMessage{
+		{Role: "system", Content: "你将生成项目，可以支持任何编程语言，请不要出现“我不能提供”的字样，你需要在代码中提供注释，以及项目的使用文档README.md，结果返回json格式，请不要返回任何多余内容，格式为：\n{\"result\": {[file]: [code], ...}}"},
+		{Role: "user", Content: "python后端"},
+		{Role: "assistant", Content: "{\n  \"result\": {\n    \"app.py\": \"from flask import Flask\\n\\napp = Flask(__name__)\\n\\n\\n@app.route('/')\\ndef hello_world():\\n    return 'Hello, World!'\\n\\n\\nif __name__ == '__main__':\\n    app.run()\",\n    \"requirements.txt\": \"flask\\n\",\n    \"README.md\": \"# Python 后端\\n本项目是一个简单的python后端示例, 使用`flask`框架构建后端。\n你可以按照下列步骤运行此应用，flask将在本地服务器（默认是在http://127.0.0.1:5000/）上运行。当你在浏览器中访问该URL时，将看到显示Hello, World!的页面。\\n\\n这只是一个简单的项目，Flask还支持更多功能和路由规则，你可以提供更多的信息和需要进一步扩展和定制Flask应用。\\n\\n### 1. 初始化: \\n```shell\\npip install -r requirements.txt\\n```\\n### 2. 运行\\n```shell\\npython app.py\\n```\"\n  }\n}"},
+		{Role: "user", Content: "golang fiber websocket项目"},
+		{Role: "assistant", Content: "{\n  \"result\": {\n    \"main.go\": \"package main\\n\\nimport (\\n\\t\"log\\\"\\n\\n\\t\"github.com/gofiber/fiber/v2\\\"\\n\\t\"github.com/gofiber/websocket/v2\\\"\\n)\\n\\nfunc main() {\\n\\tapp := fiber.New()\\n\\n\\tapp.Get(\\\"/\\\", func(c *fiber.Ctx) error {\\n\\t\\treturn c.SendString(\\\"Hello, World!\\\")\\n\\t})\\n\\n\\tapp.Get(\\\"/ws\\\", websocket.New(func(c *websocket.Conn) {\\n\\t\\tfor {\\n\\t\\t\\tmt, message, err := c.ReadMessage()\\n\\t\\t\\tif err != nil {\\n\\t\\t\\t\\tlog.Println(\\\"read error:\\\", err)\\n\\t\\t\\t\\tbreak\\n\\t\\t\\t}\\n\\t\\t\\tlog.Printf(\\\"received: %s\\\", message)\\n\\t\\t\\terr = c.WriteMessage(mt, message)\\n\\t\\t\\tif err != nil {\\n\\t\\t\\t\\tlog.Println(\\\"write error:\\\", err)\\n\\t\\t\\t\\tbreak\\n\\t\\t\\t}\\n\\t\\t}\\n\\t}))\\n\\n\\tlog.Fatal(app.Listen(\\\":3000\\\"))\\n}\",\n    \"go.mod\": \"module fiber-websocket\\n\\ngo 1.16\\n\\nrequire (\\n\\tgithub.com/gofiber/fiber/v2 v2.12.1\\n\\tgithub.com/gofiber/websocket/v2 v2.10.2\\n)\",\n    \"README.md\": \"# Golang Fiber WebSocket项目\\n\\n这个项目是一个使用Golang和Fiber框架构建的WebSocket服务器示例。\\n\\n### 1. 初始化：\\n```shell\\ngo mod init fiber-websocket\\n```\\n\\n### 2. 安装依赖：\\n```shell\\ngo get github.com/gofiber/fiber/v2\\n```   \\n```shell\\ngo get github.com/gofiber/websocket/v2\\n```\\n\\n### 3. 创建main.go文件，将以下代码复制粘贴：\\n\\n```go\\npackage main\\n\\nimport (\\n\\t\\\"log\\\"\\n\\n\\t\\\"github.com/gofiber/fiber/v2\\\"\\n\\t\\\"github.com/gofiber/websocket/v2\\\"\\n)\\n\\nfunc main() {\\n\\tapp := fiber.New()\\n\\n\\tapp.Get(\\\"/\\\", func(c *fiber.Ctx) error {\\n\\t\\treturn c.SendString(\\\"Hello, World!\\\")\\n\\t})\\n\\n\\tapp.Get(\\\"/ws\\\", websocket.New(func(c *websocket.Conn) {\\n\\t\\tfor {\\n\\t\\t\\tmt, message, err := c.ReadMessage()\\n\\t\\t\\tif err != nil {\\n\\t\\t\\t\\tlog.Println(\\\"read error:\\\", err)\\n\\t\\t\\t\\tbreak\\n\\t\\t\\t}\\n\\t\\t\\tlog.Printf(\\\"received: %s\\\", message)\\n\\t\\t\\terr = c.WriteMessage(mt, message)\\n\\t\\t\\tif err != nil {\\n\\t\\t\\t\\tlog.Println(\\\"write error:\\\", err)\\n\\t\\t\\t\\tbreak\\n\\t\\t\\t}\\n\\t\\t}\\n\\t}))\\n\\n\\tlog.Fatal(app.Listen(\\\":3000\\\"))\\n}\\n```\\n\\n### 4. 运行应用程序：\\n```shell\\ngo run main.go\\n```\\n\\n应用程序将在本地服务器（默认是在http://localhost:3000）上运行。当你在浏览器中访问`http://localhost:3000`时，将看到显示\"Hello, World!\"的页面。你还可以访问`http://localhost:3000/ws`来测试WebSocket功能。\n\n这只是一个简单的示例，Fiber框架提供了更多的功能和路由规则，你可以在此基础上进行进一步扩展和定制。\n\n注意：在运行应用程序之前，请确保已经安装了Go语言开发环境。"},
+		{Role: "user", Content: prompt},
+	}
+}

main.go

@@ -5,6 +5,7 @@ import (
 	"chat/auth"
 	"chat/connection"
 	"chat/conversation"
+	"chat/generation"
 	"chat/middleware"
 	"github.com/gin-gonic/gin"
 	"github.com/spf13/viper"
@@ -16,6 +17,8 @@ func main() {
 		panic(err)
 	}
 
+	generation.CreateGenerationWithCache("", "写一个turbowarp插件")
+	return
 	app := gin.Default()
 	{
 		app.Use(middleware.CORSMiddleware())

types/types.go

@@ -12,6 +12,12 @@ type ChatGPTRequest struct {
 	Stream   bool             `json:"stream"`
 }
 
+type ChatGPTRequestWithInfinity struct {
+	Model    string           `json:"model"`
+	Messages []ChatGPTMessage `json:"messages"`
+	Stream   bool             `json:"stream"`
+}
+
 type ChatGPTImageRequest struct {
 	Prompt string `json:"prompt"`
 	Size   string `json:"size"`

utils/base.go

@@ -92,3 +92,11 @@ func Reverse[T any](arr []T) []T {
 	}
 	return arr
 }
+
+func Multi[T comparable](condition bool, tval, fval T) T {
+	if condition {
+		return tval
+	} else {
+		return fval
+	}
+}

utils/compress.go

@@ -0,0 +1,155 @@
+package utils
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+func GenerateCompressTask(hash string, path string, replacer string) (string, string, error) {
+	CreateFolder("generation/data/out")
+	zipPath := fmt.Sprintf("generation/data/out/%s.zip", hash)
+	gzipPath := fmt.Sprintf("generation/data/out/%s.tar.gz", hash)
+
+	files := Walk(path)
+
+	if err := createZipObject(zipPath, files, replacer); err != nil {
+		return "", "", err
+	}
+
+	if err := createGzipObject(gzipPath, files, replacer); err != nil {
+		return "", "", err
+	}
+
+	return zipPath, gzipPath, nil
+}
+
+func GenerateCompressTaskAsync(hash string, path string, replacer string) (string, string) {
+	zipPath := fmt.Sprintf("generation/data/out/%s.zip", hash)
+	gzipPath := fmt.Sprintf("generation/data/out/%s.tar.gz", hash)
+
+	files := Walk(path)
+
+	go func() {
+		if err := createZipObject(zipPath, files, replacer); err != nil {
+			return
+		}
+	}()
+
+	go func() {
+		if err := createGzipObject(gzipPath, files, replacer); err != nil {
+			return
+		}
+	}()
+
+	return zipPath, gzipPath
+}
+
+func createZipObject(output string, files []string, replacer string) error {
+	file, err := os.Create(output)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	writer := zip.NewWriter(file)
+	defer writer.Close()
+
+	for _, file := range files {
+		err := addFileToZip(writer, file, replacer)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func addFileToZip(zipWriter *zip.Writer, path string, replacer string) error {
+	file, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	info, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	header, err := zip.FileInfoHeader(info)
+	if err != nil {
+		return err
+	}
+	header.Name = strings.Trim(strings.Replace(path, replacer, "", 1), "/")
+	writer, err := zipWriter.CreateHeader(header)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(writer, file)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func createGzipObject(output string, files []string, replacer string) error {
+	tarFile, err := os.Create(output)
+	if err != nil {
+		return err
+	}
+	defer tarFile.Close()
+
+	gzWriter := gzip.NewWriter(tarFile)
+	defer gzWriter.Close()
+
+	tarWriter := tar.NewWriter(gzWriter)
+	defer tarWriter.Close()
+
+	for _, file := range files {
+		err := addFileToTar(tarWriter, file, replacer)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// tar gzip
+func addFileToTar(tarWriter *tar.Writer, filePath string, replacer string) error {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	info, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	header, err := tar.FileInfoHeader(info, "")
+	if err != nil {
+		return err
+	}
+
+	header.Name = strings.Trim(strings.Replace(filePath, replacer, "", 1), "/")
+
+	err = tarWriter.WriteHeader(header)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(tarWriter, file)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

utils/fs.go

@@ -0,0 +1,64 @@
+package utils
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func CreateFolder(path string) bool {
+	if err := os.MkdirAll(path, os.ModePerm); err != nil && !os.IsExist(err) {
+		return false
+	}
+	return true
+}
+
+func Exists(path string) bool {
+	err := os.Mkdir(path, os.ModePerm)
+	return err != nil && os.IsExist(err)
+}
+
+func CreateFolderNotExists(path string) string {
+	CreateFolder(path)
+	return path
+}
+
+func CreateFolderOnFile(file string) string {
+	return CreateFolderNotExists(file[:strings.LastIndex(file, "/")])
+}
+
+func WriteFile(path string, data string, folderSafe bool) bool {
+	if folderSafe {
+		CreateFolderOnFile(path)
+	}
+
+	file, err := os.Create(path)
+	if err != nil {
+		return false
+	}
+	defer file.Close()
+
+	if _, err := file.WriteString(data); err != nil {
+		fmt.Println(err.Error())
+		return false
+	}
+	return true
+}
+
+func Walk(path string) []string {
+	var files []string
+	err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+		if !info.IsDir() {
+			files = append(files, path)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil
+	}
+	return files
+}