Skip to content

Commit 812edf1

Browse files
johnlannijohnlanni
authored
add ai cache plugin (#1010)
1 parent b00f79f commit 812edf1

File tree

10 files changed

+543
-5
lines changed

10 files changed

+543
-5
lines changed

‎pkg/cmd/hgctl/plugin/init/init.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"fmt"
1919
"io"
2020
"os"
21+
"os/exec"
2122

2223
"github.com/alibaba/higress/pkg/cmd/hgctl/plugin/option"
2324
"github.com/alibaba/higress/pkg/cmd/hgctl/plugin/utils"
@@ -86,6 +87,12 @@ func runInit(w io.Writer, target string) (err error) {
8687
return errors.Wrap(err, "failed to create option.yaml")
8788
}
8889

90+
cmd := exec.Command("go", "mod", "tidy")
91+
cmd.Dir = dir
92+
if err := cmd.Run(); err != nil {
93+
return errors.Wrap(err, "failed to run go mod tidy")
94+
}
95+
8996
fmt.Fprintf(w, "Initialized the project in %q\n", dir)
9097

9198
return nil

‎pkg/cmd/hgctl/plugin/init/templates.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ package main
3131
3232
import (
3333
"github.com/tidwall/gjson"
34-
"github.com/tetratelabs/proxy-wasm-go-sdk/proxywasm"
35-
"github.com/tetratelabs/proxy-wasm-go-sdk/proxywasm/types"
34+
"github.com/higress-group/proxy-wasm-go-sdk/proxywasm"
35+
"github.com/higress-group/proxy-wasm-go-sdk/proxywasm/types"
3636
"github.com/alibaba/higress/plugins/wasm-go/pkg/wrapper"
3737
)
3838
@@ -93,8 +93,8 @@ module {{ .Name }}
9393
go 1.19
9494
9595
require (
96-
github.com/alibaba/higress/plugins/wasm-go v0.0.0-20231019123123-86b223bc75f1
97-
github.com/tetratelabs/proxy-wasm-go-sdk v0.22.0
96+
github.com/alibaba/higress/plugins/wasm-go main
97+
github.com/higress-group/proxy-wasm-go-sdk main
9898
github.com/tidwall/gjson v1.14.3
9999
)
100100
`

‎plugins/wasm-go/extensions/ai-cache/.gitignore

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# File generated by hgctl. Modify as required.
2+
3+
*
4+
5+
!/.gitignore
6+
7+
!*.go
8+
!go.sum
9+
!go.mod
10+
11+
!LICENSE
12+
!*.md
13+
!*.yaml
14+
!*.yml
15+
16+
!*/
17+
18+
/out
19+
/test

‎plugins/wasm-go/extensions/ai-cache/README.md

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
## 简介
2+
3+
**Note**
4+
5+
> 需要数据面的proxy wasm版本大于等于0.2.100
6+
7+
> 编译时，需要带上版本的tag，例如：`tinygo build -o main.wasm -scheduler=none -target=wasi -gc=custom -tags="custommalloc nottinygc_finalizer proxy_wasm_version_0_2_100" ./`
8+
9+
LLM 结果缓存插件，默认配置方式可以直接用于 openai 协议的结果缓存，同时支持流式和非流式响应的缓存。
10+
11+
## 配置说明
12+
13+
| Name | Type | Requirement | Default | Description |
14+
| -------- | -------- | -------- | -------- | -------- |
15+
| cacheKeyFrom.requestBody | string | optional | "[email protected]" | 从请求 Body 中基于 [GJSON PATH](https://github.com/tidwall/gjson/blob/master/SYNTAX.md) 语法提取字符串 |
16+
| cacheValueFrom.responseBody | string | optional | "choices.0.message.content" | 从响应 Body 中基于 [GJSON PATH](https://github.com/tidwall/gjson/blob/master/SYNTAX.md) 语法提取字符串 |
17+
| cacheStreamValueFrom.responseBody | string | optional | "choices.0.delta.content" | 从流式响应 Body 中基于 [GJSON PATH](https://github.com/tidwall/gjson/blob/master/SYNTAX.md) 语法提取字符串 |
18+
| cacheKeyPrefix | string | optional | "higress-ai-cache:" | Redis缓存Key的前缀 |
19+
| cacheTTL | integer | optional | 0 | 缓存的过期时间，单位是秒，默认值为0，即永不过期 |
20+
| redis.serviceName | string | requried | - | redis 服务名称，带服务类型的完整 FQDN 名称，例如 my-redis.dns、redis.my-ns.svc.cluster.local |
21+
| redis.servicePort | integer | optional | 6379 | redis 服务端口 |
22+
| redis.timeout | integer | optional | 1000 | 请求 redis 的超时时间，单位为毫秒 |
23+
| redis.username | string | optional | - | 登陆 redis 的用户名 |
24+
| redis.password | string | optional | - | 登陆 redis 的密码 |
25+
| returnResponseTemplate | string | optional | `{"id":"from-cache","choices":[%s],"model":"gpt-4o","object":"chat.completion","usage":{"prompt_tokens":0,"completion_tokens":0,"total_tokens":0}}` | 返回 HTTP 响应的模版，用 %s 标记需要被 cache value 替换的部分 |
26+
| returnStreamResponseTemplate | string | optional | `data:{"id":"from-cache","choices":[{"index":0,"delta":{"role":"assistant","content":"%s"},"finish_reason":"stop"}],"model":"gpt-4o","object":"chat.completion","usage":{"prompt_tokens":0,"completion_tokens":0,"total_tokens":0}}\n\ndata:[DONE]\n\n` | 返回流式 HTTP 响应的模版，用 %s 标记需要被 cache value 替换的部分 |
27+
28+
## 配置示例
29+
30+
```yaml
31+
redis:
32+
serviceName: my-redis.dns
33+
timeout: 2000
34+
```

‎plugins/wasm-go/extensions/ai-cache/go.mod

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// File generated by hgctl. Modify as required.
2+
3+
module github.com/alibaba/higress/plugins/wasm-go/extensions/ai-cache
4+
5+
go 1.19
6+
7+
replace github.com/alibaba/higress/plugins/wasm-go => ../..
8+
9+
require (
10+
github.com/alibaba/higress/plugins/wasm-go v1.3.6-0.20240528060522-53bccf89f441
11+
github.com/higress-group/proxy-wasm-go-sdk v0.0.0-20240327114451-d6b7174a84fc
12+
github.com/tidwall/gjson v1.14.3
13+
github.com/tidwall/resp v0.1.1
14+
github.com/tidwall/sjson v1.2.5
15+
)
16+
17+
require (
18+
github.com/google/uuid v1.3.0 // indirect
19+
github.com/higress-group/nottinygc v0.0.0-20231101025119-e93c4c2f8520 // indirect
20+
github.com/magefile/mage v1.14.0 // indirect
21+
github.com/tidwall/match v1.1.1 // indirect
22+
github.com/tidwall/pretty v1.2.0 // indirect
23+
)

‎plugins/wasm-go/extensions/ai-cache/go.sum

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
2+
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
3+
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
4+
github.com/higress-group/nottinygc v0.0.0-20231101025119-e93c4c2f8520 h1:IHDghbGQ2DTIXHBHxWfqCYQW1fKjyJ/I7W1pMyUDeEA=
5+
github.com/higress-group/nottinygc v0.0.0-20231101025119-e93c4c2f8520/go.mod h1:Nz8ORLaFiLWotg6GeKlJMhv8cci8mM43uEnLA5t8iew=
6+
github.com/higress-group/proxy-wasm-go-sdk v0.0.0-20240327114451-d6b7174a84fc h1:t2AT8zb6N/59Y78lyRWedVoVWHNRSCBh0oWCC+bluTQ=
7+
github.com/higress-group/proxy-wasm-go-sdk v0.0.0-20240327114451-d6b7174a84fc/go.mod h1:hNFjhrLUIq+kJ9bOcs8QtiplSQ61GZXtd2xHKx4BYRo=
8+
github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo=
9+
github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
10+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
11+
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
12+
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
13+
github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw=
14+
github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
15+
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
16+
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
17+
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
18+
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
19+
github.com/tidwall/resp v0.1.1 h1:Ly20wkhqKTmDUPlyM1S7pWo5kk0tDu8OoC/vFArXmwE=
20+
github.com/tidwall/resp v0.1.1/go.mod h1:3/FrruOBAxPTPtundW0VXgmsQ4ZBA0Aw714lVYgwFa0=
21+
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
22+
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
23+
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

0 commit comments

Comments
 (0)