From 125a0044e2238a440afdbc7a392b755df566e921 Mon Sep 17 00:00:00 2001 From: liamyu Date: Tue, 17 Dec 2024 19:25:53 +0800 Subject: [PATCH 1/2] 1 Fixed the bug that no corresponding parameter in the rename operation packet would be replaced with a new parameter and set to nil. The change behavior of the parameter was ignored 2 Upgrade add_body to specify parameter types for requests with Content-Type application/json --- plugin/transformer/request/README.md | 4 + plugin/transformer/request/plugin.go | 131 +++++++++++++++++++--- plugin/transformer/request/plugin_test.go | 125 ++++++++++++++++++--- plugin/transformer/response/plugin.go | 6 +- 4 files changed, 232 insertions(+), 34 deletions(-) diff --git a/plugin/transformer/request/README.md b/plugin/transformer/request/README.md index 44fbf6a..531a0dc 100644 --- a/plugin/transformer/request/README.md +++ b/plugin/transformer/request/README.md @@ -46,6 +46,10 @@ router: # Router configuration - key: val add_body: # Add request body parameters - key: val + # support for specified types:number、string、bool + - key_bool:true:bool + - key_num:1:number + - key_str:hi:string add_query_str: # Add URL query parameters - key: val reserve_headers: # Only keep specified request headers, higher priority than remove_headers, -1 means remove all diff --git a/plugin/transformer/request/plugin.go b/plugin/transformer/request/plugin.go index 2c932b0..b1e8d0e 100644 --- a/plugin/transformer/request/plugin.go +++ b/plugin/transformer/request/plugin.go @@ -17,7 +17,9 @@ package request import ( "bytes" "context" + "fmt" "runtime/debug" + "strconv" "strings" "github.com/tidwall/gjson" @@ -40,8 +42,19 @@ const ( pluginName = "request_transformer" // DelAllKey is a placeholder for deleting all operations DelAllKey = "-1" + + typeString valType = "string" + typeNumber valType = "number" + typeBool valType = "bool" + typeDefault valType = "" + + // ErrInvalidJSONType invalid value type + ErrInvalidJSONType = 10002 ) +// valType JSON field type +type valType string + var ( strPostArgsContentType = []byte("application/x-www-form-urlencoded") strMultipartFormData = []byte("multipart/form-data") @@ -71,6 +84,8 @@ func (p *Plugin) Setup(string, plugin.Decoder) error { type KV struct { Key string Val string + // converted value for Val,current supported types include: bool number string + ConvertedVal interface{} } // Options represents the parameter options @@ -158,10 +173,9 @@ func (p *Plugin) CheckConfig(name string, decoder plugin.Decoder) error { return gerrs.Wrap(err, "add headers config error") } } - // Parse query parameters if len(options.AddQueryStr) != 0 { - options.AddQueryStrKV, err = getKV(options.AddQueryStr) + options.AddQueryStrKV, err = getAddKV(options.AddQueryStr) if err != nil { return gerrs.Wrap(err, "add query string config error") } @@ -169,7 +183,7 @@ func (p *Plugin) CheckConfig(name string, decoder plugin.Decoder) error { // Parse configuration if len(options.AddBody) != 0 { - options.AddBodyKV, err = getKV(options.AddBody) + options.AddBodyKV, err = getAddKV(options.AddBody) if err != nil { return gerrs.Wrap(err, "add body config error") } @@ -199,6 +213,68 @@ func getKV(list []string) ([]*KV, error) { return kvList, nil } +// getAddKV retrieves the key-value configuration +func getAddKV(list []string) ([]*KV, error) { + var kvList []*KV + for _, v := range list { + if v == "" { + continue + } + kv := &KV{} + arr := strings.Split(v, ":") + if len(arr) < 2 { + return nil, errs.New(gerrs.ErrWrongConfig, "invalid kv config") + } + if arr[0] == "" { + continue + } + kv.Key = arr[0] + val := arr[1] + var defaultValType valType + if len(arr) == 2 { + // 兼容旧版本，默认 string + defaultValType = typeString + } else { + if arr[2] == "" { + return nil, errs.New(gerrs.ErrWrongConfig, "invalid add kv config:value type not config") + } + defaultValType = valType(arr[2]) + } + convertedVal, err := convertJSONVal(val, defaultValType) + if err != nil { + return nil, gerrs.Wrap(err, "convert json val err") + } + kv.Val = val + kv.ConvertedVal = convertedVal + kvList = append(kvList, kv) + } + return kvList, nil +} + +// Convert the type of JSON value +func convertJSONVal(val string, valType valType) (interface{}, error) { + switch valType { + case typeString: + return fmt.Sprint(val), nil + case typeNumber: + floatValue, err := strconv.ParseFloat(val, 64) + if err != nil { + return nil, gerrs.Wrapf(err, "to float err,val:%s", val) + } + return floatValue, nil + case typeBool: + boolVal, err := strconv.ParseBool(val) + if err != nil { + return nil, gerrs.Wrapf(err, "parse bool err:%s", val) + } + return boolVal, nil + case typeDefault: + return val, nil + default: + return nil, errs.Newf(ErrInvalidJSONType, "invalid type:%s", valType) + } +} + // ServerFilter sets up server-side CORS validation func ServerFilter(ctx context.Context, req interface{}, handler filter.ServerHandleFunc) (interface{}, error) { if err := transform(ctx); err != nil { @@ -316,7 +392,7 @@ func addBody(ctx context.Context, request *fasthttp.Request, kv *KV) { return } if bytes.HasPrefix(request.Header.ContentType(), strJSONContentType) { - newBody, _ := sjson.SetBytes(request.Body(), kv.Key, kv.Val) + newBody, _ := sjson.SetBytes(request.Body(), kv.Key, kv.ConvertedVal) request.SetBody(newBody) return } @@ -340,6 +416,9 @@ func renameOption(ctx context.Context, request *fasthttp.Request, options *Optio if len(options.RenameHeadersKV) != 0 { for _, kv := range options.RenameHeadersKV { val := request.Header.Peek(kv.Key) + if val == nil { + continue + } request.Header.DisableNormalizing() request.Header.SetBytesV(kv.Val, val) request.Header.EnableNormalizing() @@ -349,7 +428,11 @@ func renameOption(ctx context.Context, request *fasthttp.Request, options *Optio // Rename query parameters if len(options.RenameQueryStrKV) != 0 { for _, kv := range options.RenameQueryStrKV { - request.URI().QueryArgs().SetBytesV(kv.Val, request.URI().QueryArgs().Peek(kv.Key)) + val := request.URI().QueryArgs().Peek(kv.Key) + if val == nil { + continue + } + request.URI().QueryArgs().SetBytesV(kv.Val, val) request.URI().QueryArgs().Del(kv.Key) } } @@ -357,39 +440,53 @@ func renameOption(ctx context.Context, request *fasthttp.Request, options *Optio // Rename query body parameters if len(options.RenameBodyKV) != 0 { for _, kv := range options.RenameBodyKV { - renameBody(ctx, request, kv) + modified := renameBody(ctx, request, kv) + if modified { + modifyBody = true + } } - modifyBody = true } return modifyBody } -func renameBody(ctx context.Context, request *fasthttp.Request, kv *KV) { +func renameBody(ctx context.Context, request *fasthttp.Request, kv *KV) bool { // Handle different content types if bytes.HasPrefix(request.Header.ContentType(), strPostArgsContentType) { - request.PostArgs().SetBytesV(kv.Val, request.PostArgs().Peek(kv.Key)) + val := request.PostArgs().Peek(kv.Key) + if val == nil { + return false + } + request.PostArgs().SetBytesV(kv.Val, val) request.PostArgs().Del(kv.Key) - return + return true } if bytes.HasPrefix(request.Header.ContentType(), strJSONContentType) { - newBody, _ := sjson.SetBytes(request.Body(), kv.Val, gjson.GetBytes(request.Body(), kv.Key).Value()) + val := gjson.GetBytes(request.Body(), kv.Key) + if !val.Exists() { + return false + } + newBody, _ := sjson.SetBytes(request.Body(), kv.Val, val.Value()) newBody, _ = sjson.DeleteBytes(newBody, kv.Key) request.SetBody(newBody) - return + return true } if bytes.HasPrefix(request.Header.ContentType(), strMultipartFormData) { form, err := request.MultipartForm() if err != nil { - log.ErrorContextf(ctx, "get multipart error: %s", err) - return + log.ErrorContextf(ctx, "get multipart err:%s", err) + return false } - form.Value[kv.Val] = form.Value[kv.Key] + val := form.Value[kv.Key] + if val == nil { + return false + } + form.Value[kv.Val] = val delete(form.Value, kv.Key) - return + return true } // Do nothing for other content types log.WarnContextf(ctx, "unsupported content-type: %s", request.Header.ContentType()) - return + return false } // Delete operation diff --git a/plugin/transformer/request/plugin_test.go b/plugin/transformer/request/plugin_test.go index 8873db4..c788c3e 100644 --- a/plugin/transformer/request/plugin_test.go +++ b/plugin/transformer/request/plugin_test.go @@ -35,7 +35,13 @@ func TestPlugin_CheckConfig(t *testing.T) { RenameBody: []string{"", ":aaa", "old_query_body:new_query_body"}, AddHeaders: []string{"", ":aaa", "new_header:new_header_val"}, AddQueryStr: []string{"", ":aaa", "new_query_str:new_query_val"}, - AddBody: []string{"", ":aaa", "new_query_body:new_query_body_val"}, + AddBody: []string{"", ":aaa", + "new_query_body:new_query_body_val", + "key_bool:true:bool", + "key_num:1:number", + "key_float_num:1.0:number", + "key_str:hi:string", + }, } p := &Plugin{} _ = p.Setup("", nil) @@ -85,6 +91,20 @@ func TestPlugin_CheckConfig(t *testing.T) { err = p.CheckConfig("", decoder) assert.NotNil(t, err) options.AddBody = tmp + + // AddBody invalid + tmp = options.AddBody + options.AddBody = []string{ + "invalid_type_oldkey:invalid_type_newkey:a:number", + } + err = p.CheckConfig("", decoder) + assert.NotNil(t, err) + options.AddBody = []string{ + "not_support_oldkey:not_support_newkey:1:int32", + } + err = p.CheckConfig("", decoder) + assert.NotNil(t, err) + options.AddBody = tmp } func TestServerFilter(t *testing.T) { @@ -382,20 +402,56 @@ func Test_addBody(t *testing.T) { fctx := &fasthttp.RequestCtx{} fctx.Request.Header.SetContentTypeBytes(strPostArgsContentType) addBody(context.Background(), &fctx.Request, &KV{ - Key: "key", - Val: "val", + Key: "key", + Val: "val", + ConvertedVal: "val", }) assert.Equal(t, "val", string(fctx.Request.PostArgs().Peek("key"))) + addBody(context.Background(), &fctx.Request, &KV{ + Key: "key_num", + Val: "1", + ConvertedVal: 1, + }) + assert.Equal(t, "1", string(fctx.Request.PostArgs().Peek("key_num"))) + addBody(context.Background(), &fctx.Request, &KV{ + Key: "key_bool", + Val: "true", + ConvertedVal: true, + }) + assert.Equal(t, "true", string(fctx.Request.PostArgs().Peek("key_bool"))) + fctx.Request.ResetBody() fctx.Request.Header.SetContentTypeBytes(strJSONContentType) fctx.Request.SetBodyString(`{}`) addBody(context.Background(), &fctx.Request, &KV{ - Key: "key", - Val: "val", + Key: "key", + Val: "val", + ConvertedVal: "val", }) assert.Equal(t, `{"key":"val"}`, string(fctx.Request.Body())) - fctx.Request.ResetBody() + addBody(context.Background(), &fctx.Request, &KV{ + Key: "key_num", + Val: "1", + ConvertedVal: 1, + }) + assert.Equal(t, `{"key_num":1}`, string(fctx.Request.Body())) + fctx.Request.ResetBody() + addBody(context.Background(), &fctx.Request, &KV{ + Key: "key_num", + Val: "1.0", + ConvertedVal: 1.0, + }) + assert.Equal(t, `{"key_num":1}`, string(fctx.Request.Body())) + fctx.Request.ResetBody() + addBody(context.Background(), &fctx.Request, &KV{ + Key: "key_bool", + Val: "true", + ConvertedVal: true, + }) + assert.Equal(t, `{"key_bool":true}`, string(fctx.Request.Body())) + fctx.Request.ResetBody() + formBody := `------WebKitFormBoundaryWrGPeYfXRBUkQimG Content-Disposition: form-data; name="another" @@ -404,19 +460,33 @@ another_val fctx.Request.SetBodyString(formBody) fctx.Request.Header.SetContentType("multipart/form-data; boundary=----WebKitFormBoundaryWrGPeYfXRBUkQimG") addBody(context.Background(), &fctx.Request, &KV{ - Key: "key", - Val: "val", + Key: "key", + Val: "val", + ConvertedVal: "val", }) form, err := fctx.Request.MultipartForm() assert.Nil(t, err) assert.Equal(t, "val", form.Value["key"][0]) + addBody(context.Background(), &fctx.Request, &KV{ + Key: "key_num", + Val: "1", + ConvertedVal: 1, + }) + assert.Equal(t, "1", form.Value["key_num"][0]) + addBody(context.Background(), &fctx.Request, &KV{ + Key: "key_bool", + Val: "true", + ConvertedVal: true, + }) + assert.Equal(t, "true", form.Value["key_bool"][0]) fctx.Request.Header.SetContentType("application/plain") fctx.Request.SetBodyString("") addBody(context.Background(), &fctx.Request, &KV{ - Key: "key", - Val: "key1", + Key: "key", + Val: "val", + ConvertedVal: "key1", }) assert.Equal(t, ``, string(fctx.Request.Body())) } @@ -425,21 +495,40 @@ func Test_renameBody(t *testing.T) { fctx := &fasthttp.RequestCtx{} fctx.Request.Header.SetContentTypeBytes(strPostArgsContentType) fctx.Request.PostArgs().Set("key", "val") - renameBody(context.Background(), &fctx.Request, &KV{ + modified := renameBody(context.Background(), &fctx.Request, &KV{ Key: "key", Val: "key1", }) + assert.True(t, modified) assert.Equal(t, "val", string(fctx.Request.PostArgs().Peek("key1"))) + // 为找到要修改参数时，忽略变更，获取参数时应为空值 + modified = renameBody(context.Background(), &fctx.Request, &KV{ + Key: "key2", + Val: "key2", + }) + assert.False(t, modified) + assert.Equal(t, "", string(fctx.Request.PostArgs().Peek("key2"))) fctx.Request.Header.SetContentTypeBytes(strJSONContentType) fctx.Request.SetBodyString(`{"key":"val"}`) - renameBody(context.Background(), &fctx.Request, &KV{ + modified = renameBody(context.Background(), &fctx.Request, &KV{ Key: "key", Val: "key1", }) + assert.True(t, modified) assert.Equal(t, `{"key1":"val"}`, string(fctx.Request.Body())) - fctx.Request.ResetBody() + // 为找到要修改参数时，忽略变更 + fctx.Request.Header.SetContentTypeBytes(strJSONContentType) + fctx.Request.SetBodyString(`{"key":"val"}`) + modified = renameBody(context.Background(), &fctx.Request, &KV{ + Key: "key1", + Val: "key1", + }) + assert.False(t, modified) + assert.Equal(t, `{"key":"val"}`, string(fctx.Request.Body())) + fctx.Request.ResetBody() + formBody := `------WebKitFormBoundaryWrGPeYfXRBUkQimG Content-Disposition: form-data; name="key" @@ -447,15 +536,23 @@ val ------WebKitFormBoundaryWrGPeYfXRBUkQimG--` fctx.Request.SetBodyString(formBody) fctx.Request.Header.SetContentType("multipart/form-data; boundary=----WebKitFormBoundaryWrGPeYfXRBUkQimG") - renameBody(context.Background(), &fctx.Request, &KV{ + modified = renameBody(context.Background(), &fctx.Request, &KV{ Key: "key", Val: "key1", }) form, err := fctx.Request.MultipartForm() + assert.True(t, modified) assert.Nil(t, err) assert.Equal(t, "val", form.Value["key1"][0]) + modified = renameBody(context.Background(), &fctx.Request, &KV{ + Key: "key2", + Val: "key2", + }) + assert.False(t, modified) + assert.Nil(t, err) + fctx.Request.Header.SetContentType("application/plain") fctx.Request.SetBodyString("") renameBody(context.Background(), &fctx.Request, &KV{ diff --git a/plugin/transformer/response/plugin.go b/plugin/transformer/response/plugin.go index a3cb63e..3661616 100644 --- a/plugin/transformer/response/plugin.go +++ b/plugin/transformer/response/plugin.go @@ -255,11 +255,11 @@ func convertJSONVal(val string, valType valType) (interface{}, error) { case typeString: return fmt.Sprint(val), nil case typeNumber: - intVal, err := strconv.ParseInt(val, 10, 32) + floatValue, err := strconv.ParseFloat(val, 64) if err != nil { - return nil, gerrs.Wrapf(err, "convert to int error, val: %s", val) + return nil, gerrs.Wrapf(err, "to float err,val:%s", val) } - return int32(intVal), nil + return floatValue, nil case typeBool: boolVal, err := strconv.ParseBool(val) if err != nil { From 6f716c773d21bed144997dcdcf997a899d50c633 Mon Sep 17 00:00:00 2001 From: liamyu Date: Tue, 17 Dec 2024 19:29:47 +0800 Subject: [PATCH 2/2] 1 Fixed the bug that no corresponding parameter in the rename operation packet would be replaced with a new parameter and set to nil. The change behavior of the parameter was ignored 2 Upgrade add_body to specify parameter types for requests with Content-Type application/json --- plugin/transformer/request/plugin_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/plugin/transformer/request/plugin_test.go b/plugin/transformer/request/plugin_test.go index c788c3e..ae316b7 100644 --- a/plugin/transformer/request/plugin_test.go +++ b/plugin/transformer/request/plugin_test.go @@ -420,7 +420,6 @@ func Test_addBody(t *testing.T) { }) assert.Equal(t, "true", string(fctx.Request.PostArgs().Peek("key_bool"))) fctx.Request.ResetBody() - fctx.Request.Header.SetContentTypeBytes(strJSONContentType) fctx.Request.SetBodyString(`{}`) addBody(context.Background(), &fctx.Request, &KV{