Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 8 additions & 13 deletions db/dbErrors/errTools/tools.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@ package errTools
import (
"errors"
"fmt"
"strings"

"github.com/go-sql-driver/mysql"
"github.com/jackc/pgx/v5/pgconn"
"helay.net/go/utils/v3/db/dbErrors"
"helay.net/go/utils/v3/db/dbErrors/errPostgres"
"strings"
)

func Error(err error) error {
Expand Down Expand Up @@ -38,14 +39,12 @@ func IsTableNotExist(err error) bool {
return false
}
// 检查 PostgresSQL 错误
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) {
if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok {
return pgErr.Code == "42P01" // PostgreSQL 表不存在的错误码
}

// 检查 MySQL 错误
var myErr *mysql.MySQLError
if errors.As(err, &myErr) {
if myErr, ok := errors.AsType[*mysql.MySQLError](err); ok {
return myErr.Number == 1146 // MySQL 表不存在的错误码 (ER_NO_SUCH_TABLE)
}
errStr := err.Error()
Expand All @@ -67,14 +66,12 @@ func IsColumnNotExist(err error) bool {
}

// 检查 PostgreSQL 错误
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) {
if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok {
return pgErr.Code == "42703" // PostgreSQL 列不存在的错误码
}

// 检查 MySQL 错误
var myErr *mysql.MySQLError
if errors.As(err, &myErr) {
if myErr, ok := errors.AsType[*mysql.MySQLError](err); ok {
return myErr.Number == 1054 // MySQL 列不存在的错误码 (ER_BAD_FIELD_ERROR)
}

Expand All @@ -100,8 +97,7 @@ func IsDuplicateKeyError(err error) int {
errStr := err.Error()

// 检查 PostgreSQL 错误
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) {
if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok {
if pgErr.Code == "23505" { // 唯一约束违反
if strings.Contains(pgErr.ConstraintName, "pkey") || strings.Contains(pgErr.Message, "primary key") {
return 1 // 主键重复
Expand All @@ -112,8 +108,7 @@ func IsDuplicateKeyError(err error) int {
}

// 检查 MySQL 错误
var myErr *mysql.MySQLError
if errors.As(err, &myErr) {
if myErr, ok := errors.AsType[*mysql.MySQLError](err); ok {
if myErr.Number == 1062 { // ER_DUP_ENTRY
if strings.Contains(myErr.Message, "PRIMARY") || strings.Contains(myErr.Message, "primary key") {
return 1 // 主键重复
Expand Down
1 change: 1 addition & 0 deletions db/kafka/clientconf/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,7 @@ func (c *Config) toConsumer(cfg *sarama.Config) error {
cfg.Consumer.Fetch.Min = tools.Ternary(c.Consumer.Fetch.Min < 1, 1, c.Consumer.Fetch.Min)
cfg.Consumer.Fetch.Default = tools.Ternary(c.Consumer.Fetch.Default < 1, 1024*1024, c.Consumer.Fetch.Default)
cfg.Consumer.Fetch.Max = c.Consumer.Fetch.Max
cfg.Consumer.Fetch.MaxBytes = tools.Ternary(c.Consumer.Fetch.MaxBytes < 1, 1024*1024*50, c.Consumer.Fetch.MaxBytes)

cfg.Consumer.MaxWaitTime = tools.AutoTimeDuration(c.Consumer.MaxWaitTime, time.Millisecond, 500*time.Millisecond)
cfg.Consumer.MaxProcessingTime = tools.AutoTimeDuration(c.Consumer.MaxProcessingTime, time.Millisecond, 100*time.Millisecond)
Expand Down
2 changes: 2 additions & 0 deletions db/kafka/clientconf/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,8 @@ type Fetch struct {
// 类似于 JVM 的 `fetch.message.max.bytes`。
// 全局 `sarama.MaxResponseSize` 仍然适用。
Max int32 `yaml:"max" json:"max"`

MaxBytes int32 `yaml:"max_bytes" json:"max_bytes"`
}

type AutoCommit struct {
Expand Down
5 changes: 2 additions & 3 deletions db/userDb/connect/mysqlconnect/mysql.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,12 @@ import (
func InitDB(c *db.Dbbase) (*gorm.DB, error) {
//dsn = c.User + ":" + c.Pwd + "@tcp(" + strings.Join(c.Host, ",") + ")/" + c.Dbname + "?charset=utf8mb4&parseTime=True&loc=Local"
//fmt.Println(dsn)
dialector := mysql.New(mysql.Config{
return c.Connect(new(mysql.New(mysql.Config{
DSN: c.Dsn(),
DefaultStringSize: 256, // string 类型字段的默认长度
DisableDatetimePrecision: true, // 禁用 datetime 精度,MySQL 5.6 之前的数据库不支持
DontSupportRenameIndex: true, // 重命名索引时采用删除并新建的方式,MySQL 5.7 之前的数据库和 MariaDB 不支持重命名索引
DontSupportRenameColumn: true, // 用 `change` 重命名列,MySQL 8 之前的数据库和 MariaDB 不支持重命名列
SkipInitializeWithVersion: false, // 根据当前 MySQL 版本自动配置
})
return c.Connect(&dialector)
})))
}
6 changes: 2 additions & 4 deletions db/userDb/connect/postgresconnect/postgresql.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,8 @@ import (
func InitDB(c *db.Dbbase) (*gorm.DB, error) {
//postgres://user:password@host1:port1/database?target_session_attrs=read-write&TimeZone=Asia/Shanghai
//dsn = "postgres://" + c.User + ":" + c.Pwd + "@" + strings.Join(c.Host, ",") + "/" + c.Dbname + "?TimeZone=Asia/Shanghai"
dialector := postgres.New(postgres.Config{
return c.Connect(new(postgres.New(postgres.Config{
DSN: c.Dsn(),
PreferSimpleProtocol: true,
})

return c.Connect(&dialector)
})))
}
3 changes: 1 addition & 2 deletions db/userDb/connect/sqliteconnect/sqlite.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,5 @@ import (
)

func InitDB(c *db.Dbbase) (*gorm.DB, error) {
dialector := sqlite.Open(c.Dsn())
return c.Connect(&dialector)
return c.Connect(new(sqlite.Open(c.Dsn())))
}
22 changes: 11 additions & 11 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ require (
github.com/jackc/pgx/v5 v5.10.0
github.com/jlaffaye/ftp v0.2.1
github.com/malfunkt/iprange v0.9.0
github.com/minio/minio-go/v7 v7.2.0
github.com/minio/minio-go/v7 v7.2.1
github.com/nacos-group/nacos-sdk-go/v2 v2.3.5
github.com/natefinch/lumberjack v2.0.0+incompatible
github.com/pkg/errors v0.9.1
Expand All @@ -38,7 +38,7 @@ require (
github.com/vmihailenco/msgpack/v5 v5.4.1
github.com/xdg-go/scram v1.2.0
github.com/xuri/excelize/v2 v2.10.1
go.etcd.io/etcd/client/v3 v3.6.12
go.etcd.io/etcd/client/v3 v3.6.13
go.uber.org/zap v1.28.0
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba
golang.org/x/crypto v0.53.0
Expand All @@ -61,7 +61,7 @@ require (
github.com/alibabacloud-go/darabonba-array v0.1.0 // indirect
github.com/alibabacloud-go/darabonba-encode-util v0.0.2 // indirect
github.com/alibabacloud-go/darabonba-map v0.0.2 // indirect
github.com/alibabacloud-go/darabonba-openapi/v2 v2.2.2 // indirect
github.com/alibabacloud-go/darabonba-openapi/v2 v2.2.3 // indirect
github.com/alibabacloud-go/darabonba-signature-util v0.0.7 // indirect
github.com/alibabacloud-go/darabonba-string v1.0.2 // indirect
github.com/alibabacloud-go/debug v1.0.1 // indirect
Expand Down Expand Up @@ -118,8 +118,8 @@ require (
github.com/jinzhu/now v1.1.5 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.18.6 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/klauspost/compress v1.19.0 // indirect
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
Expand All @@ -139,7 +139,7 @@ require (
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.69.0 // indirect
github.com/prometheus/procfs v0.20.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
github.com/richardlehane/mscfb v1.0.7 // indirect
github.com/richardlehane/msoleps v1.0.6 // indirect
Expand All @@ -154,8 +154,8 @@ require (
github.com/xuri/efp v0.0.1 // indirect
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.etcd.io/etcd/api/v3 v3.6.12 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.6.12 // indirect
go.etcd.io/etcd/api/v3 v3.6.13 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.6.13 // indirect
go.mongodb.org/mongo-driver/v2 v2.7.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel v1.44.0 // indirect
Expand All @@ -168,9 +168,9 @@ require (
golang.org/x/sync v0.21.0 // indirect
golang.org/x/sys v0.46.0 // indirect
golang.org/x/time v0.15.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260622175928-b703f567277d // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260622175928-b703f567277d // indirect
google.golang.org/grpc v1.81.1 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect
google.golang.org/grpc v1.82.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
Expand Down
66 changes: 66 additions & 0 deletions net/http/server/logger.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package server

import (
"io"
"log"
"os"
"strings"
"sync"
)

// FilteringLogger 实现日志过滤功能
type FilteringLogger struct {
out io.Writer
config LogFilterConfig
mu sync.Mutex
logger *log.Logger
}

// NewFilteringLogger 创建一个新的过滤日志器
func NewFilteringLogger(config LogFilterConfig) *FilteringLogger {
fl := &FilteringLogger{
out: os.Stderr,
config: config,
}
fl.logger = log.New(fl, "", log.LstdFlags)
return fl
}

// Write 实现 io.Writer 接口,用于过滤日志
func (fl *FilteringLogger) Write(p []byte) (n int, err error) {
// 如果配置了屏蔽所有服务器错误
if fl.config.SuppressAllServerErrors {
return len(p), nil
}

msg := string(p)

// 检查是否包含 TLS handshake error
if fl.config.SuppressTLSHandshakeError {
if strings.Contains(msg, "TLS handshake error") {
// 过滤掉特定的 TLS 握手错误
if strings.Contains(msg, "client sent an HTTP request to an HTTPS server") ||
strings.Contains(msg, "no common certificate") ||
strings.Contains(msg, "bad certificate") {
return len(p), nil
}
}
}

// 检查是否包含客户端断开连接的错误
if fl.config.SuppressClientDisconnect {
if strings.Contains(msg, "read from client: EOF") ||
strings.Contains(msg, "write: broken pipe") ||
strings.Contains(msg, "connection reset by peer") {
return len(p), nil
}
}

// 其他日志正常输出
return fl.out.Write(p)
}

// GetErrorLog 返回一个 *log.Logger 供 http.Server 使用
func (fl *FilteringLogger) GetErrorLog() *log.Logger {
return fl.logger
}
6 changes: 6 additions & 0 deletions net/http/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,12 @@ func (s *Server[T]) httpServer() {
IdleTimeout: tools.AutoTimeDuration(s.opt.IdleTimeout, time.Second),
MaxHeaderBytes: s.opt.MaxHeaderBytes,
}

if s.opt.LogFilter.SuppressAllServerErrors || s.opt.LogFilter.SuppressClientDisconnect || s.opt.LogFilter.SuppressTLSHandshakeError {
filteringLogger := NewFilteringLogger(s.opt.LogFilter)
s.server.ErrorLog = filteringLogger.GetErrorLog()
}

}

// http3 配置处理
Expand Down
17 changes: 16 additions & 1 deletion net/http/server/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,12 @@ type Config struct {

Security SecurityConfig `ini:"security" json:"security" yaml:"security"` // 安全配置
Compression middleware.CompressionConfig `json:"compression" yaml:"compression"` // 压缩配置
Logger zaploger.Config `json:"logger" yaml:"logger"` // 日志配置

Logger zaploger.Config `json:"logger" yaml:"logger"` // 日志配置

Route route.Config `json:"route" yaml:"route"` // 路由配置
// 新增:日志过滤配置
LogFilter LogFilterConfig `json:"log_filter" yaml:"log_filter"`
}

type SecurityConfig struct {
Expand All @@ -54,6 +57,18 @@ type IPAccessConfig struct {
Debug *ipmatch.Config `ini:"debug" json:"debug" yaml:"debug"` // 调试允许的IP
}

// LogFilterConfig 日志过滤配置
type LogFilterConfig struct {
// 是否屏蔽 TLS 握手错误日志(如 HTTP 请求发到 HTTPS 端口)
SuppressTLSHandshakeError bool `json:"suppress_tls_handshake_error" yaml:"suppress_tls_handshake_error"`

// 是否屏蔽客户端主动断开连接的错误(可选)
SuppressClientDisconnect bool `json:"suppress_client_disconnect" yaml:"suppress_client_disconnect"`

// 是否屏蔽所有 HTTP 服务器内部错误日志(慎用)
SuppressAllServerErrors bool `json:"suppress_all_server_errors" yaml:"suppress_all_server_errors"`
}

type Server[T any] struct {
opt *Config
serverNames map[string]struct{} // 绑定的域名
Expand Down
3 changes: 1 addition & 2 deletions safe/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,7 @@ func (h IntegerHasher[T]) Hash(key T) uint64 {
type FloatHasher[T constraints.Float] struct{}

func (h FloatHasher[T]) Hash(key T) uint64 {
f64 := float64(key)
return xxhash.Sum64(unsafe.Slice((*byte)(unsafe.Pointer(&f64)), 8))
return xxhash.Sum64(unsafe.Slice((*byte)(unsafe.Pointer(new(float64(key)))), 8))
}

// StringHasher 字符串哈希器
Expand Down
3 changes: 1 addition & 2 deletions tools/decode/json_decode_tee/json_decode.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,7 @@ func (e *JsonDecodeError) Unwrap() error {
}

func ReadErrRaw(err error) []byte {
var jsonErr *JsonDecodeError
if errors.As(err, &jsonErr) {
if jsonErr, ok := errors.AsType[*JsonDecodeError](err); ok {
return jsonErr.Raw.Bytes()
}
return nil
Expand Down
Loading