chore: seed migration objects batch 5

This commit is contained in:
dev
2026-10-18 13:37:24 +08:00
parent 53c9c0afd8
commit c8c40b1dd2
239 changed files with 16941 additions and 1544 deletions
+46 -47
View File
@@ -1,67 +1,66 @@
// Package casbin
// @Link https://github.com/bufanyun/hotgo
// @Copyright Copyright (c) 2023 HotGo CLI
// @Author Ms <133814250@qq.com>
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
package casbin
import (
"context"
"net/http"
"testing"
casbinv2 "github.com/casbin/casbin/v2"
"github.com/casbin/casbin/v2/model"
)
// TestNew description
func TestNew(t *testing.T) {
const testModel = `
[request_definition]
r = sub, obj, act
InitEnforcer(context.TODO())
[policy_definition]
p = sub, obj, act
user := "admin"
path := "/"
method := http.MethodGet
t.Logf("\nuser:%v\npath:%v\nmethod:%v", user, path, method)
[role_definition]
g = _, _
ok, err := Enforcer.DeletePermissionsForUser(user)
[policy_effect]
e = some(where (p.eft == allow))
[matchers]
m = g(r.sub, p.sub) && keyMatch(r.obj, p.obj) && regexMatch(r.act, p.act)
`
func TestEnforcerPermissions(t *testing.T) {
m, err := model.NewModelFromString(testModel)
if err != nil {
t.Error(err)
t.Fatalf("创建Casbin测试模型失败: %v", err)
}
t.Logf("delete user premission:%v", ok)
CheckPremission(t, user, path, method)
AddPremission(t, user, "*", ActionAll)
CheckPremission(t, user, path, method)
user1 := "user1"
path1 := "/api/v1/*"
checkPathTrue := "/api/v1/user/list"
checkPathFalse := "/api/v2/user/list"
AddPremission(t, user1, path1, ActionAll)
CheckPremission(t, user1, checkPathTrue, ActionPost)
CheckPremission(t, user1, checkPathFalse, http.MethodGet)
CheckPremission(t, user1, checkPathTrue, http.MethodGet)
CheckPremission(t, user1, "/api/v1/user/list2", http.MethodGet)
ok, err = Enforcer.DeletePermissionsForUser(user1)
enforcer, err := casbinv2.NewEnforcer(m)
if err != nil {
t.Error(err)
t.Fatalf("创建Casbin测试实例失败: %v", err)
}
t.Logf("delete user premission:%v", ok)
CheckPremission(t, user1, "/api/v1/user1/list", http.MethodGet)
assertPermission(t, enforcer, "admin", "/", http.MethodGet, false)
if ok, addErr := enforcer.AddPolicy("admin", "*", ActionAll); addErr != nil || !ok {
t.Fatalf("添加管理员策略失败: ok=%v err=%v", ok, addErr)
}
assertPermission(t, enforcer, "admin", "/", http.MethodGet, true)
if ok, addErr := enforcer.AddPolicy("user1", "/api/v1/*", ActionAll); addErr != nil || !ok {
t.Fatalf("添加用户策略失败: ok=%v err=%v", ok, addErr)
}
assertPermission(t, enforcer, "user1", "/api/v1/user/list", http.MethodPost, true)
assertPermission(t, enforcer, "user1", "/api/v1/user/list", http.MethodGet, true)
assertPermission(t, enforcer, "user1", "/api/v2/user/list", http.MethodGet, false)
if ok, deleteErr := enforcer.DeletePermissionsForUser("user1"); deleteErr != nil || !ok {
t.Fatalf("删除用户策略失败: ok=%v err=%v", ok, deleteErr)
}
assertPermission(t, enforcer, "user1", "/api/v1/user/list", http.MethodGet, false)
}
// CheckPremission description
func CheckPremission(t *testing.T, user string, path string, method string) {
ok, err := Enforcer.Enforce(user, path, method)
func assertPermission(t *testing.T, enforcer *casbinv2.Enforcer, user, path, method string, expected bool) {
t.Helper()
actual, err := enforcer.Enforce(user, path, method)
if err != nil {
t.Error(err)
t.Fatalf("检查权限失败: %v", err)
}
t.Logf("check \tuser[%s] \tpremission[%s] \tpath[%s] \tallow[%v]", user, method, path, ok)
}
// Add description
func AddPremission(t *testing.T, user string, path string, method string) {
ok, err := Enforcer.AddPolicy(user, path, method)
if err != nil {
t.Error(err)
if actual != expected {
t.Fatalf("权限结果不符 user=%s path=%s method=%s expected=%v actual=%v", user, path, method, expected, actual)
}
t.Logf("add \tuser[%s] \tpremission[%s] \tpath[%s] \tresult[%v]", user, method, path, ok)
}
@@ -17,6 +17,7 @@ import (
"time"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gctx"
)
@@ -198,6 +199,9 @@ func (c *Client) parseError(resp *http.Response) error {
Reason string `json:"reason"`
}
_ = json.Unmarshal(body, &envelope)
if envelope.Message == "" && envelope.Reason == "" {
g.Log().Warningf(context.Background(), "IM API returned non-success status:%d url:%s body:%s", resp.StatusCode, resp.Request.URL.String(), string(body))
}
return stableError(envelope.Message, envelope.Reason)
}
@@ -58,6 +58,39 @@ func TestListCodesWithInviteCodeUsesSignedSearchPost(t *testing.T) {
}
}
func TestInternalPostUsesInternalSignatureAndBody(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/internal/admin-query/tenants/default/users" {
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
}
body, err := io.ReadAll(r.Body)
if err != nil {
t.Fatalf("read body: %v", err)
}
if string(body) != `{"username":"alice"}` {
t.Fatalf("unexpected body %s", body)
}
if r.Header.Get("X-Service-Name") != "hotgo-admin" || r.Header.Get("X-Signature") == "" {
t.Fatalf("missing internal authentication headers")
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"code":0,"data":{"user_id":123,"username":"alice"}}`))
}))
defer server.Close()
client := NewClient(Config{InternalBaseUrl: server.URL, InternalServiceName: "hotgo-admin", InternalServiceSecret: "secret", Timeout: time.Second})
var out struct {
UserID int64 `json:"user_id"`
Username string `json:"username"`
}
if err := client.InternalPost(context.Background(), "/internal/admin-query/tenants/default/users", map[string]string{"username": "alice"}, &out); err != nil {
t.Fatalf("InternalPost returned error: %v", err)
}
if out.UserID != 123 || out.Username != "alice" {
t.Fatalf("unexpected response: %+v", out)
}
}
func signedTestServer(t *testing.T, method string, path string, inspect func(t *testing.T, r *http.Request, body []byte)) *httptest.Server {
t.Helper()
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+18 -4
View File
@@ -14,15 +14,29 @@ import (
"time"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
)
func (c *Client) InternalGet(ctx context.Context, path string, out any) error {
req, err := c.newInternalRequest(ctx, http.MethodGet, path, nil)
return c.internalJSON(ctx, http.MethodGet, path, nil, out)
}
func (c *Client) InternalPost(ctx context.Context, path string, in any, out any) error {
body, err := json.Marshal(in)
if err != nil {
return gerror.Wrap(err, "序列化IM内部请求失败")
}
return c.internalJSON(ctx, http.MethodPost, path, body, out)
}
func (c *Client) internalJSON(ctx context.Context, method, path string, body []byte, out any) error {
req, err := c.newInternalRequest(ctx, method, path, body)
if err != nil {
return err
}
resp, err := c.http.Do(req)
if err != nil {
g.Log().Warningf(ctx, "IM internal query request failed url:%s err:%+v", req.URL.String(), err)
return gerror.New("IM内部查询服务暂不可用")
}
defer resp.Body.Close()
@@ -32,7 +46,7 @@ func (c *Client) InternalGet(ctx context.Context, path string, out any) error {
if out == nil {
return nil
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBody))
responseBody, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBody))
if err != nil {
return gerror.Wrap(err, "读取IM内部查询响应失败")
}
@@ -42,13 +56,13 @@ func (c *Client) InternalGet(ctx context.Context, path string, out any) error {
Reason string `json:"reason"`
Data json.RawMessage `json:"data"`
}
if err = json.Unmarshal(body, &envelope); err == nil && len(envelope.Data) > 0 {
if err = json.Unmarshal(responseBody, &envelope); err == nil && len(envelope.Data) > 0 {
if envelope.Code != 0 {
return stableError(envelope.Message, envelope.Reason)
}
return json.Unmarshal(envelope.Data, out)
}
return json.Unmarshal(body, out)
return json.Unmarshal(responseBody, out)
}
func (c *Client) newInternalRequest(ctx context.Context, method string, path string, bodyBytes []byte) (*http.Request, error) {
+58 -107
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"hotgo/internal/consts"
"hotgo/internal/library/imtenant"
"hotgo/internal/library/queue"
"time"
@@ -16,11 +17,6 @@ import (
"github.com/gogf/gf/v2/util/guid"
)
const (
adminCommandTable = "im_admin_commands"
adminOutboxTable = "im_admin_outbox"
)
func PublishAdminCommand(ctx context.Context, in AdminCommandInput) (res *AdminCommandModel, err error) {
if in.TenantID <= 0 || in.TenantCode == "" {
return nil, gerror.New("IM租户信息不完整")
@@ -65,41 +61,22 @@ func PublishAdminCommand(ctx context.Context, in AdminCommandInput) (res *AdminC
return nil, gerror.Wrap(err, "序列化IM命令Envelope失败")
}
db := g.DB()
db := adminDB()
err = db.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
data := g.Map{
"command_id": commandID,
"tenant_id": in.TenantID,
"tenant_code": in.TenantCode,
"event_id": eventID,
"event_type": in.EventType,
"biz_type": in.BizType,
"biz_id": in.BizID,
"idempotency_key": in.IdempotencyKey,
"status": consts.ImAdminCommandStatusPending,
"payload": string(payloadBytes),
"envelope": string(envelopeBytes),
"created_by": in.CreatedBy,
"created_at": gtime.Now(),
"updated_at": gtime.Now(),
}
if _, err := tx.Model(adminCommandTable).Safe().Ctx(ctx).Data(data).Insert(); err != nil {
nowTime := gtime.Now()
if _, err := tx.Exec(`INSERT INTO im_admin_commands
(command_id,tenant_id,tenant_code,event_id,event_type,biz_type,biz_id,idempotency_key,status,payload,envelope,created_by,created_at,updated_at)
VALUES(?,?,?,?,?,?,?,?,?,CAST(? AS JSON),CAST(? AS JSON),?,?,?)`,
commandID, in.TenantID, in.TenantCode, eventID, in.EventType, in.BizType, in.BizID,
in.IdempotencyKey, consts.ImAdminCommandStatusPending, string(payloadBytes), string(envelopeBytes),
in.CreatedBy, nowTime, nowTime); err != nil {
return gerror.Wrap(err, "创建IM后台命令失败")
}
outbox := g.Map{
"event_id": eventID,
"command_id": commandID,
"topic": consts.QueueImAdminCommandTopic,
"event_type": in.EventType,
"biz_type": in.BizType,
"biz_id": in.BizID,
"key_value": in.BizID,
"envelope": string(envelopeBytes),
"status": consts.ImAdminOutboxStatusPending,
"created_at": gtime.Now(),
"updated_at": gtime.Now(),
}
if _, err := tx.Model(adminOutboxTable).Safe().Ctx(ctx).Data(outbox).Insert(); err != nil {
if _, err := tx.Exec(`INSERT INTO im_admin_outbox
(event_id,command_id,topic,event_type,biz_type,biz_id,key_value,envelope,status,created_at,updated_at)
VALUES(?,?,?,?,?,?,?,CAST(? AS JSON),?,?,?)`,
eventID, commandID, consts.QueueImAdminCommandTopic, in.EventType, in.BizType, in.BizID,
in.BizID, string(envelopeBytes), consts.ImAdminOutboxStatusPending, nowTime, nowTime); err != nil {
return gerror.Wrap(err, "创建IM后台命令Outbox失败")
}
return nil
@@ -125,11 +102,9 @@ func PublishAdminCommand(ctx context.Context, in AdminCommandInput) (res *AdminC
}
func DispatchOutboxByEvent(ctx context.Context, eventID string) error {
var row gdb.Record
if err := g.DB().Model(adminOutboxTable).Safe().Ctx(ctx).
Where("event_id", eventID).
Where("status", consts.ImAdminOutboxStatusPending).
Scan(&row); err != nil {
row, err := adminDB().GetOne(ctx, `SELECT * FROM im_admin_outbox WHERE event_id=? AND status=? LIMIT 1`,
eventID, consts.ImAdminOutboxStatusPending)
if err != nil {
return gerror.Wrap(err, "读取IM命令Outbox失败")
}
if row.IsEmpty() {
@@ -171,19 +146,12 @@ func PublishConfigEvent(ctx context.Context, in ConfigEventInput) error {
if err != nil {
return gerror.Wrap(err, "序列化IM配置事件Envelope失败")
}
_, err = g.DB().Model(adminOutboxTable).Safe().Ctx(ctx).Data(g.Map{
"event_id": eventID,
"command_id": "",
"topic": consts.QueueImConfigEventTopic,
"event_type": in.EventType,
"biz_type": "config",
"biz_id": in.BizID,
"key_value": in.TenantCode,
"envelope": string(envelopeBytes),
"status": consts.ImAdminOutboxStatusPending,
"created_at": gtime.Now(),
"updated_at": gtime.Now(),
}).Insert()
nowTime := gtime.Now()
_, err = adminDB().Exec(ctx, `INSERT INTO im_admin_outbox
(event_id,command_id,topic,event_type,biz_type,biz_id,key_value,envelope,status,created_at,updated_at)
VALUES(?,?,?,?,?,?,?,CAST(? AS JSON),?,?,?)`,
eventID, "", consts.QueueImConfigEventTopic, in.EventType, "config", in.BizID, in.TenantCode,
string(envelopeBytes), consts.ImAdminOutboxStatusPending, nowTime, nowTime)
if err != nil {
return gerror.Wrap(err, "创建IM配置事件Outbox失败")
}
@@ -230,19 +198,12 @@ func PublishEvent(ctx context.Context, in EventInput) (eventID string, err error
if err != nil {
return "", gerror.Wrap(err, "序列化IM Kafka事件Envelope失败")
}
_, err = g.DB().Model(adminOutboxTable).Safe().Ctx(ctx).Data(g.Map{
"event_id": eventID,
"command_id": "",
"topic": in.Topic,
"event_type": in.EventType,
"biz_type": in.BizType,
"biz_id": in.BizID,
"key_value": in.BizID,
"envelope": string(envelopeBytes),
"status": consts.ImAdminOutboxStatusPending,
"created_at": gtime.Now(),
"updated_at": gtime.Now(),
}).Insert()
nowTime := gtime.Now()
_, err = adminDB().Exec(ctx, `INSERT INTO im_admin_outbox
(event_id,command_id,topic,event_type,biz_type,biz_id,key_value,envelope,status,created_at,updated_at)
VALUES(?,?,?,?,?,?,?,CAST(? AS JSON),?,?,?)`,
eventID, "", in.Topic, in.EventType, in.BizType, in.BizID, in.BizID,
string(envelopeBytes), consts.ImAdminOutboxStatusPending, nowTime, nowTime)
if err != nil {
return "", gerror.Wrap(err, "创建IM Kafka事件Outbox失败")
}
@@ -256,13 +217,9 @@ func DispatchPendingOutbox(ctx context.Context, limit int) (sent int, err error)
if limit <= 0 {
limit = 100
}
var rows []gdb.Record
err = g.DB().Model(adminOutboxTable).Safe().Ctx(ctx).
WhereIn("status", []string{consts.ImAdminOutboxStatusPending, consts.ImAdminOutboxStatusFailed}).
Where("next_retry_at IS NULL OR next_retry_at <= NOW()").
OrderAsc("id").
Limit(limit).
Scan(&rows)
rows, err := adminDB().GetAll(ctx, `SELECT * FROM im_admin_outbox
WHERE status IN (?,?) AND (next_retry_at IS NULL OR next_retry_at <= NOW())
ORDER BY id ASC LIMIT ?`, consts.ImAdminOutboxStatusPending, consts.ImAdminOutboxStatusFailed, limit)
if err != nil {
return 0, gerror.Wrap(err, "读取待投递IM命令Outbox失败")
}
@@ -294,27 +251,16 @@ func dispatchOutboxRecord(ctx context.Context, row gdb.Record) error {
return err
}
now := gtime.Now()
_, err := g.DB().Model(adminOutboxTable).Safe().Ctx(ctx).
Where("event_id", eventID).
Data(g.Map{
"status": consts.ImAdminOutboxStatusSent,
"attempts": gdb.Raw("attempts + 1"),
"sent_at": now,
"last_reason": "",
"last_error": "",
"updated_at": now,
}).Update()
_, err := adminDB().Exec(ctx, `UPDATE im_admin_outbox
SET status=?,attempts=attempts+1,sent_at=?,last_reason='',last_error='',updated_at=?
WHERE event_id=?`, consts.ImAdminOutboxStatusSent, now, now, eventID)
if err != nil {
return gerror.Wrap(err, "更新IM命令Outbox投递状态失败")
}
if commandID != "" {
_, err = g.DB().Model(adminCommandTable).Safe().Ctx(ctx).
Where("command_id", commandID).
Data(g.Map{
"status": consts.ImAdminCommandStatusRunning,
"sent_at": now,
"updated_at": now,
}).Update()
_, err = adminDB().Exec(ctx, `UPDATE im_admin_commands
SET status=?,sent_at=?,updated_at=? WHERE command_id=?`,
consts.ImAdminCommandStatusRunning, now, now, commandID)
if err != nil {
return gerror.Wrap(err, "更新IM后台命令投递状态失败")
}
@@ -324,16 +270,9 @@ func dispatchOutboxRecord(ctx context.Context, row gdb.Record) error {
func markOutboxFailed(ctx context.Context, eventID string, reason string, message string) error {
now := gtime.Now()
_, err := g.DB().Model(adminOutboxTable).Safe().Ctx(ctx).
Where("event_id", eventID).
Data(g.Map{
"status": consts.ImAdminOutboxStatusFailed,
"attempts": gdb.Raw("attempts + 1"),
"next_retry_at": now.Add(30 * time.Second),
"last_reason": reason,
"last_error": message,
"updated_at": now,
}).Update()
_, err := adminDB().Exec(ctx, `UPDATE im_admin_outbox
SET status=?,attempts=attempts+1,next_retry_at=?,last_reason=?,last_error=?,updated_at=?
WHERE event_id=?`, consts.ImAdminOutboxStatusFailed, now.Add(30*time.Second), reason, message, now, eventID)
if err != nil {
return gerror.Wrap(err, "更新IM命令Outbox失败状态失败")
}
@@ -364,15 +303,27 @@ func ApplyCommandResult(ctx context.Context, result CommandResult) error {
if len(result.Result) > 0 {
data["result"] = string(result.Result)
}
mod := g.DB().Model(adminCommandTable).Safe().Ctx(ctx)
whereColumn := "event_id"
whereValue := result.CommandEventID
if result.CommandID != "" {
mod = mod.Where("command_id", result.CommandID)
} else {
mod = mod.Where("event_id", result.CommandEventID)
whereColumn = "command_id"
whereValue = result.CommandID
}
_, err := mod.Data(data).Update()
sql := `UPDATE im_admin_commands SET status=?,reason=?,message=?,completed_at=?,updated_at=?`
args := []any{data["status"], data["reason"], data["message"], data["completed_at"], data["updated_at"]}
if resultValue, ok := data["result"]; ok {
sql += `,result=CAST(? AS JSON)`
args = append(args, resultValue)
}
sql += ` WHERE ` + whereColumn + `=?`
args = append(args, whereValue)
_, err := adminDB().Exec(ctx, sql, args...)
if err != nil {
return gerror.Wrap(err, "更新IM后台命令结果失败")
}
return nil
}
func adminDB() gdb.DB {
return imtenant.RegistryDB()
}
+58 -12
View File
@@ -7,6 +7,7 @@ import (
"hash/fnv"
"hotgo/internal/dao"
"hotgo/internal/model/entity"
"net/url"
"strings"
"sync"
@@ -20,14 +21,7 @@ var tenantDbGroups sync.Map
const registryDbGroup = "imRegistry"
// RegistryDB returns the tenant registry DB. It falls back to the default DB for
// single-database deployments where database.imRegistry is not configured.
func RegistryDB(ctx context.Context) gdb.DB {
if configs, err := gdb.GetConfigGroup(registryDbGroup); err == nil && len(configs) > 0 {
return g.DB(registryDbGroup)
}
return g.DB()
}
var registryDbGroupOnce sync.Once
// DB resolves a tenant registry record and returns the corresponding business DB.
func DB(ctx context.Context, tenantId int64) (db gdb.DB, tenant *entity.ImTenant, err error) {
@@ -35,7 +29,7 @@ func DB(ctx context.Context, tenantId int64) (db gdb.DB, tenant *entity.ImTenant
return nil, nil, gerror.New("租户ID不能为空")
}
if err = RegistryDB(ctx).Model(dao.ImTenant.Table()).Safe().Ctx(ctx).WherePri(tenantId).Scan(&tenant); err != nil {
if err = RegistryDB().Model(dao.ImTenant.Table()).Safe().Ctx(ctx).WherePri(tenantId).Scan(&tenant); err != nil {
return nil, nil, gerror.Wrap(err, "获取租户信息失败")
}
if tenant == nil || tenant.Id <= 0 {
@@ -65,9 +59,40 @@ func DB(ctx context.Context, tenantId int64) (db gdb.DB, tenant *entity.ImTenant
return db, tenant, nil
}
// RegistryDB returns the dedicated tenant registry connection, or the default
// HotGo database when a separate registry group is not configured.
func RegistryDB() gdb.DB {
registryDbGroupOnce.Do(func() {
ensureRegistryDBGroup()
})
if configs, err := gdb.GetConfigGroup(registryDbGroup); err == nil && len(configs) > 0 {
return g.DB(registryDbGroup)
}
return g.DB()
}
func ensureRegistryDBGroup() {
if configs, err := gdb.GetConfigGroup(registryDbGroup); err == nil && len(configs) > 0 {
configs[0].Prefix = ""
_ = gdb.SetConfigGroup(registryDbGroup, configs)
return
}
configs, err := gdb.GetConfigGroup("default")
if err != nil || len(configs) == 0 {
return
}
cloned := make(gdb.ConfigGroup, len(configs))
copy(cloned, configs)
for i := range cloned {
cloned[i].Prefix = ""
}
_ = gdb.SetConfigGroup(registryDbGroup, cloned)
}
func tenantLink(ctx context.Context, tenant *entity.ImTenant) (string, error) {
if tenant.DbDsn != "" {
return tenant.DbDsn, nil
return normalizeMySQLTimeOptions(tenant.DbDsn)
}
if tenant.DbName == "" {
return "", gerror.New("租户业务库未配置")
@@ -115,10 +140,31 @@ func replaceMySQLDatabase(link string, dbName string) (string, error) {
dbStart += 2
queryStart := strings.Index(link[dbStart:], "?")
if queryStart < 0 {
return link[:dbStart] + dbName, nil
return normalizeMySQLTimeOptions(link[:dbStart] + dbName)
}
queryStart += dbStart
return link[:dbStart] + dbName + link[queryStart:], nil
return normalizeMySQLTimeOptions(link[:dbStart] + dbName + link[queryStart:])
}
func normalizeMySQLTimeOptions(link string) (string, error) {
if !strings.HasPrefix(link, "mysql:") {
if !strings.Contains(link, "@tcp(") {
return "", gerror.New("租户业务库仅支持 MySQL")
}
link = "mysql:" + link
}
parts := strings.SplitN(link, "?", 2)
values := make(url.Values)
if len(parts) == 2 {
var err error
values, err = url.ParseQuery(parts[1])
if err != nil {
return "", gerror.Wrap(err, "租户业务库连接参数格式不正确")
}
}
values.Set("loc", "UTC")
values.Set("parseTime", "true")
return parts[0] + "?" + values.Encode(), nil
}
func hash(s string) uint64 {
@@ -0,0 +1,52 @@
package imtenant
import (
"context"
"strings"
"testing"
)
func TestNormalizeMySQLTimeOptions(t *testing.T) {
link, err := normalizeMySQLTimeOptions("user:pass@tcp(127.0.0.1:3306)/tinode?charset=utf8mb4")
if err != nil {
t.Fatalf("normalize raw mysql dsn: %v", err)
}
if !strings.HasPrefix(link, "mysql:user:pass@tcp(127.0.0.1:3306)/tinode?") {
t.Fatalf("unexpected normalized link: %s", link)
}
if !strings.Contains(link, "loc=UTC") || !strings.Contains(link, "parseTime=true") {
t.Fatalf("missing required time options: %s", link)
}
}
func TestNormalizeMySQLTimeOptionsRejectsUnsupportedFormat(t *testing.T) {
if _, err := normalizeMySQLTimeOptions("postgres://user:pass@127.0.0.1:5432/tinode"); err == nil {
t.Fatal("expected unsupported format error")
}
}
func TestReplaceMySQLDatabaseUsesUTC(t *testing.T) {
link, err := replaceMySQLDatabase("mysql:user:pass@tcp(127.0.0.1:3306)/hotgo?loc=Local&parseTime=false&charset=utf8mb4", "tinode")
if err != nil {
t.Fatal(err)
}
for _, expected := range []string{"/tinode?", "loc=UTC", "parseTime=true", "charset=utf8mb4"} {
if !strings.Contains(link, expected) {
t.Fatalf("link %q does not contain %q", link, expected)
}
}
}
func TestNextSnowflakeID(t *testing.T) {
first, err := NextSnowflakeID(context.Background())
if err != nil {
t.Fatal(err)
}
second, err := NextSnowflakeID(context.Background())
if err != nil {
t.Fatal(err)
}
if first <= 0 || second <= first {
t.Fatalf("Snowflake IDs are not increasing: %d, %d", first, second)
}
}
@@ -0,0 +1,56 @@
package imtenant
import (
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
"testing"
)
func TestNoDirectDefaultDBModelForIMTables(t *testing.T) {
_, file, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("runtime caller unavailable")
}
serverRoot := filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
internalRoot := filepath.Join(serverRoot, "internal")
patterns := []*regexp.Regexp{
regexp.MustCompile(`g\.DB\(\)\.Model\(\s*"im_`),
regexp.MustCompile(`g\.DB\(\)\.Model\(\s*` + "`" + `im_`),
regexp.MustCompile(`g\.DB\(\)\.(Insert|InsertIgnore|Replace|Save|Update|Delete)\([^,\n]+,\s*"im_`),
regexp.MustCompile(`g\.DB\(\)\.(Insert|InsertIgnore|Replace|Save|Update|Delete)\([^,\n]+,\s*` + "`" + `im_`),
}
var violations []string
err := filepath.WalkDir(internalRoot, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
return nil
}
content, err := os.ReadFile(path)
if err != nil {
return err
}
for _, pattern := range patterns {
if pattern.Match(content) {
rel, _ := filepath.Rel(serverRoot, path)
violations = append(violations, rel)
break
}
}
return nil
})
if err != nil {
t.Fatal(err)
}
if len(violations) > 0 {
t.Fatalf("IM public tables must not use default g.DB() model helpers because the default DB has table prefix: %s", strings.Join(violations, ", "))
}
}
@@ -281,6 +281,8 @@ func (client *Client) handleRoutineTask(ctx context.Context, task func()) {
// Conn 获取当前连接
func (client *Client) Conn() *Conn {
client.mutex.Lock()
defer client.mutex.Unlock()
return client.conn
}
+6 -2
View File
@@ -43,7 +43,6 @@ func (r *RPC) Request(ctx context.Context, msgId string, send func()) (res inter
defer func() {
isClose.Set(true)
close(resCh)
r.popCallback(msgId)
}()
@@ -57,8 +56,13 @@ func (r *RPC) Request(ctx context.Context, msgId string, send func()) (res inter
r.task(ctx, send)
timer := time.NewTimer(time.Second * RPCTimeout)
defer timer.Stop()
select {
case <-time.After(time.Second * RPCTimeout):
case <-ctx.Done():
err = gerror.Wrap(ctx.Err(), "RPC request canceled")
return
case <-timer.C:
err = gerror.New("RPC response timeout")
return
case got := <-resCh:
@@ -1,21 +1,14 @@
// Package tcp_test
// @Link https://github.com/bufanyun/hotgo
// @Copyright Copyright (c) 2023 HotGo CLI
// @Author Ms <133814250@qq.com>
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
package tcp_test
import (
"context"
"fmt"
"github.com/gogf/gf/v2/os/gctx"
"github.com/gogf/gf/v2/test/gtest"
"hotgo/internal/library/network/tcp"
"net"
"testing"
"time"
)
var T *testing.T // 声明一个全局的 *testing.T 变量
"hotgo/internal/library/network/tcp"
)
type TestMsgReq struct {
Name string `json:"name"`
@@ -33,101 +26,95 @@ type TestRPCMsgRes struct {
tcp.ServerRes
}
func onTestMsg(ctx context.Context, req *TestMsgReq) {
fmt.Printf("服务器收到消息 ==> onTestMsg:%+v\n", req)
conn := tcp.ConnFromCtx(ctx)
gtest.C(T, func(t *gtest.T) {
t.AssertNE(conn, nil)
})
res := new(TestMsgRes)
res.Message = fmt.Sprintf("你的名字:%v", req.Name)
conn.Send(ctx, res)
}
func onResponseTestMsg(ctx context.Context, req *TestMsgRes) {
fmt.Printf("客户端收到响应消息 ==> TestMsgRes:%+v\n", req)
err := req.GetError()
gtest.C(T, func(t *gtest.T) {
t.AssertNil(err)
})
}
func onTestRPCMsg(ctx context.Context, req *TestRPCMsgReq) (res *TestRPCMsgRes, err error) {
fmt.Printf("服务器收到消息 ==> onTestRPCMsg:%+v\n", req)
res = new(TestRPCMsgRes)
res.Message = fmt.Sprintf("你的名字:%v", req.Name)
return
}
func startTCPServer() {
serv := tcp.NewServer(&tcp.ServerConfig{
Name: "hotgo",
Addr: ":8002",
})
// 注册路由
serv.RegisterRouter(
onTestMsg,
)
// 注册RPC路由
serv.RegisterRPCRouter(
onTestRPCMsg,
)
// 服务监听
err := serv.Listen()
gtest.C(T, func(t *gtest.T) {
t.AssertNil(err)
})
}
// 一个基本的消息收发
func TestSendMsg(t *testing.T) {
T = t
go startTCPServer()
ctx := gctx.New()
client := tcp.NewClient(&tcp.ClientConfig{
Addr: "127.0.0.1:8002",
address := freeTCPAddress(t)
messageResponse := make(chan string, 1)
server := tcp.NewServer(&tcp.ServerConfig{Name: "hotgo-test", Addr: address})
server.RegisterRouter(func(ctx context.Context, req *TestMsgReq) {
res := &TestMsgRes{}
res.SetMessage(fmt.Sprintf("你的名字:%s", req.Name))
if err := tcp.ConnFromCtx(ctx).Send(ctx, res); err != nil {
messageResponse <- "send-error:" + err.Error()
}
})
// 注册路由
client.RegisterRouter(
onResponseTestMsg,
)
server.RegisterRPCRouter(func(ctx context.Context, req *TestRPCMsgReq) (*TestRPCMsgRes, error) {
res := &TestRPCMsgRes{}
res.SetMessage(fmt.Sprintf("你的名字:%s", req.Name))
return res, nil
})
serverErrors := make(chan error, 1)
go func() {
err := client.Start()
gtest.C(T, func(t *gtest.T) {
t.AssertNil(err)
})
serverErrors <- server.Listen()
}()
defer server.Close()
// 确保服务都启动完成
time.Sleep(time.Second * 1)
// 拿到客户端的连接
conn := client.Conn()
gtest.C(T, func(t *gtest.T) {
t.AssertNE(conn, nil)
client := tcp.NewClient(&tcp.ClientConfig{
Addr: address,
Timeout: time.Second,
ConnectInterval: 10 * time.Millisecond,
MaxConnectCount: 100,
})
// 向服务器发送tcp消息,不会阻塞程序执行
err := conn.Send(ctx, &TestMsgReq{Name: "Tom"})
gtest.C(T, func(t *gtest.T) {
t.AssertNil(err)
client.RegisterRouter(func(ctx context.Context, res *TestMsgRes) {
messageResponse <- res.Message
})
if err := client.Start(); err != nil {
t.Fatalf("启动TCP客户端失败: %v", err)
}
defer client.Stop()
// 向服务器发送rpc消息,会等待服务器响应结果,直到拿到结果或响应超时才会继续
var res TestRPCMsgRes
if err = conn.RequestScan(ctx, &TestRPCMsgReq{Name: "Tony"}, &res); err != nil {
gtest.C(T, func(t *gtest.T) {
t.AssertNil(err)
})
conn := waitForConnection(t, client, serverErrors)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := conn.Send(ctx, &TestMsgReq{Name: "Tom"}); err != nil {
t.Fatalf("发送TCP消息失败: %v", err)
}
select {
case message := <-messageResponse:
if message != "你的名字:Tom" {
t.Fatalf("TCP响应不符: %s", message)
}
case <-ctx.Done():
t.Fatal("等待TCP响应超时")
}
fmt.Printf("客户端收到RPC消息响应 ==> TestRPCMsgRes:%+v\n", res)
time.Sleep(time.Second * 1)
var rpcResponse TestRPCMsgRes
if err := conn.RequestScan(ctx, &TestRPCMsgReq{Name: "Tony"}, &rpcResponse); err != nil {
t.Fatalf("发送RPC消息失败: %v", err)
}
if rpcResponse.Message != "你的名字:Tony" {
t.Fatalf("RPC响应不符: %s", rpcResponse.Message)
}
}
func freeTCPAddress(t *testing.T) string {
t.Helper()
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("申请测试端口失败: %v", err)
}
address := listener.Addr().String()
if err = listener.Close(); err != nil {
t.Fatalf("释放测试端口失败: %v", err)
}
return address
}
func waitForConnection(t *testing.T, client *tcp.Client, serverErrors <-chan error) *tcp.Conn {
t.Helper()
deadline := time.NewTimer(2 * time.Second)
defer deadline.Stop()
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for {
select {
case err := <-serverErrors:
t.Fatalf("TCP服务启动失败: %v", err)
case <-deadline.C:
t.Fatal("等待TCP连接超时")
case <-ticker.C:
if conn := client.Conn(); conn != nil {
return conn
}
}
}
}
+2 -2
View File
@@ -55,7 +55,7 @@ func consumerListen(ctx context.Context, job Consumer) {
)
if err != nil {
Logger().Fatalf(ctx, "InstanceConsumer %s err:%+v", topic, err)
Logger().Errorf(ctx, "InstanceConsumer %s err:%+v", topic, err)
return
}
@@ -70,6 +70,6 @@ func consumerListen(ctx context.Context, job Consumer) {
// 记录消费队列日志
ConsumerLog(ctx, topic, mqMsg, err)
}); listenErr != nil {
Logger().Fatalf(ctx, "消费队列:%s 监听失败, err:%+v", topic, listenErr)
Logger().Errorf(ctx, "消费队列:%s 监听失败, err:%+v", topic, listenErr)
}
}
+112 -65
View File
@@ -7,7 +7,10 @@ package queue
import (
"context"
"crypto/tls"
"crypto/sha256"
"crypto/sha512"
"errors"
"hash"
"strings"
"time"
@@ -15,6 +18,7 @@ import (
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/xdg-go/scram"
"hotgo/internal/consts"
"hotgo/utility/simple"
)
@@ -26,17 +30,37 @@ type KafkaMq struct {
}
type KafkaConfig struct {
ClientId string
Brokers []string
GroupID string
Partitions int32
Replication int16
Version string
UserName string
Password string
SASLEnabled bool
SASLMechanism string
TLSEnabled bool
ClientId string
Brokers []string
GroupID string
Partitions int32
Replication int16
Version string
UserName string
Password string
Mechanism string
}
type xdgSCRAMClient struct {
*scram.ClientConversation
HashGeneratorFcn scram.HashGeneratorFcn
}
func (x *xdgSCRAMClient) Begin(userName, password, authzID string) error {
client, err := x.HashGeneratorFcn.NewClient(userName, password, authzID)
if err != nil {
return err
}
x.ClientConversation = client.NewConversation()
return nil
}
func (x *xdgSCRAMClient) Step(challenge string) (string, error) {
return x.ClientConversation.Step(challenge)
}
func (x *xdgSCRAMClient) Done() bool {
return x.ClientConversation.Done()
}
// SendMsg 按字符串类型生产数据
@@ -100,11 +124,16 @@ func (r *KafkaMq) ListenReceiveMsgDo(topic string, receiveDo func(mqMsg MqMsg))
go func(consumerCtx context.Context) {
for {
if err = r.consumerIns.Consume(consumerCtx, []string{topic}, &consumer); err != nil {
Logger().Fatalf(ctx, "kafka Error from consumer, err%+v", err)
Logger().Errorf(ctx, "kafka error from consumer, topic:%s err:%+v", topic, err)
select {
case <-consumerCtx.Done():
return
case <-time.After(3 * time.Second):
}
}
if consumerCtx.Err() != nil {
Logger().Debugf(ctx, "kafka consoumer stop : %v", consumerCtx.Err())
Logger().Debugf(ctx, "kafka consumer stop : %v", consumerCtx.Err())
return
}
consumer.ready = make(chan bool)
@@ -112,14 +141,18 @@ func (r *KafkaMq) ListenReceiveMsgDo(topic string, receiveDo func(mqMsg MqMsg))
}(consumerCtx)
// await till the consumer has been set up
<-consumer.ready
select {
case <-consumer.ready:
case <-time.After(10 * time.Second):
Logger().Warningf(ctx, "kafka consumer setup timeout, topic:%s", topic)
}
Logger().Debug(ctx, "kafka consumer up and running!...")
simple.Event().Register(consts.EventServerClose, func(ctx context.Context, args ...interface{}) {
Logger().Debug(ctx, "kafka consumer close...")
cancel()
if err = r.consumerIns.Close(); err != nil {
Logger().Fatalf(ctx, "kafka Error closing client, err:%+v", err)
Logger().Errorf(ctx, "kafka error closing client, err:%+v", err)
}
})
return
@@ -128,7 +161,7 @@ func (r *KafkaMq) ListenReceiveMsgDo(topic string, receiveDo func(mqMsg MqMsg))
// RegisterKafkaMqConsumer 注册消费者
func RegisterKafkaMqConsumer(connOpt KafkaConfig) (client MqConsumer, err error) {
mqIns := &KafkaMq{}
kfkVersion, err := sarama.ParseKafkaVersion(connOpt.Version)
kfkVersion, err := parseKafkaVersion(connOpt.Version)
if err != nil {
return
}
@@ -140,9 +173,7 @@ func RegisterKafkaMqConsumer(connOpt KafkaConfig) (client MqConsumer, err error)
conf := sarama.NewConfig()
conf.Consumer.Return.Errors = true
conf.Version = kfkVersion
if err = configureKafkaSecurity(conf, connOpt); err != nil {
return
}
configureKafkaSASL(conf, connOpt)
// 默认按随机方式消费
conf.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRange
@@ -161,9 +192,7 @@ func RegisterKafkaMqConsumer(connOpt KafkaConfig) (client MqConsumer, err error)
// RegisterKafkaProducer 注册并启动生产者接口实现
func RegisterKafkaProducer(connOpt KafkaConfig) (client MqProducer, err error) {
mqIns := &KafkaMq{}
if connOpt.ClientId == "" {
connOpt.ClientId = "hotgo-producer"
}
connOpt.ClientId = "HOTGO-Producer"
// 这里如果使用go程需要处理chan同步问题
if err = doRegisterKafkaProducer(connOpt, mqIns); err != nil {
@@ -175,7 +204,7 @@ func RegisterKafkaProducer(connOpt KafkaConfig) (client MqProducer, err error) {
// doRegisterKafkaProducer 注册同步类型实例
func doRegisterKafkaProducer(connOpt KafkaConfig, mqIns *KafkaMq) (err error) {
kfkVersion, err := sarama.ParseKafkaVersion(connOpt.Version)
kfkVersion, err := parseKafkaVersion(connOpt.Version)
if err != nil {
return
}
@@ -197,9 +226,7 @@ func doRegisterKafkaProducer(connOpt KafkaConfig, mqIns *KafkaMq) (err error) {
conf.ClientID = connOpt.ClientId
conf.Version = kfkVersion
if err = configureKafkaSecurity(conf, connOpt); err != nil {
return
}
configureKafkaSASL(conf, connOpt)
mqIns.producerIns, err = sarama.NewAsyncProducer(brokers, conf)
if err != nil {
@@ -213,61 +240,81 @@ func doRegisterKafkaProducer(connOpt KafkaConfig, mqIns *KafkaMq) (err error) {
return
}
func configureKafkaSecurity(conf *sarama.Config, connOpt KafkaConfig) error {
if connOpt.TLSEnabled {
conf.Net.TLS.Enable = true
conf.Net.TLS.Config = &tls.Config{MinVersion: tls.VersionTLS12}
func parseKafkaVersion(version string) (sarama.KafkaVersion, error) {
parts := strings.Split(version, ".")
if len(parts) == 4 && parts[0] != "0" && parts[3] == "0" {
version = strings.Join(parts[:3], ".")
}
if !connOpt.SASLEnabled && connOpt.UserName == "" {
return nil
}
if connOpt.UserName == "" || connOpt.Password == "" {
return gerror.New("queue kafka sasl username or password is empty")
return sarama.ParseKafkaVersion(version)
}
func configureKafkaSASL(conf *sarama.Config, connOpt KafkaConfig) {
if connOpt.UserName == "" {
return
}
conf.Net.SASL.Enable = true
conf.Net.SASL.User = connOpt.UserName
conf.Net.SASL.Password = connOpt.Password
mechanism := strings.ToUpper(strings.ReplaceAll(connOpt.SASLMechanism, "_", "-"))
mechanism := strings.ToUpper(strings.ReplaceAll(connOpt.Mechanism, "_", "-"))
if mechanism == "" {
mechanism = sarama.SASLTypeSCRAMSHA512
}
switch mechanism {
case "", "PLAIN":
conf.Net.SASL.Mechanism = sarama.SASLTypePlaintext
case "SCRAM-SHA-256":
case sarama.SASLTypeSCRAMSHA512:
conf.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA512
conf.Net.SASL.SCRAMClientGeneratorFunc = scramClientGenerator(sha512.New)
case sarama.SASLTypeSCRAMSHA256:
conf.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA256
conf.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
return &xdgSCRAMClient{HashGeneratorFcn: scram.SHA256}
}
case "SCRAM-SHA-512":
conf.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA512
conf.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
return &xdgSCRAMClient{HashGeneratorFcn: scram.SHA512}
}
conf.Net.SASL.SCRAMClientGeneratorFunc = scramClientGenerator(sha256.New)
default:
return gerror.New("queue kafka sasl mechanism is not support")
conf.Net.SASL.Mechanism = sarama.SASLMechanism(mechanism)
}
return nil
}
type xdgSCRAMClient struct {
*scram.Client
*scram.ClientConversation
scram.HashGeneratorFcn
}
func (x *xdgSCRAMClient) Begin(userName, password, authzID string) (err error) {
x.Client, err = x.HashGeneratorFcn.NewClient(userName, password, authzID)
// KafkaHealthCheck verifies broker connectivity by creating a short-lived Sarama client.
func KafkaHealthCheck(ctx context.Context) error {
if len(config.Kafka.Address) == 0 {
return errors.New("queue kafka address is empty")
}
kfkVersion, err := parseKafkaVersion(config.Kafka.Version)
if err != nil {
return err
}
x.ClientConversation = x.Client.NewConversation()
if !validateVersion(kfkVersion) {
kfkVersion = sarama.V2_4_0_0
}
conf := sarama.NewConfig()
conf.Version = kfkVersion
conf.ClientID = "hotgo-healthz"
conf.Net.DialTimeout = 2 * time.Second
conf.Net.ReadTimeout = 2 * time.Second
conf.Net.WriteTimeout = 2 * time.Second
configureKafkaSASL(conf, KafkaConfig{
UserName: config.Kafka.Username,
Password: config.Kafka.Password,
Mechanism: config.Kafka.Mechanism,
})
select {
case <-ctx.Done():
return ctx.Err()
default:
}
client, err := sarama.NewClient(config.Kafka.Address, conf)
if err != nil {
return err
}
defer func() { _ = client.Close() }()
if len(client.Brokers()) == 0 {
return errors.New("no kafka brokers available")
}
return nil
}
func (x *xdgSCRAMClient) Step(challenge string) (response string, err error) {
return x.ClientConversation.Step(challenge)
}
func (x *xdgSCRAMClient) Done() bool {
return x.ClientConversation.Done()
func scramClientGenerator(hashFunc func() hash.Hash) func() sarama.SCRAMClient {
return func() sarama.SCRAMClient {
return &xdgSCRAMClient{HashGeneratorFcn: scram.HashGeneratorFcn(hashFunc)}
}
}
// validateVersion 验证版本是否有效
+16 -51
View File
@@ -11,8 +11,6 @@ import (
"github.com/gogf/gf/v2/os/gctx"
"hotgo/internal/library/queue/disk"
"hotgo/utility/charset"
"os"
"strings"
"sync"
"time"
)
@@ -59,14 +57,11 @@ type RocketmqConf struct {
type KafkaConf struct {
Address []string `json:"address"`
Version string `json:"version"`
Username string `json:"username"`
Password string `json:"password"`
Mechanism string `json:"mechanism"`
RandClient bool `json:"randClient"`
MultiConsumer bool `json:"multiConsumer"`
ClientId string `json:"clientId"`
UserName string `json:"userName"`
Password string `json:"password"`
SASLEnabled bool `json:"saslEnabled"`
SASLMechanism string `json:"saslMechanism"`
TLSEnabled bool `json:"tlsEnabled"`
}
type MqMsg struct {
@@ -93,7 +88,6 @@ func init() {
if err := g.Cfg().MustGet(ctx, "queue").Scan(&config); err != nil {
Logger().Warningf(ctx, "queue init err:%+v", err)
}
normalizeKafkaConf()
}
// InstanceConsumer 实例化消费者
@@ -130,15 +124,12 @@ func NewProducer(groupName string) (mqClient MqProducer, err error) {
return
}
mqClient, err = RegisterKafkaProducer(KafkaConfig{
Brokers: config.Kafka.Address,
GroupID: groupName,
Version: config.Kafka.Version,
ClientId: firstKafkaClientId(config.Kafka.ClientId, "hotgo-producer"),
UserName: config.Kafka.UserName,
Password: config.Kafka.Password,
SASLEnabled: config.Kafka.SASLEnabled,
SASLMechanism: config.Kafka.SASLMechanism,
TLSEnabled: config.Kafka.TLSEnabled,
Brokers: config.Kafka.Address,
GroupID: groupName,
Version: config.Kafka.Version,
UserName: config.Kafka.Username,
Password: config.Kafka.Password,
Mechanism: config.Kafka.Mechanism,
})
case "redis":
if _, err = g.Redis().Do(ctx, "ping"); err == nil {
@@ -200,15 +191,13 @@ func NewConsumer(groupName string) (mqClient MqConsumer, err error) {
}
mqClient, err = RegisterKafkaMqConsumer(KafkaConfig{
Brokers: config.Kafka.Address,
GroupID: groupName,
Version: config.Kafka.Version,
ClientId: clientId,
UserName: config.Kafka.UserName,
Password: config.Kafka.Password,
SASLEnabled: config.Kafka.SASLEnabled,
SASLMechanism: config.Kafka.SASLMechanism,
TLSEnabled: config.Kafka.TLSEnabled,
Brokers: config.Kafka.Address,
GroupID: groupName,
Version: config.Kafka.Version,
ClientId: clientId,
UserName: config.Kafka.Username,
Password: config.Kafka.Password,
Mechanism: config.Kafka.Mechanism,
})
case "redis":
if _, err = g.Redis().Do(ctx, "ping"); err == nil {
@@ -233,30 +222,6 @@ func NewConsumer(groupName string) (mqClient MqConsumer, err error) {
return
}
func normalizeKafkaConf() {
config.Kafka.UserName = resolveEnvPlaceholder(config.Kafka.UserName)
config.Kafka.Password = resolveEnvPlaceholder(config.Kafka.Password)
if config.Kafka.SASLMechanism == "" {
config.Kafka.SASLMechanism = "SCRAM-SHA-512"
}
}
func resolveEnvPlaceholder(value string) string {
value = strings.TrimSpace(value)
if strings.HasPrefix(value, "${") && strings.HasSuffix(value, "}") {
return os.Getenv(strings.TrimSuffix(strings.TrimPrefix(value, "${"), "}"))
}
return value
}
func firstKafkaClientId(value string, fallback string) string {
value = strings.TrimSpace(value)
if value != "" {
return value
}
return fallback
}
// BodyString 返回消息体
func (m *MqMsg) BodyString() string {
return string(m.Body)
@@ -13,6 +13,7 @@ import (
"github.com/gogf/gf/v2/os/gfile"
"github.com/gogf/gf/v2/os/gtime"
"github.com/gogf/gf/v2/util/gconv"
"io"
"os"
"path/filepath"
"sort"
@@ -137,20 +138,53 @@ func (d *LocalDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res
// MergePartFile 合并分片文件
func MergePartFile(srcPath, dstPath string) (err error) {
dir, err := os.ReadDir(srcPath)
entries, err := os.ReadDir(srcPath)
if err != nil {
return err
}
sort.Slice(dir, func(i, j int) bool {
fiIndex, _ := strconv.Atoi(dir[i].Name())
fjIndex, _ := strconv.Atoi(dir[j].Name())
return fiIndex < fjIndex
type filePart struct {
index int
path string
}
parts := make([]filePart, 0, len(entries))
for _, entry := range entries {
if entry.IsDir() {
continue
}
index, parseErr := strconv.Atoi(entry.Name())
if parseErr != nil || index <= 0 {
return gerror.Newf("无效的分片文件名:%s", entry.Name())
}
parts = append(parts, filePart{index: index, path: filepath.Join(srcPath, entry.Name())})
}
if len(parts) == 0 {
return gerror.New("没有可合并的分片文件")
}
sort.Slice(parts, func(i, j int) bool {
return parts[i].index < parts[j].index
})
for _, file := range dir {
filePath := filepath.Join(srcPath, file.Name())
if err = gfile.PutBytesAppend(dstPath, gfile.GetBytes(filePath)); err != nil {
return err
if err = os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil {
return gerror.Wrap(err, "创建目标目录失败")
}
dstFile, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
if err != nil {
return gerror.Wrap(err, "创建合并文件失败")
}
defer dstFile.Close()
for _, part := range parts {
srcFile, openErr := os.Open(part.path)
if openErr != nil {
return gerror.Wrapf(openErr, "打开分片%d失败", part.index)
}
_, copyErr := io.Copy(dstFile, srcFile)
closeErr := srcFile.Close()
if copyErr != nil {
return gerror.Wrapf(copyErr, "合并分片%d失败", part.index)
}
if closeErr != nil {
return gerror.Wrapf(closeErr, "关闭分片%d失败", part.index)
}
}
return
return dstFile.Sync()
}
@@ -1,61 +1,52 @@
package storager
import (
"crypto/md5"
"errors"
"fmt"
"io"
"bytes"
"os"
"path/filepath"
"testing"
)
func TestMergePartFile(t *testing.T) {
srcFile, _ := os.Open("1.zip")
defer srcFile.Close()
fileInfo, err := srcFile.Stat()
if err != nil {
t.Error(err)
tempDir := t.TempDir()
partDir := filepath.Join(tempDir, "parts")
if err := os.MkdirAll(partDir, 0o755); err != nil {
t.Fatal(err)
}
hash := md5.New()
_, err = io.Copy(hash, srcFile)
sum := hash.Sum(nil)
fileHash := fmt.Sprintf("%x", sum)
var singleSize = int64(1024 * 1024 * 100)
chunks := int(fileInfo.Size() / singleSize)
if fileInfo.Size()%singleSize != 0 {
chunks += 1
parts := map[string][]byte{
"1": []byte("first-"),
"2": []byte("second-"),
"10": []byte("tenth"),
}
srcFile.Seek(0, io.SeekStart)
for j := 0; j < chunks; j++ {
partSize := singleSize
if j == chunks-1 {
partSize = fileInfo.Size() - int64(j)*singleSize
for name, content := range parts {
if err := os.WriteFile(filepath.Join(partDir, name), content, 0o644); err != nil {
t.Fatal(err)
}
partData := make([]byte, partSize)
_, err = io.ReadFull(srcFile, partData)
pf, _ := os.Create(fmt.Sprintf("tmp/%d", j+1))
_, err = pf.Write(partData)
if err != nil {
t.Error(err)
return
}
pf.Close()
}
err = MergePartFile("tmp/", "2.zip")
destination := filepath.Join(tempDir, "merged.bin")
if err := os.WriteFile(destination, []byte("stale-data"), 0o644); err != nil {
t.Fatal(err)
}
if err := MergePartFile(partDir, destination); err != nil {
t.Fatalf("合并分片失败: %v", err)
}
actual, err := os.ReadFile(destination)
if err != nil {
t.Error(err)
return
t.Fatal(err)
}
expected := []byte("first-second-tenth")
if !bytes.Equal(actual, expected) {
t.Fatalf("合并内容不符 expected=%q actual=%q", expected, actual)
}
}
f2, _ := os.Open("2.zip")
hash2 := md5.New()
_, err = io.Copy(hash2, f2)
sum2 := hash.Sum(nil)
fileHash2 := fmt.Sprintf("%x", sum2)
if fileHash != fileHash2 {
t.Error(errors.New("hash mismatch"))
func TestMergePartFileRejectsEmptyDirectory(t *testing.T) {
partDir := t.TempDir()
err := MergePartFile(partDir, filepath.Join(t.TempDir(), "merged.bin"))
if err == nil {
t.Fatal("空分片目录应返回错误")
}
}