pro/im_manager
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add im kafka admin integration

Browse Source
This commit is contained in:
richjohnrambo
2026-09-04 11:57:38 +08:00
parent a2abaef3a6
commit f7f8cc0ef0
74 changed files with 6254 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/library/queue/disk.go
+1 -1
View File
@@ -125,7 +125,7 @@ func (d *DiskProducerMq) getProducer(topic string) *disk.Queue {
func NewDiskQueue(topic string, config *disk.Config) *disk.Queue {
conf := &disk.Config{
Path: fmt.Sprintf(config.Path + "/" + config.GroupName + "/" + topic),
Path: fmt.Sprintf("%s/%s/%s", config.Path, config.GroupName, topic),
BatchSize: config.BatchSize,
BatchTime: config.BatchTime * time.Second,
SegmentSize: config.SegmentSize,
server/internal/library/queue/kafkamq.go
+81 -20
View File
@@ -7,13 +7,16 @@ package queue
import (
"context"
"fmt"
"crypto/tls"
"strings"
"time"
"github.com/IBM/sarama"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/xdg-go/scram"
"hotgo/internal/consts"
"hotgo/utility/simple"
"time"
)
type KafkaMq struct {
@@ -23,14 +26,17 @@ type KafkaMq struct {
}
type KafkaConfig struct {
ClientId string
Brokers []string
GroupID string
Partitions int32
Replication int16
Version string
UserName string
Password string
ClientId string
Brokers []string
GroupID string
Partitions int32
Replication int16
Version string
UserName string
Password string
SASLEnabled bool
SASLMechanism string
TLSEnabled bool
}
// SendMsg 按字符串类型生产数据
@@ -98,7 +104,7 @@ func (r *KafkaMq) ListenReceiveMsgDo(topic string, receiveDo func(mqMsg MqMsg))
}
if consumerCtx.Err() != nil {
Logger().Debugf(ctx, fmt.Sprintf("kafka consoumer stop : %v", consumerCtx.Err()))
Logger().Debugf(ctx, "kafka consoumer stop : %v", consumerCtx.Err())
return
}
consumer.ready = make(chan bool)
@@ -134,10 +140,8 @@ func RegisterKafkaMqConsumer(connOpt KafkaConfig) (client MqConsumer, err error)
conf := sarama.NewConfig()
conf.Consumer.Return.Errors = true
conf.Version = kfkVersion
if connOpt.UserName != "" {
conf.Net.SASL.Enable = true
conf.Net.SASL.User = connOpt.UserName
conf.Net.SASL.Password = connOpt.Password
if err = configureKafkaSecurity(conf, connOpt); err != nil {
return
}
// 默认按随机方式消费
@@ -157,7 +161,9 @@ func RegisterKafkaMqConsumer(connOpt KafkaConfig) (client MqConsumer, err error)
// RegisterKafkaProducer 注册并启动生产者接口实现
func RegisterKafkaProducer(connOpt KafkaConfig) (client MqProducer, err error) {
mqIns := &KafkaMq{}
connOpt.ClientId = "HOTGO-Producer"
if connOpt.ClientId == "" {
connOpt.ClientId = "hotgo-producer"
}
// 这里如果使用go程需要处理chan同步问题
if err = doRegisterKafkaProducer(connOpt, mqIns); err != nil {
@@ -191,10 +197,8 @@ func doRegisterKafkaProducer(connOpt KafkaConfig, mqIns *KafkaMq) (err error) {
conf.ClientID = connOpt.ClientId
conf.Version = kfkVersion
if connOpt.UserName != "" {
conf.Net.SASL.Enable = true
conf.Net.SASL.User = connOpt.UserName
conf.Net.SASL.Password = connOpt.Password
if err = configureKafkaSecurity(conf, connOpt); err != nil {
return
}
mqIns.producerIns, err = sarama.NewAsyncProducer(brokers, conf)
@@ -209,6 +213,63 @@ func doRegisterKafkaProducer(connOpt KafkaConfig, mqIns *KafkaMq) (err error) {
return
}
func configureKafkaSecurity(conf *sarama.Config, connOpt KafkaConfig) error {
if connOpt.TLSEnabled {
conf.Net.TLS.Enable = true
conf.Net.TLS.Config = &tls.Config{MinVersion: tls.VersionTLS12}
}
if !connOpt.SASLEnabled && connOpt.UserName == "" {
return nil
}
if connOpt.UserName == "" || connOpt.Password == "" {
return gerror.New("queue kafka sasl username or password is empty")
}
conf.Net.SASL.Enable = true
conf.Net.SASL.User = connOpt.UserName
conf.Net.SASL.Password = connOpt.Password
mechanism := strings.ToUpper(strings.ReplaceAll(connOpt.SASLMechanism, "_", "-"))
switch mechanism {
case "", "PLAIN":
conf.Net.SASL.Mechanism = sarama.SASLTypePlaintext
case "SCRAM-SHA-256":
conf.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA256
conf.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
return &xdgSCRAMClient{HashGeneratorFcn: scram.SHA256}
}
case "SCRAM-SHA-512":
conf.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA512
conf.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
return &xdgSCRAMClient{HashGeneratorFcn: scram.SHA512}
}
default:
return gerror.New("queue kafka sasl mechanism is not support")
}
return nil
}
type xdgSCRAMClient struct {
*scram.Client
*scram.ClientConversation
scram.HashGeneratorFcn
}
func (x *xdgSCRAMClient) Begin(userName, password, authzID string) (err error) {
x.Client, err = x.HashGeneratorFcn.NewClient(userName, password, authzID)
if err != nil {
return err
}
x.ClientConversation = x.Client.NewConversation()
return nil
}
func (x *xdgSCRAMClient) Step(challenge string) (response string, err error) {
return x.ClientConversation.Step(challenge)
}
func (x *xdgSCRAMClient) Done() bool {
return x.ClientConversation.Done()
}
// validateVersion 验证版本是否有效
func validateVersion(version sarama.KafkaVersion) bool {
for _, item := range sarama.SupportedVersions {
server/internal/library/queue/queue.go
+51 -7
View File
@@ -11,6 +11,8 @@ import (
"github.com/gogf/gf/v2/os/gctx"
"hotgo/internal/library/queue/disk"
"hotgo/utility/charset"
"os"
"strings"
"sync"
"time"
)
@@ -59,6 +61,12 @@ type KafkaConf struct {
Version string `json:"version"`
RandClient bool `json:"randClient"`
MultiConsumer bool `json:"multiConsumer"`
ClientId string `json:"clientId"`
UserName string `json:"userName"`
Password string `json:"password"`
SASLEnabled bool `json:"saslEnabled"`
SASLMechanism string `json:"saslMechanism"`
TLSEnabled bool `json:"tlsEnabled"`
}
type MqMsg struct {
@@ -85,6 +93,7 @@ func init() {
if err := g.Cfg().MustGet(ctx, "queue").Scan(&config); err != nil {
Logger().Warningf(ctx, "queue init err:%+v", err)
}
normalizeKafkaConf()
}
// InstanceConsumer 实例化消费者
@@ -121,9 +130,15 @@ func NewProducer(groupName string) (mqClient MqProducer, err error) {
return
}
mqClient, err = RegisterKafkaProducer(KafkaConfig{
Brokers: config.Kafka.Address,
GroupID: groupName,
Version: config.Kafka.Version,
Brokers: config.Kafka.Address,
GroupID: groupName,
Version: config.Kafka.Version,
ClientId: firstKafkaClientId(config.Kafka.ClientId, "hotgo-producer"),
UserName: config.Kafka.UserName,
Password: config.Kafka.Password,
SASLEnabled: config.Kafka.SASLEnabled,
SASLMechanism: config.Kafka.SASLMechanism,
TLSEnabled: config.Kafka.TLSEnabled,
})
case "redis":
if _, err = g.Redis().Do(ctx, "ping"); err == nil {
@@ -185,10 +200,15 @@ func NewConsumer(groupName string) (mqClient MqConsumer, err error) {
}
mqClient, err = RegisterKafkaMqConsumer(KafkaConfig{
Brokers: config.Kafka.Address,
GroupID: groupName,
Version: config.Kafka.Version,
ClientId: clientId,
Brokers: config.Kafka.Address,
GroupID: groupName,
Version: config.Kafka.Version,
ClientId: clientId,
UserName: config.Kafka.UserName,
Password: config.Kafka.Password,
SASLEnabled: config.Kafka.SASLEnabled,
SASLMechanism: config.Kafka.SASLMechanism,
TLSEnabled: config.Kafka.TLSEnabled,
})
case "redis":
if _, err = g.Redis().Do(ctx, "ping"); err == nil {
@@ -213,6 +233,30 @@ func NewConsumer(groupName string) (mqClient MqConsumer, err error) {
return
}
func normalizeKafkaConf() {
config.Kafka.UserName = resolveEnvPlaceholder(config.Kafka.UserName)
config.Kafka.Password = resolveEnvPlaceholder(config.Kafka.Password)
if config.Kafka.SASLMechanism == "" {
config.Kafka.SASLMechanism = "SCRAM-SHA-512"
}
}
func resolveEnvPlaceholder(value string) string {
value = strings.TrimSpace(value)
if strings.HasPrefix(value, "${") && strings.HasSuffix(value, "}") {
return os.Getenv(strings.TrimSuffix(strings.TrimPrefix(value, "${"), "}"))
}
return value
}
func firstKafkaClientId(value string, fallback string) string {
value = strings.TrimSpace(value)
if value != "" {
return value
}
return fallback
}
// BodyString 返回消息体
func (m *MqMsg) BodyString() string {
return string(m.Body)