pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden single-machine load handling

Browse Source
This commit is contained in:
richjohnrambo
2026-09-29 17:52:13 +08:00
parent ab73b69870
commit 4b00b5ae37
24 changed files with 1030 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
loadtest/im-pressure/prepare.go
+62 -16
View File
@@ -49,7 +49,7 @@ func runPrepare(ctx context.Context, cfg *Config, manifestPath string) error {
if err = saveManifest(manifestPath, manifest); err != nil {
return err
}
if err = prepareGroup(ctx, cfg, manifest, api); err != nil {
if err = prepareGroup(ctx, cfg, manifest, api, manifestPath); err != nil {
return err
}
if err = saveManifest(manifestPath, manifest); err != nil {
@@ -184,7 +184,7 @@ func prepareFriendships(ctx context.Context, cfg *Config, manifest *Manifest, ap
return err
}
func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *apiClient) error {
func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *apiClient, manifestPath string) error {
if err := parallelAccounts(ctx, manifest.Accounts, cfg.PrepareWorkers, func(account AccountRecord) error {
return retryPrepareOperation(ctx, "privacy-"+account.Username, func() error {
return api.disableGroupInviteVerification(ctx, account)
@@ -200,6 +200,9 @@ func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *api
return err
}
manifest.GroupTopic = group.Topic
if err = saveManifest(manifestPath, manifest); err != nil {
return fmt.Errorf("checkpoint created group: %w", err)
}
log.Printf("created load-test group topic_hash=%s", topicHash(group.Topic))
}
memberIDs := make([]string, 0, len(manifest.Accounts)-1)
@@ -214,20 +217,9 @@ func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *api
if end > len(memberIDs) {
end = len(memberIDs)
}
var response *groupInvitationResponse
err := retryPrepareOperation(ctx, fmt.Sprintf("group-invite-%03d", batch), func() error {
var inviteErr error
response, inviteErr = api.inviteGroup(ctx, owner, manifest.GroupTopic, batch, memberIDs[offset:end])
return inviteErr
})
if err != nil {
if err := ensureGroupMembers(ctx, api, owner, manifest.GroupTopic, batch, memberIDs[offset:end]); err != nil {
return fmt.Errorf("invite group batch %d: %w", batch, err)
}
for _, item := range response.Items {
if item.Status != "joined" && item.Status != "already_member" {
return fmt.Errorf("invite user %s status=%s reason=%s", item.UserID, item.Status, item.Reason)
}
}
}
count, err := api.countGroupMembers(ctx, owner, manifest.GroupTopic)
if err != nil {
@@ -240,9 +232,60 @@ func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *api
return nil
}
func retryPrepareOperation(ctx context.Context, label string, operation func() error) error {
func ensureGroupMembers(ctx context.Context, api *apiClient, owner AccountRecord, topic string, batch int, desired []string) error {
var lastErr error
for attempt := 1; attempt <= 6; attempt++ {
for attempt := 1; attempt <= 12; attempt++ {
members, err := api.groupMemberIDs(ctx, owner, topic)
if err != nil {
lastErr = fmt.Errorf("list current members: %w", err)
} else {
missing := make([]string, 0, len(desired))
for _, userID := range desired {
if !members[userID] {
missing = append(missing, userID)
}
}
if len(missing) == 0 {
return nil
}
response, inviteErr := api.inviteGroup(ctx, owner, topic, batch, missing)
if inviteErr != nil {
lastErr = inviteErr
} else {
lastErr = nil
for _, item := range response.Items {
if item.Status != "joined" && item.Status != "already_member" {
lastErr = fmt.Errorf("invite user %s status=%s reason=%s", item.UserID, item.Status, item.Reason)
break
}
}
if lastErr == nil {
continue
}
}
}
if ctx.Err() != nil {
return ctx.Err()
}
if attempt == 12 {
break
}
delay := time.Duration(attempt*5) * time.Second
log.Printf("operation=group-invite-%03d attempt=%d failed; retry_in=%s: %v", batch, attempt, delay, lastErr)
if err := waitContext(ctx, delay); err != nil {
return err
}
}
return fmt.Errorf("missing members not recovered after retries: %w", lastErr)
}
func retryPrepareOperation(ctx context.Context, label string, operation func() error) error {
return retryOperation(ctx, label, 6, operation)
}
func retryOperation(ctx context.Context, label string, attempts int, operation func() error) error {
var lastErr error
for attempt := 1; attempt <= attempts; attempt++ {
if err := operation(); err == nil {
return nil
} else {
@@ -251,6 +294,9 @@ func retryPrepareOperation(ctx context.Context, label string, operation func() e
if ctx.Err() != nil {
return ctx.Err()
}
if attempt == attempts {
break
}
delay := time.Duration(attempt*5) * time.Second
log.Printf("operation=%s attempt=%d failed; retry_in=%s: %v", label, attempt, delay, lastErr)
if err := waitContext(ctx, delay); err != nil {