package main import ( "encoding/json" "fmt" "os" "path/filepath" "sort" "time" ) type CombinedReport struct { RunID string `json:"run_id"` Scenario string `json:"scenario"` GeneratedAt time.Time `json:"generated_at"` Agents []AgentSummary `json:"agents"` ExpectedConnections int `json:"expected_connections"` ReadyConnections int `json:"ready_connections"` PublishAttempted int64 `json:"publish_attempted"` PublishAccepted int64 `json:"publish_accepted"` PublishRejected int64 `json:"publish_rejected"` PublishExplicitReject int64 `json:"publish_explicitly_rejected"` PublishOutcomeUnknown int64 `json:"publish_outcome_unknown"` PublishErrors map[string]int64 `json:"publish_errors,omitempty"` PublishSuccessRate float64 `json:"publish_success_rate"` AckP95MS float64 `json:"ack_p95_ms"` DeliveryP95MS float64 `json:"delivery_p95_ms"` DeliveryLocalP95MS float64 `json:"delivery_local_p95_ms"` DeliveryCrossP95MS float64 `json:"delivery_cross_machine_p95_ms"` CrossClockSkewSamples int64 `json:"cross_machine_negative_latency_samples"` ExpectedDeliveries int64 `json:"expected_deliveries"` ReceivedDeliveries int64 `json:"received_deliveries"` RealtimeLost int64 `json:"realtime_lost"` RealtimeLostStored int64 `json:"realtime_lost_but_persisted"` ArrivalRate float64 `json:"arrival_rate"` LossRate float64 `json:"loss_rate"` Duplicates int64 `json:"duplicates"` OutOfOrder int64 `json:"out_of_order"` OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"` Disconnects int64 `json:"disconnects"` DisconnectReasons map[string]int64 `json:"disconnect_reasons,omitempty"` DisconnectSamples []DisconnectSample `json:"disconnect_samples,omitempty"` UnexpectedMessages int64 `json:"unexpected_messages"` HistoryChecked int64 `json:"history_checked"` HistoryFound int64 `json:"history_found"` HistoryUnackedChecked int64 `json:"history_unacked_observed_checked"` HistoryUnackedFound int64 `json:"history_unacked_observed_found"` PersistenceLost int64 `json:"persistence_lost"` Missing []MissingDelivery `json:"missing,omitempty"` Passed bool `json:"passed"` FailureReasons []string `json:"failure_reasons,omitempty"` } func runMerge(aPath, bPath, output string) error { a, err := readAgentSummary(aPath) if err != nil { return err } b, err := readAgentSummary(bPath) if err != nil { return err } if a.RunID != b.RunID || a.Scenario != b.Scenario { return fmt.Errorf("agent summaries do not belong to the same run") } report := CombinedReport{RunID: a.RunID, Scenario: a.Scenario, GeneratedAt: time.Now().UTC(), Agents: []AgentSummary{a, b}, Passed: true, PublishErrors: make(map[string]int64), DisconnectReasons: make(map[string]int64)} for _, summary := range report.Agents { expectedConnections := summary.ExpectedConnections if expectedConnections == 0 { expectedConnections = 1000 } report.ExpectedConnections += expectedConnections report.ReadyConnections += summary.ReadyConnections report.PublishAttempted += summary.PublishAttempted report.PublishAccepted += summary.PublishAccepted for category, count := range summary.PublishErrors { report.PublishErrors[category] += count } unknown := summary.PublishOutcomeUnknown if unknown == 0 { unknown = summary.PublishErrors["ctrl_timeout"] + summary.PublishErrors["socket"] } explicit := summary.PublishErrors["403"] + summary.PublishErrors["502"] + summary.PublishErrors["503"] + summary.PublishErrors["other"] report.PublishOutcomeUnknown += unknown report.PublishExplicitReject += explicit report.PublishRejected += explicit report.ExpectedDeliveries += summary.ExpectedPeer + summary.ExpectedGroup report.ReceivedDeliveries += summary.ReceivedPeer + summary.ReceivedGroup report.RealtimeLost += summary.RealtimeLost report.RealtimeLostStored += summary.RealtimeLostStored report.Duplicates += summary.Duplicates report.OutOfOrder += summary.OutOfOrder report.OrderingViolations = append(report.OrderingViolations, summary.OrderingViolations...) report.Disconnects += summary.Disconnects for category, count := range summary.DisconnectReasons { report.DisconnectReasons[category] += count } if remaining := 100 - len(report.DisconnectSamples); remaining > 0 { if remaining > len(summary.DisconnectSamples) { remaining = len(summary.DisconnectSamples) } report.DisconnectSamples = append(report.DisconnectSamples, summary.DisconnectSamples[:remaining]...) } report.UnexpectedMessages += summary.UnexpectedMessages report.HistoryChecked += summary.HistoryChecked report.HistoryFound += summary.HistoryFound report.HistoryUnackedChecked += summary.HistoryUnackedChecked report.HistoryUnackedFound += summary.HistoryUnackedFound report.PersistenceLost += summary.PersistenceLost report.CrossClockSkewSamples += summary.CrossClockSkewSamples report.Missing = append(report.Missing, summary.Missing...) if len(summary.Errors) > 0 { report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("agent %s errors=%d", summary.Machine, len(summary.Errors))) } } if report.PublishAttempted > 0 { report.PublishSuccessRate = float64(report.PublishAccepted) / float64(report.PublishAttempted) } report.AckP95MS = mergedAckP95(report.Agents) report.DeliveryLocalP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatencyLocal }) report.DeliveryCrossP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatencyCross }) // delivery_p95_ms remains the compatibility field, but now uses only same-clock // samples. Cross-machine wall clocks cannot produce a valid one-way latency. report.DeliveryP95MS = report.DeliveryLocalP95MS if report.DeliveryP95MS == 0 { report.DeliveryP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatency }) } if report.ExpectedDeliveries > 0 { report.ArrivalRate = float64(report.ReceivedDeliveries) / float64(report.ExpectedDeliveries) report.LossRate = float64(report.RealtimeLost) / float64(report.ExpectedDeliveries) } if report.ReadyConnections != report.ExpectedConnections { report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("ready connections=%d expected=%d", report.ReadyConnections, report.ExpectedConnections)) } if report.PublishAttempted > 0 && report.PublishSuccessRate < 0.9999 { report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("publish success rate=%.6f expected>=0.999900", report.PublishSuccessRate)) } if report.PublishOutcomeUnknown > 0 { report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("publish outcome unknown=%d (ACK timeout or socket failure)", report.PublishOutcomeUnknown)) } if report.UnexpectedMessages > 0 { report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("deliveries observed without ACK ledger=%d", report.UnexpectedMessages)) } if report.AckP95MS > 500 { report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("ack p95=%.3fms expected<=500ms", report.AckP95MS)) } if report.DeliveryP95MS > 1000 { report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("delivery p95=%.0fms expected<=1000ms", report.DeliveryP95MS)) } if report.PublishRejected > 0 || report.RealtimeLost > 0 || report.PersistenceLost > 0 || report.Duplicates > 0 || report.OutOfOrder > 0 || report.Disconnects > 0 || report.UnexpectedMessages > 0 { report.FailureReasons = append(report.FailureReasons, "message correctness gate failed") } report.Passed = len(report.FailureReasons) == 0 sort.Slice(report.Missing, func(i, j int) bool { if report.Missing[i].ClientMid == report.Missing[j].ClientMid { return report.Missing[i].ReceiverIndex < report.Missing[j].ReceiverIndex } return report.Missing[i].ClientMid < report.Missing[j].ClientMid }) return writeJSONFile(output, report) } func mergedAckP95(agents []AgentSummary) float64 { var expected int64 latencies := make([]float64, 0) for _, agent := range agents { expected += agent.AckLatency.Count for _, message := range agent.AcceptedLocal { if message.SentAt.IsZero() || message.AckAt.Before(message.SentAt) { continue } latencies = append(latencies, float64(message.AckAt.Sub(message.SentAt))/float64(time.Millisecond)) } } if expected == 0 || int64(len(latencies)) != expected { return mergedHistogramP95(agents, func(summary AgentSummary) HistogramSnapshot { return summary.AckLatency }) } sort.Float64s(latencies) index := (len(latencies)*95 + 99) / 100 return latencies[index-1] } func mergedHistogramP95(agents []AgentSummary, selectHistogram func(AgentSummary) HistogramSnapshot) float64 { var count int64 buckets := make([]int64, len(latencyBounds)+1) maxMS := float64(0) for _, agent := range agents { histogram := selectHistogram(agent) count += histogram.Count if histogram.MaxMS > maxMS { maxMS = histogram.MaxMS } for i, bound := range latencyBounds { buckets[i] += histogram.Buckets[bound.String()] } buckets[len(latencyBounds)] += histogram.Buckets["+Inf"] } if count == 0 { return 0 } target := (count*95 + 99) / 100 var cumulative int64 for i, bucket := range buckets { cumulative += bucket if cumulative >= target { if i == len(latencyBounds) { return maxMS } return float64(latencyBounds[i]) / float64(time.Millisecond) } } return maxMS } func readAgentSummary(path string) (AgentSummary, error) { var summary AgentSummary raw, err := os.ReadFile(path) if err != nil { return summary, err } if err = json.Unmarshal(raw, &summary); err != nil { return summary, err } return summary, nil } func writeJSONFile(path string, value any) error { raw, err := json.MarshalIndent(value, "", " ") if err != nil { return err } if dir := filepath.Dir(path); dir != "." { if err = os.MkdirAll(dir, 0o750); err != nil { return err } } return os.WriteFile(path, append(raw, '\n'), 0o600) }