280 lines
12 KiB
Go
280 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
type CombinedReport struct {
|
|
RunID string `json:"run_id"`
|
|
Scenario string `json:"scenario"`
|
|
GeneratedAt time.Time `json:"generated_at"`
|
|
Agents []AgentSummary `json:"agents"`
|
|
ExpectedConnections int `json:"expected_connections"`
|
|
ReadyConnections int `json:"ready_connections"`
|
|
PublishAttempted int64 `json:"publish_attempted"`
|
|
PublishAccepted int64 `json:"publish_accepted"`
|
|
PublishLateAccepted int64 `json:"publish_late_accepted"`
|
|
PublishFinalAccepted int64 `json:"publish_final_accepted"`
|
|
PublishRejected int64 `json:"publish_rejected"`
|
|
PublishExplicitReject int64 `json:"publish_explicitly_rejected"`
|
|
PublishOutcomeUnknown int64 `json:"publish_outcome_unknown"`
|
|
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
|
|
PublishCtrlObserved int64 `json:"publish_ctrl_observed"`
|
|
PublishCtrlOver20 int64 `json:"publish_ctrl_over_20s"`
|
|
PublishLateCtrl int64 `json:"publish_ctrl_late"`
|
|
PublishCtrlUnresolved int64 `json:"publish_ctrl_unresolved"`
|
|
PublishCtrlP95MS float64 `json:"publish_ctrl_p95_ms"`
|
|
CtrlObservationSamples []CtrlObservationSample `json:"ctrl_observation_samples,omitempty"`
|
|
PublishSuccessRate float64 `json:"publish_success_rate"`
|
|
AckP95MS float64 `json:"ack_p95_ms"`
|
|
DeliveryP95MS float64 `json:"delivery_p95_ms"`
|
|
DeliveryLocalP95MS float64 `json:"delivery_local_p95_ms"`
|
|
DeliveryCrossP95MS float64 `json:"delivery_cross_machine_p95_ms"`
|
|
CrossClockSkewSamples int64 `json:"cross_machine_negative_latency_samples"`
|
|
ExpectedDeliveries int64 `json:"expected_deliveries"`
|
|
ReceivedDeliveries int64 `json:"received_deliveries"`
|
|
RealtimeLost int64 `json:"realtime_lost"`
|
|
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
|
|
ArrivalRate float64 `json:"arrival_rate"`
|
|
LossRate float64 `json:"loss_rate"`
|
|
Duplicates int64 `json:"duplicates"`
|
|
OutOfOrder int64 `json:"out_of_order"`
|
|
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
|
|
Disconnects int64 `json:"disconnects"`
|
|
DisconnectReasons map[string]int64 `json:"disconnect_reasons,omitempty"`
|
|
DisconnectSamples []DisconnectSample `json:"disconnect_samples,omitempty"`
|
|
UnexpectedMessages int64 `json:"unexpected_messages"`
|
|
HistoryChecked int64 `json:"history_checked"`
|
|
HistoryFound int64 `json:"history_found"`
|
|
HistoryUnackedChecked int64 `json:"history_unacked_observed_checked"`
|
|
HistoryUnackedFound int64 `json:"history_unacked_observed_found"`
|
|
PersistenceLost int64 `json:"persistence_lost"`
|
|
Missing []MissingDelivery `json:"missing,omitempty"`
|
|
Passed bool `json:"passed"`
|
|
FailureReasons []string `json:"failure_reasons,omitempty"`
|
|
}
|
|
|
|
func runMerge(aPath, bPath, output string) error {
|
|
a, err := readAgentSummary(aPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
b, err := readAgentSummary(bPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if a.RunID != b.RunID || a.Scenario != b.Scenario {
|
|
return fmt.Errorf("agent summaries do not belong to the same run")
|
|
}
|
|
report := CombinedReport{RunID: a.RunID, Scenario: a.Scenario, GeneratedAt: time.Now().UTC(), Agents: []AgentSummary{a, b}, Passed: true,
|
|
PublishErrors: make(map[string]int64), DisconnectReasons: make(map[string]int64)}
|
|
for _, summary := range report.Agents {
|
|
expectedConnections := summary.ExpectedConnections
|
|
if expectedConnections == 0 {
|
|
expectedConnections = 1000
|
|
}
|
|
report.ExpectedConnections += expectedConnections
|
|
report.ReadyConnections += summary.ReadyConnections
|
|
report.PublishAttempted += summary.PublishAttempted
|
|
report.PublishAccepted += summary.PublishAccepted
|
|
lateAccepted := summary.PublishLateAccepted
|
|
// Backfill summaries generated before publish_ctrl_late_accepted was added.
|
|
if summary.CtrlTrackingEnabled && lateAccepted == 0 {
|
|
for _, sample := range summary.CtrlObservationSamples {
|
|
if sample.Late && sample.Code >= 200 && sample.Code < 300 {
|
|
lateAccepted++
|
|
}
|
|
}
|
|
}
|
|
report.PublishLateAccepted += lateAccepted
|
|
report.PublishFinalAccepted += summary.PublishAccepted + lateAccepted
|
|
for category, count := range summary.PublishErrors {
|
|
report.PublishErrors[category] += count
|
|
}
|
|
unknown := summary.PublishOutcomeUnknown
|
|
if summary.CtrlTrackingEnabled {
|
|
unknown = summary.PublishCtrlUnresolved + summary.PublishErrors["socket"]
|
|
} else if unknown == 0 {
|
|
unknown = summary.PublishErrors["ctrl_timeout"] + summary.PublishErrors["socket"]
|
|
}
|
|
explicit := summary.PublishErrors["403"] + summary.PublishErrors["502"] +
|
|
summary.PublishErrors["503"] + summary.PublishErrors["other"]
|
|
report.PublishOutcomeUnknown += unknown
|
|
report.PublishCtrlObserved += summary.PublishCtrlObserved
|
|
report.PublishCtrlOver20 += summary.PublishCtrlOver20
|
|
report.PublishLateCtrl += summary.PublishLateCtrl
|
|
report.PublishCtrlUnresolved += summary.PublishCtrlUnresolved
|
|
if remaining := 1000 - len(report.CtrlObservationSamples); remaining > 0 {
|
|
if remaining > len(summary.CtrlObservationSamples) {
|
|
remaining = len(summary.CtrlObservationSamples)
|
|
}
|
|
report.CtrlObservationSamples = append(report.CtrlObservationSamples, summary.CtrlObservationSamples[:remaining]...)
|
|
}
|
|
report.PublishExplicitReject += explicit
|
|
report.PublishRejected += explicit
|
|
report.ExpectedDeliveries += summary.ExpectedPeer + summary.ExpectedGroup
|
|
report.ReceivedDeliveries += summary.ReceivedPeer + summary.ReceivedGroup
|
|
report.RealtimeLost += summary.RealtimeLost
|
|
report.RealtimeLostStored += summary.RealtimeLostStored
|
|
report.Duplicates += summary.Duplicates
|
|
report.OutOfOrder += summary.OutOfOrder
|
|
report.OrderingViolations = append(report.OrderingViolations, summary.OrderingViolations...)
|
|
report.Disconnects += summary.Disconnects
|
|
for category, count := range summary.DisconnectReasons {
|
|
report.DisconnectReasons[category] += count
|
|
}
|
|
if remaining := 100 - len(report.DisconnectSamples); remaining > 0 {
|
|
if remaining > len(summary.DisconnectSamples) {
|
|
remaining = len(summary.DisconnectSamples)
|
|
}
|
|
report.DisconnectSamples = append(report.DisconnectSamples, summary.DisconnectSamples[:remaining]...)
|
|
}
|
|
report.UnexpectedMessages += summary.UnexpectedMessages
|
|
report.HistoryChecked += summary.HistoryChecked
|
|
report.HistoryFound += summary.HistoryFound
|
|
report.HistoryUnackedChecked += summary.HistoryUnackedChecked
|
|
report.HistoryUnackedFound += summary.HistoryUnackedFound
|
|
report.PersistenceLost += summary.PersistenceLost
|
|
report.CrossClockSkewSamples += summary.CrossClockSkewSamples
|
|
report.Missing = append(report.Missing, summary.Missing...)
|
|
actionableErrors := len(summary.Errors)
|
|
lateTimeouts := int(lateAccepted)
|
|
for _, message := range summary.Errors {
|
|
if lateTimeouts > 0 && strings.Contains(message, "ctrl timeout") {
|
|
actionableErrors--
|
|
lateTimeouts--
|
|
}
|
|
}
|
|
if actionableErrors > 0 {
|
|
report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("agent %s unresolved errors=%d", summary.Machine, actionableErrors))
|
|
}
|
|
}
|
|
if report.PublishAttempted > 0 {
|
|
report.PublishSuccessRate = float64(report.PublishFinalAccepted) / float64(report.PublishAttempted)
|
|
}
|
|
report.AckP95MS = mergedAckP95(report.Agents)
|
|
report.PublishCtrlP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.PublishCtrlLatency })
|
|
report.DeliveryLocalP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatencyLocal })
|
|
report.DeliveryCrossP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatencyCross })
|
|
// delivery_p95_ms remains the compatibility field, but now uses only same-clock
|
|
// samples. Cross-machine wall clocks cannot produce a valid one-way latency.
|
|
report.DeliveryP95MS = report.DeliveryLocalP95MS
|
|
if report.DeliveryP95MS == 0 {
|
|
report.DeliveryP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatency })
|
|
}
|
|
if report.ExpectedDeliveries > 0 {
|
|
report.ArrivalRate = float64(report.ReceivedDeliveries) / float64(report.ExpectedDeliveries)
|
|
report.LossRate = float64(report.RealtimeLost) / float64(report.ExpectedDeliveries)
|
|
}
|
|
if report.ReadyConnections != report.ExpectedConnections {
|
|
report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("ready connections=%d expected=%d",
|
|
report.ReadyConnections, report.ExpectedConnections))
|
|
}
|
|
if report.PublishAttempted > 0 && report.PublishSuccessRate < 0.9999 {
|
|
report.FailureReasons = append(report.FailureReasons,
|
|
fmt.Sprintf("publish success rate=%.6f expected>=0.999900", report.PublishSuccessRate))
|
|
}
|
|
if report.PublishOutcomeUnknown > 0 {
|
|
report.FailureReasons = append(report.FailureReasons,
|
|
fmt.Sprintf("publish outcome unknown=%d (ACK timeout or socket failure)", report.PublishOutcomeUnknown))
|
|
}
|
|
if report.UnexpectedMessages > 0 {
|
|
report.FailureReasons = append(report.FailureReasons,
|
|
fmt.Sprintf("deliveries observed without ACK ledger=%d", report.UnexpectedMessages))
|
|
}
|
|
if report.PublishRejected > 0 || report.RealtimeLost > 0 || report.PersistenceLost > 0 || report.Duplicates > 0 || report.OutOfOrder > 0 || report.Disconnects > 0 || report.UnexpectedMessages > 0 {
|
|
report.FailureReasons = append(report.FailureReasons, "message correctness gate failed")
|
|
}
|
|
report.Passed = len(report.FailureReasons) == 0
|
|
sort.Slice(report.Missing, func(i, j int) bool {
|
|
if report.Missing[i].ClientMid == report.Missing[j].ClientMid {
|
|
return report.Missing[i].ReceiverIndex < report.Missing[j].ReceiverIndex
|
|
}
|
|
return report.Missing[i].ClientMid < report.Missing[j].ClientMid
|
|
})
|
|
return writeJSONFile(output, report)
|
|
}
|
|
|
|
func mergedAckP95(agents []AgentSummary) float64 {
|
|
var expected int64
|
|
latencies := make([]float64, 0)
|
|
for _, agent := range agents {
|
|
expected += agent.AckLatency.Count
|
|
for _, message := range agent.AcceptedLocal {
|
|
if message.SentAt.IsZero() || message.AckAt.Before(message.SentAt) {
|
|
continue
|
|
}
|
|
latencies = append(latencies, float64(message.AckAt.Sub(message.SentAt))/float64(time.Millisecond))
|
|
}
|
|
}
|
|
if expected == 0 || int64(len(latencies)) != expected {
|
|
return mergedHistogramP95(agents, func(summary AgentSummary) HistogramSnapshot { return summary.AckLatency })
|
|
}
|
|
sort.Float64s(latencies)
|
|
index := (len(latencies)*95 + 99) / 100
|
|
return latencies[index-1]
|
|
}
|
|
|
|
func mergedHistogramP95(agents []AgentSummary, selectHistogram func(AgentSummary) HistogramSnapshot) float64 {
|
|
var count int64
|
|
buckets := make([]int64, len(latencyBounds)+1)
|
|
maxMS := float64(0)
|
|
for _, agent := range agents {
|
|
histogram := selectHistogram(agent)
|
|
count += histogram.Count
|
|
if histogram.MaxMS > maxMS {
|
|
maxMS = histogram.MaxMS
|
|
}
|
|
for i, bound := range latencyBounds {
|
|
buckets[i] += histogram.Buckets[bound.String()]
|
|
}
|
|
buckets[len(latencyBounds)] += histogram.Buckets["+Inf"]
|
|
}
|
|
if count == 0 {
|
|
return 0
|
|
}
|
|
target := (count*95 + 99) / 100
|
|
var cumulative int64
|
|
for i, bucket := range buckets {
|
|
cumulative += bucket
|
|
if cumulative >= target {
|
|
if i == len(latencyBounds) {
|
|
return maxMS
|
|
}
|
|
return float64(latencyBounds[i]) / float64(time.Millisecond)
|
|
}
|
|
}
|
|
return maxMS
|
|
}
|
|
|
|
func readAgentSummary(path string) (AgentSummary, error) {
|
|
var summary AgentSummary
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return summary, err
|
|
}
|
|
if err = json.Unmarshal(raw, &summary); err != nil {
|
|
return summary, err
|
|
}
|
|
return summary, nil
|
|
}
|
|
|
|
func writeJSONFile(path string, value any) error {
|
|
raw, err := json.MarshalIndent(value, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if dir := filepath.Dir(path); dir != "." {
|
|
if err = os.MkdirAll(dir, 0o750); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return os.WriteFile(path, append(raw, '\n'), 0o600)
|
|
}
|