- MongoDB storage: cwmp_exchanges collection with indexes (deviceId+timestamp, deviceId+sequenceNumber unique, TTL 24h) - ExchangeStoreService: CWMP method extraction, per-device queue for ordering, retry logic, message cap (5000/device) - Exchange API: GET/DELETE/SSE stream endpoints with Last-Event-ID reconnection - CwmpLoggerMiddleware: dual-write (MongoDB + legacy files) with fire-and-forget - Frontend: SequenceViewer (CPE↔ACS lifelines, arrows, auto-scroll, virtual scroll) + InspectorPanel (formatted XML, copy, syntax highlighting) - Tr069Trace page: integrated new viewer with col-6/col-6 layout - DeviceDetail page: collapsible exchange section when device is traced - 14 property-based tests, 40+ unit tests, 5 integration tests - Health check includes index verification - Config: feature flags, SSE timeouts, retry policy
223 lines
6.1 KiB
TypeScript
223 lines
6.1 KiB
TypeScript
import logger from '@adonisjs/core/services/logger'
|
|
|
|
/**
|
|
* Tracks percentile-based timing metrics for the TR-069 Exchange Viewer subsystem.
|
|
*
|
|
* Collects:
|
|
* - MongoDB write latency (p50, p95, p99)
|
|
* - SSE push latency (persist → client receive)
|
|
* - API response times by endpoint
|
|
*
|
|
* Exposes metrics via structured logs (no Prometheus setup required).
|
|
* Periodically logs aggregated stats every `flushIntervalMs` milliseconds.
|
|
*/
|
|
|
|
export interface MetricsSummary {
|
|
count: number
|
|
p50: number
|
|
p95: number
|
|
p99: number
|
|
min: number
|
|
max: number
|
|
avg: number
|
|
}
|
|
|
|
export interface AggregatedMetrics {
|
|
mongoWriteLatency: MetricsSummary | null
|
|
ssePushLatency: MetricsSummary | null
|
|
apiResponseTimes: Record<string, MetricsSummary>
|
|
collectedAt: string
|
|
windowMs: number
|
|
}
|
|
|
|
const DEFAULT_FLUSH_INTERVAL_MS = 60_000 // 1 minute
|
|
const DEFAULT_MAX_SAMPLES = 10_000 // cap memory usage
|
|
|
|
export class ExchangeMetricsService {
|
|
private static instance: ExchangeMetricsService
|
|
|
|
private mongoWriteSamples: number[] = []
|
|
private ssePushSamples: number[] = []
|
|
private apiResponseSamples: Map<string, number[]> = new Map()
|
|
|
|
private flushIntervalMs: number
|
|
private maxSamples: number
|
|
private flushTimer: ReturnType<typeof setInterval> | null = null
|
|
private lastFlushAt: number = Date.now()
|
|
|
|
private constructor(
|
|
flushIntervalMs: number = DEFAULT_FLUSH_INTERVAL_MS,
|
|
maxSamples: number = DEFAULT_MAX_SAMPLES
|
|
) {
|
|
this.flushIntervalMs = flushIntervalMs
|
|
this.maxSamples = maxSamples
|
|
}
|
|
|
|
static getInstance(
|
|
flushIntervalMs?: number,
|
|
maxSamples?: number
|
|
): ExchangeMetricsService {
|
|
if (!ExchangeMetricsService.instance) {
|
|
ExchangeMetricsService.instance = new ExchangeMetricsService(flushIntervalMs, maxSamples)
|
|
}
|
|
return ExchangeMetricsService.instance
|
|
}
|
|
|
|
/**
|
|
* Reset singleton (for testing only).
|
|
*/
|
|
static resetInstance(): void {
|
|
if (ExchangeMetricsService.instance) {
|
|
ExchangeMetricsService.instance.stop()
|
|
}
|
|
ExchangeMetricsService.instance = undefined as any
|
|
}
|
|
|
|
/**
|
|
* Start the periodic flush timer.
|
|
* Safe to call multiple times (idempotent).
|
|
*/
|
|
start(): void {
|
|
if (this.flushTimer) return
|
|
this.lastFlushAt = Date.now()
|
|
this.flushTimer = setInterval(() => this.flush(), this.flushIntervalMs)
|
|
// Prevent the timer from keeping the process alive
|
|
if (this.flushTimer.unref) {
|
|
this.flushTimer.unref()
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Stop the periodic flush timer and perform a final flush.
|
|
*/
|
|
stop(): void {
|
|
if (this.flushTimer) {
|
|
clearInterval(this.flushTimer)
|
|
this.flushTimer = null
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Record a MongoDB write latency sample (in milliseconds).
|
|
*/
|
|
recordMongoWriteLatency(durationMs: number): void {
|
|
this.addSample(this.mongoWriteSamples, durationMs)
|
|
}
|
|
|
|
/**
|
|
* Record an SSE push latency sample (in milliseconds).
|
|
* This is the time from persist completion to the client receiving the event.
|
|
*/
|
|
recordSsePushLatency(durationMs: number): void {
|
|
this.addSample(this.ssePushSamples, durationMs)
|
|
}
|
|
|
|
/**
|
|
* Record an API response time sample (in milliseconds) for a specific endpoint.
|
|
* @param endpoint - e.g. "GET /exchanges", "DELETE /exchanges", "GET /exchanges/stream"
|
|
*/
|
|
recordApiResponseTime(endpoint: string, durationMs: number): void {
|
|
if (!this.apiResponseSamples.has(endpoint)) {
|
|
this.apiResponseSamples.set(endpoint, [])
|
|
}
|
|
this.addSample(this.apiResponseSamples.get(endpoint)!, durationMs)
|
|
}
|
|
|
|
/**
|
|
* Compute aggregated metrics without clearing samples.
|
|
* Useful for health checks or on-demand inspection.
|
|
*/
|
|
getAggregatedMetrics(): AggregatedMetrics {
|
|
const now = Date.now()
|
|
const windowMs = now - this.lastFlushAt
|
|
|
|
const apiResponseTimes: Record<string, MetricsSummary> = {}
|
|
for (const [endpoint, samples] of this.apiResponseSamples) {
|
|
const summary = this.computeSummary(samples)
|
|
if (summary) {
|
|
apiResponseTimes[endpoint] = summary
|
|
}
|
|
}
|
|
|
|
return {
|
|
mongoWriteLatency: this.computeSummary(this.mongoWriteSamples),
|
|
ssePushLatency: this.computeSummary(this.ssePushSamples),
|
|
apiResponseTimes,
|
|
collectedAt: new Date(now).toISOString(),
|
|
windowMs,
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Flush aggregated metrics to structured logs and reset samples.
|
|
*/
|
|
flush(): AggregatedMetrics {
|
|
const metrics = this.getAggregatedMetrics()
|
|
|
|
// Only log if there are any samples
|
|
const hasData =
|
|
metrics.mongoWriteLatency !== null ||
|
|
metrics.ssePushLatency !== null ||
|
|
Object.keys(metrics.apiResponseTimes).length > 0
|
|
|
|
if (hasData) {
|
|
logger.info(
|
|
{ metrics },
|
|
'[ExchangeMetrics] Aggregated performance metrics'
|
|
)
|
|
}
|
|
|
|
// Reset samples
|
|
this.mongoWriteSamples = []
|
|
this.ssePushSamples = []
|
|
this.apiResponseSamples.clear()
|
|
this.lastFlushAt = Date.now()
|
|
|
|
return metrics
|
|
}
|
|
|
|
/**
|
|
* Compute percentile summary for a set of samples.
|
|
* Returns null if no samples are available.
|
|
*/
|
|
computeSummary(samples: number[]): MetricsSummary | null {
|
|
if (samples.length === 0) return null
|
|
|
|
const sorted = [...samples].sort((a, b) => a - b)
|
|
const count = sorted.length
|
|
const sum = sorted.reduce((acc, v) => acc + v, 0)
|
|
|
|
return {
|
|
count,
|
|
p50: this.percentile(sorted, 50),
|
|
p95: this.percentile(sorted, 95),
|
|
p99: this.percentile(sorted, 99),
|
|
min: sorted[0],
|
|
max: sorted[count - 1],
|
|
avg: Math.round((sum / count) * 100) / 100,
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Calculate percentile value from a sorted array using nearest-rank method.
|
|
*/
|
|
private percentile(sorted: number[], p: number): number {
|
|
if (sorted.length === 0) return 0
|
|
if (sorted.length === 1) return sorted[0]
|
|
|
|
const index = Math.ceil((p / 100) * sorted.length) - 1
|
|
return sorted[Math.max(0, Math.min(index, sorted.length - 1))]
|
|
}
|
|
|
|
/**
|
|
* Add a sample to a samples array, respecting the max capacity.
|
|
* When at capacity, the oldest sample is dropped (FIFO).
|
|
*/
|
|
private addSample(samples: number[], value: number): void {
|
|
if (samples.length >= this.maxSamples) {
|
|
samples.shift()
|
|
}
|
|
samples.push(value)
|
|
}
|
|
}
|