Files
adam-acs/app/services/cwmp/state_machine_manager.ts
T
Kevin L'HereecandContinue 08c620ee68 fix: drain task queue on session completion to stop orphan CWMP requests
After OperationalState, the state machine was marked completed but leftover
tasks in the CWMPTaskQueue (queued by ConfigurationEvaluationState callbacks)
were being served to the CPE on subsequent empty POSTs via the continue()
task fallback path, causing spurious GetParameterNames + SetParameterValues
exchanges after the session officially ended.

Fix: call taskQueue.clearPendingTasksForDevice() in completeMachine() so no
tasks bleed into the next session.

Generated with [Continue](https://continue.dev)

Co-Authored-By: Continue <noreply@continue.dev>
2026-06-08 14:20:59 +00:00

468 lines
17 KiB
TypeScript

import { EventEmitter } from 'node:events'
import type { Device } from '../../domain/device/device.js'
import type { DeviceId } from '../../domain/device/device_id.js'
import { CWMPTaskQueue, ResultStatus } from './task_queue.js'
import { MainStateMachine } from './main_state_machine.js'
import { ImmediateCommandStateMachine } from './immediate_command_state_machine.js'
import { getDeviceHashIdentifier } from '../../utils/device_utils.js'
import { resolveHighestPriorityEvent, normalizeEventCode } from '../../utils/cwmp-events.js'
import { CommandStatus } from '../../domain/commands/command.js'
import { ModeType } from '../../domain/shared/mode.js'
import CommandModel from '../../models/command.js'
import logger from '@adonisjs/core/services/logger'
export enum MachineStatus {
CREATED = 'created',
RUNNING = 'running',
COMPLETED = 'completed',
FAILED = 'failed',
}
export interface StateMachineInstance {
id: string
deviceId: DeviceId
session: string
stateMachine: MainStateMachine | ImmediateCommandStateMachine
status: MachineStatus
createdAt: Date
completedAt?: Date
currentState: string
lastTaskId?: string
events?: string[]
}
/**
* Central state machine manager with TCP-socket-based session tracking,
* device preemption, and CWMP task queue integration.
*/
export class StateMachineManager extends EventEmitter {
private taskQueue: CWMPTaskQueue
private machines: Map<string, StateMachineInstance> = new Map()
private deviceSessions: Map<string, string> = new Map() // deviceKey -> sessionId
private sessionMachines: Map<string, string> = new Map() // sessionId -> machineId
private remoteToDevice: Map<string, string> = new Map() // remoteKey -> deviceKey
private cleanupInterval: ReturnType<typeof setInterval> | null = null
private static instance: StateMachineManager | null = null
constructor() {
super()
this.taskQueue = new CWMPTaskQueue()
}
static getInstance(): StateMachineManager {
if (!StateMachineManager.instance) {
StateMachineManager.instance = new StateMachineManager()
}
return StateMachineManager.instance
}
getTaskQueue(): CWMPTaskQueue {
return this.taskQueue
}
startCleanupTimer(intervalMs: number = 60000): void {
if (this.cleanupInterval) return
this.cleanupInterval = setInterval(() => this.cleanupCompletedMachines(), intervalMs)
}
stopCleanupTimer(): void {
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval)
this.cleanupInterval = null
}
}
/**
* Start a state machine for a device session
*/
async startStateMachine(
device: Device,
session: string,
opts?: { events?: string[] }
): Promise<StateMachineInstance> {
const deviceKey = JSON.stringify(device.deviceId)
const evts = opts?.events || []
const primaryEvent = resolveHighestPriorityEvent(evts)
const normalizedEvents = evts.map((e) => normalizeEventCode(e))
// Check for NI commands that force MainStateMachine
let forceMainForNI = false
try {
const hashId = getDeviceHashIdentifier({
manufacturer: device.deviceId.manufacturer,
oui: device.deviceId.oui,
productClass: device.deviceId.productClass,
serialNumber: device.deviceId.serialNumber,
})
const collection = CommandModel.getCollection()
const niCount = await collection.countDocuments({
deviceIdentifier: hashId,
mode: ModeType.NI,
status: { $in: [CommandStatus.PENDING, CommandStatus.WAITING] },
})
forceMainForNI = niCount > 0
} catch (e) {
logger.warn(`Failed to check NI commands: ${e}`)
}
// Handle preemption of existing machine
const existingSessionId = this.deviceSessions.get(deviceKey)
if (existingSessionId) {
const existingMachineId = this.sessionMachines.get(existingSessionId)
if (existingMachineId) {
const existingMachine = this.machines.get(existingMachineId)
if (existingMachine && existingMachine.status === MachineStatus.RUNNING) {
const isImmediate = existingMachine.stateMachine instanceof ImmediateCommandStateMachine
const onboardingEvent = ['BOOTSTRAP', 'BOOT', 'PERIODIC', 'SCHEDULED'].includes(
primaryEvent
)
const shouldPreempt = isImmediate && (forceMainForNI || onboardingEvent)
if (shouldPreempt) {
// Rebind session if changed
if (existingSessionId !== session) {
this.sessionMachines.delete(existingSessionId)
this.sessionMachines.set(session, existingMachineId)
this.deviceSessions.set(deviceKey, session)
existingMachine.session = session
}
// Replace with MainStateMachine
const newMachine = new MainStateMachine(device, session, this.taskQueue, normalizedEvents)
;(existingMachine as any).stateMachine = newMachine
existingMachine.events = evts
await newMachine.start()
existingMachine.currentState = newMachine.getCurrentStateName()
if (newMachine.isCompleted()) this.completeMachine(existingMachine)
return existingMachine
}
// For a new CONNECTION REQUEST arriving while a previous immediate session
// is still technically "running", complete the old one and create a fresh machine
if (isImmediate && primaryEvent === 'CONNECTION REQUEST') {
this.completeMachine(existingMachine)
// Fall through to create a new machine below
} else if (!isImmediate && existingSessionId !== session) {
// Stale MainStateMachine from a dropped session — the CPE started a new
// Inform cycle which means the previous session is dead. Complete the old
// machine and create a fresh one for the new session.
logger.warn(
{ oldSession: existingSessionId, newSession: session, state: existingMachine.currentState },
'[StateMachineManager] Replacing stale MainStateMachine from dropped session'
)
this.completeMachine(existingMachine)
// Fall through to create a new machine below
} else {
// Rebind session to existing machine
if (existingSessionId !== session) {
this.sessionMachines.delete(existingSessionId)
this.sessionMachines.set(session, existingMachineId)
this.deviceSessions.set(deviceKey, session)
existingMachine.session = session
}
return existingMachine
}
}
}
}
// Create new machine
const machineId = `${deviceKey}_${session}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`
let stateMachine: MainStateMachine | ImmediateCommandStateMachine
if (forceMainForNI) {
stateMachine = new MainStateMachine(device, session, this.taskQueue, normalizedEvents)
} else if (primaryEvent === 'BOOTSTRAP' || primaryEvent === 'BOOT') {
// BOOTSTRAP/BOOT always triggers full MainStateMachine onboarding
stateMachine = new MainStateMachine(device, session, this.taskQueue, normalizedEvents)
} else if (primaryEvent === 'CONNECTION REQUEST') {
// CONNECTION REQUEST only if no BOOTSTRAP/BOOT present
stateMachine = new ImmediateCommandStateMachine(device, session, this.taskQueue)
} else if (primaryEvent === 'PERIODIC' || primaryEvent === 'VALUE CHANGE' || primaryEvent === 'SCHEDULED') {
// Lightweight MainStateMachine
stateMachine = new MainStateMachine(device, session, this.taskQueue, normalizedEvents)
} else if (primaryEvent === 'DIAGNOSTICS COMPLETE' || primaryEvent === 'TRANSFER COMPLETE') {
stateMachine = new ImmediateCommandStateMachine(device, session, this.taskQueue)
} else {
// Unknown event - use MainStateMachine as default
stateMachine = new MainStateMachine(device, session, this.taskQueue, normalizedEvents)
}
const machineInstance: StateMachineInstance = {
id: machineId,
deviceId: device.deviceId,
session,
stateMachine,
status: MachineStatus.CREATED,
createdAt: new Date(),
currentState: stateMachine.getCurrentStateName(),
events: evts,
}
this.machines.set(machineId, machineInstance)
this.deviceSessions.set(deviceKey, session)
this.sessionMachines.set(session, machineId)
// Start the machine
try {
machineInstance.status = MachineStatus.RUNNING
machineInstance.currentState = stateMachine.getCurrentStateName()
if (stateMachine.isCompleted()) {
this.completeMachine(machineInstance)
}
} catch (error) {
machineInstance.status = MachineStatus.FAILED
machineInstance.completedAt = new Date()
logger.error(`Failed to start state machine: ${error}`)
}
return machineInstance
}
/**
* Bind a remote endpoint to a device for session rebinding
*/
bindRemoteToDevice(remoteKey: string, deviceId: DeviceId): void {
this.remoteToDevice.set(remoteKey, JSON.stringify(deviceId))
}
/**
* Returns the hash identifier for the device bound to a remote endpoint.
* Used by trace logging middleware to correlate non-Inform messages.
*/
getHashIdForRemote(remoteKey: string): string | undefined {
const deviceKeyJson = this.remoteToDevice.get(remoteKey)
if (!deviceKeyJson) return undefined
try {
const deviceId = JSON.parse(deviceKeyJson)
return getDeviceHashIdentifier({
manufacturer: deviceId.manufacturer ?? '',
oui: deviceId.oui,
productClass: deviceId.productClass,
serialNumber: deviceId.serialNumber,
})
} catch {
return undefined
}
}
/**
* Ensure session is bound when TCP connection changes
*/
ensureSessionForRemote(remoteKey: string, newSession: string): boolean {
const deviceKey = this.remoteToDevice.get(remoteKey)
if (!deviceKey) return false
const existingSession = this.deviceSessions.get(deviceKey)
if (!existingSession) return false
const machineId = this.sessionMachines.get(existingSession)
if (!machineId) return false
const machine = this.machines.get(machineId)
if (!machine) return false
// Don't rebind to a completed/failed machine
if (machine.status !== MachineStatus.RUNNING) {
return false
}
if (existingSession !== newSession) {
this.sessionMachines.delete(existingSession)
this.sessionMachines.set(newSession, machineId)
this.deviceSessions.set(deviceKey, newSession)
machine.session = newSession
}
return true
}
/**
* Continue when CPE sends empty body
*/
async continue(session: string): Promise<string> {
const machineId = this.sessionMachines.get(session)
if (!machineId) return ''
const machineInstance = this.machines.get(machineId)
if (!machineInstance || machineInstance.status !== MachineStatus.RUNNING) return ''
try {
// If a task is in-flight and CPE sends empty body, it means the CPE
// cannot/will not respond to our request. Treat as task failure.
if (machineInstance.lastTaskId) {
const failedTaskId = machineInstance.lastTaskId
machineInstance.lastTaskId = undefined
logger.warn(
{ session, taskId: failedTaskId },
'CPE sent empty body while task in-flight — treating as task failure'
)
this.taskQueue.failTask(failedTaskId, 'CPE sent empty body (no response)')
try {
await this.taskQueue.executeTaskCallback(failedTaskId, ResultStatus.Error, {
FaultCode: 9000,
FaultString: 'CPE did not respond (empty body)',
})
} catch (cbErr) {
logger.error(`Task callback error on empty-body failure: ${cbErr}`)
}
// Let the state machine handle the failure and advance
const response = await machineInstance.stateMachine.handleResponse(false, {
FaultCode: 9000,
FaultString: 'CPE did not respond (empty body)',
})
machineInstance.currentState = machineInstance.stateMachine.getCurrentStateName()
if (machineInstance.stateMachine.isCompleted()) {
this.completeMachine(machineInstance)
return ''
}
return response
}
// Let the state machine continue first (GPN discovery, config eval, etc.)
const response = await machineInstance.stateMachine.handleResponse(true, { isFirst: false })
logger.debug(
{
session,
stateName: machineInstance.stateMachine.getCurrentStateName(),
responseLength: response?.length || 0,
},
'[Continue] State machine handleResponse result'
)
machineInstance.currentState = machineInstance.stateMachine.getCurrentStateName()
if (machineInstance.stateMachine.isCompleted()) {
this.completeMachine(machineInstance)
return ''
}
// If state machine produced a response, send it
if (response && response !== '') {
return response
}
// Otherwise, serve queued tasks (GPV, etc.)
const pendingTask = this.taskQueue.getTaskForDevice(machineInstance.deviceId)
if (pendingTask) {
const builder = CWMPTaskQueue.task_map[pendingTask.task]
if (typeof builder === 'function') {
const envelope = builder(pendingTask.props)
if (pendingTask.id) machineInstance.lastTaskId = pendingTask.id
return envelope
}
}
return response
} catch (error) {
logger.error(`Error continuing session ${session}: ${error}`)
this.failMachine(machineInstance, String(error))
return ''
}
}
/**
* Handle a CWMP response message
*/
async handleCWMPResponse(session: string, success: boolean, message: any): Promise<string> {
const machineId = this.sessionMachines.get(session)
if (!machineId) return ''
const machineInstance = this.machines.get(machineId)
if (!machineInstance || machineInstance.status !== MachineStatus.RUNNING) return ''
try {
const responseTaskId = machineInstance.lastTaskId
const response = await machineInstance.stateMachine.handleResponse(success, message)
// Execute task callback
if (responseTaskId) {
if (success) {
this.taskQueue.completeTask(responseTaskId, message)
} else {
this.taskQueue.failTask(responseTaskId, JSON.stringify(message))
}
try {
await this.taskQueue.executeTaskCallback(
responseTaskId,
success ? ResultStatus.Success : ResultStatus.Error,
message
)
} catch (cbErr) {
logger.error(`Task callback error: ${cbErr}`)
}
if (machineInstance.lastTaskId === responseTaskId) {
machineInstance.lastTaskId = undefined
}
}
machineInstance.currentState = machineInstance.stateMachine.getCurrentStateName()
if (machineInstance.stateMachine.isCompleted()) {
this.completeMachine(machineInstance)
return ''
}
return response
} catch (error) {
logger.error(`Error handling CWMP response: ${error}`)
this.failMachine(machineInstance, String(error))
return ''
}
}
hasSession(session: string): boolean {
return this.sessionMachines.has(session)
}
/**
* Enqueue an immediate task for a device (e.g., after Inform)
*/
async enqueueImmediate(device: Device, type: string, payload: any): Promise<void> {
this.taskQueue.addCWMPTask(device.deviceId, type, payload)
}
private completeMachine(machine: StateMachineInstance): void {
machine.status = MachineStatus.COMPLETED
machine.completedAt = new Date()
const deviceKey = JSON.stringify(machine.deviceId)
this.deviceSessions.delete(deviceKey)
this.sessionMachines.delete(machine.session)
// Drain any leftover tasks so they don't bleed into the next session
this.taskQueue.clearPendingTasksForDevice(machine.deviceId)
}
private failMachine(machine: StateMachineInstance, error: string): void {
machine.status = MachineStatus.FAILED
machine.completedAt = new Date()
const deviceKey = JSON.stringify(machine.deviceId)
this.deviceSessions.delete(deviceKey)
this.sessionMachines.delete(machine.session)
logger.error(`State machine failed: ${error}`)
}
cleanupCompletedMachines(olderThanHours: number = 1): void {
const cutoff = new Date(Date.now() - olderThanHours * 60 * 60 * 1000)
for (const [id, machine] of this.machines.entries()) {
if (
(machine.status === MachineStatus.COMPLETED || machine.status === MachineStatus.FAILED) &&
machine.completedAt &&
machine.completedAt < cutoff
) {
this.machines.delete(id)
}
}
}
async shutdown(): Promise<void> {
this.stopCleanupTimer()
this.cleanupCompletedMachines(0)
}
}