Hot Standby should check for secondary server upon start

packages/connector/src/MosConnection.ts

@@ -40,6 +40,7 @@ export class MosConnection extends EventEmitter<MosConnectionEvents> implements
 	private _mosDevices: { [ncsID: string]: MosDevice } = {}
 	private _initialized = false
 	private _isListening = false
+	private _isOpenMediaHotStandby = false
 
 	// private _isListening: Promise<boolean[]>
 
@@ -81,6 +82,7 @@ export class MosConnection extends EventEmitter<MosConnectionEvents> implements
 	 */
 	async connect(connectionOptions: IMOSDeviceConnectionOptions): Promise<MosDevice> {
 		if (!this._initialized) throw Error('Not initialized, run .init() first!')
+		this._isOpenMediaHotStandby = connectionOptions.secondary?.openMediaHotStandby ?? false
 
 		// Connect to MOS-device:
 		const primary = new NCSServerConnection(
@@ -90,9 +92,21 @@ export class MosConnection extends EventEmitter<MosConnectionEvents> implements
 			connectionOptions.primary.timeout,
 			connectionOptions.primary.heartbeatInterval,
 			this._debug,
-			this.mosTypes.strict
+			this.mosTypes.strict,
+			this._isOpenMediaHotStandby
 		)
-		let secondary: NCSServerConnection | null = null
+		const secondary = connectionOptions.secondary
+			? new NCSServerConnection(
+					connectionOptions.secondary.id,
+					connectionOptions.secondary.host,
+					this._conf.mosID,
+					connectionOptions.secondary.timeout,
+					connectionOptions.secondary.heartbeatInterval,
+					this._debug,
+					this.mosTypes.strict,
+					this._isOpenMediaHotStandby
+			  )
+			: null
 		this._ncsConnections[connectionOptions.primary.host] = primary
 
 		primary.on('rawMessage', (type: string, message: string) => {
@@ -130,16 +144,7 @@ export class MosConnection extends EventEmitter<MosConnectionEvents> implements
 			)
 		}
 
-		if (connectionOptions.secondary) {
-			secondary = new NCSServerConnection(
-				connectionOptions.secondary.id,
-				connectionOptions.secondary.host,
-				this._conf.mosID,
-				connectionOptions.secondary.timeout,
-				connectionOptions.secondary.heartbeatInterval,
-				this._debug,
-				this.mosTypes.strict
-			)
+		if (secondary && connectionOptions.secondary) {
 			this._ncsConnections[connectionOptions.secondary.host] = secondary
 			secondary.on('rawMessage', (type: string, message: string) => {
 				this.emit('rawMessage', 'secondary', type, message)
@@ -506,7 +511,8 @@ export class MosConnection extends EventEmitter<MosConnectionEvents> implements
 						undefined,
 						undefined,
 						this._debug,
-						this.mosTypes.strict
+						this.mosTypes.strict,
+						this._isOpenMediaHotStandby
 					)
 					this._ncsConnections[remoteAddress] = primary
 

packages/connector/src/__tests__/OpenMediaHotStandby.spec.ts

@@ -128,7 +128,8 @@ describe('Hot Standby Feature', () => {
 
 		if (primary && secondary) {
 			expect(primary.getConnectedStatus().connected).toBe(true)
-			expect(secondary.getConnectedStatus().connected).toBe(true)
+			// Hot standby only connects one connection at a time:
+			expect(secondary.getConnectedStatus().connected).toBe(false)
 
 			// Verify heartbeat states
 			expect(primary.isHearbeatEnabled()).toBe(true)
@@ -142,7 +143,7 @@ describe('Hot Standby Feature', () => {
 
 		if (primary && secondary) {
 			expect(primary.connected).toBe(true)
-			expect(secondary.connected).toBe(true)
+			expect(secondary.connected).toBe(false)
 
 			// Disconnect primary connection:
 			await discconnectPrimary()
@@ -172,7 +173,8 @@ describe('Hot Standby Feature', () => {
 		if (primary && secondary) {
 			// Initially, primary should be connected and secondary should be connected but with heartbeats disabled
 			expect(primary.connected).toBe(true)
-			expect(secondary.connected).toBe(true)
+			// Hot standby only connects one connection at a time:
+			expect(secondary.connected).toBe(false)
 			expect(primary.isHearbeatEnabled()).toBe(true)
 			expect(secondary.isHearbeatEnabled()).toBe(false)
 
@@ -204,7 +206,8 @@ describe('Hot Standby Feature', () => {
 		if (primary && secondary) {
 			// Initially, both should be connected with primary heartbeats enabled and secondary disabled
 			expect(primary.connected).toBe(true)
-			expect(secondary.connected).toBe(true)
+			// Hot standby only connects one connection at a time:
+			expect(secondary.connected).toBe(false)
 			expect(primary.isHearbeatEnabled()).toBe(true)
 			expect(secondary.isHearbeatEnabled()).toBe(false)
 
@@ -234,9 +237,10 @@ describe('Hot Standby Feature', () => {
 		expect(secondary).toBeTruthy()
 
 		if (primary && secondary) {
-			// Initial setup - primary connected and secondary connected but with heartbeats disabled
+			// Initial setup - primary connected and secondary not but with heartbeats disabled
 			expect(primary.connected).toBe(true)
-			expect(secondary.connected).toBe(true)
+			// Hot standby only connects one connection at a time:
+			expect(secondary.connected).toBe(false)
 
 			// First disconnect primary to force secondary active
 			await discconnectPrimary()

packages/connector/src/connection/NCSServerConnection.ts

@@ -11,6 +11,7 @@ export interface ClientDescription {
 	heartbeatConnected: boolean
 	client: MosSocketClient
 	clientDescription: MosModel.PortType
+	isConnected: () => boolean
 }
 
 export interface INCSServerConnection {
@@ -47,6 +48,7 @@ export class NCSServerConnection extends EventEmitter<NCSServerConnectionEvents>
 
 	private _heartBeatsTimer?: NodeJS.Timeout
 	private _heartBeatsInterval: number
+	private _isOpenMediaHotStandby = false
 
 	constructor(
 		id: string,
@@ -55,7 +57,8 @@ export class NCSServerConnection extends EventEmitter<NCSServerConnectionEvents>
 		timeout: number | undefined,
 		heartbeatsInterval: number | undefined,
 		debug: boolean,
-		strict: boolean
+		strict: boolean,
+		isOpenMediaHotStandby: boolean
 	) {
 		super()
 		this._id = id
@@ -66,6 +69,7 @@ export class NCSServerConnection extends EventEmitter<NCSServerConnectionEvents>
 		this._connected = false
 		this._debug = debug ?? false
 		this._strict = strict ?? false
+		this._isOpenMediaHotStandby = isOpenMediaHotStandby ?? false
 	}
 	get timeout(): number {
 		return this._timeout
@@ -87,6 +91,9 @@ export class NCSServerConnection extends EventEmitter<NCSServerConnectionEvents>
 			heartbeatConnected: false,
 			client: client,
 			clientDescription: clientDescription,
+			isConnected: () => {
+				return client.isConnected
+			},
 		}
 		client.on('rawMessage', (type: string, message: string) => {
 			this.emit('rawMessage', type, message)
@@ -214,21 +221,36 @@ export class NCSServerConnection extends EventEmitter<NCSServerConnectionEvents>
 				connected: false,
 				status: 'Not connected',
 			}
+
+		// Check if we have any clients at all
+		const clientCount = Object.keys(this._clients).length
+		if (clientCount === 0) {
+			return {
+				connected: false,
+				status: 'No clients available',
+			}
+		}
+
+		let isConnectedToSomeDevice = false
+
 		let notConnectedStatus: string | undefined = undefined
 		Object.values<ClientDescription>(this._clients).forEach((client) => {
-			if (client.useHeartbeats && !client.heartbeatConnected) {
+			if (client.isConnected()) {
+				isConnectedToSomeDevice = true
+			}
+			if (client.useHeartbeats && !client.heartbeatConnected && !this._isOpenMediaHotStandby) {
 				notConnectedStatus = `No heartbeats on port ${client.clientDescription}`
 			}
 		})
-		if (!notConnectedStatus) {
+		if (!notConnectedStatus && isConnectedToSomeDevice) {
 			return {
 				connected: true,
 				status: 'Connected',
 			}
 		} else {
 			return {
 				connected: false,
-				status: notConnectedStatus,
+				status: notConnectedStatus || 'No heartbeats',
 			}
 		}
 	}

packages/connector/src/connection/mosSocketClient.ts

@@ -131,6 +131,10 @@ export class MosSocketClient extends EventEmitter<MosSocketClientEvents> {
 		}
 	}
 
+	get isConnected(): boolean {
+		return this._connected
+	}
+
 	/** */
 	disconnect(): void {
 		this.dispose()

packages/mos-dummy-device/.eslintrc.js

@@ -0,0 +1,6 @@
+module.exports = {
+	extends: '../../.eslintrc',
+	rules: {
+		'no-console': 'off',
+	},
+}

packages/mos-dummy-device/README.md

@@ -0,0 +1,132 @@
+# MOS Dummy Device
+
+A dummy MOS (Media Object Server) server for testing MOS clients with failover capabilities. This server implements the MOS protocol and allows you to simulate connection issues and manage rundowns through JSON files.
+
+The mos dummy device is primary made for debugging MOS server connections and failovers.
+For quick ingesting MOS rundowns use quick-mos
+
+## Features
+
+- MOS protocol support Profiles 0, 1, 2, 3, and 4
+- JSON-based rundown downs in the `rundowns` directory
+- Hot reloading of rundowns from the filesystem
+- Simulation of server outages for failover testing
+- Command-line interface for manual testing
+
+## Installation
+Install dependencies:
+
+```bash
+yarn install
+```
+
+### Development mode
+To run in development mode with automatic reloading:
+
+```bash
+yarn dev
+```
+
+you can add a --secondary flag to run a secondary server on a different port:
+
+```bash
+yarn dev --secondary
+```
+
+Or with file watching:
+
+```bash
+yarn run watch
+```
+
+### Run on same machine as client:
+To run on same machine as client, remember to set the clients acceptsConnections to false (here shown in Sofie):
+
+```bash
+mos: {
+		self: {
+			debug: debug,
+			// mosID: 'sofie.tv.automation',
+			mosID: 'N/A', // set by Core
+			acceptsConnections: false, // default:true
+			// accepsConnectionsFrom: ['127.0.0.1'],
+			profiles: {
+				'0': true,
+				'1': true,
+				'2': true,
+				'3': false,
+				'4': false,
+				'5': false,
+				'6': false,
+				'7': false,
+			},
+			offspecFailover: true,
+		},
+```
+
+
+### Rundown Management
+
+Rundowns are stored as JSON files in the `rundowns` directory. The server watches this directory for changes:
+
+- Adding a new `.json` file creates a new rundown
+- Modifying a file updates the rundown
+- Deleting a file removes the rundown
+
+### Rundown JSON Format
+
+Here's an example of a rundown JSON file:
+
+```json
+{
+  "ID": "EXAMPLE_RO",
+  "Slug": "Example Rundown",
+  "DefaultChannel": "A",
+  "Stories": [
+    {
+      "ID": "EXAMPLE_RO_STORY_1",
+      "Slug": "Story 1",
+      "Number": "1",
+      "Items": [
+        {
+          "ID": "EXAMPLE_RO_STORY_1_ITEM_1",
+          "Slug": "Item 1 in Story 1",
+          "ObjectID": "OBJ_EXAMPLE_RO_STORY_1_ITEM_1",
+          "MOSID": "DUMMY.MOS.SERVER",
+          "ObjectSlug": "Item 1 in Story 1",
+          "Duration": 1000,
+          "TimeBase": 100
+        }
+      ]
+    }
+  ]
+}
+```
+
+### Command-Line Interface
+
+While the server is running, you can use the following commands:
+
+- `outage [duration_ms]` - Simulate a server outage for the specified duration (default: 5000ms)
+- `exit` - Shutdown the server and exit
+
+Example: `outage 10000` will simulate a 10-second outage.
+
+## Configuration
+
+You can modify the configuration in the `SERVER_CONFIG` object in `src/index.ts`:
+
+- `mosID`: The MOS ID of the server
+- `acceptsConnections`: Whether to accept incoming connections
+- `profiles`: The MOS profiles to support
+- `debug`: Enable/disable debug logging
+- `ports`: The ports to use for MOS communication
+
+## Testing Failover
+
+To test failover with your MOS client:
+
+1. Start the server
+2. Connect your MOS client to the server
+3. Use the `outage` command to simulate an outage
+4. Observe how your client handles the disconnection and reconnection

packages/mos-dummy-device/package.json

@@ -0,0 +1,19 @@
+{
+    "name": "mos-dummy-server",
+    "version": "1.0.0",
+    "description": "A dummy MOS server for testing MOS clients with failover capabilities",
+    "main": "dist/index.js",
+    "scripts": {
+        "dev": "ts-node src/index.ts"
+    },
+    "license": "MIT",
+    "dependencies": {
+        "@mos-connection/connector": "^4.2.2",
+        "chokidar": "^3.5.3"
+    },
+    "devDependencies": {
+        "@types/node": "^20.10.5",
+        "ts-node": "^10.9.2",
+        "typescript": "^5.3.3"
+    }
+}