add support for binary JPEG (server)

cvmts/package.json

@@ -14,6 +14,7 @@
     "@cvmts/cvm-rs": "*",
     "@cvmts/qemu": "*",
     "@maxmind/geoip2-node": "^5.0.0",
+    "@ygoe/msgpack": "^1.0.3",
     "execa": "^8.0.1",
     "mnemonist": "^0.39.5",
     "sharp": "^0.33.3",

cvmts/src/CollabVMServer.ts

@@ -15,6 +15,8 @@ import { Size, Rect, Logger } from '@cvmts/shared';
 import { JPEGEncoder } from './JPEGEncoder.js';
 import VM from './VM.js';
 import { ReaderModel } from '@maxmind/geoip2-node';
+import msgpack from "@ygoe/msgpack";
+import { CollabVMProtocolMessage, CollabVMProtocolMessageType } from './protocol/CollabVMProtocolMessage.js';
 
 // Instead of strange hacks we can just use nodejs provided
 // import.meta properties, which have existed since LTS if not before
@@ -441,6 +443,21 @@ export default class CollabVMServer {
 					}
 					this.sendVoteUpdate();
 					break;
+				case "cap": {
+					if (msgArr.length < 2) return;
+					// Capabilities can only be announced before connecting to the VM
+					if (client.connectedToNode) return;
+					var caps = [];
+					for (const cap of msgArr.slice(1)) switch(cap) {
+						case "bin": {
+							if (caps.indexOf("bin") !== -1) break;
+							client.Capabilities.bin = true;
+							caps.push("bin");
+							break;
+						}
+					}
+					client.sendMsg(cvm.guacEncode("cap", ...caps));
+				}
 				case 'admin':
 					if (msgArr.length < 2) return;
 					switch (msgArr[1]) {
@@ -649,11 +666,7 @@ export default class CollabVMServer {
 										height: displaySize.height
 									});
 
-									this.clients.forEach(async (client) => {
+									this.clients.forEach(async (client) => this.SendFullScreenWithSize(client));
-										client.sendMsg(cvm.guacEncode('size', '0', displaySize.width.toString(), displaySize.height.toString()));
-										client.sendMsg(cvm.guacEncode('png', '0', '0', '0', '0', encoded));
-										client.sendMsg(cvm.guacEncode('sync', Date.now().toString()));
-									});
 									break;
 							}
 							break;
@@ -806,14 +819,27 @@ export default class CollabVMServer {
 	}
 
 	private async OnDisplayRectangle(rect: Rect) {
-		let encodedb64 = await this.MakeRectData(rect);
+		let encoded = await this.MakeRectData(rect);
-
+		let encodedb64 = encoded.toString("base64");
+		let bmsg : CollabVMProtocolMessage = {
+			type: CollabVMProtocolMessageType.rect,
+			rect: {
+				x: rect.x,
+				y: rect.y,
+				data: encoded
+			},
+		};
+		var encodedbin = msgpack.encode(bmsg);
 		this.clients
 			.filter((c) => c.connectedToNode || c.viewMode == 1)
 			.forEach((c) => {
 				if (this.screenHidden && c.rank == Rank.Unregistered) return;
-				c.sendMsg(cvm.guacEncode('png', '0', '0', rect.x.toString(), rect.y.toString(), encodedb64));
+				if (c.Capabilities.bin) {
-				c.sendMsg(cvm.guacEncode('sync', Date.now().toString()));
+					c.socket.sendBinary(encodedbin);
+				} else {
+					c.sendMsg(cvm.guacEncode('png', '0', '0', rect.x.toString(), rect.y.toString(), encodedb64));
+					c.sendMsg(cvm.guacEncode('sync', Date.now().toString()));
+				}
 			});
 	}
 
@@ -838,7 +864,20 @@ export default class CollabVMServer {
 		});
 
 		client.sendMsg(cvm.guacEncode('size', '0', displaySize.width.toString(), displaySize.height.toString()));
-		client.sendMsg(cvm.guacEncode('png', '0', '0', '0', '0', encoded));
+
+		if (client.Capabilities.bin) {
+			let msg : CollabVMProtocolMessage = {
+				type: CollabVMProtocolMessageType.rect,
+				rect: {
+					x: 0,
+					y: 0,
+					data: encoded
+				}
+			};
+			client.socket.sendBinary(msgpack.encode(msg));
+		} else {
+			client.sendMsg(cvm.guacEncode('png', '0', '0', '0', '0', encoded.toString("base64")));
+		}
 	}
 
 	private async MakeRectData(rect: Rect) {
@@ -847,11 +886,11 @@ export default class CollabVMServer {
 
 		// TODO: actually throw an error here
 		if(displaySize.width == 0 && displaySize.height == 0)
-			return "no";
+			return Buffer.from("no")
 
 		let encoded = await JPEGEncoder.Encode(display.Buffer(), displaySize, rect);
 
-		return encoded.toString('base64');
+		return encoded;
 	}
 
 	async getThumbnail(): Promise<string> {

cvmts/src/NetworkClient.ts

@@ -1,6 +1,7 @@
 export default interface NetworkClient {
 	getIP(): string;
 	send(msg: string): Promise<void>;
+	sendBinary(msg: Uint8Array): Promise<void>;
 	close(): void;
 	on(event: string, listener: (...args: any[]) => void): void;
 	off(event: string, listener: (...args: any[]) => void): void;

cvmts/src/TCP/TCPClient.ts

@@ -2,6 +2,9 @@ import EventEmitter from 'events';
 import NetworkClient from '../NetworkClient.js';
 import { Socket } from 'net';
 
+const TextHeader = 0;
+const BinaryHeader = 1;
+
 export default class TCPClient extends EventEmitter implements NetworkClient {
 	private socket: Socket;
 	private cache: string;
@@ -34,7 +37,21 @@ export default class TCPClient extends EventEmitter implements NetworkClient {
 
 	send(msg: string): Promise<void> {
 		return new Promise((res, rej) => {
-			this.socket.write(msg, (err) => {
+			let _msg = new Uint32Array([TextHeader, ...Buffer.from(msg, "utf-8")]);
+			this.socket.write(Buffer.from(_msg), (err) => {
+				if (err) {
+					rej(err);
+					return;
+				}
+				res();
+			});
+		});
+	}
+
+	sendBinary(msg: Uint8Array): Promise<void> {
+		return new Promise((res, rej) => {
+			let _msg = new Uint32Array([BinaryHeader, msg.length, ...msg]);
+			this.socket.write(Buffer.from(_msg), (err) => {
 				if (err) {
 					rej(err);
 					return;

cvmts/src/User.ts

@@ -6,6 +6,7 @@ import RateLimiter from './RateLimiter.js';
 import { execa, execaCommand, ExecaSyncError } from 'execa';
 import { Logger } from '@cvmts/shared';
 import NetworkClient from './NetworkClient.js';
+import CollabVMCapabilities from './protocol/CollabVMCapabilities.js';
 
 export class User {
 	socket: NetworkClient;
@@ -19,6 +20,7 @@ export class User {
 	msgsSent: number;
 	Config: IConfig;
 	IP: IPData;
+	Capabilities: CollabVMCapabilities;
 	// Hide flag. Only takes effect if the user is logged in.
 	noFlag: boolean = false;
 	countryCode: string | null = null;
@@ -38,6 +40,7 @@ export class User {
 		this.Config = config;
 		this.socket = socket;
 		this.msgsSent = 0;
+		this.Capabilities = new CollabVMCapabilities();
 
 		this.socket.on('disconnect', () => {
 			// Unref the ip data for this connection

cvmts/src/WebSocket/WSClient.ts

@@ -33,6 +33,7 @@ export default class WSClient extends EventEmitter implements NetworkClient {
 	getIP(): string {
 		return this.ip;
 	}
+
 	send(msg: string): Promise<void> {
 		return new Promise((res, rej) => {
 			if (!this.isOpen()) res();
@@ -47,6 +48,20 @@ export default class WSClient extends EventEmitter implements NetworkClient {
 		});
 	}
 
+	sendBinary(msg: Uint8Array): Promise<void> {
+		return new Promise((res, rej) => {
+			if (!this.isOpen()) res();
+
+			this.socket.send(msg, (err) => {
+				if (err) {
+					rej(err);
+					return;
+				}
+				res();
+			});
+		});
+	}
+
 	close(): void {
 		if (this.isOpen()) {
 			// While this seems counterintutive, do note that the WebSocket protocol

cvmts/src/protocol/CollabVMCapabilities.ts

@@ -0,0 +1,8 @@
+export default class CollabVMCapabilities {
+    // Support for JPEG screen rects in binary msgpack format
+    bin: boolean;
+
+    constructor() {
+        this.bin = false;
+    }
+}

cvmts/src/protocol/CollabVMProtocolMessage.ts

@@ -0,0 +1,11 @@
+import CollabVMRectMessage from "./CollabVMRectMessage.js";
+
+export interface CollabVMProtocolMessage {
+    type: CollabVMProtocolMessageType;
+    rect?: CollabVMRectMessage | undefined;
+}
+
+export enum CollabVMProtocolMessageType {
+    // JPEG Dirty Rectangle
+    rect = 0,
+}

cvmts/src/protocol/CollabVMRectMessage.ts

@@ -0,0 +1,5 @@
+export default interface CollabVMRectMessage {
+    x: number;
+    y: number;
+    data: Uint8Array;
+}

yarn.lock

@@ -64,6 +64,7 @@ __metadata:
     "@maxmind/geoip2-node": "npm:^5.0.0"
     "@types/node": "npm:^20.12.5"
     "@types/ws": "npm:^8.5.5"
+    "@ygoe/msgpack": "npm:^1.0.3"
     execa: "npm:^8.0.1"
     mnemonist: "npm:^0.39.5"
     prettier: "npm:^3.2.5"
@@ -1570,6 +1571,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@ygoe/msgpack@npm:^1.0.3":
+  version: 1.0.3
+  resolution: "@ygoe/msgpack@npm:1.0.3"
+  checksum: 10c0/f4a9adc86f41b6ccc07fd7756adac28b71a38e6fb741ff4943b338f3bb6283154ae9ebf0614fa40e78cd3c09bbd79f4c5e60e136a8136a99a2b8385ba4710ed7
+  languageName: node
+  linkType: hard
+
 "abbrev@npm:^2.0.0":
   version: 2.0.0
   resolution: "abbrev@npm:2.0.0"