diff --git a/cvmts/package.json b/cvmts/package.json index 773823f..81246fe 100644 --- a/cvmts/package.json +++ b/cvmts/package.json @@ -14,6 +14,7 @@ "@cvmts/cvm-rs": "*", "@cvmts/qemu": "*", "@maxmind/geoip2-node": "^5.0.0", + "@ygoe/msgpack": "^1.0.3", "execa": "^8.0.1", "mnemonist": "^0.39.5", "sharp": "^0.33.3", diff --git a/cvmts/src/CollabVMServer.ts b/cvmts/src/CollabVMServer.ts index 82d4588..fa1ece8 100644 --- a/cvmts/src/CollabVMServer.ts +++ b/cvmts/src/CollabVMServer.ts @@ -15,6 +15,8 @@ import { Size, Rect, Logger } from '@cvmts/shared'; import { JPEGEncoder } from './JPEGEncoder.js'; import VM from './VM.js'; import { ReaderModel } from '@maxmind/geoip2-node'; +import msgpack from "@ygoe/msgpack"; +import { CollabVMProtocolMessage, CollabVMProtocolMessageType } from './protocol/CollabVMProtocolMessage.js'; // Instead of strange hacks we can just use nodejs provided // import.meta properties, which have existed since LTS if not before @@ -441,6 +443,21 @@ export default class CollabVMServer { } this.sendVoteUpdate(); break; + case "cap": { + if (msgArr.length < 2) return; + // Capabilities can only be announced before connecting to the VM + if (client.connectedToNode) return; + var caps = []; + for (const cap of msgArr.slice(1)) switch(cap) { + case "bin": { + if (caps.indexOf("bin") !== -1) break; + client.Capabilities.bin = true; + caps.push("bin"); + break; + } + } + client.sendMsg(cvm.guacEncode("cap", ...caps)); + } case 'admin': if (msgArr.length < 2) return; switch (msgArr[1]) { @@ -649,11 +666,7 @@ export default class CollabVMServer { height: displaySize.height }); - this.clients.forEach(async (client) => { - client.sendMsg(cvm.guacEncode('size', '0', displaySize.width.toString(), displaySize.height.toString())); - client.sendMsg(cvm.guacEncode('png', '0', '0', '0', '0', encoded)); - client.sendMsg(cvm.guacEncode('sync', Date.now().toString())); - }); + this.clients.forEach(async (client) => this.SendFullScreenWithSize(client)); break; } break; @@ -806,14 +819,27 @@ export default class CollabVMServer { } private async OnDisplayRectangle(rect: Rect) { - let encodedb64 = await this.MakeRectData(rect); - + let encoded = await this.MakeRectData(rect); + let encodedb64 = encoded.toString("base64"); + let bmsg : CollabVMProtocolMessage = { + type: CollabVMProtocolMessageType.rect, + rect: { + x: rect.x, + y: rect.y, + data: encoded + }, + }; + var encodedbin = msgpack.encode(bmsg); this.clients .filter((c) => c.connectedToNode || c.viewMode == 1) .forEach((c) => { if (this.screenHidden && c.rank == Rank.Unregistered) return; - c.sendMsg(cvm.guacEncode('png', '0', '0', rect.x.toString(), rect.y.toString(), encodedb64)); - c.sendMsg(cvm.guacEncode('sync', Date.now().toString())); + if (c.Capabilities.bin) { + c.socket.sendBinary(encodedbin); + } else { + c.sendMsg(cvm.guacEncode('png', '0', '0', rect.x.toString(), rect.y.toString(), encodedb64)); + c.sendMsg(cvm.guacEncode('sync', Date.now().toString())); + } }); } @@ -838,7 +864,20 @@ export default class CollabVMServer { }); client.sendMsg(cvm.guacEncode('size', '0', displaySize.width.toString(), displaySize.height.toString())); - client.sendMsg(cvm.guacEncode('png', '0', '0', '0', '0', encoded)); + + if (client.Capabilities.bin) { + let msg : CollabVMProtocolMessage = { + type: CollabVMProtocolMessageType.rect, + rect: { + x: 0, + y: 0, + data: encoded + } + }; + client.socket.sendBinary(msgpack.encode(msg)); + } else { + client.sendMsg(cvm.guacEncode('png', '0', '0', '0', '0', encoded.toString("base64"))); + } } private async MakeRectData(rect: Rect) { @@ -847,11 +886,11 @@ export default class CollabVMServer { // TODO: actually throw an error here if(displaySize.width == 0 && displaySize.height == 0) - return "no"; + return Buffer.from("no") let encoded = await JPEGEncoder.Encode(display.Buffer(), displaySize, rect); - return encoded.toString('base64'); + return encoded; } async getThumbnail(): Promise { diff --git a/cvmts/src/NetworkClient.ts b/cvmts/src/NetworkClient.ts index 501da5b..600356f 100644 --- a/cvmts/src/NetworkClient.ts +++ b/cvmts/src/NetworkClient.ts @@ -1,6 +1,7 @@ export default interface NetworkClient { getIP(): string; send(msg: string): Promise; + sendBinary(msg: Uint8Array): Promise; close(): void; on(event: string, listener: (...args: any[]) => void): void; off(event: string, listener: (...args: any[]) => void): void; diff --git a/cvmts/src/TCP/TCPClient.ts b/cvmts/src/TCP/TCPClient.ts index 83e65db..f20498e 100644 --- a/cvmts/src/TCP/TCPClient.ts +++ b/cvmts/src/TCP/TCPClient.ts @@ -2,6 +2,9 @@ import EventEmitter from 'events'; import NetworkClient from '../NetworkClient.js'; import { Socket } from 'net'; +const TextHeader = 0; +const BinaryHeader = 1; + export default class TCPClient extends EventEmitter implements NetworkClient { private socket: Socket; private cache: string; @@ -34,7 +37,21 @@ export default class TCPClient extends EventEmitter implements NetworkClient { send(msg: string): Promise { return new Promise((res, rej) => { - this.socket.write(msg, (err) => { + let _msg = new Uint32Array([TextHeader, ...Buffer.from(msg, "utf-8")]); + this.socket.write(Buffer.from(_msg), (err) => { + if (err) { + rej(err); + return; + } + res(); + }); + }); + } + + sendBinary(msg: Uint8Array): Promise { + return new Promise((res, rej) => { + let _msg = new Uint32Array([BinaryHeader, msg.length, ...msg]); + this.socket.write(Buffer.from(_msg), (err) => { if (err) { rej(err); return; diff --git a/cvmts/src/User.ts b/cvmts/src/User.ts index 6f5bc2b..1590f2e 100644 --- a/cvmts/src/User.ts +++ b/cvmts/src/User.ts @@ -6,6 +6,7 @@ import RateLimiter from './RateLimiter.js'; import { execa, execaCommand, ExecaSyncError } from 'execa'; import { Logger } from '@cvmts/shared'; import NetworkClient from './NetworkClient.js'; +import CollabVMCapabilities from './protocol/CollabVMCapabilities.js'; export class User { socket: NetworkClient; @@ -19,6 +20,7 @@ export class User { msgsSent: number; Config: IConfig; IP: IPData; + Capabilities: CollabVMCapabilities; // Hide flag. Only takes effect if the user is logged in. noFlag: boolean = false; countryCode: string | null = null; @@ -38,6 +40,7 @@ export class User { this.Config = config; this.socket = socket; this.msgsSent = 0; + this.Capabilities = new CollabVMCapabilities(); this.socket.on('disconnect', () => { // Unref the ip data for this connection diff --git a/cvmts/src/WebSocket/WSClient.ts b/cvmts/src/WebSocket/WSClient.ts index cbb84ed..5b4db9b 100644 --- a/cvmts/src/WebSocket/WSClient.ts +++ b/cvmts/src/WebSocket/WSClient.ts @@ -33,6 +33,7 @@ export default class WSClient extends EventEmitter implements NetworkClient { getIP(): string { return this.ip; } + send(msg: string): Promise { return new Promise((res, rej) => { if (!this.isOpen()) res(); @@ -47,6 +48,20 @@ export default class WSClient extends EventEmitter implements NetworkClient { }); } + sendBinary(msg: Uint8Array): Promise { + return new Promise((res, rej) => { + if (!this.isOpen()) res(); + + this.socket.send(msg, (err) => { + if (err) { + rej(err); + return; + } + res(); + }); + }); + } + close(): void { if (this.isOpen()) { // While this seems counterintutive, do note that the WebSocket protocol diff --git a/cvmts/src/protocol/CollabVMCapabilities.ts b/cvmts/src/protocol/CollabVMCapabilities.ts new file mode 100644 index 0000000..c94106f --- /dev/null +++ b/cvmts/src/protocol/CollabVMCapabilities.ts @@ -0,0 +1,8 @@ +export default class CollabVMCapabilities { + // Support for JPEG screen rects in binary msgpack format + bin: boolean; + + constructor() { + this.bin = false; + } +} \ No newline at end of file diff --git a/cvmts/src/protocol/CollabVMProtocolMessage.ts b/cvmts/src/protocol/CollabVMProtocolMessage.ts new file mode 100644 index 0000000..544a7e7 --- /dev/null +++ b/cvmts/src/protocol/CollabVMProtocolMessage.ts @@ -0,0 +1,11 @@ +import CollabVMRectMessage from "./CollabVMRectMessage.js"; + +export interface CollabVMProtocolMessage { + type: CollabVMProtocolMessageType; + rect?: CollabVMRectMessage | undefined; +} + +export enum CollabVMProtocolMessageType { + // JPEG Dirty Rectangle + rect = 0, +} \ No newline at end of file diff --git a/cvmts/src/protocol/CollabVMRectMessage.ts b/cvmts/src/protocol/CollabVMRectMessage.ts new file mode 100644 index 0000000..f2a8668 --- /dev/null +++ b/cvmts/src/protocol/CollabVMRectMessage.ts @@ -0,0 +1,5 @@ +export default interface CollabVMRectMessage { + x: number; + y: number; + data: Uint8Array; +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 1181e92..81a1c39 100644 --- a/yarn.lock +++ b/yarn.lock @@ -64,6 +64,7 @@ __metadata: "@maxmind/geoip2-node": "npm:^5.0.0" "@types/node": "npm:^20.12.5" "@types/ws": "npm:^8.5.5" + "@ygoe/msgpack": "npm:^1.0.3" execa: "npm:^8.0.1" mnemonist: "npm:^0.39.5" prettier: "npm:^3.2.5" @@ -1570,6 +1571,13 @@ __metadata: languageName: node linkType: hard +"@ygoe/msgpack@npm:^1.0.3": + version: 1.0.3 + resolution: "@ygoe/msgpack@npm:1.0.3" + checksum: 10c0/f4a9adc86f41b6ccc07fd7756adac28b71a38e6fb741ff4943b338f3bb6283154ae9ebf0614fa40e78cd3c09bbd79f4c5e60e136a8136a99a2b8385ba4710ed7 + languageName: node + linkType: hard + "abbrev@npm:^2.0.0": version: 2.0.0 resolution: "abbrev@npm:2.0.0"