diff --git a/package.json b/package.json index 8f7d852..f281c80 100644 --- a/package.json +++ b/package.json @@ -14,12 +14,12 @@ "@types/sharp": "^0.31.1", "@types/ws": "^8.5.4", "async-mutex": "^0.4.0", + "canvas": "^2.11.0", "execa": "^6.1.0", "fs": "^0.0.1-security", "jimp": "^0.16.2", "mnemonist": "^0.39.5", "rfb2": "github:elijahr2411/node-rfb2", - "sharp": "^0.31.3", "toml": "^3.0.0", "typescript": "^4.9.5", "ws": "^8.12.0" diff --git a/src/Framebuffer.ts b/src/Framebuffer.ts index 38deb5f..33b1dda 100644 --- a/src/Framebuffer.ts +++ b/src/Framebuffer.ts @@ -1,7 +1,7 @@ import { Mutex } from "async-mutex"; export default class Framebuffer { - private fb : Buffer; + fb : Buffer; private writemutex : Mutex; size : {height : number, width : number}; constructor() { @@ -15,10 +15,10 @@ export default class Framebuffer { this.size.width = w; this.fb = Buffer.alloc(size); } - loadDirtyRect(rect : Buffer, x : number, y : number, width : number, height : number) { + loadDirtyRect(rect : Buffer, x : number, y : number, width : number, height : number) : Promise { if (this.fb.length < rect.length) throw new Error("Dirty rect larger than framebuffer (did you forget to set the size?)"); - this.writemutex.runExclusive(() => { + return this.writemutex.runExclusive(() => { return new Promise((res, rej) => { var byteswritten = 0; for (var i = 0; i < height; i++) { diff --git a/src/QEMUVM.ts b/src/QEMUVM.ts index f94a19e..0935471 100644 --- a/src/QEMUVM.ts +++ b/src/QEMUVM.ts @@ -4,10 +4,15 @@ import * as rfb from 'rfb2'; import * as fs from 'fs'; import { execa, ExecaChildProcess } from "execa"; import QMPClient from "./QMPClient.js"; +import BatchRects from "./RectBatcher.js"; +import { createCanvas, Canvas, CanvasRenderingContext2D, createImageData } from "canvas"; +import { Mutex } from "async-mutex"; export default class QEMUVM extends EventEmitter { vnc? : rfb.RfbClient; vncPort : number; + framebuffer : Canvas; + framebufferCtx : CanvasRenderingContext2D; qmpSock : string; qmpClient : QMPClient; qemuCmd : string; @@ -17,6 +22,9 @@ export default class QEMUVM extends EventEmitter { processRestartErrorLevel : number; expectedExit : boolean; vncOpen : boolean; + vncUpdateInterval? : NodeJS.Timer; + rects : {height:number,width:number,x:number,y:number,data:Buffer}[]; + rectMutex : Mutex; vncReconnectTimeout? : NodeJS.Timer; qmpReconnectTimeout? : NodeJS.Timer; @@ -34,6 +42,10 @@ export default class QEMUVM extends EventEmitter { this.qmpErrorLevel = 0; this.vncErrorLevel = 0; this.vncOpen = true; + this.rects = []; + this.rectMutex = new Mutex(); + this.framebuffer = createCanvas(1, 1); + this.framebufferCtx = this.framebuffer.getContext("2d"); this.processRestartErrorLevel = 0; this.expectedExit = false; this.qmpClient = new QMPClient(this.qmpSock); @@ -133,23 +145,51 @@ export default class QEMUVM extends EventEmitter { this.vncErrorLevel = 0; //@ts-ignore this.onVNCSize({height: this.vnc.height, width: this.vnc.width}); + this.vncUpdateInterval = setInterval(() => this.SendRects(), 33); } - private async onVNCRect(rect : any) { - var buff = Buffer.alloc(rect.height * rect.width * 4) - var offset = 0; - for (var i = 0; i < rect.data.length; i += 4) { - buff[offset++] = rect.data[i + 2]; - buff[offset++] = rect.data[i + 1]; - buff[offset++] = rect.data[i]; - buff[offset++] = 255; - } - this.emit("dirtyrect", buff, rect.x, rect.y, rect.width, rect.height); - if (!this.vnc) throw new Error(); - if (this.vncOpen) - this.vnc.requestUpdate(true, 0, 0, this.vnc.height, this.vnc.width); + private onVNCRect(rect : any) { + return this.rectMutex.runExclusive(async () => { + return new Promise(async (res, rej) => { + var buff = Buffer.alloc(rect.height * rect.width * 4) + var offset = 0; + for (var i = 0; i < rect.data.length; i += 4) { + buff[offset++] = rect.data[i + 2]; + buff[offset++] = rect.data[i + 1]; + buff[offset++] = rect.data[i]; + buff[offset++] = 255; + } + var imgdata = createImageData(Uint8ClampedArray.from(buff), rect.width, rect.height); + this.framebufferCtx.putImageData(imgdata, rect.x, rect.y); + this.rects.push({ + x: rect.x, + y: rect.y, + height: rect.height, + width: rect.width, + data: buff, + }); + if (!this.vnc) throw new Error(); + if (this.vncOpen) + this.vnc.requestUpdate(true, 0, 0, this.vnc.height, this.vnc.width); + res(); + }) + }); + } + + SendRects() { + if (!this.vnc || this.rects.length < 1) return; + return this.rectMutex.runExclusive(() => { + return new Promise(async (res, rej) => { + var rect = await BatchRects(this.framebuffer, [...this.rects]); + this.rects = []; + this.emit('dirtyrect', rect.data, rect.x, rect.y); + res(); + }); + }) } private onVNCSize(size : any) { + if (this.framebuffer.height !== size.height) this.framebuffer.height = size.height; + if (this.framebuffer.width !== size.width) this.framebuffer.width = size.width; this.emit("size", {height: size.height, width: size.width}); } @@ -174,6 +214,7 @@ export default class QEMUVM extends EventEmitter { this.expectedExit = true; this.vncOpen = false; this.vnc?.end(); + clearInterval(this.vncUpdateInterval); var killTimeout = setTimeout(() => { console.log("Force killing QEMU after 10 seconds of waiting for shutdown"); this.qemuProcess?.kill(9); diff --git a/src/RectBatcher.ts b/src/RectBatcher.ts new file mode 100644 index 0000000..3283929 --- /dev/null +++ b/src/RectBatcher.ts @@ -0,0 +1,28 @@ +import { Canvas, createCanvas, createImageData } from "canvas"; + +export default async function BatchRects(fb : Canvas, rects : {height:number,width:number,x:number,y:number,data:Buffer}[]) : Promise<{x:number,y:number,data:Canvas}> { + var mergedX = fb.width; + var mergedY = fb.height; + var mergedHeight = 0; + var mergedWidth = 0; + rects.forEach((r) => { + if (r.x < mergedX) mergedX = r.x; + if (r.y < mergedY) mergedY = r.y; + }); + rects.forEach(r => { + if (r.height > mergedHeight) mergedHeight = (r.height + r.y) - mergedY; + if (r.width > mergedWidth) mergedWidth = (r.width + r.x) - mergedX; + }); + var rect = createCanvas(mergedWidth, mergedHeight); + var ctx = rect.getContext("2d"); + ctx.drawImage(fb, mergedX, mergedY, mergedWidth, mergedHeight, 0, 0, mergedWidth, mergedHeight); + for (const r of rects) { + var id = createImageData(Uint8ClampedArray.from(r.data), r.width, r.height); + ctx.putImageData(id, r.x - mergedX, r.y - mergedY); + } + return { + data: rect, + x: mergedX, + y: mergedY, + } +} \ No newline at end of file diff --git a/src/WSServer.ts b/src/WSServer.ts index 1bb2ad7..2f3c3c1 100644 --- a/src/WSServer.ts +++ b/src/WSServer.ts @@ -11,8 +11,7 @@ import Queue from 'mnemonist/queue.js'; import { createHash } from 'crypto'; import { isIP } from 'net'; import QEMUVM from './QEMUVM.js'; -import Framebuffer from './Framebuffer.js'; -import sharp from 'sharp'; +import { Canvas, createCanvas, CanvasRenderingContext2D } from 'canvas'; export default class WSServer { private Config : IConfig; @@ -37,9 +36,8 @@ export default class WSServer { private voteTimeout : number; // Interval to keep track private voteTimeoutInterval? : NodeJS.Timer; - private ModPerms : number; + private ModPerms : number; private VM : QEMUVM; - private framebuffer : Framebuffer; constructor(config : IConfig, vm : QEMUVM) { this.ChatHistory = new CircularBuffer<{user:string,msg:string}>(Array, 5); this.TurnQueue = new Queue(); @@ -61,10 +59,9 @@ export default class WSServer { }); this.socket.on('connection', (ws : WebSocket, req : http.IncomingMessage) => this.onConnection(ws, req)); var initSize = vm.getSize(); - this.framebuffer = new Framebuffer(); this.newsize(initSize); this.VM = vm; - this.VM.on("dirtyrect", (j, x, y, w, h) => this.newrect(j, x, y, w, h)); + this.VM.on("dirtyrect", (j, x, y) => this.newrect(j, x, y)); this.VM.on("size", (s) => this.newsize(s)); } @@ -172,8 +169,8 @@ export default class WSServer { client.sendMsg(guacutils.encode("connect", "1", "1", "1", "0")); if (this.Config.collabvm.motd) client.sendMsg(guacutils.encode("chat", "", this.Config.collabvm.motd)); if (this.ChatHistory.size !== 0) client.sendMsg(this.getChatHistoryMsg()); - client.sendMsg(guacutils.encode("size", "0", this.framebuffer.size.width.toString(), this.framebuffer.size.height.toString())); - var jpg = await sharp(await this.framebuffer.getFb(), {raw: {height: this.framebuffer.size.height, width: this.framebuffer.size.width, channels: 4}}).jpeg().toBuffer(); + client.sendMsg(guacutils.encode("size", "0", this.VM.framebuffer.width.toString(), this.VM.framebuffer.height.toString())); + var jpg = this.VM.framebuffer.toBuffer("image/jpeg"); var jpg64 = jpg.toString("base64"); client.sendMsg(guacutils.encode("sync", Date.now().toString())); client.sendMsg(guacutils.encode("png", "0", "0", "0", "0", jpg64)); @@ -542,26 +539,25 @@ export default class WSServer { } } - private async newrect(buff : Buffer, x : number, y : number, width : number, height : number) { - var jpg = await sharp(buff, {raw: {height: height, width: width, channels: 4}}).jpeg().toBuffer(); + private async newrect(rect : Canvas, x : number, y : number) { + var jpg = rect.toBuffer("image/jpeg"); var jpg64 = jpg.toString("base64"); this.clients.filter(c => c.connectedToNode).forEach(c => { c.sendMsg(guacutils.encode("sync", Date.now().toString())); c.sendMsg(guacutils.encode("png", "0", "0", x.toString(), y.toString(), jpg64)); }); - this.framebuffer.loadDirtyRect(buff, x, y, width, height); } private newsize(size : {height:number,width:number}) { - this.framebuffer.setSize(size.width, size.height); this.clients.filter(c => c.connectedToNode).forEach(c => c.sendMsg(guacutils.encode("size", "0", size.width.toString(), size.height.toString()))); } getThumbnail() : Promise { return new Promise(async (res, rej) => { - var jpg = await sharp(await this.framebuffer.getFb(), {raw: {height: this.framebuffer.size.height, width: this.framebuffer.size.width, channels: 4}}) - .resize(400, 300, {fit: 'fill'}) - .jpeg().toBuffer(); + var cnv = createCanvas(400, 300); + var ctx = cnv.getContext("2d"); + ctx.drawImage(this.VM.framebuffer, 0, 0, 400, 300); + var jpg = cnv.toBuffer("image/jpeg"); res(jpg.toString("base64")); }) }