Add rect batching and switch to node-canvas. Currently it leaves a LOT of artifacts for some reason

package.json

@@ -14,12 +14,12 @@
     "@types/sharp": "^0.31.1",
     "@types/ws": "^8.5.4",
     "async-mutex": "^0.4.0",
+    "canvas": "^2.11.0",
     "execa": "^6.1.0",
     "fs": "^0.0.1-security",
     "jimp": "^0.16.2",
     "mnemonist": "^0.39.5",
     "rfb2": "github:elijahr2411/node-rfb2",
-    "sharp": "^0.31.3",
     "toml": "^3.0.0",
     "typescript": "^4.9.5",
     "ws": "^8.12.0"

src/Framebuffer.ts

@@ -1,7 +1,7 @@
 import { Mutex } from "async-mutex";
 
 export default class Framebuffer {
-    private fb : Buffer;
+    fb : Buffer;
     private writemutex : Mutex;
     size : {height : number, width : number};
     constructor() {
@@ -15,10 +15,10 @@ export default class Framebuffer {
         this.size.width = w;
         this.fb = Buffer.alloc(size);
     }
-    loadDirtyRect(rect : Buffer, x : number, y : number, width : number, height : number) {
+    loadDirtyRect(rect : Buffer, x : number, y : number, width : number, height : number) : Promise<void> {
         if (this.fb.length < rect.length)
             throw new Error("Dirty rect larger than framebuffer (did you forget to set the size?)");
-        this.writemutex.runExclusive(() => {
+        return this.writemutex.runExclusive(() => {
             return new Promise<void>((res, rej) => {
                 var byteswritten = 0;
                 for (var i = 0; i < height; i++) {

src/QEMUVM.ts

@@ -4,10 +4,15 @@ import * as rfb from 'rfb2';
 import * as fs from 'fs';
 import { execa, ExecaChildProcess } from "execa";
 import QMPClient from "./QMPClient.js";
+import BatchRects from "./RectBatcher.js";
+import { createCanvas, Canvas, CanvasRenderingContext2D, createImageData } from "canvas";
+import { Mutex } from "async-mutex";
 
 export default class QEMUVM extends EventEmitter {
     vnc? : rfb.RfbClient;
     vncPort : number;
+    framebuffer : Canvas;
+    framebufferCtx : CanvasRenderingContext2D;
     qmpSock : string;
     qmpClient : QMPClient;
     qemuCmd : string;
@@ -17,6 +22,9 @@ export default class QEMUVM extends EventEmitter {
     processRestartErrorLevel : number;
     expectedExit : boolean;
     vncOpen : boolean;
+    vncUpdateInterval? : NodeJS.Timer;
+    rects : {height:number,width:number,x:number,y:number,data:Buffer}[];
+    rectMutex : Mutex;
 
     vncReconnectTimeout? : NodeJS.Timer;
     qmpReconnectTimeout? : NodeJS.Timer;
@@ -34,6 +42,10 @@ export default class QEMUVM extends EventEmitter {
         this.qmpErrorLevel = 0;
         this.vncErrorLevel = 0;
         this.vncOpen = true;
+        this.rects = [];
+        this.rectMutex = new Mutex();
+        this.framebuffer = createCanvas(1, 1);
+        this.framebufferCtx = this.framebuffer.getContext("2d");
         this.processRestartErrorLevel = 0;
         this.expectedExit = false;
         this.qmpClient = new QMPClient(this.qmpSock);
@@ -133,23 +145,51 @@ export default class QEMUVM extends EventEmitter {
         this.vncErrorLevel = 0;
         //@ts-ignore
         this.onVNCSize({height: this.vnc.height, width: this.vnc.width});
+        this.vncUpdateInterval = setInterval(() => this.SendRects(), 33);
     }
-    private async onVNCRect(rect : any) {
+    private onVNCRect(rect : any) {
-        var buff = Buffer.alloc(rect.height * rect.width * 4)
+        return this.rectMutex.runExclusive(async () => {
-        var offset = 0;
+            return new Promise<void>(async (res, rej) => {
-        for (var i = 0; i < rect.data.length; i += 4) {
+                var buff = Buffer.alloc(rect.height * rect.width * 4)
-            buff[offset++] = rect.data[i + 2];
+                var offset = 0;
-            buff[offset++] = rect.data[i + 1];
+                for (var i = 0; i < rect.data.length; i += 4) {
-            buff[offset++] = rect.data[i];
+                    buff[offset++] = rect.data[i + 2];
-            buff[offset++] = 255;
+                    buff[offset++] = rect.data[i + 1];
-        }
+                    buff[offset++] = rect.data[i];
-        this.emit("dirtyrect", buff, rect.x, rect.y, rect.width, rect.height);
+                    buff[offset++] = 255;
-        if (!this.vnc) throw new Error();
+                }
-        if (this.vncOpen)
+                var imgdata = createImageData(Uint8ClampedArray.from(buff), rect.width, rect.height);
-            this.vnc.requestUpdate(true, 0, 0, this.vnc.height, this.vnc.width);
+                this.framebufferCtx.putImageData(imgdata, rect.x, rect.y);
+                this.rects.push({
+                    x: rect.x,
+                    y: rect.y,
+                    height: rect.height,
+                    width: rect.width,
+                    data: buff,
+                });
+                if (!this.vnc) throw new Error();
+                if (this.vncOpen)
+                    this.vnc.requestUpdate(true, 0, 0, this.vnc.height, this.vnc.width);
+                res();
+            })
+        });
+    }
+
+    SendRects() {
+        if (!this.vnc || this.rects.length < 1) return;
+        return this.rectMutex.runExclusive(() => {
+            return new Promise<void>(async (res, rej) => {
+                var rect = await BatchRects(this.framebuffer, [...this.rects]);
+                this.rects = [];
+                this.emit('dirtyrect', rect.data, rect.x, rect.y);
+                res();
+            });
+        })
     }
 
     private onVNCSize(size : any) {
+        if (this.framebuffer.height !== size.height) this.framebuffer.height = size.height;
+        if (this.framebuffer.width !== size.width) this.framebuffer.width = size.width;
         this.emit("size", {height: size.height, width: size.width});
     }
 
@@ -174,6 +214,7 @@ export default class QEMUVM extends EventEmitter {
             this.expectedExit = true;
             this.vncOpen = false;
             this.vnc?.end();
+            clearInterval(this.vncUpdateInterval);
             var killTimeout = setTimeout(() => {
                 console.log("Force killing QEMU after 10 seconds of waiting for shutdown");
                 this.qemuProcess?.kill(9);

src/RectBatcher.ts

@@ -0,0 +1,28 @@
+import { Canvas, createCanvas, createImageData } from "canvas";
+
+export default async function BatchRects(fb : Canvas, rects : {height:number,width:number,x:number,y:number,data:Buffer}[]) : Promise<{x:number,y:number,data:Canvas}> {
+    var mergedX = fb.width;
+    var mergedY = fb.height;
+    var mergedHeight = 0;
+    var mergedWidth = 0;
+    rects.forEach((r) => {
+        if (r.x < mergedX) mergedX = r.x;
+        if (r.y < mergedY) mergedY = r.y;
+    });
+    rects.forEach(r => {
+        if (r.height > mergedHeight) mergedHeight = (r.height + r.y) - mergedY;
+        if (r.width > mergedWidth) mergedWidth = (r.width + r.x) - mergedX; 
+    });
+    var rect = createCanvas(mergedWidth, mergedHeight);
+    var ctx = rect.getContext("2d");
+    ctx.drawImage(fb, mergedX, mergedY, mergedWidth, mergedHeight, 0, 0, mergedWidth, mergedHeight);
+    for (const r of rects) {
+        var id = createImageData(Uint8ClampedArray.from(r.data), r.width, r.height);
+        ctx.putImageData(id, r.x - mergedX, r.y - mergedY);
+    }
+    return {
+        data: rect,
+        x: mergedX,
+        y: mergedY,
+    }
+}

src/WSServer.ts

@@ -11,8 +11,7 @@ import Queue from 'mnemonist/queue.js';
 import { createHash } from 'crypto';
 import { isIP } from 'net';
 import QEMUVM from './QEMUVM.js';
-import Framebuffer from './Framebuffer.js';
+import { Canvas, createCanvas, CanvasRenderingContext2D } from 'canvas';
-import sharp from 'sharp';
 
 export default class WSServer {
     private Config : IConfig;
@@ -39,7 +38,6 @@ export default class WSServer {
     private voteTimeoutInterval? : NodeJS.Timer;
     private ModPerms : number;  
     private VM : QEMUVM;
-    private framebuffer : Framebuffer;
     constructor(config : IConfig, vm : QEMUVM) {
         this.ChatHistory = new CircularBuffer<{user:string,msg:string}>(Array, 5);
         this.TurnQueue = new Queue<User>();
@@ -61,10 +59,9 @@ export default class WSServer {
         });
         this.socket.on('connection', (ws : WebSocket, req : http.IncomingMessage) => this.onConnection(ws, req));
         var initSize = vm.getSize();
-        this.framebuffer = new Framebuffer();
         this.newsize(initSize);
         this.VM = vm;
-        this.VM.on("dirtyrect", (j, x, y, w, h) => this.newrect(j, x, y, w, h));
+        this.VM.on("dirtyrect", (j, x, y) => this.newrect(j, x, y));
         this.VM.on("size", (s) => this.newsize(s));
     }
 
@@ -172,8 +169,8 @@ export default class WSServer {
                 client.sendMsg(guacutils.encode("connect", "1", "1", "1", "0"));
                 if (this.Config.collabvm.motd) client.sendMsg(guacutils.encode("chat", "", this.Config.collabvm.motd));
                 if (this.ChatHistory.size !== 0) client.sendMsg(this.getChatHistoryMsg());
-                client.sendMsg(guacutils.encode("size", "0", this.framebuffer.size.width.toString(), this.framebuffer.size.height.toString()));
+                client.sendMsg(guacutils.encode("size", "0", this.VM.framebuffer.width.toString(), this.VM.framebuffer.height.toString()));
-                var jpg = await sharp(await this.framebuffer.getFb(), {raw: {height: this.framebuffer.size.height, width: this.framebuffer.size.width, channels: 4}}).jpeg().toBuffer();
+                var jpg = this.VM.framebuffer.toBuffer("image/jpeg");
                 var jpg64 = jpg.toString("base64");
                 client.sendMsg(guacutils.encode("sync", Date.now().toString()));
                 client.sendMsg(guacutils.encode("png", "0", "0", "0", "0", jpg64));
@@ -542,26 +539,25 @@ export default class WSServer {
         }
     }
 
-    private async newrect(buff : Buffer, x : number, y : number, width : number, height : number) {
+    private async newrect(rect : Canvas, x : number, y : number) {
-        var jpg = await sharp(buff, {raw: {height: height, width: width, channels: 4}}).jpeg().toBuffer();
+        var jpg = rect.toBuffer("image/jpeg");
         var jpg64 = jpg.toString("base64");
         this.clients.filter(c => c.connectedToNode).forEach(c => {
             c.sendMsg(guacutils.encode("sync", Date.now().toString()));
             c.sendMsg(guacutils.encode("png", "0", "0", x.toString(), y.toString(), jpg64));
         });
-        this.framebuffer.loadDirtyRect(buff, x, y, width, height);
     }
 
     private newsize(size : {height:number,width:number}) {
-        this.framebuffer.setSize(size.width, size.height);
         this.clients.filter(c => c.connectedToNode).forEach(c => c.sendMsg(guacutils.encode("size", "0", size.width.toString(), size.height.toString())));
     }
 
     getThumbnail() : Promise<string> {
         return new Promise(async (res, rej) => {
-            var jpg = await sharp(await this.framebuffer.getFb(), {raw: {height: this.framebuffer.size.height, width: this.framebuffer.size.width, channels: 4}})
+            var cnv = createCanvas(400, 300);
-                .resize(400, 300, {fit: 'fill'})
+            var ctx = cnv.getContext("2d");
-                .jpeg().toBuffer();
+            ctx.drawImage(this.VM.framebuffer, 0, 0, 400, 300);
+            var jpg = cnv.toBuffer("image/jpeg");
             res(jpg.toString("base64"));
         })
     }