cthrace/collabvm-1.2.ts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add rect batching and switch to node-canvas. Currently it leaves a LOT of artifacts for some reason

Browse Source
This commit is contained in:
elijahr2411
2023-02-07 20:51:25 -05:00
parent 63b641ec58
commit caf76a978e
5 changed files with 97 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@@ -14,12 +14,12 @@
"@types/sharp": "^0.31.1",
"@types/ws": "^8.5.4",
"async-mutex": "^0.4.0",
"canvas": "^2.11.0",
"execa": "^6.1.0",
"fs": "^0.0.1-security",
"jimp": "^0.16.2",
"mnemonist": "^0.39.5",
"rfb2": "github:elijahr2411/node-rfb2",
"sharp": "^0.31.3",
"toml": "^3.0.0",
"typescript": "^4.9.5",
"ws": "^8.12.0"

6
src/Framebuffer.ts
View File

@@ -1,7 +1,7 @@
import { Mutex } from "async-mutex";
export default class Framebuffer {
private fb : Buffer;
fb : Buffer;
private writemutex : Mutex;
size : {height : number, width : number};
constructor() {
@@ -15,10 +15,10 @@ export default class Framebuffer {
this.size.width = w;
this.fb = Buffer.alloc(size);
}
loadDirtyRect(rect : Buffer, x : number, y : number, width : number, height : number) {
loadDirtyRect(rect : Buffer, x : number, y : number, width : number, height : number) : Promise<void> {
if (this.fb.length < rect.length)
throw new Error("Dirty rect larger than framebuffer (did you forget to set the size?)");
this.writemutex.runExclusive(() => {
return this.writemutex.runExclusive(() => {
return new Promise<void>((res, rej) => {
var byteswritten = 0;
for (var i = 0; i < height; i++) {

67
src/QEMUVM.ts
View File

@@ -4,10 +4,15 @@ import * as rfb from 'rfb2';
import * as fs from 'fs';
import { execa, ExecaChildProcess } from "execa";
import QMPClient from "./QMPClient.js";
import BatchRects from "./RectBatcher.js";
import { createCanvas, Canvas, CanvasRenderingContext2D, createImageData } from "canvas";
import { Mutex } from "async-mutex";
export default class QEMUVM extends EventEmitter {
vnc? : rfb.RfbClient;
vncPort : number;
framebuffer : Canvas;
framebufferCtx : CanvasRenderingContext2D;
qmpSock : string;
qmpClient : QMPClient;
qemuCmd : string;
@@ -17,6 +22,9 @@ export default class QEMUVM extends EventEmitter {
processRestartErrorLevel : number;
expectedExit : boolean;
vncOpen : boolean;
vncUpdateInterval? : NodeJS.Timer;
rects : {height:number,width:number,x:number,y:number,data:Buffer}[];
rectMutex : Mutex;
vncReconnectTimeout? : NodeJS.Timer;
qmpReconnectTimeout? : NodeJS.Timer;
@@ -34,6 +42,10 @@ export default class QEMUVM extends EventEmitter {
this.qmpErrorLevel = 0;
this.vncErrorLevel = 0;
this.vncOpen = true;
this.rects = [];
this.rectMutex = new Mutex();
this.framebuffer = createCanvas(1, 1);
this.framebufferCtx = this.framebuffer.getContext("2d");
this.processRestartErrorLevel = 0;
this.expectedExit = false;
this.qmpClient = new QMPClient(this.qmpSock);
@@ -133,23 +145,51 @@ export default class QEMUVM extends EventEmitter {
this.vncErrorLevel = 0;
//@ts-ignore
this.onVNCSize({height: this.vnc.height, width: this.vnc.width});
this.vncUpdateInterval = setInterval(() => this.SendRects(), 33);
}
private async onVNCRect(rect : any) {
var buff = Buffer.alloc(rect.height * rect.width * 4)
var offset = 0;
for (var i = 0; i < rect.data.length; i += 4) {
buff[offset++] = rect.data[i + 2];
buff[offset++] = rect.data[i + 1];
buff[offset++] = rect.data[i];
buff[offset++] = 255;
}
this.emit("dirtyrect", buff, rect.x, rect.y, rect.width, rect.height);
if (!this.vnc) throw new Error();
if (this.vncOpen)
this.vnc.requestUpdate(true, 0, 0, this.vnc.height, this.vnc.width);
private onVNCRect(rect : any) {
return this.rectMutex.runExclusive(async () => {
return new Promise<void>(async (res, rej) => {
var buff = Buffer.alloc(rect.height * rect.width * 4)
var offset = 0;
for (var i = 0; i < rect.data.length; i += 4) {
buff[offset++] = rect.data[i + 2];
buff[offset++] = rect.data[i + 1];
buff[offset++] = rect.data[i];
buff[offset++] = 255;
}
var imgdata = createImageData(Uint8ClampedArray.from(buff), rect.width, rect.height);
this.framebufferCtx.putImageData(imgdata, rect.x, rect.y);
this.rects.push({
x: rect.x,
y: rect.y,
height: rect.height,
width: rect.width,
data: buff,
});
if (!this.vnc) throw new Error();
if (this.vncOpen)
this.vnc.requestUpdate(true, 0, 0, this.vnc.height, this.vnc.width);
res();
})
});
}
SendRects() {
if (!this.vnc || this.rects.length < 1) return;
return this.rectMutex.runExclusive(() => {
return new Promise<void>(async (res, rej) => {
var rect = await BatchRects(this.framebuffer, [...this.rects]);
this.rects = [];
this.emit('dirtyrect', rect.data, rect.x, rect.y);
res();
});
})
}
private onVNCSize(size : any) {
if (this.framebuffer.height !== size.height) this.framebuffer.height = size.height;
if (this.framebuffer.width !== size.width) this.framebuffer.width = size.width;
this.emit("size", {height: size.height, width: size.width});
}
@@ -174,6 +214,7 @@ export default class QEMUVM extends EventEmitter {
this.expectedExit = true;
this.vncOpen = false;
this.vnc?.end();
clearInterval(this.vncUpdateInterval);
var killTimeout = setTimeout(() => {
console.log("Force killing QEMU after 10 seconds of waiting for shutdown");
this.qemuProcess?.kill(9);

28
src/RectBatcher.ts Normal file
View File

@@ -0,0 +1,28 @@
import { Canvas, createCanvas, createImageData } from "canvas";
export default async function BatchRects(fb : Canvas, rects : {height:number,width:number,x:number,y:number,data:Buffer}[]) : Promise<{x:number,y:number,data:Canvas}> {
var mergedX = fb.width;
var mergedY = fb.height;
var mergedHeight = 0;
var mergedWidth = 0;
rects.forEach((r) => {
if (r.x < mergedX) mergedX = r.x;
if (r.y < mergedY) mergedY = r.y;
});
rects.forEach(r => {
if (r.height > mergedHeight) mergedHeight = (r.height + r.y) - mergedY;
if (r.width > mergedWidth) mergedWidth = (r.width + r.x) - mergedX;
});
var rect = createCanvas(mergedWidth, mergedHeight);
var ctx = rect.getContext("2d");
ctx.drawImage(fb, mergedX, mergedY, mergedWidth, mergedHeight, 0, 0, mergedWidth, mergedHeight);
for (const r of rects) {
var id = createImageData(Uint8ClampedArray.from(r.data), r.width, r.height);
ctx.putImageData(id, r.x - mergedX, r.y - mergedY);
}
return {
data: rect,
x: mergedX,
y: mergedY,
}
}

26
src/WSServer.ts
View File

@@ -11,8 +11,7 @@ import Queue from 'mnemonist/queue.js';
import { createHash } from 'crypto';
import { isIP } from 'net';
import QEMUVM from './QEMUVM.js';
import Framebuffer from './Framebuffer.js';
import sharp from 'sharp';
import { Canvas, createCanvas, CanvasRenderingContext2D } from 'canvas';
export default class WSServer {
private Config : IConfig;
@@ -37,9 +36,8 @@ export default class WSServer {
private voteTimeout : number;
// Interval to keep track
private voteTimeoutInterval? : NodeJS.Timer;
private ModPerms : number;
private ModPerms : number;
private VM : QEMUVM;
private framebuffer : Framebuffer;
constructor(config : IConfig, vm : QEMUVM) {
this.ChatHistory = new CircularBuffer<{user:string,msg:string}>(Array, 5);
this.TurnQueue = new Queue<User>();
@@ -61,10 +59,9 @@ export default class WSServer {
});
this.socket.on('connection', (ws : WebSocket, req : http.IncomingMessage) => this.onConnection(ws, req));
var initSize = vm.getSize();
this.framebuffer = new Framebuffer();
this.newsize(initSize);
this.VM = vm;
this.VM.on("dirtyrect", (j, x, y, w, h) => this.newrect(j, x, y, w, h));
this.VM.on("dirtyrect", (j, x, y) => this.newrect(j, x, y));
this.VM.on("size", (s) => this.newsize(s));
}
@@ -172,8 +169,8 @@ export default class WSServer {
client.sendMsg(guacutils.encode("connect", "1", "1", "1", "0"));
if (this.Config.collabvm.motd) client.sendMsg(guacutils.encode("chat", "", this.Config.collabvm.motd));
if (this.ChatHistory.size !== 0) client.sendMsg(this.getChatHistoryMsg());
client.sendMsg(guacutils.encode("size", "0", this.framebuffer.size.width.toString(), this.framebuffer.size.height.toString()));
var jpg = await sharp(await this.framebuffer.getFb(), {raw: {height: this.framebuffer.size.height, width: this.framebuffer.size.width, channels: 4}}).jpeg().toBuffer();
client.sendMsg(guacutils.encode("size", "0", this.VM.framebuffer.width.toString(), this.VM.framebuffer.height.toString()));
var jpg = this.VM.framebuffer.toBuffer("image/jpeg");
var jpg64 = jpg.toString("base64");
client.sendMsg(guacutils.encode("sync", Date.now().toString()));
client.sendMsg(guacutils.encode("png", "0", "0", "0", "0", jpg64));
@@ -542,26 +539,25 @@ export default class WSServer {
}
}
private async newrect(buff : Buffer, x : number, y : number, width : number, height : number) {
var jpg = await sharp(buff, {raw: {height: height, width: width, channels: 4}}).jpeg().toBuffer();
private async newrect(rect : Canvas, x : number, y : number) {
var jpg = rect.toBuffer("image/jpeg");
var jpg64 = jpg.toString("base64");
this.clients.filter(c => c.connectedToNode).forEach(c => {
c.sendMsg(guacutils.encode("sync", Date.now().toString()));
c.sendMsg(guacutils.encode("png", "0", "0", x.toString(), y.toString(), jpg64));
});
this.framebuffer.loadDirtyRect(buff, x, y, width, height);
}
private newsize(size : {height:number,width:number}) {
this.framebuffer.setSize(size.width, size.height);
this.clients.filter(c => c.connectedToNode).forEach(c => c.sendMsg(guacutils.encode("size", "0", size.width.toString(), size.height.toString())));
}
getThumbnail() : Promise<string> {
return new Promise(async (res, rej) => {
var jpg = await sharp(await this.framebuffer.getFb(), {raw: {height: this.framebuffer.size.height, width: this.framebuffer.size.width, channels: 4}})
.resize(400, 300, {fit: 'fill'})
.jpeg().toBuffer();
var cnv = createCanvas(400, 300);
var ctx = cnv.getContext("2d");
ctx.drawImage(this.VM.framebuffer, 0, 0, 400, 300);
var jpg = cnv.toBuffer("image/jpeg");
res(jpg.toString("base64"));
})
}