From a904f26961ead220969bdb5a326c3f3dbf8d0b8a Mon Sep 17 00:00:00 2001 From: modeco80 Date: Wed, 24 Apr 2024 04:18:05 -0400 Subject: [PATCH] abstract jpeg encoding away from "WSServer" Additionally make thumbnail encoding threadpooled as well, just so it (probably) doesn't block as much. --- cvmts/src/JPEGEncoder.ts | 59 +++++++++++++++++++++++++ cvmts/src/JPEGEncoderWorker.ts | 2 +- cvmts/src/ThumbnailJPEGEncoderWorker.ts | 31 +++++++++++++ cvmts/src/WSServer.ts | 57 +++--------------------- qemu/src/QemuDisplay.ts | 4 ++ 5 files changed, 102 insertions(+), 51 deletions(-) create mode 100644 cvmts/src/JPEGEncoder.ts create mode 100644 cvmts/src/ThumbnailJPEGEncoderWorker.ts diff --git a/cvmts/src/JPEGEncoder.ts b/cvmts/src/JPEGEncoder.ts new file mode 100644 index 0000000..2cc3a6c --- /dev/null +++ b/cvmts/src/JPEGEncoder.ts @@ -0,0 +1,59 @@ +import path from 'node:path'; +import Piscina from 'piscina'; + +import { Size, Rect } from '@cvmts/shared'; + +const kMaxJpegThreads = 4; +const kIdleTimeout = 25000; + +// Thread pool for doing JPEG encoding for rects. +const TheJpegEncoderPool = new Piscina({ + filename: path.join(import.meta.dirname + '/JPEGEncoderWorker.js'), + idleTimeout: kIdleTimeout, + maxThreads: kMaxJpegThreads +}); + +const TheThumbnailEncoderPool = new Piscina({ + filename: path.join(import.meta.dirname + '/ThumbnailJPEGEncoderWorker.js'), + idleTimeout: kIdleTimeout, + maxThreads: kMaxJpegThreads +}); + +// A good balance. TODO: Configurable? +let gJpegQuality = 35; + +export class JPEGEncoder { + + static SetQuality(quality: number) { + gJpegQuality = quality; + } + + static async EncodeJpeg(canvas: Buffer, displaySize: Size, rect: Rect): Promise { + let offset = (rect.y * displaySize.width + rect.x) * 4; + + let res = await TheJpegEncoderPool.run({ + buffer: canvas.subarray(offset), + width: rect.width, + height: rect.height, + stride: displaySize.width, + quality: gJpegQuality + }); + + // TODO: There's probably (definitely) a better way to fix this + if (res == undefined) return Buffer.from([]); + + // have to manually turn it back into a buffer because + // Piscina for some reason turns it into a Uint8Array + return Buffer.from(res); + } + + static async EncodeThumbnail(buffer: Buffer, size: Size) : Promise { + let res = await TheThumbnailEncoderPool.run({ + buffer: buffer, + size: size, + quality: gJpegQuality + }); + + return Buffer.from(res) + } +} diff --git a/cvmts/src/JPEGEncoderWorker.ts b/cvmts/src/JPEGEncoderWorker.ts index c5f93cf..124914b 100644 --- a/cvmts/src/JPEGEncoderWorker.ts +++ b/cvmts/src/JPEGEncoderWorker.ts @@ -9,7 +9,7 @@ export default async (opts: any) => { height: opts.height, subsampling: jpegTurbo.SAMP_422, stride: opts.stride, - quality: opts.quality + quality: opts.quality || 75 }); return Piscina.move(res); diff --git a/cvmts/src/ThumbnailJPEGEncoderWorker.ts b/cvmts/src/ThumbnailJPEGEncoderWorker.ts new file mode 100644 index 0000000..535bfd9 --- /dev/null +++ b/cvmts/src/ThumbnailJPEGEncoderWorker.ts @@ -0,0 +1,31 @@ +import { Size } from '@cvmts/shared'; +import Piscina from 'piscina'; +import sharp from 'sharp'; + +const kThumbnailSize: Size = { + width: 400, + height: 300 +}; + +// this returns appropiate Sharp options to deal with CVMTS raw framebuffers +// (which are RGBA bitmaps, essentially. We probably should abstract that out but +// that'd mean having to introduce that to rfb and oihwekjtgferklds;./tghnredsltg;erhds) +function GetRawSharpOptions(size: Size): sharp.CreateRaw { + return { + width: size.width, + height: size.height, + channels: 4 + }; +} + +export default async (opts: any) => { + let out = await sharp(opts.buffer, { raw: GetRawSharpOptions(opts.size) }) + .resize(kThumbnailSize.width, kThumbnailSize.height, { fit: 'fill' }) + .jpeg({ + quality: opts.quality || 75 + }) + .toFormat('jpeg') + .toBuffer(); + + return Piscina.move(out); +}; diff --git a/cvmts/src/WSServer.ts b/cvmts/src/WSServer.ts index ddc3105..78c6d3d 100644 --- a/cvmts/src/WSServer.ts +++ b/cvmts/src/WSServer.ts @@ -17,8 +17,7 @@ import path from 'node:path'; import AuthManager from './AuthManager.js'; import { Size, Rect, Logger } from '@cvmts/shared'; -import sharp from 'sharp'; -import Piscina from 'piscina'; +import { JPEGEncoder } from './JPEGEncoder.js'; // Instead of strange hacks we can just use nodejs provided // import.meta properties, which have existed since LTS if not before @@ -37,43 +36,8 @@ type VoteTally = { no: number; }; -// A good balance. TODO: Configurable? -const kJpegQuality = 35; -// this returns appropiate Sharp options to deal with the framebuffer -function GetRawSharpOptions(size: Size): sharp.CreateRaw { - return { - width: size.width, - height: size.height, - channels: 4 - }; -} -// Thread pool for doing JPEG encoding for rects. -const TheJpegEncoderPool = new Piscina({ - filename: path.join(import.meta.dirname + '/JPEGEncoderWorker.js'), - minThreads: 4, - maxThreads: 4 -}); - -async function EncodeJpeg(canvas: Buffer, displaySize: Size, rect: Rect): Promise { - let offset = (rect.y * displaySize.width + rect.x) * 4; - - let res = await TheJpegEncoderPool.run({ - buffer: canvas.subarray(offset), - width: rect.width, - height: rect.height, - stride: displaySize.width, - quality: kJpegQuality - }); - - // TODO: There's probably (definitely) a better way to fix this - if (res == undefined) return Buffer.from([]); - - // have to manually turn it back into a buffer because - // Piscina for some reason turns it into a Uint8Array - return Buffer.from(res); -} export default class WSServer { private Config: IConfig; @@ -930,24 +894,17 @@ export default class WSServer { let display = this.VM.GetDisplay(); let displaySize = display.Size(); - let encoded = await EncodeJpeg(display.Buffer(), displaySize, rect); + let encoded = await JPEGEncoder.EncodeJpeg(display.Buffer(), displaySize, rect); return encoded.toString('base64'); } - getThumbnail(): Promise { - return new Promise(async (res, rej) => { - let display = this.VM.GetDisplay(); - if (display == null) return; + async getThumbnail(): Promise { + let display = this.VM.GetDisplay(); + if (!display.Connected()) throw new Error('VM display is not connected'); - // TODO: pass custom options to Sharp.resize() probably - let out = await sharp(display.Buffer(), { raw: GetRawSharpOptions(display.Size()) }) - .resize(400, 300, { fit: 'fill' }) - .toFormat('jpeg') - .toBuffer(); - - res(out.toString('base64')); - }); + let buf = await JPEGEncoder.EncodeThumbnail(display.Buffer(), display.Size()); + return buf.toString('base64'); } startVote() { diff --git a/qemu/src/QemuDisplay.ts b/qemu/src/QemuDisplay.ts index cb4e9d4..efcfd47 100644 --- a/qemu/src/QemuDisplay.ts +++ b/qemu/src/QemuDisplay.ts @@ -116,6 +116,10 @@ export class QemuDisplay extends EventEmitter { this.displayVnc.disconnect(); } + Connected() { + return this.displayVnc.connected; + } + Buffer(): Buffer { return this.displayVnc.fb; }