cvmts: Add support for cgroup process resource limits on Linux

Using systemd's `Delegate=` option, it is possible to get it to let you manage your own cgroup subtree, therefore allowing you to set options and other fun stuff.

This commit adds support for doing so and configuring the resource limits in config.toml.

For later: The cgroup created has to be a threaded one. Iin theory, we can actually wait for the QEMU process to handshake qmp, grab the vCPU threads, and only limit those. For now, just limiting the entire QEMU process works, though and is the least complicated.

NOTE: Windows support should still work, even if you have resource limits configured. If you do, it should only warn and complain, but still function.

config.example.toml

@@ -57,6 +57,17 @@ qemuArgs = "qemu-system-x86_64"
 vncPort = 5900
 snapshots = true
 
+# Resource limits. Only works on Linux, with `Delegate=yes` set in your .service file.
+#
+# cpuUsageMax specifies CPU usage limits in the common top notation, so 200% means 2 CPUs, so on so forth.
+# runOnCpus specifies what CPUs the VM is allowed to run on.
+#
+# Either can be omitted or specified; however, if you want to disable it entirely,
+# it's a better idea to just comment this inline table out, 
+# since the inline table existing is used to enable cgroup support.
+resourceLimits = { cpuUsageMax = 100, runOnCpus = [ 2, 4 ] }
+
+
 # VNC options
 # Only used if vm.type is set to "vncvm"
 [vncvm]

cvmts/src/IConfig.ts

@@ -1,3 +1,4 @@
+import { CgroupLimits } from './vm/qemu_launcher';
 import VNCVMDef from './vm/vnc/VNCVMDef';
 
 export default interface IConfig {
@@ -38,6 +39,7 @@ export default interface IConfig {
 		qemuArgs: string;
 		vncPort: number;
 		snapshots: boolean;
+		resourceLimits?: CgroupLimits
 	};
 	vncvm: VNCVMDef;
 	mysql: MySQLConfig;

cvmts/src/index.ts

@@ -81,7 +81,7 @@ async function start() {
 				vncPort: Config.qemu.vncPort,
 			};
 
-			VM = new QemuVMShim(def);
+			VM = new QemuVMShim(def, Config.qemu.resourceLimits);
 			break;
 		}
 		case 'vncvm': {

cvmts/src/util/cgroup.ts

@@ -0,0 +1,87 @@
+// Cgroup management code
+// this sucks, ill mess with it later
+
+import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+
+export class CGroupController {
+	private controller;
+	private cg: CGroup;
+
+	constructor(controller: string, cg: CGroup) {
+		this.controller = controller;
+		this.cg = cg;
+	}
+
+	WriteValue(key: string, value: string) {
+		writeFileSync(path.join(this.cg.Path(), `${this.controller}.${key}`), value);
+	}
+}
+
+export class CGroup {
+	private path;
+
+	constructor(path: string) {
+		this.path = path;
+	}
+
+	private InitControllers() {
+		// Configure this "root" cgroup to provide cpu and cpuset controllers to the leaf
+		// QEMU cgroups. A bit iffy but whatever.
+		writeFileSync(path.join(this.path, 'cgroup.subtree_control'), '+cpu +cpuset');
+	}
+
+	GetController(controller: string) {
+		return new CGroupController(controller, this);
+	}
+
+	Path(): string {
+		return this.path;
+	}
+
+	HasSubgroup(name: string): boolean {
+		let subgroup_root = path.join(this.path, name);
+		if (existsSync(subgroup_root)) return true;
+		return false;
+	}
+
+	// Gets a CGroup inside of this cgroup.
+	GetSubgroup(name: string): CGroup {
+		// make the subgroup if it doesn't already exist
+		let subgroup_root = path.join(this.path, name);
+		if (!this.HasSubgroup(name)) {
+			mkdirSync(subgroup_root);
+			// We need to make the subgroup threaded before we can attach a process to it.
+			// It's a bit weird, but oh well. Blame linux people, not me.
+			writeFileSync(path.join(subgroup_root, 'cgroup.type'), 'threaded');
+		}
+		return new CGroup(subgroup_root);
+	}
+
+	// Attaches a process to this cgroup.
+	AttachProcess(pid: number) {
+		appendFileSync(path.join(this.path, 'cgroup.procs'), pid.toString());
+	}
+
+	// Returns a CGroup instance for the process' current cgroup, prepared for subgroup usage.
+	// This will only fail if you are not using systemd or elogind,
+	// since even logind user sessions are run inside of a user@[UID] slice.
+	// NOTE: This only supports cgroups2-only systems. Systemd practically enforces that so /shrug
+	static Self(): CGroup {
+		const kCgroupSelfPath = '/proc/self/cgroup';
+		if (!existsSync(kCgroupSelfPath)) throw new Error('This process is not in a CGroup.');
+		let res = readFileSync(kCgroupSelfPath, { encoding: 'utf-8' });
+
+		// Make sure the first/only line is a cgroups2 0::/path/to/cgroup entry.
+		// Legacy cgroups1 is not supported.
+		if (res[0] != '0') throw new Error('CGroup.Self() does not work with cgroups 1 systems. Please do not the cgroups 1.');
+		let cg_path = res.substring(3, res.indexOf('\n'));
+
+		let cg = new CGroup(path.join('/sys/fs/cgroup', cg_path));
+
+		// Do root level group initalization
+		cg.InitControllers();
+
+		return cg;
+	}
+}

cvmts/src/vm/qemu.ts

@@ -4,6 +4,8 @@ import { QemuVM, QemuVmDefinition, VMState } from '@computernewb/superqemu';
 import { VMDisplay } from '../display/interface.js';
 import { VncDisplay } from '../display/vnc.js';
 import pino from 'pino';
+import { CgroupLimits, QemuResourceLimitedLauncher } from './qemu_launcher.js';
+
 
 // shim over superqemu because it diverges from the VM interface
 export class QemuVMShim implements VM {
@@ -11,9 +13,22 @@ export class QemuVMShim implements VM {
 	private display: VncDisplay | null = null;
 	private logger;
 
-	constructor(def: QemuVmDefinition) {
+	constructor(def: QemuVmDefinition, resourceLimits?: CgroupLimits) {
-		this.vm = new QemuVM(def);
 		this.logger = pino({ name: `CVMTS.QemuVMShim/${def.id}` });
+
+		if (resourceLimits) {
+			if (process.platform == 'linux') {
+				this.vm = new QemuVM(def, new QemuResourceLimitedLauncher(def.id, resourceLimits));
+			} else {
+				// Just use the default Superqemu launcher on non-Linux platforms,
+				// .. regardless of if resource control is (somehow) enabled.
+				this.logger.warn({platform: process.platform}, 'Resource control is not supported on this platform. Please remove or comment it out from your configuration.');
+				this.vm = new QemuVM(def);
+			}
+		} else {
+			this.vm = new QemuVM(def);
+		}
+
 	}
 
 	Start(): Promise<void> {

cvmts/src/vm/qemu_launcher.ts

@@ -0,0 +1,104 @@
+import EventEmitter from 'events';
+import { IProcess, IProcessLauncher, ProcessLaunchOptions } from '@computernewb/superqemu';
+import { execaCommand } from 'execa';
+import { Readable, Writable } from 'stream';
+import { CGroup } from '../util/cgroup.js';
+
+export interface CgroupLimits {
+	cpuUsageMax?: number;
+	runOnCpus?: number[];
+}
+
+interface CGroupValue {
+	controller: string;
+	key: string;
+	value: string;
+}
+
+function MakeValuesFromLimits(limits: CgroupLimits): CGroupValue[] {
+	let option_array = [];
+
+	if (limits.cpuUsageMax) {
+		// cpu.max
+		option_array.push({
+			controller: 'cpu',
+			key: 'max',
+			value: `${(limits.cpuUsageMax / 100) * 10000} 10000`
+		});
+	}
+
+	if(limits.runOnCpus) {
+		// Make sure a CPU is not specified more than once. Bit hacky but oh well
+		let unique = [...new Set(limits.runOnCpus)];
+		option_array.push({
+			controller: 'cpuset',
+			key: 'cpus',
+			value: `${unique.join(',')}`
+		});
+	}
+
+	return option_array;
+}
+
+// A process automatically placed in a given cgroup.
+class CGroupLimitedProcess extends EventEmitter implements IProcess {
+	private process;
+	stdin: Writable | null = null;
+	stdout: Readable | null = null;
+	stderr: Readable | null = null;
+	private cgroup: CGroup;
+
+	constructor(cg: CGroup, command: string, opts?: ProcessLaunchOptions) {
+		super();
+		this.cgroup = cg;
+
+		this.process = execaCommand(command, opts);
+
+		this.stdin = this.process.stdin;
+		this.stdout = this.process.stdout;
+		this.stderr = this.process.stderr;
+
+		let self = this;
+		this.process.on('spawn', () => {
+			// it should have one!
+			self.cgroup.AttachProcess(self.process.pid!);
+			self.emit('spawn');
+		});
+
+		this.process.on('exit', (code) => {
+			self.emit('exit', code);
+		});
+	}
+
+	kill(signal?: number | NodeJS.Signals): boolean {
+		return this.process.kill(signal);
+	}
+
+	dispose(): void {
+		this.stdin = null;
+		this.stdout = null;
+		this.stderr = null;
+
+		this.process.removeAllListeners();
+		this.removeAllListeners();
+	}
+}
+
+export class QemuResourceLimitedLauncher implements IProcessLauncher {
+	private group;
+
+	constructor(name: string, limits: CgroupLimits) {
+		let root = CGroup.Self();
+		this.group = root.GetSubgroup(name);
+
+		// Set cgroup keys.
+		for(const val of MakeValuesFromLimits(limits)) {
+			let controller = this.group.GetController(val.controller);
+			controller.WriteValue(val.key, val.value);
+		}
+	}
+
+	launch(command: string, opts?: ProcessLaunchOptions | undefined): IProcess {
+		return new CGroupLimitedProcess(this.group, command, opts);
+	}
+}