cvmts: Only target QEMU vCPU threads by default

Previous behaviour was to limit the whole QEMU process; this only limits the vCPU threads. A bit (very) hacky how I did this but it does work.

config.example.toml

@@ -61,6 +61,9 @@ snapshots = true
 #
 # cpuUsageMax specifies CPU usage limits in the common top notation, so 200% means 2 CPUs, so on so forth.
 # runOnCpus specifies what CPUs the VM is allowed to run on.
+# limitProcess is optional (default false) and determines if only qemu vCPU threads are put into the cgroup,
+# or the entire QEMU process (incl. all its threads). This is rarely what you want, so the example does not
+# provide it.
 #
 # Either can be omitted or specified; however, if you want to disable it entirely,
 # it's a better idea to just comment this inline table out, 

cvmts/src/util/cgroup.ts

@@ -29,6 +29,7 @@ export class CGroup {
 		// Configure this "root" cgroup to provide cpu and cpuset controllers to the leaf
 		// QEMU cgroups. A bit iffy but whatever.
 		writeFileSync(path.join(this.path, 'cgroup.subtree_control'), '+cpu +cpuset');
+		//writeFileSync(path.join(this.path, 'cgroup.subtree_control'), '+cpu');
 	}
 
 	GetController(controller: string) {
@@ -63,6 +64,11 @@ export class CGroup {
 		appendFileSync(path.join(this.path, 'cgroup.procs'), pid.toString());
 	}
 
+	// Attaches a thread to this cgroup. (The CGroup is a threaded one. See above)
+	AttachThread(tid: number) {
+		appendFileSync(path.join(this.path, 'cgroup.threads'), tid.toString());
+	}
+
 	// Returns a CGroup instance for the process' current cgroup, prepared for subgroup usage.
 	// This will only fail if you are not using systemd or elogind,
 	// since even logind user sessions are run inside of a user@[UID] slice.

cvmts/src/vm/qemu.ts

@@ -6,29 +6,31 @@ import { VncDisplay } from '../display/vnc.js';
 import pino from 'pino';
 import { CgroupLimits, QemuResourceLimitedLauncher } from './qemu_launcher.js';
 
-
 // shim over superqemu because it diverges from the VM interface
 export class QemuVMShim implements VM {
 	private vm;
 	private display: VncDisplay | null = null;
 	private logger;
+	private cg_launcher: QemuResourceLimitedLauncher | null = null;
+	private resource_limits: CgroupLimits | null = null;
 
 	constructor(def: QemuVmDefinition, resourceLimits?: CgroupLimits) {
 		this.logger = pino({ name: `CVMTS.QemuVMShim/${def.id}` });
 
 		if (resourceLimits) {
 			if (process.platform == 'linux') {
-				this.vm = new QemuVM(def, new QemuResourceLimitedLauncher(def.id, resourceLimits));
+				this.resource_limits = resourceLimits;
+				this.cg_launcher = new QemuResourceLimitedLauncher(def.id, resourceLimits);
+				this.vm = new QemuVM(def, this.cg_launcher);
 			} else {
 				// Just use the default Superqemu launcher on non-Linux platforms,
 				// .. regardless of if resource control is (somehow) enabled.
-				this.logger.warn({platform: process.platform}, 'Resource control is not supported on this platform. Please remove or comment it out from your configuration.');
+				this.logger.warn({ platform: process.platform }, 'Resource control is not supported on this platform. Please remove or comment it out from your configuration.');
 				this.vm = new QemuVM(def);
 			}
 		} else {
 			this.vm = new QemuVM(def);
 		}
-
 	}
 
 	Start(): Promise<void> {
@@ -54,7 +56,24 @@ export class QemuVMShim implements VM {
 		return this.vm.MonitorCommand(command);
 	}
 
+	async PlaceVCPUThreadsIntoCGroup() {
+		if (this.cg_launcher) {
+			if (!this.resource_limits?.limitProcess) {
+				// Get all vCPUs and pin them to the CGroup.
+				let cpu_res = await this.vm.QmpCommand('query-cpus-fast', {});
+				for (let cpu of cpu_res) {
+					this.logger.info(`Placing vCPU thread with TID ${cpu['thread-id']} to cgroup`);
+					this.cg_launcher.group.AttachThread(cpu['thread-id']);
+				}
+			}
+		}
+	}
+
 	StartDisplay(): void {
+		// HACK: We should probably use another subscribed eventemitter for this. For now,
+		// this "works". I guess.
+		this.PlaceVCPUThreadsIntoCGroup();
+
 		// boot it up
 		let info = this.vm.GetDisplayInfo();
 

cvmts/src/vm/qemu_launcher.ts

@@ -7,6 +7,7 @@ import { CGroup } from '../util/cgroup.js';
 export interface CgroupLimits {
 	cpuUsageMax?: number;
 	runOnCpus?: number[];
+	limitProcess?: boolean;
 }
 
 interface CGroupValue {
@@ -47,10 +48,15 @@ class CGroupLimitedProcess extends EventEmitter implements IProcess {
 	stdout: Readable | null = null;
 	stderr: Readable | null = null;
 	private cgroup: CGroup;
+	private limits;
 
-	constructor(cg: CGroup, command: string, opts?: ProcessLaunchOptions) {
+	constructor(cg: CGroup, limits: CgroupLimits, command: string, opts?: ProcessLaunchOptions) {
 		super();
 		this.cgroup = cg;
+		this.limits = limits;
+
+		if(!this.limits.limitProcess)
+			this.limits.limitProcess = false;
 
 		this.process = execaCommand(command, opts);
 
@@ -60,8 +66,10 @@ class CGroupLimitedProcess extends EventEmitter implements IProcess {
 
 		let self = this;
 		this.process.on('spawn', () => {
+			if(self.limits.limitProcess) {
 				// it should have one!
 				self.cgroup.AttachProcess(self.process.pid!);
+			}
 			self.emit('spawn');
 		});
 
@@ -85,11 +93,13 @@ class CGroupLimitedProcess extends EventEmitter implements IProcess {
 }
 
 export class QemuResourceLimitedLauncher implements IProcessLauncher {
-	private group;
+	public group;
+	private limits;
 
 	constructor(name: string, limits: CgroupLimits) {
 		let root = CGroup.Self();
 		this.group = root.GetSubgroup(name);
+		this.limits = limits;
 
 		// Set cgroup keys.
 		for(const val of MakeValuesFromLimits(limits)) {
@@ -99,6 +109,6 @@ export class QemuResourceLimitedLauncher implements IProcessLauncher {
 	}
 
 	launch(command: string, opts?: ProcessLaunchOptions | undefined): IProcess {
-		return new CGroupLimitedProcess(this.group, command, opts);
+		return new CGroupLimitedProcess(this.group, this.limits, command, opts);
 	}
 }