From 86f1345a2dee4cb4235d6cb774a1537298eae30e Mon Sep 17 00:00:00 2001 From: modeco80 Date: Sat, 2 Nov 2024 07:46:59 -0400 Subject: [PATCH] cvmts: Only target QEMU vCPU threads by default Previous behaviour was to limit the whole QEMU process; this only limits the vCPU threads. A bit (very) hacky how I did this but it does work. --- config.example.toml | 3 +++ cvmts/src/util/cgroup.ts | 6 ++++++ cvmts/src/vm/qemu.ts | 27 +++++++++++++++++++++++---- cvmts/src/vm/qemu_launcher.ts | 20 +++++++++++++++----- 4 files changed, 47 insertions(+), 9 deletions(-) diff --git a/config.example.toml b/config.example.toml index 0a57363..48098d1 100644 --- a/config.example.toml +++ b/config.example.toml @@ -61,6 +61,9 @@ snapshots = true # # cpuUsageMax specifies CPU usage limits in the common top notation, so 200% means 2 CPUs, so on so forth. # runOnCpus specifies what CPUs the VM is allowed to run on. +# limitProcess is optional (default false) and determines if only qemu vCPU threads are put into the cgroup, +# or the entire QEMU process (incl. all its threads). This is rarely what you want, so the example does not +# provide it. # # Either can be omitted or specified; however, if you want to disable it entirely, # it's a better idea to just comment this inline table out, diff --git a/cvmts/src/util/cgroup.ts b/cvmts/src/util/cgroup.ts index e6d1eba..a1514af 100644 --- a/cvmts/src/util/cgroup.ts +++ b/cvmts/src/util/cgroup.ts @@ -29,6 +29,7 @@ export class CGroup { // Configure this "root" cgroup to provide cpu and cpuset controllers to the leaf // QEMU cgroups. A bit iffy but whatever. writeFileSync(path.join(this.path, 'cgroup.subtree_control'), '+cpu +cpuset'); + //writeFileSync(path.join(this.path, 'cgroup.subtree_control'), '+cpu'); } GetController(controller: string) { @@ -63,6 +64,11 @@ export class CGroup { appendFileSync(path.join(this.path, 'cgroup.procs'), pid.toString()); } + // Attaches a thread to this cgroup. (The CGroup is a threaded one. See above) + AttachThread(tid: number) { + appendFileSync(path.join(this.path, 'cgroup.threads'), tid.toString()); + } + // Returns a CGroup instance for the process' current cgroup, prepared for subgroup usage. // This will only fail if you are not using systemd or elogind, // since even logind user sessions are run inside of a user@[UID] slice. diff --git a/cvmts/src/vm/qemu.ts b/cvmts/src/vm/qemu.ts index 2d56343..3e2c4f7 100644 --- a/cvmts/src/vm/qemu.ts +++ b/cvmts/src/vm/qemu.ts @@ -6,29 +6,31 @@ import { VncDisplay } from '../display/vnc.js'; import pino from 'pino'; import { CgroupLimits, QemuResourceLimitedLauncher } from './qemu_launcher.js'; - // shim over superqemu because it diverges from the VM interface export class QemuVMShim implements VM { private vm; private display: VncDisplay | null = null; private logger; + private cg_launcher: QemuResourceLimitedLauncher | null = null; + private resource_limits: CgroupLimits | null = null; constructor(def: QemuVmDefinition, resourceLimits?: CgroupLimits) { this.logger = pino({ name: `CVMTS.QemuVMShim/${def.id}` }); if (resourceLimits) { if (process.platform == 'linux') { - this.vm = new QemuVM(def, new QemuResourceLimitedLauncher(def.id, resourceLimits)); + this.resource_limits = resourceLimits; + this.cg_launcher = new QemuResourceLimitedLauncher(def.id, resourceLimits); + this.vm = new QemuVM(def, this.cg_launcher); } else { // Just use the default Superqemu launcher on non-Linux platforms, // .. regardless of if resource control is (somehow) enabled. - this.logger.warn({platform: process.platform}, 'Resource control is not supported on this platform. Please remove or comment it out from your configuration.'); + this.logger.warn({ platform: process.platform }, 'Resource control is not supported on this platform. Please remove or comment it out from your configuration.'); this.vm = new QemuVM(def); } } else { this.vm = new QemuVM(def); } - } Start(): Promise { @@ -54,7 +56,24 @@ export class QemuVMShim implements VM { return this.vm.MonitorCommand(command); } + async PlaceVCPUThreadsIntoCGroup() { + if (this.cg_launcher) { + if (!this.resource_limits?.limitProcess) { + // Get all vCPUs and pin them to the CGroup. + let cpu_res = await this.vm.QmpCommand('query-cpus-fast', {}); + for (let cpu of cpu_res) { + this.logger.info(`Placing vCPU thread with TID ${cpu['thread-id']} to cgroup`); + this.cg_launcher.group.AttachThread(cpu['thread-id']); + } + } + } + } + StartDisplay(): void { + // HACK: We should probably use another subscribed eventemitter for this. For now, + // this "works". I guess. + this.PlaceVCPUThreadsIntoCGroup(); + // boot it up let info = this.vm.GetDisplayInfo(); diff --git a/cvmts/src/vm/qemu_launcher.ts b/cvmts/src/vm/qemu_launcher.ts index 5ef3aa0..30648ce 100644 --- a/cvmts/src/vm/qemu_launcher.ts +++ b/cvmts/src/vm/qemu_launcher.ts @@ -7,6 +7,7 @@ import { CGroup } from '../util/cgroup.js'; export interface CgroupLimits { cpuUsageMax?: number; runOnCpus?: number[]; + limitProcess?: boolean; } interface CGroupValue { @@ -47,10 +48,15 @@ class CGroupLimitedProcess extends EventEmitter implements IProcess { stdout: Readable | null = null; stderr: Readable | null = null; private cgroup: CGroup; + private limits; - constructor(cg: CGroup, command: string, opts?: ProcessLaunchOptions) { + constructor(cg: CGroup, limits: CgroupLimits, command: string, opts?: ProcessLaunchOptions) { super(); this.cgroup = cg; + this.limits = limits; + + if(!this.limits.limitProcess) + this.limits.limitProcess = false; this.process = execaCommand(command, opts); @@ -60,8 +66,10 @@ class CGroupLimitedProcess extends EventEmitter implements IProcess { let self = this; this.process.on('spawn', () => { - // it should have one! - self.cgroup.AttachProcess(self.process.pid!); + if(self.limits.limitProcess) { + // it should have one! + self.cgroup.AttachProcess(self.process.pid!); + } self.emit('spawn'); }); @@ -85,11 +93,13 @@ class CGroupLimitedProcess extends EventEmitter implements IProcess { } export class QemuResourceLimitedLauncher implements IProcessLauncher { - private group; + public group; + private limits; constructor(name: string, limits: CgroupLimits) { let root = CGroup.Self(); this.group = root.GetSubgroup(name); + this.limits = limits; // Set cgroup keys. for(const val of MakeValuesFromLimits(limits)) { @@ -99,6 +109,6 @@ export class QemuResourceLimitedLauncher implements IProcessLauncher { } launch(command: string, opts?: ProcessLaunchOptions | undefined): IProcess { - return new CGroupLimitedProcess(this.group, command, opts); + return new CGroupLimitedProcess(this.group, this.limits, command, opts); } }