diff --git a/config.example.toml b/config.example.toml index d0df6f8..0a57363 100644 --- a/config.example.toml +++ b/config.example.toml @@ -57,6 +57,17 @@ qemuArgs = "qemu-system-x86_64" vncPort = 5900 snapshots = true +# Resource limits. Only works on Linux, with `Delegate=yes` set in your .service file. +# +# cpuUsageMax specifies CPU usage limits in the common top notation, so 200% means 2 CPUs, so on so forth. +# runOnCpus specifies what CPUs the VM is allowed to run on. +# +# Either can be omitted or specified; however, if you want to disable it entirely, +# it's a better idea to just comment this inline table out, +# since the inline table existing is used to enable cgroup support. +resourceLimits = { cpuUsageMax = 100, runOnCpus = [ 2, 4 ] } + + # VNC options # Only used if vm.type is set to "vncvm" [vncvm] diff --git a/cvmts/src/IConfig.ts b/cvmts/src/IConfig.ts index 2217718..667c509 100644 --- a/cvmts/src/IConfig.ts +++ b/cvmts/src/IConfig.ts @@ -1,3 +1,4 @@ +import { CgroupLimits } from './vm/qemu_launcher'; import VNCVMDef from './vm/vnc/VNCVMDef'; export default interface IConfig { @@ -38,6 +39,7 @@ export default interface IConfig { qemuArgs: string; vncPort: number; snapshots: boolean; + resourceLimits?: CgroupLimits }; vncvm: VNCVMDef; mysql: MySQLConfig; diff --git a/cvmts/src/index.ts b/cvmts/src/index.ts index 47c8c51..720ec80 100644 --- a/cvmts/src/index.ts +++ b/cvmts/src/index.ts @@ -81,7 +81,7 @@ async function start() { vncPort: Config.qemu.vncPort, }; - VM = new QemuVMShim(def); + VM = new QemuVMShim(def, Config.qemu.resourceLimits); break; } case 'vncvm': { diff --git a/cvmts/src/util/cgroup.ts b/cvmts/src/util/cgroup.ts new file mode 100644 index 0000000..e6d1eba --- /dev/null +++ b/cvmts/src/util/cgroup.ts @@ -0,0 +1,87 @@ +// Cgroup management code +// this sucks, ill mess with it later + +import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; + +export class CGroupController { + private controller; + private cg: CGroup; + + constructor(controller: string, cg: CGroup) { + this.controller = controller; + this.cg = cg; + } + + WriteValue(key: string, value: string) { + writeFileSync(path.join(this.cg.Path(), `${this.controller}.${key}`), value); + } +} + +export class CGroup { + private path; + + constructor(path: string) { + this.path = path; + } + + private InitControllers() { + // Configure this "root" cgroup to provide cpu and cpuset controllers to the leaf + // QEMU cgroups. A bit iffy but whatever. + writeFileSync(path.join(this.path, 'cgroup.subtree_control'), '+cpu +cpuset'); + } + + GetController(controller: string) { + return new CGroupController(controller, this); + } + + Path(): string { + return this.path; + } + + HasSubgroup(name: string): boolean { + let subgroup_root = path.join(this.path, name); + if (existsSync(subgroup_root)) return true; + return false; + } + + // Gets a CGroup inside of this cgroup. + GetSubgroup(name: string): CGroup { + // make the subgroup if it doesn't already exist + let subgroup_root = path.join(this.path, name); + if (!this.HasSubgroup(name)) { + mkdirSync(subgroup_root); + // We need to make the subgroup threaded before we can attach a process to it. + // It's a bit weird, but oh well. Blame linux people, not me. + writeFileSync(path.join(subgroup_root, 'cgroup.type'), 'threaded'); + } + return new CGroup(subgroup_root); + } + + // Attaches a process to this cgroup. + AttachProcess(pid: number) { + appendFileSync(path.join(this.path, 'cgroup.procs'), pid.toString()); + } + + // Returns a CGroup instance for the process' current cgroup, prepared for subgroup usage. + // This will only fail if you are not using systemd or elogind, + // since even logind user sessions are run inside of a user@[UID] slice. + // NOTE: This only supports cgroups2-only systems. Systemd practically enforces that so /shrug + static Self(): CGroup { + const kCgroupSelfPath = '/proc/self/cgroup'; + if (!existsSync(kCgroupSelfPath)) throw new Error('This process is not in a CGroup.'); + let res = readFileSync(kCgroupSelfPath, { encoding: 'utf-8' }); + + // Make sure the first/only line is a cgroups2 0::/path/to/cgroup entry. + // Legacy cgroups1 is not supported. + if (res[0] != '0') throw new Error('CGroup.Self() does not work with cgroups 1 systems. Please do not the cgroups 1.'); + let cg_path = res.substring(3, res.indexOf('\n')); + + let cg = new CGroup(path.join('/sys/fs/cgroup', cg_path)); + + // Do root level group initalization + cg.InitControllers(); + + return cg; + } +} diff --git a/cvmts/src/vm/qemu.ts b/cvmts/src/vm/qemu.ts index 38eab09..2d56343 100644 --- a/cvmts/src/vm/qemu.ts +++ b/cvmts/src/vm/qemu.ts @@ -4,6 +4,8 @@ import { QemuVM, QemuVmDefinition, VMState } from '@computernewb/superqemu'; import { VMDisplay } from '../display/interface.js'; import { VncDisplay } from '../display/vnc.js'; import pino from 'pino'; +import { CgroupLimits, QemuResourceLimitedLauncher } from './qemu_launcher.js'; + // shim over superqemu because it diverges from the VM interface export class QemuVMShim implements VM { @@ -11,9 +13,22 @@ export class QemuVMShim implements VM { private display: VncDisplay | null = null; private logger; - constructor(def: QemuVmDefinition) { - this.vm = new QemuVM(def); + constructor(def: QemuVmDefinition, resourceLimits?: CgroupLimits) { this.logger = pino({ name: `CVMTS.QemuVMShim/${def.id}` }); + + if (resourceLimits) { + if (process.platform == 'linux') { + this.vm = new QemuVM(def, new QemuResourceLimitedLauncher(def.id, resourceLimits)); + } else { + // Just use the default Superqemu launcher on non-Linux platforms, + // .. regardless of if resource control is (somehow) enabled. + this.logger.warn({platform: process.platform}, 'Resource control is not supported on this platform. Please remove or comment it out from your configuration.'); + this.vm = new QemuVM(def); + } + } else { + this.vm = new QemuVM(def); + } + } Start(): Promise { diff --git a/cvmts/src/vm/qemu_launcher.ts b/cvmts/src/vm/qemu_launcher.ts new file mode 100644 index 0000000..8e9086b --- /dev/null +++ b/cvmts/src/vm/qemu_launcher.ts @@ -0,0 +1,104 @@ +import EventEmitter from 'events'; +import { IProcess, IProcessLauncher, ProcessLaunchOptions } from '@computernewb/superqemu'; +import { execaCommand } from 'execa'; +import { Readable, Writable } from 'stream'; +import { CGroup } from '../util/cgroup.js'; + +export interface CgroupLimits { + cpuUsageMax?: number; + runOnCpus?: number[]; +} + +interface CGroupValue { + controller: string; + key: string; + value: string; +} + +function MakeValuesFromLimits(limits: CgroupLimits): CGroupValue[] { + let option_array = []; + + if (limits.cpuUsageMax) { + // cpu.max + option_array.push({ + controller: 'cpu', + key: 'max', + value: `${(limits.cpuUsageMax / 100) * 10000} 10000` + }); + } + + if(limits.runOnCpus) { + // Make sure a CPU is not specified more than once. Bit hacky but oh well + let unique = [...new Set(limits.runOnCpus)]; + option_array.push({ + controller: 'cpuset', + key: 'cpus', + value: `${unique.join(',')}` + }); + } + + return option_array; +} + +// A process automatically placed in a given cgroup. +class CGroupLimitedProcess extends EventEmitter implements IProcess { + private process; + stdin: Writable | null = null; + stdout: Readable | null = null; + stderr: Readable | null = null; + private cgroup: CGroup; + + constructor(cg: CGroup, command: string, opts?: ProcessLaunchOptions) { + super(); + this.cgroup = cg; + + this.process = execaCommand(command, opts); + + this.stdin = this.process.stdin; + this.stdout = this.process.stdout; + this.stderr = this.process.stderr; + + let self = this; + this.process.on('spawn', () => { + // it should have one! + self.cgroup.AttachProcess(self.process.pid!); + self.emit('spawn'); + }); + + this.process.on('exit', (code) => { + self.emit('exit', code); + }); + } + + kill(signal?: number | NodeJS.Signals): boolean { + return this.process.kill(signal); + } + + dispose(): void { + this.stdin = null; + this.stdout = null; + this.stderr = null; + + this.process.removeAllListeners(); + this.removeAllListeners(); + } +} + +export class QemuResourceLimitedLauncher implements IProcessLauncher { + private group; + + constructor(name: string, limits: CgroupLimits) { + let root = CGroup.Self(); + this.group = root.GetSubgroup(name); + + // Set cgroup keys. + for(const val of MakeValuesFromLimits(limits)) { + let controller = this.group.GetController(val.controller); + controller.WriteValue(val.key, val.value); + } + } + + launch(command: string, opts?: ProcessLaunchOptions | undefined): IProcess { + return new CGroupLimitedProcess(this.group, command, opts); + } +}