cvmts: replace jpeg-turbo native module with new rust module

This module also does threadpooling internally, so we don't need Piscina anymore (which I'm pretty sure was actually bottlenecking.)
2024-06-20 03:20:56 -04:00
parent 39521a4b1d
commit 87a377a10f
17 changed files with 632 additions and 555 deletions
--- a/jpegturbo-rs/Cargo.lock
+++ b/jpegturbo-rs/Cargo.lock
@@ -0,0 +1,350 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "addr2line"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678"
+dependencies = [
+ "gimli",
+]
+
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
+[[package]]
+name = "anyhow"
+version = "1.0.86"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
+
+[[package]]
+name = "backtrace"
+version = "0.3.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a"
+dependencies = [
+ "addr2line",
+ "cc",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+]
+
+[[package]]
+name = "cc"
+version = "1.0.99"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "cmake"
+version = "0.1.50"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
+[[package]]
+name = "gimli"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd"
+
+[[package]]
+name = "hermit-abi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
+
+[[package]]
+name = "jpegturbo-rs"
+version = "0.1.0"
+dependencies = [
+ "neon",
+ "once_cell",
+ "tokio",
+ "turbojpeg-sys",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.155"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+
+[[package]]
+name = "libloading"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19"
+dependencies = [
+ "cfg-if",
+ "windows-targets",
+]
+
+[[package]]
+name = "memchr"
+version = "2.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
+
+[[package]]
+name = "miniz_oxide"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08"
+dependencies = [
+ "adler",
+]
+
+[[package]]
+name = "neon"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d75440242411c87dc39847b0e33e961ec1f10326a9d8ecf9c1ea64a3b3c13dc"
+dependencies = [
+ "getrandom",
+ "libloading",
+ "neon-macros",
+ "once_cell",
+ "semver",
+ "send_wrapper",
+ "smallvec",
+]
+
+[[package]]
+name = "neon-macros"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6813fde79b646e47e7ad75f480aa80ef76a5d9599e2717407961531169ee38b"
+dependencies = [
+ "quote",
+ "syn",
+ "syn-mid",
+]
+
+[[package]]
+name = "num_cpus"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
+[[package]]
+name = "object"
+version = "0.36.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"
+
+[[package]]
+name = "pkg-config"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.85"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rustc-demangle"
+version = "0.1.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
+
+[[package]]
+name = "semver"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
+
+[[package]]
+name = "send_wrapper"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73"
+
+[[package]]
+name = "smallvec"
+version = "1.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
+
+[[package]]
+name = "syn"
+version = "2.0.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn-mid"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5dc35bb08dd1ca3dfb09dce91fd2d13294d6711c88897d9a9d60acf39bce049"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tokio"
+version = "1.38.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a"
+dependencies = [
+ "backtrace",
+ "num_cpus",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "turbojpeg-sys"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fa6daade3b979fb7454cce5ebcb9772ce7a1cf476ea27ed20ed06e13d9bc983"
+dependencies = [
+ "anyhow",
+ "cmake",
+ "libc",
+ "pkg-config",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
+[[package]]
+name = "windows-targets"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
--- a/jpegturbo-rs/Cargo.toml
+++ b/jpegturbo-rs/Cargo.toml
@@ -0,0 +1,15 @@
+[package]
+name = "jpegturbo-rs"
+description = "Rust powered JPEGTurbo sex"
+version = "0.1.0"
+edition = "2021"
+exclude = ["index.node"]
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+neon = "1"
+once_cell = "1.19.0"
+tokio = { version = "1.38.0", features = [ "rt", "rt-multi-thread" ] }
+turbojpeg-sys = "1.0.0"
--- a/jpegturbo-rs/index.d.ts
+++ b/jpegturbo-rs/index.d.ts
@@ -0,0 +1,15 @@
+
+interface JpegInputArgs {
+	width: number,
+	height: number,
+	stride: number, // The width of your input framebuffer OR your image width (if encoding a full image)
+	buffer: Buffer
+
+	// TODO: Allow different formats, or export a boxed ffi object which can store a format
+	// (i.e: new JpegEncoder(FORMAT_xxx)).
+}
+
+/// Performs JPEG encoding.
+export function jpegEncode(input: JpegInputArgs) : Promise<Buffer>;
+
+// TODO: Version that can downscale?
--- a/jpegturbo-rs/index.js
+++ b/jpegturbo-rs/index.js
@@ -0,0 +1,6 @@
+// *sigh*
+import { createRequire } from 'module';
+const require = createRequire(import.meta.url);
+
+export let {jpegEncode} = require('./index.node');
+
--- a/jpegturbo-rs/package.json
+++ b/jpegturbo-rs/package.json
@@ -0,0 +1,16 @@
+{
+  "name": "@cvmts/jpegturbo-rs",
+  "version": "0.1.0",
+  "packageManager": "yarn@4.1.1",
+  "type": "module",
+  "main": "index.js",
+  "types": "index.d.ts",
+  "scripts": {
+    "build": "cargo-cp-artifact -nc index.node -- cargo build --release --message-format=json-render-diagnostics",
+    "install": "yarn build",
+    "test": "cargo test"
+  },
+  "devDependencies": {
+    "cargo-cp-artifact": "^0.1"
+  }
+}
--- a/jpegturbo-rs/src/jpeg_compressor.rs
+++ b/jpegturbo-rs/src/jpeg_compressor.rs
@@ -0,0 +1,82 @@
+use turbojpeg_sys::*;
+
+pub struct Image<'a> {
+    pub buffer: &'a [u8],
+    pub width: u32,
+    pub height: u32,
+    pub stride: u32,
+    pub format: TJPF,
+}
+
+pub struct JpegCompressor {
+    handle: tjhandle,
+    subsamp: TJSAMP,
+    quality: u32,
+}
+
+unsafe impl Send for JpegCompressor {}
+
+impl JpegCompressor {
+    pub fn new() -> Self {
+        unsafe {
+            let init = Self {
+                handle: tjInitCompress(),
+                subsamp: TJSAMP_TJSAMP_422,
+                quality: 95,
+            };
+            return init;
+        }
+    }
+
+    pub fn set_quality(&mut self, quality: u32) {
+        self.quality = quality;
+    }
+
+    pub fn set_subsamp(&mut self, samp: TJSAMP) {
+        self.subsamp = samp;
+    }
+
+    pub fn compress_buffer<'a>(&self, image: &Image<'a>) -> Vec<u8> {
+        unsafe {
+            let size: usize =
+                tjBufSize(image.width as i32, image.height as i32, self.subsamp) as usize;
+            let mut vec = Vec::with_capacity(size);
+
+            vec.resize(size, 0);
+
+            let mut ptr: *mut u8 = vec.as_mut_ptr();
+            let mut size: u64 = 0;
+
+            let res = tjCompress2(
+                self.handle,
+                image.buffer.as_ptr(),
+                image.width as i32,
+                image.stride as i32,
+                image.height as i32,
+                image.format,
+                std::ptr::addr_of_mut!(ptr),
+                std::ptr::addr_of_mut!(size),
+                self.subsamp,
+                self.quality as i32,
+                (TJFLAG_NOREALLOC) as i32,
+            );
+
+			// TODO: Result sex so we can actually notify failure
+            if res == -1 {
+                return Vec::new();
+            }
+
+			// Truncate down to the size we're given back
+			vec.truncate(size as usize);
+            return vec;
+        }
+    }
+}
+
+impl Drop for JpegCompressor {
+    fn drop(&mut self) {
+        unsafe {
+            tjDestroy(self.handle);
+        }
+    }
+}
--- a/jpegturbo-rs/src/lib.rs
+++ b/jpegturbo-rs/src/lib.rs
@@ -0,0 +1,91 @@
+use std::sync::{Arc, Mutex};
+
+use neon::prelude::*;
+use neon::types::buffer::TypedArray;
+
+use once_cell::sync::OnceCell;
+use tokio::runtime::Runtime;
+
+use std::cell::RefCell;
+
+mod jpeg_compressor;
+
+fn runtime<'a, C: Context<'a>>(cx: &mut C) -> NeonResult<&'static Runtime> {
+    static RUNTIME: OnceCell<Runtime> = OnceCell::new();
+
+    RUNTIME
+        .get_or_try_init(Runtime::new)
+        .or_else(|err| cx.throw_error(&err.to_string()))
+}
+
+thread_local! {
+    static COMPRESSOR: RefCell<jpeg_compressor::JpegCompressor> = RefCell::new(jpeg_compressor::JpegCompressor::new());
+}
+
+fn jpeg_encode_impl<'a>(cx: &mut FunctionContext<'a>) -> JsResult<'a, JsPromise> {
+    let input = cx.argument::<JsObject>(0)?;
+
+    // Get our input arguments here
+    let width: u64 = input.get::<JsNumber, _, _>(cx, "width")?.value(cx) as u64;
+    let height: u64 = input.get::<JsNumber, _, _>(cx, "height")?.value(cx) as u64;
+    let stride: u64 = input.get::<JsNumber, _, _>(cx, "stride")?.value(cx) as u64;
+    let buffer: Handle<JsBuffer> = input.get(cx, "buffer")?;
+
+    let (deferred, promise) = cx.promise();
+    let channel = cx.channel();
+    let runtime = runtime(cx)?;
+
+    let buf = buffer.as_slice(cx);
+
+    let copy: Arc<Mutex<Vec<u8>>> = Arc::new(Mutex::new(Vec::with_capacity(buf.len())));
+
+	// Copy from the node buffer to our temporary buffer
+    {
+        let mut locked = copy.lock().unwrap();
+        let cap = locked.capacity();
+        locked.resize(cap, 0);
+        locked.copy_from_slice(buf);
+    }
+
+	// Spawn off a tokio blocking pool thread that will do the work for us
+	runtime.spawn_blocking(move || {
+        let clone = Arc::clone(&copy);
+        let locked = clone.lock().unwrap();
+
+        let image: jpeg_compressor::Image = jpeg_compressor::Image {
+            buffer: locked.as_slice(),
+            width: width as u32,
+            height: height as u32,
+
+            stride: (stride * 4u64) as u32, // I think?
+            format: turbojpeg_sys::TJPF_TJPF_RGBA,
+        };
+
+        let vec = COMPRESSOR.with(|lazy| {
+            let mut b = lazy.borrow_mut();
+            b.set_quality(35);
+            b.set_subsamp(turbojpeg_sys::TJSAMP_TJSAMP_420);
+            b.compress_buffer(&image)
+        });
+
+		// Fulfill the Javascript promise with our encoded buffer
+        deferred.settle_with(&channel, move |mut cx| {
+            let mut buf = cx.buffer(vec.len())?;
+            let slice = buf.as_mut_slice(&mut cx);
+            slice.copy_from_slice(vec.as_slice());
+            Ok(buf)
+        });
+    });
+
+    Ok(promise)
+}
+
+fn jpeg_encode(mut cx: FunctionContext) -> JsResult<JsPromise> {
+    jpeg_encode_impl(&mut cx)
+}
+
+#[neon::main]
+fn main(mut cx: ModuleContext) -> NeonResult<()> {
+    cx.export_function("jpegEncode", jpeg_encode)?;
+    Ok(())
+}