numcodecs_wasm_builder/
main.rs

1#![expect(missing_docs)] // FIXME
2#![allow(clippy::multiple_crate_versions)] // windows_i686_gnullvm
3
4use std::{
5    collections::HashMap,
6    env, fs, io,
7    path::{Path, PathBuf},
8    process::Command,
9    str::FromStr,
10};
11
12use clap::Parser;
13use semver::Version;
14
15// FIXME: https://github.com/bytecodealliance/rustix/issues/1620
16use ::is_terminal as _;
17
18#[derive(Parser, Debug)]
19#[command()]
20struct Args {
21    /// Name of the numcodecs codec crate to compile
22    #[arg(name = "crate", long)]
23    crate_: String,
24
25    /// Version of the numcodecs codec crate to compile
26    #[arg(long)]
27    version: Version,
28
29    /// Path to the codec type to export, without the leading crate name
30    #[arg(long)]
31    codec: String,
32
33    /// Path to which the wasm file is output
34    #[arg(long, short)]
35    output: PathBuf,
36
37    /// Compile the local crate instead of the published one
38    #[arg(long)]
39    local: bool,
40
41    /// Compile the crate with debug information enabled
42    #[arg(long)]
43    debug: bool,
44
45    /// Enable verbose logging while compiling the crate
46    #[arg(long)]
47    verbose: bool,
48
49    /// Enable experimental features for the WASM guest
50    #[arg(long)]
51    wasm_features: Vec<String>,
52}
53
54fn main() -> io::Result<()> {
55    let args = Args::parse();
56
57    let scratch_dir = scratch::path(concat!(
58        env!("CARGO_PKG_NAME"),
59        "-",
60        env!("CARGO_PKG_VERSION"),
61    ));
62    eprintln!("scratch_dir={}", scratch_dir.display());
63
64    let target_dir = scratch_dir.join("target");
65    eprintln!("target_dir={}", target_dir.display());
66    eprintln!("creating {}", target_dir.display());
67    fs::create_dir_all(&target_dir)?;
68
69    let crate_dir = create_codec_wasm_component_crate(
70        &scratch_dir,
71        &args.crate_,
72        &args.version,
73        &args.codec,
74        args.local,
75        &args.wasm_features,
76    )?;
77    copy_buildenv_to_crate(&crate_dir)?;
78
79    let nix_env = NixEnv::new(&crate_dir)?;
80
81    let wasm = build_wasm_codec(
82        &nix_env,
83        &target_dir,
84        &crate_dir,
85        &format!("{}-wasm", args.crate_),
86        args.debug,
87        args.verbose,
88    )?;
89    let wasm = optimize_wasm_codec(&wasm, &nix_env, args.debug)?;
90    let wasm = adapt_wasi_snapshot_to_preview2(&wasm)?;
91
92    fs::copy(wasm, args.output)?;
93
94    Ok(())
95}
96
97fn create_codec_wasm_component_crate(
98    scratch_dir: &Path,
99    crate_: &str,
100    version: &Version,
101    codec: &str,
102    local: bool,
103    wasm_features: &[String],
104) -> io::Result<PathBuf> {
105    let crate_dir = scratch_dir.join(format!("{crate_}-wasm-{version}"));
106    eprintln!("crate_dir={}", crate_dir.display());
107    eprintln!("creating {}", crate_dir.display());
108    if crate_dir.exists() {
109        fs::remove_dir_all(&crate_dir)?;
110    }
111    fs::create_dir_all(&crate_dir)?;
112
113    let (numcodecs_wasm_logging_path, numcodecs_wasm_guest_path, numcodecs_my_codec_path) = if local
114    {
115        let numcodecs = Path::new(env!("CARGO_MANIFEST_DIR")).join("..").join("..");
116        eprintln!("looking for local workspace in {}", numcodecs.display());
117        let numcodecs = numcodecs.canonicalize()?;
118        let numcodecs_wasm_logging = numcodecs.join("crates").join("numcodecs-wasm-logging");
119        let numcodecs_wasm_guest = numcodecs.join("crates").join("numcodecs-wasm-guest");
120        let numcodecs_my_codec = numcodecs
121            .join("codecs")
122            .join(crate_.strip_prefix("numcodecs-").unwrap_or(crate_));
123        (
124            format!(r#" path = "{}","#, numcodecs_wasm_logging.display()),
125            format!(r#" path = "{}","#, numcodecs_wasm_guest.display()),
126            format!(r#" path = "{}","#, numcodecs_my_codec.display()),
127        )
128    } else {
129        (String::new(), String::new(), String::new())
130    };
131
132    fs::write(
133        crate_dir.join("Cargo.toml"),
134        format!(
135            r#"
136[workspace]
137
138[package]
139name = "{crate_}-wasm"
140version = "{version}"
141edition = "2024"
142
143# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
144
145[dependencies]
146numcodecs-wasm-logging = {{ version = "0.2",{numcodecs_wasm_logging_path} default-features = false }}
147numcodecs-wasm-guest = {{ version = "0.3",{numcodecs_wasm_guest_path} default-features = false, features = {wasm_features:?} }}
148numcodecs-my-codec = {{ package = "{crate_}", version = "{version}",{numcodecs_my_codec_path} default-features = false }}
149    "#
150        ),
151    )?;
152
153    fs::create_dir_all(crate_dir.join("src"))?;
154
155    fs::write(
156        crate_dir.join("src").join("lib.rs"),
157        format!(
158            "
159#![cfg_attr(not(test), no_main)]
160
161numcodecs_wasm_guest::export_codec!(
162    numcodecs_wasm_logging::LoggingCodec<numcodecs_my_codec::{codec}>
163);
164    "
165        ),
166    )?;
167
168    Ok(crate_dir)
169}
170
171fn copy_buildenv_to_crate(crate_dir: &Path) -> io::Result<()> {
172    fs::write(
173        crate_dir.join("flake.nix"),
174        include_str!("../buildenv/flake.nix"),
175    )?;
176    fs::write(
177        crate_dir.join("flake.lock"),
178        include_str!("../buildenv/flake.lock"),
179    )?;
180
181    fs::write(
182        crate_dir.join("include.h"),
183        include_str!("../buildenv/include.h"),
184    )?;
185    fs::write(
186        crate_dir.join("include.hpp"),
187        include_str!("../buildenv/include.hpp"),
188    )?;
189
190    fs::write(
191        crate_dir.join("rust-toolchain"),
192        include_str!("../buildenv/rust-toolchain"),
193    )?;
194
195    Ok(())
196}
197
198struct NixEnv {
199    llvm_version: String,
200    ar: PathBuf,
201    clang: PathBuf,
202    libclang: PathBuf,
203    lld: PathBuf,
204    nm: PathBuf,
205    ranlib: PathBuf,
206    strip: PathBuf,
207    objdump: PathBuf,
208    dlltool: PathBuf,
209    wasi_sysroot: PathBuf,
210    libclang_rt: PathBuf,
211    wasm_opt: PathBuf,
212    pkg_config: PathBuf,
213    #[expect(dead_code)]
214    python3: PathBuf,
215}
216
217impl NixEnv {
218    pub fn new(flake_parent_dir: &Path) -> io::Result<Self> {
219        fn try_read_env<T: FromStr<Err: std::error::Error>>(
220            env: &HashMap<&str, &str>,
221            key: &str,
222        ) -> Result<T, io::Error> {
223            let Some(var) = env.get(key).copied() else {
224                return Err(io::Error::new(
225                    io::ErrorKind::InvalidData,
226                    format!("missing flake env key: {key}"),
227                ));
228            };
229
230            T::from_str(var).map_err(|err| {
231                io::Error::new(
232                    io::ErrorKind::InvalidData,
233                    format!("invalid flake env variable {key}={var}: {err}"),
234                )
235            })
236        }
237
238        let mut env = Command::new("nix");
239        env.current_dir(flake_parent_dir);
240        env.arg("develop");
241        // env.arg("--store");
242        // env.arg(nix_store_path);
243        env.arg("path:.");
244        env.arg("--no-update-lock-file");
245        env.arg("--ignore-environment");
246        env.arg("--command");
247        env.arg("env");
248        eprintln!("executing {env:?}");
249        let env = env.output()?;
250        eprintln!(
251            "{}\n{}",
252            String::from_utf8_lossy(&env.stdout),
253            String::from_utf8_lossy(&env.stderr)
254        );
255        let env = std::str::from_utf8(&env.stdout).map_err(|err| {
256            io::Error::new(
257                io::ErrorKind::InvalidData,
258                format!("invalid flake env output: {err}"),
259            )
260        })?;
261        let env = env
262            .lines()
263            .filter_map(|line| line.split_once('='))
264            .collect::<HashMap<_, _>>();
265
266        Ok(Self {
267            llvm_version: try_read_env(&env, "MY_LLVM_VERSION")?,
268            ar: try_read_env(&env, "MY_AR")?,
269            clang: try_read_env(&env, "MY_CLANG")?,
270            libclang: try_read_env(&env, "MY_LIBCLANG")?,
271            lld: try_read_env(&env, "MY_LLD")?,
272            nm: try_read_env(&env, "MY_NM")?,
273            ranlib: try_read_env(&env, "MY_RANLIB")?,
274            strip: try_read_env(&env, "MY_STRIP")?,
275            objdump: try_read_env(&env, "MY_OBJDUMP")?,
276            dlltool: try_read_env(&env, "MY_DLLTOOL")?,
277            wasi_sysroot: try_read_env(&env, "MY_WASI_SYSROOT")?,
278            libclang_rt: try_read_env(&env, "MY_LIBCLANG_RT")?,
279            wasm_opt: try_read_env(&env, "MY_WASM_OPT")?,
280            pkg_config: try_read_env(&env, "MY_PKG_CONFIG")?,
281            python3: try_read_env(&env, "MY_PYTHON3")?,
282        })
283    }
284}
285
286#[expect(clippy::too_many_lines)]
287fn configure_cargo_cmd(
288    nix_env: &NixEnv,
289    target_dir: &Path,
290    crate_dir: &Path,
291    debug: bool,
292) -> Command {
293    let NixEnv {
294        llvm_version,
295        ar,
296        clang,
297        libclang,
298        lld,
299        nm,
300        ranlib,
301        strip,
302        objdump,
303        dlltool,
304        wasi_sysroot,
305        libclang_rt,
306        pkg_config,
307        ..
308    } = nix_env;
309
310    let mut cmd = Command::new("nix");
311    cmd.current_dir(crate_dir);
312    cmd.arg("develop");
313    // cmd.arg("--store");
314    // cmd.arg(nix_store_path);
315    cmd.arg("--no-update-lock-file");
316    cmd.arg("--ignore-environment");
317    cmd.arg("path:.");
318    cmd.arg("--command");
319    cmd.arg("env");
320    cmd.arg("GMP_MPFR_SYS_CACHE=");
321    cmd.arg(format!("CC={clang}", clang = clang.join("clang").display()));
322    cmd.arg(format!(
323        "CXX={clang}",
324        clang = clang.join("clang++").display()
325    ));
326    cmd.arg(format!("LD={lld}", lld = lld.join("lld").display()));
327    cmd.arg(format!("LLD={lld}", lld = lld.join("lld").display()));
328    cmd.arg(format!("AR={ar}", ar = ar.display()));
329    cmd.arg(format!("NM={nm}", nm = nm.display()));
330    cmd.arg(format!("RANLIB={ranlib}", ranlib = ranlib.display()));
331    cmd.arg(format!("STRIP={strip}", strip = strip.display()));
332    cmd.arg(format!("OBJDUMP={objdump}", objdump = objdump.display()));
333    cmd.arg(format!("DLLTOOL={dlltool}", dlltool = dlltool.display()));
334    cmd.arg(format!(
335        "PKG_CONFIG={pkg_config}",
336        pkg_config = pkg_config.display()
337    ));
338    cmd.arg(format!(
339        "LIBCLANG_PATH={libclang}",
340        libclang = libclang.display()
341    ));
342    cmd.arg(format!(
343        "CFLAGS=--target=wasm32-wasip1 -nodefaultlibs -resource-dir {resource_dir} \
344         --sysroot={wasi_sysroot} -isystem {clang_include} -isystem {wasi32_wasi_include} \
345         -isystem {include} -B {lld} -D_WASI_EMULATED_PROCESS_CLOCKS -D_WASI_EMULATED_SIGNAL \
346         -include {c_include_path} -O3 {debug} \
347         -DHAVE_STRNLEN=1 -DHAVE_MEMSET=1 -DHAVE_RAISE=1",
348        resource_dir = libclang.join("clang").join(llvm_version).display(),
349        wasi_sysroot = wasi_sysroot.display(),
350        clang_include = libclang
351            .join("clang")
352            .join(llvm_version)
353            .join("include")
354            .display(),
355        wasi32_wasi_include = wasi_sysroot.join("include").join("wasm32-wasip1").display(),
356        include = wasi_sysroot.join("include").display(),
357        lld = lld.display(),
358        c_include_path = crate_dir.join("include.h").display(),
359        debug = if debug { "-g" } else { "" },
360    ));
361    cmd.arg(format!(
362        "CXXFLAGS=--target=wasm32-wasip1 -nodefaultlibs -resource-dir {resource_dir} \
363         --sysroot={wasi_sysroot} -isystem {wasm32_wasi_cxx_include} -isystem {cxx_include} \
364         -isystem {clang_include} -isystem {wasi32_wasi_include} -isystem {include} -B {lld} \
365         -D_WASI_EMULATED_PROCESS_CLOCKS -include {cpp_include_path} -O3 {debug} \
366         -fwasm-exceptions -mllvm -wasm-use-legacy-eh=false",
367        resource_dir = libclang.join("clang").join(llvm_version).display(),
368        wasi_sysroot = wasi_sysroot.display(),
369        wasm32_wasi_cxx_include = wasi_sysroot
370            .join("include")
371            .join("wasm32-wasip1")
372            .join("c++")
373            .join("v1")
374            .display(),
375        cxx_include = wasi_sysroot
376            .join("include")
377            .join("c++")
378            .join("v1")
379            .display(),
380        clang_include = libclang
381            .join("clang")
382            .join(llvm_version)
383            .join("include")
384            .display(),
385        wasi32_wasi_include = wasi_sysroot.join("include").join("wasm32-wasip1").display(),
386        include = wasi_sysroot.join("include").display(),
387        lld = lld.display(),
388        cpp_include_path = crate_dir.join("include.hpp").display(),
389        debug = if debug { "-g" } else { "" },
390    ));
391    cmd.arg(format!(
392        "BINDGEN_EXTRA_CLANG_ARGS=--target=wasm32-wasip1 -nodefaultlibs -resource-dir \
393         {resource_dir} --sysroot={wasi_sysroot} -isystem {wasm32_wasi_cxx_include} -isystem \
394         {cxx_include} -isystem {clang_include} -isystem {wasi32_wasi_include} -isystem {include} \
395         -B {lld} -D_WASI_EMULATED_PROCESS_CLOCKS -fvisibility=default",
396        resource_dir = libclang.join("clang").join(llvm_version).display(),
397        wasi_sysroot = wasi_sysroot.display(),
398        wasm32_wasi_cxx_include = wasi_sysroot
399            .join("include")
400            .join("wasm32-wasip1")
401            .join("c++")
402            .join("v1")
403            .display(),
404        cxx_include = wasi_sysroot
405            .join("include")
406            .join("c++")
407            .join("v1")
408            .display(),
409        clang_include = libclang
410            .join("clang")
411            .join(llvm_version)
412            .join("include")
413            .display(),
414        wasi32_wasi_include = wasi_sysroot.join("include").join("wasm32-wasip1").display(),
415        include = wasi_sysroot.join("include").display(),
416        lld = lld.display(),
417    ));
418    cmd.arg("CXXSTDLIB=c++");
419    // disable default flags from cc
420    cmd.arg("CRATE_CC_NO_DEFAULTS=1");
421    cmd.arg(format!(
422        "LDFLAGS=-lc -lwasi-emulated-process-clocks -lwasi-emulated-signal \
423        -L{libclang_rt} -lclang_rt.builtins -lunwind -lc++ -lc++abi",
424        libclang_rt = libclang_rt.join("wasm32-unknown-wasip1").display(),
425    ));
426    cmd.arg("PKG_CONFIG_PATH=\"\"");
427    cmd.arg(format!(
428        "PKG_CONFIG_LIBDIR={pkg_config_lib}:{pkg_config_share}",
429        pkg_config_lib = wasi_sysroot.join("lib").join("pkgconfig").display(),
430        pkg_config_share = wasi_sysroot.join("share").join("pkgconfig").display()
431    ));
432    cmd.arg(format!(
433        "PKG_CONFIG_SYSROOT_DIR={wasi_sysroot}",
434        wasi_sysroot = wasi_sysroot.display()
435    ));
436    cmd.arg(format!(
437        "RUSTFLAGS=-C panic=abort {debug} \
438        -C link-arg=-L{wasm32_wasi_lib} \
439        -C link-arg=-L{libclang_rt} -C link-arg=-lclang_rt.builtins \
440        -C link-arg=-lunwind -C link-arg=-lc++ -C link-arg=-lc++abi \
441        -C llvm-args=-wasm-use-legacy-eh=false",
442        debug = if debug { "-g" } else { "-C strip=symbols" },
443        wasm32_wasi_lib = wasi_sysroot.join("lib").join("wasm32-wasip1").display(),
444        libclang_rt = libclang_rt.join("wasm32-unknown-wasip1").display(),
445    ));
446    cmd.arg(format!(
447        "CARGO_TARGET_DIR={target_dir}",
448        target_dir = target_dir.display()
449    ));
450
451    // we don't need nightly Rust features but need to compile std with immediate
452    // panic abort instead of compiling with nightly, we fake it and forbid the
453    // unstable_features lint
454    cmd.arg("RUSTC_BOOTSTRAP=1");
455
456    cmd.arg("cargo");
457
458    cmd
459}
460
461fn build_wasm_codec(
462    nix_env: &NixEnv,
463    target_dir: &Path,
464    crate_dir: &Path,
465    crate_name: &str,
466    debug: bool,
467    verbose: bool,
468) -> io::Result<PathBuf> {
469    let mut cmd = configure_cargo_cmd(nix_env, target_dir, crate_dir, debug);
470    cmd.arg("rustc")
471        .arg("--crate-type=cdylib")
472        .arg("-Z")
473        .arg("build-std=std,panic_abort")
474        .arg("-Z")
475        .arg("build-std-features=panic_immediate_abort")
476        .arg("--release")
477        .arg("--target=wasm32-wasip1");
478
479    if verbose {
480        cmd.arg("-vv");
481    }
482
483    eprintln!("executing {cmd:?}");
484
485    let status = cmd.status()?;
486    if !status.success() {
487        return Err(io::Error::other(format!("cargo exited with code {status}")));
488    }
489
490    Ok(target_dir
491        .join("wasm32-wasip1")
492        .join("release")
493        .join(crate_name.replace('-', "_"))
494        .with_extension("wasm"))
495}
496
497fn optimize_wasm_codec(wasm: &Path, nix_env: &NixEnv, debug: bool) -> io::Result<PathBuf> {
498    let NixEnv { wasm_opt, .. } = nix_env;
499
500    let opt_out = wasm.with_extension("opt.wasm");
501
502    let mut cmd = Command::new(wasm_opt);
503
504    cmd.arg("--enable-sign-ext")
505        .arg("--disable-threads")
506        .arg("--enable-mutable-globals")
507        .arg("--enable-nontrapping-float-to-int")
508        .arg("--enable-simd")
509        .arg("--enable-bulk-memory")
510        .arg("--enable-exception-handling")
511        .arg("--disable-tail-call")
512        .arg("--enable-reference-types")
513        .arg("--enable-multivalue")
514        .arg("--disable-gc")
515        .arg("--disable-memory64")
516        .arg("--disable-relaxed-simd")
517        .arg("--disable-extended-const")
518        .arg("--disable-strings")
519        .arg("--disable-multimemory")
520        .arg(if debug { "-g" } else { "--strip-debug" });
521
522    // FIXME: https://github.com/WebAssembly/binaryen/issues/8325
523    cmd.arg("-O3").arg("-o").arg(&opt_out).arg(wasm);
524
525    eprintln!("executing {cmd:?}");
526
527    let status = cmd.status()?;
528    if !status.success() {
529        return Err(io::Error::other(format!(
530            "wasm-opt exited with code {status}"
531        )));
532    }
533
534    Ok(opt_out)
535}
536
537fn adapt_wasi_snapshot_to_preview2(wasm: &Path) -> io::Result<PathBuf> {
538    let wasm_preview2 = wasm.with_extension("preview2.wasm");
539
540    eprintln!("reading from {}", wasm.display());
541    let wasm = fs::read(wasm)?;
542
543    let mut encoder = wit_component::ComponentEncoder::default()
544        .module(&wasm)
545        .map_err(|err| {
546            io::Error::other(
547                // FIXME: better error reporting in the build script
548                format!("wit_component::ComponentEncoder::module failed: {err:#}"),
549            )
550        })?
551        .adapter(
552            wasi_preview1_component_adapter_provider::WASI_SNAPSHOT_PREVIEW1_ADAPTER_NAME,
553            wasi_preview1_component_adapter_provider::WASI_SNAPSHOT_PREVIEW1_REACTOR_ADAPTER,
554        )
555        .map_err(|err| {
556            io::Error::other(
557                // FIXME: better error reporting in the build script
558                format!("wit_component::ComponentEncoder::adapter failed: {err:#}"),
559            )
560        })?;
561
562    let wasm = encoder.encode().map_err(|err| {
563        io::Error::other(
564            // FIXME: better error reporting in the build script
565            format!("wit_component::ComponentEncoder::encode failed: {err:#}"),
566        )
567    })?;
568
569    eprintln!("writing to {}", wasm_preview2.display());
570    fs::write(&wasm_preview2, wasm)?;
571
572    Ok(wasm_preview2)
573}
numcodecs_wasm_builder/main.rs

numcodecs_wasm_builder/
main.rs
