How to add binary I/O to boa engine?

help
1

I'm trying to add proper binary I/O capabilities to the boa engine (GitHub - boa-dev/boa: Boa is an embeddable Javascript engine written in Rust. · GitHub).

This is what I have so far

use boa_engine::{Context, Source, JsResult, JsValue, JsObject, NativeFunction, js_string};
use boa_engine::object::builtins::{JsArrayBuffer, JsUint8Array};
use std::io::{self, Read, Write};

/// High-performance synchronous block parsing from WASI stdin (fd: 0)
fn read_exactly(bytes_to_read: usize) -> Option<Vec<u8>> {
    let mut buffer = vec![0u8; bytes_to_read];
    let mut total_read = 0;
    let mut stdin = io::stdin().lock();

    while total_read < bytes_to_read {
        match stdin.read(&mut buffer[total_read..bytes_to_read]) {
            Ok(0) => return None,
            Ok(n) => total_read += n,
            Err(_) => return None,
        }
    }
    Some(buffer)
}

/// Dispatches binary blocks back across WASI stdout (fd: 1)
fn write_stdout(bytes: &[u8]) -> bool {
    let mut stdout = io::stdout().lock();
    if stdout.write_all(bytes).is_err() {
        return false;
    }
    stdout.flush().is_ok()
}

/// Forwards system panic states safely across WASI stderr (fd: 2)
fn write_stderr(message: String) {
    let mut stderr = io::stderr().lock();
    let _ = stderr.write_all(message.as_bytes());
    let _ = stderr.flush();
}

fn main() -> JsResult<()> {
    // 1. Initialize the official execution context
    let mut context = Context::default();

    // 2. Build the top-level 'std' namespace object template
    let std_obj = JsObject::with_object_proto(context.intrinsics());

    // Inject std.read
    let read_fn = NativeFunction::from_copy_closure(|_this, args, ctx| {
        let len = args.get(0).and_then(|v| v.as_number()).unwrap_or(0.0) as usize;
        
        if let Some(bytes) = read_exactly(len) {
            let ab = JsArrayBuffer::from_byte_block(bytes, ctx)?;
            let ua = JsUint8Array::from_array_buffer(ab, ctx)?;
            return Ok(JsValue::from(ua));
        }
        Ok(JsValue::null())
    });
    // FIX: Replaced from_proto_and_data with direct function builder template allocation
    let read_js_fn = context.intrinsics().templates().function().create(read_fn, vec![]);
    std_obj.set(js_string!("read"), JsValue::from(read_js_fn), true, &mut context)?;

    // Inject std.write
    let write_fn = NativeFunction::from_copy_closure(|_this, args, ctx| {
        let fd = args.get(0).and_then(|v| v.as_number()).unwrap_or(1.0) as i32;
        let js_buffer = args.get(1).and_then(|v| v.as_object().cloned());
        
        if let Some(obj) = js_buffer {
            if let Ok(typed_array) = JsUint8Array::from_object(obj) {
                // FIX: Changed .get_buffer(ctx) to .buffer(ctx)
                let array_buffer = typed_array.buffer(ctx)?;
                let bytes = array_buffer.take_buffer(ctx)?;
                if fd == 1 {
                    return Ok(JsValue::from(write_stdout(&bytes)));
                } else if fd == 2 {
                    write_stderr(String::from_utf8_lossy(&bytes).into_owned());
                    return Ok(JsValue::from(true));
                }
            }
        }
        Ok(JsValue::from(false))
    });
    // FIX: Replaced from_proto_and_data with direct function builder template allocation
    let write_js_fn = context.intrinsics().templates().function().create(write_fn, vec![]);
    std_obj.set(js_string!("write"), JsValue::from(write_js_fn), true, &mut context)?;

    // Register 'std' globally
    context.global_object().set(js_string!("std"), JsValue::from(std_obj), true, &mut context)?;

    // 3. Native Messaging loop refactored for the clean 'std' object API
    let js_code = r#"
        const MAX_CHUNK_ELEMENTS = 209715;

        async function readExactly(bytesToRead) {
          const buffer = new Uint8Array(bytesToRead);
          let totalRead = 0;
          while (totalRead < bytesToRead) {
            const chunkBytes = std.read(bytesToRead - totalRead);
            if (!chunkBytes) return null; 
            
            buffer.set(chunkBytes, totalRead);
            totalRead += chunkBytes.length;
          }
          return buffer;
        }

        async function sendNativeMessage(payloadData) {
          if (Array.isArray(payloadData) && payloadData.length > MAX_CHUNK_ELEMENTS) {
            for (let i = 0; i < payloadData.length; i += MAX_CHUNK_ELEMENTS) {
              await transmitFrame(payloadData.slice(i, i + MAX_CHUNK_ELEMENTS));
            }
          } else {
            await transmitFrame(payloadData);
          }
        }

        async function transmitFrame(obj) {
          try {
            const encodedPayload = new TextEncoder().encode(JSON.stringify(obj));
            const packet = new Uint8Array(4 + encodedPayload.length);
            const view = new DataView(packet.buffer);
            view.setUint32(0, encodedPayload.length, true);
            packet.set(encodedPayload, 4);
            std.write(1, packet);
          } catch (_) {}
        }

        async function runHostLoop() {
          while (true) {
            const headerBytes = await readExactly(4);
            if (!headerBytes) return;

            const view = new DataView(headerBytes.buffer);
            const messageLength = view.getUint32(0, true);

            const bodyBytes = await readExactly(messageLength);
            if (!bodyBytes) return;

            try {
              const parsedMessage = JSON.parse(new TextDecoder().decode(bodyBytes));
              await sendNativeMessage(parsedMessage);
            } catch (err) {
              try {
                std.write(2, new TextEncoder().encode(`Parse error: ${err.message}\n`));
              } catch (_) {}
            }
          }
        }

        runHostLoop();
    "#;

    // 4. Parse source execution cleanly using native bytes slices
    context.eval(Source::from_bytes(js_code.as_bytes()))?;

    Ok(())
}

These are the errors I'm getting when compiling

cargo build --target wasm32-wasip1 --release
   Compiling nm_wasi_host v0.1.0 (/media/user/z/boa)
error[E0624]: method `templates` is private
  --> src/main.rs:56:43
   |
56 |     let read_js_fn = context.intrinsics().templates().function().create(...
   |                                           ^^^^^^^^^ private method
   |
  ::: /media/user/x/rust/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/boa_engine-0.19.1/src/context/intrinsics.rs:68:5
   |
68 |     pub(crate) const fn templates(&self) -> &ObjectTemplates {
   |     -------------------------------------------------------- private method defined here

error[E0599]: no method named `take_buffer` found for enum `JsValue` in the current scope
  --> src/main.rs:68:42
   |
68 |                 let bytes = array_buffer.take_buffer(ctx)?;
   |                                          ^^^^^^^^^^^ method not found in `JsValue`

error[E0624]: method `templates` is private
  --> src/main.rs:80:44
   |
80 |     let write_js_fn = context.intrinsics().templates().function().create...
   |                                            ^^^^^^^^^ private method
   |
  ::: /media/user/x/rust/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/boa_engine-0.19.1/src/context/intrinsics.rs:68:5
   |
68 |     pub(crate) const fn templates(&self) -> &ObjectTemplates {
   |     -------------------------------------------------------- private method defined here

Some errors have detailed explanations: E0599, E0624.
For more information about an error, try `rustc --explain E0599`.
error: could not compile `nm_wasi_host` (bin "nm_wasi_host") due to 3 previous errors

How to fix?

2

I'm not familiar enough with that crate or your objectives to give specific advice on achieving your goal.

But in general, you can't use the private methods or fields of an upstream crate. That's what the "private method" error is about. Either find a way forward using the public API, or make changes within the upstream crate itself if you plan to PR your changes (or are okay vendoring or forking the crate), or perhaps submit a PR/feature request to make the desired interfaces public (if it would be reasonable for the crate to do so).

3

A quick scan of the documentation finds:

4

Hmm. How would I use that?

I wound up with a working implementation (after hours of testing what some free LLM spit out; back and forth) that doesn't use that, rather uses the closure I think.

I added an std global object to the context ('cause ECMAScript doesn't have I/O I'm on a mission to implement it in every JavaScript engine I test that doesn't have (binary) I/O implemented), read(), write(), err() on that object, and tested with a Native Messaging host test script and another script to test std.err()

It would be interesting to compare using that. How would I do so?

use boa_engine::{Context, Source, JsResult, JsValue, JsObject, NativeFunction, js_string};
use boa_engine::object::builtins::{JsArrayBuffer, JsUint8Array};
use boa_engine::object::FunctionObjectBuilder;
use std::io::{self, Read, Write};

/// High-performance synchronous block parsing from stdin (Works on native and WASI)
fn read_bytes(bytes_to_read: usize) -> Option<Vec<u8>> {
    if bytes_to_read == 0 {
        return Some(vec![]);
    }
    
    let mut buffer = vec![0u8; bytes_to_read];
    let mut stdin = io::stdin().lock();

    // 💡 FIX: Read whatever bytes are available up to capacity (do not use read_exact)
    match stdin.read(&mut buffer) {
        Ok(0) => None, // Clean EOF / Stream closed
        Ok(n) => {
            buffer.truncate(n);
            Some(buffer)
        }
        Err(_) => None,
    }
}

/// Dispatches binary blocks cleanly back across stdout and returns amount written
fn write_stdout(bytes: &[u8]) -> Option<usize> {
    let mut stdout = io::stdout().lock();
    if stdout.write_all(bytes).is_err() {
        return None;
    }
    if stdout.flush().is_err() {
        return None;
    }
    Some(bytes.len())
}

/// Forwards standard error diagnostics safely without corrupting the stdout pipeline
fn write_stderr(message: &str) {
    let mut stderr = io::stderr().lock();
    let _ = stderr.write_all(message.as_bytes());
    let _ = stderr.flush();
}

fn main() -> JsResult<()> {
    // DIRECTLY REFERENCE THE JS FILE IN MAIN.RS
    let js_code = include_str!("__index.js");

    // Initialize execution context
    let mut context = Context::default();

    // Build the top-level 'std' namespace object template
    let std_obj = JsObject::with_object_proto(context.intrinsics());

    // Inject std.read(typedArray) -> returns bytes read or null
    let read_fn = NativeFunction::from_copy_closure(|_this, args, ctx| {
        let js_buffer = args.get(0).and_then(|v| v.as_object().cloned());
        
        if let Some(obj) = js_buffer {
            if let Ok(typed_array) = JsUint8Array::from_object(obj) {
                let len = typed_array.length(ctx).unwrap_or(0);
                let byte_offset = typed_array.byte_offset(ctx).unwrap_or(0);

                if let Some(bytes) = read_bytes(len) {
                    let bytes_read = bytes.len();
                    
                    let ab_val = typed_array.buffer(ctx)?;
                    if let Some(ab_obj) = ab_val.as_object().cloned() {
                        let ab = JsArrayBuffer::from_object(ab_obj)?;
                        
                        let write_result = if let Some(mut gc_ref_mut) = ab.data_mut() {
                            let slice = &mut *gc_ref_mut;
                            slice[byte_offset..(byte_offset + bytes_read)].copy_from_slice(&bytes[..bytes_read]);
                            Some(JsValue::from(bytes_read))
                        } else {
                            None
                        };
                        
                        if let Some(val) = write_result {
                            return Ok(val);
                        }
                    }
                }
            }
        }
        Ok(JsValue::null())
    });
    let read_js_fn = FunctionObjectBuilder::new(context.realm(), read_fn).build();
    std_obj.set(js_string!("read"), JsValue::from(read_js_fn), true, &mut context)?;

    // Inject std.write(typedArray) -> returns bytes written or 0
    let write_fn = NativeFunction::from_copy_closure(|_this, args, ctx| {
        let js_buffer = args.get(0).and_then(|v| v.as_object().cloned());
        if let Some(obj) = js_buffer {
            if let Ok(typed_array) = JsUint8Array::from_object(obj) {
                let byte_offset = typed_array.byte_offset(ctx).unwrap_or(0);
                let length = typed_array.length(ctx).unwrap_or(0);

                let array_buffer_val = typed_array.buffer(ctx)?;
                if let Some(array_buffer_obj) = array_buffer_val.as_object().cloned() {
                    let array_buffer = JsArrayBuffer::from_object(array_buffer_obj)?;
                    
                    let write_result = if let Some(gc_ref) = array_buffer.data() {
                        let bytes: &[u8] = &*gc_ref;
                        let view_window_slice = &bytes[byte_offset..(byte_offset + length)];
                        write_stdout(view_window_slice).map(JsValue::from)
                    } else {
                        None
                    };

                    if let Some(val) = write_result {
                        return Ok(val);
                    }
                }
            }
        }
        Ok(JsValue::from(0))
    });
    let write_js_fn = FunctionObjectBuilder::new(context.realm(), write_fn).build();
    std_obj.set(js_string!("write"), JsValue::from(write_js_fn), true, &mut context)?;

    // Inject std.err(string)
    let err_fn = NativeFunction::from_copy_closure(|_this, args, _ctx| {
        let js_str = args.get(0).and_then(|v| v.as_string()).cloned().unwrap_or_default();
        write_stderr(&js_str.to_std_string_escaped());
        Ok(JsValue::undefined())
    });
    let err_js_fn = FunctionObjectBuilder::new(context.realm(), err_fn).build();
    std_obj.set(js_string!("err"), JsValue::from(err_js_fn), true, &mut context)?;

    // Register 'std' globally
    context.global_object().set(js_string!("std"), JsValue::from(std_obj), true, &mut context)?;

    // Parse execution with explicit error tracking to stderr if JavaScript crashes
    match context.eval(Source::from_bytes(js_code.as_bytes())) {
        Ok(_) => {
            context.run_jobs();
        }
        Err(js_error) => {
            eprintln!("JavaScript Exception: {}", js_error);
        }
    }

    Ok(())
}
5

As it describes, it defines a function on the global object, so it wouldn't create that std object interface you're going for, but it's a quick easy way to get a function callable from JS: just give it the name and the NativeFunction instance and you're done.

What you've got there is a fine enough approach for adding what you're after. I suppose you could define a Standard class or the like, but it's not really any real value in a single instance case like this.

6

Can you write out an example usage?