Audio I/O

// my-mic-worklet.js
class MicWorklet extends AudioWorkletProcessor {
  constructor() { super(); this._buf = []; this._frames = 0; }
  process(inputs) {
    const ch = inputs[0]?.[0];
    if (!ch) return true;
    this._buf.push(new Float32Array(ch));
    this._frames += ch.length;
    if (this._frames >= 320) {            // 20 ms at 16 kHz
      const out = new Float32Array(this._frames);
      let o = 0;
      for (const c of this._buf) { out.set(c, o); o += c.length; }
      this.port.postMessage(out);
      this._buf = []; this._frames = 0;
    }
    return true;
  }
}
registerProcessor("mic-worklet", MicWorklet);

// main thread
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const ctx = new AudioContext({ sampleRate: 16000 });
await ctx.audioWorklet.addModule("my-mic-worklet.js");
const src = ctx.createMediaStreamSource(stream);
const node = new AudioWorkletNode(ctx, "mic-worklet");
node.port.onmessage = (e) => {
  const pcm16 = floatTo16BitPCM(e.data);
  ws.send(JSON.stringify({
    type: "input_audio_buffer.append",
    audio: arrayBufferToBase64(pcm16.buffer),
  }));
};
src.connect(node);
function floatTo16BitPCM(f32) {
  const out = new Int16Array(f32.length);
  for (let i = 0; i < f32.length; i++) {
    const s = Math.max(-1, Math.min(1, f32[i]));
    out[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
  }
  return out;
}
function arrayBufferToBase64(buf) {
  let bin = "";
  const b = new Uint8Array(buf);
  for (let i = 0; i < b.length; i++) bin += String.fromCharCode(b[i]);
  return btoa(bin);
}

let playCtx = null;
let playCursor = 0;
let outRate = null;
ws.onmessage = (ev) => {
  const evt = JSON.parse(ev.data);
  if (evt.type === "session.configured") {
    outRate = evt.session.output_audio_sample_rate;
    playCtx = new AudioContext({ sampleRate: outRate });
    playCursor = playCtx.currentTime;
    return;
  }
  if (evt.type === "response.output_audio.delta") {
    playPCM16(b64ToInt16(evt.delta));
  }
};
function playPCM16(int16) {
  const buf = playCtx.createBuffer(1, int16.length, outRate);
  const ch = buf.getChannelData(0);
  for (let i = 0; i < int16.length; i++) ch[i] = int16[i] / 0x8000;
  const src = playCtx.createBufferSource();
  src.buffer = buf;
  src.connect(playCtx.destination);
  const start = Math.max(playCtx.currentTime, playCursor);
  src.start(start);
  playCursor = start + buf.duration;
}
function b64ToInt16(b64) {
  // base64 → ArrayBuffer → little-endian Int16Array
  const bin = atob(b64);
  const buf = new ArrayBuffer(bin.length);
  const view = new DataView(buf);
  for (let i = 0; i < bin.length; i++) view.setUint8(i, bin.charCodeAt(i));
  const out = new Int16Array(bin.length / 2);
  for (let i = 0; i < out.length; i++) out[i] = view.getInt16(i * 2, true);
  return out;
}