Audio I/O

1
// my-mic-worklet.js
2
class MicWorklet extends AudioWorkletProcessor {
3
  constructor() { super(); this._buf = []; this._frames = 0; }
4
  process(inputs) {
5
    const ch = inputs[0]?.[0];
6
    if (!ch) return true;
7
    this._buf.push(new Float32Array(ch));
8
    this._frames += ch.length;
9
    if (this._frames >= 320) {            // 20 ms at 16 kHz
10
      const out = new Float32Array(this._frames);
11
      let o = 0;
12
      for (const c of this._buf) { out.set(c, o); o += c.length; }
13
      this.port.postMessage(out);
14
      this._buf = []; this._frames = 0;
15
    }
16
    return true;
17
  }
18
}
19
registerProcessor("mic-worklet", MicWorklet);

1
// main thread
2
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
3
const ctx = new AudioContext({ sampleRate: 16000 });
4
await ctx.audioWorklet.addModule("my-mic-worklet.js");
5
const src = ctx.createMediaStreamSource(stream);
6
const node = new AudioWorkletNode(ctx, "mic-worklet");
7
node.port.onmessage = (e) => {
8
  const pcm16 = floatTo16BitPCM(e.data);
9
  ws.send(JSON.stringify({
10
    type: "input_audio_buffer.append",
11
    audio: arrayBufferToBase64(pcm16.buffer),
12
  }));
13
};
14
src.connect(node);
15
16
function floatTo16BitPCM(f32) {
17
  const out = new Int16Array(f32.length);
18
  for (let i = 0; i < f32.length; i++) {
19
    const s = Math.max(-1, Math.min(1, f32[i]));
20
    out[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
21
  }
22
  return out;
23
}
24
function arrayBufferToBase64(buf) {
25
  let bin = "";
26
  const b = new Uint8Array(buf);
27
  for (let i = 0; i < b.length; i++) bin += String.fromCharCode(b[i]);
28
  return btoa(bin);
29
}

1
const TTS_RATE = 48000;
2
const playCtx = new AudioContext({ sampleRate: TTS_RATE });
3
let playCursor = playCtx.currentTime;
4
5
ws.onmessage = (ev) => {
6
  const evt = JSON.parse(ev.data);
7
  if (evt.type === "response.output_audio.delta") {
8
    playPCM16(b64ToInt16(evt.delta));
9
  }
10
};
11
12
function playPCM16(int16) {
13
  const buf = playCtx.createBuffer(1, int16.length, TTS_RATE);
14
  const ch = buf.getChannelData(0);
15
  for (let i = 0; i < int16.length; i++) ch[i] = int16[i] / 0x8000;
16
  const src = playCtx.createBufferSource();
17
  src.buffer = buf;
18
  src.connect(playCtx.destination);
19
  const start = Math.max(playCtx.currentTime, playCursor);
20
  src.start(start);
21
  playCursor = start + buf.duration;
22
}
23
24
function b64ToInt16(b64) {
25
  // base64 → ArrayBuffer → little-endian Int16Array
26
  const bin = atob(b64);
27
  const buf = new ArrayBuffer(bin.length);
28
  const view = new DataView(buf);
29
  for (let i = 0; i < bin.length; i++) view.setUint8(i, bin.charCodeAt(i));
30
  const out = new Int16Array(bin.length / 2);
31
  for (let i = 0; i < out.length; i++) out[i] = view.getInt16(i * 2, true);
32
  return out;
33
}