Testing a Jev (TypeSafe AI) integration without an API key: mocks, simulators and failure branches

// T04 (pool-authored, offline): raw-fetch Jev client vs a local mock server. No Jev call.
import http from 'node:http';
const script = []; let hits = 0;
const server = http.createServer((req, res) => {
  hits++; const step = script.shift() ?? { status: 500, body: '{}' };
  if (step.hang) return;
  res.writeHead(step.status, { 'content-type': step.ctype ?? 'application/json', ...(step.headers ?? {}) });
  res.end(step.body);
});
await new Promise(r => server.listen(0, '127.0.0.1', r));
const url = `http://127.0.0.1:${server.address().port}/v1/systemone`;
const OPTIONS = ['billing', 'technical', 'other'];
const RETRY = new Set([408, 429, ...Array.from({ length: 100 }, (_, i) => 500 + i)]);
async function ask({ timeoutMs = 800, maxRetries = 2 } = {}) {
  for (let attempt = 0; ; attempt++) {
    const ctl = new AbortController(); const t = setTimeout(() => ctl.abort(), timeoutMs);
    let res;
    try {
      res = await fetch(url, { method: 'POST', signal: ctl.signal, headers: { 'content-type': 'application/json', authorization: 'Bearer test-not-a-key' },
        body: JSON.stringify({ model: 'jev-1.13.0', state: 'I was charged twice', questions: { category: { type: 'choice', instructions: 'Team?', criteria: { billing: null, technical: null, other: null } } } }) });
      var text = await res.text();
    } catch (e) {
      clearTimeout(t);
      const kind = ctl.signal.aborted ? 'timeout' : 'connection';
      if (attempt >= maxRetries) return { ok: false, kind, attempts: attempt + 1 };
      await new Promise(r => setTimeout(r, 500 * 2 ** attempt)); continue;
    }
    clearTimeout(t);
    if (!res.ok) {
      const html = !(res.headers.get('content-type') ?? '').includes('json');
      if (attempt < maxRetries && RETRY.has(res.status)) {
        const ra = Number(res.headers.get('retry-after'));
        await new Promise(r => setTimeout(r, Number.isFinite(ra) && ra >= 0 && res.headers.has('retry-after') ? ra * 1000 : 500 * 2 ** attempt)); continue;
      }
      return { ok: false, kind: 'http', status: res.status, html, attempts: attempt + 1 };
    }
    let body; try { body = JSON.parse(text); } catch { return { ok: false, kind: 'malformed-json', attempts: attempt + 1 }; }
    const a = body?.answers?.category;
    if (!a || !OPTIONS.includes(a.choice) || typeof a.confidence !== 'number') return { ok: false, kind: 'invalid-answer', got: a ?? null, attempts: attempt + 1 };
    return { ok: true, choice: a.choice, confidence: a.confidence, attempts: attempt + 1 };
  }
}
const good = JSON.stringify({ model: 'jev-1.13.0', answers: { category: { type: 'choice', choice: 'billing', probabilities: { billing: 0.9, technical: 0.06, other: 0.04 }, confidence: 0.85 } }, usage: { input_tokens: 300, output_tokens: 20 } });
const cases = [
  ['429 Retry-After:1 then 200', [{ status: 429, headers: { 'retry-after': '1' }, body: '{"error":"rate"}' }, { status: 200, body: good }]],
  ['529 x3', [{ status: 529, body: '{}' }, { status: 529, body: '{}' }, { status: 529, body: '{}' }]],
  ['HTML 403 (Cloudflare-style)', [{ status: 403, ctype: 'text/html', body: '<html>blocked</html>' }]],
  ['401', [{ status: 401, body: '{"error":"bad key"}' }]],
  ['422', [{ status: 422, body: '{"error":"state null"}' }]],
  ['200 malformed JSON', [{ status: 200, body: '{"answers": ' }]],
  ['200 choice not among options', [{ status: 200, body: good.replace('"choice":"billing"', '"choice":"legal"') }]],
  ['hang past client timeout x3', [{ hang: true }, { hang: true }, { hang: true }]],
];
for (const [name, steps] of cases) {
  script.length = 0; script.push(...steps); hits = 0;
  const t0 = Date.now(); const r = await ask();
  console.log(JSON.stringify({ case: name, result: r, server_hits: hits, ms: Date.now() - t0 }));
}
server.closeAllConnections?.(); server.close();

{"case":"429 Retry-After:1 then 200","result":{"ok":true,"choice":"billing","confidence":0.85,"attempts":2},"server_hits":2,"ms":1053}
{"case":"529 x3","result":{"ok":false,"kind":"http","status":529,"html":false,"attempts":3},"server_hits":3,"ms":1519}
{"case":"HTML 403 (Cloudflare-style)","result":{"ok":false,"kind":"http","status":403,"html":true,"attempts":1},"server_hits":1,"ms":5}
{"case":"401","result":{"ok":false,"kind":"http","status":401,"html":false,"attempts":1},"server_hits":1,"ms":4}
{"case":"422","result":{"ok":false,"kind":"http","status":422,"html":false,"attempts":1},"server_hits":1,"ms":3}
{"case":"200 malformed JSON","result":{"ok":false,"kind":"malformed-json","attempts":1},"server_hits":1,"ms":4}
{"case":"200 choice not among options","result":{"ok":false,"kind":"invalid-answer","got":{"type":"choice","choice":"legal","probabilities":{"billing":0.9,"technical":0.06,"other":0.04},"confidence":0.85},"attempts":1},"server_hits":1,"ms":5}
{"case":"hang past client timeout x3","result":{"ok":false,"kind":"timeout","attempts":3},"server_hits":3,"ms":3908}