diff --git a/backend/src/configs/configurations.ts b/backend/src/configs/configurations.ts index 473fa83..48a00ab 100644 --- a/backend/src/configs/configurations.ts +++ b/backend/src/configs/configurations.ts @@ -1,13 +1,14 @@ // backend/src/config/configuration.ts /** Matches values accepted by jsonwebtoken `expiresIn` (via ms), e.g. 7d, 15m, or plain seconds. */ -const JWT_TIMESPAN_PATTERN = - /^\d+(\.\d+)?(ms|s|m|h|d|w|y)?$/i; +const JWT_TIMESPAN_PATTERN = /^\d+(\.\d+)?(ms|s|m|h|d|w|y)?$/i; function assertJwtSecret(value: string, envKey: string): string { const trimmed = value.trim(); if (!trimmed) { - throw new Error(`❌ Environment variable ${envKey} is required but not set`); + throw new Error( + `❌ Environment variable ${envKey} is required but not set`, + ); } if (trimmed.length < 16) { throw new Error(`❌ ${envKey} must be at least 16 characters`); @@ -21,7 +22,9 @@ function assertJwtSecret(value: string, envKey: string): string { function assertJwtTimespan(value: string, envKey: string): string { const trimmed = value.trim(); if (!trimmed) { - throw new Error(`❌ Environment variable ${envKey} is required but not set`); + throw new Error( + `❌ Environment variable ${envKey} is required but not set`, + ); } if (!/^\d+$/.test(trimmed) && !JWT_TIMESPAN_PATTERN.test(trimmed)) { throw new Error( @@ -31,7 +34,10 @@ function assertJwtTimespan(value: string, envKey: string): string { return trimmed; } -function parseEnvBoolean(value: string | undefined, defaultValue: boolean): boolean { +function parseEnvBoolean( + value: string | undefined, + defaultValue: boolean, +): boolean { if (value === undefined || value.trim() === '') { return defaultValue; } @@ -67,8 +73,37 @@ export interface Config { apiKey: string | null; templateId: number; }; + voice: VoiceConfig; } +/** Only OpenRouter today. The indirection exists so a locale can diverge without code. */ +export type VoiceProviderId = 'openrouter'; + +export type VoiceProfile = { + asr: { provider: VoiceProviderId; model: string }; + llm: { provider: VoiceProviderId; model: string }; +}; + +export interface VoiceConfig { + openRouter: { + apiKey: string | null; + baseUrl: string; + }; + /** + * Locale → provider profile. A locale absent here has no microphone button at all — + * clean absence rather than a dead control. + */ + profiles: Record; + /** Recording cap in ms; null means uncapped. */ + maxRecordingMs: number | null; + throttle: { ttl: number; limit: number }; +} + +export const VOICE_LOCALES = ['fa', 'en', 'nl'] as const; + +const DEFAULT_ASR_MODEL = 'openai/whisper-1'; +const DEFAULT_LLM_MODEL = 'google/gemini-3.7-flash'; + export default (): Config => { // Helper function to get required env var with type safety const getEnvVar = (key: string): string => { @@ -129,5 +164,96 @@ export default (): Config => { apiKey: process.env.SMS_IR_API_KEY?.trim() || null, templateId: getEnvVarAsNumber('SMS_IR_TEMPLATE_ID', 123456), }, + voice: buildVoiceConfig(getEnvVarWithDefault, getEnvVarAsNumber), }; -}; \ No newline at end of file +}; + +const VOICE_PROVIDER_IDS: readonly VoiceProviderId[] = ['openrouter']; + +/** Unknown provider ids fail at boot; silently coercing a typo would ship the wrong vendor. */ +function parseProviderId( + value: string | undefined, + envKey: string, +): VoiceProviderId { + const trimmed = value?.trim(); + if (!trimmed) return 'openrouter'; + if ((VOICE_PROVIDER_IDS as readonly string[]).includes(trimmed)) { + return trimmed as VoiceProviderId; + } + throw new Error( + `❌ ${envKey}="${trimmed}" is not a known voice provider (${VOICE_PROVIDER_IDS.join(', ')})`, + ); +} + +/** + * Every enabled locale gets its own ASR and LLM provider+model, each overridable + * independently. They all point at the same OpenRouter models today; the per-locale + * indirection is kept because Persian ASR is the weakest link and swapping only `fa` must + * not be a code change. + */ +function buildVoiceConfig( + getEnvVarWithDefault: (key: string, defaultValue: string) => string, + getEnvVarAsNumber: (key: string, defaultValue: number) => number, +): VoiceConfig { + const enabled = getEnvVarWithDefault( + 'VOICE_ENABLED_LOCALES', + VOICE_LOCALES.join(','), + ) + .split(',') + .map((locale) => locale.trim().toLowerCase()) + .filter(Boolean); + for (const locale of enabled) { + // Fail loudly, like parseProviderId. Silently filtering a typo would disable the + // microphone everywhere with nothing to explain why. + if (!(VOICE_LOCALES as readonly string[]).includes(locale)) { + throw new Error( + `❌ VOICE_ENABLED_LOCALES contains unknown locale "${locale}" (known: ${VOICE_LOCALES.join(', ')})`, + ); + } + } + + const profiles: Record = {}; + for (const locale of enabled) { + const suffix = locale.toUpperCase(); + profiles[locale] = { + asr: { + provider: parseProviderId( + process.env[`VOICE_ASR_PROVIDER_${suffix}`], + `VOICE_ASR_PROVIDER_${suffix}`, + ), + model: getEnvVarWithDefault( + `VOICE_ASR_MODEL_${suffix}`, + getEnvVarWithDefault('VOICE_ASR_MODEL', DEFAULT_ASR_MODEL), + ), + }, + llm: { + provider: parseProviderId( + process.env[`VOICE_LLM_PROVIDER_${suffix}`], + `VOICE_LLM_PROVIDER_${suffix}`, + ), + model: getEnvVarWithDefault( + `VOICE_LLM_MODEL_${suffix}`, + getEnvVarWithDefault('VOICE_LLM_MODEL', DEFAULT_LLM_MODEL), + ), + }, + }; + } + + const maxRecordingMs = getEnvVarAsNumber('VOICE_MAX_RECORDING_MS', 120_000); + + return { + openRouter: { + apiKey: process.env.OPENROUTER_API_KEY?.trim() || null, + baseUrl: getEnvVarWithDefault( + 'OPENROUTER_BASE_URL', + 'https://openrouter.ai/api/v1', + ), + }, + profiles, + maxRecordingMs: maxRecordingMs > 0 ? maxRecordingMs : null, + throttle: { + ttl: getEnvVarAsNumber('VOICE_THROTTLE_TTL', 60), + limit: getEnvVarAsNumber('VOICE_THROTTLE_LIMIT', 6), + }, + }; +} diff --git a/backend/src/modules/voice/extraction.prompt.ts b/backend/src/modules/voice/extraction.prompt.ts new file mode 100644 index 0000000..6c07a23 --- /dev/null +++ b/backend/src/modules/voice/extraction.prompt.ts @@ -0,0 +1,78 @@ +import type { ExtractionCatalog } from './voice.providers'; + +/** Locale-specific guidance. Only the tooth vocabulary and numbering habits differ. */ +const LOCALE_NOTES: Record = { + fa: [ + 'The clinician is speaking Persian. Tooth references are usually quadrant-relative:', + '"شش بالا راست" = upper right six -> arch "upper", side "patient_right", position 6.', + 'Digits may appear in Persian or Latin script. Two-digit FDI notation ("یک چهار") does', + 'occur — use the "fdi" field only for that.', + ].join(' '), + nl: [ + 'The clinician is speaking Dutch and uses FDI notation, which is standard in the', + 'Netherlands. "rechtsboven zes" = upper right six. A bare two-digit number is FDI.', + ].join(' '), + en: [ + 'The clinician is speaking English. IMPORTANT: a bare two-digit number is ambiguous,', + 'because Universal numbering and FDI disagree ("tooth 14" is a different tooth in each).', + 'Set "fdi" ONLY when the speaker made the notation explicit (e.g. "FDI one four").', + 'Otherwise describe the tooth with arch/side/position, or leave it unresolved.', + ].join(' '), +}; + +function codeList(entries: { code: string; label: string }[]): string { + if (entries.length === 0) return '(none available)'; + return entries.map((e) => `- ${e.code} = ${e.label}`).join('\n'); +} + +export function buildExtractionPrompt( + transcript: string, + catalog: ExtractionCatalog, + localeHint: string, +) { + const localeNote = LOCALE_NOTES[localeHint] ?? LOCALE_NOTES.en; + + const system = [ + 'You extract structured dental treatment data from a transcript of a clinician speaking.', + 'You are a parser, not an assistant: report only what was said.', + '', + 'HARD RULES', + '1. Never invent a code. treatmentType, prosthesisDefaultType and prosthesisOverrides[].type', + ' must be codes from the lists below. labId must be an id from the lab list. If what you', + ' heard is not in a list, use null.', + '2. Never output an FDI tooth code unless the speaker used FDI notation. Prefer', + ' arch + side + position.', + '3. "side" is always the PATIENT\'s side. The patient\'s upper right is quadrant 1. Never', + " flip to the viewer's point of view.", + '4. Never do calendar arithmetic. Report the deadline as it was said, using due.kind.', + ' If no deadline was mentioned, use due.kind = "none".', + '5. Copy the exact spoken words for each tooth into "spoken", so the clinician can see', + ' what was heard.', + '6. If you are unsure about a value, use null. A missing field is recoverable; a wrong', + ' one is not.', + '', + localeNote, + '', + 'TREATMENT TYPE CODES', + codeList(catalog.treatmentTypes), + '', + 'PROSTHESIS TYPE CODES', + codeList(catalog.prosthesisTypes), + '', + 'LABS THIS CLINIC CAN SEND TO', + catalog.labs.length > 0 + ? catalog.labs.map((l) => `- ${l.id} = ${l.name}`).join('\n') + : '(none linked — labId must be null)', + '', + 'OTHER FIELDS', + '- connectedSpans: only for bridges or splinted units. Endpoints inclusive.', + '- comment: clinical notes, in the language spoken. Omit the parts already captured as', + ' treatment type, teeth or deadline.', + '- labMatchExact: true only when the spoken name matched a lab name exactly.', + ].join('\n'); + + return [ + { role: 'system' as const, content: system }, + { role: 'user' as const, content: transcript }, + ]; +} diff --git a/backend/src/modules/voice/extraction.wire.spec.ts b/backend/src/modules/voice/extraction.wire.spec.ts new file mode 100644 index 0000000..6d9efb9 --- /dev/null +++ b/backend/src/modules/voice/extraction.wire.spec.ts @@ -0,0 +1,209 @@ +import { + toVoiceIntent, + VOICE_INTENT_JSON_SCHEMA, + type WireVoiceIntent, +} from './extraction.wire'; + +const emptyDue = { + kind: 'none' as const, + weekday: null, + which: null, + unit: null, + amount: null, + jy: null, + jm: null, + jd: null, + y: null, + m: null, + d: null, +}; + +const wire = (overrides: Partial = {}): WireVoiceIntent => ({ + treatmentType: null, + teeth: [], + connectedSpans: [], + comment: null, + prosthesisDefaultType: null, + prosthesisOverrides: [], + labId: null, + labMatchExact: false, + due: emptyDue, + ...overrides, +}); + +const positionalTooth = { + spoken: 'شش بالا راست', + fdi: null, + arch: 'upper' as const, + side: 'patient_right' as const, + position: 6, +}; + +describe('VOICE_INTENT_JSON_SCHEMA', () => { + it('is strict — every property required, no extras', () => { + expect(VOICE_INTENT_JSON_SCHEMA.additionalProperties).toBe(false); + expect([...VOICE_INTENT_JSON_SCHEMA.required].sort()).toEqual( + Object.keys(VOICE_INTENT_JSON_SCHEMA.properties).sort(), + ); + }); + + it('requires every field of the due object, since strict mode allows no optionals', () => { + const due = VOICE_INTENT_JSON_SCHEMA.properties.due; + expect([...due.required].sort()).toEqual( + Object.keys(due.properties).sort(), + ); + }); +}); + +describe('toVoiceIntent', () => { + it('narrows a positional tooth', () => { + const result = toVoiceIntent(wire({ teeth: [positionalTooth] })); + expect(result.teeth[0]).toEqual({ + kind: 'positional', + arch: 'upper', + side: 'patient_right', + position: 6, + spoken: 'شش بالا راست', + }); + }); + + it('narrows an explicit FDI tooth, which wins over positional fields', () => { + const result = toVoiceIntent( + wire({ teeth: [{ ...positionalTooth, fdi: '14', spoken: 'یک چهار' }] }), + ); + expect(result.teeth[0]).toEqual({ + kind: 'explicit', + fdi: '14', + spoken: 'یک چهار', + }); + }); + + it('maps due.kind "none" to no deadline', () => { + expect(toVoiceIntent(wire()).due).toBeNull(); + }); + + it('narrows each due kind', () => { + expect( + toVoiceIntent( + wire({ + due: { + ...emptyDue, + kind: 'weekday', + weekday: 'thursday', + which: 'next', + }, + }), + ).due, + ).toEqual({ kind: 'weekday', weekday: 'thursday', which: 'next' }); + + expect( + toVoiceIntent( + wire({ due: { ...emptyDue, kind: 'offset', unit: 'week', amount: 2 } }), + ).due, + ).toEqual({ kind: 'offset', unit: 'week', amount: 2 }); + + expect( + toVoiceIntent( + wire({ due: { ...emptyDue, kind: 'jalali', jy: 1404, jm: 7, jd: 25 } }), + ).due, + ).toEqual({ kind: 'jalali', jy: 1404, jm: 7, jd: 25 }); + + expect( + toVoiceIntent( + wire({ + due: { ...emptyDue, kind: 'gregorian', y: 2025, m: 10, d: 17 }, + }), + ).due, + ).toEqual({ kind: 'gregorian', y: 2025, m: 10, d: 17 }); + }); + + it('ignores a non-FDI-shaped fdi and keeps the positional fields', () => { + // A model emitting fdi:"6" alongside correct arch/side/position must still yield 16, + // not lose the tooth to the explicit branch. + const result = toVoiceIntent( + wire({ teeth: [{ ...positionalTooth, fdi: '6' }] }), + ); + expect(result.teeth[0]).toEqual({ + kind: 'positional', + arch: 'upper', + side: 'patient_right', + position: 6, + spoken: 'شش بالا راست', + }); + }); + + it('rejects impossible FDI shapes from the explicit branch', () => { + for (const fdi of ['99', '0', '140', '9', 'ab']) { + expect( + toVoiceIntent(wire({ teeth: [{ ...positionalTooth, fdi }] })).teeth[0] + .kind, + ).toBe('positional'); + } + }); + + it('passes an unrecognised due kind through so it can be flagged', () => { + // Collapsing it to null would make a misunderstood deadline indistinguishable from + // no deadline at all, and the resolver's flagging path unreachable. + const result = toVoiceIntent( + wire({ due: { ...emptyDue, kind: 'lunar_month' as never } }), + ); + expect(result.due).toEqual({ kind: 'lunar_month' }); + }); + + it('reports no prosthesis when neither a default nor an override was given', () => { + expect(toVoiceIntent(wire()).prosthesis).toBeNull(); + }); + + it('builds a prosthesis intent from a default alone', () => { + const result = toVoiceIntent(wire({ prosthesisDefaultType: 'pfm_crown' })); + expect(result.prosthesis).toEqual({ + defaultType: 'pfm_crown', + overrides: [], + }); + }); + + it('builds a prosthesis intent from overrides alone', () => { + const result = toVoiceIntent( + wire({ + prosthesisOverrides: [{ tooth: positionalTooth, type: 'pfm_crown' }], + }), + ); + expect(result.prosthesis?.defaultType).toBeNull(); + expect(result.prosthesis?.overrides).toHaveLength(1); + }); + + it('narrows connected spans', () => { + const result = toVoiceIntent( + wire({ + connectedSpans: [ + { + from: { ...positionalTooth, fdi: '14' }, + to: { ...positionalTooth, fdi: '16' }, + }, + ], + }), + ); + expect(result.connectedSpans[0].from).toEqual({ + kind: 'explicit', + fdi: '14', + spoken: 'شش بالا راست', + }); + }); + + it('is total — a missing or malformed payload yields a resolvable shape, not a throw', () => { + // Whatever survives here is reported as unresolved downstream rather than crashing. + for (const bad of [undefined, null, {}, { teeth: 'nope', due: 5 }]) { + expect(() => toVoiceIntent(bad as never)).not.toThrow(); + const result = toVoiceIntent(bad as never); + expect(result.teeth).toEqual([]); + expect(result.due).toBeNull(); + expect(result.labMatchExact).toBe(false); + } + }); + + it('coerces a non-boolean labMatchExact to false', () => { + expect( + toVoiceIntent(wire({ labMatchExact: 'yes' as never })).labMatchExact, + ).toBe(false); + }); +}); diff --git a/backend/src/modules/voice/extraction.wire.ts b/backend/src/modules/voice/extraction.wire.ts new file mode 100644 index 0000000..af9ac1c --- /dev/null +++ b/backend/src/modules/voice/extraction.wire.ts @@ -0,0 +1,271 @@ +import type { + ConnectedSpanIntent, + DueIntent, + ProsthesisIntent, + ToothIntent, + VoiceIntent, + Weekday, +} from './voice.types'; +import { WEEKDAYS } from './voice.types'; + +/** + * The shape the model actually emits, and its JSON schema. + * + * Deliberately flat: strict `json_schema` mode has poor support for discriminated unions, + * so every variant field is present and nullable on the wire. `toVoiceIntent` narrows the + * flat shape into the internal union the resolvers consume, and is total — anything it + * cannot classify becomes a shape the resolvers will report as unresolved rather than + * something that throws here. + */ + +export type WireToothIntent = { + spoken: string; + /** Two-digit FDI code, only when the speaker genuinely used FDI notation. */ + fdi: string | null; + arch: 'upper' | 'lower' | null; + side: 'patient_right' | 'patient_left' | null; + position: number | null; +}; + +export type WireDue = { + kind: 'weekday' | 'offset' | 'jalali' | 'gregorian' | 'none'; + weekday: Weekday | null; + which: 'this' | 'next' | null; + unit: 'day' | 'week' | 'month' | null; + amount: number | null; + jy: number | null; + jm: number | null; + jd: number | null; + y: number | null; + m: number | null; + d: number | null; +}; + +export type WireVoiceIntent = { + treatmentType: string | null; + teeth: WireToothIntent[]; + connectedSpans: { from: WireToothIntent; to: WireToothIntent }[]; + comment: string | null; + prosthesisDefaultType: string | null; + prosthesisOverrides: { tooth: WireToothIntent; type: string }[]; + labId: string | null; + labMatchExact: boolean; + due: WireDue; +}; + +const TOOTH_SCHEMA = { + type: 'object', + additionalProperties: false, + required: ['spoken', 'fdi', 'arch', 'side', 'position'], + properties: { + spoken: { + type: 'string', + description: 'The exact transcript words for this tooth.', + }, + fdi: { + type: ['string', 'null'], + description: + 'Two-digit FDI code ONLY if the speaker used FDI notation. Otherwise null.', + }, + arch: { type: ['string', 'null'], enum: ['upper', 'lower', null] }, + side: { + type: ['string', 'null'], + enum: ['patient_right', 'patient_left', null], + description: "The PATIENT's side, never the viewer's.", + }, + position: { + type: ['integer', 'null'], + description: '1 = central incisor … 8 = third molar.', + }, + }, +} as const; + +export const VOICE_INTENT_JSON_SCHEMA = { + type: 'object', + additionalProperties: false, + required: [ + 'treatmentType', + 'teeth', + 'connectedSpans', + 'comment', + 'prosthesisDefaultType', + 'prosthesisOverrides', + 'labId', + 'labMatchExact', + 'due', + ], + properties: { + treatmentType: { + type: ['string', 'null'], + description: 'A treatment type CODE from the supplied list, or null.', + }, + teeth: { type: 'array', items: TOOTH_SCHEMA }, + connectedSpans: { + type: 'array', + description: 'Bridges / splinted units. Endpoints inclusive.', + items: { + type: 'object', + additionalProperties: false, + required: ['from', 'to'], + properties: { from: TOOTH_SCHEMA, to: TOOTH_SCHEMA }, + }, + }, + comment: { + type: ['string', 'null'], + description: 'Clinical notes, in the spoken language.', + }, + prosthesisDefaultType: { + type: ['string', 'null'], + description: + 'A prosthesis type CODE applied to every tooth unless overridden.', + }, + prosthesisOverrides: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['tooth', 'type'], + properties: { tooth: TOOTH_SCHEMA, type: { type: 'string' } }, + }, + }, + labId: { + type: ['string', 'null'], + description: 'An id from the supplied lab list. Never invent one.', + }, + labMatchExact: { + type: 'boolean', + description: 'True only when the spoken name matched a lab name exactly.', + }, + due: { + type: 'object', + additionalProperties: false, + required: [ + 'kind', + 'weekday', + 'which', + 'unit', + 'amount', + 'jy', + 'jm', + 'jd', + 'y', + 'm', + 'd', + ], + properties: { + kind: { + type: 'string', + enum: ['weekday', 'offset', 'jalali', 'gregorian', 'none'], + }, + weekday: { type: ['string', 'null'], enum: [...WEEKDAYS, null] }, + which: { type: ['string', 'null'], enum: ['this', 'next', null] }, + unit: { + type: ['string', 'null'], + enum: ['day', 'week', 'month', null], + }, + amount: { type: ['integer', 'null'] }, + jy: { type: ['integer', 'null'] }, + jm: { type: ['integer', 'null'] }, + jd: { type: ['integer', 'null'] }, + y: { type: ['integer', 'null'] }, + m: { type: ['integer', 'null'] }, + d: { type: ['integer', 'null'] }, + }, + }, + }, +} as const; + +/** Two digits, quadrant 1-8, position 1-8 — the only thing that can be an FDI code. */ +const FDI_SHAPE = /^[1-8][1-8]$/; + +function toToothIntent(wire: WireToothIntent | undefined | null): ToothIntent { + const spoken = typeof wire?.spoken === 'string' ? wire.spoken : ''; + const fdi = typeof wire?.fdi === 'string' ? wire.fdi.trim() : ''; + // Only take the explicit branch for something actually FDI-shaped. A model that emits + // fdi:"6" alongside correct arch/side/position would otherwise lose the tooth entirely. + if (FDI_SHAPE.test(fdi)) { + return { kind: 'explicit', fdi, spoken }; + } + return { + kind: 'positional', + arch: wire?.arch as 'upper' | 'lower', + side: wire?.side as 'patient_right' | 'patient_left', + position: typeof wire?.position === 'number' ? wire.position : Number.NaN, + spoken, + }; +} + +function toDueIntent(wire: WireDue | undefined | null): DueIntent | null { + switch (wire?.kind) { + case 'weekday': + return { + kind: 'weekday', + weekday: wire.weekday as Weekday, + which: wire.which as 'this', + }; + case 'offset': + return { + kind: 'offset', + unit: wire.unit as 'day', + amount: typeof wire.amount === 'number' ? wire.amount : Number.NaN, + }; + case 'jalali': + return { + kind: 'jalali', + jy: wire.jy as number, + jm: wire.jm as number, + jd: wire.jd as number, + }; + case 'gregorian': + return { + kind: 'gregorian', + y: wire.y as number, + m: wire.m as number, + d: wire.d as number, + }; + case 'none': + case undefined: + return null; + default: + // An unrecognised kind means a deadline WAS spoken and we failed to classify it. + // Passing it through lets the resolver flag it; collapsing it to null would make a + // misunderstood deadline indistinguishable from no deadline at all. + return { kind: wire?.kind } as unknown as DueIntent; + } +} + +export function toVoiceIntent(wire: WireVoiceIntent): VoiceIntent { + const teeth = Array.isArray(wire?.teeth) ? wire.teeth : []; + const spans = Array.isArray(wire?.connectedSpans) ? wire.connectedSpans : []; + const overrides = Array.isArray(wire?.prosthesisOverrides) + ? wire.prosthesisOverrides + : []; + + const connectedSpans: ConnectedSpanIntent[] = spans.map((span) => ({ + from: toToothIntent(span?.from), + to: toToothIntent(span?.to), + })); + + const hasProsthesis = + wire?.prosthesisDefaultType != null || overrides.length > 0; + const prosthesis: ProsthesisIntent | null = hasProsthesis + ? { + defaultType: wire?.prosthesisDefaultType ?? null, + overrides: overrides.map((o) => ({ + tooth: toToothIntent(o?.tooth), + type: o?.type, + })), + } + : null; + + return { + treatmentType: wire?.treatmentType ?? null, + teeth: teeth.map(toToothIntent), + connectedSpans, + comment: wire?.comment ?? null, + prosthesis, + labId: wire?.labId ?? null, + labMatchExact: wire?.labMatchExact === true, + due: toDueIntent(wire?.due), + }; +} diff --git a/backend/src/modules/voice/openrouter.provider.ts b/backend/src/modules/voice/openrouter.provider.ts new file mode 100644 index 0000000..422c56a --- /dev/null +++ b/backend/src/modules/voice/openrouter.provider.ts @@ -0,0 +1,171 @@ +import { Logger } from '@nestjs/common'; +import { + type AsrProvider, + type AsrResult, + type AudioInput, + type ExtractionCatalog, + type ExtractionProvider, + type ExtractionResult, + VoiceProviderError, +} from './voice.providers'; +import { buildExtractionPrompt } from './extraction.prompt'; +import { + VOICE_INTENT_JSON_SCHEMA, + toVoiceIntent, + type WireVoiceIntent, +} from './extraction.wire'; + +type OpenRouterConfig = { + apiKey: string; + baseUrl: string; + model: string; +}; + +type TranscriptionResponse = { + text?: unknown; + usage?: { seconds?: unknown; cost?: unknown }; +}; + +type ChatResponse = { + choices?: { message?: { content?: unknown } }[]; + usage?: { cost?: unknown }; +}; + +function numberOrNull(value: unknown): number | null { + return typeof value === 'number' && Number.isFinite(value) ? value : null; +} + +const errorLogger = new Logger('OpenRouterVoice'); + +async function readError( + response: Response, + stage: 'asr' | 'extraction', +): Promise { + // A vendor 4xx can echo the request back, transcript included. Keep the body out of the + // thrown message — which callers log and could forward — and out of the default log + // level; it stays available at debug when someone is actively diagnosing. + try { + errorLogger.debug( + `${stage} ${response.status} body: ${(await response.text()).slice(0, 500)}`, + ); + } catch { + errorLogger.debug(`${stage} ${response.status} body unreadable`); + } + throw new VoiceProviderError( + stage, + `OpenRouter ${stage} failed with status ${response.status}`, + response.status, + ); +} + +/** Speech → text via OpenRouter's transcription endpoint (whisper-1 and friends). */ +export class OpenRouterAsrProvider implements AsrProvider { + private readonly logger = new Logger(OpenRouterAsrProvider.name); + + constructor(private readonly config: OpenRouterConfig) {} + + async transcribe( + audio: AudioInput, + localeHint: string, + signal?: AbortSignal, + ): Promise { + const response = await fetch( + `${this.config.baseUrl}/audio/transcriptions`, + { + method: 'POST', + signal, + headers: { + Authorization: `Bearer ${this.config.apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: this.config.model, + input_audio: { data: audio.data, format: audio.format }, + language: localeHint, + }), + }, + ); + + if (!response.ok) await readError(response, 'asr'); + + const body = (await response.json()) as TranscriptionResponse; + const text = typeof body.text === 'string' ? body.text.trim() : ''; + this.logger.debug( + `transcribed ${numberOrNull(body.usage?.seconds) ?? '?'}s`, + ); + + return { + text, + usage: { + seconds: numberOrNull(body.usage?.seconds), + costUsd: numberOrNull(body.usage?.cost), + }, + }; + } +} + +/** Transcript → VoiceIntent via OpenRouter chat completions with a JSON schema. */ +export class OpenRouterExtractionProvider implements ExtractionProvider { + constructor(private readonly config: OpenRouterConfig) {} + + async extract( + transcript: string, + catalog: ExtractionCatalog, + localeHint: string, + signal?: AbortSignal, + ): Promise { + const response = await fetch(`${this.config.baseUrl}/chat/completions`, { + method: 'POST', + signal, + headers: { + Authorization: `Bearer ${this.config.apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: this.config.model, + temperature: 0, + // Only route to endpoints that actually honour the JSON schema. Without this, + // OpenRouter may pick a provider that treats it as a hint and returns prose, + // which fails parsing intermittently and unreproducibly. + provider: { require_parameters: true }, + messages: buildExtractionPrompt(transcript, catalog, localeHint), + response_format: { + type: 'json_schema', + json_schema: { + name: 'voice_intent', + strict: true, + schema: VOICE_INTENT_JSON_SCHEMA, + }, + }, + }), + }); + + if (!response.ok) await readError(response, 'extraction'); + + const body = (await response.json()) as ChatResponse; + const content = body.choices?.[0]?.message?.content; + if (typeof content !== 'string' || !content.trim()) { + throw new VoiceProviderError( + 'extraction', + 'OpenRouter returned no content', + ); + } + + let wire: WireVoiceIntent; + try { + wire = JSON.parse(content) as WireVoiceIntent; + } catch { + // Schema-constrained output should be valid JSON; if it is not, the resolvers can do + // nothing with it, so fail here rather than pass rubbish downstream. + throw new VoiceProviderError( + 'extraction', + 'OpenRouter returned unparseable JSON', + ); + } + + return { + intent: toVoiceIntent(wire), + costUsd: numberOrNull(body.usage?.cost), + }; + } +} diff --git a/backend/src/modules/voice/voice.providers.ts b/backend/src/modules/voice/voice.providers.ts new file mode 100644 index 0000000..d44976e --- /dev/null +++ b/backend/src/modules/voice/voice.providers.ts @@ -0,0 +1,68 @@ +import type { VoiceIntent } from './voice.types'; + +/** + * ASR and extraction are separate, independently swappable roles — they will not come + * from the same vendor for every locale. Both are resolved per locale from + * `config.voice.profiles`, so pointing `fa` at a Persian-specialist vendor while `en` + * and `nl` keep OpenRouter is configuration, not code. + */ + +export type AudioInput = { + /** Raw base64, no data: prefix. */ + data: string; + /** Container as the recorder produced it: webm, mp4, m4a, wav, … */ + format: string; +}; + +export type AsrUsage = { + seconds: number | null; + costUsd: number | null; +}; + +export type AsrResult = { + text: string; + usage: AsrUsage; +}; + +export interface AsrProvider { + /** `localeHint` is ISO-639-1 and materially improves accuracy — always pass it. */ + transcribe( + audio: AudioInput, + localeHint: string, + signal?: AbortSignal, + ): Promise; +} + +export type ExtractionCatalog = { + /** Catalog codes with their labels in the actor's locale, so the model matches spoken words. */ + treatmentTypes: { code: string; label: string }[]; + prosthesisTypes: { code: string; label: string }[]; + /** The clinic's linked labs — a closed choice list. */ + labs: { id: string; name: string }[]; +}; + +export type ExtractionResult = { + intent: VoiceIntent; + costUsd: number | null; +}; + +export interface ExtractionProvider { + extract( + transcript: string, + catalog: ExtractionCatalog, + localeHint: string, + signal?: AbortSignal, + ): Promise; +} + +/** Raised when a vendor call fails; the service maps this onto the staged error codes. */ +export class VoiceProviderError extends Error { + constructor( + readonly stage: 'asr' | 'extraction', + message: string, + readonly status?: number, + ) { + super(message); + this.name = 'VoiceProviderError'; + } +}