diff --git a/ventures/havre/app.js b/ventures/havre/app.js index dfe126d..5adf402 100644 --- a/ventures/havre/app.js +++ b/ventures/havre/app.js @@ -2291,6 +2291,7 @@ function runWorkflow(id, { door = "chat", idea = "", name = "" } = {}) { if (id === "new") return writeFromIdea(idea, { door, name }); if (id === "photoshoot") return photoshoot(idea, { door }); if (id === "recall") return recall(idea); + if (id === "whisper") return whisper(idea); } /** the chat a workflow's result lands in: the one of theirs on screen, else their latest, else a new one */ function threadFor(c) { @@ -2404,6 +2405,96 @@ async function photoshoot(arg, { door = "chat" } = {}) { balanceSoon(); } } +/** + * /whisper: a short ASMR voice note. One call writes the script in their voice from the chat (and what it's about, + * if given); a whisper preset speaks it (W.WHISPER_PRESETS: luna on Inworld, eve on Grok, picked from the + * asmr.gandora.lol bake-off); it's mixed over soft rain made here, and lands in their chat as a voice note. + */ +let whispering = false; +async function whisper(arg) { + const c = card(); + if (!c || c.plain) { err("/whisper needs a character: pick one in the harbor first."); return; } + if (whispering) { toast("A whisper is already on its way."); return; } + const { preset: named, brief } = W.parseWhisper(arg); + const key = named || (c.whisper && W.WHISPER_PRESETS[c.whisper] ? c.whisper : W.WHISPER_DEFAULT), p = W.WHISPER_PRESETS[key]; + await loadList("tts"); + const vm = (S.lists.tts || []).find((m) => m.id === p.model); + const guess = vm ? A.voicePrice(vm, 600) : null; + if (!confirm(`A whispered voice note from ${c.name} (${p.voice})${brief ? " about " + brief : ""}? About ${guess != null ? A.usd(guess) : "a cent"} on your Venice key, plus the words.`)) return; + const t = threadFor(c); + whispering = true; + S.inflight++; + doing(t, "is writing something to whisper…"); + try { + const lines = t.msgs.filter((m) => !m.hidden && (m.role === "user" || m.role === "assistant")).slice(-10) + .map((m) => `${m.role === "user" ? "Them" : c.name}: ${String(m.content || "").replace(/\s+/g, " ").trim().slice(0, 300)}`); + const o = await askJson(await jsonModel(c), { messages: W.whisperMessages(c, { brief, lines }), schema: W.WHISPER_SCHEMA, maxTokens: 400, temperature: 0.9, what: "the whisper" }); + const script = W.cleanWhisper(o); + if (!script) throw new Error("The model sent back nothing to whisper. Try again, or pick another model on the card."); + doing(t, "is whispering…"); + const input = W.whisperInput(script, p); + const voice = await new Venice(S.key).speak({ model: p.model, voice: p.voice, input, speed: p.speed, format: p.format }); + let blob = voice; + try { blob = await overRain(voice); } catch { /* the dry take is still a whisper */ } + const price = vm ? A.voicePrice(vm, input.length) : null; + done(t); + deliver(t, { role: "assistant", content: script, voice: { id: await M.put(blob), model: vm ? vm.name : p.model, voice: p.voice, price, whisper: true } }); + if (named) { const now = S.cards.find((x) => x.id === c.id); if (now && now.whisper !== named) replaceCard({ ...now, whisper: named }); } + } catch (e) { + done(t); + err(e.message || String(e)); + } finally { + whispering = false; + S.inflight--; + balanceSoon(); + } +} +/** the voice, close-mic'd (a little low end, gentle compression), over soft rain made here; a wav back */ +async function overRain(blob) { + const AC = window.OfflineAudioContext || window.webkitOfflineAudioContext; + const probe = new (window.AudioContext || window.webkitAudioContext)(); + const v = await probe.decodeAudioData(await blob.arrayBuffer()); + probe.close && probe.close(); + const rate = 44100, pad = 1.5, len = Math.ceil((v.duration + pad * 2) * rate); + const ctx = new AC(2, len, rate); + // rain: pink-ish hiss with a slow flutter, over a low brown rumble + const noise = ctx.createBuffer(2, len, rate); + for (let ch = 0; ch < 2; ch++) { + const d = noise.getChannelData(ch); let b0 = 0, b1 = 0, b2 = 0, br = 0; + for (let i = 0; i < len; i++) { + const w = Math.random() * 2 - 1; + b0 = 0.997 * b0 + 0.029 * w; b1 = 0.985 * b1 + 0.032 * w; b2 = 0.95 * b2 + 0.048 * w; + br = (br + 0.02 * w) / 1.02; + d[i] = (b0 + b1 + b2 + w * 0.05) * 0.6 + br * 2.5; + } + } + const rain = ctx.createBufferSource(); rain.buffer = noise; + const hp = ctx.createBiquadFilter(); hp.type = "highpass"; hp.frequency.value = 120; + const lp = ctx.createBiquadFilter(); lp.type = "lowpass"; lp.frequency.value = 6500; + const rg = ctx.createGain(); const end = len / rate; + rg.gain.setValueAtTime(0, 0); rg.gain.linearRampToValueAtTime(0.09, 1.2); rg.gain.setValueAtTime(0.09, end - 1.2); rg.gain.linearRampToValueAtTime(0, end); + rain.connect(hp); hp.connect(lp); lp.connect(rg); rg.connect(ctx.destination); + const src = ctx.createBufferSource(); src.buffer = v; + const lo = ctx.createBiquadFilter(); lo.type = "lowshelf"; lo.frequency.value = 180; lo.gain.value = 4; + const cmp = ctx.createDynamicsCompressor(); cmp.threshold.value = -30; cmp.ratio.value = 4; cmp.attack.value = 0.01; cmp.release.value = 0.25; + const vg = ctx.createGain(); vg.gain.value = 1.6; + src.connect(lo); lo.connect(cmp); cmp.connect(vg); vg.connect(ctx.destination); + rain.start(0); src.start(pad); + return wavBlob(await ctx.startRendering()); +} +/** an AudioBuffer as a 16-bit wav */ +function wavBlob(buf) { + const n = buf.numberOfChannels, len = buf.length, out = new DataView(new ArrayBuffer(44 + len * n * 2)); + const s = (o, t) => { for (let i = 0; i < t.length; i++) out.setUint8(o + i, t.charCodeAt(i)); }; + s(0, "RIFF"); out.setUint32(4, 36 + len * n * 2, true); s(8, "WAVE"); s(12, "fmt "); out.setUint32(16, 16, true); + out.setUint16(20, 1, true); out.setUint16(22, n, true); out.setUint32(24, buf.sampleRate, true); + out.setUint32(28, buf.sampleRate * n * 2, true); out.setUint16(32, n * 2, true); out.setUint16(34, 16, true); + s(36, "data"); out.setUint32(40, len * n * 2, true); + const chans = []; for (let c = 0; c < n; c++) chans.push(buf.getChannelData(c)); + let o = 44; + for (let i = 0; i < len; i++) for (let c = 0; c < n; c++) { const x = Math.max(-1, Math.min(1, chans[c][i])); out.setInt16(o, x < 0 ? x * 0x8000 : x * 0x7fff, true); o += 2; } + return new Blob([out], { type: "audio/wav" }); +} /** * A photo of them from the chat, kept on their card: as their portrait (as = "portrait"; the look it was drawn from goes * on the card too if it had none), or as one of the pictures their photos and clips are made from (as = ""). diff --git a/ventures/havre/venice.js b/ventures/havre/venice.js index 654d7b9..d10723d 100644 --- a/ventures/havre/venice.js +++ b/ventures/havre/venice.js @@ -141,8 +141,9 @@ export class Venice { } /** a voice note: words read out by a voice model. mood goes to models that take a style prompt; the rest ignore it. */ - async speak({ model, voice, input, mood = "", speed = 1, signal } = {}) { + async speak({ model, voice, input, mood = "", speed = 1, format = "", signal } = {}) { const body = { input: String(input || "").slice(0, 4096), model, speed }; + if (format) body.response_format = format; if (voice) body.voice = voice; if (mood) body.prompt = String(mood).slice(0, 500); const res = await this.raw("/audio/speech", { body, signal }); diff --git a/ventures/havre/workflows.js b/ventures/havre/workflows.js index 1e5d13e..18d50b3 100644 --- a/ventures/havre/workflows.js +++ b/ventures/havre/workflows.js @@ -33,6 +33,11 @@ export const WORKFLOWS = [ about: "a few photos of them from the pictures on their card, one shoot, each a different moment (3 unless you say)", usage: "/photoshoot [how many] [what the shoot is about]", }, + { + id: "whisper", command: "whisper", title: "a whisper", takes: "card", gives: "voice note", doors: ["chat"], + about: "a short ASMR voice note from them, whispered over soft rain (luna or eve; add what it's about)", + usage: "/whisper [luna|eve] [what it's about]", + }, { id: "recall", command: "recall", title: "recall", takes: "words", gives: "memory", doors: ["chat"], about: "what they remember about something (their notes, then the chats themselves), brought to mind for a while", @@ -128,6 +133,51 @@ export function cardFromIdea(o, { model } = {}) { return normalizeCard({ ...base, look: { ...base.look, description: look } }); } +// ---------- whisper: a short ASMR voice note, over soft rain ---------- + +// The voices that whisper well and cheaply (picked in the asmr.gandora.lol bake-off): Venice model, voice, how the +// words are wrapped so the model whispers, and the pace (0.9 of their usual reading speed sounded best). +export const WHISPER_PRESETS = { + luna: { model: "tts-inworld-1-5-max", voice: "Luna", lead: "[whispering] ", tail: "", speed: 0.77, format: "wav" }, + eve: { model: "tts-xai-v1", voice: "eve", lead: "", tail: "", speed: 0.77, format: "mp3" }, +}; +export const WHISPER_DEFAULT = "luna"; + +/** the words after /whisper: a preset name first if there is one, the rest what it's about */ +export function parseWhisper(arg = "") { + const m = /^\s*(\w+)(?:\s+|$)([\s\S]*)$/.exec(str(arg, 2000)); + const first = m ? m[1].toLowerCase() : ""; + if (WHISPER_PRESETS[first]) return { preset: first, brief: m[2].replace(/\s+/g, " ").trim() }; + return { preset: "", brief: str(arg, 2000).replace(/\s+/g, " ").trim() }; +} + +export const WHISPER_SCHEMA = { + type: "object", additionalProperties: false, required: ["script"], + properties: { script: { type: "string" } }, +}; + +export function whisperMessages(card, { brief = "", lines = [] } = {}) { + const system = `You are ${card.name}. Write a short ASMR voice note you whisper to the person you're talking with: ` + + "60 to 110 words, in your own voice and manner, intimate and calming, present tense, short phrases with pauses " + + "written as \"...\". Gentle sensory detail (breath, rain, a soft sound nearby) is welcome. No stage directions, no " + + "emoji, no markup, nothing in brackets: only the words to be whispered. Return one JSON object: script."; + const persona = card.persona ? clip(card.persona.replace(/\s+/g, " "), 900) : ""; + const user = [ + persona ? `Who you are: ${persona}` : "", + lines.length ? "The conversation so far, latest last:\n" + lines.join("\n") : "You haven't talked yet.", + "What it's about: " + (str(brief, 500).trim() || "winding down together; help them relax."), + ].filter(Boolean).join("\n\n"); + return [{ role: "system", content: system }, { role: "user", content: user }]; +} + +/** the script, cleaned of anything the model shouldn't have added (brackets, asterisks, emoji-ish markup) */ +export function cleanWhisper(o) { + return str(o && o.script, 1500).replace(/\[[^\]]*\]|\([^)]*\)|<[^>]*>|\*[^*]*\*/g, "").replace(/\s+/g, " ").trim(); +} + +/** what the voice model is sent: the script inside the preset's whisper markup */ +export const whisperInput = (script, preset) => (preset.lead || "") + script + (preset.tail || ""); + // ---------- photoshoot: a few photos of them, from the pictures on their card ---------- export const SHOOT_MIN = 1, SHOOT_MAX = 6, SHOOT_DEFAULT = 3;