From 9f226c2ea948d5abbb73605e6634d67b269eb754 Mon Sep 17 00:00:00 2001 From: techartdev Date: Fri, 20 Feb 2026 21:31:01 +0200 Subject: [PATCH] Add voice input provider options and update integration settings - Introduced configurable voice input provider option: `browser` or `assist_stt`. - Enhanced chat card to support Home Assistant STT transcription mode for manual mic input. - Updated integration settings to expose voice provider through websocket payload. - Adjusted continuous voice mode availability based on selected voice provider. - Updated version to 0.1.32 in manifest and chat card files. --- CHANGELOG.md | 10 + README.md | 1 + custom_components/openclaw/__init__.py | 8 +- custom_components/openclaw/config_flow.py | 9 + custom_components/openclaw/const.py | 2 + custom_components/openclaw/manifest.json | 2 +- custom_components/openclaw/strings.json | 3 +- .../openclaw/translations/en.json | 3 +- .../openclaw/www/openclaw-chat-card.js | 285 +++++++++++++++++- www/openclaw-chat-card.js | 2 +- 10 files changed, 317 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7646a7f..a2a48ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ All notable changes to the OpenClaw Home Assistant Integration will be documented in this file. +## [0.1.32] - 2026-02-20 + +### Added +- Added configurable voice input provider option: `browser` or `assist_stt`. +- Chat card now supports Home Assistant STT transcription mode (`assist_stt`) for manual mic input. + +### Changed +- Voice provider is now exposed through integration settings websocket payload and card configuration handling. +- Continuous voice mode remains available only for browser voice provider. + ## [0.1.31] - 2026-02-20 ### Fixed diff --git a/README.md b/README.md index c257d7e..d688a7e 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,7 @@ When enabled, OpenClaw tool-call responses can execute Home Assistant services. - **Wake word enabled** - **Wake word** (default: `hey openclaw`) - **Always voice mode** (continuous listening while card is open) +- **Voice input provider** (`browser` or `assist_stt`) --- diff --git a/custom_components/openclaw/__init__.py b/custom_components/openclaw/__init__.py index 3d956f2..53c6766 100644 --- a/custom_components/openclaw/__init__.py +++ b/custom_components/openclaw/__init__.py @@ -47,6 +47,7 @@ from .const import ( CONF_WAKE_WORD_ENABLED, CONF_ALWAYS_VOICE_MODE, CONF_ALLOW_BRAVE_WEBSPEECH, + CONF_VOICE_PROVIDER, CONTEXT_STRATEGY_TRUNCATE, DEFAULT_CONTEXT_MAX_CHARS, DEFAULT_CONTEXT_STRATEGY, @@ -56,6 +57,7 @@ from .const import ( DEFAULT_WAKE_WORD_ENABLED, DEFAULT_ALWAYS_VOICE_MODE, DEFAULT_ALLOW_BRAVE_WEBSPEECH, + DEFAULT_VOICE_PROVIDER, DOMAIN, EVENT_MESSAGE_RECEIVED, OPENCLAW_CONFIG_REL_PATH, @@ -76,7 +78,7 @@ _CARD_PATH = Path(__file__).parent / "www" / _CARD_FILENAME # URL at which the card JS is served (registered via register_static_path) _CARD_STATIC_URL = f"/openclaw/{_CARD_FILENAME}" # Versioned URL used for Lovelace resource registration to avoid stale browser cache -_CARD_URL = f"{_CARD_STATIC_URL}?v=0.1.30" +_CARD_URL = f"{_CARD_STATIC_URL}?v=0.1.32" OpenClawConfigEntry = ConfigEntry @@ -689,6 +691,10 @@ def _async_register_websocket_api(hass: HomeAssistant) -> None: CONF_ALLOW_BRAVE_WEBSPEECH, DEFAULT_ALLOW_BRAVE_WEBSPEECH, ), + CONF_VOICE_PROVIDER: options.get( + CONF_VOICE_PROVIDER, + DEFAULT_VOICE_PROVIDER, + ), "language": hass.config.language, }, ) diff --git a/custom_components/openclaw/config_flow.py b/custom_components/openclaw/config_flow.py index 6507048..6062c30 100644 --- a/custom_components/openclaw/config_flow.py +++ b/custom_components/openclaw/config_flow.py @@ -46,6 +46,7 @@ from .const import ( CONF_WAKE_WORD_ENABLED, CONF_ALWAYS_VOICE_MODE, CONF_ALLOW_BRAVE_WEBSPEECH, + CONF_VOICE_PROVIDER, CONTEXT_STRATEGY_CLEAR, CONTEXT_STRATEGY_TRUNCATE, DEFAULT_GATEWAY_HOST, @@ -58,6 +59,7 @@ from .const import ( DEFAULT_WAKE_WORD_ENABLED, DEFAULT_ALWAYS_VOICE_MODE, DEFAULT_ALLOW_BRAVE_WEBSPEECH, + DEFAULT_VOICE_PROVIDER, DOMAIN, OPENCLAW_CONFIG_REL_PATH, ) @@ -474,6 +476,13 @@ class OpenClawOptionsFlow(OptionsFlow): DEFAULT_ALLOW_BRAVE_WEBSPEECH, ), ): bool, + vol.Optional( + CONF_VOICE_PROVIDER, + default=options.get( + CONF_VOICE_PROVIDER, + DEFAULT_VOICE_PROVIDER, + ), + ): vol.In(["browser", "assist_stt"]), } ), ) diff --git a/custom_components/openclaw/const.py b/custom_components/openclaw/const.py index b9fe3d2..81e4b98 100644 --- a/custom_components/openclaw/const.py +++ b/custom_components/openclaw/const.py @@ -32,6 +32,7 @@ CONF_WAKE_WORD_ENABLED = "wake_word_enabled" CONF_WAKE_WORD = "wake_word" CONF_ALWAYS_VOICE_MODE = "always_voice_mode" CONF_ALLOW_BRAVE_WEBSPEECH = "allow_brave_webspeech" +CONF_VOICE_PROVIDER = "voice_provider" DEFAULT_INCLUDE_EXPOSED_CONTEXT = True DEFAULT_CONTEXT_MAX_CHARS = 13000 @@ -41,6 +42,7 @@ DEFAULT_WAKE_WORD_ENABLED = False DEFAULT_WAKE_WORD = "hey openclaw" DEFAULT_ALWAYS_VOICE_MODE = False DEFAULT_ALLOW_BRAVE_WEBSPEECH = False +DEFAULT_VOICE_PROVIDER = "browser" CONTEXT_STRATEGY_TRUNCATE = "truncate" CONTEXT_STRATEGY_CLEAR = "clear" diff --git a/custom_components/openclaw/manifest.json b/custom_components/openclaw/manifest.json index 1afdc16..16651d4 100644 --- a/custom_components/openclaw/manifest.json +++ b/custom_components/openclaw/manifest.json @@ -8,7 +8,7 @@ "iot_class": "local_polling", "issue_tracker": "https://github.com/techartdev/OpenClawHomeAssistant/issues", "requirements": [], - "version": "0.1.31", + "version": "0.1.32", "dependencies": ["conversation"], "after_dependencies": ["hassio", "lovelace"] } diff --git a/custom_components/openclaw/strings.json b/custom_components/openclaw/strings.json index c39a3f1..e2401ea 100644 --- a/custom_components/openclaw/strings.json +++ b/custom_components/openclaw/strings.json @@ -44,7 +44,8 @@ "wake_word_enabled": "Require wake word", "wake_word": "Wake word", "always_voice_mode": "Always-on voice mode", - "allow_brave_webspeech": "Allow Web Speech in Brave (experimental)" + "allow_brave_webspeech": "Allow Web Speech in Brave (experimental)", + "voice_provider": "Voice input provider (browser or HA STT)" } } } diff --git a/custom_components/openclaw/translations/en.json b/custom_components/openclaw/translations/en.json index c39a3f1..e2401ea 100644 --- a/custom_components/openclaw/translations/en.json +++ b/custom_components/openclaw/translations/en.json @@ -44,7 +44,8 @@ "wake_word_enabled": "Require wake word", "wake_word": "Wake word", "always_voice_mode": "Always-on voice mode", - "allow_brave_webspeech": "Allow Web Speech in Brave (experimental)" + "allow_brave_webspeech": "Allow Web Speech in Brave (experimental)", + "voice_provider": "Voice input provider (browser or HA STT)" } } } diff --git a/custom_components/openclaw/www/openclaw-chat-card.js b/custom_components/openclaw/www/openclaw-chat-card.js index d9eb274..1fb7fc4 100644 --- a/custom_components/openclaw/www/openclaw-chat-card.js +++ b/custom_components/openclaw/www/openclaw-chat-card.js @@ -13,7 +13,7 @@ * + subscribes to openclaw_message_received events. */ -const CARD_VERSION = "0.2.9"; +const CARD_VERSION = "0.3.0"; // Max time (ms) to show the thinking indicator before falling back to an error const THINKING_TIMEOUT_MS = 120_000; @@ -72,8 +72,19 @@ class OpenClawChatCard extends HTMLElement { this._integrationVoiceLanguage = null; this._integrationTtsLanguage = null; this._allowBraveWebSpeechIntegration = false; + this._voiceProviderIntegration = "browser"; + this._preferredAssistSttEngine = null; this._voiceBackendBlocked = false; this._voiceStopRequested = false; + this._assistRecordingActive = false; + this._assistAudioStream = null; + this._assistAudioContext = null; + this._assistAudioSource = null; + this._assistAudioProcessor = null; + this._assistAudioSilenceGain = null; + this._assistAudioChunks = []; + this._assistInputSampleRate = 16000; + this._assistAutoStopTimer = null; } // ── HA card interface ─────────────────────────────────────────────── @@ -94,6 +105,7 @@ class OpenClawChatCard extends HTMLElement { show_voice_button: config.show_voice_button !== false, show_clear_button: config.show_clear_button !== false, allow_brave_webspeech: config.allow_brave_webspeech === true, + voice_provider: config.voice_provider || null, session_id: config.session_id || null, ...config, }; @@ -317,6 +329,8 @@ class OpenClawChatCard extends HTMLElement { this._wakeWord = (result?.wake_word || "hey openclaw").toString().trim().toLowerCase(); this._alwaysVoiceMode = !!result?.always_voice_mode; this._allowBraveWebSpeechIntegration = !!result?.allow_brave_webspeech; + this._voiceProviderIntegration = + result?.voice_provider === "assist_stt" ? "assist_stt" : "browser"; this._integrationVoiceLanguage = result?.language ? this._normalizeSpeechLanguage(result.language) : null; @@ -337,6 +351,7 @@ class OpenClawChatCard extends HTMLElement { : []; const preferredPipeline = pipelines.find((pipeline) => pipeline?.id === preferredPipelineId) || pipelines[0]; + this._preferredAssistSttEngine = preferredPipeline?.stt_engine || null; const sttLanguage = preferredPipeline?.stt_language || preferredPipeline?.language; const ttsLanguage = @@ -536,7 +551,258 @@ class OpenClawChatCard extends HTMLElement { return brandMatch || ua.includes("brave") || !!navigator.brave; } + _getVoiceProvider() { + const configured = this._config.voice_provider; + if (configured === "assist_stt" || configured === "browser") { + return configured; + } + return this._voiceProviderIntegration || "browser"; + } + _startVoiceRecognition() { + const provider = this._getVoiceProvider(); + if (provider === "assist_stt") { + this._startAssistSttRecognition(); + return; + } + this._startBrowserVoiceRecognition(); + } + + async _startAssistSttRecognition() { + if (!this._hass) return; + + if (this._isVoiceMode) { + this._voiceStatus = + "Continuous voice mode currently requires browser speech. Switch voice provider to browser or disable continuous mode."; + this._render(); + return; + } + + if (!navigator.mediaDevices?.getUserMedia) { + this._voiceStatus = "Microphone capture not supported by this browser."; + this._render(); + return; + } + + if (!this._preferredAssistSttEngine) { + this._voiceStatus = + "No Assist STT engine found in preferred voice pipeline. Configure Voice settings first."; + this._render(); + return; + } + + try { + this._assistAudioChunks = []; + this._assistAudioStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + this._assistAudioContext = new (window.AudioContext || window.webkitAudioContext)({ + sampleRate: 16000, + }); + this._assistInputSampleRate = this._assistAudioContext.sampleRate || 16000; + + this._assistAudioSource = this._assistAudioContext.createMediaStreamSource(this._assistAudioStream); + this._assistAudioProcessor = this._assistAudioContext.createScriptProcessor(4096, 1, 1); + this._assistAudioSilenceGain = this._assistAudioContext.createGain(); + this._assistAudioSilenceGain.gain.value = 0; + + this._assistAudioProcessor.onaudioprocess = (event) => { + const input = event.inputBuffer.getChannelData(0); + this._assistAudioChunks.push(new Float32Array(input)); + }; + + this._assistAudioSource.connect(this._assistAudioProcessor); + this._assistAudioProcessor.connect(this._assistAudioSilenceGain); + this._assistAudioSilenceGain.connect(this._assistAudioContext.destination); + + this._assistRecordingActive = true; + this._recognition = { engine: "assist_stt" }; + this._voiceStatus = "Listening with Home Assistant STT…"; + this._render(); + + if (this._assistAutoStopTimer) { + clearTimeout(this._assistAutoStopTimer); + } + this._assistAutoStopTimer = setTimeout(() => { + this._stopAssistSttRecognition(true); + }, 4500); + } catch (err) { + console.error("OpenClaw: failed to start Assist STT recording", err); + this._voiceStatus = "Could not start microphone recording for Assist STT."; + this._assistRecordingActive = false; + this._recognition = null; + this._render(); + } + } + + async _stopAssistSttRecognition(processAudio) { + if (this._assistAutoStopTimer) { + clearTimeout(this._assistAutoStopTimer); + this._assistAutoStopTimer = null; + } + + if (this._assistAudioProcessor) { + this._assistAudioProcessor.disconnect(); + this._assistAudioProcessor.onaudioprocess = null; + this._assistAudioProcessor = null; + } + if (this._assistAudioSource) { + this._assistAudioSource.disconnect(); + this._assistAudioSource = null; + } + if (this._assistAudioSilenceGain) { + this._assistAudioSilenceGain.disconnect(); + this._assistAudioSilenceGain = null; + } + if (this._assistAudioStream) { + for (const track of this._assistAudioStream.getTracks()) { + track.stop(); + } + this._assistAudioStream = null; + } + if (this._assistAudioContext) { + try { + await this._assistAudioContext.close(); + } catch (e) { + // ignore + } + this._assistAudioContext = null; + } + + this._assistRecordingActive = false; + this._recognition = null; + + if (!processAudio) { + this._voiceStatus = ""; + this._render(); + return; + } + + if (!this._assistAudioChunks.length) { + this._voiceStatus = "No speech detected. Tap mic and try again."; + this._render(); + return; + } + + this._voiceStatus = "Transcribing with Home Assistant STT…"; + this._render(); + + try { + const language = this._getSpeechRecognitionLanguage(); + const wavBuffer = this._encodeWavFromChunks(this._assistAudioChunks, this._assistInputSampleRate); + this._assistAudioChunks = []; + + const token = this._hass?.auth?.data?.access_token; + const response = await fetch( + `/api/stt/${encodeURIComponent(this._preferredAssistSttEngine)}`, + { + method: "POST", + headers: { + "Content-Type": "audio/wav", + "X-Speech-Content": + `format=wav; codec=pcm; sample_rate=16000; bit_rate=16; channel=1; language=${language}`, + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }, + body: new Blob([wavBuffer], { type: "audio/wav" }), + } + ); + + if (!response.ok) { + this._voiceStatus = `Assist STT error (${response.status}).`; + this._render(); + return; + } + + const sttResult = await response.json(); + const transcript = String(sttResult?.text || "").trim(); + if (!transcript) { + this._voiceStatus = "No speech recognized by Assist STT."; + this._render(); + return; + } + + this._voiceStatus = "Sending…"; + this._render(); + this._sendMessage(transcript); + } catch (err) { + console.error("OpenClaw: Assist STT transcription failed", err); + this._voiceStatus = "Assist STT transcription failed."; + this._render(); + } + } + + _downsampleBuffer(input, inputSampleRate, outputSampleRate) { + if (outputSampleRate >= inputSampleRate) { + return input; + } + const sampleRateRatio = inputSampleRate / outputSampleRate; + const outputLength = Math.round(input.length / sampleRateRatio); + const outputBuffer = new Float32Array(outputLength); + + let outputOffset = 0; + let inputOffset = 0; + while (outputOffset < outputLength) { + const nextInputOffset = Math.round((outputOffset + 1) * sampleRateRatio); + let accumulator = 0; + let count = 0; + for (let idx = inputOffset; idx < nextInputOffset && idx < input.length; idx += 1) { + accumulator += input[idx]; + count += 1; + } + outputBuffer[outputOffset] = count > 0 ? accumulator / count : 0; + outputOffset += 1; + inputOffset = nextInputOffset; + } + + return outputBuffer; + } + + _encodeWavFromChunks(chunks, inputSampleRate) { + let totalLength = 0; + for (const chunk of chunks) { + totalLength += chunk.length; + } + + const merged = new Float32Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + merged.set(chunk, offset); + offset += chunk.length; + } + + const downsampled = this._downsampleBuffer(merged, inputSampleRate, 16000); + const buffer = new ArrayBuffer(44 + downsampled.length * 2); + const view = new DataView(buffer); + + const writeString = (dataView, start, value) => { + for (let idx = 0; idx < value.length; idx += 1) { + dataView.setUint8(start + idx, value.charCodeAt(idx)); + } + }; + + writeString(view, 0, "RIFF"); + view.setUint32(4, 36 + downsampled.length * 2, true); + writeString(view, 8, "WAVE"); + writeString(view, 12, "fmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); + view.setUint32(24, 16000, true); + view.setUint32(28, 16000 * 2, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + writeString(view, 36, "data"); + view.setUint32(40, downsampled.length * 2, true); + + let wavOffset = 44; + for (let idx = 0; idx < downsampled.length; idx += 1) { + const sample = Math.max(-1, Math.min(1, downsampled[idx])); + view.setInt16(wavOffset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true); + wavOffset += 2; + } + + return buffer; + } + + _startBrowserVoiceRecognition() { this._voiceBackendBlocked = false; this._voiceStopRequested = false; const allowBraveWebSpeech = @@ -683,9 +949,15 @@ class OpenClawChatCard extends HTMLElement { } _stopVoiceRecognition() { + if (this._assistRecordingActive) { + this._stopAssistSttRecognition(true); + } + if (this._recognition) { - this._voiceStopRequested = true; - this._recognition.abort(); + if (this._recognition.engine !== "assist_stt") { + this._voiceStopRequested = true; + this._recognition.abort(); + } this._recognition = null; } if (this._voiceRetryTimer) { @@ -725,6 +997,13 @@ class OpenClawChatCard extends HTMLElement { } _toggleVoiceMode() { + if (!this._isVoiceMode && this._getVoiceProvider() === "assist_stt") { + this._voiceStatus = + "Continuous voice mode is only available with browser voice provider."; + this._render(); + return; + } + this._isVoiceMode = !this._isVoiceMode; if (this._isVoiceMode) { this._startVoiceRecognition(); diff --git a/www/openclaw-chat-card.js b/www/openclaw-chat-card.js index 1c7b6a4..d098aa3 100644 --- a/www/openclaw-chat-card.js +++ b/www/openclaw-chat-card.js @@ -1,7 +1,7 @@ (async () => { try { if (!customElements.get("openclaw-chat-card")) { - const src = "/openclaw/openclaw-chat-card.js?v=0.1.30"; + const src = "/openclaw/openclaw-chat-card.js?v=0.1.32"; console.info("OpenClaw loader importing", src); await import(src); }