From e5020d24febf65a5db05e253ec88c351a071e28e Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Tue, 8 Sep 2026 15:26:44 +0100 Subject: [PATCH] Pin model downloads: fixed commit, SHA-256 and size checks, transfer deadline --- README.md | 2 +- daemon/sttd.py | 75 ++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index c4dcc73..90247e7 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Dictation for Omarchy: press a key, talk, press it again, and the words are past - **Never miss the first words**: Settings → Advanced → "Keep the microphone open" keeps the stream running between recordings, so a recording starts instantly and even includes the half second before the key press. (Not for a Bluetooth headset: it would stay in headset mode all the time.) - **Stop is instant**: the recording is transcribed at every pause while you talk, so only the last phrase is left when you stop. - **History** of every recording (text + audio) with play, copy and delete, searchable, in the bar popup; kept for a month by default (Settings). -- **Languages** picked from Whisper's list; the model a language needs is downloaded by itself the first time (about 150 MB for the default model; the bar shows the progress). English-only models are swapped for the multilingual one. +- **Languages** picked from Whisper's list; the model a language needs is downloaded by itself the first time, from a pinned commit of ggerganov/whisper.cpp and only kept when its size and SHA-256 match the checksums shipped in the daemon (about 150 MB for the default model; the bar shows the progress). English-only models are swapped for the multilingual one. - **Engine**: Omarchy's own dictation engine, [voxtype](https://github.com/peteonrails/voxtype) (local Whisper), by default, so there is nothing new to install. whisper.cpp (`whisper-cli`) and any custom command are also supported (Settings → Advanced). - **Safe key bindings**: a key that anything else already uses is refused, never taken over. - Local only. Nothing leaves your machine. diff --git a/daemon/sttd.py b/daemon/sttd.py index 6db81c9..b9b8680 100755 --- a/daemon/sttd.py +++ b/daemon/sttd.py @@ -9,6 +9,7 @@ and the `stt` CLI are its clients. Standard library only. import asyncio import fcntl +import hashlib import json import math import os @@ -91,7 +92,47 @@ UI_STRINGS = { "ca": {"listening": "Escoltant…", "opening": "Obrint el micròfon…", "transcribing": "Transcrivint…"}, } VOXTYPE_MODELS = os.environ.get("STT_MODELS_DIR") or os.path.join(os.environ.get("XDG_DATA_HOME", os.path.join(HOME, ".local", "share")), "voxtype", "models") -MODEL_URL = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-{model}.bin" +# Models come from one pinned commit of ggerganov/whisper.cpp on Hugging Face, and a +# download is only used when its size and SHA-256 match this table (taken from that +# commit's LFS pointers). An unlisted model name is refused rather than fetched. +MODEL_COMMIT = "5359861c739e955e79d9a303bcbc70fb988958b1" +MODEL_URL = "https://huggingface.co/ggerganov/whisper.cpp/resolve/" + MODEL_COMMIT + "/ggml-{model}.bin" +MODEL_DIGESTS = { # name: (sha256, bytes) + "base": ("60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe", 147951465), + "base-q5_1": ("422f1ae452ade6f30a004d7e5c6a43195e4433bc370bf23fac9cc591f01a8898", 59707625), + "base-q8_0": ("c577b9a86e7e048a0b7eada054f4dd79a56bbfa911fbdacf900ac5b567cbb7d9", 81768585), + "base.en": ("a03779c86df3323075f5e796cb2ce5029f00ec8869eee3fdfb897afe36c6d002", 147964211), + "base.en-q5_1": ("4baf70dd0d7c4247ba2b81fafd9c01005ac77c2f9ef064e00dcf195d0e2fdd2f", 59721011), + "base.en-q8_0": ("a4d4a0768075e13cfd7e19df3ae2dbc4a68d37d36a7dad45e8410c9a34f8c87e", 81781811), + "large-v1": ("7d99f41a10525d0206bddadd86760181fa920438b6b33237e3118ff6c83bb53d", 3094623691), + "large-v2": ("9a423fe4d40c82774b6af34115b8b935f34152246eb19e80e376071d3f999487", 3094623691), + "large-v2-q5_0": ("3a214837221e4530dbc1fe8d734f302af393eb30bd0ed046042ebf4baf70f6f2", 1080732091), + "large-v2-q8_0": ("fef54e6d898246a65c8285bfa83bd1807e27fadf54d5d4e81754c47634737e8c", 1656129691), + "large-v3": ("64d182b440b98d5203c4f9bd541544d84c605196c4f7b845dfa11fb23594d1e2", 3095033483), + "large-v3-q5_0": ("d75795ecff3f83b5faa89d1900604ad8c780abd5739fae406de19f23ecd98ad1", 1081140203), + "large-v3-turbo": ("1fc70f774d38eb169993ac391eea357ef47c88757ef72ee5943879b7e8e2bc69", 1624555275), + "large-v3-turbo-q5_0": ("394221709cd5ad1f40c46e6031ca61bce88931e6e088c188294c6d5a55ffa7e2", 574041195), + "large-v3-turbo-q8_0": ("317eb69c11673c9de1e1f0d459b253999804ec71ac4c23c17ecf5fbe24e259a1", 874188075), + "medium": ("6c14d5adee5f86394037b4e4e8b59f1673b6cee10e3cf0b11bbdbee79c156208", 1533763059), + "medium-q5_0": ("19fea4b380c3a618ec4723c3eef2eb785ffba0d0538cf43f8f235e7b3b34220f", 539212467), + "medium-q8_0": ("42a1ffcbe4167d224232443396968db4d02d4e8e87e213d3ee2e03095dea6502", 823369779), + "medium.en": ("cc37e93478338ec7700281a7ac30a10128929eb8f427dda2e865faa8f6da4356", 1533774781), + "medium.en-q5_0": ("76733e26ad8fe1c7a5bf7531a9d41917b2adc0f20f2e4f5531688a8c6cd88eb0", 539225533), + "medium.en-q8_0": ("43fa2cd084de5a04399a896a9a7a786064e221365c01700cea4666005218f11c", 823382461), + "small": ("1be3a9b2063867b937e64e2ec7483364a79917e157fa98c5d94b5c1fffea987b", 487601967), + "small-q5_1": ("ae85e4a935d7a567bd102fe55afc16bb595bdb618e11b2fc7591bc08120411bb", 190085487), + "small-q8_0": ("49c8fb02b65e6049d5fa6c04f81f53b867b5ec9540406812c643f177317f779f", 264464607), + "small.en": ("c6138d6d58ecc8322097e0f987c32f1be8bb0a18532a3f88f734d1bbf9c41e5d", 487614201), + "small.en-q5_1": ("bfdff4894dcb76bbf647d56263ea2a96645423f1669176f4844a1bf8e478ad30", 190098681), + "small.en-q8_0": ("67a179f608ea6114bd3fdb9060e762b588a3fb3bd00c4387971be4d177958067", 264477561), + "tiny": ("be07e048e1e599ad46341c8d2a135645097a538221678b7acdd1b1919c6e1b21", 77691713), + "tiny-q5_1": ("818710568da3ca15689e31a743197b520007872ff9576237bda97bd1b469c3d7", 32152673), + "tiny-q8_0": ("c2085835d3f50733e2ff6e4b41ae8a2b8d8110461e18821b09a15c40c42d1cca", 43537433), + "tiny.en": ("921e4cf8686fdd993dcd081a5da5b6c365bfde1162e72b08d75ac75289920b1f", 77704715), + "tiny.en-q5_1": ("c77c5766f1cef09b6b7d47f21b546cbddd4157886b3b5d6d4f709e91e66c7c2b", 32166155), + "tiny.en-q8_0": ("5bc2b3860aa151a4c6e7bb095e1fcce7cf12c7b020ca08dcec0c6d018bb7dd94", 43550795), +} +MODEL_DOWNLOAD_DEADLINE = 3600 # seconds for the whole transfer, the largest model is ~3 GB DEFAULT_CONFIG = { "engine": "voxtype", # voxtype | whisper-cpp | command @@ -679,6 +720,14 @@ def model_path(model): return os.path.join(VOXTYPE_MODELS, f"ggml-{model}.bin") +def sha256_file(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + class EngineRun: """The transcription process now running (if any), so cancel/shutdown can kill it.""" proc = None @@ -1113,6 +1162,10 @@ class Daemon: return model async def fetch_model(self, model): + if model not in MODEL_DIGESTS: + self.fail(f"No pinned checksum for the {model} model; put ggml-{model}.bin in {VOXTYPE_MODELS} yourself") + return + want_sha, want_size = MODEL_DIGESTS[model] os.makedirs(VOXTYPE_MODELS, exist_ok=True) dest = model_path(model) fd, part = tempfile.mkstemp(prefix=f"ggml-{model}.", suffix=".part", dir=VOXTYPE_MODELS) @@ -1123,14 +1176,11 @@ class Daemon: log("downloading model", model) proc = None try: - head = await self.loop.run_in_executor(None, lambda: subprocess.run( - ["curl", "-sIL", url], capture_output=True, text=True, timeout=60)) - total = 0 - for line in head.stdout.splitlines(): - if line.lower().startswith("content-length:"): - total = int(line.split(":", 1)[1].strip() or 0) proc = await asyncio.create_subprocess_exec( - "curl", "-sSL", "--fail", "-o", part, url, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE) + "curl", "-sSL", "--fail", "--proto", "=https", + "--max-filesize", str(want_size), "--max-time", str(MODEL_DOWNLOAD_DEADLINE), + "--speed-limit", "1000", "--speed-time", "60", # give up when stalled for a minute + "-o", part, url, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE) while proc.returncode is None: try: await asyncio.wait_for(proc.wait(), timeout=0.5) @@ -1140,14 +1190,17 @@ class Daemon: done = os.path.getsize(part) except OSError: done = 0 - self.download["pct"] = int(done * 100 / total) if total else 0 + self.download["pct"] = int(done * 100 / want_size) self.broadcast() err = (await proc.stderr.read()).decode(errors="replace").strip() if proc.returncode != 0: raise RuntimeError(err.splitlines()[-1] if err else f"curl exited {proc.returncode}") size = os.path.getsize(part) - if size < 1_000_000 or (total and size != total): - raise RuntimeError(f"incomplete file ({size} of {total} bytes)") + if size != want_size: + raise RuntimeError(f"unexpected size ({size} bytes, expected {want_size})") + got_sha = await self.loop.run_in_executor(None, sha256_file, part) + if got_sha != want_sha: + raise RuntimeError("checksum mismatch; the file was discarded") os.chmod(part, 0o644) # mkstemp makes it private; models are plain shared files os.replace(part, dest) log("model ready", dest)