""" muse — request a song, get it in Navidrome. The gap this fills: Lidarr thinks in MusicBrainz *releases*, so the smallest thing it can want is an album. Soulseek trades individual files. Nothing in between existed, so requesting one song meant either hoovering a whole discography through Lidarr or hand-filing downloads out of slskd's UI. The trick that makes this simple: the user PICKS a specific track before anything is downloaded. That means we hold ground-truth metadata — artist, album, year, track number — before the file exists. Taggers like beets have to infer all of it afterwards from whatever a stranger named the file; we never guess. We tag what was asked for. Flow: /api/search Deezer track search -> the user picks one /api/request /track/{id} for exact metadata, then slskd search -> score candidates -> enqueue best worker poll slskd -> on completion: tag, then hardlink into /data/media/music// ()/NN - Title.ext Hardlink, not copy: the download stays in slskd's completed dir so it keeps being shared, while the library gets its own name for the same bytes. One file on disk, two names — the same trick the *arrs use. """ import asyncio import base64 import collections import difflib import hashlib import json import logging import os import re import secrets import sqlite3 import unicodedata import uuid from contextlib import asynccontextmanager from datetime import datetime, timedelta, timezone from pathlib import Path from urllib.parse import urlencode import httpx import mutagen from fastapi import FastAPI, HTTPException, Query, Request from fastapi.responses import (FileResponse, HTMLResponse, JSONResponse, RedirectResponse, Response) from fastapi.staticfiles import StaticFiles from pydantic import BaseModel log = logging.getLogger("muse") logging.basicConfig( level=os.getenv("MUSE_LOG_LEVEL", "INFO"), format="[%(levelname)s|%(name)s] %(asctime)s: %(message)s", datefmt="%Y-%m-%dT%H:%M:%S%z", ) SLSKD_URL = os.getenv("SLSKD_URL", "http://192.168.1.87:5030").rstrip("/") SLSKD_API_KEY = os.getenv("SLSKD_API_KEY", "") DOWNLOAD_DIR = Path(os.getenv("MUSE_DOWNLOAD_DIR", "/data/torrents/soulseek/complete")) LIBRARY_DIR = Path(os.getenv("MUSE_LIBRARY_DIR", "/data/media/music")) DB_PATH = Path(os.getenv("MUSE_DB", "/config/muse.db")) # Ordered best-first. A candidate's extension must appear here or it is # rejected outright — this is what keeps .m4a rips and stray .cue files out. FORMATS = [f.strip() for f in os.getenv("MUSE_FORMATS", "flac,mp3").split(",") if f.strip()] # Below this, the remote filename doesn't plausibly contain the track title. # Soulseek filenames are chaotic ("08-radiohead-creep_(accoustic)-ent.mp3"), # so this compares against a normalised form, not the raw string. MIN_MATCH = float(os.getenv("MUSE_MIN_MATCH", "0.45")) # The title alone must clear this before the artist is even consulted. Without # a floor of its own, a file by the right artist scored well enough on the # combined ratio to be fetched no matter which of their songs it actually was. TITLE_FLOOR = float(os.getenv("MUSE_TITLE_FLOOR", "0.5")) # How many ranked candidates to keep per job. When a transfer fails, the next # one is tried instead of failing the request -- "File not shared." is a # per-file condition, so candidate two usually just works. CANDIDATES = int(os.getenv("MUSE_CANDIDATES", "8")) # Defaults to the size of the ladder, because keeping eight candidates and then # refusing to try three of them is a cap with no reasoning behind it. Measured # on a live instance: slskd answers 500 whenever it cannot open a connection to # a peer, which is common enough that requests routinely need four or five # attempts. One succeeded on exactly the fifth; another gave up at the cap with # three candidates still in hand. MAX_ATTEMPTS = int(os.getenv("MUSE_MAX_ATTEMPTS", str(CANDIDATES))) # A download that has moved no bytes for this long is not going to. The peer # queued it and vanished, or slskd lost the transfer; either way the job used # to sit in `downloading` forever behind a cheerful pulsing dot. STALL_MINUTES = float(os.getenv("MUSE_STALL_MINUTES", "20")) # Query variants to try before giving up. Each costs a real Soulseek search, # so this escalates only when a search underdelivers -- see search_queries(). MAX_QUERIES = int(os.getenv("MUSE_MAX_QUERIES", "4")) MIN_CANDIDATES = int(os.getenv("MUSE_MIN_CANDIDATES", "3")) # Issuing a search is a POST that occasionally blips. Nine production failures # were one slskd hiccup during a bulk import, every queued job dying inside the # same second, each one permanently. SEARCH_RETRIES = int(os.getenv("MUSE_SEARCH_RETRIES", "3")) # Soulseek results trickle in: what you have at t=5s is much worse than t=15s. SEARCH_POLLS = int(os.getenv("MUSE_SEARCH_POLLS", "20")) # Pause between candidate attempts. Without one, a ladder of 8 walks the whole # list inside a single bad slskd window -- each enqueue blocks ~5s on a peer # connection, so eight attempts is 40 seconds of hammering one subsystem, and # it takes other people's searches down with it. ATTEMPT_BACKOFF = float(os.getenv("MUSE_ATTEMPT_BACKOFF", "4")) # Consecutive systemic failures before we stop walking the ladder. A file that # is not shared says something about that file; a timeout says only that slskd # is having a moment, and burning eight candidates to learn it once is waste. SYSTEMIC_LIMIT = int(os.getenv("MUSE_SYSTEMIC_LIMIT", "2")) # How many Soulseek searches may run at once. # # This is a hard cap, not a suggestion. Each search takes 10-40s of polling # and real load on slskd and the Soulseek server, so firing one task per # request — which is what an earlier version did — means pasting a 50-track # list launches 50 concurrent searches and drowns the client. Everything goes # through a bounded queue instead: requests return instantly, work drains at a # civil rate. WORKERS = int(os.getenv("MUSE_WORKERS", "3")) # Search uses Deezer's public API: no key, no OAuth, and vastly better # relevance than MusicBrainz for the way people actually type. # # MusicBrainz was tried first and rejected on measurements, not taste. It is a # superb database but a poor search engine: "radiohead creep" returned eight # covers and mashups before the Radiohead recording, and "bohemian rhapsody" # never surfaced Queen at all across 90 results — because its text index # matches "radiohead" inside a cover's *title* as readily as in the artist # field. Client-side re-ranking can only reorder what came back, so it could # not fix a result set that lacked the right row. Deezer returns # "Creep / Radiohead / Pablo Honey" first for the same query. # # Deezer search omits track number and release date, so those are fetched from # /track/{id} at request time — one extra call per request, not per result. DEEZER = "https://api.deezer.com" # Derivative versions to push down when the user did not ask for them. # Matched on WORD BOUNDARIES, not as substrings: a bare "live" is needed to # catch "Come As You Are (Live)", but substring matching would also penalise # "Deliver Me" and "Olive". Same trap for "demo" vs "Demolition". # # A term only counts against a result when the user did NOT type it, so asking # for "oasis live forever" or "nirvana unplugged demo" still works. DERIVATIVE = ("cover", "covers", "karaoke", "tribute", "remix", "remixed", "instrumental", "acapella", "originally performed", "made popular by", "in the style of", "demo", "rehearsal", "live", "unplugged", "re recorded", "rerecorded", "sped up", "slowed", "reverb", "8d", "nightcore") DERIVATIVE_RE = re.compile(r"\b(" + "|".join(DERIVATIVE) + r")\b") # ─────────────────────────── auth ─────────────────────────── # # OAuth against GitHub and/or Google, built so this can face the internet. # # FAILS CLOSED. Auth is on unless MUSE_AUTH is explicitly set to "off", and if # it is on with incomplete config the app refuses to start. The failure mode # worth engineering against is not a broken login page — it is a misconfigured # deploy quietly serving an unauthenticated download client to the internet. # You should have to *mean* it to run without auth. # # Sessions are server-side: the cookie carries a 256-bit random token and # nothing else, and the database stores only its SHA-256. There is no signed # payload to get wrong, a stolen database yields no usable cookies, and logout # and revocation actually revoke. # # WHO MAY LOG IN is two rules, checked in this order: # 1. MUSE_ALLOWED_USERS — the bootstrap allowlist. Always admitted. This is # how the very first admin gets in when the users table is empty, and it # is the recovery path if someone disables themselves. # 2. A row in `users` that is not disabled — how everyone invited later gets # in, without an env change or a redeploy. # Anyone matching neither is rejected and the attempt is logged. AUTH_MODE = os.getenv("MUSE_AUTH", "github").strip().lower() ENABLED_PROVIDERS = [p for p in (x.strip() for x in AUTH_MODE.split(",")) if p and p != "off"] # Absolute external URL, e.g. https://muse.aryan-singh.dev. Used to build the # OAuth callback. Taken from config rather than from X-Forwarded-* headers on # purpose: those are attacker-controllable and this is a redirect target. BASE_URL = os.getenv("MUSE_BASE_URL", "").rstrip("/") ALLOWED_USERS = {u.strip().lower() for u in os.getenv("MUSE_ALLOWED_USERS", "").split(",") if u.strip()} COOKIE_SECURE = os.getenv("MUSE_COOKIE_SECURE", "true").lower() != "false" SESSION_DAYS = int(os.getenv("MUSE_SESSION_DAYS", "7")) SESSION_COOKIE = "muse_session" STATE_COOKIE = "muse_oauth_state" # One callback URL for every provider — the provider is carried in the state # COOKIE, not the query string, so adding Google needs no change to the GitHub # OAuth app that is already registered against /auth/callback. PROVIDERS: dict[str, dict] = { "github": { "label": "GitHub", "authorize": "https://github.com/login/oauth/authorize", "token": "https://github.com/login/oauth/access_token", "scope": "read:user", "client_id": os.getenv("MUSE_GITHUB_CLIENT_ID", "").strip(), "client_secret": os.getenv("MUSE_GITHUB_CLIENT_SECRET", "").strip(), "id_env": "MUSE_GITHUB_CLIENT_ID", "secret_env": "MUSE_GITHUB_CLIENT_SECRET", }, "google": { "label": "Google", "authorize": "https://accounts.google.com/o/oauth2/v2/auth", "token": "https://oauth2.googleapis.com/token", "scope": "openid email profile", "client_id": os.getenv("MUSE_GOOGLE_CLIENT_ID", "").strip(), "client_secret": os.getenv("MUSE_GOOGLE_CLIENT_SECRET", "").strip(), "id_env": "MUSE_GOOGLE_CLIENT_ID", "secret_env": "MUSE_GOOGLE_CLIENT_SECRET", }, } GH_API = "https://api.github.com/user" GOOGLE_API = "https://openidconnect.googleapis.com/v1/userinfo" # Paths reachable without a session. Deliberately tiny: /healthz reports a bare # ok so the container healthcheck works without leaking configuration. PUBLIC_PATHS = {"/healthz", "/auth/login", "/auth/callback", "/auth/logout", "/login.css"} ROLES = ("admin", "user", "guest") def auth_preflight(): if AUTH_MODE == "off": log.warning("=" * 62) log.warning("MUSE_AUTH=off — every endpoint is UNAUTHENTICATED.") log.warning("Never expose this to the internet in this state.") log.warning("=" * 62) return if not ENABLED_PROVIDERS: raise RuntimeError(f"MUSE_AUTH must list providers or be 'off', got {AUTH_MODE!r}") unknown = [p for p in ENABLED_PROVIDERS if p not in PROVIDERS] if unknown: raise RuntimeError( f"MUSE_AUTH names unknown provider(s) {unknown}. " f"Valid: {', '.join(PROVIDERS)}, or 'off'.") missing: list[str] = [] for p in ENABLED_PROVIDERS: cfg = PROVIDERS[p] if not cfg["client_id"]: missing.append(cfg["id_env"]) if not cfg["client_secret"]: missing.append(cfg["secret_env"]) if not BASE_URL: missing.append("MUSE_BASE_URL") # The bootstrap allowlist is only mandatory while nobody exists yet — # otherwise a working multi-user install would be held hostage by an env # var it has outgrown. But an EMPTY database and no allowlist means nobody # can ever log in, which is a misconfiguration worth refusing to start on. if not ALLOWED_USERS and not _any_users(): missing.append("MUSE_ALLOWED_USERS") if missing: raise RuntimeError( "Auth is enabled but " + ", ".join(sorted(set(missing))) + " not set. " "Refusing to start: set them, or set MUSE_AUTH=off to run without " "authentication (LAN only).") if not BASE_URL.startswith("https://") and COOKIE_SECURE: log.warning("MUSE_BASE_URL is not https but cookies are Secure — the " "browser will refuse to send them. Set MUSE_COOKIE_SECURE=false " "only if you are testing over plain http on a LAN.") log.info("auth: providers=%s bootstrap_allowlist=%d", ",".join(ENABLED_PROVIDERS), len(ALLOWED_USERS)) def _any_users() -> bool: try: with db() as c: return c.execute("SELECT 1 FROM users LIMIT 1").fetchone() is not None except sqlite3.Error: return False def _hash(tok: str) -> str: return hashlib.sha256(tok.encode()).hexdigest() def new_session(username: str) -> str: token = secrets.token_urlsafe(32) exp = datetime.now(timezone.utc) + timedelta(days=SESSION_DAYS) with db() as c: c.execute("INSERT INTO sessions (token_hash,username,expires_at) VALUES (?,?,?)", (_hash(token), username, exp.isoformat())) # Opportunistic sweep; no scheduler needed for a handful of rows. c.execute("DELETE FROM sessions WHERE expires_at < ?", (datetime.now(timezone.utc).isoformat(),)) return token def may_login(username: str, row: sqlite3.Row | None, provider: str | None = None) -> bool: """The single authorization rule, used at login AND on every request. Bootstrap allowlist first, then a live, non-disabled user row. Deliberately one function: an authorization check that runs only at login is not a revocation mechanism, it is a delay — disabling someone would leave their session valid for up to MUSE_SESSION_DAYS. An invite is bound to the provider it was issued for. Invite friend@gmail.com for Google and Google is the only way in for that identity; there is no second, weaker door onto the same account. `provider` is None on the per-request re-check, where the question is only whether the account is still live. """ if username.lower() in ALLOWED_USERS: return True if not row or row["disabled"]: return False return provider is None or row["provider"] == provider def current_user(request: Request) -> sqlite3.Row | None: """The signed-in user's row, or None. Re-validated on every request.""" tok = request.cookies.get(SESSION_COOKIE) if not tok: return None with db() as c: row = c.execute( "SELECT username, expires_at FROM sessions WHERE token_hash=?", (_hash(tok),)).fetchone() if not row: return None try: if datetime.fromisoformat(row["expires_at"]) < datetime.now(timezone.utc): return None except ValueError: return None u = c.execute("SELECT * FROM users WHERE username=?", (row["username"],)).fetchone() if not may_login(row["username"], u): return None if u is None: # Allowlisted but no row — an admin deleted the bootstrap account while # its session was still live. The allowlist is exactly the recovery # path for that, so put the row back rather than half-authenticating # them into a state where every role check fails. u = upsert_user(row["username"]) log.info("auth: restored deleted allowlisted user %s", row["username"]) return u def session_user(request: Request) -> str | None: u = current_user(request) return u["username"] if u else None def user_role(request: Request) -> str: u = current_user(request) if u: return u["role"] # With auth off there is nobody to be, so everyone is an admin — otherwise # the admin panel would be unreachable on a LAN-only install. return "admin" if AUTH_MODE == "off" else "guest" def require_admin(request: Request) -> str: if AUTH_MODE == "off": return "local" u = current_user(request) if not u: raise HTTPException(401, "unauthenticated") if u["role"] != "admin": log.warning("admin: DENIED %s (role=%s) -> %s", u["username"], u["role"], request.url.path) raise HTTPException(403, "admin only") return u["username"] SAFE = re.compile(r'[<>:"/\\|?*\x00-\x1f]') NOISE = re.compile(r"[^a-z0-9]+") YEAR = re.compile(r"^\d{4}$") # Apostrophes vanish rather than becoming a separator, so "Don't" normalises # onto a filename's "Dont". Everything else non-alphanumeric becomes a space, # so "R&B" stays two tokens instead of collapsing to the unrelated word "rb". QUOTES = re.compile("['\u2018\u2019\u02bc\u0060\"\u201c\u201d]") # Letters NFKD leaves alone because they are atomic rather than base + accent. FOLD = str.maketrans({"\u00f8": "o", "\u00e6": "ae", "\u0153": "oe", "\u00f0": "d", "\u00fe": "th", "\u0142": "l", "\u00df": "ss", "\u0111": "d", "\u0131": "i"}) # A leading track number: "02 - ", "02.", "A1 ". Anchored and bounded so it # cannot eat a title that merely opens with a digit ("1979", "99 Problems"). TRACKNO = re.compile(r"^\s*(?:[a-d]?\d{1,3})\s*(?:[-._)\]]\s*)+", re.I) # Technical release tags. Deliberately NOT "live", "remix" or "acoustic": # confidence() reads those to decide whether this is the recording that was # asked for, and stripping them here would blind it. JUNK = re.compile( r"[\[({]\s*(?:\d{2,4}\s*k(?:bps)?|v\d|web|cd\s*\d*|vinyl|flac|mp3|" r"lossless|hi-?res|\d{2,3}\s*bit|\d{2}\s*khz)[^\])}]*[\])}]" r"|\b(?:\d{3}\s*kbps|web-?dl|cdrip|vbr|cbr)\b", re.I) # Most filesystems cap a path component at 255 bytes. Track titles can be far # longer than that once a stranger has appended their own notes, and the # resulting ENAMETOOLONG would fail the import after the download had finished. MAX_COMPONENT = 150 def safe(name: str) -> str: """Filesystem-safe path component. Trailing dots/spaces break on some FS.""" return SAFE.sub("_", name).strip(". ")[:MAX_COMPONENT].strip() or "Unknown" def safe_cover(url) -> str | None: """Artwork URL, stored and later rendered as an . The CSP already stops a foreign host loading, so this is not an XSS gate — it stops the field being used as free storage. `data:` is permitted by img-src, so without a check any client could park megabytes of base64 in the jobs table on every request. """ u = str(url or "").strip() return u if u.startswith("https://") and len(u) <= 300 else None def safe_year(year) -> str | None: """A year, or nothing. This lands in a directory name, so it is checked against a whitelist rather than merely scrubbed.""" y = str(year or "").strip() return y if YEAR.match(y) else None def norm(s: str) -> str: """Lowercase, fold accents away, and reduce to alphanumeric tokens. The accent fold is load-bearing. Deezer returns "Beyonc\u00e9", "Bj\u00f6rk" and "Sigur R\u00f3s"; the strangers who named the files on Soulseek wrote "Beyonce", "Bjork", "Sigur Ros". Without it the accented character is not in [a-z0-9] and became a *space*, so "beyonce" was compared against "beyonc " -- a containment miss and a mediocre ratio, on a track that was sitting right there in the results. """ s = unicodedata.normalize("NFKD", str(s).lower().translate(FOLD)) s = "".join(c for c in s if not unicodedata.combining(c)) return NOISE.sub(" ", QUOTES.sub("", s)).strip() def bare_name(remote_path: str) -> str: """A remote filename with extension, track number and release tags gone. All of that is decoration a peer added that says nothing about which recording the file holds. Leaving it in drags the similarity ratio down on files that are in fact exactly right. """ base = remote_path.replace("\\", "/").rsplit("/", 1)[-1] if "." in base: base = base.rsplit(".", 1)[0] return JUNK.sub(" ", TRACKNO.sub("", base)).strip() def similarity(want: str, got: str) -> float: """Best of a sequence ratio and a token-coverage ratio, 0..1. difflib alone is order-sensitive: "creep radiohead" against "radiohead creep" scores far below what a human would call a match, and Soulseek filenames put the fields in whatever order the ripper felt like. The token measure is order-free, so taking the better of the two absorbs reordering and interleaved junk without having to loosen MIN_MATCH. """ if not want or not got: return 0.0 seq = difflib.SequenceMatcher(None, want, got).ratio() a, b = set(want.split()), set(got.split()) return max(seq, len(a & b) / len(a) if a else 0.0) def artist_confirmed(remote_path: str, artist: str) -> bool: """Is the requested artist actually named somewhere in this path? This is a gate rather than a score because of what the query ladder does. Once a filtered artist name forces the search down to the bare title, the result set stops being "this song from various peers" and becomes "every song anyone ever called Chicago" -- The Fall, Groove Armada, Sufjan Stevens. The artist is then the only thing distinguishing the right recording from a completely different song, and a weighted contribution is not enough: 0.75 for a perfect title with no artist still clears MIN_MATCH, and an available stranger will out-score a queued match every time. """ na = norm(artist) if not na: return False path = norm(remote_path.replace("\\", "/")) if na in path: return True # Every distinctive word, in any order: catches "Jackson, Michael" and # "MJ - Chicago (Michael Jackson)" without matching on "the" or "and". words = [w for w in na.split() if len(w) > 3] return bool(words) and all(w in path for w in words) def match_ratio(remote_path: str, artist: str, title: str) -> float: """How well a remote path matches the track asked for, 0..1. Title and artist are measured separately and the title has to stand on its own. A single ratio over the blob "artist title" let a strong artist match carry a weak title -- the same failure the Deezer ranking keeps a dedicated term for, where `Oasis - Live Forever` once resolved to `Cloudburst` because the artist agreed and nothing else had to. The artist is looked for in the whole path rather than the filename: a file at `Radiohead/Pablo Honey/02 Creep.flac` names its artist in the folder, which is the commonest well-organised case there is. """ stem = norm(bare_name(remote_path)) path = norm(remote_path.replace("\\", "/")) nt, na = norm(title), norm(artist) t = 1.0 if (nt and nt in stem) else similarity(nt, stem) if t < TITLE_FLOOR: return 0.0 # wrong song; a matching artist must not rescue it a = 1.0 if (na and na in path) else similarity(na, path) return 0.75 * t + 0.25 * a # ─────────────────────────── storage ─────────────────────────── def db(): c = sqlite3.connect(DB_PATH, timeout=30) c.row_factory = sqlite3.Row return c def init_db(): DB_PATH.parent.mkdir(parents=True, exist_ok=True) with db() as c: c.execute(""" CREATE TABLE IF NOT EXISTS jobs ( id TEXT PRIMARY KEY, state TEXT NOT NULL, -- queued|searching|downloading|review|imported|failed detail TEXT, artist TEXT, title TEXT, album TEXT, year TEXT, track INTEGER, recording_mbid TEXT, release_mbid TEXT, username TEXT, remote_file TEXT, size INTEGER, imported_path TEXT, created_at TEXT DEFAULT (datetime('now')) ) """) c.execute(""" CREATE TABLE IF NOT EXISTS sessions ( token_hash TEXT PRIMARY KEY, -- sha256 of the cookie value username TEXT NOT NULL, created_at TEXT DEFAULT (datetime('now')), expires_at TEXT NOT NULL ) """) # Users exist as rows even while the allowlist is a single name in an # env var. Roles and quotas are read from here from the start, so # opening this up to other people later is an INSERT and a config # change rather than a schema migration against live job history. c.execute(""" CREATE TABLE IF NOT EXISTS users ( username TEXT PRIMARY KEY COLLATE NOCASE, provider TEXT NOT NULL DEFAULT 'github', role TEXT NOT NULL DEFAULT 'user', -- admin|user|guest display TEXT, avatar TEXT, quota INTEGER NOT NULL DEFAULT 0, -- requests/month, 0 = unlimited disabled INTEGER NOT NULL DEFAULT 0, created_at TEXT DEFAULT (datetime('now')), last_seen TEXT ) """) # Added after the first release, so existing databases need it grafted # on rather than recreated -- job history is the whole point of the file. cols = {r["name"] for r in c.execute("PRAGMA table_info(jobs)").fetchall()} for col, decl in (("requested_by", "TEXT"), ("cover", "TEXT"), ("genre", "TEXT"), ("duration", "INTEGER"), ("confidence", "REAL"), ("match_why", "TEXT"), ("reviewed_by", "TEXT"), # The candidate ladder. Keeping the runners-up on the # row is what lets a rejected transfer fall through to # the next peer minutes later, in a different task, # after the search result set is long gone. ("candidates", "TEXT"), ("tried", "TEXT"), ("attempts", "INTEGER NOT NULL DEFAULT 0"), ("started_at", "TEXT")): if col not in cols: c.execute(f"ALTER TABLE jobs ADD COLUMN {col} {decl}") ucols = {r["name"] for r in c.execute("PRAGMA table_info(users)").fetchall()} for col, decl in (("storage_quota", "INTEGER NOT NULL DEFAULT 0"), ("invited_by", "TEXT"), ("email", "TEXT"), ("notify", "INTEGER NOT NULL DEFAULT 1")): if col not in ucols: c.execute(f"ALTER TABLE users ADD COLUMN {col} {decl}") c.execute("CREATE INDEX IF NOT EXISTS jobs_by_user ON jobs (requested_by)") c.execute("CREATE INDEX IF NOT EXISTS jobs_by_state ON jobs (state)") c.execute("CREATE INDEX IF NOT EXISTS jobs_by_created ON jobs (created_at)") def upsert_user(username: str, avatar: str = "", display: str = "", provider: str = "github") -> sqlite3.Row: """Record the sign-in. The first user to ever log in becomes admin — no bootstrap ceremony, no default password to forget. Everyone after that is a plain user until an admin says otherwise; role is never touched on subsequent logins, or every sign-in would silently undo a demotion. """ with db() as c: first = c.execute("SELECT COUNT(*) n FROM users").fetchone()["n"] == 0 c.execute( "INSERT INTO users (username,provider,role,display,avatar,last_seen) " "VALUES (?,?,?,?,?,datetime('now')) " "ON CONFLICT(username) DO UPDATE SET " " last_seen=datetime('now'), " # provider is NOT refreshed from the login. It is part of the # identity an admin invited, and may_login now enforces it — so # overwriting it here would let a successful login silently # rewrite the very rule that permitted it. " avatar=COALESCE(NULLIF(excluded.avatar,''), users.avatar), " " display=COALESCE(NULLIF(excluded.display,''), users.display)", (username, provider, "admin" if first else "user", display, avatar)) return c.execute("SELECT * FROM users WHERE username=?", (username,)).fetchone() def set_state(job_id: str, state: str, detail: str = "", **cols): sets = ", ".join(f"{k}=?" for k in cols) sql = f"UPDATE jobs SET state=?, detail=?{', ' + sets if sets else ''} WHERE id=?" with db() as c: c.execute(sql, [state, detail, *cols.values(), job_id]) log.info("job %s -> %s %s", job_id[:8], state, detail) # ─────────────────────────── slskd ─────────────────────────── def slskd_client(timeout: float = 30.0) -> httpx.AsyncClient: """30s suits searches and transfer polling, which are legitimately slow. Status probes pass something much shorter: they run on a UI poll, and a probe that outlasts its own poll interval stacks requests and leaves the admin panel on skeletons for the whole timeout — precisely when slskd is down and you opened the panel to find out why. """ return httpx.AsyncClient( base_url=SLSKD_URL, headers={"X-API-Key": SLSKD_API_KEY}, timeout=httpx.Timeout(timeout), ) # slskd is on the LAN, so a reachable instance answers in milliseconds. Long # enough to absorb a slow moment, short enough to stay under the 5s UI poll. PROBE_TIMEOUT = 4.0 def errstr(e: Exception) -> str: """httpx's timeout and connection errors often str() to an empty string, which would render as 'slskd unreachable — ' with nothing after it.""" return (str(e) or type(e).__name__)[:120] def ext_of(remote_path: str) -> str: # Soulseek paths are Windows-style; the API's `extension` field comes back # empty in practice, so derive it ourselves. return remote_path.rsplit(".", 1)[-1].lower() if "." in remote_path else "" def confidence(filename: str, artist: str, title: str, album: str | None = None) -> tuple[float, list[str]]: """How sure are we this file is the song that was asked for, 0..1. Deliberately separate from score(). score() decides *which* candidate to fetch and is right to weigh peer speed and queue length; none of that says anything about whether the file is the correct recording. Mixing the two would let a fast peer raise the confidence of a wrong match. Returns the score and the human-readable reasons behind it, because a number alone gives the reviewer nothing to act on. """ stem = norm(bare_name(filename)) path = norm(filename.replace("\\", "/")) na, nt = norm(artist), norm(title) why: list[str] = [] # Title is the load-bearing signal: a wrong title is a wrong song, whereas # a missing artist is routine in files organised into per-artist folders. t_ratio = similarity(nt, stem) if nt and nt in stem: t_score, t_ratio = 1.0, max(t_ratio, 1.0) why.append("title matches") elif t_ratio >= 0.75: t_score = t_ratio why.append(f"title close ({t_ratio:.0%})") else: t_score = t_ratio why.append(f"title differs ({t_ratio:.0%})") # Artist may legitimately live in the path rather than the filename. if na and na in stem: a_score = 1.0 why.append("artist in filename") elif na and na in path: a_score = 0.9 why.append("artist in folder") elif na and set(na.split()) & set(path.split()): a_score = 0.5 why.append("artist partly matches") else: a_score = 0.0 why.append("artist not found in path") # 0.75/0.25 rather than an even split. A file named exactly right but # sitting in an unorganised folder is the commonest honest case there is, # and it has to land in the review band, not below it — 0.65 would have # binned it alongside genuinely wrong songs. c = 0.75 * t_score + 0.25 * a_score if album and norm(album) and norm(album) in path: c = min(1.0, c + 0.05) why.append("album in path") # A live or remix version of the right song is still the wrong file when # the request was for the studio recording, and that is exactly the case a # reviewer needs to see rather than have silently imported. So the penalty # caps confidence below AUTO instead of merely scaling it: "a human must # decide" is the whole meaning of the flag, and a perfect artist+title # match should not be able to auto-import past it. variants = [w for w in ("live", "remix", "cover", "karaoke", "instrumental", "acoustic", "demo", "edit", "radio edit") if re.search(rf"\b{w}\b", stem) and w not in nt] if variants: c = min(c * 0.8, AUTO_CONFIDENCE - 0.1) why.append(f"looks like a {variants[0]} version, which was not requested") return round(max(0.0, min(1.0, c)), 3), why # >= AUTO imports without asking. Below REVIEW the match is bad enough that a # human almost certainly rejects it, but it is still held rather than binned: # the file is already downloaded and only the reviewer knows if it is right. AUTO_CONFIDENCE = float(os.getenv("MUSE_AUTO_CONFIDENCE", "0.95")) REVIEW_CONFIDENCE = float(os.getenv("MUSE_REVIEW_CONFIDENCE", "0.70")) def score(candidate: dict, peer: dict, artist: str, title: str) -> float | None: """Rank one remote file. None = reject. Ordering intent: actually-gettable first, then correct format, then filename plausibility, then bitrate. Availability leads deliberately. It used to sit *below* format -- 100 points per format rank against 60 for a free upload slot -- which meant a FLAC behind two hundred queued people always beat an MP3 available right now. Whether the peer can serve you is the single most predictive signal for whether the job ever finishes, so it outranks the preference for a better file. Format is the tiebreak within an availability tier. """ ext = ext_of(candidate["filename"]) if ext not in FORMATS: return None ratio = match_ratio(candidate["filename"], artist, title) if ratio < MIN_MATCH: return None s = 0.0 s += 220 if peer.get("hasFreeUploadSlot") else 0 # Unsaturated: the old `max(0, 40 - queueLength)` scored a 40-deep queue # and a 5000-deep one identically, which is the difference between waiting # a minute and never being served at all. s += 120 / (1 + peer.get("queueLength", 0) / 8) s += min(peer.get("uploadSpeed", 0) / 50_000, 20) s += (len(FORMATS) - FORMATS.index(ext)) * 60 # format preference s += ratio * 90 s += min(candidate.get("bitRate") or 0, 320) / 32 # 0-10 return s PAREN = re.compile(r"[\[(\{][^\])}]*[\])}]") FEAT = re.compile(r"\b(?:feat|ft|featuring)\b\.?.*$", re.I) ARTICLE = re.compile(r"^(?:the|a|an|los|las|les)\s+", re.I) class SearchUnavailable(Exception): """slskd could not accept a search. Transient, not a bad request.""" def search_queries(artist: str, title: str) -> list[str]: """Query variants to try, loosest last. The Soulseek server filters searches containing certain artist names. That is measured, not theorised: on this network "Michael Jackson Billie Jean" and bare "Michael Jackson" both return zero responses, while "Billie Jean" returns 1083 files. Sending only `artist title` -- all this used to do -- means every request for a filtered artist dies reporting "searched 0 files", and that is exactly the roster of names in the failed jobs: Michael Jackson, The Weeknd, Kanye West, Frank Ocean. So the ladder ends at the title alone, and match_ratio() then recognises the artist from the folder path instead of from the query. The middle rungs handle the ordinary case where Deezer's metadata is simply more decorated than the filename a stranger typed. """ out: list[str] = [] seen: set[str] = set() def add(*parts: str) -> None: q = " ".join(p for p in parts if p).strip() # Dedupe on the normalised form, so "Creep (Remastered)" and "Creep" # are recognised as one query once the punctuation is gone. k = norm(q) if q and k and k not in seen: seen.add(k) out.append(q) plain_t = PAREN.sub(" ", title).strip() or title lean_t = FEAT.sub(" ", plain_t).strip() or plain_t head_t = lean_t.split("/")[0].strip() or lean_t # "A / B" -> "A" lean_a = ARTICLE.sub("", FEAT.sub(" ", artist).strip()).strip() add(artist, title) # what was sent before, and still best when it works add(artist, lean_t) # decoration and featuring credits gone add(lean_a, head_t) # leading article gone, split title add(head_t) # title alone -- this is the rung that beats the filter add(lean_t) return out[:MAX_QUERIES] async def run_search(sc: httpx.AsyncClient, text: str) -> dict: """Issue one Soulseek search and poll it to completion. The search is deleted only once it has actually finished. Deleting one still InProgress makes slskd throw an optimistic-concurrency error inside its own SearchService and lose the record; the old code deleted unconditionally after its 40s budget, so every slow search corrupted a row. """ search_id = str(uuid.uuid4()) for attempt in range(SEARCH_RETRIES): try: r = await sc.post("/api/v0/searches", json={"id": search_id, "searchText": text}) r.raise_for_status() break except Exception as e: if attempt == SEARCH_RETRIES - 1: raise SearchUnavailable(errstr(e)) from e await asyncio.sleep(2 * (attempt + 1)) results: dict = {} finished = False for _ in range(SEARCH_POLLS): await asyncio.sleep(2) try: rr = await sc.get(f"/api/v0/searches/{search_id}", params={"includeResponses": "true"}) rr.raise_for_status() results = rr.json() except Exception as e: # Swallowed silently before, which made a broken slskd look # identical to a track nobody happens to share. log.debug("poll %r: %s", text, errstr(e)) continue if str(results.get("state", "")).startswith("Completed"): finished = True break if finished: try: await sc.delete(f"/api/v0/searches/{search_id}") except Exception as e: log.debug("cleanup %r: %s", text, errstr(e)) else: log.info("search %r unfinished after %ds, leaving it for slskd to close", text, SEARCH_POLLS * 2) return results def spread_by_peer(ranked: list[tuple[float, dict, dict]]) -> list[dict]: """Order candidates so consecutive attempts land on different peers. Straight score order routinely stacks three files from one well-stocked user at the top; if that user is offline all three attempts burn on the same dead connection. Peers arrive best-first, so round-robining them keeps the strongest candidate first and still guarantees variety behind it. """ by_peer: dict[str, list[dict]] = {} for s, peer, f in ranked: by_peer.setdefault(peer["username"], []).append({ "username": peer["username"], "filename": f["filename"], "size": f.get("size") or 0, "score": round(s, 1), }) queues = list(by_peer.values()) out: list[dict] = [] while queues: for q in list(queues): out.append(q.pop(0)) if not q: queues.remove(q) return out def no_candidate_detail(seen: int, stats: "collections.Counter", queries: list[str]) -> str: """Say which filter emptied the results, not merely that they are empty. One message used to cover four unrelated situations, and 28 of 29 real occurrences turned out to be the search itself coming back empty -- which is now stated outright instead of being reported as a matching failure. """ if seen == 0: tried = ", ".join(repr(q) for q in queries) word = "y" if len(queries) == 1 else "ies" return f"Soulseek returned nothing for {len(queries)} quer{word}: {tried}" bits = [] if stats["format"]: bits.append(f"{stats['format']} not {'/'.join(FORMATS)}") if stats["weak"]: bits.append(f"{stats['weak']} did not match the title") if stats["locked"]: bits.append(f"{stats['locked']} locked") return f"{seen} files found, none usable: " + ", ".join(bits) def exhausted_detail(tried: list[dict], attempts: int) -> str: """Summarise the whole ladder instead of overwriting it with the last error. Counts `tried`, not `attempts`. They agree for anything this version enqueued, but a row carried over from before the ladder existed has attempts=0 with a real failure recorded against it, which rendered as the nonsense "tried 0 peers: 1x no bytes in 20m". The list is the evidence. """ if not tried: return "no candidate could be enqueued" n_peers = len({t.get("username") for t in tried if t.get("username")}) or len(tried) reasons = collections.Counter(t["reason"].split(":")[0] for t in tried) plural = "" if n_peers == 1 else "s" joined = ", ".join(f"{n}x {r}" for r, n in reasons.most_common(3)) return f"tried {n_peers} peer{plural}: {joined}" # Substrings in a failure that describe slskd's state rather than the file's. # "File not shared" is about one file and the next candidate is unaffected; # a timeout means the client could not talk to anyone, so the next candidate # would fail for the same reason and only costs us the candidate. SYSTEMIC = ("timed out", "timeout", "connection", "refused", "unreachable") def is_systemic(reason: str) -> bool: r = reason.lower() return any(w in r for w in SYSTEMIC) async def try_next_candidate(job_id: str, artist: str, title: str, album: str | None = None) -> bool: """Enqueue the next-best candidate. False once the ladder is exhausted. This is the difference between a request that survives one bad peer and one that does not. "Transfer rejected: File not shared." is a *per-file* condition -- the peer's share index is stale -- so candidate two almost always works, and slskd's 500 on an unreachable peer is the same shape. """ with db() as c: row = c.execute( "SELECT candidates, tried, attempts FROM jobs WHERE id=?", (job_id,)).fetchone() if not row: return False cands = json.loads(row["candidates"] or "[]") tried = json.loads(row["tried"] or "[]") attempts = row["attempts"] or 0 systemic_run = 0 first = True while cands and attempts < MAX_ATTEMPTS: # Space the attempts out. The first one is immediate; the rest wait, # because back-to-back enqueues are what saturated slskd. if not first: await asyncio.sleep(ATTEMPT_BACKOFF) first = False cand = cands.pop(0) attempts += 1 try: async with slskd_client() as sc: er = await sc.post( f"/api/v0/transfers/downloads/{cand['username']}", json=[{"filename": cand["filename"], "size": cand["size"]}]) er.raise_for_status() except Exception as e: reason = f"could not reach peer: {errstr(e)}" tried.append({"username": cand["username"], "filename": cand["filename"], "reason": reason}) if is_systemic(reason): systemic_run += 1 if systemic_run >= SYSTEMIC_LIMIT: # Stop, and leave the untried candidates on the row so a # retry resumes the ladder instead of re-running a search # that already told us the right answer. with db() as c: c.execute("UPDATE jobs SET candidates=?, tried=?," " attempts=? WHERE id=?", (json.dumps(cands), json.dumps(tried), attempts, job_id)) set_state(job_id, "failed", f"slskd could not reach any peer after " f"{attempts} attempts -- {len(cands)} candidate" f"{'' if len(cands) == 1 else 's'} left to try, " f"so a retry resumes where this stopped") return False else: systemic_run = 0 continue systemic_run = 0 conf, why = confidence(cand["filename"], artist, title, album) with db() as c: c.execute( "UPDATE jobs SET candidates=?, tried=?, attempts=? WHERE id=?", (json.dumps(cands), json.dumps(tried), attempts, job_id)) extra = f", attempt {attempts}" if attempts > 1 else "" set_state(job_id, "downloading", f"from {cand['username']} (score {cand['score']:.0f}{extra})", username=cand["username"], remote_file=cand["filename"], size=cand["size"], confidence=conf, match_why="; ".join(why), started_at=datetime.now(timezone.utc).isoformat()) return True with db() as c: c.execute("UPDATE jobs SET candidates=?, tried=?, attempts=? WHERE id=?", (json.dumps(cands), json.dumps(tried), attempts, job_id)) set_state(job_id, "failed", exhausted_detail(tried, attempts)) return False async def pick_and_enqueue(job_id: str, artist: str, title: str, album: str | None = None): """Search Soulseek for one track and queue the best candidate. Builds a ranked candidate *list* rather than a single winner, so a peer that rejects, times out or cannot be reached costs one attempt instead of the entire request. """ queries = search_queries(artist, title) pool: dict[tuple[str, str], tuple[float, dict, dict]] = {} seen = 0 stats: "collections.Counter" = collections.Counter() async with slskd_client() as sc: for i, q in enumerate(queries): try: results = await run_search(sc, q) except SearchUnavailable as e: set_state(job_id, "failed", f"slskd would not accept a search: {e}") return for peer in results.get("responses", []): for f in peer.get("files", []): seen += 1 if f.get("isLocked"): stats["locked"] += 1 continue if ext_of(f["filename"]) not in FORMATS: stats["format"] += 1 continue sc_ = score(f, peer, artist, title) if sc_ is None: stats["weak"] += 1 continue key = (peer["username"], f["filename"]) if key not in pool or pool[key][0] < sc_: pool[key] = (sc_, peer, f) log.info("job %s q%d/%d %r -> %d responses, %d files, %d candidates", job_id[:8], i + 1, len(queries), q, len(results.get("responses", [])), results.get("fileCount", 0), len(pool)) # Escalate only on underdelivery: the common case stays one search, # which is what keeps a 50-track paste civil to slskd. if len(pool) >= MIN_CANDIDATES: break if not pool: set_state(job_id, "failed", no_candidate_detail(seen, stats, queries)) return # Prefer candidates whose path actually names the artist, and only settle # for ones that do not when there is nothing else. Availability is a # tiebreak among plausible files, never a reason to take the wrong song. named = {k: v for k, v in pool.items() if artist_confirmed(v[2]["filename"], artist)} if named: if len(named) < len(pool): log.info("job %s keeping %d/%d candidates that name %r", job_id[:8], len(named), len(pool), artist) pool = named else: log.info("job %s no candidate names %r; falling back to title-only " "matches, which review will have to judge", job_id[:8], artist) ranked = sorted(pool.values(), key=lambda r: -r[0]) cands = spread_by_peer(ranked)[:CANDIDATES] with db() as c: c.execute("UPDATE jobs SET candidates=?, tried=?, attempts=0 WHERE id=?", (json.dumps(cands), "[]", job_id)) await try_next_candidate(job_id, artist, title, album) # ─────────────────────────── email ─────────────────────────── # # Optional and off unless SMTP_HOST is set. A homelab tool that cannot send # mail should still import music, so every failure here is logged and # swallowed — a notification is never worth failing an import over. SMTP_HOST = os.getenv("MUSE_SMTP_HOST", "").strip() SMTP_PORT = int(os.getenv("MUSE_SMTP_PORT", "587")) SMTP_USER = os.getenv("MUSE_SMTP_USER", "").strip() SMTP_PASS = os.getenv("MUSE_SMTP_PASS", "") SMTP_FROM = os.getenv("MUSE_SMTP_FROM", "").strip() or SMTP_USER SMTP_TLS = os.getenv("MUSE_SMTP_TLS", "starttls").strip().lower() MAIL_ON = bool(SMTP_HOST and SMTP_FROM) def _send_sync(to: str, subject: str, body: str): import smtplib from email.message import EmailMessage msg = EmailMessage() msg["From"], msg["To"], msg["Subject"] = SMTP_FROM, to, subject msg.set_content(body) if SMTP_TLS == "ssl": s = smtplib.SMTP_SSL(SMTP_HOST, SMTP_PORT, timeout=20) else: s = smtplib.SMTP(SMTP_HOST, SMTP_PORT, timeout=20) with s: if SMTP_TLS == "starttls": s.starttls() if SMTP_USER: s.login(SMTP_USER, SMTP_PASS) s.send_message(msg) async def send_mail(to: str, subject: str, body: str): """Fire-and-forget. smtplib is blocking, so it runs off the event loop — otherwise a slow relay would stall the import worker for every job.""" if not (MAIL_ON and to): return try: await asyncio.to_thread(_send_sync, to, subject, body) log.info("mail: sent %r to %s", subject, to) except Exception as e: log.warning("mail: sending to %s failed: %s", to, errstr(e)) def recipient(username: str | None) -> str | None: """Where to write to, and whether they want to hear from us at all.""" if not username: return None with db() as c: r = c.execute("SELECT email, notify, provider FROM users WHERE username=?", (username,)).fetchone() if not r or not r["notify"]: return None # A Google identity IS an email address, so there is nothing to configure # for those accounts unless they want mail somewhere else. if r["email"]: return r["email"] return username if r["provider"] == "google" and "@" in username else None async def notify_invited(row: sqlite3.Row, by: str, to: str): """Tell someone they've been invited. There is no token in this link and none is needed. The row already names the exact identity that may sign in, and Google will not hand us an unverified address — so possession of the email is not what authorises anyone, and a leaked link grants nothing. It is a courtesy, not a credential, which is also why it can be resent freely. """ label = PROVIDERS.get(row["provider"], {}).get("label", row["provider"]) where = BASE_URL or "your muse instance" await send_mail( to, f"{by} invited you to muse", f"You've been given access to muse — request a song, and it turns up\n" f"in the music library, correctly tagged.\n\n" f" {where}\n\n" f"Sign in with {label}, using the account for {row['username']}.\n" f"That address is what authorises you, so it has to be that one.\n\n" f"Playback lives in Navidrome, not here — muse only decides what gets\n" f"added to the library.\n") async def notify_imported(job, dest): to = recipient(job["requested_by"]) if not to: return album = job["album"] or "Singles" if job["year"]: album += f" ({job['year']})" await send_mail( to, f"muse: {job['title']} is in your library", f"{job['artist']} — {job['title']}\n{album}\n\n" f"Filed as: {dest}\n\nIt should appear in Navidrome on the next scan.\n") async def notify_review(job): to = recipient(job["requested_by"]) if not to: return await send_mail( to, f"muse: {job['title']} needs a look", f"{job['artist']} — {job['title']}\n\n" f"The best match found was only {(job['confidence'] or 0):.0%} confident, " f"so it has not been added to your library yet.\n\n" f"Why: {job['match_why'] or 'unknown'}\n" f"File: {job['remote_file']}\n\n" f"Approve or reject it under Review in muse.\n") # ─────────────────────────── import ─────────────────────────── def tag_and_file(job: sqlite3.Row, src: Path) -> Path: """Write known-good tags, then hardlink into the library.""" from mutagen import File as MFile from mutagen.easyid3 import EasyID3 from mutagen.flac import FLAC from mutagen.mp3 import MP3 try: if src.suffix.lower() == ".flac": audio = FLAC(src) else: try: audio = EasyID3(src) except Exception: m = MP3(src) m.add_tags() m.save() audio = EasyID3(src) audio["title"] = job["title"] audio["artist"] = job["artist"] audio["albumartist"] = job["artist"] if job["album"]: audio["album"] = job["album"] if job["year"]: audio["date"] = str(job["year"]) if job["track"]: audio["tracknumber"] = str(job["track"]) # Deliberately NOT writing the Deezer id into musicbrainz_trackid -- # that field means a specific thing and a wrong value there would # mislead Picard, beets, or anything else that later reads this file. audio.save() except Exception as e: # A tagging failure must not strand the file — Navidrome can still # read whatever tags it shipped with. log.warning("tagging failed for %s: %s", src.name, e) album = f"{safe(job['album'] or 'Singles')}" year = safe_year(job["year"]) if year: album += f" ({year})" dest_dir = LIBRARY_DIR / safe(job["artist"]) / album num = f"{int(job['track']):02d} - " if job["track"] else "" dest = dest_dir / f"{num}{safe(job['title'])}{src.suffix.lower()}" # Belt and braces. Every component above is either safe()'d or matched # against a whitelist, but that is a property of five call sites that must # each stay correct; this is one check that cannot be forgotten. It exists # because `year` was for a while appended AFTER safe() ran, so a request # with year="../../../../tmp/x" wrote outside the library. root = LIBRARY_DIR.resolve() if not Path(os.path.normpath(dest)).resolve().is_relative_to(root): raise ValueError(f"refusing to write outside the library: {dest}") dest_dir.mkdir(parents=True, exist_ok=True) if dest.exists(): return dest try: os.link(src, dest) # same filesystem -> free, and keeps sharing except OSError: import shutil shutil.copy2(src, dest) # cross-device fallback return dest # How far a file's length may drift from what Deezer said before it stops being # plausibly the same recording. Generous, because edits, fades, hidden tracks # and a stray second of silence are all normal; a different song is not. LENGTH_TOLERANCE = float(os.getenv("MUSE_LENGTH_TOLERANCE", "0.2")) def length_check(src: Path, want_ms) -> tuple[float, str] | None: """Compare the downloaded audio's real length against the request. The one signal filename matching cannot fake. A file named exactly right, in a folder named exactly right, is still the wrong song if it runs six minutes when the track is four -- and that is not hypothetical: The Fall's "Chicago, Now!" (359s) was filed as Michael Jackson's "Chicago" (245s) because every *textual* signal agreed. Returns a multiplier and a reason, or None when there is nothing to say. """ want = (want_ms or 0) / 1000.0 if want <= 0: return None try: got = mutagen.File(str(src)).info.length except Exception: return None # unreadable is review's problem if got <= 0: return None drift = abs(got - want) / want if drift <= LENGTH_TOLERANCE: return None return (0.35, f"runs {mmss(got)} but the track is {mmss(want)} " f"({drift:.0%} out), so this is probably a different recording") def mmss(seconds: float) -> str: return f"{int(seconds) // 60}:{int(seconds) % 60:02d}" def downloaded_index() -> dict[str, Path]: """Every completed file, by basename. slskd nests a download under the remote's parent directory name, so the basename is the only reliable key. Built once per poll cycle and shared by every job in it: the previous version walked the whole tree separately for each in-flight job, every ten seconds, which on a large downloads directory is a lot of stat() calls to answer the same question. """ out: dict[str, Path] = {} try: for p in DOWNLOAD_DIR.rglob("*"): if p.is_file(): out.setdefault(p.name, p) except OSError as e: log.warning("cannot walk %s: %s", DOWNLOAD_DIR, errstr(e)) return out # ─────────────────────────── search queue ─────────────────────────── work_q: "asyncio.Queue[str]" = asyncio.Queue() async def search_worker(n: int): """Drain the search queue, one job at a time, WORKERS of these running.""" while True: job_id = await work_q.get() try: with db() as c: job = c.execute("SELECT * FROM jobs WHERE id=?", (job_id,)).fetchone() if job and job["state"] in ("queued", "searching"): set_state(job_id, "searching", "") # A ladder left over from a run that stopped on a slskd stall # is still the right answer -- the search that produced it has # not become wrong. Resume it rather than paying for another. keys = job.keys() left = json.loads( (job["candidates"] if "candidates" in keys else None) or "[]") if left: log.info("job %s resuming ladder, %d candidate(s) left", job_id[:8], len(left)) await try_next_candidate(job_id, job["artist"], job["title"], job["album"]) else: await pick_and_enqueue(job_id, job["artist"], job["title"], job["album"]) except Exception as e: log.warning("worker %d: %s", n, e) try: set_state(job_id, "failed", str(e)[:200]) except Exception: pass finally: work_q.task_done() def transfer_index(payload: list) -> dict[str, dict]: """Flatten slskd's user -> directories -> files nesting. Keeps the whole file object, not just its state. `exception` is where the literal "Transfer rejected: File not shared." lives, and discarding it was why a failed job could only ever say "transfer Completed, Rejected" with no reason a person could act on. """ out: dict[str, dict] = {} for user in payload: for d in user.get("directories", []): for f in d.get("files", []): out[f.get("filename", "")] = f return out def stalled_for(job: sqlite3.Row) -> float: """Minutes since this transfer last had anything to show for itself. Falls back to created_at, which is what makes this work on rows written before started_at existed. Those are `downloading` with no clock at all, so returning zero for them would exempt precisely the jobs that have been stuck the longest -- the ones this check exists to catch. """ keys = job.keys() started = ((job["started_at"] if "started_at" in keys else None) or (job["created_at"] if "created_at" in keys else None)) if not started: return 0.0 try: t = datetime.fromisoformat(started) except ValueError: return 0.0 if t.tzinfo is None: t = t.replace(tzinfo=timezone.utc) return (datetime.now(timezone.utc) - t).total_seconds() / 60.0 async def abandon(job: sqlite3.Row, reason: str) -> None: """Give up on the current peer and fall through to the next candidate.""" with db() as c: row = c.execute("SELECT tried FROM jobs WHERE id=?", (job["id"],)).fetchone() tried = json.loads((row["tried"] if row else None) or "[]") tried.append({"username": job["username"], "filename": job["remote_file"], "reason": reason}) with db() as c: c.execute("UPDATE jobs SET tried=? WHERE id=?", (json.dumps(tried), job["id"])) log.info("job %s giving up on %s: %s", job["id"][:8], job["username"], reason) await try_next_candidate(job["id"], job["artist"], job["title"], job["album"]) async def cancel_transfer(username: str, filename: str) -> None: """Ask slskd to drop a transfer we have stopped waiting for.""" try: async with slskd_client(timeout=PROBE_TIMEOUT) as sc: await sc.delete(f"/api/v0/transfers/downloads/{username}", params={"filename": filename}) except Exception as e: log.debug("cancel %s/%s: %s", username, filename, errstr(e)) async def import_worker(): """Poll slskd for in-flight downloads and import the finished ones.""" missing: collections.Counter = collections.Counter() while True: await asyncio.sleep(10) try: with db() as c: jobs = c.execute( "SELECT * FROM jobs WHERE state='downloading'").fetchall() if not jobs: missing.clear() continue async with slskd_client() as sc: r = await sc.get("/api/v0/transfers/downloads") r.raise_for_status() index = transfer_index(r.json()) # One directory walk per cycle, shared by every job. This used to be # a full recursive rglob per stuck job, every ten seconds, forever. done_files: dict[str, Path] | None = None for job in jobs: t = index.get(job["remote_file"]) st = str((t or {}).get("state", "")) if "Completed, Succeeded" in st: if done_files is None: done_files = downloaded_index() src = done_files.get( job["remote_file"].replace("\\", "/").rsplit("/", 1)[-1]) if not src: # Normally just still being moved out of incomplete, but # unbounded before: a misconfigured DOWNLOAD_DIR meant # polling until the heat death of the process. if stalled_for(job) > STALL_MINUTES: set_state(job["id"], "failed", "downloaded but the file never appeared in " f"{DOWNLOAD_DIR} -- check MUSE_DOWNLOAD_DIR") continue try: actual = src.stat().st_size except OSError: actual = job["size"] conf = job["confidence"] why = job["match_why"] or "" # Applied here rather than in confidence(), because the # file does not exist yet when the candidate is chosen. penalty = length_check(src, job["duration"]) if penalty is not None and conf is not None: mult, reason = penalty conf = round(conf * mult, 3) why = f"{why}; {reason}" if why else reason with db() as c: c.execute("UPDATE jobs SET confidence=?," " match_why=? WHERE id=?", (conf, why, job["id"])) log.info("job %s length mismatch: %s", job["id"][:8], reason) if conf is not None and conf < AUTO_CONFIDENCE: band = ("almost certainly wrong" if conf < REVIEW_CONFIDENCE else "worth a look") set_state(job["id"], "review", f"{conf:.0%} confident, {band} -- {why}", size=actual) await notify_review(job) continue dest = tag_and_file(job, src) set_state(job["id"], "imported", str(dest), imported_path=str(dest), size=actual) await notify_imported(job, dest) elif "Completed" in st and "Succeeded" not in st: # Every terminal non-success used to end the request here. # "File not shared." is per-file and per-peer, so the only # sane response is to try the next candidate. why = (t or {}).get("exception") or st await abandon(job, str(why)[:120]) elif t is None: # Absent from slskd entirely: it restarted, or the transfer # was cleared. Tolerate a couple of polls in case it is a # race, then stop waiting for something nobody is doing. missing[job["id"]] += 1 if missing[job["id"]] >= 3: del missing[job["id"]] await abandon(job, "slskd forgot the transfer") else: missing.pop(job["id"], None) # Queued behind a wall, or connected and sending nothing. # There was no clock on this at all, which is how a job sat # in `downloading` for days behind a cheerful pulsing dot. if not (t or {}).get("bytesTransferred") \ and stalled_for(job) > STALL_MINUTES: place = (t or {}).get("placeInQueue") where = f" at #{place} in the queue" if place else "" await cancel_transfer(job["username"], job["remote_file"]) await abandon(job, f"no bytes in {STALL_MINUTES:.0f}m{where}") except Exception as e: log.warning("import worker: %s", errstr(e)) # ─────────────────────────── api ─────────────────────────── @asynccontextmanager async def lifespan(_: FastAPI): # init_db first: the preflight consults the users table to decide whether # the bootstrap allowlist is still mandatory. init_db() auth_preflight() # raises and stops the container if misconfigured # The queue lives in memory, so a restart mid-import would otherwise # strand every unstarted job in 'queued' forever with nothing to pick it # up. Anything not yet handed to slskd goes back on the queue. with db() as c: orphans = [r["id"] for r in c.execute( "SELECT id FROM jobs WHERE state IN ('queued','searching')").fetchall()] for jid in orphans: work_q.put_nowait(jid) if orphans: log.info("requeued %d job(s) left over from the last run", len(orphans)) tasks = [asyncio.create_task(import_worker())] tasks += [asyncio.create_task(search_worker(i)) for i in range(WORKERS)] log.info("muse up — slskd=%s library=%s formats=%s workers=%d", SLSKD_URL, LIBRARY_DIR, ",".join(FORMATS), WORKERS) yield for t in tasks: t.cancel() # ─────────────────────── static front end / CSP ─────────────────────── # # The UI is a Next.js static export: `next build` writes plain files into # web/out and this process serves them, so the deployment stays one container, # one process, one healthcheck, with no Node runtime in the image. # # That export contains inline ", re.S) def script_hashes() -> list[str]: out: set[str] = set() for page in ("index.html", "404.html"): f = WEB / page if not f.exists(): continue for body in _INLINE_SCRIPT.findall(f.read_bytes()): digest = base64.b64encode(hashlib.sha256(body).digest()).decode() out.add(f"'sha256-{digest}'") return sorted(out) SCRIPT_HASHES = " ".join(script_hashes()) # Framer Motion writes its pre-hydration `initial` state into style="" on the # server-rendered markup, so those attributes have to be allowed or every # animated element flashes in at full opacity before hydration corrects it. # # style-src-attr is the narrow lever for that: it permits style ATTRIBUTES # while style-src 'self' still blocks injected