Aryan SinghandClaude Opus 5
0ebc9515b4
MAX_ATTEMPTS was 5 while CANDIDATES was 8, so three ranked candidates were gathered and then never tried. The cap was arbitrary and the live numbers show what it cost. slskd answers 500 whenever it cannot open a connection to a peer, and that is common: retrying "Michael Jackson - Billie Jean" walked four unreachable peers before the fifth served it -- succeeding on precisely the last attempt the cap allowed. "The Weeknd - Die For You" ran the same gauntlet and lost, giving up with three untried candidates still on the row. Defaults to CANDIDATES, so the ladder is the ladder. Anyone wanting the old behaviour can still set MUSE_MAX_ATTEMPTS. The cap test lowers the value explicitly now; with the two equal by default it was only ever proving the list ran out, which is the other branch. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
3010 lines
129 KiB
Python
3010 lines
129 KiB
Python
"""
|
||
muse — request a song, get it in Navidrome.
|
||
|
||
The gap this fills: Lidarr thinks in MusicBrainz *releases*, so the smallest
|
||
thing it can want is an album. Soulseek trades individual files. Nothing in
|
||
between existed, so requesting one song meant either hoovering a whole
|
||
discography through Lidarr or hand-filing downloads out of slskd's UI.
|
||
|
||
The trick that makes this simple: the user PICKS a specific track before
|
||
anything is downloaded. That means we hold ground-truth metadata — artist,
|
||
album, year, track number — before the file exists. Taggers like beets have to
|
||
infer all of it afterwards from whatever a stranger named the file; we never
|
||
guess. We tag what was asked for.
|
||
|
||
Flow:
|
||
/api/search Deezer track search -> the user picks one
|
||
/api/request /track/{id} for exact metadata, then slskd search ->
|
||
score candidates -> enqueue best
|
||
worker poll slskd -> on completion: tag, then hardlink into
|
||
/data/media/music/<Artist>/<Album> (<Year>)/NN - Title.ext
|
||
|
||
Hardlink, not copy: the download stays in slskd's completed dir so it keeps
|
||
being shared, while the library gets its own name for the same bytes. One
|
||
file on disk, two names — the same trick the *arrs use.
|
||
"""
|
||
|
||
import asyncio
|
||
import base64
|
||
import collections
|
||
import difflib
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import os
|
||
import re
|
||
import secrets
|
||
import sqlite3
|
||
import unicodedata
|
||
import uuid
|
||
from contextlib import asynccontextmanager
|
||
from datetime import datetime, timedelta, timezone
|
||
from pathlib import Path
|
||
from urllib.parse import urlencode
|
||
|
||
import httpx
|
||
from fastapi import FastAPI, HTTPException, Query, Request
|
||
from fastapi.responses import (FileResponse, HTMLResponse, JSONResponse,
|
||
RedirectResponse, Response)
|
||
from fastapi.staticfiles import StaticFiles
|
||
from pydantic import BaseModel
|
||
|
||
log = logging.getLogger("muse")
|
||
logging.basicConfig(
|
||
level=os.getenv("MUSE_LOG_LEVEL", "INFO"),
|
||
format="[%(levelname)s|%(name)s] %(asctime)s: %(message)s",
|
||
datefmt="%Y-%m-%dT%H:%M:%S%z",
|
||
)
|
||
|
||
SLSKD_URL = os.getenv("SLSKD_URL", "http://192.168.1.87:5030").rstrip("/")
|
||
SLSKD_API_KEY = os.getenv("SLSKD_API_KEY", "")
|
||
DOWNLOAD_DIR = Path(os.getenv("MUSE_DOWNLOAD_DIR", "/data/torrents/soulseek/complete"))
|
||
LIBRARY_DIR = Path(os.getenv("MUSE_LIBRARY_DIR", "/data/media/music"))
|
||
DB_PATH = Path(os.getenv("MUSE_DB", "/config/muse.db"))
|
||
|
||
# Ordered best-first. A candidate's extension must appear here or it is
|
||
# rejected outright — this is what keeps .m4a rips and stray .cue files out.
|
||
FORMATS = [f.strip() for f in os.getenv("MUSE_FORMATS", "flac,mp3").split(",") if f.strip()]
|
||
|
||
# Below this, the remote filename doesn't plausibly contain the track title.
|
||
# Soulseek filenames are chaotic ("08-radiohead-creep_(accoustic)-ent.mp3"),
|
||
# so this compares against a normalised form, not the raw string.
|
||
MIN_MATCH = float(os.getenv("MUSE_MIN_MATCH", "0.45"))
|
||
|
||
# The title alone must clear this before the artist is even consulted. Without
|
||
# a floor of its own, a file by the right artist scored well enough on the
|
||
# combined ratio to be fetched no matter which of their songs it actually was.
|
||
TITLE_FLOOR = float(os.getenv("MUSE_TITLE_FLOOR", "0.5"))
|
||
|
||
# How many ranked candidates to keep per job. When a transfer fails, the next
|
||
# one is tried instead of failing the request -- "File not shared." is a
|
||
# per-file condition, so candidate two usually just works.
|
||
CANDIDATES = int(os.getenv("MUSE_CANDIDATES", "8"))
|
||
|
||
# Defaults to the size of the ladder, because keeping eight candidates and then
|
||
# refusing to try three of them is a cap with no reasoning behind it. Measured
|
||
# on a live instance: slskd answers 500 whenever it cannot open a connection to
|
||
# a peer, which is common enough that requests routinely need four or five
|
||
# attempts. One succeeded on exactly the fifth; another gave up at the cap with
|
||
# three candidates still in hand.
|
||
MAX_ATTEMPTS = int(os.getenv("MUSE_MAX_ATTEMPTS", str(CANDIDATES)))
|
||
|
||
# A download that has moved no bytes for this long is not going to. The peer
|
||
# queued it and vanished, or slskd lost the transfer; either way the job used
|
||
# to sit in `downloading` forever behind a cheerful pulsing dot.
|
||
STALL_MINUTES = float(os.getenv("MUSE_STALL_MINUTES", "20"))
|
||
|
||
# Query variants to try before giving up. Each costs a real Soulseek search,
|
||
# so this escalates only when a search underdelivers -- see search_queries().
|
||
MAX_QUERIES = int(os.getenv("MUSE_MAX_QUERIES", "4"))
|
||
MIN_CANDIDATES = int(os.getenv("MUSE_MIN_CANDIDATES", "3"))
|
||
|
||
# Issuing a search is a POST that occasionally blips. Nine production failures
|
||
# were one slskd hiccup during a bulk import, every queued job dying inside the
|
||
# same second, each one permanently.
|
||
SEARCH_RETRIES = int(os.getenv("MUSE_SEARCH_RETRIES", "3"))
|
||
|
||
# Soulseek results trickle in: what you have at t=5s is much worse than t=15s.
|
||
SEARCH_POLLS = int(os.getenv("MUSE_SEARCH_POLLS", "20"))
|
||
|
||
# Pause between candidate attempts. Without one, a ladder of 8 walks the whole
|
||
# list inside a single bad slskd window -- each enqueue blocks ~5s on a peer
|
||
# connection, so eight attempts is 40 seconds of hammering one subsystem, and
|
||
# it takes other people's searches down with it.
|
||
ATTEMPT_BACKOFF = float(os.getenv("MUSE_ATTEMPT_BACKOFF", "4"))
|
||
|
||
# Consecutive systemic failures before we stop walking the ladder. A file that
|
||
# is not shared says something about that file; a timeout says only that slskd
|
||
# is having a moment, and burning eight candidates to learn it once is waste.
|
||
SYSTEMIC_LIMIT = int(os.getenv("MUSE_SYSTEMIC_LIMIT", "2"))
|
||
|
||
# How many Soulseek searches may run at once.
|
||
#
|
||
# This is a hard cap, not a suggestion. Each search takes 10-40s of polling
|
||
# and real load on slskd and the Soulseek server, so firing one task per
|
||
# request — which is what an earlier version did — means pasting a 50-track
|
||
# list launches 50 concurrent searches and drowns the client. Everything goes
|
||
# through a bounded queue instead: requests return instantly, work drains at a
|
||
# civil rate.
|
||
WORKERS = int(os.getenv("MUSE_WORKERS", "3"))
|
||
|
||
# Search uses Deezer's public API: no key, no OAuth, and vastly better
|
||
# relevance than MusicBrainz for the way people actually type.
|
||
#
|
||
# MusicBrainz was tried first and rejected on measurements, not taste. It is a
|
||
# superb database but a poor search engine: "radiohead creep" returned eight
|
||
# covers and mashups before the Radiohead recording, and "bohemian rhapsody"
|
||
# never surfaced Queen at all across 90 results — because its text index
|
||
# matches "radiohead" inside a cover's *title* as readily as in the artist
|
||
# field. Client-side re-ranking can only reorder what came back, so it could
|
||
# not fix a result set that lacked the right row. Deezer returns
|
||
# "Creep / Radiohead / Pablo Honey" first for the same query.
|
||
#
|
||
# Deezer search omits track number and release date, so those are fetched from
|
||
# /track/{id} at request time — one extra call per request, not per result.
|
||
DEEZER = "https://api.deezer.com"
|
||
|
||
# Derivative versions to push down when the user did not ask for them.
|
||
# Matched on WORD BOUNDARIES, not as substrings: a bare "live" is needed to
|
||
# catch "Come As You Are (Live)", but substring matching would also penalise
|
||
# "Deliver Me" and "Olive". Same trap for "demo" vs "Demolition".
|
||
#
|
||
# A term only counts against a result when the user did NOT type it, so asking
|
||
# for "oasis live forever" or "nirvana unplugged demo" still works.
|
||
DERIVATIVE = ("cover", "covers", "karaoke", "tribute", "remix", "remixed",
|
||
"instrumental", "acapella", "originally performed",
|
||
"made popular by", "in the style of", "demo", "rehearsal",
|
||
"live", "unplugged", "re recorded", "rerecorded",
|
||
"sped up", "slowed", "reverb", "8d", "nightcore")
|
||
DERIVATIVE_RE = re.compile(r"\b(" + "|".join(DERIVATIVE) + r")\b")
|
||
|
||
# ─────────────────────────── auth ───────────────────────────
|
||
#
|
||
# OAuth against GitHub and/or Google, built so this can face the internet.
|
||
#
|
||
# FAILS CLOSED. Auth is on unless MUSE_AUTH is explicitly set to "off", and if
|
||
# it is on with incomplete config the app refuses to start. The failure mode
|
||
# worth engineering against is not a broken login page — it is a misconfigured
|
||
# deploy quietly serving an unauthenticated download client to the internet.
|
||
# You should have to *mean* it to run without auth.
|
||
#
|
||
# Sessions are server-side: the cookie carries a 256-bit random token and
|
||
# nothing else, and the database stores only its SHA-256. There is no signed
|
||
# payload to get wrong, a stolen database yields no usable cookies, and logout
|
||
# and revocation actually revoke.
|
||
#
|
||
# WHO MAY LOG IN is two rules, checked in this order:
|
||
# 1. MUSE_ALLOWED_USERS — the bootstrap allowlist. Always admitted. This is
|
||
# how the very first admin gets in when the users table is empty, and it
|
||
# is the recovery path if someone disables themselves.
|
||
# 2. A row in `users` that is not disabled — how everyone invited later gets
|
||
# in, without an env change or a redeploy.
|
||
# Anyone matching neither is rejected and the attempt is logged.
|
||
AUTH_MODE = os.getenv("MUSE_AUTH", "github").strip().lower()
|
||
ENABLED_PROVIDERS = [p for p in (x.strip() for x in AUTH_MODE.split(","))
|
||
if p and p != "off"]
|
||
|
||
# Absolute external URL, e.g. https://muse.aryan-singh.dev. Used to build the
|
||
# OAuth callback. Taken from config rather than from X-Forwarded-* headers on
|
||
# purpose: those are attacker-controllable and this is a redirect target.
|
||
BASE_URL = os.getenv("MUSE_BASE_URL", "").rstrip("/")
|
||
ALLOWED_USERS = {u.strip().lower()
|
||
for u in os.getenv("MUSE_ALLOWED_USERS", "").split(",") if u.strip()}
|
||
COOKIE_SECURE = os.getenv("MUSE_COOKIE_SECURE", "true").lower() != "false"
|
||
SESSION_DAYS = int(os.getenv("MUSE_SESSION_DAYS", "7"))
|
||
|
||
SESSION_COOKIE = "muse_session"
|
||
STATE_COOKIE = "muse_oauth_state"
|
||
|
||
# One callback URL for every provider — the provider is carried in the state
|
||
# COOKIE, not the query string, so adding Google needs no change to the GitHub
|
||
# OAuth app that is already registered against /auth/callback.
|
||
PROVIDERS: dict[str, dict] = {
|
||
"github": {
|
||
"label": "GitHub",
|
||
"authorize": "https://github.com/login/oauth/authorize",
|
||
"token": "https://github.com/login/oauth/access_token",
|
||
"scope": "read:user",
|
||
"client_id": os.getenv("MUSE_GITHUB_CLIENT_ID", "").strip(),
|
||
"client_secret": os.getenv("MUSE_GITHUB_CLIENT_SECRET", "").strip(),
|
||
"id_env": "MUSE_GITHUB_CLIENT_ID",
|
||
"secret_env": "MUSE_GITHUB_CLIENT_SECRET",
|
||
},
|
||
"google": {
|
||
"label": "Google",
|
||
"authorize": "https://accounts.google.com/o/oauth2/v2/auth",
|
||
"token": "https://oauth2.googleapis.com/token",
|
||
"scope": "openid email profile",
|
||
"client_id": os.getenv("MUSE_GOOGLE_CLIENT_ID", "").strip(),
|
||
"client_secret": os.getenv("MUSE_GOOGLE_CLIENT_SECRET", "").strip(),
|
||
"id_env": "MUSE_GOOGLE_CLIENT_ID",
|
||
"secret_env": "MUSE_GOOGLE_CLIENT_SECRET",
|
||
},
|
||
}
|
||
|
||
GH_API = "https://api.github.com/user"
|
||
GOOGLE_API = "https://openidconnect.googleapis.com/v1/userinfo"
|
||
|
||
# Paths reachable without a session. Deliberately tiny: /healthz reports a bare
|
||
# ok so the container healthcheck works without leaking configuration.
|
||
PUBLIC_PATHS = {"/healthz", "/auth/login", "/auth/callback", "/auth/logout",
|
||
"/login.css"}
|
||
|
||
ROLES = ("admin", "user", "guest")
|
||
|
||
|
||
def auth_preflight():
|
||
if AUTH_MODE == "off":
|
||
log.warning("=" * 62)
|
||
log.warning("MUSE_AUTH=off — every endpoint is UNAUTHENTICATED.")
|
||
log.warning("Never expose this to the internet in this state.")
|
||
log.warning("=" * 62)
|
||
return
|
||
if not ENABLED_PROVIDERS:
|
||
raise RuntimeError(f"MUSE_AUTH must list providers or be 'off', got {AUTH_MODE!r}")
|
||
|
||
unknown = [p for p in ENABLED_PROVIDERS if p not in PROVIDERS]
|
||
if unknown:
|
||
raise RuntimeError(
|
||
f"MUSE_AUTH names unknown provider(s) {unknown}. "
|
||
f"Valid: {', '.join(PROVIDERS)}, or 'off'.")
|
||
|
||
missing: list[str] = []
|
||
for p in ENABLED_PROVIDERS:
|
||
cfg = PROVIDERS[p]
|
||
if not cfg["client_id"]:
|
||
missing.append(cfg["id_env"])
|
||
if not cfg["client_secret"]:
|
||
missing.append(cfg["secret_env"])
|
||
if not BASE_URL:
|
||
missing.append("MUSE_BASE_URL")
|
||
# The bootstrap allowlist is only mandatory while nobody exists yet —
|
||
# otherwise a working multi-user install would be held hostage by an env
|
||
# var it has outgrown. But an EMPTY database and no allowlist means nobody
|
||
# can ever log in, which is a misconfiguration worth refusing to start on.
|
||
if not ALLOWED_USERS and not _any_users():
|
||
missing.append("MUSE_ALLOWED_USERS")
|
||
if missing:
|
||
raise RuntimeError(
|
||
"Auth is enabled but " + ", ".join(sorted(set(missing))) + " not set. "
|
||
"Refusing to start: set them, or set MUSE_AUTH=off to run without "
|
||
"authentication (LAN only).")
|
||
|
||
if not BASE_URL.startswith("https://") and COOKIE_SECURE:
|
||
log.warning("MUSE_BASE_URL is not https but cookies are Secure — the "
|
||
"browser will refuse to send them. Set MUSE_COOKIE_SECURE=false "
|
||
"only if you are testing over plain http on a LAN.")
|
||
log.info("auth: providers=%s bootstrap_allowlist=%d",
|
||
",".join(ENABLED_PROVIDERS), len(ALLOWED_USERS))
|
||
|
||
|
||
def _any_users() -> bool:
|
||
try:
|
||
with db() as c:
|
||
return c.execute("SELECT 1 FROM users LIMIT 1").fetchone() is not None
|
||
except sqlite3.Error:
|
||
return False
|
||
|
||
|
||
def _hash(tok: str) -> str:
|
||
return hashlib.sha256(tok.encode()).hexdigest()
|
||
|
||
|
||
def new_session(username: str) -> str:
|
||
token = secrets.token_urlsafe(32)
|
||
exp = datetime.now(timezone.utc) + timedelta(days=SESSION_DAYS)
|
||
with db() as c:
|
||
c.execute("INSERT INTO sessions (token_hash,username,expires_at) VALUES (?,?,?)",
|
||
(_hash(token), username, exp.isoformat()))
|
||
# Opportunistic sweep; no scheduler needed for a handful of rows.
|
||
c.execute("DELETE FROM sessions WHERE expires_at < ?",
|
||
(datetime.now(timezone.utc).isoformat(),))
|
||
return token
|
||
|
||
|
||
def may_login(username: str, row: sqlite3.Row | None,
|
||
provider: str | None = None) -> bool:
|
||
"""The single authorization rule, used at login AND on every request.
|
||
|
||
Bootstrap allowlist first, then a live, non-disabled user row. Deliberately
|
||
one function: an authorization check that runs only at login is not a
|
||
revocation mechanism, it is a delay — disabling someone would leave their
|
||
session valid for up to MUSE_SESSION_DAYS.
|
||
|
||
An invite is bound to the provider it was issued for. Invite
|
||
friend@gmail.com for Google and Google is the only way in for that
|
||
identity; there is no second, weaker door onto the same account. `provider`
|
||
is None on the per-request re-check, where the question is only whether the
|
||
account is still live.
|
||
"""
|
||
if username.lower() in ALLOWED_USERS:
|
||
return True
|
||
if not row or row["disabled"]:
|
||
return False
|
||
return provider is None or row["provider"] == provider
|
||
|
||
|
||
def current_user(request: Request) -> sqlite3.Row | None:
|
||
"""The signed-in user's row, or None. Re-validated on every request."""
|
||
tok = request.cookies.get(SESSION_COOKIE)
|
||
if not tok:
|
||
return None
|
||
with db() as c:
|
||
row = c.execute(
|
||
"SELECT username, expires_at FROM sessions WHERE token_hash=?",
|
||
(_hash(tok),)).fetchone()
|
||
if not row:
|
||
return None
|
||
try:
|
||
if datetime.fromisoformat(row["expires_at"]) < datetime.now(timezone.utc):
|
||
return None
|
||
except ValueError:
|
||
return None
|
||
u = c.execute("SELECT * FROM users WHERE username=?",
|
||
(row["username"],)).fetchone()
|
||
if not may_login(row["username"], u):
|
||
return None
|
||
if u is None:
|
||
# Allowlisted but no row — an admin deleted the bootstrap account while
|
||
# its session was still live. The allowlist is exactly the recovery
|
||
# path for that, so put the row back rather than half-authenticating
|
||
# them into a state where every role check fails.
|
||
u = upsert_user(row["username"])
|
||
log.info("auth: restored deleted allowlisted user %s", row["username"])
|
||
return u
|
||
|
||
|
||
def session_user(request: Request) -> str | None:
|
||
u = current_user(request)
|
||
return u["username"] if u else None
|
||
|
||
|
||
def user_role(request: Request) -> str:
|
||
u = current_user(request)
|
||
if u:
|
||
return u["role"]
|
||
# With auth off there is nobody to be, so everyone is an admin — otherwise
|
||
# the admin panel would be unreachable on a LAN-only install.
|
||
return "admin" if AUTH_MODE == "off" else "guest"
|
||
|
||
|
||
def require_admin(request: Request) -> str:
|
||
if AUTH_MODE == "off":
|
||
return "local"
|
||
u = current_user(request)
|
||
if not u:
|
||
raise HTTPException(401, "unauthenticated")
|
||
if u["role"] != "admin":
|
||
log.warning("admin: DENIED %s (role=%s) -> %s",
|
||
u["username"], u["role"], request.url.path)
|
||
raise HTTPException(403, "admin only")
|
||
return u["username"]
|
||
|
||
|
||
SAFE = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
|
||
NOISE = re.compile(r"[^a-z0-9]+")
|
||
YEAR = re.compile(r"^\d{4}$")
|
||
|
||
# Apostrophes vanish rather than becoming a separator, so "Don't" normalises
|
||
# onto a filename's "Dont". Everything else non-alphanumeric becomes a space,
|
||
# so "R&B" stays two tokens instead of collapsing to the unrelated word "rb".
|
||
QUOTES = re.compile("['\u2018\u2019\u02bc\u0060\"\u201c\u201d]")
|
||
|
||
# Letters NFKD leaves alone because they are atomic rather than base + accent.
|
||
FOLD = str.maketrans({"\u00f8": "o", "\u00e6": "ae", "\u0153": "oe",
|
||
"\u00f0": "d", "\u00fe": "th", "\u0142": "l",
|
||
"\u00df": "ss", "\u0111": "d", "\u0131": "i"})
|
||
|
||
# A leading track number: "02 - ", "02.", "A1 ". Anchored and bounded so it
|
||
# cannot eat a title that merely opens with a digit ("1979", "99 Problems").
|
||
TRACKNO = re.compile(r"^\s*(?:[a-d]?\d{1,3})\s*(?:[-._)\]]\s*)+", re.I)
|
||
|
||
# Technical release tags. Deliberately NOT "live", "remix" or "acoustic":
|
||
# confidence() reads those to decide whether this is the recording that was
|
||
# asked for, and stripping them here would blind it.
|
||
JUNK = re.compile(
|
||
r"[\[({]\s*(?:\d{2,4}\s*k(?:bps)?|v\d|web|cd\s*\d*|vinyl|flac|mp3|"
|
||
r"lossless|hi-?res|\d{2,3}\s*bit|\d{2}\s*khz)[^\])}]*[\])}]"
|
||
r"|\b(?:\d{3}\s*kbps|web-?dl|cdrip|vbr|cbr)\b", re.I)
|
||
|
||
# Most filesystems cap a path component at 255 bytes. Track titles can be far
|
||
# longer than that once a stranger has appended their own notes, and the
|
||
# resulting ENAMETOOLONG would fail the import after the download had finished.
|
||
MAX_COMPONENT = 150
|
||
|
||
|
||
def safe(name: str) -> str:
|
||
"""Filesystem-safe path component. Trailing dots/spaces break on some FS."""
|
||
return SAFE.sub("_", name).strip(". ")[:MAX_COMPONENT].strip() or "Unknown"
|
||
|
||
|
||
def safe_cover(url) -> str | None:
|
||
"""Artwork URL, stored and later rendered as an <img src>.
|
||
|
||
The CSP already stops a foreign host loading, so this is not an XSS gate —
|
||
it stops the field being used as free storage. `data:` is permitted by
|
||
img-src, so without a check any client could park megabytes of base64 in
|
||
the jobs table on every request.
|
||
"""
|
||
u = str(url or "").strip()
|
||
return u if u.startswith("https://") and len(u) <= 300 else None
|
||
|
||
|
||
def safe_year(year) -> str | None:
|
||
"""A year, or nothing. This lands in a directory name, so it is checked
|
||
against a whitelist rather than merely scrubbed."""
|
||
y = str(year or "").strip()
|
||
return y if YEAR.match(y) else None
|
||
|
||
|
||
def norm(s: str) -> str:
|
||
"""Lowercase, fold accents away, and reduce to alphanumeric tokens.
|
||
|
||
The accent fold is load-bearing. Deezer returns "Beyonc\u00e9", "Bj\u00f6rk" and
|
||
"Sigur R\u00f3s"; the strangers who named the files on Soulseek wrote
|
||
"Beyonce", "Bjork", "Sigur Ros". Without it the accented character is not
|
||
in [a-z0-9] and became a *space*, so "beyonce" was compared against
|
||
"beyonc " -- a containment miss and a mediocre ratio, on a track that was
|
||
sitting right there in the results.
|
||
"""
|
||
s = unicodedata.normalize("NFKD", str(s).lower().translate(FOLD))
|
||
s = "".join(c for c in s if not unicodedata.combining(c))
|
||
return NOISE.sub(" ", QUOTES.sub("", s)).strip()
|
||
|
||
|
||
def bare_name(remote_path: str) -> str:
|
||
"""A remote filename with extension, track number and release tags gone.
|
||
|
||
All of that is decoration a peer added that says nothing about which
|
||
recording the file holds. Leaving it in drags the similarity ratio down on
|
||
files that are in fact exactly right.
|
||
"""
|
||
base = remote_path.replace("\\", "/").rsplit("/", 1)[-1]
|
||
if "." in base:
|
||
base = base.rsplit(".", 1)[0]
|
||
return JUNK.sub(" ", TRACKNO.sub("", base)).strip()
|
||
|
||
|
||
def similarity(want: str, got: str) -> float:
|
||
"""Best of a sequence ratio and a token-coverage ratio, 0..1.
|
||
|
||
difflib alone is order-sensitive: "creep radiohead" against "radiohead
|
||
creep" scores far below what a human would call a match, and Soulseek
|
||
filenames put the fields in whatever order the ripper felt like. The token
|
||
measure is order-free, so taking the better of the two absorbs reordering
|
||
and interleaved junk without having to loosen MIN_MATCH.
|
||
"""
|
||
if not want or not got:
|
||
return 0.0
|
||
seq = difflib.SequenceMatcher(None, want, got).ratio()
|
||
a, b = set(want.split()), set(got.split())
|
||
return max(seq, len(a & b) / len(a) if a else 0.0)
|
||
|
||
|
||
def match_ratio(remote_path: str, artist: str, title: str) -> float:
|
||
"""How well a remote path matches the track asked for, 0..1.
|
||
|
||
Title and artist are measured separately and the title has to stand on its
|
||
own. A single ratio over the blob "artist title" let a strong artist match
|
||
carry a weak title -- the same failure the Deezer ranking keeps a dedicated
|
||
term for, where `Oasis - Live Forever` once resolved to `Cloudburst`
|
||
because the artist agreed and nothing else had to.
|
||
|
||
The artist is looked for in the whole path rather than the filename: a file
|
||
at `Radiohead/Pablo Honey/02 Creep.flac` names its artist in the folder,
|
||
which is the commonest well-organised case there is.
|
||
"""
|
||
stem = norm(bare_name(remote_path))
|
||
path = norm(remote_path.replace("\\", "/"))
|
||
nt, na = norm(title), norm(artist)
|
||
|
||
t = 1.0 if (nt and nt in stem) else similarity(nt, stem)
|
||
if t < TITLE_FLOOR:
|
||
return 0.0 # wrong song; a matching artist must not rescue it
|
||
a = 1.0 if (na and na in path) else similarity(na, path)
|
||
return 0.75 * t + 0.25 * a
|
||
|
||
|
||
# ─────────────────────────── storage ───────────────────────────
|
||
|
||
def db():
|
||
c = sqlite3.connect(DB_PATH, timeout=30)
|
||
c.row_factory = sqlite3.Row
|
||
return c
|
||
|
||
|
||
def init_db():
|
||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||
with db() as c:
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS jobs (
|
||
id TEXT PRIMARY KEY,
|
||
state TEXT NOT NULL, -- queued|searching|downloading|review|imported|failed
|
||
detail TEXT,
|
||
artist TEXT, title TEXT, album TEXT,
|
||
year TEXT, track INTEGER,
|
||
recording_mbid TEXT, release_mbid TEXT,
|
||
username TEXT, remote_file TEXT, size INTEGER,
|
||
imported_path TEXT,
|
||
created_at TEXT DEFAULT (datetime('now'))
|
||
)
|
||
""")
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS sessions (
|
||
token_hash TEXT PRIMARY KEY, -- sha256 of the cookie value
|
||
username TEXT NOT NULL,
|
||
created_at TEXT DEFAULT (datetime('now')),
|
||
expires_at TEXT NOT NULL
|
||
)
|
||
""")
|
||
# Users exist as rows even while the allowlist is a single name in an
|
||
# env var. Roles and quotas are read from here from the start, so
|
||
# opening this up to other people later is an INSERT and a config
|
||
# change rather than a schema migration against live job history.
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS users (
|
||
username TEXT PRIMARY KEY COLLATE NOCASE,
|
||
provider TEXT NOT NULL DEFAULT 'github',
|
||
role TEXT NOT NULL DEFAULT 'user', -- admin|user|guest
|
||
display TEXT,
|
||
avatar TEXT,
|
||
quota INTEGER NOT NULL DEFAULT 0, -- requests/month, 0 = unlimited
|
||
disabled INTEGER NOT NULL DEFAULT 0,
|
||
created_at TEXT DEFAULT (datetime('now')),
|
||
last_seen TEXT
|
||
)
|
||
""")
|
||
# Added after the first release, so existing databases need it grafted
|
||
# on rather than recreated -- job history is the whole point of the file.
|
||
cols = {r["name"] for r in c.execute("PRAGMA table_info(jobs)").fetchall()}
|
||
for col, decl in (("requested_by", "TEXT"),
|
||
("cover", "TEXT"),
|
||
("genre", "TEXT"),
|
||
("duration", "INTEGER"),
|
||
("confidence", "REAL"),
|
||
("match_why", "TEXT"),
|
||
("reviewed_by", "TEXT"),
|
||
# The candidate ladder. Keeping the runners-up on the
|
||
# row is what lets a rejected transfer fall through to
|
||
# the next peer minutes later, in a different task,
|
||
# after the search result set is long gone.
|
||
("candidates", "TEXT"),
|
||
("tried", "TEXT"),
|
||
("attempts", "INTEGER NOT NULL DEFAULT 0"),
|
||
("started_at", "TEXT")):
|
||
if col not in cols:
|
||
c.execute(f"ALTER TABLE jobs ADD COLUMN {col} {decl}")
|
||
ucols = {r["name"] for r in c.execute("PRAGMA table_info(users)").fetchall()}
|
||
for col, decl in (("storage_quota", "INTEGER NOT NULL DEFAULT 0"),
|
||
("invited_by", "TEXT"),
|
||
("email", "TEXT"),
|
||
("notify", "INTEGER NOT NULL DEFAULT 1")):
|
||
if col not in ucols:
|
||
c.execute(f"ALTER TABLE users ADD COLUMN {col} {decl}")
|
||
c.execute("CREATE INDEX IF NOT EXISTS jobs_by_user ON jobs (requested_by)")
|
||
c.execute("CREATE INDEX IF NOT EXISTS jobs_by_state ON jobs (state)")
|
||
c.execute("CREATE INDEX IF NOT EXISTS jobs_by_created ON jobs (created_at)")
|
||
|
||
|
||
def upsert_user(username: str, avatar: str = "", display: str = "",
|
||
provider: str = "github") -> sqlite3.Row:
|
||
"""Record the sign-in.
|
||
|
||
The first user to ever log in becomes admin — no bootstrap ceremony, no
|
||
default password to forget. Everyone after that is a plain user until an
|
||
admin says otherwise; role is never touched on subsequent logins, or every
|
||
sign-in would silently undo a demotion.
|
||
"""
|
||
with db() as c:
|
||
first = c.execute("SELECT COUNT(*) n FROM users").fetchone()["n"] == 0
|
||
c.execute(
|
||
"INSERT INTO users (username,provider,role,display,avatar,last_seen) "
|
||
"VALUES (?,?,?,?,?,datetime('now')) "
|
||
"ON CONFLICT(username) DO UPDATE SET "
|
||
" last_seen=datetime('now'), "
|
||
# provider is NOT refreshed from the login. It is part of the
|
||
# identity an admin invited, and may_login now enforces it — so
|
||
# overwriting it here would let a successful login silently
|
||
# rewrite the very rule that permitted it.
|
||
" avatar=COALESCE(NULLIF(excluded.avatar,''), users.avatar), "
|
||
" display=COALESCE(NULLIF(excluded.display,''), users.display)",
|
||
(username, provider, "admin" if first else "user", display, avatar))
|
||
return c.execute("SELECT * FROM users WHERE username=?", (username,)).fetchone()
|
||
|
||
|
||
def set_state(job_id: str, state: str, detail: str = "", **cols):
|
||
sets = ", ".join(f"{k}=?" for k in cols)
|
||
sql = f"UPDATE jobs SET state=?, detail=?{', ' + sets if sets else ''} WHERE id=?"
|
||
with db() as c:
|
||
c.execute(sql, [state, detail, *cols.values(), job_id])
|
||
log.info("job %s -> %s %s", job_id[:8], state, detail)
|
||
|
||
|
||
# ─────────────────────────── slskd ───────────────────────────
|
||
|
||
def slskd_client(timeout: float = 30.0) -> httpx.AsyncClient:
|
||
"""30s suits searches and transfer polling, which are legitimately slow.
|
||
|
||
Status probes pass something much shorter: they run on a UI poll, and a
|
||
probe that outlasts its own poll interval stacks requests and leaves the
|
||
admin panel on skeletons for the whole timeout — precisely when slskd is
|
||
down and you opened the panel to find out why.
|
||
"""
|
||
return httpx.AsyncClient(
|
||
base_url=SLSKD_URL,
|
||
headers={"X-API-Key": SLSKD_API_KEY},
|
||
timeout=httpx.Timeout(timeout),
|
||
)
|
||
|
||
|
||
# slskd is on the LAN, so a reachable instance answers in milliseconds. Long
|
||
# enough to absorb a slow moment, short enough to stay under the 5s UI poll.
|
||
PROBE_TIMEOUT = 4.0
|
||
|
||
|
||
def errstr(e: Exception) -> str:
|
||
"""httpx's timeout and connection errors often str() to an empty string,
|
||
which would render as 'slskd unreachable — ' with nothing after it."""
|
||
return (str(e) or type(e).__name__)[:120]
|
||
|
||
|
||
def ext_of(remote_path: str) -> str:
|
||
# Soulseek paths are Windows-style; the API's `extension` field comes back
|
||
# empty in practice, so derive it ourselves.
|
||
return remote_path.rsplit(".", 1)[-1].lower() if "." in remote_path else ""
|
||
|
||
|
||
def confidence(filename: str, artist: str, title: str,
|
||
album: str | None = None) -> tuple[float, list[str]]:
|
||
"""How sure are we this file is the song that was asked for, 0..1.
|
||
|
||
Deliberately separate from score(). score() decides *which* candidate to
|
||
fetch and is right to weigh peer speed and queue length; none of that says
|
||
anything about whether the file is the correct recording. Mixing the two
|
||
would let a fast peer raise the confidence of a wrong match.
|
||
|
||
Returns the score and the human-readable reasons behind it, because a
|
||
number alone gives the reviewer nothing to act on.
|
||
"""
|
||
stem = norm(bare_name(filename))
|
||
path = norm(filename.replace("\\", "/"))
|
||
na, nt = norm(artist), norm(title)
|
||
why: list[str] = []
|
||
|
||
# Title is the load-bearing signal: a wrong title is a wrong song, whereas
|
||
# a missing artist is routine in files organised into per-artist folders.
|
||
t_ratio = similarity(nt, stem)
|
||
if nt and nt in stem:
|
||
t_score, t_ratio = 1.0, max(t_ratio, 1.0)
|
||
why.append("title matches")
|
||
elif t_ratio >= 0.75:
|
||
t_score = t_ratio
|
||
why.append(f"title close ({t_ratio:.0%})")
|
||
else:
|
||
t_score = t_ratio
|
||
why.append(f"title differs ({t_ratio:.0%})")
|
||
|
||
# Artist may legitimately live in the path rather than the filename.
|
||
if na and na in stem:
|
||
a_score = 1.0
|
||
why.append("artist in filename")
|
||
elif na and na in path:
|
||
a_score = 0.9
|
||
why.append("artist in folder")
|
||
elif na and set(na.split()) & set(path.split()):
|
||
a_score = 0.5
|
||
why.append("artist partly matches")
|
||
else:
|
||
a_score = 0.0
|
||
why.append("artist not found in path")
|
||
|
||
# 0.75/0.25 rather than an even split. A file named exactly right but
|
||
# sitting in an unorganised folder is the commonest honest case there is,
|
||
# and it has to land in the review band, not below it — 0.65 would have
|
||
# binned it alongside genuinely wrong songs.
|
||
c = 0.75 * t_score + 0.25 * a_score
|
||
|
||
if album and norm(album) and norm(album) in path:
|
||
c = min(1.0, c + 0.05)
|
||
why.append("album in path")
|
||
|
||
# A live or remix version of the right song is still the wrong file when
|
||
# the request was for the studio recording, and that is exactly the case a
|
||
# reviewer needs to see rather than have silently imported. So the penalty
|
||
# caps confidence below AUTO instead of merely scaling it: "a human must
|
||
# decide" is the whole meaning of the flag, and a perfect artist+title
|
||
# match should not be able to auto-import past it.
|
||
variants = [w for w in ("live", "remix", "cover", "karaoke", "instrumental",
|
||
"acoustic", "demo", "edit", "radio edit")
|
||
if re.search(rf"\b{w}\b", stem) and w not in nt]
|
||
if variants:
|
||
c = min(c * 0.8, AUTO_CONFIDENCE - 0.1)
|
||
why.append(f"looks like a {variants[0]} version, which was not requested")
|
||
|
||
return round(max(0.0, min(1.0, c)), 3), why
|
||
|
||
|
||
# >= AUTO imports without asking. Below REVIEW the match is bad enough that a
|
||
# human almost certainly rejects it, but it is still held rather than binned:
|
||
# the file is already downloaded and only the reviewer knows if it is right.
|
||
AUTO_CONFIDENCE = float(os.getenv("MUSE_AUTO_CONFIDENCE", "0.95"))
|
||
REVIEW_CONFIDENCE = float(os.getenv("MUSE_REVIEW_CONFIDENCE", "0.70"))
|
||
|
||
|
||
def score(candidate: dict, peer: dict, artist: str, title: str) -> float | None:
|
||
"""Rank one remote file. None = reject.
|
||
|
||
Ordering intent: actually-gettable first, then correct format, then
|
||
filename plausibility, then bitrate.
|
||
|
||
Availability leads deliberately. It used to sit *below* format -- 100
|
||
points per format rank against 60 for a free upload slot -- which meant a
|
||
FLAC behind two hundred queued people always beat an MP3 available right
|
||
now. Whether the peer can serve you is the single most predictive signal
|
||
for whether the job ever finishes, so it outranks the preference for a
|
||
better file. Format is the tiebreak within an availability tier.
|
||
"""
|
||
ext = ext_of(candidate["filename"])
|
||
if ext not in FORMATS:
|
||
return None
|
||
|
||
ratio = match_ratio(candidate["filename"], artist, title)
|
||
if ratio < MIN_MATCH:
|
||
return None
|
||
|
||
s = 0.0
|
||
s += 220 if peer.get("hasFreeUploadSlot") else 0
|
||
# Unsaturated: the old `max(0, 40 - queueLength)` scored a 40-deep queue
|
||
# and a 5000-deep one identically, which is the difference between waiting
|
||
# a minute and never being served at all.
|
||
s += 120 / (1 + peer.get("queueLength", 0) / 8)
|
||
s += min(peer.get("uploadSpeed", 0) / 50_000, 20)
|
||
s += (len(FORMATS) - FORMATS.index(ext)) * 60 # format preference
|
||
s += ratio * 90
|
||
s += min(candidate.get("bitRate") or 0, 320) / 32 # 0-10
|
||
return s
|
||
|
||
|
||
PAREN = re.compile(r"[\[(\{][^\])}]*[\])}]")
|
||
FEAT = re.compile(r"\b(?:feat|ft|featuring)\b\.?.*$", re.I)
|
||
ARTICLE = re.compile(r"^(?:the|a|an|los|las|les)\s+", re.I)
|
||
|
||
|
||
class SearchUnavailable(Exception):
|
||
"""slskd could not accept a search. Transient, not a bad request."""
|
||
|
||
|
||
def search_queries(artist: str, title: str) -> list[str]:
|
||
"""Query variants to try, loosest last.
|
||
|
||
The Soulseek server filters searches containing certain artist names. That
|
||
is measured, not theorised: on this network "Michael Jackson Billie Jean"
|
||
and bare "Michael Jackson" both return zero responses, while "Billie Jean"
|
||
returns 1083 files. Sending only `artist title` -- all this used to do --
|
||
means every request for a filtered artist dies reporting "searched 0
|
||
files", and that is exactly the roster of names in the failed jobs:
|
||
Michael Jackson, The Weeknd, Kanye West, Frank Ocean.
|
||
|
||
So the ladder ends at the title alone, and match_ratio() then recognises
|
||
the artist from the folder path instead of from the query. The middle
|
||
rungs handle the ordinary case where Deezer's metadata is simply more
|
||
decorated than the filename a stranger typed.
|
||
"""
|
||
out: list[str] = []
|
||
seen: set[str] = set()
|
||
|
||
def add(*parts: str) -> None:
|
||
q = " ".join(p for p in parts if p).strip()
|
||
# Dedupe on the normalised form, so "Creep (Remastered)" and "Creep"
|
||
# are recognised as one query once the punctuation is gone.
|
||
k = norm(q)
|
||
if q and k and k not in seen:
|
||
seen.add(k)
|
||
out.append(q)
|
||
|
||
plain_t = PAREN.sub(" ", title).strip() or title
|
||
lean_t = FEAT.sub(" ", plain_t).strip() or plain_t
|
||
head_t = lean_t.split("/")[0].strip() or lean_t # "A / B" -> "A"
|
||
lean_a = ARTICLE.sub("", FEAT.sub(" ", artist).strip()).strip()
|
||
|
||
add(artist, title) # what was sent before, and still best when it works
|
||
add(artist, lean_t) # decoration and featuring credits gone
|
||
add(lean_a, head_t) # leading article gone, split title
|
||
add(head_t) # title alone -- this is the rung that beats the filter
|
||
add(lean_t)
|
||
return out[:MAX_QUERIES]
|
||
|
||
|
||
async def run_search(sc: httpx.AsyncClient, text: str) -> dict:
|
||
"""Issue one Soulseek search and poll it to completion.
|
||
|
||
The search is deleted only once it has actually finished. Deleting one
|
||
still InProgress makes slskd throw an optimistic-concurrency error inside
|
||
its own SearchService and lose the record; the old code deleted
|
||
unconditionally after its 40s budget, so every slow search corrupted a row.
|
||
"""
|
||
search_id = str(uuid.uuid4())
|
||
for attempt in range(SEARCH_RETRIES):
|
||
try:
|
||
r = await sc.post("/api/v0/searches",
|
||
json={"id": search_id, "searchText": text})
|
||
r.raise_for_status()
|
||
break
|
||
except Exception as e:
|
||
if attempt == SEARCH_RETRIES - 1:
|
||
raise SearchUnavailable(errstr(e)) from e
|
||
await asyncio.sleep(2 * (attempt + 1))
|
||
|
||
results: dict = {}
|
||
finished = False
|
||
for _ in range(SEARCH_POLLS):
|
||
await asyncio.sleep(2)
|
||
try:
|
||
rr = await sc.get(f"/api/v0/searches/{search_id}",
|
||
params={"includeResponses": "true"})
|
||
rr.raise_for_status()
|
||
results = rr.json()
|
||
except Exception as e:
|
||
# Swallowed silently before, which made a broken slskd look
|
||
# identical to a track nobody happens to share.
|
||
log.debug("poll %r: %s", text, errstr(e))
|
||
continue
|
||
if str(results.get("state", "")).startswith("Completed"):
|
||
finished = True
|
||
break
|
||
|
||
if finished:
|
||
try:
|
||
await sc.delete(f"/api/v0/searches/{search_id}")
|
||
except Exception as e:
|
||
log.debug("cleanup %r: %s", text, errstr(e))
|
||
else:
|
||
log.info("search %r unfinished after %ds, leaving it for slskd to close",
|
||
text, SEARCH_POLLS * 2)
|
||
return results
|
||
|
||
|
||
def spread_by_peer(ranked: list[tuple[float, dict, dict]]) -> list[dict]:
|
||
"""Order candidates so consecutive attempts land on different peers.
|
||
|
||
Straight score order routinely stacks three files from one well-stocked
|
||
user at the top; if that user is offline all three attempts burn on the
|
||
same dead connection. Peers arrive best-first, so round-robining them keeps
|
||
the strongest candidate first and still guarantees variety behind it.
|
||
"""
|
||
by_peer: dict[str, list[dict]] = {}
|
||
for s, peer, f in ranked:
|
||
by_peer.setdefault(peer["username"], []).append({
|
||
"username": peer["username"],
|
||
"filename": f["filename"],
|
||
"size": f.get("size") or 0,
|
||
"score": round(s, 1),
|
||
})
|
||
queues = list(by_peer.values())
|
||
out: list[dict] = []
|
||
while queues:
|
||
for q in list(queues):
|
||
out.append(q.pop(0))
|
||
if not q:
|
||
queues.remove(q)
|
||
return out
|
||
|
||
|
||
def no_candidate_detail(seen: int, stats: "collections.Counter",
|
||
queries: list[str]) -> str:
|
||
"""Say which filter emptied the results, not merely that they are empty.
|
||
|
||
One message used to cover four unrelated situations, and 28 of 29 real
|
||
occurrences turned out to be the search itself coming back empty -- which
|
||
is now stated outright instead of being reported as a matching failure.
|
||
"""
|
||
if seen == 0:
|
||
tried = ", ".join(repr(q) for q in queries)
|
||
word = "y" if len(queries) == 1 else "ies"
|
||
return f"Soulseek returned nothing for {len(queries)} quer{word}: {tried}"
|
||
bits = []
|
||
if stats["format"]:
|
||
bits.append(f"{stats['format']} not {'/'.join(FORMATS)}")
|
||
if stats["weak"]:
|
||
bits.append(f"{stats['weak']} did not match the title")
|
||
if stats["locked"]:
|
||
bits.append(f"{stats['locked']} locked")
|
||
return f"{seen} files found, none usable: " + ", ".join(bits)
|
||
|
||
|
||
def exhausted_detail(tried: list[dict], attempts: int) -> str:
|
||
"""Summarise the whole ladder instead of overwriting it with the last error.
|
||
|
||
Counts `tried`, not `attempts`. They agree for anything this version
|
||
enqueued, but a row carried over from before the ladder existed has
|
||
attempts=0 with a real failure recorded against it, which rendered as the
|
||
nonsense "tried 0 peers: 1x no bytes in 20m". The list is the evidence.
|
||
"""
|
||
if not tried:
|
||
return "no candidate could be enqueued"
|
||
n_peers = len({t.get("username") for t in tried if t.get("username")}) or len(tried)
|
||
reasons = collections.Counter(t["reason"].split(":")[0] for t in tried)
|
||
plural = "" if n_peers == 1 else "s"
|
||
joined = ", ".join(f"{n}x {r}" for r, n in reasons.most_common(3))
|
||
return f"tried {n_peers} peer{plural}: {joined}"
|
||
|
||
|
||
# Substrings in a failure that describe slskd's state rather than the file's.
|
||
# "File not shared" is about one file and the next candidate is unaffected;
|
||
# a timeout means the client could not talk to anyone, so the next candidate
|
||
# would fail for the same reason and only costs us the candidate.
|
||
SYSTEMIC = ("timed out", "timeout", "connection", "refused", "unreachable")
|
||
|
||
|
||
def is_systemic(reason: str) -> bool:
|
||
r = reason.lower()
|
||
return any(w in r for w in SYSTEMIC)
|
||
|
||
|
||
async def try_next_candidate(job_id: str, artist: str, title: str,
|
||
album: str | None = None) -> bool:
|
||
"""Enqueue the next-best candidate. False once the ladder is exhausted.
|
||
|
||
This is the difference between a request that survives one bad peer and one
|
||
that does not. "Transfer rejected: File not shared." is a *per-file*
|
||
condition -- the peer's share index is stale -- so candidate two almost
|
||
always works, and slskd's 500 on an unreachable peer is the same shape.
|
||
"""
|
||
with db() as c:
|
||
row = c.execute(
|
||
"SELECT candidates, tried, attempts FROM jobs WHERE id=?",
|
||
(job_id,)).fetchone()
|
||
if not row:
|
||
return False
|
||
cands = json.loads(row["candidates"] or "[]")
|
||
tried = json.loads(row["tried"] or "[]")
|
||
attempts = row["attempts"] or 0
|
||
|
||
systemic_run = 0
|
||
first = True
|
||
while cands and attempts < MAX_ATTEMPTS:
|
||
# Space the attempts out. The first one is immediate; the rest wait,
|
||
# because back-to-back enqueues are what saturated slskd.
|
||
if not first:
|
||
await asyncio.sleep(ATTEMPT_BACKOFF)
|
||
first = False
|
||
|
||
cand = cands.pop(0)
|
||
attempts += 1
|
||
try:
|
||
async with slskd_client() as sc:
|
||
er = await sc.post(
|
||
f"/api/v0/transfers/downloads/{cand['username']}",
|
||
json=[{"filename": cand["filename"],
|
||
"size": cand["size"]}])
|
||
er.raise_for_status()
|
||
except Exception as e:
|
||
reason = f"could not reach peer: {errstr(e)}"
|
||
tried.append({"username": cand["username"],
|
||
"filename": cand["filename"],
|
||
"reason": reason})
|
||
if is_systemic(reason):
|
||
systemic_run += 1
|
||
if systemic_run >= SYSTEMIC_LIMIT:
|
||
# Stop, and leave the untried candidates on the row so a
|
||
# retry resumes the ladder instead of re-running a search
|
||
# that already told us the right answer.
|
||
with db() as c:
|
||
c.execute("UPDATE jobs SET candidates=?, tried=?,"
|
||
" attempts=? WHERE id=?",
|
||
(json.dumps(cands), json.dumps(tried),
|
||
attempts, job_id))
|
||
set_state(job_id, "failed",
|
||
f"slskd could not reach any peer after "
|
||
f"{attempts} attempts -- {len(cands)} candidate"
|
||
f"{'' if len(cands) == 1 else 's'} left to try, "
|
||
f"so a retry resumes where this stopped")
|
||
return False
|
||
else:
|
||
systemic_run = 0
|
||
continue
|
||
|
||
systemic_run = 0
|
||
|
||
conf, why = confidence(cand["filename"], artist, title, album)
|
||
with db() as c:
|
||
c.execute(
|
||
"UPDATE jobs SET candidates=?, tried=?, attempts=? WHERE id=?",
|
||
(json.dumps(cands), json.dumps(tried), attempts, job_id))
|
||
extra = f", attempt {attempts}" if attempts > 1 else ""
|
||
set_state(job_id, "downloading",
|
||
f"from {cand['username']} (score {cand['score']:.0f}{extra})",
|
||
username=cand["username"], remote_file=cand["filename"],
|
||
size=cand["size"], confidence=conf, match_why="; ".join(why),
|
||
started_at=datetime.now(timezone.utc).isoformat())
|
||
return True
|
||
|
||
with db() as c:
|
||
c.execute("UPDATE jobs SET candidates=?, tried=?, attempts=? WHERE id=?",
|
||
(json.dumps(cands), json.dumps(tried), attempts, job_id))
|
||
set_state(job_id, "failed", exhausted_detail(tried, attempts))
|
||
return False
|
||
|
||
|
||
async def pick_and_enqueue(job_id: str, artist: str, title: str,
|
||
album: str | None = None):
|
||
"""Search Soulseek for one track and queue the best candidate.
|
||
|
||
Builds a ranked candidate *list* rather than a single winner, so a peer
|
||
that rejects, times out or cannot be reached costs one attempt instead of
|
||
the entire request.
|
||
"""
|
||
queries = search_queries(artist, title)
|
||
pool: dict[tuple[str, str], tuple[float, dict, dict]] = {}
|
||
seen = 0
|
||
stats: "collections.Counter" = collections.Counter()
|
||
|
||
async with slskd_client() as sc:
|
||
for i, q in enumerate(queries):
|
||
try:
|
||
results = await run_search(sc, q)
|
||
except SearchUnavailable as e:
|
||
set_state(job_id, "failed",
|
||
f"slskd would not accept a search: {e}")
|
||
return
|
||
|
||
for peer in results.get("responses", []):
|
||
for f in peer.get("files", []):
|
||
seen += 1
|
||
if f.get("isLocked"):
|
||
stats["locked"] += 1
|
||
continue
|
||
if ext_of(f["filename"]) not in FORMATS:
|
||
stats["format"] += 1
|
||
continue
|
||
sc_ = score(f, peer, artist, title)
|
||
if sc_ is None:
|
||
stats["weak"] += 1
|
||
continue
|
||
key = (peer["username"], f["filename"])
|
||
if key not in pool or pool[key][0] < sc_:
|
||
pool[key] = (sc_, peer, f)
|
||
|
||
log.info("job %s q%d/%d %r -> %d responses, %d files, %d candidates",
|
||
job_id[:8], i + 1, len(queries), q,
|
||
len(results.get("responses", [])),
|
||
results.get("fileCount", 0), len(pool))
|
||
# Escalate only on underdelivery: the common case stays one search,
|
||
# which is what keeps a 50-track paste civil to slskd.
|
||
if len(pool) >= MIN_CANDIDATES:
|
||
break
|
||
|
||
if not pool:
|
||
set_state(job_id, "failed", no_candidate_detail(seen, stats, queries))
|
||
return
|
||
|
||
ranked = sorted(pool.values(), key=lambda r: -r[0])
|
||
cands = spread_by_peer(ranked)[:CANDIDATES]
|
||
with db() as c:
|
||
c.execute("UPDATE jobs SET candidates=?, tried=?, attempts=0 WHERE id=?",
|
||
(json.dumps(cands), "[]", job_id))
|
||
await try_next_candidate(job_id, artist, title, album)
|
||
|
||
|
||
# ─────────────────────────── email ───────────────────────────
|
||
#
|
||
# Optional and off unless SMTP_HOST is set. A homelab tool that cannot send
|
||
# mail should still import music, so every failure here is logged and
|
||
# swallowed — a notification is never worth failing an import over.
|
||
|
||
SMTP_HOST = os.getenv("MUSE_SMTP_HOST", "").strip()
|
||
SMTP_PORT = int(os.getenv("MUSE_SMTP_PORT", "587"))
|
||
SMTP_USER = os.getenv("MUSE_SMTP_USER", "").strip()
|
||
SMTP_PASS = os.getenv("MUSE_SMTP_PASS", "")
|
||
SMTP_FROM = os.getenv("MUSE_SMTP_FROM", "").strip() or SMTP_USER
|
||
SMTP_TLS = os.getenv("MUSE_SMTP_TLS", "starttls").strip().lower()
|
||
MAIL_ON = bool(SMTP_HOST and SMTP_FROM)
|
||
|
||
|
||
def _send_sync(to: str, subject: str, body: str):
|
||
import smtplib
|
||
from email.message import EmailMessage
|
||
|
||
msg = EmailMessage()
|
||
msg["From"], msg["To"], msg["Subject"] = SMTP_FROM, to, subject
|
||
msg.set_content(body)
|
||
if SMTP_TLS == "ssl":
|
||
s = smtplib.SMTP_SSL(SMTP_HOST, SMTP_PORT, timeout=20)
|
||
else:
|
||
s = smtplib.SMTP(SMTP_HOST, SMTP_PORT, timeout=20)
|
||
with s:
|
||
if SMTP_TLS == "starttls":
|
||
s.starttls()
|
||
if SMTP_USER:
|
||
s.login(SMTP_USER, SMTP_PASS)
|
||
s.send_message(msg)
|
||
|
||
|
||
async def send_mail(to: str, subject: str, body: str):
|
||
"""Fire-and-forget. smtplib is blocking, so it runs off the event loop —
|
||
otherwise a slow relay would stall the import worker for every job."""
|
||
if not (MAIL_ON and to):
|
||
return
|
||
try:
|
||
await asyncio.to_thread(_send_sync, to, subject, body)
|
||
log.info("mail: sent %r to %s", subject, to)
|
||
except Exception as e:
|
||
log.warning("mail: sending to %s failed: %s", to, errstr(e))
|
||
|
||
|
||
def recipient(username: str | None) -> str | None:
|
||
"""Where to write to, and whether they want to hear from us at all."""
|
||
if not username:
|
||
return None
|
||
with db() as c:
|
||
r = c.execute("SELECT email, notify, provider FROM users WHERE username=?",
|
||
(username,)).fetchone()
|
||
if not r or not r["notify"]:
|
||
return None
|
||
# A Google identity IS an email address, so there is nothing to configure
|
||
# for those accounts unless they want mail somewhere else.
|
||
if r["email"]:
|
||
return r["email"]
|
||
return username if r["provider"] == "google" and "@" in username else None
|
||
|
||
|
||
async def notify_invited(row: sqlite3.Row, by: str, to: str):
|
||
"""Tell someone they've been invited.
|
||
|
||
There is no token in this link and none is needed. The row already names
|
||
the exact identity that may sign in, and Google will not hand us an
|
||
unverified address — so possession of the email is not what authorises
|
||
anyone, and a leaked link grants nothing. It is a courtesy, not a
|
||
credential, which is also why it can be resent freely.
|
||
"""
|
||
label = PROVIDERS.get(row["provider"], {}).get("label", row["provider"])
|
||
where = BASE_URL or "your muse instance"
|
||
await send_mail(
|
||
to, f"{by} invited you to muse",
|
||
f"You've been given access to muse — request a song, and it turns up\n"
|
||
f"in the music library, correctly tagged.\n\n"
|
||
f" {where}\n\n"
|
||
f"Sign in with {label}, using the account for {row['username']}.\n"
|
||
f"That address is what authorises you, so it has to be that one.\n\n"
|
||
f"Playback lives in Navidrome, not here — muse only decides what gets\n"
|
||
f"added to the library.\n")
|
||
|
||
|
||
async def notify_imported(job, dest):
|
||
to = recipient(job["requested_by"])
|
||
if not to:
|
||
return
|
||
album = job["album"] or "Singles"
|
||
if job["year"]:
|
||
album += f" ({job['year']})"
|
||
await send_mail(
|
||
to, f"muse: {job['title']} is in your library",
|
||
f"{job['artist']} — {job['title']}\n{album}\n\n"
|
||
f"Filed as: {dest}\n\nIt should appear in Navidrome on the next scan.\n")
|
||
|
||
|
||
async def notify_review(job):
|
||
to = recipient(job["requested_by"])
|
||
if not to:
|
||
return
|
||
await send_mail(
|
||
to, f"muse: {job['title']} needs a look",
|
||
f"{job['artist']} — {job['title']}\n\n"
|
||
f"The best match found was only {(job['confidence'] or 0):.0%} confident, "
|
||
f"so it has not been added to your library yet.\n\n"
|
||
f"Why: {job['match_why'] or 'unknown'}\n"
|
||
f"File: {job['remote_file']}\n\n"
|
||
f"Approve or reject it under Review in muse.\n")
|
||
|
||
|
||
# ─────────────────────────── import ───────────────────────────
|
||
|
||
def tag_and_file(job: sqlite3.Row, src: Path) -> Path:
|
||
"""Write known-good tags, then hardlink into the library."""
|
||
from mutagen import File as MFile
|
||
from mutagen.easyid3 import EasyID3
|
||
from mutagen.flac import FLAC
|
||
from mutagen.mp3 import MP3
|
||
|
||
try:
|
||
if src.suffix.lower() == ".flac":
|
||
audio = FLAC(src)
|
||
else:
|
||
try:
|
||
audio = EasyID3(src)
|
||
except Exception:
|
||
m = MP3(src)
|
||
m.add_tags()
|
||
m.save()
|
||
audio = EasyID3(src)
|
||
|
||
audio["title"] = job["title"]
|
||
audio["artist"] = job["artist"]
|
||
audio["albumartist"] = job["artist"]
|
||
if job["album"]:
|
||
audio["album"] = job["album"]
|
||
if job["year"]:
|
||
audio["date"] = str(job["year"])
|
||
if job["track"]:
|
||
audio["tracknumber"] = str(job["track"])
|
||
# Deliberately NOT writing the Deezer id into musicbrainz_trackid --
|
||
# that field means a specific thing and a wrong value there would
|
||
# mislead Picard, beets, or anything else that later reads this file.
|
||
audio.save()
|
||
except Exception as e:
|
||
# A tagging failure must not strand the file — Navidrome can still
|
||
# read whatever tags it shipped with.
|
||
log.warning("tagging failed for %s: %s", src.name, e)
|
||
|
||
album = f"{safe(job['album'] or 'Singles')}"
|
||
year = safe_year(job["year"])
|
||
if year:
|
||
album += f" ({year})"
|
||
dest_dir = LIBRARY_DIR / safe(job["artist"]) / album
|
||
|
||
num = f"{int(job['track']):02d} - " if job["track"] else ""
|
||
dest = dest_dir / f"{num}{safe(job['title'])}{src.suffix.lower()}"
|
||
|
||
# Belt and braces. Every component above is either safe()'d or matched
|
||
# against a whitelist, but that is a property of five call sites that must
|
||
# each stay correct; this is one check that cannot be forgotten. It exists
|
||
# because `year` was for a while appended AFTER safe() ran, so a request
|
||
# with year="../../../../tmp/x" wrote outside the library.
|
||
root = LIBRARY_DIR.resolve()
|
||
if not Path(os.path.normpath(dest)).resolve().is_relative_to(root):
|
||
raise ValueError(f"refusing to write outside the library: {dest}")
|
||
|
||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
if dest.exists():
|
||
return dest
|
||
try:
|
||
os.link(src, dest) # same filesystem -> free, and keeps sharing
|
||
except OSError:
|
||
import shutil
|
||
shutil.copy2(src, dest) # cross-device fallback
|
||
return dest
|
||
|
||
|
||
def downloaded_index() -> dict[str, Path]:
|
||
"""Every completed file, by basename.
|
||
|
||
slskd nests a download under the remote's parent directory name, so the
|
||
basename is the only reliable key. Built once per poll cycle and shared by
|
||
every job in it: the previous version walked the whole tree separately for
|
||
each in-flight job, every ten seconds, which on a large downloads
|
||
directory is a lot of stat() calls to answer the same question.
|
||
"""
|
||
out: dict[str, Path] = {}
|
||
try:
|
||
for p in DOWNLOAD_DIR.rglob("*"):
|
||
if p.is_file():
|
||
out.setdefault(p.name, p)
|
||
except OSError as e:
|
||
log.warning("cannot walk %s: %s", DOWNLOAD_DIR, errstr(e))
|
||
return out
|
||
|
||
|
||
# ─────────────────────────── search queue ───────────────────────────
|
||
|
||
work_q: "asyncio.Queue[str]" = asyncio.Queue()
|
||
|
||
|
||
async def search_worker(n: int):
|
||
"""Drain the search queue, one job at a time, WORKERS of these running."""
|
||
while True:
|
||
job_id = await work_q.get()
|
||
try:
|
||
with db() as c:
|
||
job = c.execute("SELECT * FROM jobs WHERE id=?", (job_id,)).fetchone()
|
||
if job and job["state"] in ("queued", "searching"):
|
||
set_state(job_id, "searching", "")
|
||
# A ladder left over from a run that stopped on a slskd stall
|
||
# is still the right answer -- the search that produced it has
|
||
# not become wrong. Resume it rather than paying for another.
|
||
keys = job.keys()
|
||
left = json.loads(
|
||
(job["candidates"] if "candidates" in keys else None) or "[]")
|
||
if left:
|
||
log.info("job %s resuming ladder, %d candidate(s) left",
|
||
job_id[:8], len(left))
|
||
await try_next_candidate(job_id, job["artist"],
|
||
job["title"], job["album"])
|
||
else:
|
||
await pick_and_enqueue(job_id, job["artist"], job["title"],
|
||
job["album"])
|
||
except Exception as e:
|
||
log.warning("worker %d: %s", n, e)
|
||
try:
|
||
set_state(job_id, "failed", str(e)[:200])
|
||
except Exception:
|
||
pass
|
||
finally:
|
||
work_q.task_done()
|
||
|
||
|
||
def transfer_index(payload: list) -> dict[str, dict]:
|
||
"""Flatten slskd's user -> directories -> files nesting.
|
||
|
||
Keeps the whole file object, not just its state. `exception` is where the
|
||
literal "Transfer rejected: File not shared." lives, and discarding it was
|
||
why a failed job could only ever say "transfer Completed, Rejected" with no
|
||
reason a person could act on.
|
||
"""
|
||
out: dict[str, dict] = {}
|
||
for user in payload:
|
||
for d in user.get("directories", []):
|
||
for f in d.get("files", []):
|
||
out[f.get("filename", "")] = f
|
||
return out
|
||
|
||
|
||
def stalled_for(job: sqlite3.Row) -> float:
|
||
"""Minutes since this transfer last had anything to show for itself.
|
||
|
||
Falls back to created_at, which is what makes this work on rows written
|
||
before started_at existed. Those are `downloading` with no clock at all,
|
||
so returning zero for them would exempt precisely the jobs that have been
|
||
stuck the longest -- the ones this check exists to catch.
|
||
"""
|
||
keys = job.keys()
|
||
started = ((job["started_at"] if "started_at" in keys else None)
|
||
or (job["created_at"] if "created_at" in keys else None))
|
||
if not started:
|
||
return 0.0
|
||
try:
|
||
t = datetime.fromisoformat(started)
|
||
except ValueError:
|
||
return 0.0
|
||
if t.tzinfo is None:
|
||
t = t.replace(tzinfo=timezone.utc)
|
||
return (datetime.now(timezone.utc) - t).total_seconds() / 60.0
|
||
|
||
|
||
async def abandon(job: sqlite3.Row, reason: str) -> None:
|
||
"""Give up on the current peer and fall through to the next candidate."""
|
||
with db() as c:
|
||
row = c.execute("SELECT tried FROM jobs WHERE id=?",
|
||
(job["id"],)).fetchone()
|
||
tried = json.loads((row["tried"] if row else None) or "[]")
|
||
tried.append({"username": job["username"], "filename": job["remote_file"],
|
||
"reason": reason})
|
||
with db() as c:
|
||
c.execute("UPDATE jobs SET tried=? WHERE id=?",
|
||
(json.dumps(tried), job["id"]))
|
||
log.info("job %s giving up on %s: %s", job["id"][:8], job["username"], reason)
|
||
await try_next_candidate(job["id"], job["artist"], job["title"], job["album"])
|
||
|
||
|
||
async def cancel_transfer(username: str, filename: str) -> None:
|
||
"""Ask slskd to drop a transfer we have stopped waiting for."""
|
||
try:
|
||
async with slskd_client(timeout=PROBE_TIMEOUT) as sc:
|
||
await sc.delete(f"/api/v0/transfers/downloads/{username}",
|
||
params={"filename": filename})
|
||
except Exception as e:
|
||
log.debug("cancel %s/%s: %s", username, filename, errstr(e))
|
||
|
||
|
||
async def import_worker():
|
||
"""Poll slskd for in-flight downloads and import the finished ones."""
|
||
missing: collections.Counter = collections.Counter()
|
||
while True:
|
||
await asyncio.sleep(10)
|
||
try:
|
||
with db() as c:
|
||
jobs = c.execute(
|
||
"SELECT * FROM jobs WHERE state='downloading'").fetchall()
|
||
if not jobs:
|
||
missing.clear()
|
||
continue
|
||
|
||
async with slskd_client() as sc:
|
||
r = await sc.get("/api/v0/transfers/downloads")
|
||
r.raise_for_status()
|
||
index = transfer_index(r.json())
|
||
|
||
# One directory walk per cycle, shared by every job. This used to be
|
||
# a full recursive rglob per stuck job, every ten seconds, forever.
|
||
done_files: dict[str, Path] | None = None
|
||
|
||
for job in jobs:
|
||
t = index.get(job["remote_file"])
|
||
st = str((t or {}).get("state", ""))
|
||
|
||
if "Completed, Succeeded" in st:
|
||
if done_files is None:
|
||
done_files = downloaded_index()
|
||
src = done_files.get(
|
||
job["remote_file"].replace("\\", "/").rsplit("/", 1)[-1])
|
||
if not src:
|
||
# Normally just still being moved out of incomplete, but
|
||
# unbounded before: a misconfigured DOWNLOAD_DIR meant
|
||
# polling until the heat death of the process.
|
||
if stalled_for(job) > STALL_MINUTES:
|
||
set_state(job["id"], "failed",
|
||
"downloaded but the file never appeared in "
|
||
f"{DOWNLOAD_DIR} -- check MUSE_DOWNLOAD_DIR")
|
||
continue
|
||
try:
|
||
actual = src.stat().st_size
|
||
except OSError:
|
||
actual = job["size"]
|
||
|
||
conf = job["confidence"]
|
||
if conf is not None and conf < AUTO_CONFIDENCE:
|
||
band = ("almost certainly wrong"
|
||
if conf < REVIEW_CONFIDENCE else "worth a look")
|
||
set_state(job["id"], "review",
|
||
f"{conf:.0%} confident, {band} -- "
|
||
f"{job['match_why'] or ''}", size=actual)
|
||
await notify_review(job)
|
||
continue
|
||
|
||
dest = tag_and_file(job, src)
|
||
set_state(job["id"], "imported", str(dest),
|
||
imported_path=str(dest), size=actual)
|
||
await notify_imported(job, dest)
|
||
|
||
elif "Completed" in st and "Succeeded" not in st:
|
||
# Every terminal non-success used to end the request here.
|
||
# "File not shared." is per-file and per-peer, so the only
|
||
# sane response is to try the next candidate.
|
||
why = (t or {}).get("exception") or st
|
||
await abandon(job, str(why)[:120])
|
||
|
||
elif t is None:
|
||
# Absent from slskd entirely: it restarted, or the transfer
|
||
# was cleared. Tolerate a couple of polls in case it is a
|
||
# race, then stop waiting for something nobody is doing.
|
||
missing[job["id"]] += 1
|
||
if missing[job["id"]] >= 3:
|
||
del missing[job["id"]]
|
||
await abandon(job, "slskd forgot the transfer")
|
||
|
||
else:
|
||
missing.pop(job["id"], None)
|
||
# Queued behind a wall, or connected and sending nothing.
|
||
# There was no clock on this at all, which is how a job sat
|
||
# in `downloading` for days behind a cheerful pulsing dot.
|
||
if not (t or {}).get("bytesTransferred") \
|
||
and stalled_for(job) > STALL_MINUTES:
|
||
place = (t or {}).get("placeInQueue")
|
||
where = f" at #{place} in the queue" if place else ""
|
||
await cancel_transfer(job["username"], job["remote_file"])
|
||
await abandon(job, f"no bytes in {STALL_MINUTES:.0f}m{where}")
|
||
except Exception as e:
|
||
log.warning("import worker: %s", errstr(e))
|
||
|
||
|
||
# ─────────────────────────── api ───────────────────────────
|
||
|
||
@asynccontextmanager
|
||
async def lifespan(_: FastAPI):
|
||
# init_db first: the preflight consults the users table to decide whether
|
||
# the bootstrap allowlist is still mandatory.
|
||
init_db()
|
||
auth_preflight() # raises and stops the container if misconfigured
|
||
|
||
# The queue lives in memory, so a restart mid-import would otherwise
|
||
# strand every unstarted job in 'queued' forever with nothing to pick it
|
||
# up. Anything not yet handed to slskd goes back on the queue.
|
||
with db() as c:
|
||
orphans = [r["id"] for r in c.execute(
|
||
"SELECT id FROM jobs WHERE state IN ('queued','searching')").fetchall()]
|
||
for jid in orphans:
|
||
work_q.put_nowait(jid)
|
||
if orphans:
|
||
log.info("requeued %d job(s) left over from the last run", len(orphans))
|
||
|
||
tasks = [asyncio.create_task(import_worker())]
|
||
tasks += [asyncio.create_task(search_worker(i)) for i in range(WORKERS)]
|
||
log.info("muse up — slskd=%s library=%s formats=%s workers=%d",
|
||
SLSKD_URL, LIBRARY_DIR, ",".join(FORMATS), WORKERS)
|
||
yield
|
||
for t in tasks:
|
||
t.cancel()
|
||
|
||
|
||
# ─────────────────────── static front end / CSP ───────────────────────
|
||
#
|
||
# The UI is a Next.js static export: `next build` writes plain files into
|
||
# web/out and this process serves them, so the deployment stays one container,
|
||
# one process, one healthcheck, with no Node runtime in the image.
|
||
#
|
||
# That export contains inline <script> blocks (React's hydration payload),
|
||
# which `script-src 'self'` blocks outright. The usual fix is a per-request
|
||
# nonce, but a STATIC export has a better option: the scripts never change
|
||
# between requests, so their SHA-256 can go straight into the policy. Hashing
|
||
# them at startup means the CSP needs no per-request work, no HTML rewriting,
|
||
# and no maintenance when the bundle changes -- rebuild and the hashes follow.
|
||
WEB = Path(__file__).parent / "web" / "out"
|
||
|
||
_INLINE_SCRIPT = re.compile(rb"<script(?![^>]*\bsrc=)[^>]*>(.*?)</script>", re.S)
|
||
|
||
|
||
def script_hashes() -> list[str]:
|
||
out: set[str] = set()
|
||
for page in ("index.html", "404.html"):
|
||
f = WEB / page
|
||
if not f.exists():
|
||
continue
|
||
for body in _INLINE_SCRIPT.findall(f.read_bytes()):
|
||
digest = base64.b64encode(hashlib.sha256(body).digest()).decode()
|
||
out.add(f"'sha256-{digest}'")
|
||
return sorted(out)
|
||
|
||
|
||
SCRIPT_HASHES = " ".join(script_hashes())
|
||
|
||
# Framer Motion writes its pre-hydration `initial` state into style="" on the
|
||
# server-rendered markup, so those attributes have to be allowed or every
|
||
# animated element flashes in at full opacity before hydration corrects it.
|
||
#
|
||
# style-src-attr is the narrow lever for that: it permits style ATTRIBUTES
|
||
# while style-src 'self' still blocks injected <style> blocks and remote
|
||
# stylesheets. It is not a free pass either -- CSS-based exfiltration goes
|
||
# through url(), and img-src below already limits where those can point.
|
||
CSP = (
|
||
"default-src 'self'; "
|
||
f"script-src 'self' {SCRIPT_HASHES}; "
|
||
"style-src 'self'; style-src-attr 'unsafe-inline'; "
|
||
"img-src 'self' data: https://*.dzcdn.net https://*.githubusercontent.com; "
|
||
"media-src 'self' https://*.dzcdn.net; "
|
||
"connect-src 'self'; font-src 'self'; object-src 'none'; "
|
||
"frame-ancestors 'none'; base-uri 'none'; form-action 'self'"
|
||
)
|
||
|
||
|
||
app = FastAPI(
|
||
title="muse",
|
||
lifespan=lifespan,
|
||
# No interactive docs on an internet-facing app: they enumerate every
|
||
# endpoint and schema for anyone who reaches the login page.
|
||
docs_url=None, redoc_url=None, openapi_url=None,
|
||
)
|
||
|
||
|
||
@app.middleware("http")
|
||
async def guard(request: Request, call_next):
|
||
"""Require a session for everything except PUBLIC_PATHS, and stamp
|
||
security headers on every response."""
|
||
# /_next/* is NOT public. Unlike the old hand-written page there is no
|
||
# local login screen to style — an unauthenticated visitor is redirected
|
||
# straight to GitHub and never receives our HTML — so the bundle can sit
|
||
# behind the session too, and not hand out a map of the app to anyone who
|
||
# finds the URL. It loads fine after the callback, cookie in hand.
|
||
path = request.url.path
|
||
if AUTH_MODE == "off" or path in PUBLIC_PATHS:
|
||
resp = await call_next(request)
|
||
elif session_user(request):
|
||
resp = await call_next(request)
|
||
elif path.startswith("/api/"):
|
||
# An API caller gets a status code, not an HTML redirect it can't use.
|
||
resp = JSONResponse({"error": "unauthenticated"}, status_code=401)
|
||
else:
|
||
resp = RedirectResponse("/auth/login", status_code=302)
|
||
|
||
# See CSP above. img-src and media-src name Deezer's CDN explicitly rather
|
||
# than allowing https: wholesale — under a blanket https: any injected
|
||
# markup could beacon to an arbitrary host by requesting an image from it,
|
||
# which is a working exfiltration channel even with connect-src locked to
|
||
# 'self'. Artwork comes from dzcdn.net and avatars from githubusercontent.
|
||
resp.headers["Content-Security-Policy"] = CSP
|
||
resp.headers["X-Content-Type-Options"] = "nosniff"
|
||
resp.headers["X-Frame-Options"] = "DENY"
|
||
resp.headers["Referrer-Policy"] = "no-referrer"
|
||
resp.headers["Permissions-Policy"] = "geolocation=(), microphone=(), camera=()"
|
||
if COOKIE_SECURE:
|
||
resp.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
|
||
return resp
|
||
|
||
|
||
def _set_cookie(resp, name, value, max_age):
|
||
resp.set_cookie(name, value, max_age=max_age, httponly=True,
|
||
secure=COOKIE_SECURE, samesite="lax", path="/")
|
||
|
||
|
||
@app.get("/healthz")
|
||
async def healthz():
|
||
"""Unauthenticated on purpose, for the container healthcheck. Returns
|
||
nothing but a literal ok — no versions, no config, no counts."""
|
||
return {"ok": True}
|
||
|
||
|
||
# The sign-in page is the first screen anyone sees, and for a while it was the
|
||
# only one built out of hardcoded hex with no relation to the rest of the app.
|
||
# It cannot use the Next bundle -- /_next/* sits behind the session on purpose,
|
||
# so an unauthenticated visitor never receives it -- so it gets one small
|
||
# stylesheet of its own, served from a public path. `style-src 'self'` in the
|
||
# CSP permits exactly that; an inline <style> block would be refused.
|
||
LOGIN_CSS = """
|
||
:root{
|
||
--bg:#0a0b0f; --surface:#121319; --surface-2:#181a22;
|
||
--fg:#f4f6fb; --dim:#a0a8bb; --mute:#6d7488;
|
||
--edge:rgb(255 255 255/.09); --edge-hi:rgb(255 255 255/.16);
|
||
--accent:#0a84ff; --accent-ink:#fff;
|
||
--e2:0 2px 6px rgb(0 0 0/.34), 0 12px 32px rgb(0 0 0/.26);
|
||
color-scheme:light dark;
|
||
}
|
||
@media (prefers-color-scheme:light){
|
||
:root{
|
||
--bg:#f6f7f9; --surface:#fff; --surface-2:#f2f3f6;
|
||
--fg:#0d1117; --dim:#596273; --mute:#858da0;
|
||
--edge:rgb(10 14 25/.1); --edge-hi:rgb(10 14 25/.18);
|
||
--accent:#0071e3;
|
||
--e2:0 2px 6px rgb(20 30 60/.07), 0 12px 32px rgb(20 30 60/.08);
|
||
}
|
||
}
|
||
*{box-sizing:border-box}
|
||
body{
|
||
margin:0; min-height:100dvh; display:grid; place-items:center; padding:24px;
|
||
background:var(--bg); color:var(--fg);
|
||
font:15px/1.55 -apple-system,BlinkMacSystemFont,"SF Pro Text",ui-sans-serif,
|
||
system-ui,"Segoe UI",Roboto,sans-serif;
|
||
letter-spacing:-.008em; -webkit-font-smoothing:antialiased;
|
||
}
|
||
.card{
|
||
width:100%; max-width:352px; padding:32px 28px; text-align:center;
|
||
background:var(--surface); border:1px solid var(--edge);
|
||
border-radius:18px; box-shadow:var(--e2);
|
||
}
|
||
.mark{
|
||
width:44px; height:44px; margin:0 auto 18px; display:grid; place-items:center;
|
||
background:var(--accent); border-radius:12px; color:var(--accent-ink);
|
||
}
|
||
h1{
|
||
margin:0; font-size:23px; font-weight:700; letter-spacing:-.024em; line-height:1.18;
|
||
font-family:-apple-system,BlinkMacSystemFont,"SF Pro Display",ui-sans-serif,system-ui,sans-serif;
|
||
}
|
||
p.sub{margin:6px 0 24px; font-size:13px; color:var(--dim)}
|
||
a.btn{
|
||
display:flex; align-items:center; justify-content:center; gap:9px;
|
||
margin:8px 0; padding:11px 18px; border-radius:10px; text-decoration:none;
|
||
background:var(--surface-2); border:1px solid var(--edge-hi);
|
||
color:var(--fg); font-size:13px; font-weight:600;
|
||
transition:border-color .15s, background .15s, transform .1s;
|
||
}
|
||
a.btn:hover{border-color:var(--accent); background:var(--bg)}
|
||
a.btn:active{transform:scale(.985)}
|
||
a.btn:focus-visible{outline:2px solid var(--accent); outline-offset:2px}
|
||
p.foot{margin:22px 0 0; font-size:11px; color:var(--mute)}
|
||
"""
|
||
|
||
CHOOSER = """<!doctype html><html lang=en><head><meta charset=utf-8>
|
||
<meta name=viewport content="width=device-width,initial-scale=1">
|
||
<meta name=theme-color media="(prefers-color-scheme: light)" content="#f6f7f9">
|
||
<meta name=theme-color media="(prefers-color-scheme: dark)" content="#0a0b0f">
|
||
<title>Sign in · muse</title>
|
||
<link rel=stylesheet href="/login.css"></head>
|
||
<body><main class=card>
|
||
<div class=mark><svg width=22 height=22 viewBox="0 0 24 24" fill=none
|
||
stroke=currentColor stroke-width=2 stroke-linecap=round stroke-linejoin=round
|
||
aria-hidden="true"><path d="M9 18V5l12-2v13"/><circle cx="6" cy="18" r="3"/>
|
||
<circle cx="18" cy="16" r="3"/></svg></div>
|
||
<h1>muse</h1>
|
||
<p class=sub>Request a song, get it in your library.</p>
|
||
{buttons}
|
||
<p class=foot>You’ll be sent to your provider to sign in.</p>
|
||
</main></body></html>"""
|
||
|
||
_BTN = ('<a class=btn href="/auth/login?provider={p}">Continue with {label}</a>')
|
||
|
||
|
||
@app.get("/login.css")
|
||
async def login_css():
|
||
"""The sign-in page's stylesheet. Public, because it is needed by someone
|
||
who by definition has no session yet."""
|
||
return Response(LOGIN_CSS, media_type="text/css",
|
||
headers={"Cache-Control": "public, max-age=3600"})
|
||
|
||
|
||
@app.get("/auth/login")
|
||
async def auth_login(provider: str = ""):
|
||
"""Start OAuth. With several providers enabled and none named, offer a
|
||
choice — a tiny self-contained page that uses only style attributes, which
|
||
the CSP already permits, so it needs no stylesheet and no extra hash."""
|
||
if AUTH_MODE == "off":
|
||
return RedirectResponse("/", status_code=302)
|
||
|
||
if not provider:
|
||
if len(ENABLED_PROVIDERS) == 1:
|
||
provider = ENABLED_PROVIDERS[0]
|
||
else:
|
||
buttons = "".join(
|
||
_BTN.format(p=p, label=PROVIDERS[p]["label"]) for p in ENABLED_PROVIDERS)
|
||
return HTMLResponse(CHOOSER.replace("{buttons}", buttons))
|
||
|
||
if provider not in ENABLED_PROVIDERS:
|
||
raise HTTPException(404, "unknown provider")
|
||
|
||
cfg = PROVIDERS[provider]
|
||
nonce = secrets.token_urlsafe(24)
|
||
params = {
|
||
"client_id": cfg["client_id"],
|
||
"redirect_uri": f"{BASE_URL}/auth/callback",
|
||
"scope": cfg["scope"],
|
||
"state": nonce,
|
||
"response_type": "code",
|
||
}
|
||
if provider == "github":
|
||
params["allow_signup"] = "false"
|
||
if provider == "google":
|
||
# Force the account chooser; otherwise Google silently reuses whichever
|
||
# account the browser last used, which on a shared machine signs you in
|
||
# as someone else with no visible prompt.
|
||
params["prompt"] = "select_account"
|
||
|
||
resp = RedirectResponse(f"{cfg['authorize']}?{urlencode(params)}", status_code=302)
|
||
# CSRF defence: the nonce is echoed back by the provider and must match
|
||
# this cookie, so a callback the user did not initiate is rejected. The
|
||
# provider name rides along in the COOKIE, never the query string, so the
|
||
# callback cannot be steered to a different provider by the redirect.
|
||
_set_cookie(resp, STATE_COOKIE, f"{provider}|{nonce}", 600)
|
||
return resp
|
||
|
||
|
||
async def fetch_identity(provider: str, token: str) -> tuple[str, str, str]:
|
||
"""(username, display, avatar) for a provider access token."""
|
||
async with httpx.AsyncClient(timeout=20) as c:
|
||
if provider == "github":
|
||
r = await c.get(GH_API, headers={"Authorization": f"Bearer {token}",
|
||
"Accept": "application/vnd.github+json"})
|
||
if r.status_code != 200:
|
||
raise HTTPException(401, "github user lookup failed")
|
||
p = r.json() or {}
|
||
return p.get("login") or "", p.get("name") or "", p.get("avatar_url") or ""
|
||
|
||
r = await c.get(GOOGLE_API, headers={"Authorization": f"Bearer {token}"})
|
||
if r.status_code != 200:
|
||
raise HTTPException(401, "google user lookup failed")
|
||
p = r.json() or {}
|
||
# Unverified Google emails must never become an identity: anyone can
|
||
# put someone else's address on an account they control, and the
|
||
# allowlist is keyed on exactly this string.
|
||
if not p.get("email") or not p.get("email_verified"):
|
||
raise HTTPException(403, "google account has no verified email")
|
||
return p["email"], p.get("name") or "", p.get("picture") or ""
|
||
|
||
|
||
@app.get("/auth/callback")
|
||
async def auth_callback(request: Request, code: str = "", state: str = ""):
|
||
if AUTH_MODE == "off":
|
||
return RedirectResponse("/", status_code=302)
|
||
|
||
cookie = request.cookies.get(STATE_COOKIE) or ""
|
||
provider, _, nonce = cookie.partition("|")
|
||
if (not code or not state or not nonce
|
||
or provider not in ENABLED_PROVIDERS
|
||
or not secrets.compare_digest(state, nonce)):
|
||
log.warning("auth: rejected callback with bad or missing state")
|
||
return JSONResponse({"error": "invalid state"}, status_code=400)
|
||
|
||
cfg = PROVIDERS[provider]
|
||
async with httpx.AsyncClient(timeout=20) as c:
|
||
t = await c.post(cfg["token"], headers={"Accept": "application/json"},
|
||
data={"client_id": cfg["client_id"],
|
||
"client_secret": cfg["client_secret"],
|
||
"code": code,
|
||
"grant_type": "authorization_code",
|
||
"redirect_uri": f"{BASE_URL}/auth/callback"})
|
||
token = (t.json() or {}).get("access_token") if t.status_code == 200 else None
|
||
if not token:
|
||
log.warning("auth: %s token exchange failed (%s)", provider, t.status_code)
|
||
return JSONResponse({"error": "token exchange failed"}, status_code=401)
|
||
|
||
username, display, avatar = await fetch_identity(provider, token)
|
||
if not username:
|
||
return JSONResponse({"error": "no username from provider"}, status_code=401)
|
||
|
||
with db() as c:
|
||
existing = c.execute("SELECT * FROM users WHERE username=?",
|
||
(username,)).fetchone()
|
||
if not may_login(username, existing, provider):
|
||
# Log the attempt: on an internet-facing box this is the signal that
|
||
# someone found the login page.
|
||
if not existing:
|
||
why = "not invited"
|
||
elif existing["disabled"]:
|
||
why = "disabled"
|
||
else:
|
||
why = f"invited for {existing['provider']}, tried {provider}"
|
||
log.warning("auth: DENIED %s user %r (%s)", provider, username, why)
|
||
return JSONResponse({"error": "not authorised"}, status_code=403)
|
||
|
||
row = upsert_user(username, avatar, display, provider)
|
||
log.info("auth: signed in %s via %s (%s)", username, provider, row["role"])
|
||
resp = RedirectResponse("/", status_code=302)
|
||
_set_cookie(resp, SESSION_COOKIE, new_session(username), SESSION_DAYS * 86400)
|
||
resp.delete_cookie(STATE_COOKIE, path="/")
|
||
return resp
|
||
|
||
|
||
@app.get("/auth/logout")
|
||
async def auth_logout(request: Request):
|
||
tok = request.cookies.get(SESSION_COOKIE)
|
||
if tok:
|
||
with db() as c:
|
||
c.execute("DELETE FROM sessions WHERE token_hash=?", (_hash(tok),))
|
||
resp = RedirectResponse("/auth/login", status_code=302)
|
||
resp.delete_cookie(SESSION_COOKIE, path="/")
|
||
return resp
|
||
|
||
|
||
# ─────────────────────────── quotas ───────────────────────────
|
||
#
|
||
# Two limits, both per user, both 0 = unlimited:
|
||
# quota requests per CALENDAR month (resets on the 1st)
|
||
# storage_quota bytes of imported audio attributable to this user
|
||
#
|
||
# Calendar month rather than a rolling 30-day window on purpose: "you have 12
|
||
# left until the 1st" is something a person can reason about, where a rolling
|
||
# window silently drips capacity back at times nobody can predict.
|
||
|
||
def month_start() -> str:
|
||
return datetime.now(timezone.utc).strftime("%Y-%m-01 00:00:00")
|
||
|
||
|
||
def usage_for(username: str | None) -> dict:
|
||
"""Requests this month and bytes imported, for one user."""
|
||
if not username:
|
||
return {"requests": 0, "storage": 0}
|
||
with db() as c:
|
||
n = c.execute(
|
||
"SELECT COUNT(*) n FROM jobs WHERE requested_by=? AND created_at >= ?",
|
||
(username, month_start())).fetchone()["n"]
|
||
b = c.execute(
|
||
"SELECT COALESCE(SUM(size),0) b FROM jobs "
|
||
"WHERE requested_by=? AND state='imported'", (username,)).fetchone()["b"]
|
||
return {"requests": n, "storage": b}
|
||
|
||
|
||
def check_quota(request: Request, adding: int = 1):
|
||
"""Raise 429 if this request would exceed the caller's monthly allowance.
|
||
|
||
Admins are exempt — someone has to be able to fix the library when a quota
|
||
is what is in the way.
|
||
"""
|
||
u = current_user(request)
|
||
if not u or u["role"] == "admin":
|
||
return
|
||
used = usage_for(u["username"])
|
||
if u["quota"] and used["requests"] + adding > u["quota"]:
|
||
raise HTTPException(429, (
|
||
f"Monthly request quota reached ({used['requests']}/{u['quota']}). "
|
||
f"Resets on the 1st."))
|
||
if u["storage_quota"] and used["storage"] >= u["storage_quota"]:
|
||
raise HTTPException(429, (
|
||
f"Storage quota reached ({used['storage'] // 2**30} GB of "
|
||
f"{u['storage_quota'] // 2**30} GB)."))
|
||
|
||
|
||
@app.get("/api/me")
|
||
async def me(request: Request):
|
||
u = current_user(request)
|
||
if not u:
|
||
return {"user": None, "auth": AUTH_MODE,
|
||
"role": "admin" if AUTH_MODE == "off" else None,
|
||
"providers": ENABLED_PROVIDERS}
|
||
return {
|
||
"user": u["username"], "auth": AUTH_MODE, "role": u["role"],
|
||
"display": u["display"], "avatar": u["avatar"], "provider": u["provider"],
|
||
"quota": u["quota"], "storage_quota": u["storage_quota"],
|
||
"usage": usage_for(u["username"]),
|
||
"providers": ENABLED_PROVIDERS,
|
||
"email": u["email"], "notify": bool(u["notify"]),
|
||
"mail_configured": MAIL_ON,
|
||
}
|
||
|
||
|
||
class SettingsBody(BaseModel):
|
||
email: str | None = None
|
||
notify: bool | None = None
|
||
|
||
|
||
EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s.]+\.[^@\s]+$")
|
||
|
||
|
||
@app.patch("/api/me")
|
||
async def update_me(body: SettingsBody, request: Request):
|
||
"""Your own notification settings. Not an admin route on purpose — an
|
||
address you cannot change yourself is a support ticket."""
|
||
me = session_user(request)
|
||
if not me:
|
||
raise HTTPException(401, "unauthenticated")
|
||
sets, vals = [], []
|
||
if body.email is not None:
|
||
e = body.email.strip()
|
||
if e and not EMAIL_RE.match(e):
|
||
raise HTTPException(400, "that does not look like an email address")
|
||
sets.append("email=?"); vals.append(e or None)
|
||
if body.notify is not None:
|
||
sets.append("notify=?"); vals.append(1 if body.notify else 0)
|
||
if not sets:
|
||
raise HTTPException(400, "nothing to change")
|
||
with db() as c:
|
||
c.execute(f"UPDATE users SET {', '.join(sets)} WHERE username=?",
|
||
[*vals, me])
|
||
return {"ok": True, "mail_configured": MAIL_ON}
|
||
|
||
|
||
class RequestBody(BaseModel):
|
||
artist: str
|
||
title: str
|
||
album: str | None = None
|
||
year: str | None = None
|
||
track: int | None = None
|
||
deezer_id: int | None = None
|
||
cover: str | None = None
|
||
genre: str | None = None
|
||
length: int | None = None
|
||
|
||
|
||
# Album metadata (release year, genre, label) is not in the track search
|
||
# payload, so it costs one extra call per distinct album. Cached forever on
|
||
# purpose: a release date and a genre are immutable, so there is nothing to
|
||
# invalidate, and repeat searches for the same artist stop hitting Deezer at
|
||
# all. Trimmed only to stop it growing without bound on a long-lived process.
|
||
_album_cache: dict[int, dict] = {}
|
||
_album_lock = asyncio.Semaphore(6) # Deezer rate-limits ~50 req/5s per IP
|
||
|
||
|
||
async def album_meta(album_id) -> dict:
|
||
if not album_id:
|
||
return {}
|
||
key = int(album_id)
|
||
if key in _album_cache:
|
||
return _album_cache[key]
|
||
async with _album_lock:
|
||
if key in _album_cache: # filled while we waited for the slot
|
||
return _album_cache[key]
|
||
try:
|
||
async with httpx.AsyncClient(timeout=15) as c:
|
||
r = await c.get(f"{DEEZER}/album/{key}")
|
||
d = r.json() if r.status_code == 200 else {}
|
||
except Exception as e:
|
||
log.debug("album %s lookup failed: %s", key, e)
|
||
return {}
|
||
genres = [g.get("name") for g in (d.get("genres") or {}).get("data", []) if g.get("name")]
|
||
meta = {
|
||
"year": (d.get("release_date") or "")[:4] or None,
|
||
"genre": genres[0] if genres else None,
|
||
"genres": genres,
|
||
"label": d.get("label"),
|
||
"nb_tracks": d.get("nb_tracks"),
|
||
"record_type": d.get("record_type"),
|
||
}
|
||
if len(_album_cache) > 5000:
|
||
for k in list(_album_cache)[:1000]:
|
||
_album_cache.pop(k, None)
|
||
_album_cache[key] = meta
|
||
return meta
|
||
|
||
|
||
def track_row(t: dict) -> dict:
|
||
"""One Deezer track object -> the shape the UI and the job table use."""
|
||
artist = t.get("artist") or {}
|
||
album = t.get("album") or {}
|
||
return {
|
||
"deezer_id": t.get("id"),
|
||
"title": t.get("title") or "",
|
||
"artist": artist.get("name") or "",
|
||
"artist_id": artist.get("id"),
|
||
"artist_pic": artist.get("picture_medium"),
|
||
"album": album.get("title") or "",
|
||
"album_id": album.get("id"),
|
||
"cover": album.get("cover_medium"),
|
||
"cover_big": album.get("cover_big") or album.get("cover_medium"),
|
||
"length": (t.get("duration") or 0) * 1000,
|
||
# 30-second clip from Deezer's CDN. This is what lets you confirm you
|
||
# picked the right recording BEFORE committing a Soulseek search to it
|
||
# -- the single most useful thing in the payload for avoiding a wrong
|
||
# import, and it costs nothing extra.
|
||
"preview": t.get("preview") or None,
|
||
"explicit": bool(t.get("explicit_lyrics")),
|
||
"isrc": t.get("isrc"),
|
||
"rank": t.get("rank") or 0,
|
||
"year": None, "genre": None, "track": None,
|
||
}
|
||
|
||
|
||
async def add_album_meta(rows: list[dict]) -> list[dict]:
|
||
"""Fill year and genre on a result list, concurrently and de-duplicated."""
|
||
ids = {r["album_id"] for r in rows if r.get("album_id")}
|
||
if not ids:
|
||
return rows
|
||
metas = dict(zip(ids, await asyncio.gather(*(album_meta(i) for i in ids))))
|
||
for r in rows:
|
||
m = metas.get(r.get("album_id")) or {}
|
||
r["year"] = r["year"] or m.get("year")
|
||
r["genre"] = r["genre"] or m.get("genre")
|
||
return rows
|
||
|
||
|
||
async def deezer_search(q: str, limit: int = 25) -> list[dict]:
|
||
"""Ranked Deezer track search. Shared by /api/search and /api/import."""
|
||
if not q.strip():
|
||
return []
|
||
async with httpx.AsyncClient(timeout=20) as c:
|
||
r = await c.get(f"{DEEZER}/search", params={"q": q, "limit": limit * 2})
|
||
# Deezer's WAF answers 403 to anything that looks like an injection
|
||
# attempt -- "'; DROP TABLE" and friends. That is a statement about the
|
||
# query, not an upstream outage, so it belongs on the empty-results
|
||
# path; a 502 would tell the user the service is broken when it is not.
|
||
if r.status_code == 403:
|
||
log.info("deezer refused the query %r", q[:60])
|
||
return []
|
||
if r.status_code != 200:
|
||
raise HTTPException(502, f"deezer returned {r.status_code}")
|
||
data = r.json()
|
||
|
||
qn = norm(q)
|
||
scored = []
|
||
for t in data.get("data", []):
|
||
title = t.get("title") or ""
|
||
artist = (t.get("artist") or {}).get("name") or ""
|
||
|
||
s = float(t.get("rank") or 0) / 10_000 # Deezer popularity, 0-~100
|
||
|
||
# An artist match separates the real track from a covers compilation.
|
||
at = set(norm(artist).split())
|
||
if norm(artist) and norm(artist) in qn:
|
||
s += 300
|
||
|
||
# Title relevance. This has to dominate everything else: without it a
|
||
# completely different song by the right artist competes on equal
|
||
# footing, and "Oasis - Live Forever" resolved to "Cloudburst" because
|
||
# a variant penalty was enough to tip between two unrelated tracks.
|
||
# Whatever query words the artist name did not explain should show up
|
||
# in the title.
|
||
rest = set(qn.split()) - at
|
||
if rest:
|
||
s += 400 * len(rest & set(norm(title).split())) / len(rest)
|
||
for w in set(DERIVATIVE_RE.findall(norm(title))):
|
||
if not re.search(rf"\b{re.escape(w)}\b", qn):
|
||
s -= 150
|
||
# Catch-all for variants the keyword list will never cover in full
|
||
# ("Devonshire Mix", "Radio Edit", "2011 Version"): a plain title beats
|
||
# a parenthesised one. Deliberately small -- it should break ties
|
||
# between versions of the SAME song, never outweigh title relevance
|
||
# above and pick a different song.
|
||
if ("(" in title or "[" in title) and "(" not in q:
|
||
s -= 40
|
||
scored.append((s, track_row(t)))
|
||
|
||
seen, out = set(), []
|
||
for _, item in sorted(scored, key=lambda x: -x[0]):
|
||
key = (norm(item["artist"]), norm(item["title"]))
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
out.append(item)
|
||
return out[:limit]
|
||
|
||
|
||
@app.get("/api/search")
|
||
async def search(q: str, limit: int = Query(25, ge=1, le=50)):
|
||
"""Track search. Returns individual songs, not albums — the whole point.
|
||
|
||
Year and genre are grafted on afterwards because Deezer's track search
|
||
omits both. Enrichment happens only here, not in deezer_search itself, so
|
||
the bulk-import path stays one HTTP call per line.
|
||
"""
|
||
return {"results": await add_album_meta(await deezer_search(q, limit))}
|
||
|
||
|
||
@app.get("/api/suggest")
|
||
async def suggest(q: str):
|
||
"""Autocomplete. Deliberately does NOT enrich or re-rank: this fires on
|
||
every few keystrokes, so it has to be one upstream call and nothing else.
|
||
Deezer's own autocomplete ordering is good enough for a dropdown, and the
|
||
real ranking still happens when the search is actually submitted."""
|
||
q = q.strip()
|
||
if len(q) < 2:
|
||
return {"tracks": [], "albums": [], "artists": []}
|
||
try:
|
||
async with httpx.AsyncClient(timeout=8) as c:
|
||
r = await c.get(f"{DEEZER}/search/autocomplete",
|
||
params={"q": q, "limit": 5})
|
||
d = r.json() if r.status_code == 200 else {}
|
||
except Exception:
|
||
return {"tracks": [], "albums": [], "artists": []}
|
||
|
||
def rows(key):
|
||
return (d.get(key) or {}).get("data", []) or []
|
||
|
||
return {
|
||
"tracks": [{"deezer_id": t.get("id"), "title": t.get("title"),
|
||
"artist": (t.get("artist") or {}).get("name"),
|
||
"cover": (t.get("album") or {}).get("cover_small"),
|
||
"preview": t.get("preview")}
|
||
for t in rows("tracks")[:5]],
|
||
"albums": [{"album_id": a.get("id"), "title": a.get("title"),
|
||
"artist": (a.get("artist") or {}).get("name"),
|
||
"cover": a.get("cover_small"),
|
||
"nb_tracks": a.get("nb_tracks")}
|
||
for a in rows("albums")[:3]],
|
||
"artists": [{"artist_id": a.get("id"), "name": a.get("name"),
|
||
"cover": a.get("picture_small"),
|
||
"nb_album": a.get("nb_album")}
|
||
for a in rows("artists")[:3]],
|
||
}
|
||
|
||
|
||
@app.get("/api/album/{album_id}")
|
||
async def album_detail(album_id: int):
|
||
"""Full track list for an album, so 'Get all' can queue it in one go."""
|
||
# Two calls, not one. /album/{id} embeds a track list, but those nested
|
||
# objects have no track_position -- and taking the list index instead would
|
||
# misnumber every multi-disc release, writing "13 - ..." for disc 2 track 1.
|
||
# /album/{id}/tracks is the only source that carries track_position and
|
||
# disk_number, and those numbers end up in the filename and the tags.
|
||
try:
|
||
async with httpx.AsyncClient(timeout=20) as c:
|
||
r, tr = await asyncio.gather(
|
||
c.get(f"{DEEZER}/album/{album_id}"),
|
||
c.get(f"{DEEZER}/album/{album_id}/tracks", params={"limit": 300}))
|
||
except Exception as e:
|
||
raise HTTPException(502, f"deezer: {e}")
|
||
if r.status_code != 200:
|
||
raise HTTPException(404, "album not found")
|
||
d = r.json()
|
||
|
||
artist = (d.get("artist") or {}).get("name") or ""
|
||
year = (d.get("release_date") or "")[:4] or None
|
||
genres = [g.get("name") for g in (d.get("genres") or {}).get("data", []) if g.get("name")]
|
||
raw = (tr.json().get("data", []) if tr.status_code == 200
|
||
else (d.get("tracks") or {}).get("data", []))
|
||
tracks = []
|
||
for t in raw:
|
||
row = track_row(t)
|
||
# These nested track objects carry no album of their own, so inherit
|
||
# what the parent already told us rather than looking it up again.
|
||
row.update(album=d.get("title") or "", album_id=d.get("id"),
|
||
cover=d.get("cover_medium"), cover_big=d.get("cover_big"),
|
||
artist=row["artist"] or artist, year=year,
|
||
genre=genres[0] if genres else None,
|
||
track=t.get("track_position"),
|
||
disc=t.get("disk_number"))
|
||
tracks.append(row)
|
||
|
||
return {"album": {"album_id": d.get("id"), "title": d.get("title"),
|
||
"artist": artist, "artist_id": (d.get("artist") or {}).get("id"),
|
||
"cover_big": d.get("cover_big"), "cover": d.get("cover_medium"),
|
||
"year": year, "genres": genres, "label": d.get("label"),
|
||
"nb_tracks": d.get("nb_tracks"),
|
||
"record_type": d.get("record_type"),
|
||
"duration": d.get("duration")},
|
||
"tracks": tracks}
|
||
|
||
|
||
@app.get("/api/artist/{artist_id}")
|
||
async def artist_detail(artist_id: int):
|
||
"""Artist header plus their top tracks and albums."""
|
||
try:
|
||
async with httpx.AsyncClient(timeout=20) as c:
|
||
info, top, albums = await asyncio.gather(
|
||
c.get(f"{DEEZER}/artist/{artist_id}"),
|
||
c.get(f"{DEEZER}/artist/{artist_id}/top", params={"limit": 15}),
|
||
c.get(f"{DEEZER}/artist/{artist_id}/albums", params={"limit": 24}))
|
||
except Exception as e:
|
||
raise HTTPException(502, f"deezer: {e}")
|
||
if info.status_code != 200:
|
||
raise HTTPException(404, "artist not found")
|
||
a = info.json()
|
||
tracks = [track_row(t) for t in (top.json().get("data", []) if top.status_code == 200 else [])]
|
||
await add_album_meta(tracks)
|
||
return {
|
||
"artist": {"artist_id": a.get("id"), "name": a.get("name"),
|
||
"picture": a.get("picture_big"), "nb_fan": a.get("nb_fan"),
|
||
"nb_album": a.get("nb_album")},
|
||
"tracks": tracks,
|
||
"albums": [{"album_id": x.get("id"), "title": x.get("title"),
|
||
"cover": x.get("cover_medium"), "cover_big": x.get("cover_big"),
|
||
"year": (x.get("release_date") or "")[:4] or None,
|
||
"record_type": x.get("record_type"),
|
||
"nb_tracks": x.get("nb_tracks")}
|
||
for x in (albums.json().get("data", []) if albums.status_code == 200 else [])],
|
||
}
|
||
|
||
|
||
async def enqueue_job(artist: str, title: str, album=None, year=None,
|
||
track=None, deezer_id=None, requested_by=None,
|
||
cover=None, genre=None, duration=None) -> str:
|
||
"""Create a job and hand it to the search queue."""
|
||
job_id = str(uuid.uuid4())
|
||
# The single choke point for all three request paths, so normalise here.
|
||
# year reaches a directory name, and artist/title/album reach filenames.
|
||
year = safe_year(year)
|
||
cover = safe_cover(cover)
|
||
artist, title = artist[:MAX_COMPONENT], title[:MAX_COMPONENT]
|
||
album = album[:MAX_COMPONENT] if album else album
|
||
genre = genre[:80] if genre else genre
|
||
with db() as c:
|
||
c.execute(
|
||
"INSERT INTO jobs (id,state,detail,artist,title,album,year,track,"
|
||
"recording_mbid,release_mbid,requested_by,cover,genre,duration) "
|
||
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||
(job_id, "queued", "", artist, title, album, year, track,
|
||
str(deezer_id or ""), None, requested_by, cover, genre, duration))
|
||
await work_q.put(job_id)
|
||
return job_id
|
||
|
||
|
||
async def enrich(deezer_id) -> dict:
|
||
"""Deezer's search payload has no track number or release date, and both
|
||
go into the filename and tags. One lookup per request, not per result."""
|
||
if not deezer_id:
|
||
return {}
|
||
try:
|
||
async with httpx.AsyncClient(timeout=15) as c:
|
||
r = await c.get(f"{DEEZER}/track/{deezer_id}")
|
||
if r.status_code != 200:
|
||
return {}
|
||
d = r.json()
|
||
rd = d.get("release_date") or (d.get("album") or {}).get("release_date")
|
||
return {
|
||
"track": d.get("track_position"),
|
||
"year": (rd or "")[:4] or None,
|
||
"album": (d.get("album") or {}).get("title"),
|
||
"artist": (d.get("artist") or {}).get("name"),
|
||
}
|
||
except Exception as e:
|
||
log.warning("deezer track lookup failed: %s", e)
|
||
return {}
|
||
|
||
|
||
@app.post("/api/request")
|
||
async def request_track(body: RequestBody, request: Request):
|
||
check_quota(request)
|
||
e = await enrich(body.deezer_id)
|
||
return {"id": await enqueue_job(
|
||
e.get("artist") or body.artist, body.title,
|
||
e.get("album") or body.album, e.get("year") or body.year,
|
||
e.get("track") or body.track, body.deezer_id,
|
||
requested_by=session_user(request), cover=body.cover,
|
||
genre=body.genre, duration=body.length)}
|
||
|
||
|
||
class BatchBody(BaseModel):
|
||
tracks: list[RequestBody]
|
||
|
||
|
||
@app.post("/api/request/batch")
|
||
async def request_batch(body: BatchBody, request: Request):
|
||
"""Queue a whole album at once.
|
||
|
||
Unlike /api/request this skips the per-track /track/{id} enrichment: the
|
||
caller got these rows from /api/album/{id}, which already carries the
|
||
album, year and track position for every one of them. Re-fetching would be
|
||
nb_tracks extra calls to learn what we were just told.
|
||
"""
|
||
# Checked against the whole batch, not one track: queueing a 30-track album
|
||
# when 5 requests remain should be refused up front, not accepted and then
|
||
# blocked halfway through.
|
||
check_quota(request, adding=len(body.tracks))
|
||
who = session_user(request)
|
||
ids = [await enqueue_job(t.artist, t.title, t.album, t.year, t.track,
|
||
t.deezer_id, requested_by=who, cover=t.cover,
|
||
genre=t.genre, duration=t.length)
|
||
for t in body.tracks]
|
||
log.info("batch: queued %d track(s) for %s", len(ids), who or "anon")
|
||
return {"queued": len(ids), "ids": ids, "pending": work_q.qsize()}
|
||
|
||
|
||
class ImportBody(BaseModel):
|
||
text: str
|
||
|
||
|
||
# "Artist - Title", "Artist – Title" (en dash), "Title by Artist". Anything
|
||
# that doesn't split just goes to Deezer whole — its search copes fine with
|
||
# free text, and guessing wrong is worse than not guessing.
|
||
SPLIT = re.compile(r"\s+[-–—|]\s+|\s+by\s+", re.I)
|
||
DEEZER_URL = re.compile(r"deezer\.com/(?:[a-z]{2}/)?(album|playlist)/(\d+)")
|
||
|
||
# Generous for a human pasting an album or a year-end list, small enough that
|
||
# one request cannot turn into thousands of upstream calls.
|
||
MAX_IMPORT_LINES = 500
|
||
MAX_IMPORT_TRACKS = 1000
|
||
|
||
|
||
async def expand_url(kind: str, ident: str) -> list[str]:
|
||
"""Turn a Deezer album/playlist URL into 'Artist - Title' lines."""
|
||
out = []
|
||
try:
|
||
async with httpx.AsyncClient(timeout=25) as c:
|
||
r = await c.get(f"{DEEZER}/{kind}/{ident}")
|
||
if r.status_code != 200:
|
||
return []
|
||
for t in (r.json().get("tracks") or {}).get("data", []):
|
||
a = (t.get("artist") or {}).get("name") or ""
|
||
out.append(f"{a} - {t.get('title')}" if a else t.get("title", ""))
|
||
except Exception as e:
|
||
log.warning("expanding %s/%s failed: %s", kind, ident, e)
|
||
return [x for x in out if x]
|
||
|
||
|
||
@app.post("/api/import")
|
||
async def import_list(body: ImportBody, request: Request):
|
||
"""Paste a list, get it all queued.
|
||
|
||
Each line is resolved against Deezer and the top-ranked hit is taken
|
||
automatically — there is no picker in bulk mode. Resolved artist/album are
|
||
recorded on every job so a wrong auto-pick is visible in the list rather
|
||
than silently landing the wrong song in the library.
|
||
"""
|
||
lines, queued, skipped = [], [], []
|
||
# Capped before expansion, not after. Each URL line costs an upstream call,
|
||
# and the quota check below cannot run until expansion has finished — so
|
||
# without a cap here a paste of 10,000 album URLs is 10,000 requests to
|
||
# Deezer from this IP before anyone's quota is consulted.
|
||
raw_lines = body.text.splitlines()
|
||
if len(raw_lines) > MAX_IMPORT_LINES:
|
||
raise HTTPException(413, f"paste at most {MAX_IMPORT_LINES} lines at a time "
|
||
f"(got {len(raw_lines)})")
|
||
for raw in raw_lines:
|
||
line = raw.strip().lstrip("-•*0123456789. )").strip()
|
||
if not line:
|
||
continue
|
||
m = DEEZER_URL.search(line)
|
||
lines.extend(await expand_url(*m.groups()) if m else [line])
|
||
if len(lines) > MAX_IMPORT_TRACKS:
|
||
raise HTTPException(413, f"that expands to more than "
|
||
f"{MAX_IMPORT_TRACKS} tracks")
|
||
|
||
# After expansion — a single playlist URL can be 200 tracks, and the point
|
||
# of the check is the real number, not the number of lines pasted.
|
||
check_quota(request, adding=len(lines))
|
||
|
||
for line in lines:
|
||
# Search the line as typed; the ranker already weights an artist-name
|
||
# match heavily, so "Artist - Title" needs no special handling.
|
||
try:
|
||
# Ask for a real candidate pool, not one row. deezer_search ranks
|
||
# what it is given, so limit=1 fetched two results and handed back
|
||
# the better of a demo and a rehearsal take while the studio
|
||
# version -- 6x the popularity -- sat further down the list unseen.
|
||
hits = await deezer_search(SPLIT.sub(" ", line), limit=10)
|
||
except Exception as e:
|
||
skipped.append({"line": line, "why": errstr(e)})
|
||
continue
|
||
if not hits:
|
||
skipped.append({"line": line, "why": "no match on Deezer"})
|
||
continue
|
||
h = hits[0]
|
||
e = await enrich(h["deezer_id"])
|
||
jid = await enqueue_job(
|
||
e.get("artist") or h["artist"], h["title"],
|
||
e.get("album") or h["album"], e.get("year"), e.get("track"),
|
||
h["deezer_id"], requested_by=session_user(request),
|
||
cover=h.get("cover"), duration=h.get("length"))
|
||
queued.append({"id": jid, "line": line, "cover": h.get("cover"),
|
||
"matched": f"{h['artist']} — {h['title']}"})
|
||
|
||
log.info("import: %d queued, %d skipped", len(queued), len(skipped))
|
||
return {"queued": queued, "skipped": skipped, "pending": work_q.qsize()}
|
||
|
||
|
||
class ReviewBody(BaseModel):
|
||
action: str # approve | reject
|
||
|
||
|
||
@app.get("/api/review")
|
||
async def review_queue(request: Request):
|
||
"""Downloads held back because the match was not convincing enough.
|
||
|
||
Scoped like /api/jobs: your own unless you are an admin, because a held
|
||
file is a record of what someone asked for.
|
||
"""
|
||
me = session_user(request)
|
||
admin = user_role(request) == "admin"
|
||
with db() as c:
|
||
if admin:
|
||
rows = c.execute(
|
||
"SELECT * FROM jobs WHERE state='review' ORDER BY created_at DESC"
|
||
).fetchall()
|
||
else:
|
||
rows = c.execute(
|
||
"SELECT * FROM jobs WHERE state='review' AND requested_by IS ? "
|
||
"ORDER BY created_at DESC", (me,)).fetchall()
|
||
return {"items": [{
|
||
"id": r["id"], "artist": r["artist"], "title": r["title"],
|
||
"album": r["album"], "year": r["year"], "cover": r["cover"],
|
||
"confidence": r["confidence"], "why": r["match_why"],
|
||
"file": (r["remote_file"] or "").replace("\\", "/").rsplit("/", 1)[-1],
|
||
"path": r["remote_file"], "size": r["size"],
|
||
"requested_by": r["requested_by"], "created_at": r["created_at"],
|
||
} for r in rows], "thresholds": {"auto": AUTO_CONFIDENCE,
|
||
"review": REVIEW_CONFIDENCE}}
|
||
|
||
|
||
@app.post("/api/review/{job_id}")
|
||
async def review_decide(job_id: str, body: ReviewBody, request: Request):
|
||
me = session_user(request)
|
||
admin = user_role(request) == "admin"
|
||
if body.action not in ("approve", "reject"):
|
||
raise HTTPException(400, "action must be approve or reject")
|
||
|
||
with db() as c:
|
||
job = c.execute("SELECT * FROM jobs WHERE id=? AND state='review'",
|
||
(job_id,)).fetchone()
|
||
# 404 rather than 403 for someone else's job: a distinct error would
|
||
# confirm the id exists, same as /api/retry.
|
||
if not job or (not admin and job["requested_by"] != me):
|
||
raise HTTPException(404, "no such review item")
|
||
|
||
base = (job["remote_file"] or "").replace("\\", "/").rsplit("/", 1)[-1]
|
||
src = downloaded_index().get(base)
|
||
if body.action == "reject":
|
||
if src:
|
||
try:
|
||
src.unlink()
|
||
except OSError as e:
|
||
log.warning("review: could not delete %s: %s", src, e)
|
||
set_state(job_id, "failed", "rejected in review", reviewed_by=me)
|
||
return {"ok": True, "state": "failed"}
|
||
|
||
if not src:
|
||
raise HTTPException(409, "the downloaded file is no longer on disk")
|
||
dest = tag_and_file(job, src)
|
||
set_state(job_id, "imported", str(dest), imported_path=str(dest),
|
||
reviewed_by=me)
|
||
log.info("review: %s approved %s", me, job_id[:8])
|
||
return {"ok": True, "state": "imported", "path": str(dest)}
|
||
|
||
|
||
@app.get("/api/discover")
|
||
async def discover(request: Request):
|
||
"""What to listen to next, derived from what you have actually asked for.
|
||
|
||
Deliberately built on muse's own request history plus Deezer's related
|
||
artists, not on listening data — Navidrome owns play counts and history,
|
||
and duplicating them here would mean two disagreeing sources of truth.
|
||
"""
|
||
me = session_user(request)
|
||
with db() as c:
|
||
top = c.execute(
|
||
"SELECT artist, COUNT(*) n FROM jobs "
|
||
"WHERE requested_by IS ? AND state='imported' "
|
||
"GROUP BY artist COLLATE NOCASE ORDER BY n DESC LIMIT 5",
|
||
(me,)).fetchall()
|
||
if not top:
|
||
return {"seeds": [], "artists": [], "tracks": []}
|
||
|
||
async def related(name: str) -> list[dict]:
|
||
try:
|
||
async with httpx.AsyncClient(timeout=15) as c:
|
||
s = await c.get(f"{DEEZER}/search/artist",
|
||
params={"q": name, "limit": 1})
|
||
hits = (s.json() or {}).get("data") or []
|
||
if not hits:
|
||
return []
|
||
r = await c.get(f"{DEEZER}/artist/{hits[0]['id']}/related",
|
||
params={"limit": 6})
|
||
return (r.json() or {}).get("data") or []
|
||
except Exception as e:
|
||
log.info("discover: related for %r failed: %s", name, errstr(e))
|
||
return []
|
||
|
||
seeds = [r["artist"] for r in top]
|
||
groups = await asyncio.gather(*(related(s) for s in seeds))
|
||
|
||
have = {norm(r["artist"]) for r in top}
|
||
seen, artists = set(), []
|
||
for g in groups:
|
||
for a in g:
|
||
k = norm(a.get("name") or "")
|
||
if not k or k in have or k in seen:
|
||
continue
|
||
seen.add(k)
|
||
artists.append({"id": a.get("id"), "name": a.get("name"),
|
||
"picture": a.get("picture_medium"),
|
||
"fans": a.get("nb_fan")})
|
||
artists.sort(key=lambda a: -(a["fans"] or 0))
|
||
return {"seeds": seeds, "artists": artists[:18]}
|
||
|
||
|
||
@app.get("/api/recap")
|
||
async def recap(request: Request, days: int = Query(365, ge=7, le=3650)):
|
||
"""Your year in muse — from request history, which muse does own."""
|
||
me = session_user(request)
|
||
since = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat(" ")[:19]
|
||
with db() as c:
|
||
base = ("FROM jobs WHERE requested_by IS ? AND created_at >= ? "
|
||
"AND state='imported'")
|
||
total = c.execute(f"SELECT COUNT(*) n, COALESCE(SUM(size),0) b {base}",
|
||
(me, since)).fetchone()
|
||
artists = c.execute(
|
||
f"SELECT artist, COUNT(*) n {base} GROUP BY artist COLLATE NOCASE "
|
||
"ORDER BY n DESC LIMIT 10", (me, since)).fetchall()
|
||
genres = c.execute(
|
||
f"SELECT genre, COUNT(*) n {base} AND genre IS NOT NULL AND genre<>'' "
|
||
"GROUP BY genre ORDER BY n DESC LIMIT 8", (me, since)).fetchall()
|
||
decades = c.execute(
|
||
f"SELECT substr(year,1,3) d, COUNT(*) n {base} "
|
||
"AND year GLOB '[0-9][0-9][0-9][0-9]' GROUP BY d ORDER BY d",
|
||
(me, since)).fetchall()
|
||
busiest = c.execute(
|
||
f"SELECT substr(created_at,1,10) d, COUNT(*) n {base} "
|
||
"GROUP BY d ORDER BY n DESC LIMIT 1", (me, since)).fetchone()
|
||
return {
|
||
"days": days,
|
||
"tracks": total["n"], "bytes": total["b"],
|
||
"artists": [{"name": r["artist"], "n": r["n"]} for r in artists],
|
||
"genres": [{"name": r["genre"], "n": r["n"]} for r in genres],
|
||
"decades": [{"decade": f"{r['d']}0s", "n": r["n"]} for r in decades],
|
||
"busiest_day": ({"date": busiest["d"], "n": busiest["n"]}
|
||
if busiest and busiest["n"] else None),
|
||
}
|
||
|
||
|
||
def job_dict(r: sqlite3.Row) -> dict:
|
||
"""A job as the UI wants it.
|
||
|
||
`candidates` is deliberately dropped: it is the untried remainder of the
|
||
ladder, several KB of peer names and paths per job, and the jobs list is
|
||
polled every four seconds. `tried` is sent parsed, because it is the answer
|
||
to "why did this fail" and the UI renders it as a list.
|
||
"""
|
||
d = dict(r)
|
||
d.pop("candidates", None)
|
||
try:
|
||
d["tried"] = json.loads(d.get("tried") or "[]")
|
||
except (TypeError, ValueError):
|
||
d["tried"] = []
|
||
d["attempts"] = d.get("attempts") or 0
|
||
d["remaining"] = max(0, MAX_ATTEMPTS - d["attempts"])
|
||
return d
|
||
|
||
|
||
@app.get("/api/jobs")
|
||
async def jobs(request: Request, limit: int = Query(200, ge=1, le=1000),
|
||
mine: bool = False):
|
||
"""Own requests by default; admins see everyone's.
|
||
|
||
Scoped rather than global because on a shared instance one person's
|
||
request history is a list of what they listen to, which is nobody else's
|
||
business by default.
|
||
"""
|
||
u = current_user(request)
|
||
everyone = (AUTH_MODE == "off") or (u is not None and u["role"] == "admin" and not mine)
|
||
who = None if everyone else (u["username"] if u else None)
|
||
|
||
where, args = ("", []) if everyone else ("WHERE requested_by IS ?", [who])
|
||
with db() as c:
|
||
rows = c.execute(
|
||
f"SELECT * FROM jobs {where} ORDER BY created_at DESC, rowid DESC LIMIT ?",
|
||
[*args, limit]).fetchall()
|
||
counts = {r["state"]: r["n"] for r in c.execute(
|
||
f"SELECT state, COUNT(*) n FROM jobs {where} GROUP BY state",
|
||
args).fetchall()}
|
||
return {"jobs": [job_dict(r) for r in rows], "counts": counts,
|
||
"pending": work_q.qsize(), "scope": "all" if everyone else "mine"}
|
||
|
||
|
||
@app.post("/api/retry/{job_id}")
|
||
async def retry(job_id: str, request: Request):
|
||
"""Put a failed job back on the queue. Soulseek failures are usually the
|
||
peer going offline, not the request being wrong, so retrying later often
|
||
just works."""
|
||
u = current_user(request)
|
||
with db() as c:
|
||
row = c.execute("SELECT id, state, requested_by FROM jobs WHERE id=?",
|
||
(job_id,)).fetchone()
|
||
if not row:
|
||
raise HTTPException(404, "no such job")
|
||
# Same 404 rather than 403 for someone else's job: a distinct error would
|
||
# confirm the id exists, which is a small oracle over other people's
|
||
# requests.
|
||
if AUTH_MODE != "off" and u and u["role"] != "admin" \
|
||
and row["requested_by"] != u["username"]:
|
||
raise HTTPException(404, "no such job")
|
||
# Only failed jobs. Requeueing one in `review` used to strand its
|
||
# downloaded file on disk with no row left pointing at it, and requeueing
|
||
# one already `imported` would re-download a track that is in the library.
|
||
if row["state"] != "failed":
|
||
raise HTTPException(409, f"only failed jobs can be retried "
|
||
f"(this one is {row['state']})")
|
||
# Clear the last transfer, but keep any candidates the previous run did
|
||
# not get to: those are left behind deliberately when slskd could not
|
||
# reach anyone, and re-running the search would only rediscover them.
|
||
with db() as c:
|
||
row = c.execute("SELECT candidates FROM jobs WHERE id=?",
|
||
(job_id,)).fetchone()
|
||
keep = bool(json.loads((row["candidates"] if row else None) or "[]"))
|
||
c.execute("UPDATE jobs SET username=NULL, remote_file=NULL, size=NULL,"
|
||
" confidence=NULL, match_why=NULL, attempts=0,"
|
||
" started_at=NULL"
|
||
+ ("" if keep else ", candidates=NULL, tried=NULL")
|
||
+ " WHERE id=?", (job_id,))
|
||
set_state(job_id, "queued", "")
|
||
await work_q.put(job_id)
|
||
return {"ok": True}
|
||
|
||
|
||
# ─────────────────────────── admin ───────────────────────────
|
||
|
||
class NewUser(BaseModel):
|
||
username: str
|
||
provider: str = "github"
|
||
role: str = "user"
|
||
quota: int = 0
|
||
storage_quota_gb: float = 0
|
||
# Where to send the invitation. Omit for a Google invite — the username
|
||
# already is the address. Only needed for GitHub, where a login is not one.
|
||
email: str | None = None
|
||
notify: bool = True
|
||
|
||
|
||
class PatchUser(BaseModel):
|
||
role: str | None = None
|
||
quota: int | None = None
|
||
storage_quota_gb: float | None = None
|
||
disabled: bool | None = None
|
||
# So a typo in an invited address is fixable without deleting the account
|
||
# and losing its request history.
|
||
email: str | None = None
|
||
|
||
|
||
def user_dict(r: sqlite3.Row) -> dict:
|
||
u = usage_for(r["username"])
|
||
return {
|
||
"username": r["username"], "provider": r["provider"], "role": r["role"],
|
||
"display": r["display"], "avatar": r["avatar"],
|
||
"quota": r["quota"], "storage_quota": r["storage_quota"],
|
||
"disabled": bool(r["disabled"]), "created_at": r["created_at"],
|
||
"last_seen": r["last_seen"], "invited_by": r["invited_by"],
|
||
"usage": u, "email": r["email"], "notify": bool(r["notify"]),
|
||
# Surfaced so the UI doesn't have to know the allowlist rule: these are
|
||
# the accounts that cannot actually be locked out by disabling them.
|
||
"bootstrap": r["username"].lower() in ALLOWED_USERS,
|
||
}
|
||
|
||
|
||
@app.get("/api/admin/users")
|
||
async def admin_users(request: Request):
|
||
require_admin(request)
|
||
with db() as c:
|
||
rows = c.execute("SELECT * FROM users ORDER BY role='admin' DESC, username").fetchall()
|
||
return {"users": [user_dict(r) for r in rows], "roles": list(ROLES)}
|
||
|
||
|
||
@app.post("/api/admin/users")
|
||
async def admin_add_user(body: NewUser, request: Request):
|
||
"""Invite someone.
|
||
|
||
The row IS the authorisation: once it exists, that identity can sign in
|
||
with the provider it was invited for, and only that one. The email is a
|
||
notification on top, sent when there is an address and SMTP is configured.
|
||
"""
|
||
who = require_admin(request)
|
||
name = body.username.strip()
|
||
if not name:
|
||
raise HTTPException(400, "username required")
|
||
if body.role not in ROLES:
|
||
raise HTTPException(400, f"role must be one of {', '.join(ROLES)}")
|
||
if body.provider not in PROVIDERS:
|
||
raise HTTPException(400, f"provider must be one of {', '.join(PROVIDERS)}")
|
||
if body.provider not in ENABLED_PROVIDERS:
|
||
raise HTTPException(
|
||
400, f"{body.provider} is not enabled on this instance "
|
||
f"(MUSE_AUTH={AUTH_MODE})")
|
||
|
||
# A Google identity is its own address, so require it to look like one --
|
||
# otherwise the invite silently authorises a username Google can never
|
||
# return, and the person just gets "not authorised" with no explanation.
|
||
if body.provider == "google" and not EMAIL_RE.match(name):
|
||
raise HTTPException(400, "a Google invite must be the full email address")
|
||
|
||
email = (body.email or "").strip() or (name if body.provider == "google" else "")
|
||
if email and not EMAIL_RE.match(email):
|
||
raise HTTPException(400, "that does not look like an email address")
|
||
|
||
with db() as c:
|
||
if c.execute("SELECT 1 FROM users WHERE username=?", (name,)).fetchone():
|
||
raise HTTPException(409, "that user already exists")
|
||
c.execute(
|
||
"INSERT INTO users (username,provider,role,quota,storage_quota,"
|
||
"invited_by,email,notify) VALUES (?,?,?,?,?,?,?,?)",
|
||
(name, body.provider, body.role, max(0, body.quota),
|
||
int(max(0.0, body.storage_quota_gb) * 2**30), who,
|
||
email or None, 1 if body.notify else 0))
|
||
r = c.execute("SELECT * FROM users WHERE username=?", (name,)).fetchone()
|
||
|
||
log.info("admin: %s invited %s as %s (%s)", who, name, body.role, body.provider)
|
||
sent = bool(MAIL_ON and email)
|
||
if sent:
|
||
await notify_invited(r, who, email)
|
||
return {**user_dict(r), "invite_sent": sent,
|
||
"invite_why": None if sent else
|
||
("no SMTP configured on this instance" if not MAIL_ON
|
||
else "no email address for this account")}
|
||
|
||
|
||
@app.post("/api/admin/users/{username}/resend")
|
||
async def admin_resend_invite(username: str, request: Request):
|
||
"""Send the invitation again. Safe to call repeatedly — the link carries
|
||
no token, so resending neither extends nor grants anything."""
|
||
who = require_admin(request)
|
||
with db() as c:
|
||
r = c.execute("SELECT * FROM users WHERE username=?", (username,)).fetchone()
|
||
if not r:
|
||
raise HTTPException(404, "no such user")
|
||
if not MAIL_ON:
|
||
raise HTTPException(409, "no SMTP configured on this instance")
|
||
if not r["email"]:
|
||
raise HTTPException(409, "no email address on file for that account")
|
||
await notify_invited(r, who, r["email"])
|
||
return {"ok": True, "sent_to": r["email"]}
|
||
|
||
|
||
@app.patch("/api/admin/users/{username}")
|
||
async def admin_patch_user(username: str, body: PatchUser, request: Request):
|
||
who = require_admin(request)
|
||
with db() as c:
|
||
r = c.execute("SELECT * FROM users WHERE username=?", (username,)).fetchone()
|
||
if not r:
|
||
raise HTTPException(404, "no such user")
|
||
|
||
# An install with no admin cannot be administered, and nothing in the
|
||
# UI can undo it. Refuse to remove the last one — whether by demotion
|
||
# or by disabling them.
|
||
losing_admin = r["role"] == "admin" and (
|
||
(body.role is not None and body.role != "admin") or body.disabled is True)
|
||
if losing_admin:
|
||
others = c.execute(
|
||
"SELECT COUNT(*) n FROM users "
|
||
"WHERE role='admin' AND disabled=0 AND username<>?",
|
||
(username,)).fetchone()["n"]
|
||
if others == 0:
|
||
raise HTTPException(
|
||
409, "this is the only active admin — promote someone else first")
|
||
|
||
sets, vals = [], []
|
||
if body.role is not None:
|
||
if body.role not in ROLES:
|
||
raise HTTPException(400, f"role must be one of {', '.join(ROLES)}")
|
||
sets.append("role=?"); vals.append(body.role)
|
||
if body.quota is not None:
|
||
sets.append("quota=?"); vals.append(max(0, body.quota))
|
||
if body.storage_quota_gb is not None:
|
||
sets.append("storage_quota=?")
|
||
vals.append(int(max(0.0, body.storage_quota_gb) * 2**30))
|
||
if body.disabled is not None:
|
||
sets.append("disabled=?"); vals.append(1 if body.disabled else 0)
|
||
if body.email is not None:
|
||
e = body.email.strip()
|
||
if e and not EMAIL_RE.match(e):
|
||
raise HTTPException(400, "that does not look like an email address")
|
||
sets.append("email=?"); vals.append(e or None)
|
||
if not sets:
|
||
raise HTTPException(400, "nothing to change")
|
||
|
||
c.execute(f"UPDATE users SET {', '.join(sets)} WHERE username=?",
|
||
[*vals, username])
|
||
# Disabling must take effect now. session_user re-checks on every
|
||
# request anyway, but dropping the rows makes it unambiguous.
|
||
if body.disabled:
|
||
c.execute("DELETE FROM sessions WHERE username=?", (username,))
|
||
r = c.execute("SELECT * FROM users WHERE username=?", (username,)).fetchone()
|
||
log.info("admin: %s updated %s (%s)", who, username, ", ".join(sets))
|
||
return user_dict(r)
|
||
|
||
|
||
@app.delete("/api/admin/users/{username}")
|
||
async def admin_delete_user(username: str, request: Request):
|
||
who = require_admin(request)
|
||
with db() as c:
|
||
r = c.execute("SELECT * FROM users WHERE username=?", (username,)).fetchone()
|
||
if not r:
|
||
raise HTTPException(404, "no such user")
|
||
if r["role"] == "admin":
|
||
others = c.execute(
|
||
"SELECT COUNT(*) n FROM users "
|
||
"WHERE role='admin' AND disabled=0 AND username<>?",
|
||
(username,)).fetchone()["n"]
|
||
if others == 0:
|
||
raise HTTPException(409, "this is the only active admin")
|
||
c.execute("DELETE FROM sessions WHERE username=?", (username,))
|
||
c.execute("DELETE FROM users WHERE username=?", (username,))
|
||
# Their jobs are deliberately kept: the history is a record of what is in
|
||
# the library and why, and deleting a person should not rewrite it.
|
||
log.info("admin: %s deleted user %s (jobs retained)", who, username)
|
||
return {"ok": True, "jobs_retained": True}
|
||
|
||
|
||
@app.get("/api/admin/activity")
|
||
async def admin_activity(request: Request, days: int = Query(30, ge=1, le=365)):
|
||
require_admin(request)
|
||
days = max(1, min(days, 365))
|
||
since = (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
|
||
with db() as c:
|
||
daily = c.execute(
|
||
"SELECT substr(created_at,1,10) d, COUNT(*) n, "
|
||
" SUM(state='imported') imported, SUM(state='failed') failed "
|
||
"FROM jobs WHERE created_at >= ? GROUP BY d ORDER BY d", (since,)).fetchall()
|
||
per_user = c.execute(
|
||
"SELECT COALESCE(requested_by,'(unattributed)') u, COUNT(*) n, "
|
||
" COALESCE(SUM(CASE WHEN state='imported' THEN size END),0) bytes "
|
||
"FROM jobs WHERE created_at >= ? GROUP BY u ORDER BY n DESC",
|
||
(since,)).fetchall()
|
||
top = c.execute(
|
||
"SELECT artist, COUNT(*) n FROM jobs WHERE created_at >= ? "
|
||
"GROUP BY artist ORDER BY n DESC LIMIT 8", (since,)).fetchall()
|
||
|
||
# Zero-fill: a bar chart with gaps for quiet days misreads as missing data
|
||
# rather than as nothing having happened.
|
||
by_day = {r["d"]: r for r in daily}
|
||
today = datetime.now(timezone.utc).date()
|
||
series = []
|
||
for i in range(days - 1, -1, -1):
|
||
d = (today - timedelta(days=i)).isoformat()
|
||
r = by_day.get(d)
|
||
series.append({"date": d, "n": r["n"] if r else 0,
|
||
"imported": (r["imported"] or 0) if r else 0,
|
||
"failed": (r["failed"] or 0) if r else 0})
|
||
return {"days": days, "series": series,
|
||
"per_user": [dict(r) for r in per_user],
|
||
"top_artists": [dict(r) for r in top]}
|
||
|
||
|
||
def _proc_stat() -> tuple[int, int]:
|
||
"""(busy, total) jiffies from /proc/stat."""
|
||
with open("/proc/stat") as f:
|
||
parts = [int(x) for x in f.readline().split()[1:]]
|
||
idle = parts[3] + (parts[4] if len(parts) > 4 else 0)
|
||
return sum(parts) - idle, sum(parts)
|
||
|
||
|
||
_cpu_prev: tuple[int, int] | None = None
|
||
|
||
|
||
def system_stats() -> dict:
|
||
"""Host CPU/RAM plus disk for the paths that matter.
|
||
|
||
Reads /proc directly rather than adding psutil: it is ~30 lines, keeps the
|
||
image free of a compiled dependency, and /proc in a container reports the
|
||
HOST's CPU and memory — which is what you actually want to see for a
|
||
homelab box. Everything is individually guarded so a non-Linux dev machine
|
||
degrades to nulls instead of 500ing the whole panel.
|
||
"""
|
||
global _cpu_prev
|
||
out: dict = {}
|
||
|
||
try:
|
||
busy, total = _proc_stat()
|
||
if _cpu_prev:
|
||
db_, dt = busy - _cpu_prev[0], total - _cpu_prev[1]
|
||
out["cpu_pct"] = round(100 * db_ / dt, 1) if dt > 0 else None
|
||
else:
|
||
out["cpu_pct"] = None # first call has no baseline to diff
|
||
_cpu_prev = (busy, total)
|
||
except Exception:
|
||
out["cpu_pct"] = None
|
||
|
||
try:
|
||
mem = {}
|
||
with open("/proc/meminfo") as f:
|
||
for line in f:
|
||
k, _, v = line.partition(":")
|
||
mem[k] = int(v.split()[0]) * 1024
|
||
total_b = mem.get("MemTotal", 0)
|
||
avail = mem.get("MemAvailable", 0)
|
||
out["mem_total"] = total_b
|
||
out["mem_used"] = total_b - avail
|
||
out["mem_pct"] = round(100 * (total_b - avail) / total_b, 1) if total_b else None
|
||
except Exception:
|
||
out["mem_total"] = out["mem_used"] = out["mem_pct"] = None
|
||
|
||
try:
|
||
out["load"] = [round(x, 2) for x in os.getloadavg()]
|
||
out["cpus"] = os.cpu_count()
|
||
except Exception:
|
||
out["load"] = None
|
||
|
||
disks = {}
|
||
for label, p in (("library", LIBRARY_DIR), ("downloads", DOWNLOAD_DIR)):
|
||
try:
|
||
s = os.statvfs(p)
|
||
total_b = s.f_blocks * s.f_frsize
|
||
free = s.f_bavail * s.f_frsize
|
||
disks[label] = {"total": total_b, "used": total_b - free, "free": free,
|
||
"pct": round(100 * (total_b - free) / total_b, 1) if total_b else None}
|
||
except Exception:
|
||
disks[label] = None
|
||
out["disks"] = disks
|
||
return out
|
||
|
||
|
||
@app.get("/api/admin/system")
|
||
async def admin_system(request: Request):
|
||
require_admin(request)
|
||
out = {"system": system_stats(), "workers": WORKERS, "pending": work_q.qsize()}
|
||
|
||
with db() as c:
|
||
out["jobs"] = {r["state"]: r["n"] for r in c.execute(
|
||
"SELECT state, COUNT(*) n FROM jobs GROUP BY state").fetchall()}
|
||
out["library"] = {
|
||
"tracks": c.execute(
|
||
"SELECT COUNT(*) n FROM jobs WHERE state='imported'").fetchone()["n"],
|
||
"bytes": c.execute(
|
||
"SELECT COALESCE(SUM(size),0) b FROM jobs "
|
||
"WHERE state='imported'").fetchone()["b"],
|
||
}
|
||
out["users"] = c.execute("SELECT COUNT(*) n FROM users").fetchone()["n"]
|
||
out["sessions"] = c.execute(
|
||
"SELECT COUNT(*) n FROM sessions WHERE expires_at > ?",
|
||
(datetime.now(timezone.utc).isoformat(),)).fetchone()["n"]
|
||
|
||
slskd: dict = {"ok": False}
|
||
try:
|
||
async with slskd_client(PROBE_TIMEOUT) as sc:
|
||
r = await sc.get("/api/v0/application")
|
||
slskd["ok"] = r.status_code == 200
|
||
if r.status_code == 200:
|
||
d = r.json()
|
||
srv = d.get("server") or {}
|
||
slskd["connected"] = bool(srv.get("isConnected"))
|
||
slskd["logged_in"] = bool(srv.get("isLoggedIn"))
|
||
slskd["version"] = d.get("version")
|
||
shares = d.get("shares") or {}
|
||
slskd["shared_files"] = (shares.get("directories") or {}).get("fileCount") \
|
||
or shares.get("fileCount")
|
||
else:
|
||
slskd["error"] = f"HTTP {r.status_code}"
|
||
except Exception as e:
|
||
slskd["error"] = errstr(e)
|
||
out["slskd"] = slskd
|
||
return out
|
||
|
||
|
||
@app.get("/api/health")
|
||
async def health():
|
||
ok, detail = True, {}
|
||
async with slskd_client(PROBE_TIMEOUT) as sc:
|
||
try:
|
||
r = await sc.get("/api/v0/application")
|
||
detail["slskd"] = r.status_code
|
||
ok = ok and r.status_code == 200
|
||
# Reachable and logged out is the failure worth naming: every
|
||
# search returns nothing and every request fails, but the container
|
||
# is up and healthy so nothing else reports a problem. This used to
|
||
# be visible only on the admin panel, which meant an ordinary user
|
||
# watched their requests fail with no way to learn why.
|
||
if r.status_code == 200:
|
||
app_state = r.json()
|
||
srv = app_state.get("server") or {}
|
||
detail["logged_in"] = bool(srv.get("isLoggedIn"))
|
||
detail["shared_files"] = (
|
||
((app_state.get("user") or {}).get("statistics") or {})
|
||
.get("fileCount"))
|
||
ok = ok and detail["logged_in"]
|
||
except Exception as e:
|
||
detail["slskd"] = errstr(e)
|
||
ok = False
|
||
detail["library_writable"] = os.access(LIBRARY_DIR, os.W_OK)
|
||
detail["downloads_readable"] = os.access(DOWNLOAD_DIR, os.R_OK)
|
||
return {"ok": ok and detail["library_writable"], **detail}
|
||
|
||
|
||
@app.get("/")
|
||
async def index():
|
||
f = WEB / "index.html"
|
||
if not f.exists():
|
||
raise HTTPException(500, "UI not built — run `npm run build` in web/")
|
||
return FileResponse(f)
|
||
|
||
|
||
# Next.js emits its immutable, content-hashed chunks under /_next/static.
|
||
#
|
||
# Guarded rather than mounted unconditionally so that importing this module
|
||
# does not require build artifacts: CI typechecks the API before the UI has
|
||
# been built, and tests should not have to run npm first. A deploy that really
|
||
# is missing its UI says so at startup and again on the first request, rather
|
||
# than silently serving 404s.
|
||
if (WEB / "_next").is_dir():
|
||
app.mount("/_next", StaticFiles(directory=WEB / "_next"), name="next")
|
||
else:
|
||
log.warning("web/out/_next is missing — the UI was not built. "
|
||
"API endpoints work; the web page will not.")
|