Aryan SinghandClaude Opus 5
c781193ec9
A job carried over from before the candidate ladder has attempts=0 while carrying a real failure in `tried`, so the summary rendered as "tried 0 peers: 1x no bytes in 20m" -- a count of zero next to the evidence contradicting it. The live instance produced exactly that when the stall check finally cleared its long-stuck job. Counts distinct usernames in `tried` instead. The two agree for anything this version enqueued; the list is the one that is right in both cases. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2933 lines
125 KiB
Python
2933 lines
125 KiB
Python
"""
|
||
muse — request a song, get it in Navidrome.
|
||
|
||
The gap this fills: Lidarr thinks in MusicBrainz *releases*, so the smallest
|
||
thing it can want is an album. Soulseek trades individual files. Nothing in
|
||
between existed, so requesting one song meant either hoovering a whole
|
||
discography through Lidarr or hand-filing downloads out of slskd's UI.
|
||
|
||
The trick that makes this simple: the user PICKS a specific track before
|
||
anything is downloaded. That means we hold ground-truth metadata — artist,
|
||
album, year, track number — before the file exists. Taggers like beets have to
|
||
infer all of it afterwards from whatever a stranger named the file; we never
|
||
guess. We tag what was asked for.
|
||
|
||
Flow:
|
||
/api/search Deezer track search -> the user picks one
|
||
/api/request /track/{id} for exact metadata, then slskd search ->
|
||
score candidates -> enqueue best
|
||
worker poll slskd -> on completion: tag, then hardlink into
|
||
/data/media/music/<Artist>/<Album> (<Year>)/NN - Title.ext
|
||
|
||
Hardlink, not copy: the download stays in slskd's completed dir so it keeps
|
||
being shared, while the library gets its own name for the same bytes. One
|
||
file on disk, two names — the same trick the *arrs use.
|
||
"""
|
||
|
||
import asyncio
|
||
import base64
|
||
import collections
|
||
import difflib
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import os
|
||
import re
|
||
import secrets
|
||
import sqlite3
|
||
import unicodedata
|
||
import uuid
|
||
from contextlib import asynccontextmanager
|
||
from datetime import datetime, timedelta, timezone
|
||
from pathlib import Path
|
||
from urllib.parse import urlencode
|
||
|
||
import httpx
|
||
from fastapi import FastAPI, HTTPException, Query, Request
|
||
from fastapi.responses import (FileResponse, HTMLResponse, JSONResponse,
|
||
RedirectResponse, Response)
|
||
from fastapi.staticfiles import StaticFiles
|
||
from pydantic import BaseModel
|
||
|
||
log = logging.getLogger("muse")
|
||
logging.basicConfig(
|
||
level=os.getenv("MUSE_LOG_LEVEL", "INFO"),
|
||
format="[%(levelname)s|%(name)s] %(asctime)s: %(message)s",
|
||
datefmt="%Y-%m-%dT%H:%M:%S%z",
|
||
)
|
||
|
||
SLSKD_URL = os.getenv("SLSKD_URL", "http://192.168.1.87:5030").rstrip("/")
|
||
SLSKD_API_KEY = os.getenv("SLSKD_API_KEY", "")
|
||
DOWNLOAD_DIR = Path(os.getenv("MUSE_DOWNLOAD_DIR", "/data/torrents/soulseek/complete"))
|
||
LIBRARY_DIR = Path(os.getenv("MUSE_LIBRARY_DIR", "/data/media/music"))
|
||
DB_PATH = Path(os.getenv("MUSE_DB", "/config/muse.db"))
|
||
|
||
# Ordered best-first. A candidate's extension must appear here or it is
|
||
# rejected outright — this is what keeps .m4a rips and stray .cue files out.
|
||
FORMATS = [f.strip() for f in os.getenv("MUSE_FORMATS", "flac,mp3").split(",") if f.strip()]
|
||
|
||
# Below this, the remote filename doesn't plausibly contain the track title.
|
||
# Soulseek filenames are chaotic ("08-radiohead-creep_(accoustic)-ent.mp3"),
|
||
# so this compares against a normalised form, not the raw string.
|
||
MIN_MATCH = float(os.getenv("MUSE_MIN_MATCH", "0.45"))
|
||
|
||
# The title alone must clear this before the artist is even consulted. Without
|
||
# a floor of its own, a file by the right artist scored well enough on the
|
||
# combined ratio to be fetched no matter which of their songs it actually was.
|
||
TITLE_FLOOR = float(os.getenv("MUSE_TITLE_FLOOR", "0.5"))
|
||
|
||
# How many ranked candidates to keep per job. When a transfer fails, the next
|
||
# one is tried instead of failing the request -- "File not shared." is a
|
||
# per-file condition, so candidate two usually just works.
|
||
CANDIDATES = int(os.getenv("MUSE_CANDIDATES", "8"))
|
||
MAX_ATTEMPTS = int(os.getenv("MUSE_MAX_ATTEMPTS", "5"))
|
||
|
||
# A download that has moved no bytes for this long is not going to. The peer
|
||
# queued it and vanished, or slskd lost the transfer; either way the job used
|
||
# to sit in `downloading` forever behind a cheerful pulsing dot.
|
||
STALL_MINUTES = float(os.getenv("MUSE_STALL_MINUTES", "20"))
|
||
|
||
# Query variants to try before giving up. Each costs a real Soulseek search,
|
||
# so this escalates only when a search underdelivers -- see search_queries().
|
||
MAX_QUERIES = int(os.getenv("MUSE_MAX_QUERIES", "4"))
|
||
MIN_CANDIDATES = int(os.getenv("MUSE_MIN_CANDIDATES", "3"))
|
||
|
||
# Issuing a search is a POST that occasionally blips. Nine production failures
|
||
# were one slskd hiccup during a bulk import, every queued job dying inside the
|
||
# same second, each one permanently.
|
||
SEARCH_RETRIES = int(os.getenv("MUSE_SEARCH_RETRIES", "3"))
|
||
|
||
# Soulseek results trickle in: what you have at t=5s is much worse than t=15s.
|
||
SEARCH_POLLS = int(os.getenv("MUSE_SEARCH_POLLS", "20"))
|
||
|
||
# How many Soulseek searches may run at once.
|
||
#
|
||
# This is a hard cap, not a suggestion. Each search takes 10-40s of polling
|
||
# and real load on slskd and the Soulseek server, so firing one task per
|
||
# request — which is what an earlier version did — means pasting a 50-track
|
||
# list launches 50 concurrent searches and drowns the client. Everything goes
|
||
# through a bounded queue instead: requests return instantly, work drains at a
|
||
# civil rate.
|
||
WORKERS = int(os.getenv("MUSE_WORKERS", "3"))
|
||
|
||
# Search uses Deezer's public API: no key, no OAuth, and vastly better
|
||
# relevance than MusicBrainz for the way people actually type.
|
||
#
|
||
# MusicBrainz was tried first and rejected on measurements, not taste. It is a
|
||
# superb database but a poor search engine: "radiohead creep" returned eight
|
||
# covers and mashups before the Radiohead recording, and "bohemian rhapsody"
|
||
# never surfaced Queen at all across 90 results — because its text index
|
||
# matches "radiohead" inside a cover's *title* as readily as in the artist
|
||
# field. Client-side re-ranking can only reorder what came back, so it could
|
||
# not fix a result set that lacked the right row. Deezer returns
|
||
# "Creep / Radiohead / Pablo Honey" first for the same query.
|
||
#
|
||
# Deezer search omits track number and release date, so those are fetched from
|
||
# /track/{id} at request time — one extra call per request, not per result.
|
||
DEEZER = "https://api.deezer.com"
|
||
|
||
# Derivative versions to push down when the user did not ask for them.
|
||
# Matched on WORD BOUNDARIES, not as substrings: a bare "live" is needed to
|
||
# catch "Come As You Are (Live)", but substring matching would also penalise
|
||
# "Deliver Me" and "Olive". Same trap for "demo" vs "Demolition".
|
||
#
|
||
# A term only counts against a result when the user did NOT type it, so asking
|
||
# for "oasis live forever" or "nirvana unplugged demo" still works.
|
||
DERIVATIVE = ("cover", "covers", "karaoke", "tribute", "remix", "remixed",
|
||
"instrumental", "acapella", "originally performed",
|
||
"made popular by", "in the style of", "demo", "rehearsal",
|
||
"live", "unplugged", "re recorded", "rerecorded",
|
||
"sped up", "slowed", "reverb", "8d", "nightcore")
|
||
DERIVATIVE_RE = re.compile(r"\b(" + "|".join(DERIVATIVE) + r")\b")
|
||
|
||
# ─────────────────────────── auth ───────────────────────────
|
||
#
|
||
# OAuth against GitHub and/or Google, built so this can face the internet.
|
||
#
|
||
# FAILS CLOSED. Auth is on unless MUSE_AUTH is explicitly set to "off", and if
|
||
# it is on with incomplete config the app refuses to start. The failure mode
|
||
# worth engineering against is not a broken login page — it is a misconfigured
|
||
# deploy quietly serving an unauthenticated download client to the internet.
|
||
# You should have to *mean* it to run without auth.
|
||
#
|
||
# Sessions are server-side: the cookie carries a 256-bit random token and
|
||
# nothing else, and the database stores only its SHA-256. There is no signed
|
||
# payload to get wrong, a stolen database yields no usable cookies, and logout
|
||
# and revocation actually revoke.
|
||
#
|
||
# WHO MAY LOG IN is two rules, checked in this order:
|
||
# 1. MUSE_ALLOWED_USERS — the bootstrap allowlist. Always admitted. This is
|
||
# how the very first admin gets in when the users table is empty, and it
|
||
# is the recovery path if someone disables themselves.
|
||
# 2. A row in `users` that is not disabled — how everyone invited later gets
|
||
# in, without an env change or a redeploy.
|
||
# Anyone matching neither is rejected and the attempt is logged.
|
||
AUTH_MODE = os.getenv("MUSE_AUTH", "github").strip().lower()
|
||
ENABLED_PROVIDERS = [p for p in (x.strip() for x in AUTH_MODE.split(","))
|
||
if p and p != "off"]
|
||
|
||
# Absolute external URL, e.g. https://muse.aryan-singh.dev. Used to build the
|
||
# OAuth callback. Taken from config rather than from X-Forwarded-* headers on
|
||
# purpose: those are attacker-controllable and this is a redirect target.
|
||
BASE_URL = os.getenv("MUSE_BASE_URL", "").rstrip("/")
|
||
ALLOWED_USERS = {u.strip().lower()
|
||
for u in os.getenv("MUSE_ALLOWED_USERS", "").split(",") if u.strip()}
|
||
COOKIE_SECURE = os.getenv("MUSE_COOKIE_SECURE", "true").lower() != "false"
|
||
SESSION_DAYS = int(os.getenv("MUSE_SESSION_DAYS", "7"))
|
||
|
||
SESSION_COOKIE = "muse_session"
|
||
STATE_COOKIE = "muse_oauth_state"
|
||
|
||
# One callback URL for every provider — the provider is carried in the state
|
||
# COOKIE, not the query string, so adding Google needs no change to the GitHub
|
||
# OAuth app that is already registered against /auth/callback.
|
||
PROVIDERS: dict[str, dict] = {
|
||
"github": {
|
||
"label": "GitHub",
|
||
"authorize": "https://github.com/login/oauth/authorize",
|
||
"token": "https://github.com/login/oauth/access_token",
|
||
"scope": "read:user",
|
||
"client_id": os.getenv("MUSE_GITHUB_CLIENT_ID", "").strip(),
|
||
"client_secret": os.getenv("MUSE_GITHUB_CLIENT_SECRET", "").strip(),
|
||
"id_env": "MUSE_GITHUB_CLIENT_ID",
|
||
"secret_env": "MUSE_GITHUB_CLIENT_SECRET",
|
||
},
|
||
"google": {
|
||
"label": "Google",
|
||
"authorize": "https://accounts.google.com/o/oauth2/v2/auth",
|
||
"token": "https://oauth2.googleapis.com/token",
|
||
"scope": "openid email profile",
|
||
"client_id": os.getenv("MUSE_GOOGLE_CLIENT_ID", "").strip(),
|
||
"client_secret": os.getenv("MUSE_GOOGLE_CLIENT_SECRET", "").strip(),
|
||
"id_env": "MUSE_GOOGLE_CLIENT_ID",
|
||
"secret_env": "MUSE_GOOGLE_CLIENT_SECRET",
|
||
},
|
||
}
|
||
|
||
GH_API = "https://api.github.com/user"
|
||
GOOGLE_API = "https://openidconnect.googleapis.com/v1/userinfo"
|
||
|
||
# Paths reachable without a session. Deliberately tiny: /healthz reports a bare
|
||
# ok so the container healthcheck works without leaking configuration.
|
||
PUBLIC_PATHS = {"/healthz", "/auth/login", "/auth/callback", "/auth/logout",
|
||
"/login.css"}
|
||
|
||
ROLES = ("admin", "user", "guest")
|
||
|
||
|
||
def auth_preflight():
|
||
if AUTH_MODE == "off":
|
||
log.warning("=" * 62)
|
||
log.warning("MUSE_AUTH=off — every endpoint is UNAUTHENTICATED.")
|
||
log.warning("Never expose this to the internet in this state.")
|
||
log.warning("=" * 62)
|
||
return
|
||
if not ENABLED_PROVIDERS:
|
||
raise RuntimeError(f"MUSE_AUTH must list providers or be 'off', got {AUTH_MODE!r}")
|
||
|
||
unknown = [p for p in ENABLED_PROVIDERS if p not in PROVIDERS]
|
||
if unknown:
|
||
raise RuntimeError(
|
||
f"MUSE_AUTH names unknown provider(s) {unknown}. "
|
||
f"Valid: {', '.join(PROVIDERS)}, or 'off'.")
|
||
|
||
missing: list[str] = []
|
||
for p in ENABLED_PROVIDERS:
|
||
cfg = PROVIDERS[p]
|
||
if not cfg["client_id"]:
|
||
missing.append(cfg["id_env"])
|
||
if not cfg["client_secret"]:
|
||
missing.append(cfg["secret_env"])
|
||
if not BASE_URL:
|
||
missing.append("MUSE_BASE_URL")
|
||
# The bootstrap allowlist is only mandatory while nobody exists yet —
|
||
# otherwise a working multi-user install would be held hostage by an env
|
||
# var it has outgrown. But an EMPTY database and no allowlist means nobody
|
||
# can ever log in, which is a misconfiguration worth refusing to start on.
|
||
if not ALLOWED_USERS and not _any_users():
|
||
missing.append("MUSE_ALLOWED_USERS")
|
||
if missing:
|
||
raise RuntimeError(
|
||
"Auth is enabled but " + ", ".join(sorted(set(missing))) + " not set. "
|
||
"Refusing to start: set them, or set MUSE_AUTH=off to run without "
|
||
"authentication (LAN only).")
|
||
|
||
if not BASE_URL.startswith("https://") and COOKIE_SECURE:
|
||
log.warning("MUSE_BASE_URL is not https but cookies are Secure — the "
|
||
"browser will refuse to send them. Set MUSE_COOKIE_SECURE=false "
|
||
"only if you are testing over plain http on a LAN.")
|
||
log.info("auth: providers=%s bootstrap_allowlist=%d",
|
||
",".join(ENABLED_PROVIDERS), len(ALLOWED_USERS))
|
||
|
||
|
||
def _any_users() -> bool:
|
||
try:
|
||
with db() as c:
|
||
return c.execute("SELECT 1 FROM users LIMIT 1").fetchone() is not None
|
||
except sqlite3.Error:
|
||
return False
|
||
|
||
|
||
def _hash(tok: str) -> str:
|
||
return hashlib.sha256(tok.encode()).hexdigest()
|
||
|
||
|
||
def new_session(username: str) -> str:
|
||
token = secrets.token_urlsafe(32)
|
||
exp = datetime.now(timezone.utc) + timedelta(days=SESSION_DAYS)
|
||
with db() as c:
|
||
c.execute("INSERT INTO sessions (token_hash,username,expires_at) VALUES (?,?,?)",
|
||
(_hash(token), username, exp.isoformat()))
|
||
# Opportunistic sweep; no scheduler needed for a handful of rows.
|
||
c.execute("DELETE FROM sessions WHERE expires_at < ?",
|
||
(datetime.now(timezone.utc).isoformat(),))
|
||
return token
|
||
|
||
|
||
def may_login(username: str, row: sqlite3.Row | None,
|
||
provider: str | None = None) -> bool:
|
||
"""The single authorization rule, used at login AND on every request.
|
||
|
||
Bootstrap allowlist first, then a live, non-disabled user row. Deliberately
|
||
one function: an authorization check that runs only at login is not a
|
||
revocation mechanism, it is a delay — disabling someone would leave their
|
||
session valid for up to MUSE_SESSION_DAYS.
|
||
|
||
An invite is bound to the provider it was issued for. Invite
|
||
friend@gmail.com for Google and Google is the only way in for that
|
||
identity; there is no second, weaker door onto the same account. `provider`
|
||
is None on the per-request re-check, where the question is only whether the
|
||
account is still live.
|
||
"""
|
||
if username.lower() in ALLOWED_USERS:
|
||
return True
|
||
if not row or row["disabled"]:
|
||
return False
|
||
return provider is None or row["provider"] == provider
|
||
|
||
|
||
def current_user(request: Request) -> sqlite3.Row | None:
|
||
"""The signed-in user's row, or None. Re-validated on every request."""
|
||
tok = request.cookies.get(SESSION_COOKIE)
|
||
if not tok:
|
||
return None
|
||
with db() as c:
|
||
row = c.execute(
|
||
"SELECT username, expires_at FROM sessions WHERE token_hash=?",
|
||
(_hash(tok),)).fetchone()
|
||
if not row:
|
||
return None
|
||
try:
|
||
if datetime.fromisoformat(row["expires_at"]) < datetime.now(timezone.utc):
|
||
return None
|
||
except ValueError:
|
||
return None
|
||
u = c.execute("SELECT * FROM users WHERE username=?",
|
||
(row["username"],)).fetchone()
|
||
if not may_login(row["username"], u):
|
||
return None
|
||
if u is None:
|
||
# Allowlisted but no row — an admin deleted the bootstrap account while
|
||
# its session was still live. The allowlist is exactly the recovery
|
||
# path for that, so put the row back rather than half-authenticating
|
||
# them into a state where every role check fails.
|
||
u = upsert_user(row["username"])
|
||
log.info("auth: restored deleted allowlisted user %s", row["username"])
|
||
return u
|
||
|
||
|
||
def session_user(request: Request) -> str | None:
|
||
u = current_user(request)
|
||
return u["username"] if u else None
|
||
|
||
|
||
def user_role(request: Request) -> str:
|
||
u = current_user(request)
|
||
if u:
|
||
return u["role"]
|
||
# With auth off there is nobody to be, so everyone is an admin — otherwise
|
||
# the admin panel would be unreachable on a LAN-only install.
|
||
return "admin" if AUTH_MODE == "off" else "guest"
|
||
|
||
|
||
def require_admin(request: Request) -> str:
|
||
if AUTH_MODE == "off":
|
||
return "local"
|
||
u = current_user(request)
|
||
if not u:
|
||
raise HTTPException(401, "unauthenticated")
|
||
if u["role"] != "admin":
|
||
log.warning("admin: DENIED %s (role=%s) -> %s",
|
||
u["username"], u["role"], request.url.path)
|
||
raise HTTPException(403, "admin only")
|
||
return u["username"]
|
||
|
||
|
||
SAFE = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
|
||
NOISE = re.compile(r"[^a-z0-9]+")
|
||
YEAR = re.compile(r"^\d{4}$")
|
||
|
||
# Apostrophes vanish rather than becoming a separator, so "Don't" normalises
|
||
# onto a filename's "Dont". Everything else non-alphanumeric becomes a space,
|
||
# so "R&B" stays two tokens instead of collapsing to the unrelated word "rb".
|
||
QUOTES = re.compile("['\u2018\u2019\u02bc\u0060\"\u201c\u201d]")
|
||
|
||
# Letters NFKD leaves alone because they are atomic rather than base + accent.
|
||
FOLD = str.maketrans({"\u00f8": "o", "\u00e6": "ae", "\u0153": "oe",
|
||
"\u00f0": "d", "\u00fe": "th", "\u0142": "l",
|
||
"\u00df": "ss", "\u0111": "d", "\u0131": "i"})
|
||
|
||
# A leading track number: "02 - ", "02.", "A1 ". Anchored and bounded so it
|
||
# cannot eat a title that merely opens with a digit ("1979", "99 Problems").
|
||
TRACKNO = re.compile(r"^\s*(?:[a-d]?\d{1,3})\s*(?:[-._)\]]\s*)+", re.I)
|
||
|
||
# Technical release tags. Deliberately NOT "live", "remix" or "acoustic":
|
||
# confidence() reads those to decide whether this is the recording that was
|
||
# asked for, and stripping them here would blind it.
|
||
JUNK = re.compile(
|
||
r"[\[({]\s*(?:\d{2,4}\s*k(?:bps)?|v\d|web|cd\s*\d*|vinyl|flac|mp3|"
|
||
r"lossless|hi-?res|\d{2,3}\s*bit|\d{2}\s*khz)[^\])}]*[\])}]"
|
||
r"|\b(?:\d{3}\s*kbps|web-?dl|cdrip|vbr|cbr)\b", re.I)
|
||
|
||
# Most filesystems cap a path component at 255 bytes. Track titles can be far
|
||
# longer than that once a stranger has appended their own notes, and the
|
||
# resulting ENAMETOOLONG would fail the import after the download had finished.
|
||
MAX_COMPONENT = 150
|
||
|
||
|
||
def safe(name: str) -> str:
|
||
"""Filesystem-safe path component. Trailing dots/spaces break on some FS."""
|
||
return SAFE.sub("_", name).strip(". ")[:MAX_COMPONENT].strip() or "Unknown"
|
||
|
||
|
||
def safe_cover(url) -> str | None:
|
||
"""Artwork URL, stored and later rendered as an <img src>.
|
||
|
||
The CSP already stops a foreign host loading, so this is not an XSS gate —
|
||
it stops the field being used as free storage. `data:` is permitted by
|
||
img-src, so without a check any client could park megabytes of base64 in
|
||
the jobs table on every request.
|
||
"""
|
||
u = str(url or "").strip()
|
||
return u if u.startswith("https://") and len(u) <= 300 else None
|
||
|
||
|
||
def safe_year(year) -> str | None:
|
||
"""A year, or nothing. This lands in a directory name, so it is checked
|
||
against a whitelist rather than merely scrubbed."""
|
||
y = str(year or "").strip()
|
||
return y if YEAR.match(y) else None
|
||
|
||
|
||
def norm(s: str) -> str:
|
||
"""Lowercase, fold accents away, and reduce to alphanumeric tokens.
|
||
|
||
The accent fold is load-bearing. Deezer returns "Beyonc\u00e9", "Bj\u00f6rk" and
|
||
"Sigur R\u00f3s"; the strangers who named the files on Soulseek wrote
|
||
"Beyonce", "Bjork", "Sigur Ros". Without it the accented character is not
|
||
in [a-z0-9] and became a *space*, so "beyonce" was compared against
|
||
"beyonc " -- a containment miss and a mediocre ratio, on a track that was
|
||
sitting right there in the results.
|
||
"""
|
||
s = unicodedata.normalize("NFKD", str(s).lower().translate(FOLD))
|
||
s = "".join(c for c in s if not unicodedata.combining(c))
|
||
return NOISE.sub(" ", QUOTES.sub("", s)).strip()
|
||
|
||
|
||
def bare_name(remote_path: str) -> str:
|
||
"""A remote filename with extension, track number and release tags gone.
|
||
|
||
All of that is decoration a peer added that says nothing about which
|
||
recording the file holds. Leaving it in drags the similarity ratio down on
|
||
files that are in fact exactly right.
|
||
"""
|
||
base = remote_path.replace("\\", "/").rsplit("/", 1)[-1]
|
||
if "." in base:
|
||
base = base.rsplit(".", 1)[0]
|
||
return JUNK.sub(" ", TRACKNO.sub("", base)).strip()
|
||
|
||
|
||
def similarity(want: str, got: str) -> float:
|
||
"""Best of a sequence ratio and a token-coverage ratio, 0..1.
|
||
|
||
difflib alone is order-sensitive: "creep radiohead" against "radiohead
|
||
creep" scores far below what a human would call a match, and Soulseek
|
||
filenames put the fields in whatever order the ripper felt like. The token
|
||
measure is order-free, so taking the better of the two absorbs reordering
|
||
and interleaved junk without having to loosen MIN_MATCH.
|
||
"""
|
||
if not want or not got:
|
||
return 0.0
|
||
seq = difflib.SequenceMatcher(None, want, got).ratio()
|
||
a, b = set(want.split()), set(got.split())
|
||
return max(seq, len(a & b) / len(a) if a else 0.0)
|
||
|
||
|
||
def match_ratio(remote_path: str, artist: str, title: str) -> float:
|
||
"""How well a remote path matches the track asked for, 0..1.
|
||
|
||
Title and artist are measured separately and the title has to stand on its
|
||
own. A single ratio over the blob "artist title" let a strong artist match
|
||
carry a weak title -- the same failure the Deezer ranking keeps a dedicated
|
||
term for, where `Oasis - Live Forever` once resolved to `Cloudburst`
|
||
because the artist agreed and nothing else had to.
|
||
|
||
The artist is looked for in the whole path rather than the filename: a file
|
||
at `Radiohead/Pablo Honey/02 Creep.flac` names its artist in the folder,
|
||
which is the commonest well-organised case there is.
|
||
"""
|
||
stem = norm(bare_name(remote_path))
|
||
path = norm(remote_path.replace("\\", "/"))
|
||
nt, na = norm(title), norm(artist)
|
||
|
||
t = 1.0 if (nt and nt in stem) else similarity(nt, stem)
|
||
if t < TITLE_FLOOR:
|
||
return 0.0 # wrong song; a matching artist must not rescue it
|
||
a = 1.0 if (na and na in path) else similarity(na, path)
|
||
return 0.75 * t + 0.25 * a
|
||
|
||
|
||
# ─────────────────────────── storage ───────────────────────────
|
||
|
||
def db():
|
||
c = sqlite3.connect(DB_PATH, timeout=30)
|
||
c.row_factory = sqlite3.Row
|
||
return c
|
||
|
||
|
||
def init_db():
|
||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||
with db() as c:
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS jobs (
|
||
id TEXT PRIMARY KEY,
|
||
state TEXT NOT NULL, -- queued|searching|downloading|review|imported|failed
|
||
detail TEXT,
|
||
artist TEXT, title TEXT, album TEXT,
|
||
year TEXT, track INTEGER,
|
||
recording_mbid TEXT, release_mbid TEXT,
|
||
username TEXT, remote_file TEXT, size INTEGER,
|
||
imported_path TEXT,
|
||
created_at TEXT DEFAULT (datetime('now'))
|
||
)
|
||
""")
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS sessions (
|
||
token_hash TEXT PRIMARY KEY, -- sha256 of the cookie value
|
||
username TEXT NOT NULL,
|
||
created_at TEXT DEFAULT (datetime('now')),
|
||
expires_at TEXT NOT NULL
|
||
)
|
||
""")
|
||
# Users exist as rows even while the allowlist is a single name in an
|
||
# env var. Roles and quotas are read from here from the start, so
|
||
# opening this up to other people later is an INSERT and a config
|
||
# change rather than a schema migration against live job history.
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS users (
|
||
username TEXT PRIMARY KEY COLLATE NOCASE,
|
||
provider TEXT NOT NULL DEFAULT 'github',
|
||
role TEXT NOT NULL DEFAULT 'user', -- admin|user|guest
|
||
display TEXT,
|
||
avatar TEXT,
|
||
quota INTEGER NOT NULL DEFAULT 0, -- requests/month, 0 = unlimited
|
||
disabled INTEGER NOT NULL DEFAULT 0,
|
||
created_at TEXT DEFAULT (datetime('now')),
|
||
last_seen TEXT
|
||
)
|
||
""")
|
||
# Added after the first release, so existing databases need it grafted
|
||
# on rather than recreated -- job history is the whole point of the file.
|
||
cols = {r["name"] for r in c.execute("PRAGMA table_info(jobs)").fetchall()}
|
||
for col, decl in (("requested_by", "TEXT"),
|
||
("cover", "TEXT"),
|
||
("genre", "TEXT"),
|
||
("duration", "INTEGER"),
|
||
("confidence", "REAL"),
|
||
("match_why", "TEXT"),
|
||
("reviewed_by", "TEXT"),
|
||
# The candidate ladder. Keeping the runners-up on the
|
||
# row is what lets a rejected transfer fall through to
|
||
# the next peer minutes later, in a different task,
|
||
# after the search result set is long gone.
|
||
("candidates", "TEXT"),
|
||
("tried", "TEXT"),
|
||
("attempts", "INTEGER NOT NULL DEFAULT 0"),
|
||
("started_at", "TEXT")):
|
||
if col not in cols:
|
||
c.execute(f"ALTER TABLE jobs ADD COLUMN {col} {decl}")
|
||
ucols = {r["name"] for r in c.execute("PRAGMA table_info(users)").fetchall()}
|
||
for col, decl in (("storage_quota", "INTEGER NOT NULL DEFAULT 0"),
|
||
("invited_by", "TEXT"),
|
||
("email", "TEXT"),
|
||
("notify", "INTEGER NOT NULL DEFAULT 1")):
|
||
if col not in ucols:
|
||
c.execute(f"ALTER TABLE users ADD COLUMN {col} {decl}")
|
||
c.execute("CREATE INDEX IF NOT EXISTS jobs_by_user ON jobs (requested_by)")
|
||
c.execute("CREATE INDEX IF NOT EXISTS jobs_by_state ON jobs (state)")
|
||
c.execute("CREATE INDEX IF NOT EXISTS jobs_by_created ON jobs (created_at)")
|
||
|
||
|
||
def upsert_user(username: str, avatar: str = "", display: str = "",
|
||
provider: str = "github") -> sqlite3.Row:
|
||
"""Record the sign-in.
|
||
|
||
The first user to ever log in becomes admin — no bootstrap ceremony, no
|
||
default password to forget. Everyone after that is a plain user until an
|
||
admin says otherwise; role is never touched on subsequent logins, or every
|
||
sign-in would silently undo a demotion.
|
||
"""
|
||
with db() as c:
|
||
first = c.execute("SELECT COUNT(*) n FROM users").fetchone()["n"] == 0
|
||
c.execute(
|
||
"INSERT INTO users (username,provider,role,display,avatar,last_seen) "
|
||
"VALUES (?,?,?,?,?,datetime('now')) "
|
||
"ON CONFLICT(username) DO UPDATE SET "
|
||
" last_seen=datetime('now'), "
|
||
# provider is NOT refreshed from the login. It is part of the
|
||
# identity an admin invited, and may_login now enforces it — so
|
||
# overwriting it here would let a successful login silently
|
||
# rewrite the very rule that permitted it.
|
||
" avatar=COALESCE(NULLIF(excluded.avatar,''), users.avatar), "
|
||
" display=COALESCE(NULLIF(excluded.display,''), users.display)",
|
||
(username, provider, "admin" if first else "user", display, avatar))
|
||
return c.execute("SELECT * FROM users WHERE username=?", (username,)).fetchone()
|
||
|
||
|
||
def set_state(job_id: str, state: str, detail: str = "", **cols):
|
||
sets = ", ".join(f"{k}=?" for k in cols)
|
||
sql = f"UPDATE jobs SET state=?, detail=?{', ' + sets if sets else ''} WHERE id=?"
|
||
with db() as c:
|
||
c.execute(sql, [state, detail, *cols.values(), job_id])
|
||
log.info("job %s -> %s %s", job_id[:8], state, detail)
|
||
|
||
|
||
# ─────────────────────────── slskd ───────────────────────────
|
||
|
||
def slskd_client(timeout: float = 30.0) -> httpx.AsyncClient:
|
||
"""30s suits searches and transfer polling, which are legitimately slow.
|
||
|
||
Status probes pass something much shorter: they run on a UI poll, and a
|
||
probe that outlasts its own poll interval stacks requests and leaves the
|
||
admin panel on skeletons for the whole timeout — precisely when slskd is
|
||
down and you opened the panel to find out why.
|
||
"""
|
||
return httpx.AsyncClient(
|
||
base_url=SLSKD_URL,
|
||
headers={"X-API-Key": SLSKD_API_KEY},
|
||
timeout=httpx.Timeout(timeout),
|
||
)
|
||
|
||
|
||
# slskd is on the LAN, so a reachable instance answers in milliseconds. Long
|
||
# enough to absorb a slow moment, short enough to stay under the 5s UI poll.
|
||
PROBE_TIMEOUT = 4.0
|
||
|
||
|
||
def errstr(e: Exception) -> str:
|
||
"""httpx's timeout and connection errors often str() to an empty string,
|
||
which would render as 'slskd unreachable — ' with nothing after it."""
|
||
return (str(e) or type(e).__name__)[:120]
|
||
|
||
|
||
def ext_of(remote_path: str) -> str:
|
||
# Soulseek paths are Windows-style; the API's `extension` field comes back
|
||
# empty in practice, so derive it ourselves.
|
||
return remote_path.rsplit(".", 1)[-1].lower() if "." in remote_path else ""
|
||
|
||
|
||
def confidence(filename: str, artist: str, title: str,
|
||
album: str | None = None) -> tuple[float, list[str]]:
|
||
"""How sure are we this file is the song that was asked for, 0..1.
|
||
|
||
Deliberately separate from score(). score() decides *which* candidate to
|
||
fetch and is right to weigh peer speed and queue length; none of that says
|
||
anything about whether the file is the correct recording. Mixing the two
|
||
would let a fast peer raise the confidence of a wrong match.
|
||
|
||
Returns the score and the human-readable reasons behind it, because a
|
||
number alone gives the reviewer nothing to act on.
|
||
"""
|
||
stem = norm(bare_name(filename))
|
||
path = norm(filename.replace("\\", "/"))
|
||
na, nt = norm(artist), norm(title)
|
||
why: list[str] = []
|
||
|
||
# Title is the load-bearing signal: a wrong title is a wrong song, whereas
|
||
# a missing artist is routine in files organised into per-artist folders.
|
||
t_ratio = similarity(nt, stem)
|
||
if nt and nt in stem:
|
||
t_score, t_ratio = 1.0, max(t_ratio, 1.0)
|
||
why.append("title matches")
|
||
elif t_ratio >= 0.75:
|
||
t_score = t_ratio
|
||
why.append(f"title close ({t_ratio:.0%})")
|
||
else:
|
||
t_score = t_ratio
|
||
why.append(f"title differs ({t_ratio:.0%})")
|
||
|
||
# Artist may legitimately live in the path rather than the filename.
|
||
if na and na in stem:
|
||
a_score = 1.0
|
||
why.append("artist in filename")
|
||
elif na and na in path:
|
||
a_score = 0.9
|
||
why.append("artist in folder")
|
||
elif na and set(na.split()) & set(path.split()):
|
||
a_score = 0.5
|
||
why.append("artist partly matches")
|
||
else:
|
||
a_score = 0.0
|
||
why.append("artist not found in path")
|
||
|
||
# 0.75/0.25 rather than an even split. A file named exactly right but
|
||
# sitting in an unorganised folder is the commonest honest case there is,
|
||
# and it has to land in the review band, not below it — 0.65 would have
|
||
# binned it alongside genuinely wrong songs.
|
||
c = 0.75 * t_score + 0.25 * a_score
|
||
|
||
if album and norm(album) and norm(album) in path:
|
||
c = min(1.0, c + 0.05)
|
||
why.append("album in path")
|
||
|
||
# A live or remix version of the right song is still the wrong file when
|
||
# the request was for the studio recording, and that is exactly the case a
|
||
# reviewer needs to see rather than have silently imported. So the penalty
|
||
# caps confidence below AUTO instead of merely scaling it: "a human must
|
||
# decide" is the whole meaning of the flag, and a perfect artist+title
|
||
# match should not be able to auto-import past it.
|
||
variants = [w for w in ("live", "remix", "cover", "karaoke", "instrumental",
|
||
"acoustic", "demo", "edit", "radio edit")
|
||
if re.search(rf"\b{w}\b", stem) and w not in nt]
|
||
if variants:
|
||
c = min(c * 0.8, AUTO_CONFIDENCE - 0.1)
|
||
why.append(f"looks like a {variants[0]} version, which was not requested")
|
||
|
||
return round(max(0.0, min(1.0, c)), 3), why
|
||
|
||
|
||
# >= AUTO imports without asking. Below REVIEW the match is bad enough that a
|
||
# human almost certainly rejects it, but it is still held rather than binned:
|
||
# the file is already downloaded and only the reviewer knows if it is right.
|
||
AUTO_CONFIDENCE = float(os.getenv("MUSE_AUTO_CONFIDENCE", "0.95"))
|
||
REVIEW_CONFIDENCE = float(os.getenv("MUSE_REVIEW_CONFIDENCE", "0.70"))
|
||
|
||
|
||
def score(candidate: dict, peer: dict, artist: str, title: str) -> float | None:
|
||
"""Rank one remote file. None = reject.
|
||
|
||
Ordering intent: actually-gettable first, then correct format, then
|
||
filename plausibility, then bitrate.
|
||
|
||
Availability leads deliberately. It used to sit *below* format -- 100
|
||
points per format rank against 60 for a free upload slot -- which meant a
|
||
FLAC behind two hundred queued people always beat an MP3 available right
|
||
now. Whether the peer can serve you is the single most predictive signal
|
||
for whether the job ever finishes, so it outranks the preference for a
|
||
better file. Format is the tiebreak within an availability tier.
|
||
"""
|
||
ext = ext_of(candidate["filename"])
|
||
if ext not in FORMATS:
|
||
return None
|
||
|
||
ratio = match_ratio(candidate["filename"], artist, title)
|
||
if ratio < MIN_MATCH:
|
||
return None
|
||
|
||
s = 0.0
|
||
s += 220 if peer.get("hasFreeUploadSlot") else 0
|
||
# Unsaturated: the old `max(0, 40 - queueLength)` scored a 40-deep queue
|
||
# and a 5000-deep one identically, which is the difference between waiting
|
||
# a minute and never being served at all.
|
||
s += 120 / (1 + peer.get("queueLength", 0) / 8)
|
||
s += min(peer.get("uploadSpeed", 0) / 50_000, 20)
|
||
s += (len(FORMATS) - FORMATS.index(ext)) * 60 # format preference
|
||
s += ratio * 90
|
||
s += min(candidate.get("bitRate") or 0, 320) / 32 # 0-10
|
||
return s
|
||
|
||
|
||
PAREN = re.compile(r"[\[(\{][^\])}]*[\])}]")
|
||
FEAT = re.compile(r"\b(?:feat|ft|featuring)\b\.?.*$", re.I)
|
||
ARTICLE = re.compile(r"^(?:the|a|an|los|las|les)\s+", re.I)
|
||
|
||
|
||
class SearchUnavailable(Exception):
|
||
"""slskd could not accept a search. Transient, not a bad request."""
|
||
|
||
|
||
def search_queries(artist: str, title: str) -> list[str]:
|
||
"""Query variants to try, loosest last.
|
||
|
||
The Soulseek server filters searches containing certain artist names. That
|
||
is measured, not theorised: on this network "Michael Jackson Billie Jean"
|
||
and bare "Michael Jackson" both return zero responses, while "Billie Jean"
|
||
returns 1083 files. Sending only `artist title` -- all this used to do --
|
||
means every request for a filtered artist dies reporting "searched 0
|
||
files", and that is exactly the roster of names in the failed jobs:
|
||
Michael Jackson, The Weeknd, Kanye West, Frank Ocean.
|
||
|
||
So the ladder ends at the title alone, and match_ratio() then recognises
|
||
the artist from the folder path instead of from the query. The middle
|
||
rungs handle the ordinary case where Deezer's metadata is simply more
|
||
decorated than the filename a stranger typed.
|
||
"""
|
||
out: list[str] = []
|
||
seen: set[str] = set()
|
||
|
||
def add(*parts: str) -> None:
|
||
q = " ".join(p for p in parts if p).strip()
|
||
# Dedupe on the normalised form, so "Creep (Remastered)" and "Creep"
|
||
# are recognised as one query once the punctuation is gone.
|
||
k = norm(q)
|
||
if q and k and k not in seen:
|
||
seen.add(k)
|
||
out.append(q)
|
||
|
||
plain_t = PAREN.sub(" ", title).strip() or title
|
||
lean_t = FEAT.sub(" ", plain_t).strip() or plain_t
|
||
head_t = lean_t.split("/")[0].strip() or lean_t # "A / B" -> "A"
|
||
lean_a = ARTICLE.sub("", FEAT.sub(" ", artist).strip()).strip()
|
||
|
||
add(artist, title) # what was sent before, and still best when it works
|
||
add(artist, lean_t) # decoration and featuring credits gone
|
||
add(lean_a, head_t) # leading article gone, split title
|
||
add(head_t) # title alone -- this is the rung that beats the filter
|
||
add(lean_t)
|
||
return out[:MAX_QUERIES]
|
||
|
||
|
||
async def run_search(sc: httpx.AsyncClient, text: str) -> dict:
|
||
"""Issue one Soulseek search and poll it to completion.
|
||
|
||
The search is deleted only once it has actually finished. Deleting one
|
||
still InProgress makes slskd throw an optimistic-concurrency error inside
|
||
its own SearchService and lose the record; the old code deleted
|
||
unconditionally after its 40s budget, so every slow search corrupted a row.
|
||
"""
|
||
search_id = str(uuid.uuid4())
|
||
for attempt in range(SEARCH_RETRIES):
|
||
try:
|
||
r = await sc.post("/api/v0/searches",
|
||
json={"id": search_id, "searchText": text})
|
||
r.raise_for_status()
|
||
break
|
||
except Exception as e:
|
||
if attempt == SEARCH_RETRIES - 1:
|
||
raise SearchUnavailable(errstr(e)) from e
|
||
await asyncio.sleep(2 * (attempt + 1))
|
||
|
||
results: dict = {}
|
||
finished = False
|
||
for _ in range(SEARCH_POLLS):
|
||
await asyncio.sleep(2)
|
||
try:
|
||
rr = await sc.get(f"/api/v0/searches/{search_id}",
|
||
params={"includeResponses": "true"})
|
||
rr.raise_for_status()
|
||
results = rr.json()
|
||
except Exception as e:
|
||
# Swallowed silently before, which made a broken slskd look
|
||
# identical to a track nobody happens to share.
|
||
log.debug("poll %r: %s", text, errstr(e))
|
||
continue
|
||
if str(results.get("state", "")).startswith("Completed"):
|
||
finished = True
|
||
break
|
||
|
||
if finished:
|
||
try:
|
||
await sc.delete(f"/api/v0/searches/{search_id}")
|
||
except Exception as e:
|
||
log.debug("cleanup %r: %s", text, errstr(e))
|
||
else:
|
||
log.info("search %r unfinished after %ds, leaving it for slskd to close",
|
||
text, SEARCH_POLLS * 2)
|
||
return results
|
||
|
||
|
||
def spread_by_peer(ranked: list[tuple[float, dict, dict]]) -> list[dict]:
|
||
"""Order candidates so consecutive attempts land on different peers.
|
||
|
||
Straight score order routinely stacks three files from one well-stocked
|
||
user at the top; if that user is offline all three attempts burn on the
|
||
same dead connection. Peers arrive best-first, so round-robining them keeps
|
||
the strongest candidate first and still guarantees variety behind it.
|
||
"""
|
||
by_peer: dict[str, list[dict]] = {}
|
||
for s, peer, f in ranked:
|
||
by_peer.setdefault(peer["username"], []).append({
|
||
"username": peer["username"],
|
||
"filename": f["filename"],
|
||
"size": f.get("size") or 0,
|
||
"score": round(s, 1),
|
||
})
|
||
queues = list(by_peer.values())
|
||
out: list[dict] = []
|
||
while queues:
|
||
for q in list(queues):
|
||
out.append(q.pop(0))
|
||
if not q:
|
||
queues.remove(q)
|
||
return out
|
||
|
||
|
||
def no_candidate_detail(seen: int, stats: "collections.Counter",
|
||
queries: list[str]) -> str:
|
||
"""Say which filter emptied the results, not merely that they are empty.
|
||
|
||
One message used to cover four unrelated situations, and 28 of 29 real
|
||
occurrences turned out to be the search itself coming back empty -- which
|
||
is now stated outright instead of being reported as a matching failure.
|
||
"""
|
||
if seen == 0:
|
||
tried = ", ".join(repr(q) for q in queries)
|
||
word = "y" if len(queries) == 1 else "ies"
|
||
return f"Soulseek returned nothing for {len(queries)} quer{word}: {tried}"
|
||
bits = []
|
||
if stats["format"]:
|
||
bits.append(f"{stats['format']} not {'/'.join(FORMATS)}")
|
||
if stats["weak"]:
|
||
bits.append(f"{stats['weak']} did not match the title")
|
||
if stats["locked"]:
|
||
bits.append(f"{stats['locked']} locked")
|
||
return f"{seen} files found, none usable: " + ", ".join(bits)
|
||
|
||
|
||
def exhausted_detail(tried: list[dict], attempts: int) -> str:
|
||
"""Summarise the whole ladder instead of overwriting it with the last error.
|
||
|
||
Counts `tried`, not `attempts`. They agree for anything this version
|
||
enqueued, but a row carried over from before the ladder existed has
|
||
attempts=0 with a real failure recorded against it, which rendered as the
|
||
nonsense "tried 0 peers: 1x no bytes in 20m". The list is the evidence.
|
||
"""
|
||
if not tried:
|
||
return "no candidate could be enqueued"
|
||
n_peers = len({t.get("username") for t in tried if t.get("username")}) or len(tried)
|
||
reasons = collections.Counter(t["reason"].split(":")[0] for t in tried)
|
||
plural = "" if n_peers == 1 else "s"
|
||
joined = ", ".join(f"{n}x {r}" for r, n in reasons.most_common(3))
|
||
return f"tried {n_peers} peer{plural}: {joined}"
|
||
|
||
|
||
async def try_next_candidate(job_id: str, artist: str, title: str,
|
||
album: str | None = None) -> bool:
|
||
"""Enqueue the next-best candidate. False once the ladder is exhausted.
|
||
|
||
This is the difference between a request that survives one bad peer and one
|
||
that does not. "Transfer rejected: File not shared." is a *per-file*
|
||
condition -- the peer's share index is stale -- so candidate two almost
|
||
always works, and slskd's 500 on an unreachable peer is the same shape.
|
||
"""
|
||
with db() as c:
|
||
row = c.execute(
|
||
"SELECT candidates, tried, attempts FROM jobs WHERE id=?",
|
||
(job_id,)).fetchone()
|
||
if not row:
|
||
return False
|
||
cands = json.loads(row["candidates"] or "[]")
|
||
tried = json.loads(row["tried"] or "[]")
|
||
attempts = row["attempts"] or 0
|
||
|
||
while cands and attempts < MAX_ATTEMPTS:
|
||
cand = cands.pop(0)
|
||
attempts += 1
|
||
try:
|
||
async with slskd_client() as sc:
|
||
er = await sc.post(
|
||
f"/api/v0/transfers/downloads/{cand['username']}",
|
||
json=[{"filename": cand["filename"],
|
||
"size": cand["size"]}])
|
||
er.raise_for_status()
|
||
except Exception as e:
|
||
tried.append({"username": cand["username"],
|
||
"filename": cand["filename"],
|
||
"reason": f"could not reach peer: {errstr(e)}"})
|
||
continue
|
||
|
||
conf, why = confidence(cand["filename"], artist, title, album)
|
||
with db() as c:
|
||
c.execute(
|
||
"UPDATE jobs SET candidates=?, tried=?, attempts=? WHERE id=?",
|
||
(json.dumps(cands), json.dumps(tried), attempts, job_id))
|
||
extra = f", attempt {attempts}" if attempts > 1 else ""
|
||
set_state(job_id, "downloading",
|
||
f"from {cand['username']} (score {cand['score']:.0f}{extra})",
|
||
username=cand["username"], remote_file=cand["filename"],
|
||
size=cand["size"], confidence=conf, match_why="; ".join(why),
|
||
started_at=datetime.now(timezone.utc).isoformat())
|
||
return True
|
||
|
||
with db() as c:
|
||
c.execute("UPDATE jobs SET candidates=?, tried=?, attempts=? WHERE id=?",
|
||
(json.dumps(cands), json.dumps(tried), attempts, job_id))
|
||
set_state(job_id, "failed", exhausted_detail(tried, attempts))
|
||
return False
|
||
|
||
|
||
async def pick_and_enqueue(job_id: str, artist: str, title: str,
|
||
album: str | None = None):
|
||
"""Search Soulseek for one track and queue the best candidate.
|
||
|
||
Builds a ranked candidate *list* rather than a single winner, so a peer
|
||
that rejects, times out or cannot be reached costs one attempt instead of
|
||
the entire request.
|
||
"""
|
||
queries = search_queries(artist, title)
|
||
pool: dict[tuple[str, str], tuple[float, dict, dict]] = {}
|
||
seen = 0
|
||
stats: "collections.Counter" = collections.Counter()
|
||
|
||
async with slskd_client() as sc:
|
||
for i, q in enumerate(queries):
|
||
try:
|
||
results = await run_search(sc, q)
|
||
except SearchUnavailable as e:
|
||
set_state(job_id, "failed",
|
||
f"slskd would not accept a search: {e}")
|
||
return
|
||
|
||
for peer in results.get("responses", []):
|
||
for f in peer.get("files", []):
|
||
seen += 1
|
||
if f.get("isLocked"):
|
||
stats["locked"] += 1
|
||
continue
|
||
if ext_of(f["filename"]) not in FORMATS:
|
||
stats["format"] += 1
|
||
continue
|
||
sc_ = score(f, peer, artist, title)
|
||
if sc_ is None:
|
||
stats["weak"] += 1
|
||
continue
|
||
key = (peer["username"], f["filename"])
|
||
if key not in pool or pool[key][0] < sc_:
|
||
pool[key] = (sc_, peer, f)
|
||
|
||
log.info("job %s q%d/%d %r -> %d responses, %d files, %d candidates",
|
||
job_id[:8], i + 1, len(queries), q,
|
||
len(results.get("responses", [])),
|
||
results.get("fileCount", 0), len(pool))
|
||
# Escalate only on underdelivery: the common case stays one search,
|
||
# which is what keeps a 50-track paste civil to slskd.
|
||
if len(pool) >= MIN_CANDIDATES:
|
||
break
|
||
|
||
if not pool:
|
||
set_state(job_id, "failed", no_candidate_detail(seen, stats, queries))
|
||
return
|
||
|
||
ranked = sorted(pool.values(), key=lambda r: -r[0])
|
||
cands = spread_by_peer(ranked)[:CANDIDATES]
|
||
with db() as c:
|
||
c.execute("UPDATE jobs SET candidates=?, tried=?, attempts=0 WHERE id=?",
|
||
(json.dumps(cands), "[]", job_id))
|
||
await try_next_candidate(job_id, artist, title, album)
|
||
|
||
|
||
# ─────────────────────────── email ───────────────────────────
|
||
#
|
||
# Optional and off unless SMTP_HOST is set. A homelab tool that cannot send
|
||
# mail should still import music, so every failure here is logged and
|
||
# swallowed — a notification is never worth failing an import over.
|
||
|
||
SMTP_HOST = os.getenv("MUSE_SMTP_HOST", "").strip()
|
||
SMTP_PORT = int(os.getenv("MUSE_SMTP_PORT", "587"))
|
||
SMTP_USER = os.getenv("MUSE_SMTP_USER", "").strip()
|
||
SMTP_PASS = os.getenv("MUSE_SMTP_PASS", "")
|
||
SMTP_FROM = os.getenv("MUSE_SMTP_FROM", "").strip() or SMTP_USER
|
||
SMTP_TLS = os.getenv("MUSE_SMTP_TLS", "starttls").strip().lower()
|
||
MAIL_ON = bool(SMTP_HOST and SMTP_FROM)
|
||
|
||
|
||
def _send_sync(to: str, subject: str, body: str):
|
||
import smtplib
|
||
from email.message import EmailMessage
|
||
|
||
msg = EmailMessage()
|
||
msg["From"], msg["To"], msg["Subject"] = SMTP_FROM, to, subject
|
||
msg.set_content(body)
|
||
if SMTP_TLS == "ssl":
|
||
s = smtplib.SMTP_SSL(SMTP_HOST, SMTP_PORT, timeout=20)
|
||
else:
|
||
s = smtplib.SMTP(SMTP_HOST, SMTP_PORT, timeout=20)
|
||
with s:
|
||
if SMTP_TLS == "starttls":
|
||
s.starttls()
|
||
if SMTP_USER:
|
||
s.login(SMTP_USER, SMTP_PASS)
|
||
s.send_message(msg)
|
||
|
||
|
||
async def send_mail(to: str, subject: str, body: str):
|
||
"""Fire-and-forget. smtplib is blocking, so it runs off the event loop —
|
||
otherwise a slow relay would stall the import worker for every job."""
|
||
if not (MAIL_ON and to):
|
||
return
|
||
try:
|
||
await asyncio.to_thread(_send_sync, to, subject, body)
|
||
log.info("mail: sent %r to %s", subject, to)
|
||
except Exception as e:
|
||
log.warning("mail: sending to %s failed: %s", to, errstr(e))
|
||
|
||
|
||
def recipient(username: str | None) -> str | None:
|
||
"""Where to write to, and whether they want to hear from us at all."""
|
||
if not username:
|
||
return None
|
||
with db() as c:
|
||
r = c.execute("SELECT email, notify, provider FROM users WHERE username=?",
|
||
(username,)).fetchone()
|
||
if not r or not r["notify"]:
|
||
return None
|
||
# A Google identity IS an email address, so there is nothing to configure
|
||
# for those accounts unless they want mail somewhere else.
|
||
if r["email"]:
|
||
return r["email"]
|
||
return username if r["provider"] == "google" and "@" in username else None
|
||
|
||
|
||
async def notify_invited(row: sqlite3.Row, by: str, to: str):
|
||
"""Tell someone they've been invited.
|
||
|
||
There is no token in this link and none is needed. The row already names
|
||
the exact identity that may sign in, and Google will not hand us an
|
||
unverified address — so possession of the email is not what authorises
|
||
anyone, and a leaked link grants nothing. It is a courtesy, not a
|
||
credential, which is also why it can be resent freely.
|
||
"""
|
||
label = PROVIDERS.get(row["provider"], {}).get("label", row["provider"])
|
||
where = BASE_URL or "your muse instance"
|
||
await send_mail(
|
||
to, f"{by} invited you to muse",
|
||
f"You've been given access to muse — request a song, and it turns up\n"
|
||
f"in the music library, correctly tagged.\n\n"
|
||
f" {where}\n\n"
|
||
f"Sign in with {label}, using the account for {row['username']}.\n"
|
||
f"That address is what authorises you, so it has to be that one.\n\n"
|
||
f"Playback lives in Navidrome, not here — muse only decides what gets\n"
|
||
f"added to the library.\n")
|
||
|
||
|
||
async def notify_imported(job, dest):
|
||
to = recipient(job["requested_by"])
|
||
if not to:
|
||
return
|
||
album = job["album"] or "Singles"
|
||
if job["year"]:
|
||
album += f" ({job['year']})"
|
||
await send_mail(
|
||
to, f"muse: {job['title']} is in your library",
|
||
f"{job['artist']} — {job['title']}\n{album}\n\n"
|
||
f"Filed as: {dest}\n\nIt should appear in Navidrome on the next scan.\n")
|
||
|
||
|
||
async def notify_review(job):
|
||
to = recipient(job["requested_by"])
|
||
if not to:
|
||
return
|
||
await send_mail(
|
||
to, f"muse: {job['title']} needs a look",
|
||
f"{job['artist']} — {job['title']}\n\n"
|
||
f"The best match found was only {(job['confidence'] or 0):.0%} confident, "
|
||
f"so it has not been added to your library yet.\n\n"
|
||
f"Why: {job['match_why'] or 'unknown'}\n"
|
||
f"File: {job['remote_file']}\n\n"
|
||
f"Approve or reject it under Review in muse.\n")
|
||
|
||
|
||
# ─────────────────────────── import ───────────────────────────
|
||
|
||
def tag_and_file(job: sqlite3.Row, src: Path) -> Path:
|
||
"""Write known-good tags, then hardlink into the library."""
|
||
from mutagen import File as MFile
|
||
from mutagen.easyid3 import EasyID3
|
||
from mutagen.flac import FLAC
|
||
from mutagen.mp3 import MP3
|
||
|
||
try:
|
||
if src.suffix.lower() == ".flac":
|
||
audio = FLAC(src)
|
||
else:
|
||
try:
|
||
audio = EasyID3(src)
|
||
except Exception:
|
||
m = MP3(src)
|
||
m.add_tags()
|
||
m.save()
|
||
audio = EasyID3(src)
|
||
|
||
audio["title"] = job["title"]
|
||
audio["artist"] = job["artist"]
|
||
audio["albumartist"] = job["artist"]
|
||
if job["album"]:
|
||
audio["album"] = job["album"]
|
||
if job["year"]:
|
||
audio["date"] = str(job["year"])
|
||
if job["track"]:
|
||
audio["tracknumber"] = str(job["track"])
|
||
# Deliberately NOT writing the Deezer id into musicbrainz_trackid --
|
||
# that field means a specific thing and a wrong value there would
|
||
# mislead Picard, beets, or anything else that later reads this file.
|
||
audio.save()
|
||
except Exception as e:
|
||
# A tagging failure must not strand the file — Navidrome can still
|
||
# read whatever tags it shipped with.
|
||
log.warning("tagging failed for %s: %s", src.name, e)
|
||
|
||
album = f"{safe(job['album'] or 'Singles')}"
|
||
year = safe_year(job["year"])
|
||
if year:
|
||
album += f" ({year})"
|
||
dest_dir = LIBRARY_DIR / safe(job["artist"]) / album
|
||
|
||
num = f"{int(job['track']):02d} - " if job["track"] else ""
|
||
dest = dest_dir / f"{num}{safe(job['title'])}{src.suffix.lower()}"
|
||
|
||
# Belt and braces. Every component above is either safe()'d or matched
|
||
# against a whitelist, but that is a property of five call sites that must
|
||
# each stay correct; this is one check that cannot be forgotten. It exists
|
||
# because `year` was for a while appended AFTER safe() ran, so a request
|
||
# with year="../../../../tmp/x" wrote outside the library.
|
||
root = LIBRARY_DIR.resolve()
|
||
if not Path(os.path.normpath(dest)).resolve().is_relative_to(root):
|
||
raise ValueError(f"refusing to write outside the library: {dest}")
|
||
|
||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
if dest.exists():
|
||
return dest
|
||
try:
|
||
os.link(src, dest) # same filesystem -> free, and keeps sharing
|
||
except OSError:
|
||
import shutil
|
||
shutil.copy2(src, dest) # cross-device fallback
|
||
return dest
|
||
|
||
|
||
def downloaded_index() -> dict[str, Path]:
|
||
"""Every completed file, by basename.
|
||
|
||
slskd nests a download under the remote's parent directory name, so the
|
||
basename is the only reliable key. Built once per poll cycle and shared by
|
||
every job in it: the previous version walked the whole tree separately for
|
||
each in-flight job, every ten seconds, which on a large downloads
|
||
directory is a lot of stat() calls to answer the same question.
|
||
"""
|
||
out: dict[str, Path] = {}
|
||
try:
|
||
for p in DOWNLOAD_DIR.rglob("*"):
|
||
if p.is_file():
|
||
out.setdefault(p.name, p)
|
||
except OSError as e:
|
||
log.warning("cannot walk %s: %s", DOWNLOAD_DIR, errstr(e))
|
||
return out
|
||
|
||
|
||
# ─────────────────────────── search queue ───────────────────────────
|
||
|
||
work_q: "asyncio.Queue[str]" = asyncio.Queue()
|
||
|
||
|
||
async def search_worker(n: int):
|
||
"""Drain the search queue, one job at a time, WORKERS of these running."""
|
||
while True:
|
||
job_id = await work_q.get()
|
||
try:
|
||
with db() as c:
|
||
job = c.execute("SELECT * FROM jobs WHERE id=?", (job_id,)).fetchone()
|
||
if job and job["state"] in ("queued", "searching"):
|
||
set_state(job_id, "searching", "")
|
||
await pick_and_enqueue(job_id, job["artist"], job["title"],
|
||
job["album"])
|
||
except Exception as e:
|
||
log.warning("worker %d: %s", n, e)
|
||
try:
|
||
set_state(job_id, "failed", str(e)[:200])
|
||
except Exception:
|
||
pass
|
||
finally:
|
||
work_q.task_done()
|
||
|
||
|
||
def transfer_index(payload: list) -> dict[str, dict]:
|
||
"""Flatten slskd's user -> directories -> files nesting.
|
||
|
||
Keeps the whole file object, not just its state. `exception` is where the
|
||
literal "Transfer rejected: File not shared." lives, and discarding it was
|
||
why a failed job could only ever say "transfer Completed, Rejected" with no
|
||
reason a person could act on.
|
||
"""
|
||
out: dict[str, dict] = {}
|
||
for user in payload:
|
||
for d in user.get("directories", []):
|
||
for f in d.get("files", []):
|
||
out[f.get("filename", "")] = f
|
||
return out
|
||
|
||
|
||
def stalled_for(job: sqlite3.Row) -> float:
|
||
"""Minutes since this transfer last had anything to show for itself.
|
||
|
||
Falls back to created_at, which is what makes this work on rows written
|
||
before started_at existed. Those are `downloading` with no clock at all,
|
||
so returning zero for them would exempt precisely the jobs that have been
|
||
stuck the longest -- the ones this check exists to catch.
|
||
"""
|
||
keys = job.keys()
|
||
started = ((job["started_at"] if "started_at" in keys else None)
|
||
or (job["created_at"] if "created_at" in keys else None))
|
||
if not started:
|
||
return 0.0
|
||
try:
|
||
t = datetime.fromisoformat(started)
|
||
except ValueError:
|
||
return 0.0
|
||
if t.tzinfo is None:
|
||
t = t.replace(tzinfo=timezone.utc)
|
||
return (datetime.now(timezone.utc) - t).total_seconds() / 60.0
|
||
|
||
|
||
async def abandon(job: sqlite3.Row, reason: str) -> None:
|
||
"""Give up on the current peer and fall through to the next candidate."""
|
||
with db() as c:
|
||
row = c.execute("SELECT tried FROM jobs WHERE id=?",
|
||
(job["id"],)).fetchone()
|
||
tried = json.loads((row["tried"] if row else None) or "[]")
|
||
tried.append({"username": job["username"], "filename": job["remote_file"],
|
||
"reason": reason})
|
||
with db() as c:
|
||
c.execute("UPDATE jobs SET tried=? WHERE id=?",
|
||
(json.dumps(tried), job["id"]))
|
||
log.info("job %s giving up on %s: %s", job["id"][:8], job["username"], reason)
|
||
await try_next_candidate(job["id"], job["artist"], job["title"], job["album"])
|
||
|
||
|
||
async def cancel_transfer(username: str, filename: str) -> None:
|
||
"""Ask slskd to drop a transfer we have stopped waiting for."""
|
||
try:
|
||
async with slskd_client(timeout=PROBE_TIMEOUT) as sc:
|
||
await sc.delete(f"/api/v0/transfers/downloads/{username}",
|
||
params={"filename": filename})
|
||
except Exception as e:
|
||
log.debug("cancel %s/%s: %s", username, filename, errstr(e))
|
||
|
||
|
||
async def import_worker():
|
||
"""Poll slskd for in-flight downloads and import the finished ones."""
|
||
missing: collections.Counter = collections.Counter()
|
||
while True:
|
||
await asyncio.sleep(10)
|
||
try:
|
||
with db() as c:
|
||
jobs = c.execute(
|
||
"SELECT * FROM jobs WHERE state='downloading'").fetchall()
|
||
if not jobs:
|
||
missing.clear()
|
||
continue
|
||
|
||
async with slskd_client() as sc:
|
||
r = await sc.get("/api/v0/transfers/downloads")
|
||
r.raise_for_status()
|
||
index = transfer_index(r.json())
|
||
|
||
# One directory walk per cycle, shared by every job. This used to be
|
||
# a full recursive rglob per stuck job, every ten seconds, forever.
|
||
done_files: dict[str, Path] | None = None
|
||
|
||
for job in jobs:
|
||
t = index.get(job["remote_file"])
|
||
st = str((t or {}).get("state", ""))
|
||
|
||
if "Completed, Succeeded" in st:
|
||
if done_files is None:
|
||
done_files = downloaded_index()
|
||
src = done_files.get(
|
||
job["remote_file"].replace("\\", "/").rsplit("/", 1)[-1])
|
||
if not src:
|
||
# Normally just still being moved out of incomplete, but
|
||
# unbounded before: a misconfigured DOWNLOAD_DIR meant
|
||
# polling until the heat death of the process.
|
||
if stalled_for(job) > STALL_MINUTES:
|
||
set_state(job["id"], "failed",
|
||
"downloaded but the file never appeared in "
|
||
f"{DOWNLOAD_DIR} -- check MUSE_DOWNLOAD_DIR")
|
||
continue
|
||
try:
|
||
actual = src.stat().st_size
|
||
except OSError:
|
||
actual = job["size"]
|
||
|
||
conf = job["confidence"]
|
||
if conf is not None and conf < AUTO_CONFIDENCE:
|
||
band = ("almost certainly wrong"
|
||
if conf < REVIEW_CONFIDENCE else "worth a look")
|
||
set_state(job["id"], "review",
|
||
f"{conf:.0%} confident, {band} -- "
|
||
f"{job['match_why'] or ''}", size=actual)
|
||
await notify_review(job)
|
||
continue
|
||
|
||
dest = tag_and_file(job, src)
|
||
set_state(job["id"], "imported", str(dest),
|
||
imported_path=str(dest), size=actual)
|
||
await notify_imported(job, dest)
|
||
|
||
elif "Completed" in st and "Succeeded" not in st:
|
||
# Every terminal non-success used to end the request here.
|
||
# "File not shared." is per-file and per-peer, so the only
|
||
# sane response is to try the next candidate.
|
||
why = (t or {}).get("exception") or st
|
||
await abandon(job, str(why)[:120])
|
||
|
||
elif t is None:
|
||
# Absent from slskd entirely: it restarted, or the transfer
|
||
# was cleared. Tolerate a couple of polls in case it is a
|
||
# race, then stop waiting for something nobody is doing.
|
||
missing[job["id"]] += 1
|
||
if missing[job["id"]] >= 3:
|
||
del missing[job["id"]]
|
||
await abandon(job, "slskd forgot the transfer")
|
||
|
||
else:
|
||
missing.pop(job["id"], None)
|
||
# Queued behind a wall, or connected and sending nothing.
|
||
# There was no clock on this at all, which is how a job sat
|
||
# in `downloading` for days behind a cheerful pulsing dot.
|
||
if not (t or {}).get("bytesTransferred") \
|
||
and stalled_for(job) > STALL_MINUTES:
|
||
place = (t or {}).get("placeInQueue")
|
||
where = f" at #{place} in the queue" if place else ""
|
||
await cancel_transfer(job["username"], job["remote_file"])
|
||
await abandon(job, f"no bytes in {STALL_MINUTES:.0f}m{where}")
|
||
except Exception as e:
|
||
log.warning("import worker: %s", errstr(e))
|
||
|
||
|
||
# ─────────────────────────── api ───────────────────────────
|
||
|
||
@asynccontextmanager
|
||
async def lifespan(_: FastAPI):
|
||
# init_db first: the preflight consults the users table to decide whether
|
||
# the bootstrap allowlist is still mandatory.
|
||
init_db()
|
||
auth_preflight() # raises and stops the container if misconfigured
|
||
|
||
# The queue lives in memory, so a restart mid-import would otherwise
|
||
# strand every unstarted job in 'queued' forever with nothing to pick it
|
||
# up. Anything not yet handed to slskd goes back on the queue.
|
||
with db() as c:
|
||
orphans = [r["id"] for r in c.execute(
|
||
"SELECT id FROM jobs WHERE state IN ('queued','searching')").fetchall()]
|
||
for jid in orphans:
|
||
work_q.put_nowait(jid)
|
||
if orphans:
|
||
log.info("requeued %d job(s) left over from the last run", len(orphans))
|
||
|
||
tasks = [asyncio.create_task(import_worker())]
|
||
tasks += [asyncio.create_task(search_worker(i)) for i in range(WORKERS)]
|
||
log.info("muse up — slskd=%s library=%s formats=%s workers=%d",
|
||
SLSKD_URL, LIBRARY_DIR, ",".join(FORMATS), WORKERS)
|
||
yield
|
||
for t in tasks:
|
||
t.cancel()
|
||
|
||
|
||
# ─────────────────────── static front end / CSP ───────────────────────
|
||
#
|
||
# The UI is a Next.js static export: `next build` writes plain files into
|
||
# web/out and this process serves them, so the deployment stays one container,
|
||
# one process, one healthcheck, with no Node runtime in the image.
|
||
#
|
||
# That export contains inline <script> blocks (React's hydration payload),
|
||
# which `script-src 'self'` blocks outright. The usual fix is a per-request
|
||
# nonce, but a STATIC export has a better option: the scripts never change
|
||
# between requests, so their SHA-256 can go straight into the policy. Hashing
|
||
# them at startup means the CSP needs no per-request work, no HTML rewriting,
|
||
# and no maintenance when the bundle changes -- rebuild and the hashes follow.
|
||
WEB = Path(__file__).parent / "web" / "out"
|
||
|
||
_INLINE_SCRIPT = re.compile(rb"<script(?![^>]*\bsrc=)[^>]*>(.*?)</script>", re.S)
|
||
|
||
|
||
def script_hashes() -> list[str]:
|
||
out: set[str] = set()
|
||
for page in ("index.html", "404.html"):
|
||
f = WEB / page
|
||
if not f.exists():
|
||
continue
|
||
for body in _INLINE_SCRIPT.findall(f.read_bytes()):
|
||
digest = base64.b64encode(hashlib.sha256(body).digest()).decode()
|
||
out.add(f"'sha256-{digest}'")
|
||
return sorted(out)
|
||
|
||
|
||
SCRIPT_HASHES = " ".join(script_hashes())
|
||
|
||
# Framer Motion writes its pre-hydration `initial` state into style="" on the
|
||
# server-rendered markup, so those attributes have to be allowed or every
|
||
# animated element flashes in at full opacity before hydration corrects it.
|
||
#
|
||
# style-src-attr is the narrow lever for that: it permits style ATTRIBUTES
|
||
# while style-src 'self' still blocks injected <style> blocks and remote
|
||
# stylesheets. It is not a free pass either -- CSS-based exfiltration goes
|
||
# through url(), and img-src below already limits where those can point.
|
||
CSP = (
|
||
"default-src 'self'; "
|
||
f"script-src 'self' {SCRIPT_HASHES}; "
|
||
"style-src 'self'; style-src-attr 'unsafe-inline'; "
|
||
"img-src 'self' data: https://*.dzcdn.net https://*.githubusercontent.com; "
|
||
"media-src 'self' https://*.dzcdn.net; "
|
||
"connect-src 'self'; font-src 'self'; object-src 'none'; "
|
||
"frame-ancestors 'none'; base-uri 'none'; form-action 'self'"
|
||
)
|
||
|
||
|
||
app = FastAPI(
|
||
title="muse",
|
||
lifespan=lifespan,
|
||
# No interactive docs on an internet-facing app: they enumerate every
|
||
# endpoint and schema for anyone who reaches the login page.
|
||
docs_url=None, redoc_url=None, openapi_url=None,
|
||
)
|
||
|
||
|
||
@app.middleware("http")
|
||
async def guard(request: Request, call_next):
|
||
"""Require a session for everything except PUBLIC_PATHS, and stamp
|
||
security headers on every response."""
|
||
# /_next/* is NOT public. Unlike the old hand-written page there is no
|
||
# local login screen to style — an unauthenticated visitor is redirected
|
||
# straight to GitHub and never receives our HTML — so the bundle can sit
|
||
# behind the session too, and not hand out a map of the app to anyone who
|
||
# finds the URL. It loads fine after the callback, cookie in hand.
|
||
path = request.url.path
|
||
if AUTH_MODE == "off" or path in PUBLIC_PATHS:
|
||
resp = await call_next(request)
|
||
elif session_user(request):
|
||
resp = await call_next(request)
|
||
elif path.startswith("/api/"):
|
||
# An API caller gets a status code, not an HTML redirect it can't use.
|
||
resp = JSONResponse({"error": "unauthenticated"}, status_code=401)
|
||
else:
|
||
resp = RedirectResponse("/auth/login", status_code=302)
|
||
|
||
# See CSP above. img-src and media-src name Deezer's CDN explicitly rather
|
||
# than allowing https: wholesale — under a blanket https: any injected
|
||
# markup could beacon to an arbitrary host by requesting an image from it,
|
||
# which is a working exfiltration channel even with connect-src locked to
|
||
# 'self'. Artwork comes from dzcdn.net and avatars from githubusercontent.
|
||
resp.headers["Content-Security-Policy"] = CSP
|
||
resp.headers["X-Content-Type-Options"] = "nosniff"
|
||
resp.headers["X-Frame-Options"] = "DENY"
|
||
resp.headers["Referrer-Policy"] = "no-referrer"
|
||
resp.headers["Permissions-Policy"] = "geolocation=(), microphone=(), camera=()"
|
||
if COOKIE_SECURE:
|
||
resp.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
|
||
return resp
|
||
|
||
|
||
def _set_cookie(resp, name, value, max_age):
|
||
resp.set_cookie(name, value, max_age=max_age, httponly=True,
|
||
secure=COOKIE_SECURE, samesite="lax", path="/")
|
||
|
||
|
||
@app.get("/healthz")
|
||
async def healthz():
|
||
"""Unauthenticated on purpose, for the container healthcheck. Returns
|
||
nothing but a literal ok — no versions, no config, no counts."""
|
||
return {"ok": True}
|
||
|
||
|
||
# The sign-in page is the first screen anyone sees, and for a while it was the
|
||
# only one built out of hardcoded hex with no relation to the rest of the app.
|
||
# It cannot use the Next bundle -- /_next/* sits behind the session on purpose,
|
||
# so an unauthenticated visitor never receives it -- so it gets one small
|
||
# stylesheet of its own, served from a public path. `style-src 'self'` in the
|
||
# CSP permits exactly that; an inline <style> block would be refused.
|
||
LOGIN_CSS = """
|
||
:root{
|
||
--bg:#0a0b0f; --surface:#121319; --surface-2:#181a22;
|
||
--fg:#f4f6fb; --dim:#a0a8bb; --mute:#6d7488;
|
||
--edge:rgb(255 255 255/.09); --edge-hi:rgb(255 255 255/.16);
|
||
--accent:#0a84ff; --accent-ink:#fff;
|
||
--e2:0 2px 6px rgb(0 0 0/.34), 0 12px 32px rgb(0 0 0/.26);
|
||
color-scheme:light dark;
|
||
}
|
||
@media (prefers-color-scheme:light){
|
||
:root{
|
||
--bg:#f6f7f9; --surface:#fff; --surface-2:#f2f3f6;
|
||
--fg:#0d1117; --dim:#596273; --mute:#858da0;
|
||
--edge:rgb(10 14 25/.1); --edge-hi:rgb(10 14 25/.18);
|
||
--accent:#0071e3;
|
||
--e2:0 2px 6px rgb(20 30 60/.07), 0 12px 32px rgb(20 30 60/.08);
|
||
}
|
||
}
|
||
*{box-sizing:border-box}
|
||
body{
|
||
margin:0; min-height:100dvh; display:grid; place-items:center; padding:24px;
|
||
background:var(--bg); color:var(--fg);
|
||
font:15px/1.55 -apple-system,BlinkMacSystemFont,"SF Pro Text",ui-sans-serif,
|
||
system-ui,"Segoe UI",Roboto,sans-serif;
|
||
letter-spacing:-.008em; -webkit-font-smoothing:antialiased;
|
||
}
|
||
.card{
|
||
width:100%; max-width:352px; padding:32px 28px; text-align:center;
|
||
background:var(--surface); border:1px solid var(--edge);
|
||
border-radius:18px; box-shadow:var(--e2);
|
||
}
|
||
.mark{
|
||
width:44px; height:44px; margin:0 auto 18px; display:grid; place-items:center;
|
||
background:var(--accent); border-radius:12px; color:var(--accent-ink);
|
||
}
|
||
h1{
|
||
margin:0; font-size:23px; font-weight:700; letter-spacing:-.024em; line-height:1.18;
|
||
font-family:-apple-system,BlinkMacSystemFont,"SF Pro Display",ui-sans-serif,system-ui,sans-serif;
|
||
}
|
||
p.sub{margin:6px 0 24px; font-size:13px; color:var(--dim)}
|
||
a.btn{
|
||
display:flex; align-items:center; justify-content:center; gap:9px;
|
||
margin:8px 0; padding:11px 18px; border-radius:10px; text-decoration:none;
|
||
background:var(--surface-2); border:1px solid var(--edge-hi);
|
||
color:var(--fg); font-size:13px; font-weight:600;
|
||
transition:border-color .15s, background .15s, transform .1s;
|
||
}
|
||
a.btn:hover{border-color:var(--accent); background:var(--bg)}
|
||
a.btn:active{transform:scale(.985)}
|
||
a.btn:focus-visible{outline:2px solid var(--accent); outline-offset:2px}
|
||
p.foot{margin:22px 0 0; font-size:11px; color:var(--mute)}
|
||
"""
|
||
|
||
CHOOSER = """<!doctype html><html lang=en><head><meta charset=utf-8>
|
||
<meta name=viewport content="width=device-width,initial-scale=1">
|
||
<meta name=theme-color media="(prefers-color-scheme: light)" content="#f6f7f9">
|
||
<meta name=theme-color media="(prefers-color-scheme: dark)" content="#0a0b0f">
|
||
<title>Sign in · muse</title>
|
||
<link rel=stylesheet href="/login.css"></head>
|
||
<body><main class=card>
|
||
<div class=mark><svg width=22 height=22 viewBox="0 0 24 24" fill=none
|
||
stroke=currentColor stroke-width=2 stroke-linecap=round stroke-linejoin=round
|
||
aria-hidden="true"><path d="M9 18V5l12-2v13"/><circle cx="6" cy="18" r="3"/>
|
||
<circle cx="18" cy="16" r="3"/></svg></div>
|
||
<h1>muse</h1>
|
||
<p class=sub>Request a song, get it in your library.</p>
|
||
{buttons}
|
||
<p class=foot>You’ll be sent to your provider to sign in.</p>
|
||
</main></body></html>"""
|
||
|
||
_BTN = ('<a class=btn href="/auth/login?provider={p}">Continue with {label}</a>')
|
||
|
||
|
||
@app.get("/login.css")
|
||
async def login_css():
|
||
"""The sign-in page's stylesheet. Public, because it is needed by someone
|
||
who by definition has no session yet."""
|
||
return Response(LOGIN_CSS, media_type="text/css",
|
||
headers={"Cache-Control": "public, max-age=3600"})
|
||
|
||
|
||
@app.get("/auth/login")
|
||
async def auth_login(provider: str = ""):
|
||
"""Start OAuth. With several providers enabled and none named, offer a
|
||
choice — a tiny self-contained page that uses only style attributes, which
|
||
the CSP already permits, so it needs no stylesheet and no extra hash."""
|
||
if AUTH_MODE == "off":
|
||
return RedirectResponse("/", status_code=302)
|
||
|
||
if not provider:
|
||
if len(ENABLED_PROVIDERS) == 1:
|
||
provider = ENABLED_PROVIDERS[0]
|
||
else:
|
||
buttons = "".join(
|
||
_BTN.format(p=p, label=PROVIDERS[p]["label"]) for p in ENABLED_PROVIDERS)
|
||
return HTMLResponse(CHOOSER.replace("{buttons}", buttons))
|
||
|
||
if provider not in ENABLED_PROVIDERS:
|
||
raise HTTPException(404, "unknown provider")
|
||
|
||
cfg = PROVIDERS[provider]
|
||
nonce = secrets.token_urlsafe(24)
|
||
params = {
|
||
"client_id": cfg["client_id"],
|
||
"redirect_uri": f"{BASE_URL}/auth/callback",
|
||
"scope": cfg["scope"],
|
||
"state": nonce,
|
||
"response_type": "code",
|
||
}
|
||
if provider == "github":
|
||
params["allow_signup"] = "false"
|
||
if provider == "google":
|
||
# Force the account chooser; otherwise Google silently reuses whichever
|
||
# account the browser last used, which on a shared machine signs you in
|
||
# as someone else with no visible prompt.
|
||
params["prompt"] = "select_account"
|
||
|
||
resp = RedirectResponse(f"{cfg['authorize']}?{urlencode(params)}", status_code=302)
|
||
# CSRF defence: the nonce is echoed back by the provider and must match
|
||
# this cookie, so a callback the user did not initiate is rejected. The
|
||
# provider name rides along in the COOKIE, never the query string, so the
|
||
# callback cannot be steered to a different provider by the redirect.
|
||
_set_cookie(resp, STATE_COOKIE, f"{provider}|{nonce}", 600)
|
||
return resp
|
||
|
||
|
||
async def fetch_identity(provider: str, token: str) -> tuple[str, str, str]:
|
||
"""(username, display, avatar) for a provider access token."""
|
||
async with httpx.AsyncClient(timeout=20) as c:
|
||
if provider == "github":
|
||
r = await c.get(GH_API, headers={"Authorization": f"Bearer {token}",
|
||
"Accept": "application/vnd.github+json"})
|
||
if r.status_code != 200:
|
||
raise HTTPException(401, "github user lookup failed")
|
||
p = r.json() or {}
|
||
return p.get("login") or "", p.get("name") or "", p.get("avatar_url") or ""
|
||
|
||
r = await c.get(GOOGLE_API, headers={"Authorization": f"Bearer {token}"})
|
||
if r.status_code != 200:
|
||
raise HTTPException(401, "google user lookup failed")
|
||
p = r.json() or {}
|
||
# Unverified Google emails must never become an identity: anyone can
|
||
# put someone else's address on an account they control, and the
|
||
# allowlist is keyed on exactly this string.
|
||
if not p.get("email") or not p.get("email_verified"):
|
||
raise HTTPException(403, "google account has no verified email")
|
||
return p["email"], p.get("name") or "", p.get("picture") or ""
|
||
|
||
|
||
@app.get("/auth/callback")
|
||
async def auth_callback(request: Request, code: str = "", state: str = ""):
|
||
if AUTH_MODE == "off":
|
||
return RedirectResponse("/", status_code=302)
|
||
|
||
cookie = request.cookies.get(STATE_COOKIE) or ""
|
||
provider, _, nonce = cookie.partition("|")
|
||
if (not code or not state or not nonce
|
||
or provider not in ENABLED_PROVIDERS
|
||
or not secrets.compare_digest(state, nonce)):
|
||
log.warning("auth: rejected callback with bad or missing state")
|
||
return JSONResponse({"error": "invalid state"}, status_code=400)
|
||
|
||
cfg = PROVIDERS[provider]
|
||
async with httpx.AsyncClient(timeout=20) as c:
|
||
t = await c.post(cfg["token"], headers={"Accept": "application/json"},
|
||
data={"client_id": cfg["client_id"],
|
||
"client_secret": cfg["client_secret"],
|
||
"code": code,
|
||
"grant_type": "authorization_code",
|
||
"redirect_uri": f"{BASE_URL}/auth/callback"})
|
||
token = (t.json() or {}).get("access_token") if t.status_code == 200 else None
|
||
if not token:
|
||
log.warning("auth: %s token exchange failed (%s)", provider, t.status_code)
|
||
return JSONResponse({"error": "token exchange failed"}, status_code=401)
|
||
|
||
username, display, avatar = await fetch_identity(provider, token)
|
||
if not username:
|
||
return JSONResponse({"error": "no username from provider"}, status_code=401)
|
||
|
||
with db() as c:
|
||
existing = c.execute("SELECT * FROM users WHERE username=?",
|
||
(username,)).fetchone()
|
||
if not may_login(username, existing, provider):
|
||
# Log the attempt: on an internet-facing box this is the signal that
|
||
# someone found the login page.
|
||
if not existing:
|
||
why = "not invited"
|
||
elif existing["disabled"]:
|
||
why = "disabled"
|
||
else:
|
||
why = f"invited for {existing['provider']}, tried {provider}"
|
||
log.warning("auth: DENIED %s user %r (%s)", provider, username, why)
|
||
return JSONResponse({"error": "not authorised"}, status_code=403)
|
||
|
||
row = upsert_user(username, avatar, display, provider)
|
||
log.info("auth: signed in %s via %s (%s)", username, provider, row["role"])
|
||
resp = RedirectResponse("/", status_code=302)
|
||
_set_cookie(resp, SESSION_COOKIE, new_session(username), SESSION_DAYS * 86400)
|
||
resp.delete_cookie(STATE_COOKIE, path="/")
|
||
return resp
|
||
|
||
|
||
@app.get("/auth/logout")
|
||
async def auth_logout(request: Request):
|
||
tok = request.cookies.get(SESSION_COOKIE)
|
||
if tok:
|
||
with db() as c:
|
||
c.execute("DELETE FROM sessions WHERE token_hash=?", (_hash(tok),))
|
||
resp = RedirectResponse("/auth/login", status_code=302)
|
||
resp.delete_cookie(SESSION_COOKIE, path="/")
|
||
return resp
|
||
|
||
|
||
# ─────────────────────────── quotas ───────────────────────────
|
||
#
|
||
# Two limits, both per user, both 0 = unlimited:
|
||
# quota requests per CALENDAR month (resets on the 1st)
|
||
# storage_quota bytes of imported audio attributable to this user
|
||
#
|
||
# Calendar month rather than a rolling 30-day window on purpose: "you have 12
|
||
# left until the 1st" is something a person can reason about, where a rolling
|
||
# window silently drips capacity back at times nobody can predict.
|
||
|
||
def month_start() -> str:
|
||
return datetime.now(timezone.utc).strftime("%Y-%m-01 00:00:00")
|
||
|
||
|
||
def usage_for(username: str | None) -> dict:
|
||
"""Requests this month and bytes imported, for one user."""
|
||
if not username:
|
||
return {"requests": 0, "storage": 0}
|
||
with db() as c:
|
||
n = c.execute(
|
||
"SELECT COUNT(*) n FROM jobs WHERE requested_by=? AND created_at >= ?",
|
||
(username, month_start())).fetchone()["n"]
|
||
b = c.execute(
|
||
"SELECT COALESCE(SUM(size),0) b FROM jobs "
|
||
"WHERE requested_by=? AND state='imported'", (username,)).fetchone()["b"]
|
||
return {"requests": n, "storage": b}
|
||
|
||
|
||
def check_quota(request: Request, adding: int = 1):
|
||
"""Raise 429 if this request would exceed the caller's monthly allowance.
|
||
|
||
Admins are exempt — someone has to be able to fix the library when a quota
|
||
is what is in the way.
|
||
"""
|
||
u = current_user(request)
|
||
if not u or u["role"] == "admin":
|
||
return
|
||
used = usage_for(u["username"])
|
||
if u["quota"] and used["requests"] + adding > u["quota"]:
|
||
raise HTTPException(429, (
|
||
f"Monthly request quota reached ({used['requests']}/{u['quota']}). "
|
||
f"Resets on the 1st."))
|
||
if u["storage_quota"] and used["storage"] >= u["storage_quota"]:
|
||
raise HTTPException(429, (
|
||
f"Storage quota reached ({used['storage'] // 2**30} GB of "
|
||
f"{u['storage_quota'] // 2**30} GB)."))
|
||
|
||
|
||
@app.get("/api/me")
|
||
async def me(request: Request):
|
||
u = current_user(request)
|
||
if not u:
|
||
return {"user": None, "auth": AUTH_MODE,
|
||
"role": "admin" if AUTH_MODE == "off" else None,
|
||
"providers": ENABLED_PROVIDERS}
|
||
return {
|
||
"user": u["username"], "auth": AUTH_MODE, "role": u["role"],
|
||
"display": u["display"], "avatar": u["avatar"], "provider": u["provider"],
|
||
"quota": u["quota"], "storage_quota": u["storage_quota"],
|
||
"usage": usage_for(u["username"]),
|
||
"providers": ENABLED_PROVIDERS,
|
||
"email": u["email"], "notify": bool(u["notify"]),
|
||
"mail_configured": MAIL_ON,
|
||
}
|
||
|
||
|
||
class SettingsBody(BaseModel):
|
||
email: str | None = None
|
||
notify: bool | None = None
|
||
|
||
|
||
EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s.]+\.[^@\s]+$")
|
||
|
||
|
||
@app.patch("/api/me")
|
||
async def update_me(body: SettingsBody, request: Request):
|
||
"""Your own notification settings. Not an admin route on purpose — an
|
||
address you cannot change yourself is a support ticket."""
|
||
me = session_user(request)
|
||
if not me:
|
||
raise HTTPException(401, "unauthenticated")
|
||
sets, vals = [], []
|
||
if body.email is not None:
|
||
e = body.email.strip()
|
||
if e and not EMAIL_RE.match(e):
|
||
raise HTTPException(400, "that does not look like an email address")
|
||
sets.append("email=?"); vals.append(e or None)
|
||
if body.notify is not None:
|
||
sets.append("notify=?"); vals.append(1 if body.notify else 0)
|
||
if not sets:
|
||
raise HTTPException(400, "nothing to change")
|
||
with db() as c:
|
||
c.execute(f"UPDATE users SET {', '.join(sets)} WHERE username=?",
|
||
[*vals, me])
|
||
return {"ok": True, "mail_configured": MAIL_ON}
|
||
|
||
|
||
class RequestBody(BaseModel):
|
||
artist: str
|
||
title: str
|
||
album: str | None = None
|
||
year: str | None = None
|
||
track: int | None = None
|
||
deezer_id: int | None = None
|
||
cover: str | None = None
|
||
genre: str | None = None
|
||
length: int | None = None
|
||
|
||
|
||
# Album metadata (release year, genre, label) is not in the track search
|
||
# payload, so it costs one extra call per distinct album. Cached forever on
|
||
# purpose: a release date and a genre are immutable, so there is nothing to
|
||
# invalidate, and repeat searches for the same artist stop hitting Deezer at
|
||
# all. Trimmed only to stop it growing without bound on a long-lived process.
|
||
_album_cache: dict[int, dict] = {}
|
||
_album_lock = asyncio.Semaphore(6) # Deezer rate-limits ~50 req/5s per IP
|
||
|
||
|
||
async def album_meta(album_id) -> dict:
|
||
if not album_id:
|
||
return {}
|
||
key = int(album_id)
|
||
if key in _album_cache:
|
||
return _album_cache[key]
|
||
async with _album_lock:
|
||
if key in _album_cache: # filled while we waited for the slot
|
||
return _album_cache[key]
|
||
try:
|
||
async with httpx.AsyncClient(timeout=15) as c:
|
||
r = await c.get(f"{DEEZER}/album/{key}")
|
||
d = r.json() if r.status_code == 200 else {}
|
||
except Exception as e:
|
||
log.debug("album %s lookup failed: %s", key, e)
|
||
return {}
|
||
genres = [g.get("name") for g in (d.get("genres") or {}).get("data", []) if g.get("name")]
|
||
meta = {
|
||
"year": (d.get("release_date") or "")[:4] or None,
|
||
"genre": genres[0] if genres else None,
|
||
"genres": genres,
|
||
"label": d.get("label"),
|
||
"nb_tracks": d.get("nb_tracks"),
|
||
"record_type": d.get("record_type"),
|
||
}
|
||
if len(_album_cache) > 5000:
|
||
for k in list(_album_cache)[:1000]:
|
||
_album_cache.pop(k, None)
|
||
_album_cache[key] = meta
|
||
return meta
|
||
|
||
|
||
def track_row(t: dict) -> dict:
|
||
"""One Deezer track object -> the shape the UI and the job table use."""
|
||
artist = t.get("artist") or {}
|
||
album = t.get("album") or {}
|
||
return {
|
||
"deezer_id": t.get("id"),
|
||
"title": t.get("title") or "",
|
||
"artist": artist.get("name") or "",
|
||
"artist_id": artist.get("id"),
|
||
"artist_pic": artist.get("picture_medium"),
|
||
"album": album.get("title") or "",
|
||
"album_id": album.get("id"),
|
||
"cover": album.get("cover_medium"),
|
||
"cover_big": album.get("cover_big") or album.get("cover_medium"),
|
||
"length": (t.get("duration") or 0) * 1000,
|
||
# 30-second clip from Deezer's CDN. This is what lets you confirm you
|
||
# picked the right recording BEFORE committing a Soulseek search to it
|
||
# -- the single most useful thing in the payload for avoiding a wrong
|
||
# import, and it costs nothing extra.
|
||
"preview": t.get("preview") or None,
|
||
"explicit": bool(t.get("explicit_lyrics")),
|
||
"isrc": t.get("isrc"),
|
||
"rank": t.get("rank") or 0,
|
||
"year": None, "genre": None, "track": None,
|
||
}
|
||
|
||
|
||
async def add_album_meta(rows: list[dict]) -> list[dict]:
|
||
"""Fill year and genre on a result list, concurrently and de-duplicated."""
|
||
ids = {r["album_id"] for r in rows if r.get("album_id")}
|
||
if not ids:
|
||
return rows
|
||
metas = dict(zip(ids, await asyncio.gather(*(album_meta(i) for i in ids))))
|
||
for r in rows:
|
||
m = metas.get(r.get("album_id")) or {}
|
||
r["year"] = r["year"] or m.get("year")
|
||
r["genre"] = r["genre"] or m.get("genre")
|
||
return rows
|
||
|
||
|
||
async def deezer_search(q: str, limit: int = 25) -> list[dict]:
|
||
"""Ranked Deezer track search. Shared by /api/search and /api/import."""
|
||
if not q.strip():
|
||
return []
|
||
async with httpx.AsyncClient(timeout=20) as c:
|
||
r = await c.get(f"{DEEZER}/search", params={"q": q, "limit": limit * 2})
|
||
# Deezer's WAF answers 403 to anything that looks like an injection
|
||
# attempt -- "'; DROP TABLE" and friends. That is a statement about the
|
||
# query, not an upstream outage, so it belongs on the empty-results
|
||
# path; a 502 would tell the user the service is broken when it is not.
|
||
if r.status_code == 403:
|
||
log.info("deezer refused the query %r", q[:60])
|
||
return []
|
||
if r.status_code != 200:
|
||
raise HTTPException(502, f"deezer returned {r.status_code}")
|
||
data = r.json()
|
||
|
||
qn = norm(q)
|
||
scored = []
|
||
for t in data.get("data", []):
|
||
title = t.get("title") or ""
|
||
artist = (t.get("artist") or {}).get("name") or ""
|
||
|
||
s = float(t.get("rank") or 0) / 10_000 # Deezer popularity, 0-~100
|
||
|
||
# An artist match separates the real track from a covers compilation.
|
||
at = set(norm(artist).split())
|
||
if norm(artist) and norm(artist) in qn:
|
||
s += 300
|
||
|
||
# Title relevance. This has to dominate everything else: without it a
|
||
# completely different song by the right artist competes on equal
|
||
# footing, and "Oasis - Live Forever" resolved to "Cloudburst" because
|
||
# a variant penalty was enough to tip between two unrelated tracks.
|
||
# Whatever query words the artist name did not explain should show up
|
||
# in the title.
|
||
rest = set(qn.split()) - at
|
||
if rest:
|
||
s += 400 * len(rest & set(norm(title).split())) / len(rest)
|
||
for w in set(DERIVATIVE_RE.findall(norm(title))):
|
||
if not re.search(rf"\b{re.escape(w)}\b", qn):
|
||
s -= 150
|
||
# Catch-all for variants the keyword list will never cover in full
|
||
# ("Devonshire Mix", "Radio Edit", "2011 Version"): a plain title beats
|
||
# a parenthesised one. Deliberately small -- it should break ties
|
||
# between versions of the SAME song, never outweigh title relevance
|
||
# above and pick a different song.
|
||
if ("(" in title or "[" in title) and "(" not in q:
|
||
s -= 40
|
||
scored.append((s, track_row(t)))
|
||
|
||
seen, out = set(), []
|
||
for _, item in sorted(scored, key=lambda x: -x[0]):
|
||
key = (norm(item["artist"]), norm(item["title"]))
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
out.append(item)
|
||
return out[:limit]
|
||
|
||
|
||
@app.get("/api/search")
|
||
async def search(q: str, limit: int = Query(25, ge=1, le=50)):
|
||
"""Track search. Returns individual songs, not albums — the whole point.
|
||
|
||
Year and genre are grafted on afterwards because Deezer's track search
|
||
omits both. Enrichment happens only here, not in deezer_search itself, so
|
||
the bulk-import path stays one HTTP call per line.
|
||
"""
|
||
return {"results": await add_album_meta(await deezer_search(q, limit))}
|
||
|
||
|
||
@app.get("/api/suggest")
|
||
async def suggest(q: str):
|
||
"""Autocomplete. Deliberately does NOT enrich or re-rank: this fires on
|
||
every few keystrokes, so it has to be one upstream call and nothing else.
|
||
Deezer's own autocomplete ordering is good enough for a dropdown, and the
|
||
real ranking still happens when the search is actually submitted."""
|
||
q = q.strip()
|
||
if len(q) < 2:
|
||
return {"tracks": [], "albums": [], "artists": []}
|
||
try:
|
||
async with httpx.AsyncClient(timeout=8) as c:
|
||
r = await c.get(f"{DEEZER}/search/autocomplete",
|
||
params={"q": q, "limit": 5})
|
||
d = r.json() if r.status_code == 200 else {}
|
||
except Exception:
|
||
return {"tracks": [], "albums": [], "artists": []}
|
||
|
||
def rows(key):
|
||
return (d.get(key) or {}).get("data", []) or []
|
||
|
||
return {
|
||
"tracks": [{"deezer_id": t.get("id"), "title": t.get("title"),
|
||
"artist": (t.get("artist") or {}).get("name"),
|
||
"cover": (t.get("album") or {}).get("cover_small"),
|
||
"preview": t.get("preview")}
|
||
for t in rows("tracks")[:5]],
|
||
"albums": [{"album_id": a.get("id"), "title": a.get("title"),
|
||
"artist": (a.get("artist") or {}).get("name"),
|
||
"cover": a.get("cover_small"),
|
||
"nb_tracks": a.get("nb_tracks")}
|
||
for a in rows("albums")[:3]],
|
||
"artists": [{"artist_id": a.get("id"), "name": a.get("name"),
|
||
"cover": a.get("picture_small"),
|
||
"nb_album": a.get("nb_album")}
|
||
for a in rows("artists")[:3]],
|
||
}
|
||
|
||
|
||
@app.get("/api/album/{album_id}")
|
||
async def album_detail(album_id: int):
|
||
"""Full track list for an album, so 'Get all' can queue it in one go."""
|
||
# Two calls, not one. /album/{id} embeds a track list, but those nested
|
||
# objects have no track_position -- and taking the list index instead would
|
||
# misnumber every multi-disc release, writing "13 - ..." for disc 2 track 1.
|
||
# /album/{id}/tracks is the only source that carries track_position and
|
||
# disk_number, and those numbers end up in the filename and the tags.
|
||
try:
|
||
async with httpx.AsyncClient(timeout=20) as c:
|
||
r, tr = await asyncio.gather(
|
||
c.get(f"{DEEZER}/album/{album_id}"),
|
||
c.get(f"{DEEZER}/album/{album_id}/tracks", params={"limit": 300}))
|
||
except Exception as e:
|
||
raise HTTPException(502, f"deezer: {e}")
|
||
if r.status_code != 200:
|
||
raise HTTPException(404, "album not found")
|
||
d = r.json()
|
||
|
||
artist = (d.get("artist") or {}).get("name") or ""
|
||
year = (d.get("release_date") or "")[:4] or None
|
||
genres = [g.get("name") for g in (d.get("genres") or {}).get("data", []) if g.get("name")]
|
||
raw = (tr.json().get("data", []) if tr.status_code == 200
|
||
else (d.get("tracks") or {}).get("data", []))
|
||
tracks = []
|
||
for t in raw:
|
||
row = track_row(t)
|
||
# These nested track objects carry no album of their own, so inherit
|
||
# what the parent already told us rather than looking it up again.
|
||
row.update(album=d.get("title") or "", album_id=d.get("id"),
|
||
cover=d.get("cover_medium"), cover_big=d.get("cover_big"),
|
||
artist=row["artist"] or artist, year=year,
|
||
genre=genres[0] if genres else None,
|
||
track=t.get("track_position"),
|
||
disc=t.get("disk_number"))
|
||
tracks.append(row)
|
||
|
||
return {"album": {"album_id": d.get("id"), "title": d.get("title"),
|
||
"artist": artist, "artist_id": (d.get("artist") or {}).get("id"),
|
||
"cover_big": d.get("cover_big"), "cover": d.get("cover_medium"),
|
||
"year": year, "genres": genres, "label": d.get("label"),
|
||
"nb_tracks": d.get("nb_tracks"),
|
||
"record_type": d.get("record_type"),
|
||
"duration": d.get("duration")},
|
||
"tracks": tracks}
|
||
|
||
|
||
@app.get("/api/artist/{artist_id}")
|
||
async def artist_detail(artist_id: int):
|
||
"""Artist header plus their top tracks and albums."""
|
||
try:
|
||
async with httpx.AsyncClient(timeout=20) as c:
|
||
info, top, albums = await asyncio.gather(
|
||
c.get(f"{DEEZER}/artist/{artist_id}"),
|
||
c.get(f"{DEEZER}/artist/{artist_id}/top", params={"limit": 15}),
|
||
c.get(f"{DEEZER}/artist/{artist_id}/albums", params={"limit": 24}))
|
||
except Exception as e:
|
||
raise HTTPException(502, f"deezer: {e}")
|
||
if info.status_code != 200:
|
||
raise HTTPException(404, "artist not found")
|
||
a = info.json()
|
||
tracks = [track_row(t) for t in (top.json().get("data", []) if top.status_code == 200 else [])]
|
||
await add_album_meta(tracks)
|
||
return {
|
||
"artist": {"artist_id": a.get("id"), "name": a.get("name"),
|
||
"picture": a.get("picture_big"), "nb_fan": a.get("nb_fan"),
|
||
"nb_album": a.get("nb_album")},
|
||
"tracks": tracks,
|
||
"albums": [{"album_id": x.get("id"), "title": x.get("title"),
|
||
"cover": x.get("cover_medium"), "cover_big": x.get("cover_big"),
|
||
"year": (x.get("release_date") or "")[:4] or None,
|
||
"record_type": x.get("record_type"),
|
||
"nb_tracks": x.get("nb_tracks")}
|
||
for x in (albums.json().get("data", []) if albums.status_code == 200 else [])],
|
||
}
|
||
|
||
|
||
async def enqueue_job(artist: str, title: str, album=None, year=None,
|
||
track=None, deezer_id=None, requested_by=None,
|
||
cover=None, genre=None, duration=None) -> str:
|
||
"""Create a job and hand it to the search queue."""
|
||
job_id = str(uuid.uuid4())
|
||
# The single choke point for all three request paths, so normalise here.
|
||
# year reaches a directory name, and artist/title/album reach filenames.
|
||
year = safe_year(year)
|
||
cover = safe_cover(cover)
|
||
artist, title = artist[:MAX_COMPONENT], title[:MAX_COMPONENT]
|
||
album = album[:MAX_COMPONENT] if album else album
|
||
genre = genre[:80] if genre else genre
|
||
with db() as c:
|
||
c.execute(
|
||
"INSERT INTO jobs (id,state,detail,artist,title,album,year,track,"
|
||
"recording_mbid,release_mbid,requested_by,cover,genre,duration) "
|
||
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||
(job_id, "queued", "", artist, title, album, year, track,
|
||
str(deezer_id or ""), None, requested_by, cover, genre, duration))
|
||
await work_q.put(job_id)
|
||
return job_id
|
||
|
||
|
||
async def enrich(deezer_id) -> dict:
|
||
"""Deezer's search payload has no track number or release date, and both
|
||
go into the filename and tags. One lookup per request, not per result."""
|
||
if not deezer_id:
|
||
return {}
|
||
try:
|
||
async with httpx.AsyncClient(timeout=15) as c:
|
||
r = await c.get(f"{DEEZER}/track/{deezer_id}")
|
||
if r.status_code != 200:
|
||
return {}
|
||
d = r.json()
|
||
rd = d.get("release_date") or (d.get("album") or {}).get("release_date")
|
||
return {
|
||
"track": d.get("track_position"),
|
||
"year": (rd or "")[:4] or None,
|
||
"album": (d.get("album") or {}).get("title"),
|
||
"artist": (d.get("artist") or {}).get("name"),
|
||
}
|
||
except Exception as e:
|
||
log.warning("deezer track lookup failed: %s", e)
|
||
return {}
|
||
|
||
|
||
@app.post("/api/request")
|
||
async def request_track(body: RequestBody, request: Request):
|
||
check_quota(request)
|
||
e = await enrich(body.deezer_id)
|
||
return {"id": await enqueue_job(
|
||
e.get("artist") or body.artist, body.title,
|
||
e.get("album") or body.album, e.get("year") or body.year,
|
||
e.get("track") or body.track, body.deezer_id,
|
||
requested_by=session_user(request), cover=body.cover,
|
||
genre=body.genre, duration=body.length)}
|
||
|
||
|
||
class BatchBody(BaseModel):
|
||
tracks: list[RequestBody]
|
||
|
||
|
||
@app.post("/api/request/batch")
|
||
async def request_batch(body: BatchBody, request: Request):
|
||
"""Queue a whole album at once.
|
||
|
||
Unlike /api/request this skips the per-track /track/{id} enrichment: the
|
||
caller got these rows from /api/album/{id}, which already carries the
|
||
album, year and track position for every one of them. Re-fetching would be
|
||
nb_tracks extra calls to learn what we were just told.
|
||
"""
|
||
# Checked against the whole batch, not one track: queueing a 30-track album
|
||
# when 5 requests remain should be refused up front, not accepted and then
|
||
# blocked halfway through.
|
||
check_quota(request, adding=len(body.tracks))
|
||
who = session_user(request)
|
||
ids = [await enqueue_job(t.artist, t.title, t.album, t.year, t.track,
|
||
t.deezer_id, requested_by=who, cover=t.cover,
|
||
genre=t.genre, duration=t.length)
|
||
for t in body.tracks]
|
||
log.info("batch: queued %d track(s) for %s", len(ids), who or "anon")
|
||
return {"queued": len(ids), "ids": ids, "pending": work_q.qsize()}
|
||
|
||
|
||
class ImportBody(BaseModel):
|
||
text: str
|
||
|
||
|
||
# "Artist - Title", "Artist – Title" (en dash), "Title by Artist". Anything
|
||
# that doesn't split just goes to Deezer whole — its search copes fine with
|
||
# free text, and guessing wrong is worse than not guessing.
|
||
SPLIT = re.compile(r"\s+[-–—|]\s+|\s+by\s+", re.I)
|
||
DEEZER_URL = re.compile(r"deezer\.com/(?:[a-z]{2}/)?(album|playlist)/(\d+)")
|
||
|
||
# Generous for a human pasting an album or a year-end list, small enough that
|
||
# one request cannot turn into thousands of upstream calls.
|
||
MAX_IMPORT_LINES = 500
|
||
MAX_IMPORT_TRACKS = 1000
|
||
|
||
|
||
async def expand_url(kind: str, ident: str) -> list[str]:
|
||
"""Turn a Deezer album/playlist URL into 'Artist - Title' lines."""
|
||
out = []
|
||
try:
|
||
async with httpx.AsyncClient(timeout=25) as c:
|
||
r = await c.get(f"{DEEZER}/{kind}/{ident}")
|
||
if r.status_code != 200:
|
||
return []
|
||
for t in (r.json().get("tracks") or {}).get("data", []):
|
||
a = (t.get("artist") or {}).get("name") or ""
|
||
out.append(f"{a} - {t.get('title')}" if a else t.get("title", ""))
|
||
except Exception as e:
|
||
log.warning("expanding %s/%s failed: %s", kind, ident, e)
|
||
return [x for x in out if x]
|
||
|
||
|
||
@app.post("/api/import")
|
||
async def import_list(body: ImportBody, request: Request):
|
||
"""Paste a list, get it all queued.
|
||
|
||
Each line is resolved against Deezer and the top-ranked hit is taken
|
||
automatically — there is no picker in bulk mode. Resolved artist/album are
|
||
recorded on every job so a wrong auto-pick is visible in the list rather
|
||
than silently landing the wrong song in the library.
|
||
"""
|
||
lines, queued, skipped = [], [], []
|
||
# Capped before expansion, not after. Each URL line costs an upstream call,
|
||
# and the quota check below cannot run until expansion has finished — so
|
||
# without a cap here a paste of 10,000 album URLs is 10,000 requests to
|
||
# Deezer from this IP before anyone's quota is consulted.
|
||
raw_lines = body.text.splitlines()
|
||
if len(raw_lines) > MAX_IMPORT_LINES:
|
||
raise HTTPException(413, f"paste at most {MAX_IMPORT_LINES} lines at a time "
|
||
f"(got {len(raw_lines)})")
|
||
for raw in raw_lines:
|
||
line = raw.strip().lstrip("-•*0123456789. )").strip()
|
||
if not line:
|
||
continue
|
||
m = DEEZER_URL.search(line)
|
||
lines.extend(await expand_url(*m.groups()) if m else [line])
|
||
if len(lines) > MAX_IMPORT_TRACKS:
|
||
raise HTTPException(413, f"that expands to more than "
|
||
f"{MAX_IMPORT_TRACKS} tracks")
|
||
|
||
# After expansion — a single playlist URL can be 200 tracks, and the point
|
||
# of the check is the real number, not the number of lines pasted.
|
||
check_quota(request, adding=len(lines))
|
||
|
||
for line in lines:
|
||
# Search the line as typed; the ranker already weights an artist-name
|
||
# match heavily, so "Artist - Title" needs no special handling.
|
||
try:
|
||
# Ask for a real candidate pool, not one row. deezer_search ranks
|
||
# what it is given, so limit=1 fetched two results and handed back
|
||
# the better of a demo and a rehearsal take while the studio
|
||
# version -- 6x the popularity -- sat further down the list unseen.
|
||
hits = await deezer_search(SPLIT.sub(" ", line), limit=10)
|
||
except Exception as e:
|
||
skipped.append({"line": line, "why": errstr(e)})
|
||
continue
|
||
if not hits:
|
||
skipped.append({"line": line, "why": "no match on Deezer"})
|
||
continue
|
||
h = hits[0]
|
||
e = await enrich(h["deezer_id"])
|
||
jid = await enqueue_job(
|
||
e.get("artist") or h["artist"], h["title"],
|
||
e.get("album") or h["album"], e.get("year"), e.get("track"),
|
||
h["deezer_id"], requested_by=session_user(request),
|
||
cover=h.get("cover"), duration=h.get("length"))
|
||
queued.append({"id": jid, "line": line, "cover": h.get("cover"),
|
||
"matched": f"{h['artist']} — {h['title']}"})
|
||
|
||
log.info("import: %d queued, %d skipped", len(queued), len(skipped))
|
||
return {"queued": queued, "skipped": skipped, "pending": work_q.qsize()}
|
||
|
||
|
||
class ReviewBody(BaseModel):
|
||
action: str # approve | reject
|
||
|
||
|
||
@app.get("/api/review")
|
||
async def review_queue(request: Request):
|
||
"""Downloads held back because the match was not convincing enough.
|
||
|
||
Scoped like /api/jobs: your own unless you are an admin, because a held
|
||
file is a record of what someone asked for.
|
||
"""
|
||
me = session_user(request)
|
||
admin = user_role(request) == "admin"
|
||
with db() as c:
|
||
if admin:
|
||
rows = c.execute(
|
||
"SELECT * FROM jobs WHERE state='review' ORDER BY created_at DESC"
|
||
).fetchall()
|
||
else:
|
||
rows = c.execute(
|
||
"SELECT * FROM jobs WHERE state='review' AND requested_by IS ? "
|
||
"ORDER BY created_at DESC", (me,)).fetchall()
|
||
return {"items": [{
|
||
"id": r["id"], "artist": r["artist"], "title": r["title"],
|
||
"album": r["album"], "year": r["year"], "cover": r["cover"],
|
||
"confidence": r["confidence"], "why": r["match_why"],
|
||
"file": (r["remote_file"] or "").replace("\\", "/").rsplit("/", 1)[-1],
|
||
"path": r["remote_file"], "size": r["size"],
|
||
"requested_by": r["requested_by"], "created_at": r["created_at"],
|
||
} for r in rows], "thresholds": {"auto": AUTO_CONFIDENCE,
|
||
"review": REVIEW_CONFIDENCE}}
|
||
|
||
|
||
@app.post("/api/review/{job_id}")
|
||
async def review_decide(job_id: str, body: ReviewBody, request: Request):
|
||
me = session_user(request)
|
||
admin = user_role(request) == "admin"
|
||
if body.action not in ("approve", "reject"):
|
||
raise HTTPException(400, "action must be approve or reject")
|
||
|
||
with db() as c:
|
||
job = c.execute("SELECT * FROM jobs WHERE id=? AND state='review'",
|
||
(job_id,)).fetchone()
|
||
# 404 rather than 403 for someone else's job: a distinct error would
|
||
# confirm the id exists, same as /api/retry.
|
||
if not job or (not admin and job["requested_by"] != me):
|
||
raise HTTPException(404, "no such review item")
|
||
|
||
base = (job["remote_file"] or "").replace("\\", "/").rsplit("/", 1)[-1]
|
||
src = downloaded_index().get(base)
|
||
if body.action == "reject":
|
||
if src:
|
||
try:
|
||
src.unlink()
|
||
except OSError as e:
|
||
log.warning("review: could not delete %s: %s", src, e)
|
||
set_state(job_id, "failed", "rejected in review", reviewed_by=me)
|
||
return {"ok": True, "state": "failed"}
|
||
|
||
if not src:
|
||
raise HTTPException(409, "the downloaded file is no longer on disk")
|
||
dest = tag_and_file(job, src)
|
||
set_state(job_id, "imported", str(dest), imported_path=str(dest),
|
||
reviewed_by=me)
|
||
log.info("review: %s approved %s", me, job_id[:8])
|
||
return {"ok": True, "state": "imported", "path": str(dest)}
|
||
|
||
|
||
@app.get("/api/discover")
|
||
async def discover(request: Request):
|
||
"""What to listen to next, derived from what you have actually asked for.
|
||
|
||
Deliberately built on muse's own request history plus Deezer's related
|
||
artists, not on listening data — Navidrome owns play counts and history,
|
||
and duplicating them here would mean two disagreeing sources of truth.
|
||
"""
|
||
me = session_user(request)
|
||
with db() as c:
|
||
top = c.execute(
|
||
"SELECT artist, COUNT(*) n FROM jobs "
|
||
"WHERE requested_by IS ? AND state='imported' "
|
||
"GROUP BY artist COLLATE NOCASE ORDER BY n DESC LIMIT 5",
|
||
(me,)).fetchall()
|
||
if not top:
|
||
return {"seeds": [], "artists": [], "tracks": []}
|
||
|
||
async def related(name: str) -> list[dict]:
|
||
try:
|
||
async with httpx.AsyncClient(timeout=15) as c:
|
||
s = await c.get(f"{DEEZER}/search/artist",
|
||
params={"q": name, "limit": 1})
|
||
hits = (s.json() or {}).get("data") or []
|
||
if not hits:
|
||
return []
|
||
r = await c.get(f"{DEEZER}/artist/{hits[0]['id']}/related",
|
||
params={"limit": 6})
|
||
return (r.json() or {}).get("data") or []
|
||
except Exception as e:
|
||
log.info("discover: related for %r failed: %s", name, errstr(e))
|
||
return []
|
||
|
||
seeds = [r["artist"] for r in top]
|
||
groups = await asyncio.gather(*(related(s) for s in seeds))
|
||
|
||
have = {norm(r["artist"]) for r in top}
|
||
seen, artists = set(), []
|
||
for g in groups:
|
||
for a in g:
|
||
k = norm(a.get("name") or "")
|
||
if not k or k in have or k in seen:
|
||
continue
|
||
seen.add(k)
|
||
artists.append({"id": a.get("id"), "name": a.get("name"),
|
||
"picture": a.get("picture_medium"),
|
||
"fans": a.get("nb_fan")})
|
||
artists.sort(key=lambda a: -(a["fans"] or 0))
|
||
return {"seeds": seeds, "artists": artists[:18]}
|
||
|
||
|
||
@app.get("/api/recap")
|
||
async def recap(request: Request, days: int = Query(365, ge=7, le=3650)):
|
||
"""Your year in muse — from request history, which muse does own."""
|
||
me = session_user(request)
|
||
since = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat(" ")[:19]
|
||
with db() as c:
|
||
base = ("FROM jobs WHERE requested_by IS ? AND created_at >= ? "
|
||
"AND state='imported'")
|
||
total = c.execute(f"SELECT COUNT(*) n, COALESCE(SUM(size),0) b {base}",
|
||
(me, since)).fetchone()
|
||
artists = c.execute(
|
||
f"SELECT artist, COUNT(*) n {base} GROUP BY artist COLLATE NOCASE "
|
||
"ORDER BY n DESC LIMIT 10", (me, since)).fetchall()
|
||
genres = c.execute(
|
||
f"SELECT genre, COUNT(*) n {base} AND genre IS NOT NULL AND genre<>'' "
|
||
"GROUP BY genre ORDER BY n DESC LIMIT 8", (me, since)).fetchall()
|
||
decades = c.execute(
|
||
f"SELECT substr(year,1,3) d, COUNT(*) n {base} "
|
||
"AND year GLOB '[0-9][0-9][0-9][0-9]' GROUP BY d ORDER BY d",
|
||
(me, since)).fetchall()
|
||
busiest = c.execute(
|
||
f"SELECT substr(created_at,1,10) d, COUNT(*) n {base} "
|
||
"GROUP BY d ORDER BY n DESC LIMIT 1", (me, since)).fetchone()
|
||
return {
|
||
"days": days,
|
||
"tracks": total["n"], "bytes": total["b"],
|
||
"artists": [{"name": r["artist"], "n": r["n"]} for r in artists],
|
||
"genres": [{"name": r["genre"], "n": r["n"]} for r in genres],
|
||
"decades": [{"decade": f"{r['d']}0s", "n": r["n"]} for r in decades],
|
||
"busiest_day": ({"date": busiest["d"], "n": busiest["n"]}
|
||
if busiest and busiest["n"] else None),
|
||
}
|
||
|
||
|
||
def job_dict(r: sqlite3.Row) -> dict:
|
||
"""A job as the UI wants it.
|
||
|
||
`candidates` is deliberately dropped: it is the untried remainder of the
|
||
ladder, several KB of peer names and paths per job, and the jobs list is
|
||
polled every four seconds. `tried` is sent parsed, because it is the answer
|
||
to "why did this fail" and the UI renders it as a list.
|
||
"""
|
||
d = dict(r)
|
||
d.pop("candidates", None)
|
||
try:
|
||
d["tried"] = json.loads(d.get("tried") or "[]")
|
||
except (TypeError, ValueError):
|
||
d["tried"] = []
|
||
d["attempts"] = d.get("attempts") or 0
|
||
d["remaining"] = max(0, MAX_ATTEMPTS - d["attempts"])
|
||
return d
|
||
|
||
|
||
@app.get("/api/jobs")
|
||
async def jobs(request: Request, limit: int = Query(200, ge=1, le=1000),
|
||
mine: bool = False):
|
||
"""Own requests by default; admins see everyone's.
|
||
|
||
Scoped rather than global because on a shared instance one person's
|
||
request history is a list of what they listen to, which is nobody else's
|
||
business by default.
|
||
"""
|
||
u = current_user(request)
|
||
everyone = (AUTH_MODE == "off") or (u is not None and u["role"] == "admin" and not mine)
|
||
who = None if everyone else (u["username"] if u else None)
|
||
|
||
where, args = ("", []) if everyone else ("WHERE requested_by IS ?", [who])
|
||
with db() as c:
|
||
rows = c.execute(
|
||
f"SELECT * FROM jobs {where} ORDER BY created_at DESC, rowid DESC LIMIT ?",
|
||
[*args, limit]).fetchall()
|
||
counts = {r["state"]: r["n"] for r in c.execute(
|
||
f"SELECT state, COUNT(*) n FROM jobs {where} GROUP BY state",
|
||
args).fetchall()}
|
||
return {"jobs": [job_dict(r) for r in rows], "counts": counts,
|
||
"pending": work_q.qsize(), "scope": "all" if everyone else "mine"}
|
||
|
||
|
||
@app.post("/api/retry/{job_id}")
|
||
async def retry(job_id: str, request: Request):
|
||
"""Put a failed job back on the queue. Soulseek failures are usually the
|
||
peer going offline, not the request being wrong, so retrying later often
|
||
just works."""
|
||
u = current_user(request)
|
||
with db() as c:
|
||
row = c.execute("SELECT id, state, requested_by FROM jobs WHERE id=?",
|
||
(job_id,)).fetchone()
|
||
if not row:
|
||
raise HTTPException(404, "no such job")
|
||
# Same 404 rather than 403 for someone else's job: a distinct error would
|
||
# confirm the id exists, which is a small oracle over other people's
|
||
# requests.
|
||
if AUTH_MODE != "off" and u and u["role"] != "admin" \
|
||
and row["requested_by"] != u["username"]:
|
||
raise HTTPException(404, "no such job")
|
||
# Only failed jobs. Requeueing one in `review` used to strand its
|
||
# downloaded file on disk with no row left pointing at it, and requeueing
|
||
# one already `imported` would re-download a track that is in the library.
|
||
if row["state"] != "failed":
|
||
raise HTTPException(409, f"only failed jobs can be retried "
|
||
f"(this one is {row['state']})")
|
||
# Clear the previous attempt, or the ladder resumes mid-way through a
|
||
# candidate list gathered for a search that has long since expired.
|
||
with db() as c:
|
||
c.execute("UPDATE jobs SET username=NULL, remote_file=NULL, size=NULL,"
|
||
" confidence=NULL, match_why=NULL, candidates=NULL,"
|
||
" tried=NULL, attempts=0, started_at=NULL WHERE id=?",
|
||
(job_id,))
|
||
set_state(job_id, "queued", "")
|
||
await work_q.put(job_id)
|
||
return {"ok": True}
|
||
|
||
|
||
# ─────────────────────────── admin ───────────────────────────
|
||
|
||
class NewUser(BaseModel):
|
||
username: str
|
||
provider: str = "github"
|
||
role: str = "user"
|
||
quota: int = 0
|
||
storage_quota_gb: float = 0
|
||
# Where to send the invitation. Omit for a Google invite — the username
|
||
# already is the address. Only needed for GitHub, where a login is not one.
|
||
email: str | None = None
|
||
notify: bool = True
|
||
|
||
|
||
class PatchUser(BaseModel):
|
||
role: str | None = None
|
||
quota: int | None = None
|
||
storage_quota_gb: float | None = None
|
||
disabled: bool | None = None
|
||
# So a typo in an invited address is fixable without deleting the account
|
||
# and losing its request history.
|
||
email: str | None = None
|
||
|
||
|
||
def user_dict(r: sqlite3.Row) -> dict:
|
||
u = usage_for(r["username"])
|
||
return {
|
||
"username": r["username"], "provider": r["provider"], "role": r["role"],
|
||
"display": r["display"], "avatar": r["avatar"],
|
||
"quota": r["quota"], "storage_quota": r["storage_quota"],
|
||
"disabled": bool(r["disabled"]), "created_at": r["created_at"],
|
||
"last_seen": r["last_seen"], "invited_by": r["invited_by"],
|
||
"usage": u, "email": r["email"], "notify": bool(r["notify"]),
|
||
# Surfaced so the UI doesn't have to know the allowlist rule: these are
|
||
# the accounts that cannot actually be locked out by disabling them.
|
||
"bootstrap": r["username"].lower() in ALLOWED_USERS,
|
||
}
|
||
|
||
|
||
@app.get("/api/admin/users")
|
||
async def admin_users(request: Request):
|
||
require_admin(request)
|
||
with db() as c:
|
||
rows = c.execute("SELECT * FROM users ORDER BY role='admin' DESC, username").fetchall()
|
||
return {"users": [user_dict(r) for r in rows], "roles": list(ROLES)}
|
||
|
||
|
||
@app.post("/api/admin/users")
|
||
async def admin_add_user(body: NewUser, request: Request):
|
||
"""Invite someone.
|
||
|
||
The row IS the authorisation: once it exists, that identity can sign in
|
||
with the provider it was invited for, and only that one. The email is a
|
||
notification on top, sent when there is an address and SMTP is configured.
|
||
"""
|
||
who = require_admin(request)
|
||
name = body.username.strip()
|
||
if not name:
|
||
raise HTTPException(400, "username required")
|
||
if body.role not in ROLES:
|
||
raise HTTPException(400, f"role must be one of {', '.join(ROLES)}")
|
||
if body.provider not in PROVIDERS:
|
||
raise HTTPException(400, f"provider must be one of {', '.join(PROVIDERS)}")
|
||
if body.provider not in ENABLED_PROVIDERS:
|
||
raise HTTPException(
|
||
400, f"{body.provider} is not enabled on this instance "
|
||
f"(MUSE_AUTH={AUTH_MODE})")
|
||
|
||
# A Google identity is its own address, so require it to look like one --
|
||
# otherwise the invite silently authorises a username Google can never
|
||
# return, and the person just gets "not authorised" with no explanation.
|
||
if body.provider == "google" and not EMAIL_RE.match(name):
|
||
raise HTTPException(400, "a Google invite must be the full email address")
|
||
|
||
email = (body.email or "").strip() or (name if body.provider == "google" else "")
|
||
if email and not EMAIL_RE.match(email):
|
||
raise HTTPException(400, "that does not look like an email address")
|
||
|
||
with db() as c:
|
||
if c.execute("SELECT 1 FROM users WHERE username=?", (name,)).fetchone():
|
||
raise HTTPException(409, "that user already exists")
|
||
c.execute(
|
||
"INSERT INTO users (username,provider,role,quota,storage_quota,"
|
||
"invited_by,email,notify) VALUES (?,?,?,?,?,?,?,?)",
|
||
(name, body.provider, body.role, max(0, body.quota),
|
||
int(max(0.0, body.storage_quota_gb) * 2**30), who,
|
||
email or None, 1 if body.notify else 0))
|
||
r = c.execute("SELECT * FROM users WHERE username=?", (name,)).fetchone()
|
||
|
||
log.info("admin: %s invited %s as %s (%s)", who, name, body.role, body.provider)
|
||
sent = bool(MAIL_ON and email)
|
||
if sent:
|
||
await notify_invited(r, who, email)
|
||
return {**user_dict(r), "invite_sent": sent,
|
||
"invite_why": None if sent else
|
||
("no SMTP configured on this instance" if not MAIL_ON
|
||
else "no email address for this account")}
|
||
|
||
|
||
@app.post("/api/admin/users/{username}/resend")
|
||
async def admin_resend_invite(username: str, request: Request):
|
||
"""Send the invitation again. Safe to call repeatedly — the link carries
|
||
no token, so resending neither extends nor grants anything."""
|
||
who = require_admin(request)
|
||
with db() as c:
|
||
r = c.execute("SELECT * FROM users WHERE username=?", (username,)).fetchone()
|
||
if not r:
|
||
raise HTTPException(404, "no such user")
|
||
if not MAIL_ON:
|
||
raise HTTPException(409, "no SMTP configured on this instance")
|
||
if not r["email"]:
|
||
raise HTTPException(409, "no email address on file for that account")
|
||
await notify_invited(r, who, r["email"])
|
||
return {"ok": True, "sent_to": r["email"]}
|
||
|
||
|
||
@app.patch("/api/admin/users/{username}")
|
||
async def admin_patch_user(username: str, body: PatchUser, request: Request):
|
||
who = require_admin(request)
|
||
with db() as c:
|
||
r = c.execute("SELECT * FROM users WHERE username=?", (username,)).fetchone()
|
||
if not r:
|
||
raise HTTPException(404, "no such user")
|
||
|
||
# An install with no admin cannot be administered, and nothing in the
|
||
# UI can undo it. Refuse to remove the last one — whether by demotion
|
||
# or by disabling them.
|
||
losing_admin = r["role"] == "admin" and (
|
||
(body.role is not None and body.role != "admin") or body.disabled is True)
|
||
if losing_admin:
|
||
others = c.execute(
|
||
"SELECT COUNT(*) n FROM users "
|
||
"WHERE role='admin' AND disabled=0 AND username<>?",
|
||
(username,)).fetchone()["n"]
|
||
if others == 0:
|
||
raise HTTPException(
|
||
409, "this is the only active admin — promote someone else first")
|
||
|
||
sets, vals = [], []
|
||
if body.role is not None:
|
||
if body.role not in ROLES:
|
||
raise HTTPException(400, f"role must be one of {', '.join(ROLES)}")
|
||
sets.append("role=?"); vals.append(body.role)
|
||
if body.quota is not None:
|
||
sets.append("quota=?"); vals.append(max(0, body.quota))
|
||
if body.storage_quota_gb is not None:
|
||
sets.append("storage_quota=?")
|
||
vals.append(int(max(0.0, body.storage_quota_gb) * 2**30))
|
||
if body.disabled is not None:
|
||
sets.append("disabled=?"); vals.append(1 if body.disabled else 0)
|
||
if body.email is not None:
|
||
e = body.email.strip()
|
||
if e and not EMAIL_RE.match(e):
|
||
raise HTTPException(400, "that does not look like an email address")
|
||
sets.append("email=?"); vals.append(e or None)
|
||
if not sets:
|
||
raise HTTPException(400, "nothing to change")
|
||
|
||
c.execute(f"UPDATE users SET {', '.join(sets)} WHERE username=?",
|
||
[*vals, username])
|
||
# Disabling must take effect now. session_user re-checks on every
|
||
# request anyway, but dropping the rows makes it unambiguous.
|
||
if body.disabled:
|
||
c.execute("DELETE FROM sessions WHERE username=?", (username,))
|
||
r = c.execute("SELECT * FROM users WHERE username=?", (username,)).fetchone()
|
||
log.info("admin: %s updated %s (%s)", who, username, ", ".join(sets))
|
||
return user_dict(r)
|
||
|
||
|
||
@app.delete("/api/admin/users/{username}")
|
||
async def admin_delete_user(username: str, request: Request):
|
||
who = require_admin(request)
|
||
with db() as c:
|
||
r = c.execute("SELECT * FROM users WHERE username=?", (username,)).fetchone()
|
||
if not r:
|
||
raise HTTPException(404, "no such user")
|
||
if r["role"] == "admin":
|
||
others = c.execute(
|
||
"SELECT COUNT(*) n FROM users "
|
||
"WHERE role='admin' AND disabled=0 AND username<>?",
|
||
(username,)).fetchone()["n"]
|
||
if others == 0:
|
||
raise HTTPException(409, "this is the only active admin")
|
||
c.execute("DELETE FROM sessions WHERE username=?", (username,))
|
||
c.execute("DELETE FROM users WHERE username=?", (username,))
|
||
# Their jobs are deliberately kept: the history is a record of what is in
|
||
# the library and why, and deleting a person should not rewrite it.
|
||
log.info("admin: %s deleted user %s (jobs retained)", who, username)
|
||
return {"ok": True, "jobs_retained": True}
|
||
|
||
|
||
@app.get("/api/admin/activity")
|
||
async def admin_activity(request: Request, days: int = Query(30, ge=1, le=365)):
|
||
require_admin(request)
|
||
days = max(1, min(days, 365))
|
||
since = (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
|
||
with db() as c:
|
||
daily = c.execute(
|
||
"SELECT substr(created_at,1,10) d, COUNT(*) n, "
|
||
" SUM(state='imported') imported, SUM(state='failed') failed "
|
||
"FROM jobs WHERE created_at >= ? GROUP BY d ORDER BY d", (since,)).fetchall()
|
||
per_user = c.execute(
|
||
"SELECT COALESCE(requested_by,'(unattributed)') u, COUNT(*) n, "
|
||
" COALESCE(SUM(CASE WHEN state='imported' THEN size END),0) bytes "
|
||
"FROM jobs WHERE created_at >= ? GROUP BY u ORDER BY n DESC",
|
||
(since,)).fetchall()
|
||
top = c.execute(
|
||
"SELECT artist, COUNT(*) n FROM jobs WHERE created_at >= ? "
|
||
"GROUP BY artist ORDER BY n DESC LIMIT 8", (since,)).fetchall()
|
||
|
||
# Zero-fill: a bar chart with gaps for quiet days misreads as missing data
|
||
# rather than as nothing having happened.
|
||
by_day = {r["d"]: r for r in daily}
|
||
today = datetime.now(timezone.utc).date()
|
||
series = []
|
||
for i in range(days - 1, -1, -1):
|
||
d = (today - timedelta(days=i)).isoformat()
|
||
r = by_day.get(d)
|
||
series.append({"date": d, "n": r["n"] if r else 0,
|
||
"imported": (r["imported"] or 0) if r else 0,
|
||
"failed": (r["failed"] or 0) if r else 0})
|
||
return {"days": days, "series": series,
|
||
"per_user": [dict(r) for r in per_user],
|
||
"top_artists": [dict(r) for r in top]}
|
||
|
||
|
||
def _proc_stat() -> tuple[int, int]:
|
||
"""(busy, total) jiffies from /proc/stat."""
|
||
with open("/proc/stat") as f:
|
||
parts = [int(x) for x in f.readline().split()[1:]]
|
||
idle = parts[3] + (parts[4] if len(parts) > 4 else 0)
|
||
return sum(parts) - idle, sum(parts)
|
||
|
||
|
||
_cpu_prev: tuple[int, int] | None = None
|
||
|
||
|
||
def system_stats() -> dict:
|
||
"""Host CPU/RAM plus disk for the paths that matter.
|
||
|
||
Reads /proc directly rather than adding psutil: it is ~30 lines, keeps the
|
||
image free of a compiled dependency, and /proc in a container reports the
|
||
HOST's CPU and memory — which is what you actually want to see for a
|
||
homelab box. Everything is individually guarded so a non-Linux dev machine
|
||
degrades to nulls instead of 500ing the whole panel.
|
||
"""
|
||
global _cpu_prev
|
||
out: dict = {}
|
||
|
||
try:
|
||
busy, total = _proc_stat()
|
||
if _cpu_prev:
|
||
db_, dt = busy - _cpu_prev[0], total - _cpu_prev[1]
|
||
out["cpu_pct"] = round(100 * db_ / dt, 1) if dt > 0 else None
|
||
else:
|
||
out["cpu_pct"] = None # first call has no baseline to diff
|
||
_cpu_prev = (busy, total)
|
||
except Exception:
|
||
out["cpu_pct"] = None
|
||
|
||
try:
|
||
mem = {}
|
||
with open("/proc/meminfo") as f:
|
||
for line in f:
|
||
k, _, v = line.partition(":")
|
||
mem[k] = int(v.split()[0]) * 1024
|
||
total_b = mem.get("MemTotal", 0)
|
||
avail = mem.get("MemAvailable", 0)
|
||
out["mem_total"] = total_b
|
||
out["mem_used"] = total_b - avail
|
||
out["mem_pct"] = round(100 * (total_b - avail) / total_b, 1) if total_b else None
|
||
except Exception:
|
||
out["mem_total"] = out["mem_used"] = out["mem_pct"] = None
|
||
|
||
try:
|
||
out["load"] = [round(x, 2) for x in os.getloadavg()]
|
||
out["cpus"] = os.cpu_count()
|
||
except Exception:
|
||
out["load"] = None
|
||
|
||
disks = {}
|
||
for label, p in (("library", LIBRARY_DIR), ("downloads", DOWNLOAD_DIR)):
|
||
try:
|
||
s = os.statvfs(p)
|
||
total_b = s.f_blocks * s.f_frsize
|
||
free = s.f_bavail * s.f_frsize
|
||
disks[label] = {"total": total_b, "used": total_b - free, "free": free,
|
||
"pct": round(100 * (total_b - free) / total_b, 1) if total_b else None}
|
||
except Exception:
|
||
disks[label] = None
|
||
out["disks"] = disks
|
||
return out
|
||
|
||
|
||
@app.get("/api/admin/system")
|
||
async def admin_system(request: Request):
|
||
require_admin(request)
|
||
out = {"system": system_stats(), "workers": WORKERS, "pending": work_q.qsize()}
|
||
|
||
with db() as c:
|
||
out["jobs"] = {r["state"]: r["n"] for r in c.execute(
|
||
"SELECT state, COUNT(*) n FROM jobs GROUP BY state").fetchall()}
|
||
out["library"] = {
|
||
"tracks": c.execute(
|
||
"SELECT COUNT(*) n FROM jobs WHERE state='imported'").fetchone()["n"],
|
||
"bytes": c.execute(
|
||
"SELECT COALESCE(SUM(size),0) b FROM jobs "
|
||
"WHERE state='imported'").fetchone()["b"],
|
||
}
|
||
out["users"] = c.execute("SELECT COUNT(*) n FROM users").fetchone()["n"]
|
||
out["sessions"] = c.execute(
|
||
"SELECT COUNT(*) n FROM sessions WHERE expires_at > ?",
|
||
(datetime.now(timezone.utc).isoformat(),)).fetchone()["n"]
|
||
|
||
slskd: dict = {"ok": False}
|
||
try:
|
||
async with slskd_client(PROBE_TIMEOUT) as sc:
|
||
r = await sc.get("/api/v0/application")
|
||
slskd["ok"] = r.status_code == 200
|
||
if r.status_code == 200:
|
||
d = r.json()
|
||
srv = d.get("server") or {}
|
||
slskd["connected"] = bool(srv.get("isConnected"))
|
||
slskd["logged_in"] = bool(srv.get("isLoggedIn"))
|
||
slskd["version"] = d.get("version")
|
||
shares = d.get("shares") or {}
|
||
slskd["shared_files"] = (shares.get("directories") or {}).get("fileCount") \
|
||
or shares.get("fileCount")
|
||
else:
|
||
slskd["error"] = f"HTTP {r.status_code}"
|
||
except Exception as e:
|
||
slskd["error"] = errstr(e)
|
||
out["slskd"] = slskd
|
||
return out
|
||
|
||
|
||
@app.get("/api/health")
|
||
async def health():
|
||
ok, detail = True, {}
|
||
async with slskd_client(PROBE_TIMEOUT) as sc:
|
||
try:
|
||
r = await sc.get("/api/v0/application")
|
||
detail["slskd"] = r.status_code
|
||
ok = ok and r.status_code == 200
|
||
# Reachable and logged out is the failure worth naming: every
|
||
# search returns nothing and every request fails, but the container
|
||
# is up and healthy so nothing else reports a problem. This used to
|
||
# be visible only on the admin panel, which meant an ordinary user
|
||
# watched their requests fail with no way to learn why.
|
||
if r.status_code == 200:
|
||
app_state = r.json()
|
||
srv = app_state.get("server") or {}
|
||
detail["logged_in"] = bool(srv.get("isLoggedIn"))
|
||
detail["shared_files"] = (
|
||
((app_state.get("user") or {}).get("statistics") or {})
|
||
.get("fileCount"))
|
||
ok = ok and detail["logged_in"]
|
||
except Exception as e:
|
||
detail["slskd"] = errstr(e)
|
||
ok = False
|
||
detail["library_writable"] = os.access(LIBRARY_DIR, os.W_OK)
|
||
detail["downloads_readable"] = os.access(DOWNLOAD_DIR, os.R_OK)
|
||
return {"ok": ok and detail["library_writable"], **detail}
|
||
|
||
|
||
@app.get("/")
|
||
async def index():
|
||
f = WEB / "index.html"
|
||
if not f.exists():
|
||
raise HTTPException(500, "UI not built — run `npm run build` in web/")
|
||
return FileResponse(f)
|
||
|
||
|
||
# Next.js emits its immutable, content-hashed chunks under /_next/static.
|
||
#
|
||
# Guarded rather than mounted unconditionally so that importing this module
|
||
# does not require build artifacts: CI typechecks the API before the UI has
|
||
# been built, and tests should not have to run npm first. A deploy that really
|
||
# is missing its UI says so at startup and again on the first request, rather
|
||
# than silently serving 404s.
|
||
if (WEB / "_next").is_dir():
|
||
app.mount("/_next", StaticFiles(directory=WEB / "_next"), name="next")
|
||
else:
|
||
log.warning("web/out/_next is missing — the UI was not built. "
|
||
"API endpoints work; the web page will not.")
|