diff --git a/README.md b/README.md index d92469b..af4dedd 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,61 @@ Deezer's API needs no key and no OAuth. Its search payload omits track number and release date, so those come from `/track/{id}` at request time — one extra call per request, not per search result. +## Bulk import + +Open **Import a list**, paste, hit **Queue all**. These all parse: + +``` +Radiohead - Karma Police +Juice WRLD – Lucid Dreams (en dash) +3. Gorillaz - Feel Good Inc (numbered list) +Black Hole Sun by Soundgarden +Come As You Are (bare title — Deezer copes) +https://deezer.com/album/12345 (expanded into its tracks) +https://deezer.com/playlist/6789 (same) +``` + +In bulk mode there's no picker — the top-ranked Deezer hit is taken +automatically. Every job shows what it *resolved to*, so a bad auto-pick is +visible in the list rather than quietly landing the wrong song. Lines that +match nothing are reported back as skipped instead of failing the batch. + +Failed jobs get a **Retry** button. Most Soulseek failures are the peer going +offline, not a bad request, so retrying later usually just works. + +### Everything goes through a bounded queue + +`MUSE_WORKERS` (default 3) caps concurrent Soulseek searches. This matters: +each search is 10-40s of polling plus real load on slskd, so one task per +request — which an earlier version did — means a 50-track paste launches 50 +concurrent searches and drowns the client. Requests return instantly and the +queue drains at a civil rate. + +The queue is in memory, so anything still `queued` or `searching` is put back +on it at startup. A restart mid-import resumes instead of stranding jobs. + +## How results are ranked + +Two separate rankings, and they fail in different ways. + +**Picking the track (Deezer).** Title relevance dominates — whatever query +words the artist name doesn't explain must appear in the title. That term +exists because without it a *completely different song by the right artist* +competes on equal footing: `Oasis - Live Forever` once resolved to +`Cloudburst`, because a small variant penalty was enough to tip between two +unrelated tracks. Artist match, popularity, and a penalty for derivative +versions (cover, karaoke, live, demo, remix…) come after. + +Derivative terms match on **word boundaries**, not substrings — a bare `live` +is needed to catch `Come As You Are (Live)`, but substring matching would also +penalise `Deliver Me` and `Olive`. And a term only counts against a result when +you didn't type it, so `oasis live forever` and `nirvana unplugged` still work. + +**Picking the file (Soulseek).** Format preference first, then whether the peer +can actually serve you (free upload slot, short queue), then filename +plausibility, then bitrate. A marginally better file behind 200 queued users +loses to a good-enough one available now. + ## Running it ```yaml @@ -99,6 +154,7 @@ stat -c "%h" "/data/media/music// ()/ - .flac" | `MUSE_DB` | `/config/muse.db` | job history | | `MUSE_FORMATS` | `flac,mp3` | ordered preference; anything else is rejected | | `MUSE_MIN_MATCH` | `0.45` | filename-to-title similarity floor | +| `MUSE_WORKERS` | `3` | concurrent Soulseek searches | | `MUSE_LOG_LEVEL` | `INFO` | | `MUSE_FORMATS` is a preference list, best first — rejecting everything not on @@ -143,6 +199,8 @@ package page → Package settings → Change visibility. | `GET /` | web UI | | `GET /api/search?q=` | Deezer track search | | `POST /api/request` | queue a track | +| `POST /api/import` | queue many from pasted text | +| `POST /api/retry/{id}` | requeue a failed job | | `GET /api/jobs` | request history and state | | `GET /api/health` | slskd reachability, mount writability | diff --git a/app.py b/app.py index f8f3977..5934100 100644 --- a/app.py +++ b/app.py @@ -62,6 +62,16 @@ FORMATS = [f.strip() for f in os.getenv("MUSE_FORMATS", "flac,mp3").split(",") i # so this compares against a normalised form, not the raw string. MIN_MATCH = float(os.getenv("MUSE_MIN_MATCH", "0.45")) +# How many Soulseek searches may run at once. +# +# This is a hard cap, not a suggestion. Each search takes 10-40s of polling +# and real load on slskd and the Soulseek server, so firing one task per +# request — which is what an earlier version did — means pasting a 50-track +# list launches 50 concurrent searches and drowns the client. Everything goes +# through a bounded queue instead: requests return instantly, work drains at a +# civil rate. +WORKERS = int(os.getenv("MUSE_WORKERS", "3")) + # Search uses Deezer's public API: no key, no OAuth, and vastly better # relevance than MusicBrainz for the way people actually type. # @@ -79,8 +89,18 @@ MIN_MATCH = float(os.getenv("MUSE_MIN_MATCH", "0.45")) DEEZER = "https://api.deezer.com" # Derivative versions to push down when the user did not ask for them. -DERIVATIVE = ("cover", "karaoke", "tribute", "remix", "instrumental", - "originally performed", "made popular by", "in the style of") +# Matched on WORD BOUNDARIES, not as substrings: a bare "live" is needed to +# catch "Come As You Are (Live)", but substring matching would also penalise +# "Deliver Me" and "Olive". Same trap for "demo" vs "Demolition". +# +# A term only counts against a result when the user did NOT type it, so asking +# for "oasis live forever" or "nirvana unplugged demo" still works. +DERIVATIVE = ("cover", "covers", "karaoke", "tribute", "remix", "remixed", + "instrumental", "acapella", "originally performed", + "made popular by", "in the style of", "demo", "rehearsal", + "live", "unplugged", "re recorded", "rerecorded", + "sped up", "slowed", "reverb", "8d", "nightcore") +DERIVATIVE_RE = re.compile(r"\b(" + "|".join(DERIVATIVE) + r")\b") SAFE = re.compile(r'[<>:"/\\|?*\x00-\x1f]') NOISE = re.compile(r"[^a-z0-9]+") @@ -305,7 +325,32 @@ def find_downloaded(remote_file: str) -> Path | None: return None -async def worker(): +# ─────────────────────────── search queue ─────────────────────────── + +work_q: "asyncio.Queue[str]" = asyncio.Queue() + + +async def search_worker(n: int): + """Drain the search queue, one job at a time, WORKERS of these running.""" + while True: + job_id = await work_q.get() + try: + with db() as c: + job = c.execute("SELECT * FROM jobs WHERE id=?", (job_id,)).fetchone() + if job and job["state"] in ("queued", "searching"): + set_state(job_id, "searching", "") + await pick_and_enqueue(job_id, job["artist"], job["title"]) + except Exception as e: + log.warning("worker %d: %s", n, e) + try: + set_state(job_id, "failed", str(e)[:200]) + except Exception: + pass + finally: + work_q.task_done() + + +async def import_worker(): """Poll slskd for in-flight downloads and import the finished ones.""" while True: await asyncio.sleep(10) @@ -347,11 +392,25 @@ async def worker(): @asynccontextmanager async def lifespan(_: FastAPI): init_db() - task = asyncio.create_task(worker()) - log.info("muse up — slskd=%s library=%s formats=%s", - SLSKD_URL, LIBRARY_DIR, ",".join(FORMATS)) + + # The queue lives in memory, so a restart mid-import would otherwise + # strand every unstarted job in 'queued' forever with nothing to pick it + # up. Anything not yet handed to slskd goes back on the queue. + with db() as c: + orphans = [r["id"] for r in c.execute( + "SELECT id FROM jobs WHERE state IN ('queued','searching')").fetchall()] + for jid in orphans: + work_q.put_nowait(jid) + if orphans: + log.info("requeued %d job(s) left over from the last run", len(orphans)) + + tasks = [asyncio.create_task(import_worker())] + tasks += [asyncio.create_task(search_worker(i)) for i in range(WORKERS)] + log.info("muse up — slskd=%s library=%s formats=%s workers=%d", + SLSKD_URL, LIBRARY_DIR, ",".join(FORMATS), WORKERS) yield - task.cancel() + for t in tasks: + t.cancel() app = FastAPI(title="muse", lifespan=lifespan) @@ -366,11 +425,10 @@ class RequestBody(BaseModel): deezer_id: int | None = None -@app.get("/api/search") -async def search(q: str, limit: int = 25): - """Track search. Returns individual songs, not albums — the whole point.""" +async def deezer_search(q: str, limit: int = 25) -> list[dict]: + """Ranked Deezer track search. Shared by /api/search and /api/import.""" if not q.strip(): - return {"results": []} + return [] async with httpx.AsyncClient(timeout=20) as c: r = await c.get(f"{DEEZER}/search", params={"q": q, "limit": limit * 2}) if r.status_code != 200: @@ -385,13 +443,31 @@ async def search(q: str, limit: int = 25): album = (t.get("album") or {}).get("title") or "" s = float(t.get("rank") or 0) / 10_000 # Deezer popularity, 0-~100 - # An artist match is a far stronger signal than a title match: it is - # what separates the real track from a covers compilation. + + # An artist match separates the real track from a covers compilation. + at = set(norm(artist).split()) if norm(artist) and norm(artist) in qn: s += 300 - for w in DERIVATIVE: - if w in norm(title) and w not in qn: + + # Title relevance. This has to dominate everything else: without it a + # completely different song by the right artist competes on equal + # footing, and "Oasis - Live Forever" resolved to "Cloudburst" because + # a variant penalty was enough to tip between two unrelated tracks. + # Whatever query words the artist name did not explain should show up + # in the title. + rest = set(qn.split()) - at + if rest: + s += 400 * len(rest & set(norm(title).split())) / len(rest) + for w in set(DERIVATIVE_RE.findall(norm(title))): + if not re.search(rf"\b{re.escape(w)}\b", qn): s -= 150 + # Catch-all for variants the keyword list will never cover in full + # ("Devonshire Mix", "Radio Edit", "2011 Version"): a plain title beats + # a parenthesised one. Deliberately small -- it should break ties + # between versions of the SAME song, never outweigh title relevance + # above and pick a different song. + if ("(" in title or "[" in title) and "(" not in q: + s -= 40 scored.append((s, { "deezer_id": t.get("id"), "title": title, @@ -410,48 +486,157 @@ async def search(q: str, limit: int = 25): continue seen.add(key) out.append(item) - return {"results": out[:limit]} + return out[:limit] -@app.post("/api/request") -async def request_track(body: RequestBody): - artist, title = body.artist, body.title - album, year, track = body.album, body.year, body.track +@app.get("/api/search") +async def search(q: str, limit: int = 25): + """Track search. Returns individual songs, not albums — the whole point.""" + return {"results": await deezer_search(q, limit)} - # Deezer's search payload has no track number or release date, and both go - # into the filename and tags. One lookup here beats one per search result. - if body.deezer_id: - try: - async with httpx.AsyncClient(timeout=15) as c: - r = await c.get(f"{DEEZER}/track/{body.deezer_id}") - if r.status_code == 200: - d = r.json() - track = d.get("track_position") or track - rd = d.get("release_date") or (d.get("album") or {}).get("release_date") - year = (rd or "")[:4] or year - album = (d.get("album") or {}).get("title") or album - artist = (d.get("artist") or {}).get("name") or artist - except Exception as e: - log.warning("deezer track lookup failed: %s", e) +async def enqueue_job(artist: str, title: str, album=None, year=None, + track=None, deezer_id=None) -> str: + """Create a job and hand it to the search queue.""" job_id = str(uuid.uuid4()) with db() as c: c.execute( "INSERT INTO jobs (id,state,detail,artist,title,album,year,track," "recording_mbid,release_mbid) VALUES (?,?,?,?,?,?,?,?,?,?)", - (job_id, "searching", "", artist, title, album, - year, track, str(body.deezer_id or ""), None)) - asyncio.create_task(pick_and_enqueue(job_id, artist, title)) - return {"id": job_id} + (job_id, "queued", "", artist, title, album, year, track, + str(deezer_id or ""), None)) + await work_q.put(job_id) + return job_id + + +async def enrich(deezer_id) -> dict: + """Deezer's search payload has no track number or release date, and both + go into the filename and tags. One lookup per request, not per result.""" + if not deezer_id: + return {} + try: + async with httpx.AsyncClient(timeout=15) as c: + r = await c.get(f"{DEEZER}/track/{deezer_id}") + if r.status_code != 200: + return {} + d = r.json() + rd = d.get("release_date") or (d.get("album") or {}).get("release_date") + return { + "track": d.get("track_position"), + "year": (rd or "")[:4] or None, + "album": (d.get("album") or {}).get("title"), + "artist": (d.get("artist") or {}).get("name"), + } + except Exception as e: + log.warning("deezer track lookup failed: %s", e) + return {} + + +@app.post("/api/request") +async def request_track(body: RequestBody): + e = await enrich(body.deezer_id) + return {"id": await enqueue_job( + e.get("artist") or body.artist, body.title, + e.get("album") or body.album, e.get("year") or body.year, + e.get("track") or body.track, body.deezer_id)} + + +class ImportBody(BaseModel): + text: str + + +# "Artist - Title", "Artist – Title" (en dash), "Title by Artist". Anything +# that doesn't split just goes to Deezer whole — its search copes fine with +# free text, and guessing wrong is worse than not guessing. +SPLIT = re.compile(r"\s+[-–—|]\s+|\s+by\s+", re.I) +DEEZER_URL = re.compile(r"deezer\.com/(?:[a-z]{2}/)?(album|playlist)/(\d+)") + + +async def expand_url(kind: str, ident: str) -> list[str]: + """Turn a Deezer album/playlist URL into 'Artist - Title' lines.""" + out = [] + try: + async with httpx.AsyncClient(timeout=25) as c: + r = await c.get(f"{DEEZER}/{kind}/{ident}") + if r.status_code != 200: + return [] + for t in (r.json().get("tracks") or {}).get("data", []): + a = (t.get("artist") or {}).get("name") or "" + out.append(f"{a} - {t.get('title')}" if a else t.get("title", "")) + except Exception as e: + log.warning("expanding %s/%s failed: %s", kind, ident, e) + return [x for x in out if x] + + +@app.post("/api/import") +async def import_list(body: ImportBody): + """Paste a list, get it all queued. + + Each line is resolved against Deezer and the top-ranked hit is taken + automatically — there is no picker in bulk mode. Resolved artist/album are + recorded on every job so a wrong auto-pick is visible in the list rather + than silently landing the wrong song in the library. + """ + lines, queued, skipped = [], [], [] + for raw in body.text.splitlines(): + line = raw.strip().lstrip("-•*0123456789. )").strip() + if not line: + continue + m = DEEZER_URL.search(line) + lines.extend(await expand_url(*m.groups()) if m else [line]) + + for line in lines: + # Search the line as typed; the ranker already weights an artist-name + # match heavily, so "Artist - Title" needs no special handling. + try: + # Ask for a real candidate pool, not one row. deezer_search ranks + # what it is given, so limit=1 fetched two results and handed back + # the better of a demo and a rehearsal take while the studio + # version -- 6x the popularity -- sat further down the list unseen. + hits = await deezer_search(SPLIT.sub(" ", line), limit=10) + except Exception as e: + skipped.append({"line": line, "why": str(e)[:120]}) + continue + if not hits: + skipped.append({"line": line, "why": "no match on Deezer"}) + continue + h = hits[0] + e = await enrich(h["deezer_id"]) + jid = await enqueue_job( + e.get("artist") or h["artist"], h["title"], + e.get("album") or h["album"], e.get("year"), e.get("track"), + h["deezer_id"]) + queued.append({"id": jid, "line": line, + "matched": f"{h['artist']} — {h['title']}"}) + + log.info("import: %d queued, %d skipped", len(queued), len(skipped)) + return {"queued": queued, "skipped": skipped, "pending": work_q.qsize()} @app.get("/api/jobs") -async def jobs(limit: int = 50): +async def jobs(limit: int = 200): with db() as c: rows = c.execute( "SELECT * FROM jobs ORDER BY created_at DESC, rowid DESC LIMIT ?", (limit,)).fetchall() - return {"jobs": [dict(r) for r in rows]} + counts = {r["state"]: r["n"] for r in c.execute( + "SELECT state, COUNT(*) n FROM jobs GROUP BY state").fetchall()} + return {"jobs": [dict(r) for r in rows], "counts": counts, + "pending": work_q.qsize()} + + +@app.post("/api/retry/{job_id}") +async def retry(job_id: str): + """Put a failed job back on the queue. Soulseek failures are usually the + peer going offline, not the request being wrong, so retrying later often + just works.""" + with db() as c: + row = c.execute("SELECT id FROM jobs WHERE id=?", (job_id,)).fetchone() + if not row: + raise HTTPException(404, "no such job") + set_state(job_id, "queued", "") + await work_q.put(job_id) + return {"ok": True} @app.get("/api/health") diff --git a/static/index.html b/static/index.html index 8b46b34..bb1adf9 100644 --- a/static/index.html +++ b/static/index.html @@ -40,7 +40,23 @@ padding:7px 14px;font-size:14px} .get.done{border-color:var(--ok);color:var(--ok)} h2{font-size:13px;text-transform:uppercase;letter-spacing:.6px;color:var(--dim); - margin:26px 0 10px} + margin:26px 0 10px;display:flex;align-items:center;gap:10px} + h2 .count{text-transform:none;letter-spacing:0;font-weight:400;font-size:12px} + details{margin-top:10px} + summary{cursor:pointer;color:var(--dim);font-size:13px;user-select:none} + summary:hover{color:var(--fg)} + textarea{width:100%;min-height:120px;margin-top:9px;padding:11px 13px; + border-radius:9px;border:1px solid var(--line);background:var(--panel); + color:var(--fg);font:13px/1.6 ui-monospace,SFMono-Regular,Menlo,monospace; + resize:vertical} + textarea:focus{outline:2px solid var(--accent);outline-offset:-1px} + .hint{color:var(--dim);font-size:12px;margin-top:7px} + .bar{height:3px;border-radius:2px;background:var(--line);overflow:hidden; + margin-top:10px} + .bar i{display:block;height:100%;background:var(--accent);transition:width .4s} + .retry{background:transparent;border:1px solid var(--line);color:var(--dim); + padding:5px 11px;font-size:12px;font-weight:400} + .skipped{color:var(--err);font-size:12px;margin-top:8px} .pill{font-size:12px;padding:2px 9px;border-radius:99px;border:1px solid var(--line); color:var(--dim);white-space:nowrap} .pill.imported{color:var(--ok);border-color:var(--ok)} @@ -56,11 +72,27 @@ <input id="q" placeholder="artist, song title…" autocomplete="off" autofocus> <button id="go">Search</button> </form> + <details id="imp"> + <summary>Import a list — paste many songs at once</summary> + <textarea id="bulk" placeholder="One per line: + +Radiohead - Creep +Juice WRLD - Lucid Dreams +Tory Lanez - The Color Violet + +Or paste a Deezer album / playlist URL to expand it."></textarea> + <div style="display:flex;gap:9px;align-items:center;margin-top:9px"> + <button id="imprun" type="button">Queue all</button> + <span class="hint" id="imphint">Top Deezer match is taken automatically — check the list below.</span> + </div> + <div id="impout"></div> + </details> </div></header> <main class="wrap"> <div id="results"></div> - <h2>Requests</h2> + <h2>Requests <span class="count" id="qcount"></span></h2> + <div class="bar" id="bar" style="display:none"><i style="width:0"></i></div> <div id="jobs"><div class="empty">nothing requested yet</div></div> </main> @@ -115,21 +147,74 @@ function render(list) { }); } +$('#imprun').onclick = async () => { + const text = $('#bulk').value.trim(); + if (!text) return; + const b = $('#imprun'); + b.disabled = true; b.textContent = 'Queueing…'; + $('#impout').innerHTML = ''; + try { + const r = await fetch('/api/import', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text }) + }); + const d = await r.json(); + $('#impout').innerHTML = + `<div class="hint">Queued ${d.queued.length}.</div>` + + (d.skipped.length + ? `<div class="skipped">Skipped ${d.skipped.length}: ` + + d.skipped.map(s => esc(s.line) + ' (' + esc(s.why) + ')').join('; ') + '</div>' + : ''); + if (d.queued.length) $('#bulk').value = ''; + poll(); + } catch (e) { + $('#impout').innerHTML = '<div class="skipped">import failed: ' + esc(e.message) + '</div>'; + } + b.disabled = false; b.textContent = 'Queue all'; +}; + async function poll() { try { - const { jobs } = await (await fetch('/api/jobs')).json(); + const { jobs, counts, pending } = await (await fetch('/api/jobs')).json(); + + const active = (counts.queued || 0) + (counts.searching || 0) + (counts.downloading || 0); + const done = counts.imported || 0; + $('#qcount').textContent = active + ? `${active} in flight${pending ? ` · ${pending} waiting` : ''}` + : (done ? `${done} imported` : ''); + + // Progress only means something during a batch. + const bar = $('#bar'), total = active + done; + if (active && total) { + bar.style.display = 'block'; + bar.firstElementChild.style.width = (done / total * 100).toFixed(1) + '%'; + } else { + bar.style.display = 'none'; + } + $('#jobs').innerHTML = jobs.length ? jobs.map(j => ` <div class="row"> <div class="meta"> <div class="t">${esc(j.title)}</div> <div class="s">${esc(j.artist)}${j.detail ? ' · ' + esc(j.detail) : ''}</div> </div> + ${j.state === 'failed' + ? `<button class="retry" data-id="${esc(j.id)}">Retry</button>` : ''} <span class="pill ${esc(j.state)}">${esc(j.state)}</span> </div>`).join('') : '<div class="empty">nothing requested yet</div>'; + + $('#jobs').querySelectorAll('.retry').forEach(b => { + b.onclick = async () => { + b.disabled = true; + await fetch('/api/retry/' + b.dataset.id, { method: 'POST' }); + poll(); + }; + }); } catch {} } poll(); -setInterval(poll, 5000); +setInterval(poll, 4000); </script> </body> </html>