From ef1e80911a132d74e24ee43f79ad72b5b0d0e8e2 Mon Sep 17 00:00:00 2001 From: Dallas Groot Date: Sat, 28 Mar 2026 14:19:37 -0700 Subject: [PATCH] Bundled companionapi required for batch processing and smartdj Bundled companionapi required for batch processing and smartdj. --- companion-api/Dockerfile | 34 ++ companion-api/diagnose.py | 175 +++++++ companion-api/docker-compose.yml | 45 ++ companion-api/main.py | 786 +++++++++++++++++++++++++++++++ companion-api/pre_analyze.py | 302 ++++++++++++ 5 files changed, 1342 insertions(+) create mode 100644 companion-api/Dockerfile create mode 100644 companion-api/diagnose.py create mode 100644 companion-api/docker-compose.yml create mode 100644 companion-api/main.py create mode 100644 companion-api/pre_analyze.py diff --git a/companion-api/Dockerfile b/companion-api/Dockerfile new file mode 100644 index 0000000..fedb99f --- /dev/null +++ b/companion-api/Dockerfile @@ -0,0 +1,34 @@ +# Location: /home/pi/docker/navidrome/companion_api/Dockerfile + +FROM python:3.11-slim + +# System dependencies +RUN apt-get update && apt-get install -y \ + ffmpeg \ + libsndfile1 \ + curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Python dependencies +RUN pip install --no-cache-dir \ + fastapi \ + "uvicorn[standard]" \ + mutagen \ + httpx \ + python-multipart \ + librosa \ + numpy \ + tqdm \ + websockets + +# Copy application code +COPY . . + +# Data directories (mounted as volume, created as fallback) +RUN mkdir -p /app/data /app/data/vis_cache + +EXPOSE 8000 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/companion-api/diagnose.py b/companion-api/diagnose.py new file mode 100644 index 0000000..ae34a0a --- /dev/null +++ b/companion-api/diagnose.py @@ -0,0 +1,175 @@ +""" +Diagnostic script — run inside the music-companion container: + docker exec -it music-companion python diagnose.py + +Checks: + 1. What MUSIC_DIR points to and what's in it + 2. What paths are stored in smart_dj.db + 3. Whether those DB paths actually exist on disk + 4. Tests resolve_path() with sample paths +""" +import os, sqlite3, hashlib + +MUSIC_DIR = os.getenv("MUSIC_DIR", "/music") +DB_PATH = os.getenv("DB_PATH", "/app/data/smart_dj.db") +VIS_CACHE_DIR = os.getenv("VIS_CACHE_DIR", "/app/data/vis_cache") + +print("=" * 60) +print("COMPANION API DIAGNOSTICS") +print("=" * 60) + +# 1. Check MUSIC_DIR +print(f"\n1. MUSIC_DIR = {MUSIC_DIR}") +print(f" Exists: {os.path.isdir(MUSIC_DIR)}") + +if os.path.isdir(MUSIC_DIR): + top = os.listdir(MUSIC_DIR) + dirs = sorted([d for d in top if os.path.isdir(os.path.join(MUSIC_DIR, d))]) + files = [f for f in top if os.path.isfile(os.path.join(MUSIC_DIR, f))] + print(f" Top-level: {len(dirs)} folders, {len(files)} files") + + # Show first 10 folders + for d in dirs[:10]: + print(f" šŸ“ {d}") + if len(dirs) > 10: + print(f" ... and {len(dirs) - 10} more") + + # Check if there's a nested "music" folder (common misconfiguration) + if "music" in dirs: + nested = os.path.join(MUSIC_DIR, "music") + nested_contents = os.listdir(nested) + print(f"\n āš ļø FOUND nested 'music/' folder inside MUSIC_DIR!") + print(f" {nested} has {len(nested_contents)} items") + print(f" This usually means docker-compose mounts the parent instead of the music dir") + print(f" Your files are probably at /music/music/Artist/Album/song.flac") + print(f" Fix: change volume mount from '/home/pi/navidrome:/music' to '/home/pi/navidrome/music:/music'") + + # Count total audio files + audio_count = 0 + sample_paths = [] + for root, _, fnames in os.walk(MUSIC_DIR): + for f in fnames: + if f.lower().endswith(('.mp3', '.flac', '.m4a', '.ogg', '.opus', '.wav')): + audio_count += 1 + fp = os.path.join(root, f) + rel = os.path.relpath(fp, MUSIC_DIR) + if len(sample_paths) < 5: + sample_paths.append((fp, rel)) + + print(f"\n Total audio files: {audio_count}") + print(f"\n Sample file paths (absolute → relative):") + for abs_path, rel_path in sample_paths: + print(f" ABS: {abs_path}") + print(f" REL: {rel_path}") + print() +else: + print(" āŒ MUSIC_DIR does not exist!") + +# 2. Check DB +print(f"\n2. DATABASE = {DB_PATH}") +print(f" Exists: {os.path.isfile(DB_PATH)}") + +db_paths = [] +if os.path.isfile(DB_PATH): + with sqlite3.connect(DB_PATH) as c: + count = c.execute("SELECT COUNT(*) FROM dj_profiles").fetchone()[0] + print(f" Profiles: {count}") + + rows = c.execute("SELECT file_path FROM dj_profiles LIMIT 10").fetchall() + db_paths = [r[0] for r in rows] + + print(f"\n Sample DB paths:") + for p in db_paths: + exists = os.path.isfile(p) + icon = "āœ“" if exists else "āœ—" + print(f" {icon} {p}") + if not exists: + # Try to find what DOES exist + basename = os.path.basename(p) + for root, _, fnames in os.walk(MUSIC_DIR): + if basename in fnames: + actual = os.path.join(root, basename) + print(f" → FOUND at: {actual}") + break + + # Check how many DB paths actually exist + all_paths = c.execute("SELECT file_path FROM dj_profiles").fetchall() + existing = sum(1 for (p,) in all_paths if os.path.isfile(p)) + missing = len(all_paths) - existing + print(f"\n DB path health: {existing} exist, {missing} broken") + if missing > 0: + print(f" āš ļø {missing} profiles point to files that don't exist!") + print(f" This means the DB was built with a different mount path") +else: + print(" āŒ Database does not exist! Run pre_analyze.py first") + +# 3. Check vis cache +print(f"\n3. VIS CACHE = {VIS_CACHE_DIR}") +if os.path.isdir(VIS_CACHE_DIR): + vis_files = os.listdir(VIS_CACHE_DIR) + print(f" Cached: {len(vis_files)} files") +else: + print(f" Does not exist") + +# 4. Test resolve_path with the sample paths +print(f"\n4. RESOLVE_PATH TESTS") +print(f" Testing with various path formats...") + +from pathlib import Path + +def resolve_path(relative): + cleaned = relative.lstrip("/") + direct = os.path.join(MUSIC_DIR, cleaned) + if os.path.isfile(direct): + return ("direct", direct) + parts = Path(cleaned).parts + for i in range(1, len(parts)): + sub = os.path.join(MUSIC_DIR, *parts[i:]) + if os.path.isfile(sub): + return (f"strip-{i}", sub) + target = os.path.basename(cleaned) + if target: + for root, _, files in os.walk(MUSIC_DIR): + if target in files: + return ("filename-search", os.path.join(root, target)) + return ("FAILED", None) + +if sample_paths: + for abs_path, rel_path in sample_paths: + # Test various formats the iOS app might send + tests = [ + ("exact relative", rel_path), + ("with music/ prefix", f"music/{rel_path}"), + ("with leading /", f"/{rel_path}"), + ("absolute", abs_path), + ] + print(f"\n File: {os.path.basename(rel_path)}") + for label, test_path in tests: + strategy, result = resolve_path(test_path) + icon = "āœ“" if result else "āœ—" + print(f" {icon} {label}: '{test_path}' → {strategy}") + break # Just test one file + +# 5. Summary +print(f"\n{'=' * 60}") +print("SUMMARY") +print(f"{'=' * 60}") + +if os.path.isdir(MUSIC_DIR): + if "music" in [d for d in os.listdir(MUSIC_DIR) if os.path.isdir(os.path.join(MUSIC_DIR, d))]: + print("āŒ LIKELY ISSUE: Nested 'music' folder detected.") + print(" Your docker-compose probably mounts /home/pi/navidrome:/music") + print(" but should mount /home/pi/navidrome/music:/music") + elif audio_count == 0: + print("āŒ No audio files found in MUSIC_DIR!") + elif db_paths and not os.path.isfile(db_paths[0]): + print("āŒ DB paths don't match filesystem. The DB was built with") + print(" a different mount configuration. Fix mount and re-run:") + print(" docker exec -it music-companion python pre_analyze.py --force") + else: + print("āœ“ Configuration looks correct") + print(f" {audio_count} audio files, {count if os.path.isfile(DB_PATH) else 0} profiles") +else: + print("āŒ MUSIC_DIR doesn't exist — check docker-compose volumes") + +print() diff --git a/companion-api/docker-compose.yml b/companion-api/docker-compose.yml new file mode 100644 index 0000000..6402f62 --- /dev/null +++ b/companion-api/docker-compose.yml @@ -0,0 +1,45 @@ +# Location: /home/pi/docker/navidrome/docker-compose.yml + +services: + navidrome: + image: deluan/navidrome:latest + container_name: navidrome + restart: unless-stopped + ports: + - "4533:4533" + environment: + - ND_SCANSCHEDULE=1h + - ND_BASEURL=/navidrome + # ... other env vars ... + volumes: + # Absolute path to your music + - /home/pi/navidrome:/music:ro + - ./navidrome_data:/data + + music-companion: + build: ./companion_api + container_name: music-companion + restart: unless-stopped + ports: + - "8000:8000" + volumes: + # Mount the ACTUAL music directory — not the parent navidrome folder + # Host: /home/pi/navidrome/music → Container: /music + - /home/pi/navidrome/music:/music:rw + # Persistent data: Smart DJ DB + visualizer frame cache + - ./companion_data:/app/data + environment: + - MUSIC_DIR=/music + - DB_PATH=/app/data/smart_dj.db + - VIS_CACHE_DIR=/app/data/vis_cache + - NAVIDROME_URL=http://navidrome:4533/navidrome + - SUBSONIC_USER=dallas + - SUBSONIC_TOKEN=your_token + - SUBSONIC_SALT=your_salt + deploy: + resources: + limits: + cpus: '2.0' + memory: 1G + depends_on: + - navidrome diff --git a/companion-api/main.py b/companion-api/main.py new file mode 100644 index 0000000..a397894 --- /dev/null +++ b/companion-api/main.py @@ -0,0 +1,786 @@ +""" +Navidrome Companion API +Location: /home/pi/docker/navidrome/companion_api/main.py + +Endpoints: + GET /health + PATCH /edit-metadata + PATCH /batch-edit-metadata — edit tags on multiple files at once + POST /upload-track — single file upload + POST /upload-tracks — multi-file batch upload with per-track metadata + GET /smart-dj/profile + GET /smart-dj/bulk-profiles + POST /bulk-fix + GET /visualizer/frames + POST /visualizer/precompute + WS /ws/push +""" +import os, re, json, asyncio, hashlib, sqlite3, subprocess, shutil, time, warnings +from pathlib import Path +from typing import Optional, List +from contextlib import asynccontextmanager +from urllib.parse import unquote + +import httpx +import numpy as np +from fastapi import (FastAPI, HTTPException, UploadFile, File, Form, + BackgroundTasks, WebSocket, WebSocketDisconnect, Query) +from fastapi.responses import JSONResponse +from pydantic import BaseModel + +warnings.filterwarnings("ignore", category=UserWarning) +warnings.filterwarnings("ignore", category=FutureWarning) +from mutagen import File as MutagenFile + +MUSIC_DIR = os.getenv("MUSIC_DIR", "/music") +DB_PATH = os.getenv("DB_PATH", "/app/data/smart_dj.db") +VIS_CACHE_DIR = os.getenv("VIS_CACHE_DIR", "/app/data/vis_cache") +NAVIDROME_URL = os.getenv("NAVIDROME_URL", "http://navidrome:4533/navidrome") +SUBSONIC_USER = os.getenv("SUBSONIC_USER") +SUBSONIC_TOKEN = os.getenv("SUBSONIC_TOKEN") +SUBSONIC_SALT = os.getenv("SUBSONIC_SALT") + + +# ── DB ───────────────────────────────────────────────────── +def init_db(): + os.makedirs(os.path.dirname(DB_PATH), exist_ok=True) + with sqlite3.connect(DB_PATH) as c: + c.execute("""CREATE TABLE IF NOT EXISTS dj_profiles ( + file_path TEXT PRIMARY KEY, bpm REAL, + silence_start REAL, silence_end REAL, + loudness_lufs REAL, + analyzed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)""") + c.execute("""CREATE TABLE IF NOT EXISTS file_index ( + basename TEXT, + full_path TEXT, + title_words TEXT, + PRIMARY KEY (basename, full_path))""") + + +def build_file_index(): + """Scan MUSIC_DIR and index every audio file by basename and title words.""" + exts = ('.mp3', '.flac', '.m4a', '.ogg', '.opus', '.wav') + with sqlite3.connect(DB_PATH) as c: + c.execute("DELETE FROM file_index") + count = 0 + for root, _, files in os.walk(MUSIC_DIR): + for f in files: + if f.lower().endswith(exts): + fp = os.path.join(root, f) + stem = Path(f).stem.lower() + words = set(re.split(r'[\s\-_\.]+', stem)) + words = {w for w in words if len(w) > 1 and not w.isdigit()} + c.execute("INSERT OR REPLACE INTO file_index VALUES (?,?,?)", + (f.lower(), fp, " ".join(sorted(words)))) + count += 1 + print(f" File index built: {count} files", flush=True) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + init_db() + os.makedirs(VIS_CACHE_DIR, exist_ok=True) + print(f"šŸŽµ Companion API ready", flush=True) + print(f" MUSIC_DIR = {MUSIC_DIR}", flush=True) + print(f" DB_PATH = {DB_PATH}", flush=True) + print(f" VIS_CACHE = {VIS_CACHE_DIR}", flush=True) + try: + top = os.listdir(MUSIC_DIR) + dirs = [d for d in top if os.path.isdir(os.path.join(MUSIC_DIR, d))] + print(f" MUSIC_DIR has {len(dirs)} folders at top level", flush=True) + except Exception as e: + print(f" āš ļø Cannot list MUSIC_DIR: {e}", flush=True) + build_file_index() + yield + +app = FastAPI(title="Navidrome Companion API", lifespan=lifespan) + + +# ── Models ───────────────────────────────────────────────── +class MetadataUpdate(BaseModel): + relative_path: str + title: Optional[str] = None + artist: Optional[str] = None + album: Optional[str] = None + album_artist: Optional[str] = None + genre: Optional[str] = None + year: Optional[int] = None + track_number: Optional[int] = None + +class BatchMetadataUpdate(BaseModel): + """Edit the same tags on multiple files at once.""" + relative_paths: List[str] + title: Optional[str] = None + artist: Optional[str] = None + album: Optional[str] = None + album_artist: Optional[str] = None + genre: Optional[str] = None + year: Optional[int] = None + +class TrackUploadMeta(BaseModel): + """Per-track metadata for multi-file upload.""" + filename: str + title: str + artist: str + album: str + track_number: Optional[int] = None + genre: Optional[str] = None + year: Optional[int] = None + + +# ── Push Manager (WebSocket) ────────────────────────────── +class PushManager: + def __init__(self): + self.connections: list[WebSocket] = [] + + async def connect(self, ws: WebSocket): + await ws.accept() + self.connections.append(ws) + print(f"šŸ“± Client connected ({len(self.connections)} total)") + + def disconnect(self, ws: WebSocket): + if ws in self.connections: + self.connections.remove(ws) + print(f"šŸ“± Client disconnected ({len(self.connections)} total)") + + async def broadcast(self, event: str, data: dict): + msg = json.dumps({"event": event, "data": data}) + dead = [] + for ws in self.connections: + try: + await ws.send_text(msg) + except: + dead.append(ws) + for ws in dead: + self.connections.remove(ws) + + async def send_to(self, ws: WebSocket, event: str, data: dict): + await ws.send_text(json.dumps({"event": event, "data": data})) + +push = PushManager() + + +# ── Path Resolution ─────────────────────────────────────── +def resolve_path(relative: str) -> Optional[str]: + """ + Resolve a Navidrome song.path to an actual file inside the container. + + Handles: + - URL-encoded paths (David%20Graham → David Graham) + - Double/triple encoding (%2520 → %20 → space) + - Navidrome's "music/" prefix when mounts differ + - Filename-only search as fallback + """ + # Fully URL-decode — keep decoding until stable + decoded = relative + for _ in range(5): + next_decoded = unquote(decoded) + if next_decoded == decoded: + break + decoded = next_decoded + + cleaned = decoded.lstrip("/") + + # Log what we're looking for + print(f" resolve_path: '{relative}' → decoded: '{cleaned}'", flush=True) + + direct = os.path.join(MUSIC_DIR, cleaned) + if os.path.isfile(direct): + return direct + + parts = Path(cleaned).parts + for i in range(1, len(parts)): + sub = os.path.join(MUSIC_DIR, *parts[i:]) + if os.path.isfile(sub): + return sub + + target = os.path.basename(cleaned) + if target: + for root, _, files in os.walk(MUSIC_DIR): + if target in files: + return os.path.join(root, target) + + # Strategy 4: Fuzzy match via file_index DB + target_stem = Path(target).stem.lower() if target else "" + target_ext = Path(target).suffix.lower() if target else "" + # Remove track number prefix like "09 - " or "09. " + title_part = re.sub(r'^\d+[\s\.\-]+', '', target_stem).strip() + search_words = set(re.split(r'[\s\-_\.]+', title_part)) + search_words = {w for w in search_words if len(w) > 1 and not w.isdigit()} + + if search_words: + try: + with sqlite3.connect(DB_PATH) as c: + rows = c.execute("SELECT basename, full_path, title_words FROM file_index").fetchall() + + best_match = None + best_score = 0.0 + + for basename, full_path, title_words_str in rows: + if not basename.endswith(target_ext): + continue + file_words = set(title_words_str.split()) + if not file_words: + continue + overlap = len(search_words & file_words) + score = overlap / len(search_words) + if score > best_score: + best_score = score + best_match = full_path + + if best_match and best_score >= 0.5: + print(f" resolve: fuzzy ({best_score:.0%}) → {os.path.basename(best_match)}", flush=True) + return best_match + except Exception as e: + print(f" resolve: fuzzy error: {e}", flush=True) + + print(f" resolve_path: FAILED for '{cleaned}'", flush=True) + return None + + +# ── Navidrome Scan ──────────────────────────────────────── +async def trigger_scan(): + if not all([SUBSONIC_USER, SUBSONIC_TOKEN, SUBSONIC_SALT]): + print("āš ļø Subsonic credentials not set — skipping scan") + return + params = { + "u": SUBSONIC_USER, "t": SUBSONIC_TOKEN, "s": SUBSONIC_SALT, + "v": "1.16.1", "c": "CompanionAPI", "f": "json" + } + async with httpx.AsyncClient() as client: + try: + r = await client.get(f"{NAVIDROME_URL}/rest/startScan.view", params=params) + print(f"Navidrome scan: {r.status_code}") + except Exception as e: + print(f"Scan failed: {e}") + + +# ── Metadata ────────────────────────────────────────────── +def apply_tags(path: str, u: MetadataUpdate): + audio = MutagenFile(path, easy=True) + if audio is None: + raise ValueError(f"Unsupported format: {path}") + if u.title: audio['title'] = u.title + if u.artist: audio['artist'] = u.artist + if u.album: audio['album'] = u.album + if u.album_artist: audio['albumartist'] = u.album_artist + if u.genre: audio['genre'] = u.genre + if u.year: audio['date'] = str(u.year) + if u.track_number: audio['tracknumber'] = str(u.track_number) + audio.save() + + +def apply_tags_dict(path: str, tags: dict): + """Apply tags from a plain dict — used by batch/multi-track endpoints.""" + audio = MutagenFile(path, easy=True) + if audio is None: + raise ValueError(f"Unsupported format: {path}") + mapping = { + 'title': 'title', 'artist': 'artist', 'album': 'album', + 'album_artist': 'albumartist', 'genre': 'genre', + 'year': 'date', 'track_number': 'tracknumber' + } + for key, tag_name in mapping.items(): + val = tags.get(key) + if val is not None and val != "": + audio[tag_name] = str(val) + audio.save() + + +# ── Smart DJ Analysis ──────────────────────────────────── +def analyze(full_path: str) -> dict: + import librosa + cmd = ["ffmpeg", "-hide_banner", "-i", full_path, + "-af", "silencedetect=noise=-50dB:d=0.5,ebur128", "-f", "null", "-"] + out = subprocess.run(cmd, capture_output=True, text=True, timeout=120).stderr + + # ffmpeg silencedetect outputs pairs: + # silence_start: 0 (leading silence begins) + # silence_end: 0.5 (leading silence ends) ← skip to here + # silence_start: 237 (trailing silence begins) ← crossfade trigger + # silence_end: 240 (trailing silence ends) + # + # We need: + # silence_start = LAST silence_start = where trailing silence begins + # silence_end = FIRST silence_end = where leading silence ends + all_starts = re.findall(r"silence_start: ([\d\.]+)", out) + all_ends = re.findall(r"silence_end: ([\d\.]+)", out) + lu = re.search(r"I:\s+([\-\d\.]+) LUFS", out) + + # trailing silence start = last silence_start (for crossfade trigger) + sil_start = float(all_starts[-1]) if all_starts else 0.0 + # leading silence end = first silence_end (for skip-to point) + sil_end = float(all_ends[0]) if all_ends else 0.0 + loudness = float(lu.group(1)) if lu else -14.0 + + # Sanity: if sil_end is > 10s, it's probably not leading silence + if sil_end > 10.0: + sil_end = 0.0 + # Sanity: if sil_start is < 10s, it's probably not trailing silence + # (could be leading silence start = 0) + # Get track duration from ffmpeg to validate + dur_match = re.search(r"Duration: (\d+):(\d+):(\d+\.\d+)", out) + if dur_match: + total_dur = int(dur_match.group(1)) * 3600 + int(dur_match.group(2)) * 60 + float(dur_match.group(3)) + if sil_start < total_dur * 0.5: + sil_start = total_dur # No meaningful trailing silence found + + print(f" analyze: trailing_start={sil_start:.1f}s, leading_end={sil_end:.1f}s, LUFS={loudness:.1f}", flush=True) + + y, sr = librosa.load(full_path, sr=22050, duration=30) + tempo, _ = librosa.beat.beat_track(y=y, sr=sr) + del y + import gc; gc.collect() + + try: + bpm = float(tempo) + except TypeError: + bpm = float(tempo[0]) if tempo is not None else 0.0 + + profile = {"bpm": round(bpm, 1), "silence_start": round(sil_start, 3), + "silence_end": round(sil_end, 3), "loudness_lufs": round(loudness, 1)} + + with sqlite3.connect(DB_PATH) as c: + c.execute("INSERT OR REPLACE INTO dj_profiles VALUES (?,?,?,?,?,CURRENT_TIMESTAMP)", + (full_path, profile["bpm"], profile["silence_start"], + profile["silence_end"], profile["loudness_lufs"])) + return profile + + +# ── Mitsuha Visualizer Frames ───────────────────────────── +def gen_vis_frames(path: str, fps=30.0, fft_size=1024, pts=20) -> list: + import librosa + y, sr = librosa.load(path, sr=22050, mono=True) + hop = int(sr / fps) + frames = [] + for start in range(0, len(y) - fft_size, hop): + chunk = y[start:start + fft_size] * np.hanning(fft_size) + spec = np.sqrt(np.abs(np.fft.rfft(chunk)) / fft_size) + half = len(spec) + cutoff = min(half - 1, 90) + fp = [] + for i in range(pts): + ni = (i + 1) / pts + li = np.log10(ni * 9 + 1) + cb = li * cutoff + bw = max(1, cutoff / pts * li) + sb = max(1, int(cb - bw / 2)) + eb = min(cutoff, int(cb + bw / 2)) + avg = float(np.mean(spec[sb:eb + 1])) if sb <= eb < half else 0 + fp.append(avg * (1 + i / pts * 3.5)) + frames.append(fp) + del y; import gc; gc.collect() + vals = sorted(v for f in frames for v in f if v > 0.001) + if vals: + p95 = vals[min(int(len(vals) * 0.95), len(vals) - 1)] + if p95 > 0.001: + s = 0.8 / p95 + frames = [[min(1.0, v * s) for v in f] for f in frames] + return frames + + +def vis_cache_file(path: str) -> str: + return os.path.join(VIS_CACHE_DIR, hashlib.md5(path.encode()).hexdigest() + ".json") + + +def get_vis(path: str): + cf = vis_cache_file(path) + if os.path.exists(cf): + try: + with open(cf) as f: + return json.load(f) + except: + pass + if not os.path.isfile(path): + return None + try: + frames = gen_vis_frames(path) + with open(cf, "w") as f: + json.dump(frames, f) + return frames + except Exception as e: + print(f"Vis gen failed for {os.path.basename(path)}: {e}") + return None + + +# ═══════════════════════════════════════════════════════════ +# ENDPOINTS +# ═══════════════════════════════════════════════════════════ + +@app.get("/health") +async def health(): + pc = 0 + fc = 0 + try: + with sqlite3.connect(DB_PATH) as c: + pc = c.execute("SELECT COUNT(*) FROM dj_profiles").fetchone()[0] + fc = c.execute("SELECT COUNT(*) FROM file_index").fetchone()[0] + except: + pass + vc = len(os.listdir(VIS_CACHE_DIR)) if os.path.isdir(VIS_CACHE_DIR) else 0 + return { + "status": "healthy", + "music_dir": MUSIC_DIR, + "profiles": pc, + "file_index": fc, + "vis_cached": vc, + "ws_clients": len(push.connections) + } + + +@app.post("/reindex") +async def reindex(): + """Rebuild the file index (call after adding/renaming files).""" + build_file_index() + return {"status": "reindexed"} + + +# ── Single Metadata Edit ───────────────────────────────── +@app.patch("/edit-metadata") +async def edit_metadata(update: MetadataUpdate): + fp = resolve_path(update.relative_path) + if not fp: + decoded = unquote(unquote(update.relative_path)) + raise HTTPException(404, + f"File not found. " + f"raw='{update.relative_path}' | " + f"decoded='{decoded}' | " + f"MUSIC_DIR='{MUSIC_DIR}'") + try: + apply_tags(fp, update) + await trigger_scan() + await push.broadcast("metadata_updated", { + "path": update.relative_path, + "title": update.title or "", + "artist": update.artist or "", + "album": update.album or "" + }) + return {"status": "success", "file": update.relative_path, "resolved": fp} + except Exception as e: + raise HTTPException(500, str(e)) + + +# ── Batch Metadata Edit ────────────────────────────────── +@app.patch("/batch-edit-metadata") +async def batch_edit_metadata(update: BatchMetadataUpdate): + """Apply the same tag changes to multiple files at once.""" + results = {"succeeded": [], "failed": []} + tags = {} + if update.title: tags["title"] = update.title + if update.artist: tags["artist"] = update.artist + if update.album: tags["album"] = update.album + if update.album_artist: tags["album_artist"] = update.album_artist + if update.genre: tags["genre"] = update.genre + if update.year: tags["year"] = str(update.year) + + for rp in update.relative_paths: + fp = resolve_path(rp) + if not fp: + results["failed"].append({"path": rp, "error": "File not found"}) + continue + try: + apply_tags_dict(fp, tags) + results["succeeded"].append(rp) + except Exception as e: + results["failed"].append({"path": rp, "error": str(e)}) + + # Single scan after all edits + await trigger_scan() + await push.broadcast("batch_metadata_updated", { + "count": len(results["succeeded"]), + "album": update.album or "" + }) + return results + + +# ── Single File Upload ──────────────────────────────────── +@app.post("/upload-track") +async def upload_track( + file: UploadFile = File(...), + title: str = Form(...), + artist: str = Form(...), + album: str = Form(...) +): + dest = os.path.join(MUSIC_DIR, "uploads") + os.makedirs(dest, exist_ok=True) + fp = os.path.join(dest, file.filename) + with open(fp, "wb") as buf: + shutil.copyfileobj(file.file, buf) + try: + u = MetadataUpdate(relative_path=f"uploads/{file.filename}", + title=title, artist=artist, album=album) + apply_tags(fp, u) + profile = analyze(fp) + await trigger_scan() + await push.broadcast("track_uploaded", { + "filename": file.filename, + "profile": json.dumps(profile) + }) + return {"status": "uploaded", "path": f"uploads/{file.filename}", "profile": profile} + except Exception as e: + if os.path.exists(fp): + os.remove(fp) + raise HTTPException(500, str(e)) + + +# ── Multi-File Upload ───────────────────────────────────── +@app.post("/upload-tracks") +async def upload_tracks( + files: List[UploadFile] = File(...), + metadata_json: str = Form(...) +): + """ + Upload multiple audio files with per-track metadata. + + metadata_json is a JSON string containing a list of TrackUploadMeta objects: + [ + {"filename": "01 - Song.mp3", "title": "Song", "artist": "Artist", "album": "Album", "track_number": 1}, + ... + ] + + Each entry's filename must match one of the uploaded files. + Files are saved to MUSIC_DIR/Artist/Album/ with proper directory structure. + """ + try: + meta_list = json.loads(metadata_json) + except json.JSONDecodeError: + raise HTTPException(400, "Invalid metadata_json — must be a JSON array") + + # Index metadata by filename + meta_by_name = {} + for m in meta_list: + meta_by_name[m["filename"]] = m + + results = {"uploaded": [], "failed": []} + + for file in files: + meta = meta_by_name.get(file.filename) + if not meta: + # No metadata provided — use filename as title + meta = { + "filename": file.filename, + "title": Path(file.filename).stem, + "artist": "Unknown Artist", + "album": "Uploads" + } + + # Build directory: MUSIC_DIR/Artist/Album/ + artist_name = meta.get("artist", "Unknown Artist") + album_name = meta.get("album", "Uploads") + safe_artist = re.sub(r'[<>:"/\\|?*]', '_', artist_name) + safe_album = re.sub(r'[<>:"/\\|?*]', '_', album_name) + dest_dir = os.path.join(MUSIC_DIR, safe_artist, safe_album) + os.makedirs(dest_dir, exist_ok=True) + + fp = os.path.join(dest_dir, file.filename) + try: + with open(fp, "wb") as buf: + shutil.copyfileobj(file.file, buf) + + # Apply tags + tags = { + "title": meta.get("title", Path(file.filename).stem), + "artist": artist_name, + "album": album_name, + } + if meta.get("track_number"): + tags["track_number"] = str(meta["track_number"]) + if meta.get("genre"): + tags["genre"] = meta["genre"] + if meta.get("year"): + tags["year"] = str(meta["year"]) + if meta.get("album_artist"): + tags["album_artist"] = meta["album_artist"] + else: + tags["album_artist"] = artist_name + + apply_tags_dict(fp, tags) + + # Analyze (non-blocking — catch errors but don't fail the upload) + try: + analyze(fp) + except Exception as e: + print(f" Analysis failed for {file.filename}: {e}") + + rel = os.path.relpath(fp, MUSIC_DIR) + results["uploaded"].append({ + "filename": file.filename, + "path": rel + }) + + except Exception as e: + if os.path.exists(fp): + os.remove(fp) + results["failed"].append({ + "filename": file.filename, + "error": str(e) + }) + + # Single scan after all uploads + await trigger_scan() + await push.broadcast("tracks_uploaded", { + "count": len(results["uploaded"]), + "album": meta_list[0].get("album", "") if meta_list else "" + }) + + return results + + +# ── Smart DJ ────────────────────────────────────────────── +@app.get("/smart-dj/profile") +async def get_profile(relative_path: str): + fp = resolve_path(relative_path) + + if fp: + try: + with sqlite3.connect(DB_PATH) as c: + row = c.execute( + "SELECT bpm, silence_start, silence_end, loudness_lufs " + "FROM dj_profiles WHERE file_path = ?", (fp,) + ).fetchone() + if row: + return {"bpm": row[0], "silence_start": row[1], + "silence_end": row[2], "loudness_lufs": row[3]} + except sqlite3.OperationalError as e: + print(f"DB error: {e}", flush=True) + return analyze(fp) + + # File not found by path — try DB search by filename + decoded = relative_path + for _ in range(5): + nd = unquote(decoded) + if nd == decoded: break + decoded = nd + target = os.path.basename(decoded).lower() + if target: + try: + with sqlite3.connect(DB_PATH) as c: + rows = c.execute( + "SELECT file_path, bpm, silence_start, silence_end, loudness_lufs " + "FROM dj_profiles WHERE LOWER(file_path) LIKE ?", + (f"%{target}",) + ).fetchall() + if rows: + print(f" DB fallback: {rows[0][0]}", flush=True) + return {"bpm": rows[0][1], "silence_start": rows[0][2], + "silence_end": rows[0][3], "loudness_lufs": rows[0][4]} + except Exception as e: + print(f" DB fallback error: {e}", flush=True) + + raise HTTPException(404, + f"Not found. decoded='{decoded}', MUSIC_DIR='{MUSIC_DIR}'") + + +@app.get("/smart-dj/bulk-profiles") +async def bulk_profiles(paths: str = Query(...)): + results = {} + for rp in (p.strip() for p in paths.split(",") if p.strip()): + fp = resolve_path(rp) + if not fp: + results[rp] = None + continue + try: + with sqlite3.connect(DB_PATH) as c: + row = c.execute( + "SELECT bpm, silence_start, silence_end, loudness_lufs " + "FROM dj_profiles WHERE file_path = ?", (fp,) + ).fetchone() + if row: + results[rp] = {"bpm": row[0], "silence_start": row[1], + "silence_end": row[2], "loudness_lufs": row[3]} + else: + results[rp] = analyze(fp) + except: + results[rp] = None + return results + + +# ── Visualizer ──────────────────────────────────────────── +@app.get("/visualizer/frames") +async def vis_frames(relative_path: str): + fp = resolve_path(relative_path) + if not fp: + raise HTTPException(404, "Not found") + frames = get_vis(fp) + if not frames: + raise HTTPException(500, "Generation failed") + return {"frame_count": len(frames), "fps": 30.0, "points": 20, "frames": frames} + + +@app.post("/visualizer/precompute") +async def precompute(background_tasks: BackgroundTasks, relative_path: str = ""): + def compute_all(): + exts = ('.mp3', '.flac', '.m4a', '.ogg', '.opus', '.wav') + n = 0 + for root, _, files in os.walk(MUSIC_DIR): + for f in files: + if f.lower().endswith(exts): + fp = os.path.join(root, f) + if not os.path.exists(vis_cache_file(fp)): + try: + get_vis(fp) + n += 1 + except: + pass + print(f"✨ Pre-computed {n} vis caches") + + if relative_path: + background_tasks.add_task(lambda: get_vis(resolve_path(relative_path) or "")) + return {"message": f"Computing: {relative_path}"} + background_tasks.add_task(compute_all) + return {"message": "Background vis computation started"} + + +@app.post("/bulk-fix") +async def bulk_fix(background_tasks: BackgroundTasks): + background_tasks.add_task(trigger_scan) + return {"message": "Scan triggered"} + + +# ── WebSocket Push ──────────────────────────────────────── +@app.websocket("/ws/push") +async def ws_push(ws: WebSocket): + await push.connect(ws) + try: + while True: + data = json.loads(await ws.receive_text()) + act = data.get("action") + if act == "ping": + await push.send_to(ws, "pong", {"t": str(time.time())}) + elif act == "get_profile": + rp = data.get("path", "") + fp = resolve_path(rp) + if fp: + try: + with sqlite3.connect(DB_PATH) as c: + row = c.execute( + "SELECT bpm, silence_start, silence_end, loudness_lufs " + "FROM dj_profiles WHERE file_path = ?", (fp,) + ).fetchone() + if row: + await push.send_to(ws, "profile", { + "path": rp, "bpm": str(row[0]), + "silence_start": str(row[1]), + "silence_end": str(row[2]), + "loudness_lufs": str(row[3]) + }) + else: + await push.send_to(ws, "profile", + {"path": rp, "error": "not_analyzed"}) + except Exception as e: + await push.send_to(ws, "error", {"message": str(e)}) + elif act == "get_vis": + rp = data.get("path", "") + fp = resolve_path(rp) + if fp: + frames = get_vis(fp) + if frames: + await push.send_to(ws, "vis_frames", { + "path": rp, "count": str(len(frames)), + "fps": "30", "frames": json.dumps(frames) + }) + except WebSocketDisconnect: + push.disconnect(ws) + except Exception as e: + print(f"WS error: {e}") + push.disconnect(ws) diff --git a/companion-api/pre_analyze.py b/companion-api/pre_analyze.py new file mode 100644 index 0000000..08bafbb --- /dev/null +++ b/companion-api/pre_analyze.py @@ -0,0 +1,302 @@ +""" +Pre-Analyzer (crash-safe) +Location: /home/pi/docker/navidrome/companion_api/pre_analyze.py + +Each track is analyzed in a subprocess. If librosa OOMs or hangs on a large FLAC, +only the child process dies — the parent logs the failure and moves on. + +Usage: + docker compose exec music-companion python pre_analyze.py + docker compose exec music-companion python pre_analyze.py --force + docker compose exec music-companion python pre_analyze.py --dj + docker compose exec music-companion python pre_analyze.py --vis + docker compose exec music-companion python pre_analyze.py --skip-large 500 +""" +import os, sys, json, hashlib, sqlite3, subprocess, time, warnings, multiprocessing + +warnings.filterwarnings("ignore") + +MUSIC_DIR = os.getenv("MUSIC_DIR", "/music") +DB_PATH = os.getenv("DB_PATH", "/app/data/smart_dj.db") +VIS_CACHE_DIR = os.getenv("VIS_CACHE_DIR", "/app/data/vis_cache") +SUPPORTED = ('.mp3', '.flac', '.m4a', '.ogg', '.opus', '.wav') +TRACK_TIMEOUT = int(os.getenv("TRACK_TIMEOUT", "180")) # 3 min per track + + +def init_db(): + os.makedirs(os.path.dirname(DB_PATH), exist_ok=True) + with sqlite3.connect(DB_PATH) as c: + c.execute("""CREATE TABLE IF NOT EXISTS dj_profiles ( + file_path TEXT PRIMARY KEY, bpm REAL, + silence_start REAL, silence_end REAL, + loudness_lufs REAL, + analyzed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)""") + + +def is_dj_done(path): + try: + with sqlite3.connect(DB_PATH) as c: + return c.execute("SELECT 1 FROM dj_profiles WHERE file_path=?", (path,)).fetchone() is not None + except: + return False + + +def vis_path(path): + return os.path.join(VIS_CACHE_DIR, hashlib.md5(path.encode()).hexdigest() + ".json") + + +def is_vis_done(path): + return os.path.exists(vis_path(path)) + + +def fmt(mb): + return f"{mb/1024:.2f} GB" if mb >= 1024 else f"{mb:.1f} MB" + + +# ═══════════════════════════════════════════════════════════ +# CHILD PROCESS — runs in isolation, can be killed without +# taking down the parent +# ═══════════════════════════════════════════════════════════ + +def _worker(full_path, do_dj, do_vis, result_dict): + """Runs in a child process. Writes results to shared dict.""" + import re, gc, warnings + warnings.filterwarnings("ignore") + import numpy as np + + dj_ok = False + vis_ok = False + error_msg = None + + # ── DJ Analysis ─────────────────────────────────────── + if do_dj: + try: + import librosa + + # ffmpeg for silence + loudness (streams, low memory) + cmd = ["ffmpeg", "-hide_banner", "-i", full_path, + "-af", "silencedetect=noise=-50dB:d=0.5,ebur128", + "-f", "null", "-"] + r = subprocess.run(cmd, capture_output=True, text=True, timeout=90) + out = r.stderr + + ss = re.findall(r"silence_start: ([\d\.]+)", out) + se = re.findall(r"silence_end: ([\d\.]+)", out) + lu = re.search(r"I:\s+([\-\d\.]+) LUFS", out) + + # trailing silence start = last silence_start (crossfade trigger) + sil_start = float(ss[-1]) if ss else 0.0 + # leading silence end = first silence_end (skip-to point) + sil_end = float(se[0]) if se else 0.0 + loudness = float(lu.group(1)) if lu else -14.0 + + # Sanity checks + if sil_end > 10.0: + sil_end = 0.0 + dur_match = re.search(r"Duration: (\d+):(\d+):(\d+\.\d+)", out) + if dur_match: + total_dur = int(dur_match.group(1)) * 3600 + int(dur_match.group(2)) * 60 + float(dur_match.group(3)) + if sil_start < total_dur * 0.5: + sil_start = total_dur + + # BPM — load only 30s at low rate + y, sr = librosa.load(full_path, sr=22050, duration=30) + tempo, _ = librosa.beat.beat_track(y=y, sr=sr) + del y; gc.collect() + + try: + bpm = float(tempo) + except TypeError: + bpm = float(tempo[0]) if tempo is not None else 0.0 + + with sqlite3.connect(DB_PATH) as c: + c.execute( + "INSERT OR REPLACE INTO dj_profiles VALUES (?,?,?,?,?,CURRENT_TIMESTAMP)", + (full_path, round(bpm,1), round(sil_start,3), + round(sil_end,3), round(loudness,1))) + dj_ok = True + + except subprocess.TimeoutExpired: + error_msg = "ffmpeg timeout (>90s)" + except MemoryError: + error_msg = "OUT OF MEMORY during DJ analysis" + except Exception as e: + error_msg = f"DJ: {e}" + + # ── Vis Frames ──────────────────────────────────────── + if do_vis: + cache_file = vis_path(full_path) + if os.path.exists(cache_file): + vis_ok = False # already cached + else: + try: + import librosa + + # Load at 22050 to save memory + y, sr = librosa.load(full_path, sr=22050, mono=True) + fps = 30.0; fft_size = 1024; pts = 20 + hop = int(sr / fps) + frames = [] + + for start in range(0, len(y) - fft_size, hop): + chunk = y[start:start+fft_size] * np.hanning(fft_size) + spec = np.sqrt(np.abs(np.fft.rfft(chunk)) / fft_size) + half = len(spec); cutoff = min(half-1, 90) + fp = [] + for i in range(pts): + ni = (i+1)/pts; li = np.log10(ni*9+1) + cb = li*cutoff; bw = max(1, cutoff/pts*li) + sb = max(1, int(cb-bw/2)); eb = min(cutoff, int(cb+bw/2)) + avg = float(np.mean(spec[sb:eb+1])) if sb<=eb 0.001) + if vals: + p95 = vals[min(int(len(vals)*0.95), len(vals)-1)] + if p95 > 0.001: + s = 0.8/p95 + frames = [[min(1.0, v*s) for v in f] for f in frames] + + with open(cache_file, "w") as f: + json.dump(frames, f) + del frames; gc.collect() + vis_ok = True + + except MemoryError: + error_msg = (error_msg + " | " if error_msg else "") + "OUT OF MEMORY during vis" + except Exception as e: + error_msg = (error_msg + " | " if error_msg else "") + f"Vis: {e}" + + result_dict["dj"] = dj_ok + result_dict["vis"] = vis_ok + result_dict["error"] = error_msg + + +# ═══════════════════════════════════════════════════════════ +# MAIN — runs workers as subprocesses +# ═══════════════════════════════════════════════════════════ + +def scan(force=False, dj_only=False, vis_only=False, skip_large_mb=0): + init_db() + os.makedirs(VIS_CACHE_DIR, exist_ok=True) + + print(f"šŸ” Scanning {MUSIC_DIR}...") + tracks = [] + total_bytes = 0 + for root, _, files in os.walk(MUSIC_DIR): + for f in files: + if f.lower().endswith(SUPPORTED): + fp = os.path.join(root, f) + tracks.append(fp) + total_bytes += os.path.getsize(fp) + + total_mb = total_bytes / 1048576 + mode = "FORCE" if force else "missing" + what = "DJ only" if dj_only else "Vis only" if vis_only else "DJ + Vis" + print(f"šŸš€ {len(tracks)} tracks ({fmt(total_mb)}) — {what} ({mode})") + print(f" Timeout: {TRACK_TIMEOUT}s per track") + if skip_large_mb: + print(f" Skipping files > {skip_large_mb} MB") + print(f" Each track runs in a subprocess (crash-safe)") + print() + + dj_n = vis_n = skip_n = fail_n = 0 + t0 = time.time() + + for idx, path in enumerate(tracks): + name = os.path.basename(path) + size_mb = os.path.getsize(path) / 1048576 + tag = f"[{idx+1}/{len(tracks)}]" + + # Skip oversized + if skip_large_mb and size_mb > skip_large_mb: + print(f" {tag} ā­ SKIP ({fmt(size_mb)} > {skip_large_mb}MB): {name}") + skip_n += 1 + continue + + # Check what's needed + need_dj = not vis_only and (force or not is_dj_done(path)) + need_vis = not dj_only and (force or not is_vis_done(path)) + + if not need_dj and not need_vis: + skip_n += 1 + continue + + tasks = [] + if need_dj: tasks.append("DJ") + if need_vis: tasks.append("Vis") + + print(f" {tag} šŸŽµ {'+'.join(tasks)} ({fmt(size_mb)}): {name}", end="", flush=True) + t1 = time.time() + + # Run in subprocess with shared dict for results + manager = multiprocessing.Manager() + result = manager.dict({"dj": False, "vis": False, "error": None}) + + proc = multiprocessing.Process( + target=_worker, + args=(path, need_dj, need_vis, result) + ) + proc.start() + proc.join(timeout=TRACK_TIMEOUT) + + elapsed = time.time() - t1 + + if proc.is_alive(): + # Timed out — kill it + proc.kill() + proc.join() + print(f" ā° KILLED after {elapsed:.0f}s (timeout)") + fail_n += 1 + continue + + if proc.exitcode != 0 and proc.exitcode is not None: + # Crashed (OOM, segfault, etc.) + print(f" šŸ’„ CRASHED (exit code {proc.exitcode}, {elapsed:.1f}s)") + fail_n += 1 + continue + + # Success path + err = result.get("error") + if result.get("dj"): + dj_n += 1 + if result.get("vis"): + vis_n += 1 + + if err: + print(f" ⚠ {elapsed:.1f}s — {err}") + fail_n += 1 + else: + print(f" āœ“ {elapsed:.1f}s") + + total_elapsed = time.time() - t0 + m = int(total_elapsed // 60) + s = int(total_elapsed % 60) + + print(f"\n✨ Done in {m}m {s}s") + print(f" DJ profiles: {dj_n} new") + print(f" Vis frames: {vis_n} new") + print(f" Skipped: {skip_n} (already done)") + if fail_n: + print(f" ⚠ Failed: {fail_n} (see errors above)") + print(f" Tip: re-run with --skip-large 200 to skip huge FLACs") + + +if __name__ == "__main__": + args = sys.argv[1:] + force = "--force" in args + dj_only = "--dj" in args + vis_only = "--vis" in args + + skip_large = 0 + if "--skip-large" in args: + i = args.index("--skip-large") + if i + 1 < len(args): + try: skip_large = int(args[i+1]) + except: pass + + scan(force=force, dj_only=dj_only, vis_only=vis_only, skip_large_mb=skip_large)