From 935cb0a981dfd338b84175005049d6c6613e7e8d Mon Sep 17 00:00:00 2001 From: evilchili Date: Sun, 20 Nov 2022 16:26:40 -0800 Subject: [PATCH] Add filesystem scanner --- .test_env | 2 ++ groove/cli.py | 27 +++++++++++++++++-- groove/db/manager.py | 2 +- groove/db/scanner.py | 62 ++++++++++++++++++++++++++++++++++++++++++++ groove/helper.py | 2 +- test/test_scanner.py | 49 ++++++++++++++++++++++++++++++++++ 6 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 groove/db/scanner.py create mode 100644 test/test_scanner.py diff --git a/.test_env b/.test_env index 034e075..7313ff4 100644 --- a/.test_env +++ b/.test_env @@ -2,3 +2,5 @@ HOST=127.0.0.1 DEBUG=1 USERNAME=test_username PASSWORD=test_password +MEDIA_ROOT=/test +MEDIA_GLOB=*.flac,*.mp3 diff --git a/groove/cli.py b/groove/cli.py index 6473e54..bb7c4f4 100644 --- a/groove/cli.py +++ b/groove/cli.py @@ -2,10 +2,13 @@ import logging import os import typer +from pathlib import Path +from typing import Optional from dotenv import load_dotenv from groove import webserver from groove.db.manager import database_manager +from groove.db.scanner import media_scanner app = typer.Typer() @@ -18,6 +21,26 @@ def initialize(): level=logging.DEBUG if debug else logging.INFO) +@app.command() +def scan( + root: Optional[Path] = typer.Option( + None, + help="The path to the root of your media." + ), + debug: bool = typer.Option( + False, + help='Enable debugging output' + ), +): + """ + Scan the filesystem and create track entries in the database. + """ + initialize() + with database_manager() as manager: + scanner = media_scanner(root=root, db=manager.session) + count = scanner.scan() + logging.info(f"Imported {count} new tracks from {root}.") + @app.command() def server( host: str = typer.Argument( @@ -36,9 +59,9 @@ def server( """ Start the Groove on Demand playlsit server. """ - print("Starting Groove On Demand...") initialize() - with database_manager as manager: + print("Starting Groove On Demand...") + with database_manager() as manager: manager.import_from_filesystem() webserver.start(host=host, port=port, debug=debug) diff --git a/groove/db/manager.py b/groove/db/manager.py index 92a14a1..0a50029 100644 --- a/groove/db/manager.py +++ b/groove/db/manager.py @@ -40,4 +40,4 @@ class DatabaseManager: self.session.close() -database_manager = DatabaseManager() +database_manager = DatabaseManager diff --git a/groove/db/scanner.py b/groove/db/scanner.py new file mode 100644 index 0000000..eeb2f46 --- /dev/null +++ b/groove/db/scanner.py @@ -0,0 +1,62 @@ +import logging +import os +import sys + +from pathlib import Path +from typing import Callable, Union, Iterable +from sqlalchemy import func + +import groove.db + + +class MediaScanner: + """ + Scan a directory structure containing audio files and import them into the database. + """ + def __init__(self, root: Path, db: Callable, glob: Union[str, None] = None) -> None: + self._db = db + self._glob = tuple((glob or os.environ.get('MEDIA_GLOB')).split(',')) + try: + self._root = root or Path(os.environ.get('MEDIA_ROOT')) + except TypeError: + logging.error("Could not find media root. Do you need to define MEDIA_ROOT in your environment?") + sys.exit(1) + logging.debug(f"Configured media scanner for root: {self._root}") + + @property + def db(self) -> Callable: + return self._db + + @property + def root(self) -> Path: + return self._root + + @property + def glob(self) -> tuple: + return self._glob + + def find_sources(self, pattern): + return self.root.rglob(pattern) # pragma: no-cover + + def import_tracks(self, sources: Iterable) -> None: + for path in sources: + relpath = str(path.relative_to(self.root)) + stmt = groove.db.track.insert({'relpath': relpath}).prefix_with('OR IGNORE') + self.db.execute(stmt) + self.db.commit() + + def scan(self) -> int: + """ + Walk the media root and insert Track table entries for each media file + found. Existing entries will be ignored. + """ + count = self.db.query(func.count(groove.db.track.c.relpath)).scalar() + logging.debug(f"Track table currently contains {count} entries.") + for pattern in self.glob: + self.import_tracks(self.find_sources(pattern)) + newcount = self.db.query(func.count(groove.db.track.c.relpath)).scalar() - count + logging.debug(f"Inserted {newcount} new tracks so far this run...") + return newcount + + +media_scanner = MediaScanner diff --git a/groove/helper.py b/groove/helper.py index 55500e5..bc67023 100644 --- a/groove/helper.py +++ b/groove/helper.py @@ -25,7 +25,7 @@ class PlaylistDatabaseHelper: """ playlist = {} - query = db.playlist.select(db.playlist.c.slug==bindparam('slug')) + query = db.playlist.select(db.playlist.c.slug == bindparam('slug')) logging.debug(f"playlist: '{slug}' requested. Query: {query}") results = self.conn.execute(str(query), {'slug': slug}).fetchone() if not results: diff --git a/test/test_scanner.py b/test/test_scanner.py new file mode 100644 index 0000000..227e9fa --- /dev/null +++ b/test/test_scanner.py @@ -0,0 +1,49 @@ +import pytest +from pathlib import Path +from unittest.mock import MagicMock +from sqlalchemy import func + +from groove.db import scanner, track + +fixture_tracks = [ + "/test/Spookey Ruben/Modes of Transportation, Volume 1/Spookey Ruben - Modes of Transportation, Volume 1 - 01 Terra Magnifica.flac", + "/test/Spookey Ruben/Modes of Transportation, Volume 1/Spookey Ruben - Modes of Transportation, Volume 1 - 02 These Days Are Old.flac", + "/test/Spookey Ruben/Modes of Transportation, Volume 1/Spookey Ruben - Modes of Transportation, Volume 1 - 03 Crystal Cradle.flac", + "/test/Spookey Ruben/Modes of Transportation, Volume 1/Spookey Ruben - Modes of Transportation, Volume 1 - 04 Running Away.flac", + "/test/Spookey Ruben/Modes of Transportation, Volume 1/Spookey Ruben - Modes of Transportation, Volume 1 - 05 Welcome to the House of Food.flac", + "/test/Spookey Ruben/Modes of Transportation, Volume 1/Spookey Ruben - Modes of Transportation, Volume 1 - 06 Wendy McDonald.flac", + "/test/Spookey Ruben/Modes of Transportation, Volume 1/Spookey Ruben - Modes of Transportation, Volume 1 - 07 The Size of You.flac", + "/test/Spookey Ruben/Modes of Transportation, Volume 1/Spookey Ruben - Modes of Transportation, Volume 1 - 08 Its Not What You Do Its You.flac", + "/test/Spookey Ruben/Modes of Transportation, Volume 1/Spookey Ruben - Modes of Transportation, Volume 1 - 09 Mars.flac", + "/test/Spookey Ruben/Modes of Transportation, Volume 1/Spookey Ruben - Modes of Transportation, Volume 1 - 10 Leave the City.flac", + "/test/Spookey Ruben/Modes of Transportation, Volume 1/Spookey Ruben - Modes of Transportation, Volume 1 - 11 Growing Up is Over.flac", + "/test/Spookey Ruben/Modes of Transportation, Volume 1/Spookey Ruben - Modes of Transportation, Volume 1 - 12 Donate Your Heart to a Stranger....flac", + "/test/Spookey Ruben/Modes of Transportation, Volume 1/Spookey Ruben - Modes of Transportation, Volume 1 - 13 Life Insurance.flac", +] + + +@pytest.fixture +def media(): + def fixture(): + for t in fixture_tracks: + yield Path(t) + return fixture + + +def test_scanner(monkeypatch, in_memory_db, media): + + # replace the filesystem glob with the test fixture generator + monkeypatch.setattr(scanner.MediaScanner, 'find_sources', MagicMock(return_value=media())) + test_scanner = scanner.media_scanner(root=Path('/test'), db=in_memory_db) + expected = len(fixture_tracks) + + # verify all entries are scanned + assert test_scanner.scan() == expected + + # readback; verify entries are in the db + query = func.count(track.c.relpath) + query = query.filter(track.c.relpath.ilike('%Spookey%')) + assert in_memory_db.query(query).scalar() == expected + + # verify idempotency + assert test_scanner.scan() == 0