add password input validators

src/grung/db.py

@@ -1,9 +1,15 @@
 import inspect
+from collections.abc import Iterable
+from pathlib import Path
+from typing import List
 
 from tinydb import TinyDB, table
+from tinydb.storages import MemoryStorage
 from tinydb.table import Document
 
-from grung.types import Record
+from grung.exceptions import CircularReferenceError
+from grung.objects import Record
+from grung.validators import TypeValidator
 
 
 class RecordTable(table.Table):
@@ -11,25 +17,72 @@ class RecordTable(table.Table):
     Wrapper around tinydb Tables that handles Records instead of dicts.
     """
 
-    def __init__(self, storage, name, document_class: Document = Record, **kwargs):
+    def __init__(self, name: str, db: TinyDB, document_class: Document = Record, **kwargs):
         self.document_class = document_class
-        super().__init__(storage, name, **kwargs)
+        self.db = db
+        super().__init__(db.storage, name, **kwargs)
 
     def insert(self, document):
-        self._satisfy_constraints(document)
-        if document.doc_id:
-            last_insert_id = super().upsert(document)[0]
+        document.before_insert(self.db)
+
+        # check field types before attempting serialization
+        validator = TypeValidator()
+        for field in document._metadata.fields.values():
+            validator.validate(document, field, self.db)
+
+        doc = document.serialize()
+        self._check_constraints(doc)
+
+        if doc.doc_id:
+            last_insert_id = super().upsert(doc)[0]
         else:
-            last_insert_id = super().insert(dict(document))
-        return self.get(doc_id=last_insert_id)
+            last_insert_id = super().insert(dict(doc))
+            doc.doc_id = last_insert_id
+        doc.after_insert(self.db)
+        return doc.deserialize(self.db)
+
+    def get(self, *args, doc_id: int = None, recurse: bool = False, **kwargs):
+        """
+        Return exactly zero or one records from the database matching the supplied criteria.
+        If more than one records match the criteria, return the first one. Criteria are ignored
+        if doc_id is specified.
+
+        Usage:
+            Table.get(doc_id=1)
+            Table.get(where("uid") == "abcdef")
+
+        """
+        if doc_id:
+            document = super().get(doc_id=doc_id)
+            if document:
+                return document.deserialize(self.db, recurse=recurse)
+            return None
+
+        matches = self.search(*args, recurse=recurse, **kwargs)
+        if matches:
+            return matches[0]
+
+    def search(self, *args, recurse: bool = False, **kwargs) -> List[Record]:
+        results = super().search(*args, **kwargs)
+        return [r.deserialize(self.db, recurse=recurse) for r in results]
 
     def remove(self, document):
         if document.doc_id:
             super().remove(doc_ids=[document.doc_id])
 
-    def _satisfy_constraints(self, document):
-        # check for uniqueness, etc.
-        pass
+    def _check_constraints(self, document) -> bool:
+        self._check_for_recursion(document)
+        for field in document._metadata.fields.values():
+            field.validate(document, self.db)
+
+    def _check_for_recursion(self, document) -> bool:
+        ref = document.reference
+        for field in document._metadata.fields.values():
+            if isinstance(field.default, Iterable) and ref in document[field.name]:
+                raise CircularReferenceError(document, field, ref, "builtin")
+            elif document[field.name] == ref:
+                raise CircularReferenceError(document, field, ref, "builtin")
+        return True
 
 
 class GrungDB(TinyDB):
@@ -41,7 +94,10 @@ class GrungDB(TinyDB):
     default_table_name = "Record"
     _tables = {}
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, path: Path, *args, **kwargs):
+        self.path = path.parent
+        if kwargs.get("storage") != MemoryStorage:
+            args = (path,) + args
         super().__init__(*args, **kwargs)
         self.create_table(Record)
 
@@ -53,7 +109,7 @@ class GrungDB(TinyDB):
     def create_table(self, table_class):
         name = table_class.__name__
         if name not in self._tables:
-            self._tables[name] = RecordTable(self.storage, name, document_class=table_class)
+            self._tables[name] = RecordTable(name, db=self, document_class=table_class)
         return self.table(name)
 
     def save(self, record):
@@ -74,8 +130,8 @@ class GrungDB(TinyDB):
         return super().__getattr__(attr_name)
 
     @classmethod
-    def with_schema(cls, schema_module, *args, **kwargs):
-        db = GrungDB(*args, **kwargs)
+    def with_schema(cls, schema_module, path: Path | None, *args, **kwargs):
+        db = GrungDB(path=path, *args, **kwargs)
         for name, obj in inspect.getmembers(schema_module):
             if type(obj) == type and issubclass(obj, Record):
                 db.create_table(obj)

src/grung/examples.py

@@ -1,11 +1,66 @@
-from typing import List
+import re
 
-from grung.types import Field, Record
+from grung.objects import (
+    BackReference,
+    BinaryFilePointer,
+    Collection,
+    DateTime,
+    Dict,
+    Integer,
+    List,
+    Password,
+    Record,
+    RecordDict,
+    String,
+    TextFilePointer,
+    Timestamp,
+)
+from grung.validators import LengthValidator, MinMaxValidator, PatternValidator
 
 
 class User(Record):
-    _fields = [Field("name"), Field("email", unique=True)]
+    @classmethod
+    def fields(cls):
+        return [
+            *super().fields(),
+            String("name", primary_key=True, validators=[LengthValidator(min=3, max=30)]),
+            Integer("number", default=0, validators=[MinMaxValidator(min=0, max=255)]),
+            String("email", unique=True, validators=[PatternValidator(re.compile(r"[^@]+@[\w\-\.]+$"))]),
+            Password("password"),
+            DateTime("created"),
+            Timestamp("last_updated"),
+            BackReference("groups", value_type=Group),
+        ]
 
 
 class Group(Record):
-    _fields = [Field("name", unique=True), Field("users", List[User])]
+    @classmethod
+    def fields(cls):
+        return [
+            *super().fields(),
+            String("name", primary_key=True),
+            Collection("members", member_type=User),
+            Collection("groups", member_type=Group),
+            BackReference("parent", value_type=Group),
+        ]
+
+
+class Album(Record):
+    @classmethod
+    def fields(cls):
+        inherited = [f for f in super().fields() if f.name != "name"]
+        return inherited + [
+            String("name"),
+            Dict("credits"),
+            List("tracks"),
+            BackReference("artist", value_type=Artist),
+            BinaryFilePointer("cover", extension=".jpg"),
+            TextFilePointer("review"),
+        ]
+
+
+class Artist(User):
+    @classmethod
+    def fields(cls):
+        inherited = [f for f in super().fields() if f.name != "name"]
+        return inherited + [String("name"), RecordDict("albums", member_type=Album)]

src/grung/exceptions.py

@@ -0,0 +1,92 @@
+class ValidationError(Exception):
+    """
+    Thrown when a record's field does not meet validation criteria.
+    """
+
+    messages = []
+    template = (
+        "\n"
+        "  * Record:    {_record}\n"
+        "  * Field:     {_field}\n"
+        "  * Value:     {_value}\n"
+        "  * Validator: {_validator}\n"
+        "\n"
+        "{_messages}"
+    )
+
+    def __init__(self, record, field, validator, messages=[], **kwargs):
+        super().__init__(
+            self.template.format(
+                _record=dict(record),
+                _field=field,
+                _value=record[field.name],
+                _validator=validator,
+                _messages="\n".join(messages or self.__class__.messages),
+                **kwargs,
+            )
+        )
+
+
+class InvalidFieldTypeError(ValidationError):
+    """
+    Thrown when a document's field value does not match the field value_type.
+    """
+
+    messages = ["The value of the field is not an instance of the field's value_type."]
+
+
+class UniqueConstraintError(ValidationError):
+    """
+    Thrown when a db write operation cannot complete due to a field's unique constraint.
+    """
+
+    def __init__(self, record, field, validator, query, matches):
+        super().__init__(
+            record,
+            field,
+            validator,
+            messages=[
+                f"Query: {query}",
+                "The record matches the following existing records:\n\n" + "\n".join(str(m) for m in matches),
+            ],
+        )
+
+
+class CircularReferenceError(ValidationError):
+    """
+    Thrown when a record contains a reference to itself.
+    """
+
+    messages = ["This record contains a reference to itself. This will lead to infinite recursion."]
+
+
+class MalformedPointerError(ValidationError):
+    """
+    Thrown when a Pointer's value is not a valid reference string.
+    """
+
+    messages = ["A Pointer's value must follow the format 'TABLE_NAME::PRIMARY_KEY_NAME::PRIMARY_KEY_VALUE'."]
+
+
+class PointerReferenceError(Exception):
+    """
+    Thrown when a record field containing a record could not be resolve to an existing record in the database.
+    """
+
+
+class InvalidLengthError(ValidationError):
+    """
+    Thrown when a field does not meet its length constraint.
+    """
+
+
+class InvalidSizeError(ValidationError):
+    """
+    Thrown when a field's size is too large or too small.
+    """
+
+
+class PatternMatchError(ValidationError):
+    """
+    Thrown when a field does not match the specified pattern.
+    """

src/grung/objects.py

@@ -0,0 +1,455 @@
+from __future__ import annotations
+
+import hashlib
+import hmac
+import os
+import re
+import typing
+from collections import namedtuple
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+
+import nanoid
+from tinydb import TinyDB, where
+
+import grung.types
+from grung.exceptions import PointerReferenceError
+from grung.validators import LengthValidator, PatternValidator, PointerReferenceValidator, UniqueValidator
+
+Metadata = namedtuple("Metadata", ["table", "fields", "backrefs", "primary_key"])
+
+
+@dataclass
+class Field(grung.types.Field):
+    """
+    Represents a single field in a Record.
+    """
+
+    name: str
+    default: str = None
+    unique: bool = False
+    primary_key: bool = False
+    validators: list = field(default_factory=lambda: [])
+
+    value_type = str
+
+    def before_insert(self, value: value_type, db: TinyDB, record: Record) -> None:
+        pass
+
+    def after_insert(self, db: TinyDB, record: Record) -> None:
+        pass
+
+    def serialize(self, value: value_type, record: Record | None = None) -> str:
+        if value is not None:
+            return str(value)
+
+    def deserialize(self, value: str, db: TinyDB, recurse: bool = False) -> value_type:
+        return value
+
+    def validate(self, record: Record, db: TinyDB):
+        if self.unique:
+            UniqueValidator().validate(record, self, db)
+        for validator in self.validators:
+            validator.validate(record, self, db)
+
+
+class Record(grung.types.Record):
+    """
+    Base type for a single database record.
+    """
+
+    def __init__(self, raw_doc: dict = {}, doc_id: int = None, **params):
+        self.doc_id = doc_id
+        fields = self.__class__.fields()
+
+        pkey = [field for field in fields if field.primary_key]
+        if len(pkey) > 1:
+            raise Exception(f"Cannnot have more than one primary key: {pkey}")
+        elif pkey:
+            pkey = pkey[0]
+        else:
+            # 1% collision rate at ~2M records
+            pkey = Field("uid", default=nanoid.generate(size=8), primary_key=True)
+            fields.append(pkey)
+
+        pkey.unique = True
+
+        self._metadata = Metadata(
+            table=self.__class__.__name__,
+            primary_key=pkey.name,
+            fields={f.name: f for f in fields},
+            backrefs=lambda value_type: (
+                field for field in fields if type(field) == BackReference and field.value_type == value_type
+            ),
+        )
+        super().__init__(dict({field.name: field.default for field in fields}, **raw_doc, **params))
+
+    @classmethod
+    def fields(cls):
+        return []
+
+    def serialize(self):
+        """
+        Serialize every field on the record
+        """
+        rec = {}
+        for name, _field in self._metadata.fields.items():
+            rec[name] = _field.serialize(self[name], record=self) if isinstance(_field, Field) else _field
+        return self.__class__(rec, doc_id=self.doc_id)
+
+    def deserialize(self, db, recurse: bool = True):
+        """
+        Deserialize every field on the record
+        """
+        rec = {}
+        for name, _field in self._metadata.fields.items():
+            rec[name] = _field.deserialize(self[name], db, recurse=recurse)
+        return self.__class__(rec, doc_id=self.doc_id)
+
+    def before_insert(self, db: TinyDB) -> None:
+        for name, _field in self._metadata.fields.items():
+            _field.before_insert(self[name], db, self)
+
+    def after_insert(self, db: TinyDB) -> None:
+        for name, _field in self._metadata.fields.items():
+            _field.after_insert(db, self)
+
+    def update(self, **data):
+        for key, value in data.items():
+            self[key] = value
+
+    @property
+    def reference(self):
+        return Pointer.reference(self)
+
+    @property
+    def path(self):
+        return Path(self._metadata.table) / self[self._metadata.primary_key]
+
+    def __setattr__(self, key, value):
+        if key in self:
+            self[key] = value
+        super().__setattr__(key, value)
+
+    def __getattr__(self, attr_name):
+        if attr_name in self:
+            return self.get(attr_name)
+        raise AttributeError(f"No such attribute: {attr_name}")
+
+    def __hash__(self):
+        return hash(str(dict(self)))
+
+    def __repr__(self):
+        return (
+            f"{self.__class__.__name__}[{self.doc_id}]("
+            + ", ".join([f"{key}={val}" for (key, val) in self.items()])
+            + ")"
+        )
+
+
+@dataclass
+class String(Field):
+    pass
+
+
+@dataclass
+class Integer(Field):
+    value_type = int
+    default: int = 0
+
+    def deserialize(self, value: str, db: TinyDB, recurse: bool = False) -> value_type:
+        return int(value)
+
+
+@dataclass
+class Dict(Field):
+    default: dict = field(default_factory=lambda: {})
+
+    value_type = dict
+
+    def serialize(self, values: dict, record: Record | None = None) -> Dict[(str, str)]:
+        return dict((key, str(value)) for key, value in values.items())
+
+    def deserialize(self, values: dict, db: TinyDB, recurse: bool = False) -> Dict[(str, str)]:
+        return values
+
+
+@dataclass
+class List(Field):
+    default: list = field(default_factory=lambda: [])
+
+    value_type = list
+
+    def serialize(self, values: list, record: Record | None = None) -> Dict[(str, str)]:
+        return values
+
+    def deserialize(self, values: list, db: TinyDB, recurse: bool = False) -> typing.List[str]:
+        return values
+
+
+@dataclass
+class DateTime(Field):
+    default: datetime = datetime.utcfromtimestamp(0)
+
+    value_type = datetime
+
+    def serialize(self, value: value_type, record: Record | None = None) -> str:
+        return (value - datetime.utcfromtimestamp(0)).total_seconds()
+
+    def deserialize(self, value: str, db: TinyDB, recurse: bool = False) -> value_type:
+        return datetime.utcfromtimestamp(int(value))
+
+    def before_insert(self, value: value_type, db: TinyDB, record: Record) -> None:
+        if not value:
+            record[self.name] = datetime.utcnow().replace(microsecond=0)
+
+
+@dataclass
+class Timestamp(DateTime):
+    value_type = datetime
+
+    def before_insert(self, value: value_type, db: TinyDB, record: Record) -> None:
+        super().before_insert(None, db, record)
+
+
+@dataclass
+class Password(Field):
+    value_type = str
+    default: str = None
+
+    # Relatively weak. Consider using stronger initial values in production applications.
+    salt_size = 4
+    digest_size = 16
+
+    input_validators = [
+        PatternValidator(re.compile(r"(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[-+_!@#$%^&*.,?<>()])")),
+        LengthValidator(min=8, max=64),
+    ]
+
+    @classmethod
+    def is_digest(cls, passwd: str):
+        if not passwd:
+            return False
+        offset = 2 * cls.salt_size  # each byte is 2 hex chars
+        try:
+            if passwd[offset] != ":":
+                return False
+            digest = passwd[(offset + 1) :]  # noqa
+            if len(digest) != cls.digest_size * 2:
+                return False
+            return re.match(r"^[0-9a-f]+$", digest)
+        except IndexError:
+            return False
+
+    @classmethod
+    def get_digest(cls, passwd: str, salt: bytes = None):
+        if not salt:
+            salt = os.urandom(cls.salt_size)
+        digest = hashlib.blake2b(passwd.encode(), digest_size=cls.digest_size, salt=salt).hexdigest()
+        return digest, salt.hex()
+
+    @classmethod
+    def compare(cls, passwd: value_type, stored: value_type):
+        stored_salt, stored_digest = stored.split(":")
+        input_digest, input_salt = cls.get_digest(passwd, bytes.fromhex(stored_salt))
+        return hmac.compare_digest(input_digest, stored_digest)
+
+    def before_insert(self, value: value_type, db: TinyDB, record: Record) -> None:
+        if value and not self.__class__.is_digest(value):
+            for validator in self.input_validators:
+                validator.validate(record, self, db)
+            digest, salt = self.__class__.get_digest(value)
+            record[self.name] = f"{salt}:{digest}"
+
+
+@dataclass
+class Pointer(Field):
+    """
+    Store a string reference to a record.
+    """
+
+    name: str = ""
+    value_type: grung.types.Record = Record
+
+    def serialize(self, value: value_type | str, record: Record | None = None) -> str:
+        return Pointer.reference(value)
+
+    def deserialize(self, value: str, db: TinyDB, recurse: bool = True) -> value_type:
+        return Pointer.dereference(value, db, recurse)
+
+    @classmethod
+    def reference(cls, value: Record | str):
+        if isinstance(value, str):
+            PointerReferenceValidator().validate_string(value)
+            return value
+
+        if value:
+            return f"{value._metadata.table}::{value._metadata.primary_key}::{value[value._metadata.primary_key]}"
+
+        return None
+
+    @classmethod
+    def dereference(cls, value: str, db: TinyDB, recurse: bool = True):
+        if not value:
+            return
+        elif type(value) == str:
+            table_name, pkey, pval = value.split("::")
+            if pval:
+                table = db.table(table_name)
+                rec = table.get(where(pkey) == pval, recurse=recurse)
+                if not rec:
+                    raise PointerReferenceError(f"Expected a {table_name} with {pkey}=={pval} but did not find one!")
+                return rec
+        return value
+
+
+@dataclass
+class BackReference(Pointer):
+    pass
+
+
+@dataclass
+class BinaryFilePointer(Field):
+    """
+    Write the contents of this field to disk and store the path in the db.
+    """
+
+    name: str
+    extension: str = ".blob"
+
+    value_type = bytes
+
+    def relpath(self, record):
+        return Path(record._metadata.table) / record[record._metadata.primary_key] / f"{self.name}{self.extension}"
+
+    def reference(self, record):
+        return f"/::{self.relpath(record)}"
+
+    def dereference(self, reference, db):
+        relpath = reference.replace("/::", "", 1)
+        try:
+            return (db.path / relpath).read_bytes()
+        except FileNotFoundError:
+            return None
+
+    def serialize(self, value: value_type | str, record: Record | None = None) -> str:
+        return self.reference(record)
+
+    def deserialize(self, value: str, db: TinyDB, recurse: bool = True) -> value_type:
+        if not value:
+            return None
+        return self.dereference(value, db)
+
+    def prepare(self, data: value_type):
+        """
+        Return bytes to be written to disk
+        """
+        if not data:
+            return
+        if not isinstance(data, self.value_type):
+            return data.encode()
+        return data
+
+    def before_insert(self, value: value_type, db: TinyDB, record: Record) -> None:
+        if not value:
+            return
+        relpath = self.relpath(record)
+        path = db.path / relpath
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_bytes(self.prepare(value))
+
+
+@dataclass
+class TextFilePointer(BinaryFilePointer):
+    """
+    Write the contents of this field to disk and store the path in the db.
+    """
+
+    name: str
+    extension: str = ".txt"
+
+    value_type = str
+
+    def prepare(self, data: value_type):
+        if isinstance(data, bytes):
+            return data
+        return str(data).encode()
+
+    def deserialize(self, value: str, db: TinyDB, recurse: bool = True) -> value_type:
+        if not value:
+            return None
+        buf = super().deserialize(value, db)
+        return buf.decode() if buf else None
+
+
+@dataclass
+class Collection(Field):
+    """
+    A collection of pointers.
+    """
+
+    default: typing.List[value_type] = field(default_factory=lambda: [])
+    member_type: type = Record
+
+    value_type = list
+
+    def serialize(self, values: typing.List[value_type], record: Record | None = None) -> typing.List[str]:
+        return [Pointer.reference(val) for val in values]
+
+    def deserialize(self, values: typing.List[str], db: TinyDB, recurse: bool = False) -> typing.List[value_type]:
+        """
+        Recursively deserialize the objects in this collection
+        """
+        recs = []
+        if not recurse:
+            return values
+        for val in values:
+            recs.append(Pointer.dereference(val, db=db, recurse=False))
+        return recs
+
+    def after_insert(self, db: TinyDB, record: Record) -> None:
+        """
+        Populate any backreferences in the members of this collection with the parent record's uid.
+        """
+        if not record[self.name]:
+            return
+
+        for member in record[self.name]:
+            target = Pointer.dereference(member, db=db, recurse=False)
+            for backref in target._metadata.backrefs(type(record)):
+                target[backref.name] = record
+                db.save(target)
+
+
+@dataclass
+class RecordDict(Field):
+    default: typing.Dict[(str, Record)] = field(default_factory=lambda: {})
+    member_type: type = Record
+
+    value_type = dict
+
+    def serialize(
+        self, values: typing.Dict[(str, value_type)], record: Record | None = None
+    ) -> typing.Dict[(str, str)]:
+        return dict((key, Pointer.reference(val)) for (key, val) in values.items())
+
+    def deserialize(
+        self, values: typing.Dict[(str, str)], db: TinyDB, recurse: bool = False
+    ) -> typing.Dict[(str, value_type)]:
+        if not recurse:
+            return values
+        return dict((key, Pointer.dereference(val, db=db, recurse=False)) for (key, val) in values.items())
+
+    def after_insert(self, db: TinyDB, record: Record) -> None:
+        """
+        Populate any backreferences in the members of this mapping with the parent record's uid.
+        """
+        if not record[self.name]:
+            return
+
+        for key, pointer in record[self.name].items():
+            target = Pointer.dereference(pointer, db=db, recurse=False)
+            for backref in target._metadata.backrefs(type(record)):
+                target[backref.name] = record
+                db.save(target)

src/grung/types.py

@@ -1,47 +1,9 @@
-from collections import namedtuple
-from dataclasses import dataclass
-
-import nanoid
-
-Metadata = namedtuple("Metadata", ["table", "fields"])
+import typing
 
 
-@dataclass
 class Field:
-    """
-    Represents a single field in a Record.
-    """
-
-    name: str
-    value_type: type = str
-    default: value_type | None = None
-    unique: bool = False
+    pass
 
 
-class Record(dict):
-    """
-    Base type for a single database record.
-    """
-
-    def __init__(self, raw_doc: dict = {}, doc_id: int = None, **params):
-        # populate the metadata
-        self._fields.append(
-            # 1% collision rate at ~2M records
-            Field("uid", default=nanoid.generate(size=8), unique=True)
-        )
-        self._metadata = Metadata(table=self.__class__.__name__, fields={f.name: f for f in self._fields})
-        self.doc_id = doc_id
-        super().__init__(dict({field.name: field.default for field in self._fields}, **raw_doc, **params))
-
-    def __setattr__(self, key, value):
-        if key in self:
-            self[key] = value
-        super().__setattr__(key, value)
-
-    def __getattr__(self, attr_name):
-        if attr_name in self:
-            return self.get(attr_name)
-        return super().__getattr__(attr_name)
-
-    def __repr__(self):
-        return f"{self.__class__.__name__}[{self.doc_id}]: {self.items()}"
+class Record(typing.Dict[(str, Field)]):
+    pass

src/grung/validators.py

@@ -0,0 +1,199 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from tinydb import Query, TinyDB
+
+from grung.exceptions import (
+    InvalidFieldTypeError,
+    InvalidLengthError,
+    InvalidSizeError,
+    MalformedPointerError,
+    PatternMatchError,
+    UniqueConstraintError,
+    ValidationError,
+)
+from grung.types import Field, Record
+
+
+@dataclass
+class Validator:
+    def validate(self, record: Record, field: Field, db: TinyDB = None) -> bool:
+        raise ValidationError(record, field, self)
+
+
+@dataclass
+class TypeValidator:
+    def validate_list(self, record: Record, field: Field, db: TinyDB = None) -> bool:
+        messages = []
+        for i in range(len(record[field.name])):
+            member = record[field.name][i]
+            if isinstance(member, str):
+                class_name = field.member_type.__name__
+                try:
+                    return PointerReferenceValidator().validate_string(member, class_name)
+                except MalformedPointerError as e:
+                    messages.append(str(e))
+            elif not isinstance(member, field.member_type):
+                messages.append(f"{field.name}[{i}] must be a {field.member_type}, not a {type(member)}.")
+        if messages:
+            raise InvalidFieldTypeError(record, field, self, messages=messages)
+
+    def validate_dict(self, record: Record, field: Field, db: TinyDB = None) -> bool:
+        for key, member in record[field.name].items():
+            if not isinstance(member, field.member_type):
+                raise InvalidFieldTypeError(
+                    record,
+                    field,
+                    self,
+                    messages=[f"{field.name}[{key} must be a {field.member_type}, not a {type(member)}."],
+                )
+        return True
+
+    def validate(self, record: Record, field: Field, db: TinyDB = None) -> bool:
+        if record[field.name] is None:
+            return True
+
+        if not isinstance(record[field.name], field.value_type):
+            raise InvalidFieldTypeError(
+                record,
+                field,
+                self,
+                messages=[f"{field.name} must be a {field.value_type}, not a {type(record[field.name])}."],
+            )
+        if not hasattr(field, "member_type"):
+            return True
+
+        if field.value_type == dict:
+            self.validate_dict(record, field, db)
+        elif field.value_type == list:
+            self.validate_list(record, field, db)
+        else:
+            raise RuntimeError("Expected a validation for iterable but didn't get one!")
+        return True
+
+
+@dataclass
+class PointerReferenceValidator(Validator):
+    """
+    Verify that the Pointer is either a string reference to the correct member type,
+    or a record instance of member_type that hasn't been serialized.
+    """
+
+    def validate_string(self, value: str, type_name: str = "") -> bool:
+        (table, primary_key, val) = value.split("::")
+        if type_name and table != type_name:
+            raise MalformedPointerError(
+                {"string": value},
+                "",
+                self,
+                messages=[f"field should reference '{type_name}', not '{table}'."],
+            )
+        if not primary_key:
+            raise MalformedPointerError(
+                {"string": value},
+                "",
+                self,
+                messages=["Pointers must specify the primary_key field name."],
+            )
+
+    def validate(self, record: Record | str, field: Field, db: TinyDB = None) -> bool:
+        if record[field.name] is None:
+            return True
+
+        if isinstance(record, str):
+            try:
+                self.validate_string(record, field.value_type.__name__)
+            except ValueError:
+                raise MalformedPointerError({field.name: record}, field, self)
+            return True
+
+        if not isinstance(record, Record):
+            raise MalformedPointerError(record, field, self)
+
+        if not isinstance(record[field.name], field.value_type):
+            raise MalformedPointerError(record, field, self)
+
+        return True
+
+
+@dataclass
+class UniqueValidator(Validator):
+    def validate(self, record: Record, field: Field, db: TinyDB) -> bool:
+        """
+        Returns true if the field's value is unique across all records in the table.
+        """
+        if record[field.name] is None:
+            return True
+
+        query = Query()[field.name].matches(f"^{record[field.name]}$", flags=re.IGNORECASE)
+        table = db.table(record._metadata.table)
+        matches = [dict(match) for match in table.search(query) if match.doc_id != record.doc_id]
+        if matches != []:
+            raise UniqueConstraintError(record, field, self, query=query, matches=matches)
+        return True
+
+
+@dataclass
+class LengthValidator(Validator):
+    min: int = 0
+    max: int = 0
+
+    def validate(self, record: Record, field: Field, db: TinyDB = None) -> bool:
+        """
+        Returns True if the length of the field's value is between min and max, inclusive.
+        """
+        if record[field.name] is None:
+            return True
+
+        length = len(record[field.name])
+        if length < self.min or length > self.max:
+            raise InvalidLengthError(
+                record,
+                field,
+                self,
+                messages=[f"The field length must be between {self.min} and {self.max}, inclusive."],
+            )
+        return True
+
+
+@dataclass
+class MinMaxValidator(Validator):
+    min: int = 0
+    max: int = 0
+
+    def validate(self, record: Record, field: Field, db: TinyDB = None) -> bool:
+        """
+        Returns True if the size of the field's integer value is between min and max, inclusive.
+        """
+        if record[field.name] is None:
+            return True
+
+        size = int(record[field.name])
+        if size < self.min or size > self.max:
+            raise InvalidSizeError(
+                record,
+                field,
+                self,
+                messages=[f"The field size must be between {self.min} and {self.max}, inclusive."],
+            )
+        return True
+
+
+@dataclass
+class PatternValidator(Validator):
+    pattern: re.Pattern
+
+    def validate(self, record: Record, field: Field, db: TinyDB = None) -> bool:
+        if record[field.name] is None:
+            return True
+
+        if not self.pattern.match(record[field.name]):
+            raise PatternMatchError(
+                record,
+                field,
+                self,
+                messages=[f"The field value must match the pattern {self.pattern}"],
+            )
+        return True

test/test_db.py

@@ -1,21 +1,37 @@
+import tempfile
+from datetime import datetime
+from pathlib import Path
+from pprint import pprint as print
+from time import sleep
+
 import pytest
+from tinydb import Query
 from tinydb.storages import MemoryStorage
 
 from grung import examples
 from grung.db import GrungDB
+from grung.exceptions import (
+    CircularReferenceError,
+    InvalidFieldTypeError,
+    InvalidLengthError,
+    InvalidSizeError,
+    PatternMatchError,
+    UniqueConstraintError,
+)
 
 
 @pytest.fixture
 def db():
-    _db = GrungDB.with_schema(examples, storage=MemoryStorage)
+    with tempfile.TemporaryDirectory() as path:
+        _db = GrungDB.with_schema(examples, path=Path(path), storage=MemoryStorage)
         yield _db
         print(_db)
 
 
 def test_crud(db):
-    user = examples.User(name="john", email="john@foo")
-    assert user.uid
-    assert user._metadata.fields["uid"].unique
+    user = examples.User(name="john", number=23, email="john@foo")
+    assert user._metadata.fields[user._metadata.primary_key].unique
+    assert user._metadata.fields[user._metadata.primary_key].primary_key
 
     # insert
     john_something = db.save(user)
@@ -24,8 +40,8 @@ def test_crud(db):
     # read back
     assert db.User.get(doc_id=last_insert_id) == john_something
     assert john_something.name == user.name
+    assert john_something.number == 23
     assert john_something.email == user.email
-    assert john_something.uid == user.uid
 
     # update
     john_something.name = "james?"
@@ -34,18 +50,187 @@ def test_crud(db):
     assert after_update == john_something
     assert before_update != after_update
 
-    # pointers
-    players = examples.Group(name="players", users=[john_something])
-    players = db.save(players)
-    players.users[0]["name"] = "fnord"
-    db.save(players)
-
-    # modify records
-    players.users = []
-    db.save(players)
-    after_update = db.Group.get(doc_id=players.doc_id)
-    assert after_update.users == []
-
     # delete
-    db.delete(players)
-    assert len(db.Group) == 0
+    db.delete(john_something)
+    assert len(db.User) == 0
+
+
+def test_pointers(db):
+    user = db.save(examples.User(name="john", email="john@foo"))
+    players = db.save(examples.Group(name="players", members=[user]))
+
+    user = db.table("User").get(doc_id=user.doc_id)
+    assert user.groups.name == players.name
+
+    assert players.members[0].groups.name == players.name
+
+
+def test_subgroups(db):
+    kirk = db.save(examples.User(name="James T. Kirk", email="riskybiznez@starfleet"))
+    pike = db.save(examples.User(name="Christopher Pike", email="hitit@starfleet"))
+
+    tos = db.save(examples.Group(name="The Original Series", members=[kirk]))
+    snw = db.save(examples.Group(name="Strange New Worlds", members=[pike]))
+
+    trek = db.save(examples.Group(name="trek", groups=[tos, snw]))
+    tos = db.table("Group").get(doc_id=tos.doc_id)
+    snw = db.table("Group").get(doc_id=snw.doc_id)
+    assert tos in trek.groups
+    assert snw in trek.groups
+
+    assert trek.parent is None
+    assert tos.parent.name == trek.name
+    assert snw.parent.name == trek.name
+
+    unique_users = set([user for group in trek.groups for user in group.members])
+
+    kirk = db.table("User").get(doc_id=kirk.doc_id)
+    assert kirk.reference in unique_users
+
+    # recursion!
+    with pytest.raises(CircularReferenceError):
+        tos.groups = [tos]
+        db.save(tos)
+
+
+def test_unique(db):
+    user1 = examples.User(name="john", email="john@foo")
+    user2 = examples.User(name="john", email="john@foo")
+
+    user1 = db.save(user1)
+    with pytest.raises(UniqueConstraintError):
+        user2 = db.save(user2)
+    db.save(user1)
+
+
+def test_search(db):
+    # create crew members
+    kirk = db.save(examples.User(name="Captain James T. Kirk", email="riskybiznez@starfleet"))
+    bones = db.save(examples.User(name="Doctor McCoy", email="dammitjim@starfleet"))
+    ricky = db.save(examples.User(name="Ensign Ricky Redshirt", email="invincible@starfleet"))
+
+    # create the crew record
+    crew = db.save(examples.Group(name="Crew", members=[kirk, bones, ricky]))
+
+    User = Query()
+    captains = db.User.search(User.name.matches("Captain"))
+    assert len(captains) == 1
+
+    # update the crew members so they have the backreference to crew
+    kirk = db.table("User").get(doc_id=kirk.doc_id)
+    bones = db.table("User").get(doc_id=bones.doc_id)
+    ricky = db.table("User").get(doc_id=ricky.doc_id)
+
+    assert kirk in crew.members
+    assert bones in crew.members
+    assert ricky in crew.members
+
+    Group = Query()
+    crew = db.Group.get(Group.name == "Crew", recurse=False)
+    assert kirk.reference in crew.members
+
+
+def test_password(db):
+    user = db.save(examples.User(name="john", email="john@foo", password="Fnord!@#%5"))
+
+    # make sure we don't compute the digest on an existing digest
+    user = db.save(user)
+
+    assert ":" in user.password
+    assert user.password != "Fnord!@#%5"
+
+    check = user._metadata.fields["password"].compare
+    assert check("Fnord!@#%5", user.password)
+    assert not check("wrong password", user.password)
+    assert not check("", user.password)
+
+
+def test_datetime(db):
+    user = db.save(examples.User(name="john", email="john@foo", password="Fnord!@#%5", created=datetime.utcnow()))
+    assert user.created > datetime.utcfromtimestamp(0)
+    assert user.created < datetime.utcnow()
+    assert user.last_updated == user.created
+
+    sleep(1)
+    user = db.save(user)
+    assert user.last_updated >= user.created
+
+
+def test_mapping(db):
+    album = db.save(
+        examples.Album(
+            name="The Impossible Kid",
+            credits={"Produced By": "Aesop Rock", "Lyrics By": "Aesop Rock", "Puke in the MeowMix By": "Kirby"},
+            tracks=["Mystery Fish", "Rings", "Lotta Years", "Dorks"],
+        )
+    )
+    assert album.credits["Produced By"] == "Aesop Rock"
+    assert album.tracks[0] == "Mystery Fish"
+
+    aes = db.save(
+        examples.Artist(
+            name="Aesop Rock",
+            albums={"The Impossible Kid": album},
+        )
+    )
+
+    album = db.Album.get(doc_id=album.doc_id)
+    assert album.artist.uid == aes.uid
+    assert album.name in aes.albums
+    assert aes.albums[album.name].uid == album.uid
+    assert "Kirby" in aes.albums[album.name].credits.values()
+
+
+def test_file_pointers(db):
+    album = db.save(
+        examples.Album(
+            name="The Impossible Kid",
+            credits={"Produced By": "Aesop Rock", "Lyrics By": "Aesop Rock", "Puke in the MeowMix By": "Kirby"},
+            tracks=["Mystery Fish", "Rings", "Lotta Years", "Dorks"],
+            cover=b"some jpg data",
+            review="10/10 no notes",
+        )
+    )
+    assert album.cover == b"some jpg data"
+    assert album.review == "10/10 no notes"
+
+    db.save(
+        examples.Artist(
+            name="Aesop Rock",
+            albums={"The Impossible Kid": album},
+        )
+    )
+
+    album = db.Album.get(doc_id=album.doc_id)
+
+    location_on_disk = db.path / album._metadata.fields["cover"].relpath(album)
+    assert location_on_disk.read_bytes() == album.cover
+
+    location_on_disk = db.path / album._metadata.fields["review"].relpath(album)
+    assert location_on_disk.read_text() == album.review
+
+
+@pytest.mark.parametrize(
+    "updates, expected",
+    [
+        ({"name": ""}, InvalidLengthError),
+        ({"name": "a name longer than 30 characters is what we have here"}, InvalidLengthError),
+        ({"name": 23}, InvalidFieldTypeError),
+        ({"number": -1}, InvalidSizeError),
+        ({"number": 256}, InvalidSizeError),
+        ({"email": "foo+alias@"}, PatternMatchError),
+    ],
+    ids=[
+        "name too short",
+        "name too long",
+        "name is not a string",
+        "number too small",
+        "number too big",
+        "invalid email addres",
+    ],
+)
+def test_validators(updates, expected, db):
+    user = db.save(examples.User(name="john", email="john@foo", password="Fnord!@#%5", created=datetime.utcnow()))
+    with pytest.raises(expected):
+        user.update(**updates)
+        db.save(user)