hopdown/dist/tokenizer.js

"use strict";
/*
 * tokenizer.ts —  markdown tokenizer.
 *
 * Scans markdown text left-to-right producing a typed token stream.
 * Tokens carry their semantic role (delimiter, text, link, etc.)
 * so downstream consumers can make correct escaping and pairing
 * decisions without regex heuristics.
 *
 *   const tokenizer = new Tokenizer(delimiterDefs);
 *   const tokens = tokenizer.tokenize('hello **bold** end');
 *   // [text "hello "] [open "**"] [text "bold"] [close "**"] [text " end"]
 */
Object.defineProperty(exports, "__esModule", { value: true });
exports.Tokenizer = void 0;
/**
 * Characters that count as punctuation for flanking delimiter rules.
 * A delimiter is left-flanking if preceded by whitespace/punctuation
 * and followed by non-whitespace. Right-flanking is the reverse.
 */
const PUNCTUATION = new Set(' \t\n\u00A0.,;:!?\'"()[]{}/<>\\-~#@&^|*`_'.split(''));
/**
 * Characters that can be backslash-escaped in markdown.
 */
const ESCAPABLE = new Set('\\`*_{}[]()#+-.!~|><'.split(''));
/**
 * Named HTML entities recognized by the tokenizer.
 */
const NAMED_ENTITIES = {
    'amp': '&',
    'lt': '<',
    'gt': '>',
    'quot': '"',
    'apos': "'",
    'nbsp': '\u00A0',
};
/**
 * Scans markdown text into a stream of typed tokens. Handles
 * backslash escapes, entities, flanking rules, links,
 * autolinks, HTML tags, and hard line breaks.
 *
 *   const tokenizer = new Tokenizer([
 *       { delimiter: '**', htmlTag: 'strong', recursive: true, precedence: 40 },
 *       { delimiter: '*', htmlTag: 'em', recursive: true, precedence: 50 },
 *   ]);
 *   const tokens = tokenizer.tokenize('**bold**');
 */
class Tokenizer {
    tags;
    constructor(tags) {
        this.tags = tags;
    }
    /**
     * Tokenize a markdown string into an inline token stream.
     *
     *   tokenizer.tokenize('hello **world**')
     *   // [text "hello "] [open "**"] [text "world"] [close "**"]
     */
    tokenize(source) {
        const tokens = [];
        var position = 0;
        var textBuffer = '';
        const flushText = () => {
            if (textBuffer.length > 0) {
                tokens.push({
                    role: 'block',
                    value: textBuffer,
                });
                textBuffer = '';
            }
        };
        while (position < source.length) {
            const remaining = source.slice(position);
            // Backslash escape: \X → literal X
            if (source[position] === '\\' && position + 1 < source.length) {
                const nextChar = source[position + 1];
                if (ESCAPABLE.has(nextChar)) {
                    textBuffer += '\\' + nextChar;
                    position += 2;
                    continue;
                }
            }
            // HTML entity resolution: &name; or &#digits; or &#xhex;
            if (source[position] === '&') {
                const resolved = this.resolveEntity(remaining);
                if (resolved) {
                    textBuffer += resolved.character;
                    position += resolved.length;
                    continue;
                }
            }
            var match = this.matchTag(source, position);
            if (match && match.value?.length) {
                flushText();
                tokens.push(match);
                position += match.value?.length;
                continue;
            }
            // check for autolinks
            const anchor = this.tags.getByElementName('A');
            if (anchor) {
                match = anchor.tokenize('block', remaining[0], remaining);
                if (match && match.value?.length) {
                    flushText();
                    tokens.push(match);
                    position += match.value?.length;
                    continue;
                }
            }
            // Plain character
            textBuffer += source[position];
            position++;
        }
        flushText();
        return tokens;
    }
    /**
     * Try to resolve an HTML entity at the start of the string.
     * Returns the resolved character and the length consumed, or null.
     */
    resolveEntity(text) {
        const namedPattern = /^&(?<name>[a-zA-Z]+);/;
        const numericPattern = /^&#(?<code>\d+);/;
        const hexPattern = /^&#x(?<hex>[0-9a-fA-F]+);/;
        const named = text.match(namedPattern);
        if (named?.groups) {
            const resolved = NAMED_ENTITIES[named.groups.name.toLowerCase()];
            if (resolved) {
                return {
                    character: resolved,
                    length: named[0].length,
                };
            }
        }
        const numeric = text.match(numericPattern);
        if (numeric?.groups) {
            return {
                character: String.fromCharCode(parseInt(numeric.groups.code, 10)),
                length: numeric[0].length,
            };
        }
        const hex = text.match(hexPattern);
        if (hex?.groups) {
            return {
                character: String.fromCharCode(parseInt(hex.groups.hex, 16)),
                length: hex[0].length,
            };
        }
        return null;
    }
    /**
     * Try to match a delimiter at the given position. For runs of the
     * same character (e.g. *** = 3 asterisks), the run is split into
     * the longest registered delimiter that fits, then the remainder.
     * This handles cases like **bold***italic* where *** must split
     * into ** (close bold) + * (open italic).
     */
    matchTag(source, position) {
        // Count the full run of the same character
        var sequence = source[position];
        let runLength = 0;
        while (position + runLength < source.length && source[position + runLength] === sequence) {
            runLength++;
        }
        if (runLength === 0) {
            return null;
        }
        sequence = sequence.repeat(runLength);
        var matches = this.tags.map.get(sequence);
        if (!matches && runLength == 1) {
            sequence = source.slice(position, 3);
            for (var i = 0; i < 2; i++) {
                sequence = sequence.slice(0, sequence.length - 1);
                matches = this.tags.map.get(sequence);
                if (matches) {
                    break;
                }
            }
        }
        if (!matches || !sequence) {
            return null;
        }
        for (const tag of matches) {
            const charBefore = position > 0 ? source[position - 1] : '\n';
            const charAfter = source[position + sequence.length];
            const leftFlanking = (charBefore === undefined || PUNCTUATION.has(charBefore) || charBefore === '\n')
                && charAfter !== undefined && charAfter !== ' ' && charAfter !== '\n' && charAfter !== '\t' && charAfter !== '\u00A0';
            const rightFlanking = charBefore !== undefined && charBefore !== ' ' && charBefore !== '\n' && charBefore !== '\t' && charBefore !== '\u00A0'
                && (charAfter === undefined || PUNCTUATION.has(charAfter) || charAfter === '\n');
            const role = leftFlanking ? 'open' : rightFlanking ? 'close' : 'block';
            var token = tag.tokenize(role, sequence, source);
            if (token) {
                return token;
            }
        }
        ;
        return null;
    }
}
exports.Tokenizer = Tokenizer;