42 lines
1.5 KiB
TypeScript
42 lines
1.5 KiB
TypeScript
/**
|
|
* A single token in the inline token stream. The `role` field
|
|
* distinguishes structural markers from literal content, which
|
|
* is the key insight that makes round-trip escaping correct.
|
|
*/
|
|
import type { Token, TagCollectionType } from "./types";
|
|
/**
|
|
* Scans markdown text into a stream of typed tokens. Handles
|
|
* backslash escapes, entities, flanking rules, links,
|
|
* autolinks, HTML tags, and hard line breaks.
|
|
*
|
|
* const tokenizer = new Tokenizer([
|
|
* { delimiter: '**', htmlTag: 'strong', recursive: true, precedence: 40 },
|
|
* { delimiter: '*', htmlTag: 'em', recursive: true, precedence: 50 },
|
|
* ]);
|
|
* const tokens = tokenizer.tokenize('**bold**');
|
|
*/
|
|
export declare class Tokenizer {
|
|
private tags;
|
|
constructor(tags: TagCollectionType);
|
|
/**
|
|
* Tokenize a markdown string into an inline token stream.
|
|
*
|
|
* tokenizer.tokenize('hello **world**')
|
|
* // [text "hello "] [open "**"] [text "world"] [close "**"]
|
|
*/
|
|
tokenize(source: string): Token[];
|
|
/**
|
|
* Try to resolve an HTML entity at the start of the string.
|
|
* Returns the resolved character and the length consumed, or null.
|
|
*/
|
|
private resolveEntity;
|
|
/**
|
|
* Try to match a delimiter at the given position. For runs of the
|
|
* same character (e.g. *** = 3 asterisks), the run is split into
|
|
* the longest registered delimiter that fits, then the remainder.
|
|
* This handles cases like **bold***italic* where *** must split
|
|
* into ** (close bold) + * (open italic).
|
|
*/
|
|
private matchTag;
|
|
}
|