325 lines
11 KiB
JavaScript
325 lines
11 KiB
JavaScript
"use strict";
|
|
/*
|
|
* tokenizer.ts — markdown tokenizer.
|
|
*
|
|
* Scans markdown text left-to-right producing a typed token stream.
|
|
* Tokens carry their semantic role (delimiter, text, link, etc.)
|
|
* so downstream consumers can make correct escaping and pairing
|
|
* decisions without regex heuristics.
|
|
*
|
|
* const tokenizer = new Tokenizer(delimiterDefs);
|
|
* const tokens = tokenizer.tokenize('hello **bold** end');
|
|
* // [text "hello "] [open "**"] [text "bold"] [close "**"] [text " end"]
|
|
*/
|
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
exports.Tokenizer = void 0;
|
|
/**
|
|
* Characters that count as punctuation for flanking delimiter rules.
|
|
* A delimiter is left-flanking if preceded by whitespace/punctuation
|
|
* and followed by non-whitespace. Right-flanking is the reverse.
|
|
*/
|
|
const PUNCTUATION = new Set(' \t\n\u00A0.,;:!?\'"()[]{}/<>\\-~#@&^|*`_'.split(''));
|
|
/**
|
|
* Characters that can be backslash-escaped in markdown.
|
|
*/
|
|
const ESCAPABLE = new Set('\\`*_{}[]()#+-.!~|><'.split(''));
|
|
/**
|
|
* Named HTML entities recognized by the tokenizer.
|
|
*/
|
|
const NAMED_ENTITIES = {
|
|
'amp': '&',
|
|
'lt': '<',
|
|
'gt': '>',
|
|
'quot': '"',
|
|
'apos': "'",
|
|
'nbsp': '\u00A0',
|
|
};
|
|
/**
|
|
* Scans markdown text into a stream of typed tokens. Handles
|
|
* backslash escapes, entities, flanking rules, links,
|
|
* autolinks, HTML tags, and hard line breaks.
|
|
*
|
|
* const tokenizer = new Tokenizer([
|
|
* { delimiter: '**', htmlTag: 'strong', recursive: true, precedence: 40 },
|
|
* { delimiter: '*', htmlTag: 'em', recursive: true, precedence: 50 },
|
|
* ]);
|
|
* const tokens = tokenizer.tokenize('**bold**');
|
|
*/
|
|
class Tokenizer {
|
|
tags;
|
|
constructor(tags) {
|
|
this.tags = tags.ordered;
|
|
}
|
|
/**
|
|
* Tokenize a markdown string into an inline token stream.
|
|
*
|
|
* tokenizer.tokenize('hello **world**')
|
|
* // [text "hello "] [open "**"] [text "world"] [close "**"]
|
|
*/
|
|
tokenize(source) {
|
|
const tokens = [];
|
|
let position = 0;
|
|
let textBuffer = '';
|
|
const flushText = () => {
|
|
if (textBuffer.length > 0) {
|
|
tokens.push({
|
|
role: 'text',
|
|
value: textBuffer,
|
|
});
|
|
textBuffer = '';
|
|
}
|
|
};
|
|
while (position < source.length) {
|
|
const remaining = source.slice(position);
|
|
// Backslash escape: \X → literal X
|
|
if (source[position] === '\\' && position + 1 < source.length) {
|
|
const nextChar = source[position + 1];
|
|
if (ESCAPABLE.has(nextChar)) {
|
|
textBuffer += '\\' + nextChar;
|
|
position += 2;
|
|
continue;
|
|
}
|
|
}
|
|
// Hard line break: two+ trailing spaces before newline
|
|
if (source[position] === ' ') {
|
|
const spaceMatch = remaining.match(/^(?<spaces> {2,})\n/);
|
|
if (spaceMatch?.groups) {
|
|
flushText();
|
|
tokens.push({ role: 'break', value: '<br>' });
|
|
position += spaceMatch[0].length;
|
|
continue;
|
|
}
|
|
}
|
|
// HTML entity resolution: &name; or &#digits; or &#xhex;
|
|
if (source[position] === '&') {
|
|
const resolved = this.resolveEntity(remaining);
|
|
if (resolved) {
|
|
textBuffer += resolved.character;
|
|
position += resolved.length;
|
|
continue;
|
|
}
|
|
}
|
|
// Link: [text](url) or [text](url "title")
|
|
if (source[position] === '[') {
|
|
const link = this.matchLink(source, position);
|
|
if (link) {
|
|
flushText();
|
|
tokens.push({
|
|
role: 'link',
|
|
value: link.text,
|
|
href: link.href,
|
|
title: link.title,
|
|
});
|
|
position += link.length;
|
|
continue;
|
|
}
|
|
}
|
|
// Autolink: <url> and/or HTML
|
|
if (source[position] === '<') {
|
|
const autolink = this.matchAutolink(remaining);
|
|
if (autolink) {
|
|
flushText();
|
|
tokens.push({
|
|
role: 'link',
|
|
value: autolink.url,
|
|
href: autolink.url,
|
|
tag: this.tags.filter(t => { return t.constructor.name == 'Anchor'; })[0]
|
|
});
|
|
position += autolink.length;
|
|
continue;
|
|
}
|
|
// HTML tag passthrough
|
|
const htmlTagMatch = this.matchHtmlTag(remaining);
|
|
if (htmlTagMatch) {
|
|
flushText();
|
|
tokens.push({
|
|
role: 'html',
|
|
value: htmlTagMatch.tag,
|
|
});
|
|
position += htmlTagMatch.length;
|
|
continue;
|
|
}
|
|
}
|
|
// autolink bare URLs
|
|
const bareUrl = this.matchBareUrl(remaining);
|
|
if (bareUrl) {
|
|
flushText();
|
|
tokens.push({
|
|
role: 'link',
|
|
value: bareUrl.url,
|
|
href: bareUrl.url,
|
|
tag: this.tags.filter(t => { return t.constructor.name == 'Anchor'; })[0]
|
|
});
|
|
position += bareUrl.length;
|
|
continue;
|
|
}
|
|
// Delimiter: check each registered delimiter
|
|
const match = this.matchTag(source, position);
|
|
if (match && match.sequence?.length) {
|
|
flushText();
|
|
tokens.push(match);
|
|
position += match.sequence?.length;
|
|
continue;
|
|
}
|
|
// Plain character
|
|
textBuffer += source[position];
|
|
position++;
|
|
}
|
|
flushText();
|
|
return tokens;
|
|
}
|
|
/**
|
|
* Try to resolve an HTML entity at the start of the string.
|
|
* Returns the resolved character and the length consumed, or null.
|
|
*/
|
|
resolveEntity(text) {
|
|
const namedPattern = /^&(?<name>[a-zA-Z]+);/;
|
|
const numericPattern = /^&#(?<code>\d+);/;
|
|
const hexPattern = /^&#x(?<hex>[0-9a-fA-F]+);/;
|
|
const named = text.match(namedPattern);
|
|
if (named?.groups) {
|
|
const resolved = NAMED_ENTITIES[named.groups.name.toLowerCase()];
|
|
if (resolved) {
|
|
return {
|
|
character: resolved,
|
|
length: named[0].length,
|
|
};
|
|
}
|
|
}
|
|
const numeric = text.match(numericPattern);
|
|
if (numeric?.groups) {
|
|
return {
|
|
character: String.fromCharCode(parseInt(numeric.groups.code, 10)),
|
|
length: numeric[0].length,
|
|
};
|
|
}
|
|
const hex = text.match(hexPattern);
|
|
if (hex?.groups) {
|
|
return {
|
|
character: String.fromCharCode(parseInt(hex.groups.hex, 16)),
|
|
length: hex[0].length,
|
|
};
|
|
}
|
|
return null;
|
|
}
|
|
/**
|
|
* Match a code span starting at the given position.
|
|
* Handles single backtick delimiters only (not multi-backtick).
|
|
*/
|
|
matchCodeSpan(source, position) {
|
|
if (source[position] !== '`') {
|
|
return null;
|
|
}
|
|
const closeIndex = source.indexOf('`', position + 1);
|
|
if (closeIndex === -1) {
|
|
return null;
|
|
}
|
|
const content = source.slice(position + 1, closeIndex);
|
|
return {
|
|
content,
|
|
raw: source.slice(position, closeIndex + 1),
|
|
};
|
|
}
|
|
/**
|
|
* Match a markdown link [text](url) or [text](url "title")
|
|
* starting at the given position. Disallows [ in link text
|
|
* to prevent nested link ambiguity.
|
|
*/
|
|
matchLink(source, position) {
|
|
const linkPattern = /^\[(?<text>[^\[\]]+)\]\((?<href>[^\s)]+)(?:\s+"(?<title>[^"]*)")?\)/;
|
|
const match = source.slice(position).match(linkPattern);
|
|
if (!match?.groups) {
|
|
return null;
|
|
}
|
|
return {
|
|
text: match.groups.text,
|
|
href: match.groups.href,
|
|
title: match.groups.title,
|
|
length: match[0].length,
|
|
};
|
|
}
|
|
/**
|
|
* Match an angle-bracket autolink <url> at the start of the string.
|
|
*/
|
|
matchAutolink(text) {
|
|
const pattern = /^<(?<url>https?:\/\/[^\s>]+)>/;
|
|
const match = text.match(pattern);
|
|
if (!match?.groups) {
|
|
return null;
|
|
}
|
|
return {
|
|
url: match.groups.url,
|
|
length: match[0].length,
|
|
};
|
|
}
|
|
/**
|
|
* Match a bare URL (https://...) at the start of the string.
|
|
*/
|
|
matchBareUrl(text) {
|
|
const pattern = /^[\w]+?:\/\/[^\s<>\x00]+/;
|
|
const match = text.match(pattern);
|
|
if (!match) {
|
|
return null;
|
|
}
|
|
return {
|
|
url: match[0],
|
|
length: match[0].length,
|
|
};
|
|
}
|
|
/**
|
|
* Match an HTML tag at the start of the string.
|
|
*/
|
|
matchHtmlTag(text) {
|
|
const pattern = /^<\/?[a-zA-Z][a-zA-Z0-9]*(?:\s+[^>]*)?\s*\/?>/;
|
|
const match = text.match(pattern);
|
|
if (!match) {
|
|
return null;
|
|
}
|
|
return {
|
|
tag: match[0],
|
|
length: match[0].length,
|
|
};
|
|
}
|
|
/**
|
|
* Try to match a delimiter at the given position. For runs of the
|
|
* same character (e.g. *** = 3 asterisks), the run is split into
|
|
* the longest registered delimiter that fits, then the remainder.
|
|
* This handles cases like **bold***italic* where *** must split
|
|
* into ** (close bold) + * (open italic).
|
|
*/
|
|
matchTag(source, position) {
|
|
// Count the full run of the same character
|
|
var sequence = source[position];
|
|
let runLength = 0;
|
|
while (position + runLength < source.length && source[position + runLength] === sequence) {
|
|
runLength++;
|
|
}
|
|
if (runLength === 0) {
|
|
return null;
|
|
}
|
|
sequence = sequence.repeat(runLength);
|
|
// get the tag that matches this sequence
|
|
for (const tag of this.tags) {
|
|
const delim = tag.isBlock ? tag.opener : tag.delimiter;
|
|
if (delim !== sequence) {
|
|
continue;
|
|
}
|
|
const charBefore = position > 0 ? source[position - 1] : '\n';
|
|
const charAfter = source[position + sequence.length];
|
|
const leftFlanking = (charBefore === undefined || PUNCTUATION.has(charBefore) || charBefore === '\n')
|
|
&& charAfter !== undefined && charAfter !== ' ' && charAfter !== '\n' && charAfter !== '\t' && charAfter !== '\u00A0';
|
|
const rightFlanking = charBefore !== undefined && charBefore !== ' ' && charBefore !== '\n' && charBefore !== '\t' && charBefore !== '\u00A0'
|
|
&& (charAfter === undefined || PUNCTUATION.has(charAfter) || charAfter === '\n');
|
|
return {
|
|
role: leftFlanking ? 'open' : rightFlanking ? 'close' : 'block',
|
|
sequence: sequence,
|
|
value: sequence,
|
|
tag: tag,
|
|
};
|
|
}
|
|
;
|
|
return null;
|
|
}
|
|
}
|
|
exports.Tokenizer = Tokenizer;
|