add dist/ to src
This commit is contained in:
evilchili
Vendored
+324
@@ -0,0 +1,324 @@
|
||||
"use strict";
|
||||
/*
|
||||
* tokenizer.ts — markdown tokenizer.
|
||||
*
|
||||
* Scans markdown text left-to-right producing a typed token stream.
|
||||
* Tokens carry their semantic role (delimiter, text, link, etc.)
|
||||
* so downstream consumers can make correct escaping and pairing
|
||||
* decisions without regex heuristics.
|
||||
*
|
||||
* const tokenizer = new Tokenizer(delimiterDefs);
|
||||
* const tokens = tokenizer.tokenize('hello **bold** end');
|
||||
* // [text "hello "] [open "**"] [text "bold"] [close "**"] [text " end"]
|
||||
*/
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.Tokenizer = void 0;
|
||||
/**
|
||||
* Characters that count as punctuation for flanking delimiter rules.
|
||||
* A delimiter is left-flanking if preceded by whitespace/punctuation
|
||||
* and followed by non-whitespace. Right-flanking is the reverse.
|
||||
*/
|
||||
const PUNCTUATION = new Set(' \t\n\u00A0.,;:!?\'"()[]{}/<>\\-~#@&^|*`_'.split(''));
|
||||
/**
|
||||
* Characters that can be backslash-escaped in markdown.
|
||||
*/
|
||||
const ESCAPABLE = new Set('\\`*_{}[]()#+-.!~|><'.split(''));
|
||||
/**
|
||||
* Named HTML entities recognized by the tokenizer.
|
||||
*/
|
||||
const NAMED_ENTITIES = {
|
||||
'amp': '&',
|
||||
'lt': '<',
|
||||
'gt': '>',
|
||||
'quot': '"',
|
||||
'apos': "'",
|
||||
'nbsp': '\u00A0',
|
||||
};
|
||||
/**
|
||||
* Scans markdown text into a stream of typed tokens. Handles
|
||||
* backslash escapes, entities, flanking rules, links,
|
||||
* autolinks, HTML tags, and hard line breaks.
|
||||
*
|
||||
* const tokenizer = new Tokenizer([
|
||||
* { delimiter: '**', htmlTag: 'strong', recursive: true, precedence: 40 },
|
||||
* { delimiter: '*', htmlTag: 'em', recursive: true, precedence: 50 },
|
||||
* ]);
|
||||
* const tokens = tokenizer.tokenize('**bold**');
|
||||
*/
|
||||
class Tokenizer {
|
||||
tags;
|
||||
constructor(tags) {
|
||||
this.tags = tags.ordered;
|
||||
}
|
||||
/**
|
||||
* Tokenize a markdown string into an inline token stream.
|
||||
*
|
||||
* tokenizer.tokenize('hello **world**')
|
||||
* // [text "hello "] [open "**"] [text "world"] [close "**"]
|
||||
*/
|
||||
tokenize(source) {
|
||||
const tokens = [];
|
||||
let position = 0;
|
||||
let textBuffer = '';
|
||||
const flushText = () => {
|
||||
if (textBuffer.length > 0) {
|
||||
tokens.push({
|
||||
role: 'text',
|
||||
value: textBuffer,
|
||||
});
|
||||
textBuffer = '';
|
||||
}
|
||||
};
|
||||
while (position < source.length) {
|
||||
const remaining = source.slice(position);
|
||||
// Backslash escape: \X → literal X
|
||||
if (source[position] === '\\' && position + 1 < source.length) {
|
||||
const nextChar = source[position + 1];
|
||||
if (ESCAPABLE.has(nextChar)) {
|
||||
textBuffer += '\\' + nextChar;
|
||||
position += 2;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// Hard line break: two+ trailing spaces before newline
|
||||
if (source[position] === ' ') {
|
||||
const spaceMatch = remaining.match(/^(?<spaces> {2,})\n/);
|
||||
if (spaceMatch?.groups) {
|
||||
flushText();
|
||||
tokens.push({ role: 'break', value: '<br>' });
|
||||
position += spaceMatch[0].length;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// HTML entity resolution: &name; or &#digits; or &#xhex;
|
||||
if (source[position] === '&') {
|
||||
const resolved = this.resolveEntity(remaining);
|
||||
if (resolved) {
|
||||
textBuffer += resolved.character;
|
||||
position += resolved.length;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// Link: [text](url) or [text](url "title")
|
||||
if (source[position] === '[') {
|
||||
const link = this.matchLink(source, position);
|
||||
if (link) {
|
||||
flushText();
|
||||
tokens.push({
|
||||
role: 'link',
|
||||
value: link.text,
|
||||
href: link.href,
|
||||
title: link.title,
|
||||
});
|
||||
position += link.length;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// Autolink: <url> and/or HTML
|
||||
if (source[position] === '<') {
|
||||
const autolink = this.matchAutolink(remaining);
|
||||
if (autolink) {
|
||||
flushText();
|
||||
tokens.push({
|
||||
role: 'link',
|
||||
value: autolink.url,
|
||||
href: autolink.url,
|
||||
tag: this.tags.filter(t => { return t.constructor.name == 'Anchor'; })[0]
|
||||
});
|
||||
position += autolink.length;
|
||||
continue;
|
||||
}
|
||||
// HTML tag passthrough
|
||||
const htmlTagMatch = this.matchHtmlTag(remaining);
|
||||
if (htmlTagMatch) {
|
||||
flushText();
|
||||
tokens.push({
|
||||
role: 'html',
|
||||
value: htmlTagMatch.tag,
|
||||
});
|
||||
position += htmlTagMatch.length;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// autolink bare URLs
|
||||
const bareUrl = this.matchBareUrl(remaining);
|
||||
if (bareUrl) {
|
||||
flushText();
|
||||
tokens.push({
|
||||
role: 'link',
|
||||
value: bareUrl.url,
|
||||
href: bareUrl.url,
|
||||
tag: this.tags.filter(t => { return t.constructor.name == 'Anchor'; })[0]
|
||||
});
|
||||
position += bareUrl.length;
|
||||
continue;
|
||||
}
|
||||
// Delimiter: check each registered delimiter
|
||||
const match = this.matchTag(source, position);
|
||||
if (match && match.sequence?.length) {
|
||||
flushText();
|
||||
tokens.push(match);
|
||||
position += match.sequence?.length;
|
||||
continue;
|
||||
}
|
||||
// Plain character
|
||||
textBuffer += source[position];
|
||||
position++;
|
||||
}
|
||||
flushText();
|
||||
return tokens;
|
||||
}
|
||||
/**
|
||||
* Try to resolve an HTML entity at the start of the string.
|
||||
* Returns the resolved character and the length consumed, or null.
|
||||
*/
|
||||
resolveEntity(text) {
|
||||
const namedPattern = /^&(?<name>[a-zA-Z]+);/;
|
||||
const numericPattern = /^&#(?<code>\d+);/;
|
||||
const hexPattern = /^&#x(?<hex>[0-9a-fA-F]+);/;
|
||||
const named = text.match(namedPattern);
|
||||
if (named?.groups) {
|
||||
const resolved = NAMED_ENTITIES[named.groups.name.toLowerCase()];
|
||||
if (resolved) {
|
||||
return {
|
||||
character: resolved,
|
||||
length: named[0].length,
|
||||
};
|
||||
}
|
||||
}
|
||||
const numeric = text.match(numericPattern);
|
||||
if (numeric?.groups) {
|
||||
return {
|
||||
character: String.fromCharCode(parseInt(numeric.groups.code, 10)),
|
||||
length: numeric[0].length,
|
||||
};
|
||||
}
|
||||
const hex = text.match(hexPattern);
|
||||
if (hex?.groups) {
|
||||
return {
|
||||
character: String.fromCharCode(parseInt(hex.groups.hex, 16)),
|
||||
length: hex[0].length,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Match a code span starting at the given position.
|
||||
* Handles single backtick delimiters only (not multi-backtick).
|
||||
*/
|
||||
matchCodeSpan(source, position) {
|
||||
if (source[position] !== '`') {
|
||||
return null;
|
||||
}
|
||||
const closeIndex = source.indexOf('`', position + 1);
|
||||
if (closeIndex === -1) {
|
||||
return null;
|
||||
}
|
||||
const content = source.slice(position + 1, closeIndex);
|
||||
return {
|
||||
content,
|
||||
raw: source.slice(position, closeIndex + 1),
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Match a markdown link [text](url) or [text](url "title")
|
||||
* starting at the given position. Disallows [ in link text
|
||||
* to prevent nested link ambiguity.
|
||||
*/
|
||||
matchLink(source, position) {
|
||||
const linkPattern = /^\[(?<text>[^\[\]]+)\]\((?<href>[^\s)]+)(?:\s+"(?<title>[^"]*)")?\)/;
|
||||
const match = source.slice(position).match(linkPattern);
|
||||
if (!match?.groups) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
text: match.groups.text,
|
||||
href: match.groups.href,
|
||||
title: match.groups.title,
|
||||
length: match[0].length,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Match an angle-bracket autolink <url> at the start of the string.
|
||||
*/
|
||||
matchAutolink(text) {
|
||||
const pattern = /^<(?<url>https?:\/\/[^\s>]+)>/;
|
||||
const match = text.match(pattern);
|
||||
if (!match?.groups) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
url: match.groups.url,
|
||||
length: match[0].length,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Match a bare URL (https://...) at the start of the string.
|
||||
*/
|
||||
matchBareUrl(text) {
|
||||
const pattern = /^[\w]+?:\/\/[^\s<>\x00]+/;
|
||||
const match = text.match(pattern);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
url: match[0],
|
||||
length: match[0].length,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Match an HTML tag at the start of the string.
|
||||
*/
|
||||
matchHtmlTag(text) {
|
||||
const pattern = /^<\/?[a-zA-Z][a-zA-Z0-9]*(?:\s+[^>]*)?\s*\/?>/;
|
||||
const match = text.match(pattern);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
tag: match[0],
|
||||
length: match[0].length,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Try to match a delimiter at the given position. For runs of the
|
||||
* same character (e.g. *** = 3 asterisks), the run is split into
|
||||
* the longest registered delimiter that fits, then the remainder.
|
||||
* This handles cases like **bold***italic* where *** must split
|
||||
* into ** (close bold) + * (open italic).
|
||||
*/
|
||||
matchTag(source, position) {
|
||||
// Count the full run of the same character
|
||||
var sequence = source[position];
|
||||
let runLength = 0;
|
||||
while (position + runLength < source.length && source[position + runLength] === sequence) {
|
||||
runLength++;
|
||||
}
|
||||
if (runLength === 0) {
|
||||
return null;
|
||||
}
|
||||
sequence = sequence.repeat(runLength);
|
||||
// get the tag that matches this sequence
|
||||
for (const tag of this.tags) {
|
||||
const delim = tag.isBlock ? tag.opener : tag.delimiter;
|
||||
if (delim !== sequence) {
|
||||
continue;
|
||||
}
|
||||
const charBefore = position > 0 ? source[position - 1] : '\n';
|
||||
const charAfter = source[position + sequence.length];
|
||||
const leftFlanking = (charBefore === undefined || PUNCTUATION.has(charBefore) || charBefore === '\n')
|
||||
&& charAfter !== undefined && charAfter !== ' ' && charAfter !== '\n' && charAfter !== '\t' && charAfter !== '\u00A0';
|
||||
const rightFlanking = charBefore !== undefined && charBefore !== ' ' && charBefore !== '\n' && charBefore !== '\t' && charBefore !== '\u00A0'
|
||||
&& (charAfter === undefined || PUNCTUATION.has(charAfter) || charAfter === '\n');
|
||||
return {
|
||||
role: leftFlanking ? 'open' : rightFlanking ? 'close' : 'block',
|
||||
sequence: sequence,
|
||||
value: sequence,
|
||||
tag: tag,
|
||||
};
|
||||
}
|
||||
;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
exports.Tokenizer = Tokenizer;
|
||||
Reference in New Issue
Block a user