add dist/ to src
This commit is contained in:
evilchili
Vendored
+54
@@ -0,0 +1,54 @@
|
||||
"use strict";
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
var desc = Object.getOwnPropertyDescriptor(m, k);
|
||||
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
||||
desc = { enumerable: true, get: function() { return m[k]; } };
|
||||
}
|
||||
Object.defineProperty(o, k2, desc);
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||||
}) : function(o, v) {
|
||||
o["default"] = v;
|
||||
});
|
||||
var __importStar = (this && this.__importStar) || (function () {
|
||||
var ownKeys = function(o) {
|
||||
ownKeys = Object.getOwnPropertyNames || function (o) {
|
||||
var ar = [];
|
||||
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
||||
return ar;
|
||||
};
|
||||
return ownKeys(o);
|
||||
};
|
||||
return function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
||||
__setModuleDefault(result, mod);
|
||||
return result;
|
||||
};
|
||||
})();
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.defaultTags = void 0;
|
||||
const tags = __importStar(require("./tags"));
|
||||
exports.defaultTags = new tags.TagCollection([
|
||||
new tags.Bold(),
|
||||
new tags.Italic(),
|
||||
new tags.BoldItalic(),
|
||||
new tags.Strikethrough(),
|
||||
new tags.Code(),
|
||||
new tags.Anchor(),
|
||||
new tags.HardBreak(),
|
||||
new tags.FencedCode(),
|
||||
new tags.HorizontalRule(),
|
||||
new tags.Heading(),
|
||||
new tags.Blockquote(),
|
||||
new tags.OrderedList(),
|
||||
new tags.UnorderedList(),
|
||||
new tags.Table(),
|
||||
new tags.Paragraph()
|
||||
]);
|
||||
Vendored
+383
@@ -0,0 +1,383 @@
|
||||
"use strict";
|
||||
/*
|
||||
* hopdown.ts
|
||||
* - Configurable markdown <=> HTML converter.
|
||||
*/
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.HopDown = void 0;
|
||||
exports.escapeHtml = escapeHtml;
|
||||
const defaults_1 = require("./defaults");
|
||||
const tokenizer_1 = require("./tokenizer");
|
||||
function escapeHtml(source) {
|
||||
return source
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"');
|
||||
}
|
||||
/**
|
||||
* A configurable markdown <=> HTML converter.
|
||||
*
|
||||
* Examples:
|
||||
*
|
||||
* const hopdown = new HopDown();
|
||||
*
|
||||
* const html = hopdown.toHTML('**bold**');
|
||||
* html === '<p><strong>bold</strong></p>';
|
||||
*
|
||||
* const markdown = hopdown.toMarkdown(html);
|
||||
* markdown === '**bold**';
|
||||
*
|
||||
* const node = hopdown.toEditorNode('**bold**');
|
||||
* (node as HTMLElement).outerHTML === (
|
||||
* "<div>" +
|
||||
* "<span class='delim strong'>**</span>" +
|
||||
* "bold" +
|
||||
* "<span class='delim strong'>" +
|
||||
* "</div>"
|
||||
* );
|
||||
*/
|
||||
class HopDown {
|
||||
tags;
|
||||
referenceLinks = new Map();
|
||||
tokenizer;
|
||||
constructor(tags) {
|
||||
this.tags = tags || defaults_1.defaultTags;
|
||||
this.tokenizer = new tokenizer_1.Tokenizer(this.tags);
|
||||
}
|
||||
/**
|
||||
* Convert a markdown string to tokens.
|
||||
*/
|
||||
tokenize(source) {
|
||||
let text = source;
|
||||
// Resolve reference links before tokenizing
|
||||
text = this.resolveReferenceLinks(text);
|
||||
// Normalize _ emphasis to *
|
||||
text = this.normalizeUnderscores(text);
|
||||
const tokens = this.tokenizer.tokenize(text);
|
||||
return tokens;
|
||||
}
|
||||
/**
|
||||
* Convert an HTML string or a DocumentFragment to markdown.
|
||||
*/
|
||||
toMarkdown = (html) => {
|
||||
var md = '';
|
||||
if (typeof (html) == 'string') {
|
||||
const container = document.createElement('div');
|
||||
container.innerHTML = html;
|
||||
md = this.nodeToMarkdown(container);
|
||||
}
|
||||
else {
|
||||
md = this.nodeToMarkdown(html);
|
||||
}
|
||||
return md.replace(/\n{3,}/g, '\n\n').trim();
|
||||
};
|
||||
/**
|
||||
* Convert a markdown string to an HTML string.
|
||||
*/
|
||||
toHTML = (markdown) => {
|
||||
const output = [];
|
||||
const preprocessed = this.preprocessMarkdown(markdown);
|
||||
if (!preprocessed.lines) {
|
||||
return '';
|
||||
}
|
||||
const blockTags = this.tags.ordered.filter(t => {
|
||||
return t.isBlock && t.constructor.name !== 'Paragraph';
|
||||
});
|
||||
const pTag = this.tags.getByElementName('p');
|
||||
if (pTag === undefined) {
|
||||
throw new Error("Cannot parse without a paragraph tag!");
|
||||
}
|
||||
let lineIndex = 0;
|
||||
while (lineIndex < preprocessed.lines.length) {
|
||||
let context;
|
||||
let matched = false;
|
||||
let token;
|
||||
for (const tag of blockTags) {
|
||||
context = {
|
||||
lines: preprocessed.lines,
|
||||
index: lineIndex,
|
||||
text: '',
|
||||
offset: 0,
|
||||
};
|
||||
token = tag.match(context, this.tags);
|
||||
if (!token) {
|
||||
continue;
|
||||
}
|
||||
output.push(tag.toHTML(token, this.inlineToHTML));
|
||||
lineIndex += token.consumed;
|
||||
matched = true;
|
||||
break;
|
||||
}
|
||||
if (matched) {
|
||||
continue;
|
||||
}
|
||||
/* always check the paragraph tag last */
|
||||
token = pTag.match(context, this.tags);
|
||||
if (token) {
|
||||
output.push(pTag.toHTML(token, this.inlineToHTML));
|
||||
lineIndex += token.consumed;
|
||||
continue;
|
||||
}
|
||||
lineIndex++;
|
||||
}
|
||||
return output.join('\n');
|
||||
};
|
||||
/**
|
||||
* Convert a markdown string to a DocumentFragment consisting of wysiwyg editor nodes.
|
||||
*/
|
||||
toEditorNode = (markdown) => {
|
||||
const tokens = this.tokenize(markdown);
|
||||
return this.tokensToEditorNodes(tokens);
|
||||
};
|
||||
/**
|
||||
* Recursive entrypoint for converting markdown to html.
|
||||
*/
|
||||
inlineToHTML = (markdown) => {
|
||||
const tokens = this.tokenize(markdown);
|
||||
return this.tokensToHTML(tokens);
|
||||
};
|
||||
/**
|
||||
* Recursive entrypoint for converting markdown to editor nodes.
|
||||
*/
|
||||
inlineToEditorNodes = (markdown) => {
|
||||
const tokens = this.tokenize(markdown);
|
||||
return this.tokensToEditorNodes(tokens);
|
||||
};
|
||||
/**
|
||||
* Process a markdown string by splitting it into lines, stripping empty lines, and capturing reference links.
|
||||
*/
|
||||
preprocessMarkdown(markdown) {
|
||||
const blankLine = /^\s*$/;
|
||||
const refDefinition = /^\[(?<label>[^\]]+)\]:\s+(?<url>\S+)(?:\s+"(?<title>[^"]*)")?$/;
|
||||
const referenceLinks = new Map;
|
||||
const output = [];
|
||||
const lines = markdown.replace(/\r\n/g, '\n').split('\n');
|
||||
for (const line of lines) {
|
||||
const match = line.match(refDefinition);
|
||||
if (match?.groups) {
|
||||
this.referenceLinks.set(match.groups.label.toLowerCase(), {
|
||||
url: match.groups.url,
|
||||
title: match.groups.title,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (!blankLine.test(line)) {
|
||||
output.push(line);
|
||||
}
|
||||
}
|
||||
return {
|
||||
lines: output,
|
||||
referenceLinks: referenceLinks
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Convert an HTML node to a markdown string. Used by toMarkdown().
|
||||
*/
|
||||
nodeToMarkdown = (node) => {
|
||||
// DocumentFragment
|
||||
if (node.nodeType == 11) {
|
||||
return Array.from(node.childNodes).map(child => {
|
||||
return this.nodeToMarkdown(child);
|
||||
}).join('');
|
||||
}
|
||||
// text
|
||||
if (node.nodeType == 3) {
|
||||
return node.textContent || '';
|
||||
// not a block-level element
|
||||
}
|
||||
else if (node.nodeType !== 1) {
|
||||
return '';
|
||||
}
|
||||
const tag = this.tags.getByElementName(node.nodeName);
|
||||
if (!tag) {
|
||||
return Array.from(node.childNodes)
|
||||
.map(child => this.nodeToMarkdown(child))
|
||||
.join('');
|
||||
}
|
||||
const ret = tag.toMarkdown(node, this.nodeToMarkdown);
|
||||
return ret;
|
||||
};
|
||||
/**
|
||||
* Replace [text][ref] and [text][] with [text](url) using the
|
||||
* reference definitions collected during block parsing.
|
||||
*/
|
||||
resolveReferenceLinks(text) {
|
||||
if (this.referenceLinks.size === 0) {
|
||||
return text;
|
||||
}
|
||||
const refLink = /\[(?<text>[^\[\]]+)\]\[(?<label>[^\]]*)\]/g;
|
||||
return text.replace(refLink, (...args) => {
|
||||
const groups = args[args.length - 1];
|
||||
const label = (groups.label || groups.text).toLowerCase();
|
||||
const ref = this.referenceLinks.get(label);
|
||||
if (!ref) {
|
||||
return args[0];
|
||||
}
|
||||
const titlePart = ref.title ? ` "${ref.title}"` : '';
|
||||
return `[${groups.text}](${ref.url}${titlePart})`;
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Normalize flanking underscore runs to asterisks so the tokenizer
|
||||
* only needs to handle * delimiters for emphasis.
|
||||
*/
|
||||
normalizeUnderscores(text) {
|
||||
// Protect backslash-escaped underscores from normalization
|
||||
const escapePlaceholder = '\x00U\x00';
|
||||
const safeText = text.replace(/\\_/g, escapePlaceholder);
|
||||
const punctuation = `[\\s.,;:!?'"()\\[\\]{}<>\\-/\\\\~#@&^|]`;
|
||||
const openRun = new RegExp(`(?<=^|${punctuation})` + // preceded by start, space, or punctuation
|
||||
`(_+)` + // one or more underscores
|
||||
`(?=\\S)`, // followed by non-whitespace
|
||||
'g');
|
||||
const closeRun = new RegExp(`(?<=\\S)` + // preceded by non-whitespace
|
||||
`(_+)` + // one or more underscores
|
||||
`(?=$|${punctuation})`, // followed by end, space, or punctuation
|
||||
'g');
|
||||
const toAsterisks = (_, run) => '*'.repeat(run.length);
|
||||
const normalized = safeText
|
||||
.replace(openRun, toAsterisks)
|
||||
.replace(closeRun, toAsterisks);
|
||||
return normalized.replace(/\x00U\x00/g, '\\_');
|
||||
}
|
||||
/**
|
||||
* Convert a token stream to editor nodes.
|
||||
*/
|
||||
tokensToEditorNodes(tokens) {
|
||||
// First pass: match open/close pairs using a stack
|
||||
const paired = this.pairDelimiters(tokens);
|
||||
// Second pass: build HTML from paired tokens
|
||||
const fragment = document.createDocumentFragment();
|
||||
var grouping = [];
|
||||
const groupAsNode = () => {
|
||||
const div = document.createElement('div');
|
||||
for (const node of grouping) {
|
||||
div.appendChild(node);
|
||||
}
|
||||
return div;
|
||||
};
|
||||
for (const token of paired) {
|
||||
if (token.role == 'html') {
|
||||
const div = document.createElement('div');
|
||||
div.innerHTML = token.value;
|
||||
for (const node of Array.from(div.childNodes)) {
|
||||
grouping.push(node);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (token.tag) {
|
||||
const thisNode = token.tag.toEditorNode(token, this.inlineToEditorNodes);
|
||||
if (!thisNode) {
|
||||
continue;
|
||||
}
|
||||
if (token.role == 'open') {
|
||||
if (grouping.length) {
|
||||
fragment.append(groupAsNode());
|
||||
}
|
||||
grouping = [thisNode];
|
||||
continue;
|
||||
}
|
||||
if (token.role == 'close') {
|
||||
grouping.push(thisNode);
|
||||
fragment.append(groupAsNode());
|
||||
grouping = [];
|
||||
continue;
|
||||
}
|
||||
if (!grouping.length) {
|
||||
fragment.appendChild(thisNode.cloneNode(true));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const text = document.createTextNode(token.value);
|
||||
grouping.push(text);
|
||||
continue;
|
||||
}
|
||||
if (grouping.length) {
|
||||
fragment.appendChild(groupAsNode());
|
||||
}
|
||||
return fragment;
|
||||
}
|
||||
/**
|
||||
* Convert a token stream to HTML. Matches open/close delimiter
|
||||
* pairs and wraps their content in the appropriate HTML tags.
|
||||
* Unmatched delimiters are emitted as literal text.
|
||||
*/
|
||||
tokensToHTML(tokens) {
|
||||
// First pass: match open/close pairs using a stack
|
||||
const paired = this.pairDelimiters(tokens);
|
||||
// Second pass: build HTML from paired tokens
|
||||
let html = '';
|
||||
for (const token of paired) {
|
||||
console.log(token);
|
||||
if (token.role == 'text') {
|
||||
html += escapeHtml(token.value);
|
||||
continue;
|
||||
}
|
||||
if (token.role == 'html') {
|
||||
html += token.value;
|
||||
continue;
|
||||
}
|
||||
if (token.role == 'link') {
|
||||
html += this.tags.getByElementName('A').toHTML(token, this.inlineToHTML) || token.value;
|
||||
continue;
|
||||
}
|
||||
if (token.tag) {
|
||||
html += token.tag.toHTML(token, this.inlineToHTML);
|
||||
continue;
|
||||
}
|
||||
html += escapeHtml(token.value);
|
||||
}
|
||||
return html;
|
||||
}
|
||||
/**
|
||||
* Matchn/close delimiter pairs in a token stream. Unmatched
|
||||
* openers/closers are converted to text tokens so they render
|
||||
* as literal characters.
|
||||
*/
|
||||
pairDelimiters(tokens) {
|
||||
const openStack = [];
|
||||
const result = [...tokens];
|
||||
// Track which delimiter types are currently open to prevent
|
||||
// forbidden nesting (e.g. <del> inside <del>, <em> inside <em>)
|
||||
const openDelimiters = new Set();
|
||||
for (let index = 0; index < result.length; index++) {
|
||||
const token = result[index];
|
||||
if (token.role === 'block') {
|
||||
token.role = openDelimiters.has(token.sequence) ? 'close' : 'open';
|
||||
}
|
||||
if (token.role === 'open') {
|
||||
// Don't open a delimiter that's already open (prevents nesting)
|
||||
if (openDelimiters.has(token.sequence)) {
|
||||
result[index] = {
|
||||
role: 'text',
|
||||
value: token.value,
|
||||
};
|
||||
continue;
|
||||
}
|
||||
openStack.push(index);
|
||||
openDelimiters.add(token.sequence);
|
||||
}
|
||||
else if (token.role === 'close') {
|
||||
let matched = false;
|
||||
for (let stackIndex = openStack.length - 1; stackIndex >= 0; stackIndex--) {
|
||||
const openerIndex = openStack[stackIndex];
|
||||
if (result[openerIndex].sequence === token.sequence) {
|
||||
openStack.splice(stackIndex, 1);
|
||||
openDelimiters.delete(token.sequence);
|
||||
matched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!matched) {
|
||||
result[index] = {
|
||||
role: 'text',
|
||||
value: token.value,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
exports.HopDown = HopDown;
|
||||
Vendored
+18
@@ -0,0 +1,18 @@
|
||||
"use strict";
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
var desc = Object.getOwnPropertyDescriptor(m, k);
|
||||
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
||||
desc = { enumerable: true, get: function() { return m[k]; } };
|
||||
}
|
||||
Object.defineProperty(o, k2, desc);
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
||||
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
__exportStar(require("./hopdown"), exports);
|
||||
__exportStar(require("./defaults"), exports);
|
||||
Vendored
+717
@@ -0,0 +1,717 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.Paragraph = exports.Table = exports.OrderedList = exports.UnorderedList = exports.Blockquote = exports.Heading = exports.HorizontalRule = exports.FencedCode = exports.HardBreak = exports.Anchor = exports.Code = exports.Strikethrough = exports.BoldItalic = exports.Italic = exports.Bold = exports.TagCollection = exports.Tag = void 0;
|
||||
exports.childNodesToMarkdown = childNodesToMarkdown;
|
||||
const hopdown_1 = require("./hopdown");
|
||||
/**
|
||||
* Patterns shared across multiple tag classes. Defined once here
|
||||
* so the regexes aren't duplicated and can be referenced by name.
|
||||
*/
|
||||
const blankLine = /^\s*$/;
|
||||
const unorderedMarker = /^[*\-]\s/;
|
||||
const orderedMarker = /^\d+\.\s/;
|
||||
const escapeRegex = /[.*+?^${}()|[\]\\]/g;
|
||||
const tableSeparator = new RegExp('^'
|
||||
+ '\\|?' // optional leading pipe
|
||||
+ '\\s*:?-+:?\\s*' // first column separator (optional alignment colons)
|
||||
+ '('
|
||||
+ '\\|' // pipe between columns
|
||||
+ '\\s*:?-+:?\\s*' // subsequent column separator
|
||||
+ ')*' // zero or more additional columns
|
||||
+ '\\|?' // optional trailing pipe
|
||||
+ '\\s*$' // end of line
|
||||
);
|
||||
function childNodesToMarkdown(element, callback) {
|
||||
return Array.from(element.childNodes)
|
||||
.map(child => callback(child))
|
||||
.join('');
|
||||
}
|
||||
class Tag {
|
||||
element = [''];
|
||||
opener = '';
|
||||
closer = '';
|
||||
delimiter = null;
|
||||
precedence = 50;
|
||||
isBlock = false;
|
||||
toHTML(token, callback) {
|
||||
if (token.role == 'open') {
|
||||
return this.element.map(el => { return `<${el.toUpperCase()}>`; }).join('');
|
||||
}
|
||||
if (token.role == 'close') {
|
||||
const rev = this.element;
|
||||
rev.reverse();
|
||||
return rev.map(el => { return `</${el.toUpperCase()}>`; }).join('');
|
||||
}
|
||||
return token.value;
|
||||
}
|
||||
;
|
||||
toMarkdown(element, callback) {
|
||||
const children = childNodesToMarkdown(element, callback);
|
||||
return `${this.delimiter}${children || element.textContent || ''}${this.delimiter}`;
|
||||
}
|
||||
;
|
||||
toEditorNode(token, callback) {
|
||||
if (token.role == 'open' || token.role == 'close') {
|
||||
const span = document.createElement('span');
|
||||
span.classList.add('delim');
|
||||
for (const el of this.element) {
|
||||
span.classList.add(el.toLowerCase());
|
||||
}
|
||||
;
|
||||
if (this.isBlock) {
|
||||
span.textContent = token.role == 'open' ? this.opener : this.closer;
|
||||
}
|
||||
else {
|
||||
span.textContent = this.delimiter;
|
||||
}
|
||||
return span;
|
||||
}
|
||||
return document.createTextNode(token.value);
|
||||
}
|
||||
;
|
||||
}
|
||||
exports.Tag = Tag;
|
||||
class TagCollection {
|
||||
tags = {};
|
||||
ordered;
|
||||
openers;
|
||||
constructor(tags) {
|
||||
for (const tag of tags) {
|
||||
if (tag.constructor.name in this.tags) {
|
||||
throw new Error(`Duplicate tag instances found for class ${tag.constructor.name}! Do you need to subclass it?`);
|
||||
}
|
||||
this.tags[tag.constructor.name] = tag;
|
||||
}
|
||||
this.ordered = tags.sort((a, b) => {
|
||||
const p1 = a.precedence - (a.isBlock ? 100 : 0);
|
||||
const p2 = b.precedence - (b.isBlock ? 100 : 0);
|
||||
return p1 < p2 ? -1 : 1;
|
||||
});
|
||||
this.validate();
|
||||
const escapeRegex = /[.*+?^${}()|[\]\\]/g;
|
||||
this.openers = this.ordered.filter(t => { return t.delimiter !== null; }).map(tag => {
|
||||
const escaped = tag.delimiter.replace(escapeRegex, '\\$&');
|
||||
const escapedChar = tag.delimiter[0].replace(escapeRegex, '\\$&');
|
||||
return new RegExp(`(?<!${escapedChar})` +
|
||||
`${escaped}` +
|
||||
`(?!${escapedChar})` +
|
||||
`([^\\x01\\x02]+)$`);
|
||||
});
|
||||
}
|
||||
validate() {
|
||||
const inline = this.ordered.filter(t => { return !t.delimiter; });
|
||||
for (let outer = 0; outer < inline.length; outer++) {
|
||||
for (let inner = outer + 1; inner < inline.length; inner++) {
|
||||
const first = this.ordered[outer];
|
||||
const second = this.ordered[inner];
|
||||
if (!(first.delimiter && second.delimiter)) {
|
||||
continue;
|
||||
}
|
||||
const firstIsPrefix = second.delimiter.startsWith(first.delimiter);
|
||||
const secondIsPrefix = first.delimiter.startsWith(second.delimiter);
|
||||
if (!firstIsPrefix && !secondIsPrefix) {
|
||||
continue;
|
||||
}
|
||||
const longer = first.delimiter.length > second.delimiter.length ? first : second;
|
||||
const shorter = first.delimiter.length > second.delimiter.length ? second : first;
|
||||
if (longer.precedence >= shorter.precedence) {
|
||||
throw new Error(`Tag "${longer.constructor.name}" (delimiter "${longer.delimiter}") must have ` +
|
||||
`lower precedence than "${shorter.constructor.name}" (delimiter "${shorter.delimiter}") ` +
|
||||
`because its delimiter is a prefix match. ` +
|
||||
`Got ${longer.constructor.name}=${longer.precedence}, ${shorter.constructor.name}=${shorter.precedence}.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
getByElementName(element) {
|
||||
const el = element.toUpperCase();
|
||||
return this.ordered.filter(t => {
|
||||
return [...t.element, ...t.aliases ?? []].includes(el);
|
||||
})[0] || undefined;
|
||||
}
|
||||
}
|
||||
exports.TagCollection = TagCollection;
|
||||
class Bold extends Tag {
|
||||
element = ['STRONG'];
|
||||
aliases = ['B'];
|
||||
delimiter = '**';
|
||||
shortcut = 'Ctrl+B';
|
||||
precedence = 40;
|
||||
}
|
||||
exports.Bold = Bold;
|
||||
class Italic extends Tag {
|
||||
element = ['EM'];
|
||||
aliases = ['I'];
|
||||
delimiter = '*';
|
||||
shortcut = 'Ctrl+I';
|
||||
precedence = 30;
|
||||
}
|
||||
exports.Italic = Italic;
|
||||
class BoldItalic extends Tag {
|
||||
element = ['STRONG', 'EM'];
|
||||
delimiter = '***';
|
||||
precedence = 50;
|
||||
}
|
||||
exports.BoldItalic = BoldItalic;
|
||||
class Strikethrough extends Tag {
|
||||
element = ['DEL'];
|
||||
aliases = ['S', 'STRIKE'];
|
||||
delimiter = '~~';
|
||||
}
|
||||
exports.Strikethrough = Strikethrough;
|
||||
class Code extends Tag {
|
||||
element = ['CODE'];
|
||||
delimiter = '`';
|
||||
}
|
||||
exports.Code = Code;
|
||||
class Anchor extends Tag {
|
||||
element = ['A'];
|
||||
toHTML(token, callback) {
|
||||
const titleAttr = token.title
|
||||
? ` TITLE="${(0, hopdown_1.escapeHtml)(token.title)}"`
|
||||
: '';
|
||||
return `<A HREF="${(0, hopdown_1.escapeHtml)(token.href || '')}"${titleAttr}>${(0, hopdown_1.escapeHtml)(token.value)}</A>`;
|
||||
}
|
||||
toMarkdown(element, callback) {
|
||||
const href = element.getAttribute('href') || '';
|
||||
const title = element.getAttribute('title');
|
||||
const titlePart = title ? ` "${title}"` : '';
|
||||
return '[' + element.innerText + '](' + href + titlePart + ')';
|
||||
}
|
||||
toEditorNode(token, callback) {
|
||||
return document.createElement('A');
|
||||
}
|
||||
}
|
||||
exports.Anchor = Anchor;
|
||||
class BlockTag extends Tag {
|
||||
isBlock = true;
|
||||
delimiter = null;
|
||||
toMarkdown(element, callback) {
|
||||
const children = childNodesToMarkdown(element, callback);
|
||||
const ret = `${this.opener || ''}${children || element.textContent || ''}${this.closer || ''}`;
|
||||
return ret;
|
||||
}
|
||||
;
|
||||
match(context, tags) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
class HardBreak extends BlockTag {
|
||||
match(context) {
|
||||
return null;
|
||||
}
|
||||
toHTML() { return '<BR>'; }
|
||||
toMarkdown() { return ' \n'; }
|
||||
}
|
||||
exports.HardBreak = HardBreak;
|
||||
/**
|
||||
* Fenced code blocks: lines between ``` delimiters become <pre><code>.
|
||||
*
|
||||
* converter.toHTML('```js\nlet x = 1;\n```')
|
||||
* // <pre><code class="language-js">let x = 1;</code></pre>
|
||||
*/
|
||||
class FencedCode extends BlockTag {
|
||||
opener = '```';
|
||||
closer = '```';
|
||||
element = ['PRE'];
|
||||
button = {
|
||||
show: true,
|
||||
label: 'Code Block',
|
||||
shortcut: 'Ctrl+Shift+E',
|
||||
};
|
||||
template = '```\ncode\n```';
|
||||
replaceSelection = true;
|
||||
match(context) {
|
||||
// Accepts both ``` and ~~~ as fence delimiters
|
||||
const fencePattern = /^(?<fence>`{3,}|~{3,})(?<lang>.*)/;
|
||||
const matched = context.lines[context.index].match(fencePattern);
|
||||
if (!matched?.groups) {
|
||||
return null;
|
||||
}
|
||||
const fence = matched.groups.fence;
|
||||
const lang = matched.groups.lang.trim();
|
||||
const code = [];
|
||||
let lineIndex = context.index + 1;
|
||||
while (lineIndex < context.lines.length && !context.lines[lineIndex].startsWith(fence)) {
|
||||
code.push(context.lines[lineIndex++]);
|
||||
}
|
||||
return {
|
||||
opener: fence + lang,
|
||||
closer: '',
|
||||
content: code.join('\n'),
|
||||
raw: '',
|
||||
consumed: lineIndex + 1 - context.index,
|
||||
meta: { lang },
|
||||
};
|
||||
}
|
||||
toHTML(token, callback) {
|
||||
const langAttr = token.meta?.lang
|
||||
? ` class="language-${(0, hopdown_1.escapeHtml)(token.meta.lang)}"`
|
||||
: '';
|
||||
return `<pre${langAttr}>${(0, hopdown_1.escapeHtml)(token.content)}</pre>`;
|
||||
}
|
||||
toMarkdown(element, callback) {
|
||||
const lang = element.className.match(/language-(\S+)/)?.[1] || '';
|
||||
const content = element?.textContent || '';
|
||||
return '\n\n' + this.opener + `${lang}\n${content}\n` + this.closer + '\n\n';
|
||||
}
|
||||
}
|
||||
exports.FencedCode = FencedCode;
|
||||
/**
|
||||
* Three or more *, -, or _ on a line become <hr>.
|
||||
*
|
||||
* converter.toHTML('---') // '<hr>'
|
||||
*/
|
||||
class HorizontalRule extends BlockTag {
|
||||
element = ['HR'];
|
||||
button = {
|
||||
show: true,
|
||||
label: 'Divider',
|
||||
shortcut: 'Ctrl+Shift+-',
|
||||
};
|
||||
template = '---';
|
||||
replaceSelection = false;
|
||||
pattern = /^(\*{3,}|-{3,}|_{3,})\s*$/;
|
||||
match(context) {
|
||||
const matched = context.lines[context.index].match(this.pattern);
|
||||
if (!matched) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
opener: '',
|
||||
closer: '',
|
||||
content: '',
|
||||
raw: '',
|
||||
consumed: 1,
|
||||
};
|
||||
}
|
||||
toHTML(token, callback) {
|
||||
return `<${this.element[0].toUpperCase()}>`;
|
||||
}
|
||||
toMarkdown(element, callback) {
|
||||
return '\n***\n';
|
||||
}
|
||||
}
|
||||
exports.HorizontalRule = HorizontalRule;
|
||||
class Heading extends BlockTag {
|
||||
element = ['H1', 'H2', 'H3', 'H4', 'H5', 'H6'];
|
||||
match(context) {
|
||||
// ATX heading: # through ######, optional closing #s
|
||||
const atxPattern = /^(?<hashes>#{1,6})\s+(?<content>.*?)(?:\s+#+\s*)?$/;
|
||||
const matched = context.lines[context.index].match(atxPattern);
|
||||
if (matched?.groups) {
|
||||
return {
|
||||
opener: matched.groups.hashes + ' ',
|
||||
content: matched.groups.content.trim(),
|
||||
closer: '',
|
||||
raw: '',
|
||||
consumed: 1,
|
||||
meta: { level: String(matched.groups.hashes.length) },
|
||||
};
|
||||
}
|
||||
if (context.index + 1 < context.lines.length) {
|
||||
const nextLine = context.lines[context.index + 1];
|
||||
const setextUnderline = /^(?<marker>=+|-+)\s*$/;
|
||||
const underlineMatch = nextLine.match(setextUnderline);
|
||||
if (underlineMatch?.groups) {
|
||||
const content = context.lines[context.index].trim();
|
||||
if (content.length > 0) {
|
||||
const level = underlineMatch.groups.marker[0] === '=' ? '1' : '2';
|
||||
return {
|
||||
opener: '',
|
||||
closer: '',
|
||||
content: content,
|
||||
raw: '',
|
||||
consumed: 2,
|
||||
meta: { level },
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
toHTML(token, callback) {
|
||||
const level = token.meta.level;
|
||||
const id = this.anchorId(token.content);
|
||||
return `<h${level} id='${id}'>${callback(token.content)}</h${level}>`;
|
||||
}
|
||||
toMarkdown(element, callback) {
|
||||
const inner = childNodesToMarkdown(element, callback);
|
||||
const level = parseInt(element.nodeName[1]);
|
||||
return '\n\n' + '#'.repeat(level) + ' ' + inner + '\n\n';
|
||||
}
|
||||
/**
|
||||
* Generate a PascalCase anchor ID from heading text so that
|
||||
* in-page links like #MyHeading work without manual IDs.
|
||||
*/
|
||||
anchorId(text) {
|
||||
return text.replace(escapeRegex, ' ').trim().split(/\s+/).map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase()).join('_');
|
||||
}
|
||||
}
|
||||
exports.Heading = Heading;
|
||||
/**
|
||||
* Lines prefixed with > become blockquotes. Consecutive > lines
|
||||
* are merged into a single blockquote.
|
||||
*
|
||||
* converter.toHTML('> hello\n> world')
|
||||
* // <blockquote><p>hello\nworld</p></blockquote>
|
||||
*/
|
||||
class Blockquote extends BlockTag {
|
||||
element = ['BLOCKQUOTE'];
|
||||
hidden = false;
|
||||
template = '> Quote\n> continues here';
|
||||
match(context) {
|
||||
const quotePrefix = /^>\s?/;
|
||||
if (!quotePrefix.test(context.lines[context.index])) {
|
||||
return null;
|
||||
}
|
||||
const lines = [];
|
||||
let lineIndex = context.index;
|
||||
while (lineIndex < context.lines.length && quotePrefix.test(context.lines[lineIndex])) {
|
||||
lines.push(context.lines[lineIndex++].replace(quotePrefix, ''));
|
||||
}
|
||||
return {
|
||||
opener: '> ',
|
||||
closer: '',
|
||||
content: lines.join('\n'),
|
||||
raw: '',
|
||||
consumed: lineIndex - context.index,
|
||||
};
|
||||
}
|
||||
toHTML(token, callback) {
|
||||
const inner = callback(token.content);
|
||||
const breaks = inner.replace(/([^\n])\n(?!\n)/g, '$1<br>')
|
||||
.replace(/\n\n(.+?)(\n\n|$)/g, '<p>$1</p>');
|
||||
return '<blockquote>' + breaks + '</blockquote>';
|
||||
}
|
||||
toMarkdown(element, callback) {
|
||||
// Each <p> inside the blockquote is a paragraph group.
|
||||
// Within a <p>, <br> elements create line breaks.
|
||||
// Separate paragraphs get a blank > line between them.
|
||||
const paragraphs = [];
|
||||
for (const child of Array.from(element.childNodes)) {
|
||||
const el = child;
|
||||
if (child.nodeType === 1 && el.tagName === 'P') {
|
||||
const lines = this.paragraphToLines(el, callback);
|
||||
paragraphs.push(">");
|
||||
paragraphs.push(lines.map(line => '> ' + line).join('\n'));
|
||||
}
|
||||
else {
|
||||
const text = callback(child).trim();
|
||||
if (text) {
|
||||
paragraphs.push('> ' + text);
|
||||
}
|
||||
}
|
||||
}
|
||||
return '\n\n' + paragraphs.join('\n') + '\n\n';
|
||||
}
|
||||
/**
|
||||
* Split a paragraph's content into lines at <br> elements.
|
||||
*/
|
||||
paragraphToLines(paragraph, callback) {
|
||||
const lines = [];
|
||||
let currentLine = '';
|
||||
for (const child of Array.from(paragraph.childNodes)) {
|
||||
if (child.nodeType === 1 && child.tagName === 'BR') {
|
||||
lines.push(currentLine);
|
||||
currentLine = '';
|
||||
}
|
||||
else {
|
||||
currentLine += callback(child);
|
||||
}
|
||||
}
|
||||
if (currentLine.trim()) {
|
||||
lines.push(currentLine);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
}
|
||||
exports.Blockquote = Blockquote;
|
||||
/**
|
||||
* Ordered and unordered lists with arbitrary nesting. Indentation
|
||||
* determines depth; mixed list types (ul inside ol) are supported.
|
||||
*
|
||||
* converter.toHTML('- one\n- two\n 1. nested')
|
||||
*/
|
||||
class UnorderedList extends BlockTag {
|
||||
element = ['UL'];
|
||||
hidden = false;
|
||||
match(context) {
|
||||
const line = context.lines[context.index];
|
||||
const prefixPattern = /^((?:[*\-]|\d+\.)\s)/;
|
||||
const matched = line.match(prefixPattern);
|
||||
if (!matched) {
|
||||
return null;
|
||||
}
|
||||
// Count how many lines this list consumes so HopDown can
|
||||
// advance past them. The actual parsing happens in toHTML
|
||||
// where we have access to the inline converter.
|
||||
const consumed = this.countListLines(context.lines, context.index);
|
||||
return {
|
||||
opener: matched[1],
|
||||
closer: '',
|
||||
content: '',
|
||||
raw: '',
|
||||
consumed,
|
||||
meta: {
|
||||
startIndex: String(context.index),
|
||||
lines: JSON.stringify(context.lines.slice(context.index, context.index + consumed)),
|
||||
},
|
||||
};
|
||||
}
|
||||
toHTML(token, callback) {
|
||||
const lines = JSON.parse(token.meta.lines);
|
||||
const result = this.parseBlock(lines, 0, 0, callback);
|
||||
return result.html;
|
||||
}
|
||||
toMarkdown(element, callback) {
|
||||
return this.nodeToMarkdown(element, 0, callback);
|
||||
}
|
||||
countListLines(lines, start) {
|
||||
const indentedUnordered = /^(\s*)[*\-+]\s/;
|
||||
const indentedOrdered = /^(\s*)\d+\.\s/;
|
||||
let lineIndex = start;
|
||||
while (lineIndex < lines.length) {
|
||||
const line = lines[lineIndex];
|
||||
if (blankLine.test(line)) {
|
||||
break;
|
||||
}
|
||||
if (!indentedUnordered.test(line) && !indentedOrdered.test(line)) {
|
||||
break;
|
||||
}
|
||||
lineIndex++;
|
||||
}
|
||||
return lineIndex - start;
|
||||
}
|
||||
/**
|
||||
* Recursively parse markdown list lines into HTML, handling nested
|
||||
* sublists at arbitrary depth and mixed list types (ul/ol).
|
||||
*/
|
||||
parseBlock(lines, start, indent, callback) {
|
||||
const prefix = new RegExp('^'
|
||||
+ ' '.repeat(indent) // indentation for this nesting level
|
||||
+ '([\\*\\-+]|\\d+\\.)' // bullet or number marker
|
||||
+ '\\s' // space after marker
|
||||
);
|
||||
const isOl = orderedMarker.test(lines[start].trim());
|
||||
const tag = isOl ? 'ol' : 'ul';
|
||||
const items = [];
|
||||
let lineIndex = start;
|
||||
while (lineIndex < lines.length) {
|
||||
const line = lines[lineIndex];
|
||||
if (blankLine.test(line)) {
|
||||
break;
|
||||
}
|
||||
const leadingWhitespace = /^(\s*)/;
|
||||
const lineIndent = line.match(leadingWhitespace)[1].length;
|
||||
if (lineIndent < indent) {
|
||||
break;
|
||||
}
|
||||
if (lineIndent > indent) {
|
||||
const sub = this.parseBlock(lines, lineIndex, lineIndent, callback);
|
||||
items[items.length - 1].sub = sub.html;
|
||||
lineIndex = sub.end;
|
||||
continue;
|
||||
}
|
||||
if (!prefix.test(line)) {
|
||||
break;
|
||||
}
|
||||
items.push({
|
||||
text: line.replace(prefix, ''),
|
||||
sub: '',
|
||||
});
|
||||
lineIndex++;
|
||||
}
|
||||
const html = `<${tag}>`
|
||||
+ items.map(item => '<li>' + callback(item.text) + item.sub + '</li>').join('')
|
||||
+ `</${tag}>`;
|
||||
return {
|
||||
html,
|
||||
end: lineIndex,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Convert an HTML list back to markdown. Recurses into nested
|
||||
* sublists with 2-space indentation per depth level.
|
||||
*/
|
||||
nodeToMarkdown(node, depth, callback) {
|
||||
const isOl = node.nodeName === 'OL';
|
||||
const indent = ' '.repeat(depth);
|
||||
const lines = [];
|
||||
Array.from(node.children).forEach((listItem, index) => {
|
||||
const marker = isOl ? (index + 1) + '. ' : '* ';
|
||||
const { text, sublist } = this.partitionListItem(listItem, depth, callback);
|
||||
lines.push(indent + marker + text.trim());
|
||||
if (sublist) {
|
||||
lines.push(sublist);
|
||||
}
|
||||
});
|
||||
const result = lines.join('\n');
|
||||
return depth === 0
|
||||
? '\n\n' + result + '\n\n'
|
||||
: result;
|
||||
}
|
||||
partitionListItem(listItem, depth, callback) {
|
||||
let text = '';
|
||||
let sublist = '';
|
||||
for (const child of Array.from(listItem.childNodes)) {
|
||||
if (child.nodeType === 1 && (child.nodeName === 'UL' || child.nodeName === 'OL')) {
|
||||
sublist += this.nodeToMarkdown(child, depth + 1, callback);
|
||||
}
|
||||
else {
|
||||
text += callback(child);
|
||||
}
|
||||
}
|
||||
return {
|
||||
text,
|
||||
sublist,
|
||||
};
|
||||
}
|
||||
}
|
||||
exports.UnorderedList = UnorderedList;
|
||||
class OrderedList extends UnorderedList {
|
||||
element = ['OL'];
|
||||
}
|
||||
exports.OrderedList = OrderedList;
|
||||
/**
|
||||
* Pipe-delimited tables with optional column alignment.
|
||||
*
|
||||
* converter.toHTML('| A | B |\n|---|---|\n| 1 | 2 |')
|
||||
*/
|
||||
class Table extends BlockTag {
|
||||
element = ['TABLE'];
|
||||
hidden = false;
|
||||
template = '| Header 1 | Header 2 |\n|----------|----------|\n| Cell 1 | Cell 2 |';
|
||||
replaceSelection = false;
|
||||
match(context) {
|
||||
const { lines, index } = context;
|
||||
if (lines[index].indexOf('|') === -1
|
||||
|| index + 1 >= lines.length) {
|
||||
return null;
|
||||
}
|
||||
// Second line must be the separator row (e.g. |---|---|)
|
||||
if (!tableSeparator.test(lines[index + 1])) {
|
||||
return null;
|
||||
}
|
||||
const headers = this.parseRow(lines[index]);
|
||||
const aligns = this.parseAligns(lines[index + 1]);
|
||||
const rows = [];
|
||||
let lineIndex = index + 2;
|
||||
while (lineIndex < lines.length && lines[lineIndex].indexOf('|') !== -1 && !blankLine.test(lines[lineIndex])) {
|
||||
rows.push(this.parseRow(lines[lineIndex++]));
|
||||
}
|
||||
return {
|
||||
content: '',
|
||||
opener: '',
|
||||
closer: '',
|
||||
raw: '',
|
||||
consumed: lineIndex - index,
|
||||
meta: {
|
||||
headers: JSON.stringify(headers),
|
||||
aligns: JSON.stringify(aligns),
|
||||
rows: JSON.stringify(rows),
|
||||
},
|
||||
};
|
||||
}
|
||||
toHTML(token, callback) {
|
||||
const headers = JSON.parse(token.meta.headers);
|
||||
const aligns = JSON.parse(token.meta.aligns);
|
||||
const rows = JSON.parse(token.meta.rows);
|
||||
const cell = (tag, text, index) => {
|
||||
const align = aligns[index]
|
||||
? ` class="align-${aligns[index]}"`
|
||||
: '';
|
||||
return `<${tag}${align}>${callback(text)}</${tag}>`;
|
||||
};
|
||||
const head = '<thead><tr>'
|
||||
+ headers.map((text, column) => cell('th', text, column)).join('')
|
||||
+ '</tr></thead>';
|
||||
const body = rows.map(row => '<tr>' + row.map((text, column) => cell('td', text, column)).join('') + '</tr>').join('');
|
||||
return `<table>${head}<tbody>${body}</tbody></table>`;
|
||||
}
|
||||
toMarkdown(element, callback) {
|
||||
const rows = Array.from(element.querySelectorAll('tr'));
|
||||
if (!rows.length) {
|
||||
return '';
|
||||
}
|
||||
const headers = Array.from(rows[0].querySelectorAll('th,td'))
|
||||
.map(cell => callback(cell).trim());
|
||||
const separator = Array.from(element.querySelectorAll('th')).map(h => {
|
||||
return h.className == 'align-center' ? ':---:' :
|
||||
h.className == 'align-right' ? '---:' :
|
||||
'---';
|
||||
});
|
||||
const output = [
|
||||
'| ' + headers.join(' | ') + ' |',
|
||||
'| ' + separator.join(' | ') + ' |',
|
||||
];
|
||||
for (const row of rows.slice(1)) {
|
||||
const cells = Array.from(row.querySelectorAll('td,th'))
|
||||
.map(cell => callback(cell).trim());
|
||||
output.push('| ' + cells.join(' | ') + ' |');
|
||||
}
|
||||
return '\n\n' + output.join('\n') + '\n\n';
|
||||
}
|
||||
parseRow(line) {
|
||||
return line
|
||||
.replace(/^\|/, '')
|
||||
.replace(/\|$/, '')
|
||||
.split('|')
|
||||
.map(cell => cell.trim());
|
||||
}
|
||||
parseAligns(line) {
|
||||
const centerAlign = /^:-+:$/;
|
||||
const rightAlign = /^-+:$/;
|
||||
return this.parseRow(line).map(cell => {
|
||||
if (centerAlign.test(cell)) {
|
||||
return 'center';
|
||||
}
|
||||
if (rightAlign.test(cell)) {
|
||||
return 'right';
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
}
|
||||
exports.Table = Table;
|
||||
class Paragraph extends BlockTag {
|
||||
element = ['P'];
|
||||
opener = '';
|
||||
closer = '';
|
||||
precedence = -1;
|
||||
match(context, tags) {
|
||||
const collected = [];
|
||||
let lineIndex = context.index;
|
||||
while (lineIndex < context.lines.length) {
|
||||
const line = context.lines[lineIndex];
|
||||
const nextLine = lineIndex < context.lines.length ? context.lines[lineIndex + 1] : '';
|
||||
for (const pattern of [...tags.openers, blankLine]) {
|
||||
if (pattern.test(line)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Table: current line has pipes and next line is a separator row
|
||||
if (line.indexOf('|') !== -1) {
|
||||
if (nextLine && tableSeparator.test(nextLine)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
collected.push(context.lines[lineIndex++]);
|
||||
}
|
||||
if (!collected.length) {
|
||||
return null;
|
||||
}
|
||||
const ret = {
|
||||
opener: '',
|
||||
closer: '',
|
||||
content: collected.join('\n'),
|
||||
raw: '',
|
||||
consumed: lineIndex - context.index,
|
||||
};
|
||||
return ret;
|
||||
}
|
||||
toHTML(token, callback) {
|
||||
return '<p>' + callback(token.content) + '</p>';
|
||||
}
|
||||
}
|
||||
exports.Paragraph = Paragraph;
|
||||
Vendored
+324
@@ -0,0 +1,324 @@
|
||||
"use strict";
|
||||
/*
|
||||
* tokenizer.ts — markdown tokenizer.
|
||||
*
|
||||
* Scans markdown text left-to-right producing a typed token stream.
|
||||
* Tokens carry their semantic role (delimiter, text, link, etc.)
|
||||
* so downstream consumers can make correct escaping and pairing
|
||||
* decisions without regex heuristics.
|
||||
*
|
||||
* const tokenizer = new Tokenizer(delimiterDefs);
|
||||
* const tokens = tokenizer.tokenize('hello **bold** end');
|
||||
* // [text "hello "] [open "**"] [text "bold"] [close "**"] [text " end"]
|
||||
*/
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.Tokenizer = void 0;
|
||||
/**
|
||||
* Characters that count as punctuation for flanking delimiter rules.
|
||||
* A delimiter is left-flanking if preceded by whitespace/punctuation
|
||||
* and followed by non-whitespace. Right-flanking is the reverse.
|
||||
*/
|
||||
const PUNCTUATION = new Set(' \t\n\u00A0.,;:!?\'"()[]{}/<>\\-~#@&^|*`_'.split(''));
|
||||
/**
|
||||
* Characters that can be backslash-escaped in markdown.
|
||||
*/
|
||||
const ESCAPABLE = new Set('\\`*_{}[]()#+-.!~|><'.split(''));
|
||||
/**
|
||||
* Named HTML entities recognized by the tokenizer.
|
||||
*/
|
||||
const NAMED_ENTITIES = {
|
||||
'amp': '&',
|
||||
'lt': '<',
|
||||
'gt': '>',
|
||||
'quot': '"',
|
||||
'apos': "'",
|
||||
'nbsp': '\u00A0',
|
||||
};
|
||||
/**
|
||||
* Scans markdown text into a stream of typed tokens. Handles
|
||||
* backslash escapes, entities, flanking rules, links,
|
||||
* autolinks, HTML tags, and hard line breaks.
|
||||
*
|
||||
* const tokenizer = new Tokenizer([
|
||||
* { delimiter: '**', htmlTag: 'strong', recursive: true, precedence: 40 },
|
||||
* { delimiter: '*', htmlTag: 'em', recursive: true, precedence: 50 },
|
||||
* ]);
|
||||
* const tokens = tokenizer.tokenize('**bold**');
|
||||
*/
|
||||
class Tokenizer {
|
||||
tags;
|
||||
constructor(tags) {
|
||||
this.tags = tags.ordered;
|
||||
}
|
||||
/**
|
||||
* Tokenize a markdown string into an inline token stream.
|
||||
*
|
||||
* tokenizer.tokenize('hello **world**')
|
||||
* // [text "hello "] [open "**"] [text "world"] [close "**"]
|
||||
*/
|
||||
tokenize(source) {
|
||||
const tokens = [];
|
||||
let position = 0;
|
||||
let textBuffer = '';
|
||||
const flushText = () => {
|
||||
if (textBuffer.length > 0) {
|
||||
tokens.push({
|
||||
role: 'text',
|
||||
value: textBuffer,
|
||||
});
|
||||
textBuffer = '';
|
||||
}
|
||||
};
|
||||
while (position < source.length) {
|
||||
const remaining = source.slice(position);
|
||||
// Backslash escape: \X → literal X
|
||||
if (source[position] === '\\' && position + 1 < source.length) {
|
||||
const nextChar = source[position + 1];
|
||||
if (ESCAPABLE.has(nextChar)) {
|
||||
textBuffer += '\\' + nextChar;
|
||||
position += 2;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// Hard line break: two+ trailing spaces before newline
|
||||
if (source[position] === ' ') {
|
||||
const spaceMatch = remaining.match(/^(?<spaces> {2,})\n/);
|
||||
if (spaceMatch?.groups) {
|
||||
flushText();
|
||||
tokens.push({ role: 'break', value: '<br>' });
|
||||
position += spaceMatch[0].length;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// HTML entity resolution: &name; or &#digits; or &#xhex;
|
||||
if (source[position] === '&') {
|
||||
const resolved = this.resolveEntity(remaining);
|
||||
if (resolved) {
|
||||
textBuffer += resolved.character;
|
||||
position += resolved.length;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// Link: [text](url) or [text](url "title")
|
||||
if (source[position] === '[') {
|
||||
const link = this.matchLink(source, position);
|
||||
if (link) {
|
||||
flushText();
|
||||
tokens.push({
|
||||
role: 'link',
|
||||
value: link.text,
|
||||
href: link.href,
|
||||
title: link.title,
|
||||
});
|
||||
position += link.length;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// Autolink: <url> and/or HTML
|
||||
if (source[position] === '<') {
|
||||
const autolink = this.matchAutolink(remaining);
|
||||
if (autolink) {
|
||||
flushText();
|
||||
tokens.push({
|
||||
role: 'link',
|
||||
value: autolink.url,
|
||||
href: autolink.url,
|
||||
tag: this.tags.filter(t => { return t.constructor.name == 'Anchor'; })[0]
|
||||
});
|
||||
position += autolink.length;
|
||||
continue;
|
||||
}
|
||||
// HTML tag passthrough
|
||||
const htmlTagMatch = this.matchHtmlTag(remaining);
|
||||
if (htmlTagMatch) {
|
||||
flushText();
|
||||
tokens.push({
|
||||
role: 'html',
|
||||
value: htmlTagMatch.tag,
|
||||
});
|
||||
position += htmlTagMatch.length;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// autolink bare URLs
|
||||
const bareUrl = this.matchBareUrl(remaining);
|
||||
if (bareUrl) {
|
||||
flushText();
|
||||
tokens.push({
|
||||
role: 'link',
|
||||
value: bareUrl.url,
|
||||
href: bareUrl.url,
|
||||
tag: this.tags.filter(t => { return t.constructor.name == 'Anchor'; })[0]
|
||||
});
|
||||
position += bareUrl.length;
|
||||
continue;
|
||||
}
|
||||
// Delimiter: check each registered delimiter
|
||||
const match = this.matchTag(source, position);
|
||||
if (match && match.sequence?.length) {
|
||||
flushText();
|
||||
tokens.push(match);
|
||||
position += match.sequence?.length;
|
||||
continue;
|
||||
}
|
||||
// Plain character
|
||||
textBuffer += source[position];
|
||||
position++;
|
||||
}
|
||||
flushText();
|
||||
return tokens;
|
||||
}
|
||||
/**
|
||||
* Try to resolve an HTML entity at the start of the string.
|
||||
* Returns the resolved character and the length consumed, or null.
|
||||
*/
|
||||
resolveEntity(text) {
|
||||
const namedPattern = /^&(?<name>[a-zA-Z]+);/;
|
||||
const numericPattern = /^&#(?<code>\d+);/;
|
||||
const hexPattern = /^&#x(?<hex>[0-9a-fA-F]+);/;
|
||||
const named = text.match(namedPattern);
|
||||
if (named?.groups) {
|
||||
const resolved = NAMED_ENTITIES[named.groups.name.toLowerCase()];
|
||||
if (resolved) {
|
||||
return {
|
||||
character: resolved,
|
||||
length: named[0].length,
|
||||
};
|
||||
}
|
||||
}
|
||||
const numeric = text.match(numericPattern);
|
||||
if (numeric?.groups) {
|
||||
return {
|
||||
character: String.fromCharCode(parseInt(numeric.groups.code, 10)),
|
||||
length: numeric[0].length,
|
||||
};
|
||||
}
|
||||
const hex = text.match(hexPattern);
|
||||
if (hex?.groups) {
|
||||
return {
|
||||
character: String.fromCharCode(parseInt(hex.groups.hex, 16)),
|
||||
length: hex[0].length,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Match a code span starting at the given position.
|
||||
* Handles single backtick delimiters only (not multi-backtick).
|
||||
*/
|
||||
matchCodeSpan(source, position) {
|
||||
if (source[position] !== '`') {
|
||||
return null;
|
||||
}
|
||||
const closeIndex = source.indexOf('`', position + 1);
|
||||
if (closeIndex === -1) {
|
||||
return null;
|
||||
}
|
||||
const content = source.slice(position + 1, closeIndex);
|
||||
return {
|
||||
content,
|
||||
raw: source.slice(position, closeIndex + 1),
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Match a markdown link [text](url) or [text](url "title")
|
||||
* starting at the given position. Disallows [ in link text
|
||||
* to prevent nested link ambiguity.
|
||||
*/
|
||||
matchLink(source, position) {
|
||||
const linkPattern = /^\[(?<text>[^\[\]]+)\]\((?<href>[^\s)]+)(?:\s+"(?<title>[^"]*)")?\)/;
|
||||
const match = source.slice(position).match(linkPattern);
|
||||
if (!match?.groups) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
text: match.groups.text,
|
||||
href: match.groups.href,
|
||||
title: match.groups.title,
|
||||
length: match[0].length,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Match an angle-bracket autolink <url> at the start of the string.
|
||||
*/
|
||||
matchAutolink(text) {
|
||||
const pattern = /^<(?<url>https?:\/\/[^\s>]+)>/;
|
||||
const match = text.match(pattern);
|
||||
if (!match?.groups) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
url: match.groups.url,
|
||||
length: match[0].length,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Match a bare URL (https://...) at the start of the string.
|
||||
*/
|
||||
matchBareUrl(text) {
|
||||
const pattern = /^[\w]+?:\/\/[^\s<>\x00]+/;
|
||||
const match = text.match(pattern);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
url: match[0],
|
||||
length: match[0].length,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Match an HTML tag at the start of the string.
|
||||
*/
|
||||
matchHtmlTag(text) {
|
||||
const pattern = /^<\/?[a-zA-Z][a-zA-Z0-9]*(?:\s+[^>]*)?\s*\/?>/;
|
||||
const match = text.match(pattern);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
tag: match[0],
|
||||
length: match[0].length,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Try to match a delimiter at the given position. For runs of the
|
||||
* same character (e.g. *** = 3 asterisks), the run is split into
|
||||
* the longest registered delimiter that fits, then the remainder.
|
||||
* This handles cases like **bold***italic* where *** must split
|
||||
* into ** (close bold) + * (open italic).
|
||||
*/
|
||||
matchTag(source, position) {
|
||||
// Count the full run of the same character
|
||||
var sequence = source[position];
|
||||
let runLength = 0;
|
||||
while (position + runLength < source.length && source[position + runLength] === sequence) {
|
||||
runLength++;
|
||||
}
|
||||
if (runLength === 0) {
|
||||
return null;
|
||||
}
|
||||
sequence = sequence.repeat(runLength);
|
||||
// get the tag that matches this sequence
|
||||
for (const tag of this.tags) {
|
||||
const delim = tag.isBlock ? tag.opener : tag.delimiter;
|
||||
if (delim !== sequence) {
|
||||
continue;
|
||||
}
|
||||
const charBefore = position > 0 ? source[position - 1] : '\n';
|
||||
const charAfter = source[position + sequence.length];
|
||||
const leftFlanking = (charBefore === undefined || PUNCTUATION.has(charBefore) || charBefore === '\n')
|
||||
&& charAfter !== undefined && charAfter !== ' ' && charAfter !== '\n' && charAfter !== '\t' && charAfter !== '\u00A0';
|
||||
const rightFlanking = charBefore !== undefined && charBefore !== ' ' && charBefore !== '\n' && charBefore !== '\t' && charBefore !== '\u00A0'
|
||||
&& (charAfter === undefined || PUNCTUATION.has(charAfter) || charAfter === '\n');
|
||||
return {
|
||||
role: leftFlanking ? 'open' : rightFlanking ? 'close' : 'block',
|
||||
sequence: sequence,
|
||||
value: sequence,
|
||||
tag: tag,
|
||||
};
|
||||
}
|
||||
;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
exports.Tokenizer = Tokenizer;
|
||||
Vendored
+2
@@ -0,0 +1,2 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
Reference in New Issue
Block a user