evilchili/ribbit
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

Reimplement as a tokenizer with GFM parity

Browse Source
This commit is contained in:
gsb
2026-04-30 07:09:19 +00:00
parent 005db2f431
commit d41716c8b2
28 changed files with 4798 additions and 1398 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/integration/test_fuzz.js
+29 -18
View File
@@ -34,8 +34,8 @@ function mulberry32(seed) {
/* ── Keystroke generation ── */
const PRINTABLE = 'abcdefghijklmnopqrstuvwxyz 0123456789.,!?';
const DELIMITERS = ['*', '**', '***', '`', '~~'];
const BLOCK_PREFIXES = ['# ', '## ', '### ', '- ', '1. ', '> ', '---'];
const DELIMITERS = ['*', '**', '***', '`', '~~', '_', '__', '___'];
const BLOCK_PREFIXES = ['# ', '## ', '### ', '- ', '+ ', '1. ', '> ', '---', '~~~'];
const SPECIAL_KEYS = [
{ name: 'Enter', keys: Key.ENTER, isSpecial: true },
{ name: 'Backspace', keys: Key.BACK_SPACE, isSpecial: true },
@@ -70,8 +70,14 @@ function generateSequence(random, length) {
} else if (roll < 0.94) {
/* repeated delimiter (stress test) */
const count = 2 + Math.floor(random() * 4);
const character = '*';
const delimiters = ['*', '_', '~'];
const character = delimiters[Math.floor(random() * delimiters.length)];
sequence.push({ name: character.repeat(count), keys: character.repeat(count) });
} else if (roll < 0.97) {
/* backslash sequences */
const escaped = ['\\*', '\\_', '\\`', '\\~', '\\\\', '\\'];
const fragment = escaped[Math.floor(random() * escaped.length)];
sequence.push({ name: fragment, keys: fragment });
} else {
/* angle bracket / HTML-like content */
const fragments = ['<', '>', '<div>', '</div>', '<b>', '&amp;'];
@@ -188,7 +194,8 @@ async function checkInvariants() {
var forbiddenRules = {
'STRONG': ['STRONG','B'], 'B': ['STRONG','B'],
'EM': ['EM','I'], 'I': ['EM','I'],
'CODE': ['CODE','STRONG','B','EM','I','A'],
'CODE': ['CODE','STRONG','B','EM','I','A','DEL'],
'DEL': ['DEL','S','STRIKE'], 'S': ['DEL','S','STRIKE'], 'STRIKE': ['DEL','S','STRIKE'],
'A': ['A'],
};
var allElements = editor.querySelectorAll('*');
@@ -212,28 +219,32 @@ async function checkInvariants() {
}
/* Invariant 6: rendered HTML is stable through markdown round-trip.
md → toHTML → toMarkdown → toHTML must produce the same HTML.
The markdown representation may change (e.g. ***** → ***) but
the rendered output must be identical.
md → toHTML → toMarkdown → toHTML must eventually stabilize.
The first round-trip may change the HTML (e.g. literal <strong>
in text becomes a real element via HTML passthrough, then
serializes as **). But the second round-trip must be stable.
Skip if there are speculative elements (in-progress editing). */
var hasSpeculative = editor.querySelector('[data-speculative]');
if (!hasSpeculative) {
try {
var md = window.__ribbitEditor.getMarkdown();
var converter = window.__ribbitEditor.converter;
// Two round-trips: allow the first to normalize, check
// that the second produces identical HTML
var html1 = converter.toHTML(md);
var md2 = converter.toMarkdown(html1);
var html2 = converter.toHTML(md2);
/* Compare the rendered HTML, not the markdown */
var div1 = document.createElement('div');
div1.innerHTML = html1;
var div2 = document.createElement('div');
div2.innerHTML = html2;
var text1 = div1.textContent.replace(/\s+/g, ' ').trim();
var text2 = div2.textContent.replace(/\s+/g, ' ').trim();
if (text1 !== text2) {
return 'Round-trip HTML mismatch:\n html1: "' + text1.slice(0, 80) +
'"\n html2: "' + text2.slice(0, 80) + '"';
var md3 = converter.toMarkdown(html2);
var html3 = converter.toHTML(md3);
var normalize = function(html) {
return html
.replace(/\s*id='[^']*'/g, '')
.replace(/\s+/g, ' ')
.trim();
};
if (normalize(html2) !== normalize(html3)) {
return 'Round-trip HTML not stable after 2 passes:\n pass2: "' + normalize(html2).slice(0, 80) +
'"\n pass3: "' + normalize(html3).slice(0, 80) + '"';
}
} catch (err) {
return 'Round-trip check threw: ' + err.message;
@@ -241,7 +252,7 @@ async function checkInvariants() {
}
/* Invariant 7: only valid inline elements inside block content */
var validInline = ['STRONG','B','EM','I','CODE','A','BR'];
var validInline = ['STRONG','B','EM','I','CODE','A','BR','DEL','S','STRIKE'];
var blocks = editor.querySelectorAll('p,h1,h2,h3,h4,h5,h6,li,blockquote,td,th');
for (var b = 0; b < blocks.length; b++) {
var inlineEls = blocks[b].querySelectorAll('*');
test/integration/test_wysiwyg.js
+49
View File
@@ -433,6 +433,55 @@ async function runTests() {
assert(html.includes('<h2'), `Missing h2: ${html}`);
assert(html.includes('<li') || html.includes('<ul'), `Missing list: ${html}`);
});
console.log(' Strikethrough:');
await test('~~text~~ transforms to <del>', async () => {
await resetEditor();
await typeString('~~gone~~');
const html = await getHTML();
assert(html.includes('<del'), `No <del>: ${html}`);
assert(!html.includes('data-speculative'), `Still speculative: ${html}`);
assert(html.includes('gone'), `Missing content: ${html}`);
});
await test('~~text shows speculative strikethrough', async () => {
await resetEditor();
await typeString('~~hel');
const html = await getHTML();
assert(html.includes('data-speculative'), `No speculative: ${html}`);
assert(html.includes('<del'), `No <del>: ${html}`);
});
console.log(' Alternate syntax:');
await test('~~~ transforms to fenced code', async () => {
await resetEditor();
await typeString('~~~');
await driver.sleep(50);
const html = await getHTML();
assert(html.includes('<pre') || html.includes('<code'), `No code block: ${html}`);
});
await test('+ space transforms to unordered list', async () => {
await resetEditor();
await typeChar('+');
let html = await getHTML();
assert(!html.includes('<ul'), `Premature ul: ${html}`);
await typeChar(' ');
html = await getHTML();
assert(html.includes('<ul') || html.includes('<li'), `No list after "+ ": ${html}`);
});
console.log(' Backslash escapes:');
await test('backslash is just a character in WYSIWYG', async () => {
await resetEditor();
await typeString('hello\\world');
const html = await getHTML();
assert(html.includes('hello') && html.includes('world'), `Missing content: ${html}`);
});
}
(async () => {