feat(desktop): embed BoosAPI Desktop client as a subproject
Vendor the rebranded BoosAPI Desktop client (Tauri + Vue) into metapi as a monorepo subdirectory under desktop/. Source only — node_modules, dist, Cargo target and the upstream .git history are excluded. desktop/ keeps its own independent Cargo workspace (desktop/src-tauri with crates/*); it is not part of the backend boosapi crate, so a `cargo check` at the metapi root still compiles only the backend. Build the desktop app from desktop/src-tauri per desktop/README.md. Note: app icons are still placeholder assets pending the final BoosAPI logo. The updater signing keys live outside the repo and are not included.
This commit is contained in:
boos
@@ -0,0 +1,346 @@
|
||||
import type { MarkdownIt } from 'markdown-it-ts'
|
||||
import markdownItContainer from 'markdown-it-container'
|
||||
|
||||
function parseLooseInlineAttrs(input: string): Record<string, unknown> | null {
|
||||
const s = String(input ?? '').trim()
|
||||
if (!s.startsWith('{') || !s.endsWith('}'))
|
||||
return null
|
||||
|
||||
const inner = s.slice(1, -1).trim()
|
||||
if (!inner)
|
||||
return {}
|
||||
|
||||
// Only support a shallow object literal: {key:value, foo:"bar"}
|
||||
// No nested objects/arrays/functions.
|
||||
if (inner.includes('{') || inner.includes('[') || inner.includes(']'))
|
||||
return null
|
||||
|
||||
const parts: string[] = []
|
||||
let buf = ''
|
||||
let inSingle = false
|
||||
let inDouble = false
|
||||
for (let i = 0; i < inner.length; i++) {
|
||||
const ch = inner[i]
|
||||
if (ch === '\\') {
|
||||
buf += ch
|
||||
if (i + 1 < inner.length) {
|
||||
buf += inner[i + 1]
|
||||
i++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (!inDouble && ch === '\'') {
|
||||
inSingle = !inSingle
|
||||
buf += ch
|
||||
continue
|
||||
}
|
||||
if (!inSingle && ch === '"') {
|
||||
inDouble = !inDouble
|
||||
buf += ch
|
||||
continue
|
||||
}
|
||||
if (!inSingle && !inDouble && ch === ',') {
|
||||
parts.push(buf.trim())
|
||||
buf = ''
|
||||
continue
|
||||
}
|
||||
buf += ch
|
||||
}
|
||||
if (buf.trim())
|
||||
parts.push(buf.trim())
|
||||
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const part of parts) {
|
||||
if (!part)
|
||||
continue
|
||||
let inS = false
|
||||
let inD = false
|
||||
let split = -1
|
||||
for (let i = 0; i < part.length; i++) {
|
||||
const ch = part[i]
|
||||
if (ch === '\\') {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (!inD && ch === '\'') {
|
||||
inS = !inS
|
||||
continue
|
||||
}
|
||||
if (!inS && ch === '"') {
|
||||
inD = !inD
|
||||
continue
|
||||
}
|
||||
if (!inS && !inD && ch === ':') {
|
||||
split = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if (split === -1)
|
||||
return null
|
||||
|
||||
const rawKey = part.slice(0, split).trim()
|
||||
const rawVal = part.slice(split + 1).trim()
|
||||
if (!rawKey)
|
||||
return null
|
||||
|
||||
let key = rawKey
|
||||
if ((key.startsWith('"') && key.endsWith('"')) || (key.startsWith('\'') && key.endsWith('\''))) {
|
||||
try {
|
||||
key = JSON.parse(key.replace(/^'/, '"').replace(/'$/, '"'))
|
||||
}
|
||||
catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
// Restrict unquoted keys to simple identifiers.
|
||||
if (!/^[_$A-Z][\w$-]*$/i.test(key))
|
||||
return null
|
||||
|
||||
let value: unknown
|
||||
if (!rawVal) {
|
||||
value = ''
|
||||
}
|
||||
else if ((rawVal.startsWith('"') && rawVal.endsWith('"')) || (rawVal.startsWith('\'') && rawVal.endsWith('\''))) {
|
||||
try {
|
||||
value = JSON.parse(rawVal.replace(/^'/, '"').replace(/'$/, '"'))
|
||||
}
|
||||
catch {
|
||||
value = rawVal
|
||||
}
|
||||
}
|
||||
else if (/^-?\d+(?:\.\d+)?$/.test(rawVal)) {
|
||||
value = Number(rawVal)
|
||||
}
|
||||
else if (rawVal === 'true' || rawVal === 'false') {
|
||||
value = rawVal === 'true'
|
||||
}
|
||||
else if (rawVal === 'null') {
|
||||
value = null
|
||||
}
|
||||
else {
|
||||
value = rawVal
|
||||
}
|
||||
|
||||
out[key] = value
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
export function applyContainers(md: MarkdownIt) {
|
||||
;[
|
||||
'admonition',
|
||||
'info',
|
||||
'warning',
|
||||
'error',
|
||||
'tip',
|
||||
'danger',
|
||||
'note',
|
||||
'caution',
|
||||
].forEach((name) => {
|
||||
md.use(markdownItContainer, name, {
|
||||
render(tokens: unknown, idx: number) {
|
||||
const tokensAny = tokens as unknown as import('../types').MarkdownToken[]
|
||||
const token = tokensAny[idx]
|
||||
// `nesting` is a runtime-only property present on MarkdownIt tokens.
|
||||
// Narrow the shape with `unknown` -> specific minimal interface to avoid `as any`.
|
||||
const tokenShape = token as unknown as { nesting?: number }
|
||||
if (tokenShape.nesting === 1) {
|
||||
return `<div class="vmr-container vmr-container-${name}">`
|
||||
}
|
||||
else {
|
||||
return '</div>\n'
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// fallback for simple ::: blocks (kept for backwards compat)
|
||||
md.block.ruler.before(
|
||||
'fence',
|
||||
'vmr_container_fallback',
|
||||
(state: unknown, startLine: number, endLine: number, silent: boolean) => {
|
||||
interface ParserState {
|
||||
bMarks: number[]
|
||||
tShift: number[]
|
||||
eMarks: number[]
|
||||
src: string
|
||||
env: unknown
|
||||
push: (type: string, tag?: string, nesting?: number) => any
|
||||
tokens: any[]
|
||||
md: any
|
||||
line: number
|
||||
}
|
||||
const s = state as unknown as ParserState
|
||||
const startPos = s.bMarks[startLine] + s.tShift[startLine]
|
||||
const lineMax = s.eMarks[startLine]
|
||||
const line = s.src.slice(startPos, lineMax)
|
||||
|
||||
// Match ::: container syntax: ::: name {"json"}
|
||||
// Using separate patterns to avoid backtracking issues
|
||||
// Allow both "::: name" and ":::name", and stop before inline JSON attrs.
|
||||
const nameMatch = line.match(/^:::\s*([^\s{]+)/)
|
||||
if (!nameMatch)
|
||||
return false
|
||||
|
||||
const name = nameMatch[1]
|
||||
// Validate name is not empty (handles edge case like "::: ")
|
||||
if (!name.trim())
|
||||
return false
|
||||
|
||||
const rest = line.slice(nameMatch[0].length)
|
||||
const trimmedRest = rest.trim()
|
||||
|
||||
// Support both:
|
||||
// - ::: name {"json"}
|
||||
// - :::name args {"json"}
|
||||
// - :::name args
|
||||
let argsStr: string | undefined
|
||||
let jsonStr: string | undefined
|
||||
|
||||
const jsonStart = trimmedRest.indexOf('{')
|
||||
const jsonCandidate = jsonStart >= 0 ? trimmedRest.slice(jsonStart).trimStart() : undefined
|
||||
|
||||
if (jsonStart === -1) {
|
||||
argsStr = trimmedRest || undefined
|
||||
}
|
||||
else {
|
||||
const before = trimmedRest.slice(0, jsonStart).trim()
|
||||
argsStr = before || undefined
|
||||
|
||||
// Improved JSON matching: find balanced braces instead of simple non-greedy match
|
||||
if (jsonCandidate?.startsWith('{')) {
|
||||
let depth = 0
|
||||
let jsonEnd = -1
|
||||
for (let i = 0; i < jsonCandidate.length; i++) {
|
||||
if (jsonCandidate[i] === '{')
|
||||
depth++
|
||||
else if (jsonCandidate[i] === '}')
|
||||
depth--
|
||||
if (depth === 0) {
|
||||
jsonEnd = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
if (jsonEnd > 0) {
|
||||
jsonStr = jsonCandidate.slice(0, jsonEnd)
|
||||
}
|
||||
}
|
||||
|
||||
// Streaming mid-state: if we saw a '{' but didn't find a balanced '}',
|
||||
// treat the whole remainder as a plain args string.
|
||||
// Example: "::: viewcode:stream {xxx:yyy" => attrs.args === "{xxx:yyy"
|
||||
if (!jsonStr)
|
||||
argsStr = trimmedRest || undefined
|
||||
}
|
||||
|
||||
if (silent)
|
||||
return true
|
||||
|
||||
const envFinal = !!(s.env as any)?.__markstreamFinal
|
||||
|
||||
let nextLine = startLine + 1
|
||||
let found = false
|
||||
while (nextLine <= endLine) {
|
||||
const sPos = s.bMarks[nextLine] + s.tShift[nextLine]
|
||||
const ePos = s.eMarks[nextLine]
|
||||
if (s.src.slice(sPos, ePos).trim() === ':::') {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
nextLine++
|
||||
}
|
||||
// Streaming optimization:
|
||||
// - If we have an opening ":::name" but no closing ":::" yet, still emit a container.
|
||||
// Treat the remainder of the document as the container body.
|
||||
// - The structured parser will expose this as `loading: true` until it closes,
|
||||
// or until `final=true` is provided.
|
||||
if (!found)
|
||||
nextLine = endLine
|
||||
|
||||
const tokenOpen = s.push('vmr_container_open', 'div', 1)
|
||||
// `tokenOpen` is runtime token object; keep using runtime helpers but avoid casting `s` to `any`.
|
||||
tokenOpen.attrSet('class', `vmr-container vmr-container-${name}`)
|
||||
|
||||
// Mark unclosed containers for downstream consumers (optional).
|
||||
;(tokenOpen.meta as any) = { ...(tokenOpen.meta as any), unclosed: !found && !envFinal }
|
||||
|
||||
// If args are present (non-JSON payload right after the name), preserve them as a data attribute.
|
||||
if (argsStr)
|
||||
tokenOpen.attrSet('data-args', argsStr)
|
||||
|
||||
// If JSON attributes are present, store them as data attributes
|
||||
if (jsonStr) {
|
||||
try {
|
||||
const attrs = JSON.parse(jsonStr)
|
||||
for (const [key, value] of Object.entries(attrs)) {
|
||||
const isComplexValue = value != null && typeof value === 'object'
|
||||
tokenOpen.attrSet(
|
||||
`data-${key}`,
|
||||
isComplexValue ? JSON.stringify(value) : String(value),
|
||||
)
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// If strict JSON parsing fails, try a small loose-object syntax: {xxx:yyy}
|
||||
const loose = parseLooseInlineAttrs(jsonStr)
|
||||
if (loose) {
|
||||
for (const [key, value] of Object.entries(loose)) {
|
||||
const isComplexValue = value != null && typeof value === 'object'
|
||||
tokenOpen.attrSet(
|
||||
`data-${key}`,
|
||||
isComplexValue ? JSON.stringify(value) : String(value),
|
||||
)
|
||||
}
|
||||
}
|
||||
else {
|
||||
// If parsing fails, store the raw string as a data attribute
|
||||
tokenOpen.attrSet('data-attrs', jsonStr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parse inner content as full block markdown so block-level constructs
|
||||
// (especially fenced code blocks) are preserved inside the container.
|
||||
const contentLines: string[] = []
|
||||
for (let i = startLine + 1; i < nextLine; i++) {
|
||||
const sPos = s.bMarks[i] + s.tShift[i]
|
||||
const ePos = s.eMarks[i]
|
||||
contentLines.push(s.src.slice(sPos, ePos))
|
||||
}
|
||||
const hasContent = contentLines.some(line => line.trim().length > 0)
|
||||
if (hasContent) {
|
||||
// Always end with two newlines to ensure block separation (esp. for lists/paragraphs)
|
||||
let innerSrc = contentLines.join('\n')
|
||||
if (!innerSrc.endsWith('\n'))
|
||||
innerSrc += '\n'
|
||||
if (!innerSrc.endsWith('\n\n'))
|
||||
innerSrc += '\n'
|
||||
|
||||
// The last token should be the current vm_container_open token
|
||||
const prevToken = s.tokens[s.tokens.length - 1]
|
||||
|
||||
// Save the inner Markdown content as raw
|
||||
if (prevToken) {
|
||||
prevToken.raw = innerSrc
|
||||
}
|
||||
|
||||
const innerTokens: any[] = []
|
||||
// Use the same env as the parent block parser to ensure all block rules are available
|
||||
s.md.block.parse(innerSrc, s.md, s.env, innerTokens)
|
||||
s.tokens.push(...innerTokens)
|
||||
}
|
||||
|
||||
if (found)
|
||||
s.push('vmr_container_close', 'div', -1)
|
||||
|
||||
s.line = found ? (nextLine + 1) : nextLine
|
||||
return true
|
||||
},
|
||||
{
|
||||
// Ensure this rule can terminate an active paragraph so content immediately
|
||||
// above `:::` isn't swallowed into the same paragraph block.
|
||||
alt: ['paragraph', 'reference', 'blockquote', 'list'],
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,996 @@
|
||||
import type { MarkdownIt, Token } from 'markdown-it-ts'
|
||||
import { normalizeCustomHtmlTagName } from '../customHtmlTags'
|
||||
import { STANDARD_BLOCK_HTML_TAGS, STANDARD_HTML_TAGS, VOID_HTML_TAGS } from '../htmlTags'
|
||||
import { escapeTagForRegExp, findTagCloseIndexOutsideQuotes } from '../htmlTagUtils'
|
||||
|
||||
const VOID_TAGS = VOID_HTML_TAGS
|
||||
|
||||
// A conservative set of common HTML tags used to detect streaming mid-states.
|
||||
// We only suppress/merge partial tags for these names to avoid false positives
|
||||
// (e.g., autolinks like <http://...>).
|
||||
const BASE_COMMON_HTML_TAGS = STANDARD_HTML_TAGS
|
||||
|
||||
const BLOCK_LEVEL_HTML_TAGS = new Set<string>(STANDARD_BLOCK_HTML_TAGS)
|
||||
BLOCK_LEVEL_HTML_TAGS.delete('details')
|
||||
|
||||
const OPEN_TAG_RE = /<([A-Z][\w-]*)(?=[\s/>]|$)/gi
|
||||
const CLOSE_TAG_RE = /<\/\s*([A-Z][\w-]*)(?=[\s/>]|$)/gi
|
||||
const TAG_NAME_AT_START_RE = /^<\s*(?:\/\s*)?([A-Z][\w-]*)/i
|
||||
const STRICT_OPEN_TAG_NAME_AT_START_RE = /^<\s*([A-Z][\w:-]*)(?=[\s/>]|$)/i
|
||||
|
||||
function getHtmlInlineTagName(content: string) {
|
||||
return (content.match(TAG_NAME_AT_START_RE)?.[1] ?? '').toLowerCase()
|
||||
}
|
||||
|
||||
function isHtmlInlineClosingTag(content: string) {
|
||||
return /^\s*<\s*\//.test(content)
|
||||
}
|
||||
|
||||
function isSelfClosingHtmlInline(content: string, tag: string) {
|
||||
return VOID_TAGS.has(tag) || /\/\s*>\s*$/.test(content)
|
||||
}
|
||||
|
||||
function findMatchingCloseChildIndex(
|
||||
children: Array<{ type?: string, content?: string }>,
|
||||
tag: string,
|
||||
) {
|
||||
let depth = 0
|
||||
|
||||
for (let index = 0; index < children.length; index++) {
|
||||
const child = children[index]
|
||||
if (!child || child.type !== 'html_inline')
|
||||
continue
|
||||
|
||||
const content = String(child.content ?? '')
|
||||
const childTag = getHtmlInlineTagName(content)
|
||||
if (childTag !== tag)
|
||||
continue
|
||||
|
||||
if (isHtmlInlineClosingTag(content)) {
|
||||
if (depth === 0)
|
||||
return index
|
||||
depth--
|
||||
continue
|
||||
}
|
||||
|
||||
if (!isSelfClosingHtmlInline(content, childTag))
|
||||
depth++
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
function getTrailingOpenDepth(
|
||||
children: Array<{ type?: string, content?: string }>,
|
||||
tag: string,
|
||||
) {
|
||||
let depth = 0
|
||||
|
||||
for (const child of children) {
|
||||
if (!child || child.type !== 'html_inline')
|
||||
continue
|
||||
|
||||
const content = String(child.content ?? '')
|
||||
const childTag = getHtmlInlineTagName(content)
|
||||
if (childTag !== tag)
|
||||
continue
|
||||
|
||||
if (isHtmlInlineClosingTag(content)) {
|
||||
if (depth > 0)
|
||||
depth--
|
||||
continue
|
||||
}
|
||||
|
||||
if (!isSelfClosingHtmlInline(content, childTag))
|
||||
depth++
|
||||
}
|
||||
|
||||
return depth
|
||||
}
|
||||
|
||||
function findMatchingCloseRangeInHtml(content: string, tag: string, startIndex = 0) {
|
||||
const tokenRe = new RegExp(String.raw`<\s*(\/?)\s*${escapeTagForRegExp(tag)}(?=[\s>/])[^>]*>`, 'gi')
|
||||
tokenRe.lastIndex = Math.max(0, startIndex)
|
||||
let depth = 0
|
||||
let match: RegExpExecArray | null
|
||||
|
||||
while ((match = tokenRe.exec(content)) !== null) {
|
||||
const raw = match[0] ?? ''
|
||||
const closing = !!match[1]
|
||||
const selfClosing = !closing && /\/\s*>$/.test(raw)
|
||||
|
||||
if (closing) {
|
||||
if (depth === 0) {
|
||||
return {
|
||||
start: match.index,
|
||||
end: match.index + raw.length,
|
||||
}
|
||||
}
|
||||
depth--
|
||||
continue
|
||||
}
|
||||
|
||||
if (!selfClosing)
|
||||
depth++
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
function getTrailingCustomTagDepthInHtml(content: string, tag: string) {
|
||||
const tokenRe = new RegExp(String.raw`<\s*(\/?)\s*${escapeTagForRegExp(tag)}(?=[\s>/])[^>]*>`, 'gi')
|
||||
let depth = 0
|
||||
let match: RegExpExecArray | null
|
||||
|
||||
while ((match = tokenRe.exec(content)) !== null) {
|
||||
const raw = match[0] ?? ''
|
||||
const closing = !!match[1]
|
||||
const selfClosing = !closing && /\/\s*>$/.test(raw)
|
||||
|
||||
if (closing) {
|
||||
if (depth > 0)
|
||||
depth--
|
||||
continue
|
||||
}
|
||||
|
||||
if (!selfClosing)
|
||||
depth++
|
||||
}
|
||||
|
||||
return depth
|
||||
}
|
||||
|
||||
function tokenToRaw(token: Token) {
|
||||
const shape = token as unknown as { raw?: string, markup?: string, content?: string }
|
||||
return String(shape.raw ?? shape.content ?? shape.markup ?? '')
|
||||
}
|
||||
|
||||
function isNonElementHtmlBlock(content: string) {
|
||||
return /^\s*<\s*[!?]/.test(content)
|
||||
}
|
||||
|
||||
function buildCommonHtmlTagSet(extraTags?: readonly string[]) {
|
||||
const set = new Set(BASE_COMMON_HTML_TAGS)
|
||||
if (extraTags && Array.isArray(extraTags)) {
|
||||
for (const t of extraTags) {
|
||||
const raw = String(t ?? '').trim()
|
||||
if (!raw)
|
||||
continue
|
||||
const m = raw.match(/^[<\s/]*([A-Z][\w-]*)/i)
|
||||
if (!m)
|
||||
continue
|
||||
set.add(m[1].toLowerCase())
|
||||
}
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
function isCommonHtmlTagOrPrefix(tag: string, tagSet: Set<string>) {
|
||||
if (tagSet.has(tag))
|
||||
return true
|
||||
for (const common of tagSet) {
|
||||
if (common.startsWith(tag))
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function findFirstIncompleteTag(content: string, tagSet: Set<string>) {
|
||||
let first:
|
||||
| { index: number, tag: string, closing: boolean }
|
||||
| null = null
|
||||
|
||||
for (const m of content.matchAll(OPEN_TAG_RE)) {
|
||||
const idx = m.index ?? -1
|
||||
if (idx < 0)
|
||||
continue
|
||||
const tag = (m[1] ?? '').toLowerCase()
|
||||
// For opening tags we also accept prefixes of known HTML tags
|
||||
// (e.g., '<fo' while typing '<font ...>').
|
||||
if (!isCommonHtmlTagOrPrefix(tag, tagSet))
|
||||
continue
|
||||
const rest = content.slice(idx)
|
||||
if (findTagCloseIndexOutsideQuotes(rest) !== -1)
|
||||
continue
|
||||
if (!first || idx < first.index)
|
||||
first = { index: idx, tag, closing: false }
|
||||
}
|
||||
|
||||
for (const m of content.matchAll(CLOSE_TAG_RE)) {
|
||||
const idx = m.index ?? -1
|
||||
if (idx < 0)
|
||||
continue
|
||||
const tag = (m[1] ?? '').toLowerCase()
|
||||
// For closing tags we also accept prefixes of known HTML tags
|
||||
// (e.g., '</sp' while typing '</span>').
|
||||
if (!isCommonHtmlTagOrPrefix(tag, tagSet))
|
||||
continue
|
||||
const rest = content.slice(idx)
|
||||
if (findTagCloseIndexOutsideQuotes(rest) !== -1)
|
||||
continue
|
||||
if (!first || idx < first.index)
|
||||
first = { index: idx, tag, closing: true }
|
||||
}
|
||||
|
||||
// Also swallow bare "<" or "</" at the end while typing.
|
||||
const bareClose = /<\/\s*$/.exec(content)
|
||||
if (bareClose && typeof bareClose.index === 'number') {
|
||||
const idx = bareClose.index
|
||||
const rest = content.slice(idx)
|
||||
if (!rest.includes('>') && (!first || idx < first.index))
|
||||
first = { index: idx, tag: '', closing: true }
|
||||
}
|
||||
|
||||
const bareOpen = /<\s*$/.exec(content)
|
||||
if (bareOpen && typeof bareOpen.index === 'number') {
|
||||
const idx = bareOpen.index
|
||||
const rest = content.slice(idx)
|
||||
// Avoid matching "</" which is handled above.
|
||||
if (!rest.startsWith('</') && !rest.includes('>') && (!first || idx < first.index))
|
||||
first = { index: idx, tag: '', closing: false }
|
||||
}
|
||||
|
||||
return first
|
||||
}
|
||||
|
||||
function splitTextToken(token: Token, content: string) {
|
||||
const t = token as Token & { content?: string, raw?: string }
|
||||
// Preserve the original Token prototype (markdown-it-ts attaches helper methods).
|
||||
const nt = Object.assign(
|
||||
Object.create(Object.getPrototypeOf(t)),
|
||||
t,
|
||||
{ type: 'text', content, raw: content },
|
||||
) as Token
|
||||
return nt
|
||||
}
|
||||
|
||||
function fixStreamingHtmlInlineChildren(children: Token[], tagSet: Set<string>) {
|
||||
if (!children.length)
|
||||
return { children }
|
||||
|
||||
const out: Token[] = []
|
||||
let pending: { tag: string, buffer: string, closing: boolean } | null = null
|
||||
let pendingAtEnd: string | null = null
|
||||
|
||||
function pushTextPart(text: string, baseToken?: Token) {
|
||||
if (!text)
|
||||
return
|
||||
if (baseToken)
|
||||
out.push(splitTextToken(baseToken, text))
|
||||
else
|
||||
out.push({ type: 'text', content: text, raw: text } as any)
|
||||
}
|
||||
|
||||
function splitCompleteHtmlFromText(chunk: string, baseToken?: Token) {
|
||||
let cursor = 0
|
||||
while (cursor < chunk.length) {
|
||||
const lt = chunk.indexOf('<', cursor)
|
||||
if (lt === -1) {
|
||||
pushTextPart(chunk.slice(cursor), baseToken)
|
||||
break
|
||||
}
|
||||
pushTextPart(chunk.slice(cursor, lt), baseToken)
|
||||
const sub = chunk.slice(lt)
|
||||
const tagMatch = sub.match(TAG_NAME_AT_START_RE)
|
||||
if (!tagMatch) {
|
||||
pushTextPart('<', baseToken)
|
||||
cursor = lt + 1
|
||||
continue
|
||||
}
|
||||
const closeIdx = findTagCloseIndexOutsideQuotes(sub)
|
||||
if (closeIdx === -1) {
|
||||
pushTextPart('<', baseToken)
|
||||
cursor = lt + 1
|
||||
continue
|
||||
}
|
||||
|
||||
const tagText = sub.slice(0, closeIdx + 1)
|
||||
const tagName = (tagMatch[1] ?? '').toLowerCase()
|
||||
if (tagSet.has(tagName)) {
|
||||
out.push({
|
||||
type: 'html_inline',
|
||||
tag: '',
|
||||
content: tagText,
|
||||
raw: tagText,
|
||||
} as any)
|
||||
}
|
||||
else {
|
||||
pushTextPart(tagText, baseToken)
|
||||
}
|
||||
cursor = lt + tagText.length
|
||||
}
|
||||
}
|
||||
|
||||
function processTextChunk(chunk: string, baseToken?: Token) {
|
||||
if (!chunk)
|
||||
return
|
||||
const match = findFirstIncompleteTag(chunk, tagSet)
|
||||
if (!match) {
|
||||
splitCompleteHtmlFromText(chunk, baseToken)
|
||||
return
|
||||
}
|
||||
|
||||
const before = chunk.slice(0, match.index)
|
||||
if (before)
|
||||
splitCompleteHtmlFromText(before, baseToken)
|
||||
pending = {
|
||||
tag: match.tag,
|
||||
buffer: chunk.slice(match.index),
|
||||
closing: match.closing,
|
||||
}
|
||||
pendingAtEnd = pending.buffer
|
||||
}
|
||||
|
||||
for (const child of children) {
|
||||
if (pending) {
|
||||
pending.buffer += tokenToRaw(child)
|
||||
pendingAtEnd = pending.buffer
|
||||
const closeIdx = findTagCloseIndexOutsideQuotes(pending.buffer)
|
||||
if (closeIdx === -1) {
|
||||
// still incomplete: swallow this token to avoid rendering jitter
|
||||
continue
|
||||
}
|
||||
|
||||
const tagChunk = pending.buffer.slice(0, closeIdx + 1)
|
||||
const afterChunk = pending.buffer.slice(closeIdx + 1)
|
||||
out.push({
|
||||
type: 'html_inline',
|
||||
tag: '',
|
||||
content: tagChunk,
|
||||
raw: tagChunk,
|
||||
} as any)
|
||||
pending = null
|
||||
pendingAtEnd = null
|
||||
if (afterChunk)
|
||||
processTextChunk(afterChunk)
|
||||
continue
|
||||
}
|
||||
|
||||
if (child.type === 'html_inline') {
|
||||
const content = tokenToRaw(child)
|
||||
const tagMatch = content.match(TAG_NAME_AT_START_RE)
|
||||
const tagName = (tagMatch?.[1] ?? '').toLowerCase()
|
||||
if (tagName && tagSet.has(tagName) && findTagCloseIndexOutsideQuotes(content) === -1) {
|
||||
// markdown-it may prematurely close a tag at a ">" inside a quoted
|
||||
// attribute value (e.g. `<a href="...a>b`), producing a broken html_inline
|
||||
// token. Treat it as a streaming mid-state and swallow until we see a
|
||||
// real tag close ">" outside quotes.
|
||||
pending = {
|
||||
tag: tagName,
|
||||
buffer: content,
|
||||
closing: /^<\s*\//.test(content),
|
||||
}
|
||||
pendingAtEnd = pending.buffer
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if (child.type === 'text') {
|
||||
const content = String((child as any).content ?? '')
|
||||
if (!content.includes('<')) {
|
||||
out.push(child)
|
||||
continue
|
||||
}
|
||||
processTextChunk(content, child)
|
||||
continue
|
||||
}
|
||||
|
||||
out.push(child)
|
||||
}
|
||||
|
||||
return {
|
||||
children: out,
|
||||
pendingBuffer: pendingAtEnd ?? undefined,
|
||||
}
|
||||
}
|
||||
|
||||
export interface FixHtmlInlineOptions {
|
||||
/**
|
||||
* Custom HTML-like tag names that should participate in streaming
|
||||
* mid-state suppression and complete-tag splitting (e.g. ['thinking']).
|
||||
*/
|
||||
customHtmlTags?: readonly string[]
|
||||
}
|
||||
|
||||
export function applyFixHtmlInlineTokens(md: MarkdownIt, options: FixHtmlInlineOptions = {}) {
|
||||
const commonHtmlTags = buildCommonHtmlTagSet(options.customHtmlTags)
|
||||
// Tags that should stay inline when we auto-append a closing tag at core stage.
|
||||
const autoCloseInlineTagSet = new Set<string>([
|
||||
'a',
|
||||
'span',
|
||||
'strong',
|
||||
'em',
|
||||
'b',
|
||||
'i',
|
||||
'u',
|
||||
])
|
||||
const customTagSet = new Set<string>()
|
||||
if (options.customHtmlTags?.length) {
|
||||
for (const t of options.customHtmlTags) {
|
||||
const name = normalizeCustomHtmlTagName(t)
|
||||
if (!name)
|
||||
continue
|
||||
customTagSet.add(name)
|
||||
autoCloseInlineTagSet.add(name)
|
||||
}
|
||||
}
|
||||
const shouldMergeHtmlBlockTag = (tag: string) => customTagSet.has(tag) || !commonHtmlTags.has(tag) || BLOCK_LEVEL_HTML_TAGS.has(tag)
|
||||
const getHtmlBlockCarrierContent = (token: Token & { content?: string, children?: any[] }) => {
|
||||
if (token.type === 'html_block')
|
||||
return String(token.content ?? '')
|
||||
if (token.type !== 'inline' || !Array.isArray(token.children) || token.children.length !== 1)
|
||||
return ''
|
||||
const onlyChild = token.children[0]
|
||||
if (onlyChild?.type !== 'html_block')
|
||||
return ''
|
||||
return String(token.content ?? onlyChild.content ?? '')
|
||||
}
|
||||
const normalizeHtmlBlockCarrier = (
|
||||
token: Token & { content?: string, children?: any[], raw?: string },
|
||||
content: string,
|
||||
) => {
|
||||
token.type = 'html_block'
|
||||
token.content = content
|
||||
token.raw = content
|
||||
token.children = []
|
||||
}
|
||||
// Streaming mid-state: suppress partial inline HTML in text tokens until the
|
||||
// tag is fully closed with `>`, then allow it to be tokenized as html_inline.
|
||||
md.core.ruler.after('inline', 'fix_html_inline_streaming', (state: unknown) => {
|
||||
const s = state as unknown as { tokens?: Token[] }
|
||||
const toks = s.tokens ?? []
|
||||
for (const t of toks) {
|
||||
const tok = t as Token & { children?: Token[], content?: string, raw?: string }
|
||||
if (tok.type !== 'inline' || !Array.isArray(tok.children))
|
||||
continue
|
||||
|
||||
// markdown-it-ts may emit inline tokens with empty children when the
|
||||
// content starts with an incomplete HTML-ish fragment like "<span ...".
|
||||
// In that case, synthesize a text token so we can suppress mid-states.
|
||||
const originalContent = String(tok.content ?? '')
|
||||
const sourceChildren = tok.children.length
|
||||
? tok.children
|
||||
: (originalContent.includes('<')
|
||||
? [{ type: 'text', content: originalContent, raw: originalContent } as any]
|
||||
: null)
|
||||
|
||||
if (!sourceChildren)
|
||||
continue
|
||||
|
||||
try {
|
||||
const fixed = fixStreamingHtmlInlineChildren(sourceChildren, commonHtmlTags)
|
||||
tok.children = fixed.children
|
||||
if (fixed.pendingBuffer) {
|
||||
const idx = originalContent.lastIndexOf(fixed.pendingBuffer)
|
||||
if (idx !== -1) {
|
||||
const trimmed = originalContent.slice(0, idx)
|
||||
tok.content = trimmed
|
||||
// keep raw in sync if present
|
||||
if (typeof tok.raw === 'string')
|
||||
tok.raw = trimmed
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
console.error('[applyFixHtmlInlineTokens] failed to fix streaming html inline', e)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Fix certain single-token inline HTML cases by expanding into [openTag, text, closeTag]
|
||||
// This helps downstream inline parsers (e.g., <a>text</a>) to recognize inner text reliably.
|
||||
md.core.ruler.push('fix_html_inline_tokens', (state: unknown) => {
|
||||
const s = state as unknown as { tokens?: Token[] }
|
||||
const toks = s.tokens ?? []
|
||||
|
||||
// 有一些很特殊的场景，比如 html_block 开始 <thinking>，但是后面跟着很多段落,如果没匹配到</thinking>，中间的都应该合并为html_block的 content
|
||||
const tagStack: [string, number][] = []
|
||||
for (let i = 0; i < toks.length; i++) {
|
||||
const t = toks[i] as Token & { content?: string, children: any[] }
|
||||
|
||||
// If we're currently inside an unclosed custom-tag html_block, merge
|
||||
// everything (including other html_block tokens) into the opener until
|
||||
// the matching closing tag arrives.
|
||||
if (tagStack.length > 0) {
|
||||
const [openTag, openIndex] = tagStack[tagStack.length - 1]
|
||||
if (i !== openIndex) {
|
||||
// Remove structural paragraph wrappers that can appear in stream mode.
|
||||
if (t.type === 'paragraph_open' || t.type === 'paragraph_close') {
|
||||
toks.splice(i, 1)
|
||||
i--
|
||||
continue
|
||||
}
|
||||
|
||||
const chunk = String((t as any).content ?? (t as any).raw ?? '')
|
||||
|
||||
if (chunk) {
|
||||
const openToken = toks[openIndex] as Token & { content?: string, loading?: boolean }
|
||||
const mergedContent = `${String(openToken.content || '')}\n${chunk}`
|
||||
const openEnd = findTagCloseIndexOutsideQuotes(mergedContent)
|
||||
const closeRange = openEnd === -1
|
||||
? null
|
||||
: findMatchingCloseRangeInHtml(mergedContent, openTag, openEnd + 1)
|
||||
if (closeRange) {
|
||||
const before = mergedContent.slice(0, closeRange.end)
|
||||
const after = mergedContent.slice(closeRange.end)
|
||||
|
||||
openToken.content = before
|
||||
openToken.loading = false
|
||||
|
||||
const afterTrimmed = after.replace(/^\s+/, '')
|
||||
// Remove current token after merging.
|
||||
toks.splice(i, 1)
|
||||
// Close the stack before reinserting trailing content.
|
||||
tagStack.pop()
|
||||
if (afterTrimmed) {
|
||||
toks.splice(i, 0, afterTrimmed.startsWith('<')
|
||||
? ({ type: 'html_block', content: afterTrimmed } as any)
|
||||
: ({ type: 'inline', content: afterTrimmed, children: [{ type: 'text', content: afterTrimmed, raw: afterTrimmed }] } as any))
|
||||
}
|
||||
i--
|
||||
continue
|
||||
}
|
||||
|
||||
openToken.content = mergedContent
|
||||
if (openToken.loading !== false)
|
||||
openToken.loading = true
|
||||
}
|
||||
|
||||
// Remove current token after merging.
|
||||
toks.splice(i, 1)
|
||||
i--
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
const rawContent = getHtmlBlockCarrierContent(t)
|
||||
if (rawContent) {
|
||||
if (isNonElementHtmlBlock(rawContent))
|
||||
continue
|
||||
|
||||
// Support both opening (<tag ...>) and closing (</tag>) blocks.
|
||||
const tag = (rawContent.match(/<\s*(?:\/\s*)?([^\s>/]+)/)?.[1] ?? '').toLowerCase()
|
||||
const isClosingTag = /^\s*<\s*\//.test(rawContent)
|
||||
|
||||
// Merge configured custom tags and unknown HTML-like tags (e.g. <think>)
|
||||
// until their matching closing tag arrives. Also merge standard block
|
||||
// tags that markdown-it can split on blank lines, such as <details>.
|
||||
if (!tag || !shouldMergeHtmlBlockTag(tag))
|
||||
continue
|
||||
|
||||
normalizeHtmlBlockCarrier(t, rawContent)
|
||||
|
||||
if (!isClosingTag) {
|
||||
// 开始标签，入栈
|
||||
if (tag) {
|
||||
// If the html_block already contains its own closing tag, do NOT
|
||||
// push it onto the stack; otherwise we'd incorrectly merge the
|
||||
// following blocks into this html_block.
|
||||
const selfClosingRe = new RegExp(`^\\s*<\\s*${tag}\\b[^>]*\\/\\s*>`, 'i')
|
||||
if (!selfClosingRe.test(rawContent) && getTrailingCustomTagDepthInHtml(rawContent, tag) > 0)
|
||||
tagStack.push([tag, i])
|
||||
}
|
||||
}
|
||||
else {
|
||||
// 结束标签：如果匹配到栈顶，则把 closing token 也合并进 opener 并删除自己
|
||||
if (tagStack.length > 0 && tag && tagStack[tagStack.length - 1][0] === tag) {
|
||||
const [, openIndex] = tagStack[tagStack.length - 1]
|
||||
const openToken = toks[openIndex] as Token & { content?: string, loading?: boolean }
|
||||
openToken.content = `${String(openToken.content || '')}\n${rawContent}`
|
||||
openToken.loading = false
|
||||
tagStack.pop()
|
||||
|
||||
// Remove current closing html_block token so it doesn't become a stray node.
|
||||
toks.splice(i, 1)
|
||||
i--
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
else if (tagStack.length > 0) {
|
||||
// 如果在标签栈中，说明是未闭合标签的内容，合并到上一个 html_block
|
||||
if (t.type === 'paragraph_open' || t.type === 'paragraph_close') {
|
||||
// 应该删除这些标签
|
||||
toks.splice(i, 1)
|
||||
i-- // 调整索引
|
||||
continue
|
||||
}
|
||||
const content = t.content || ''
|
||||
const CLOSING_TAG_REGEX = new RegExp(`<\\s*\\/\\s*${tagStack[tagStack.length - 1][0]}\\s*>`, 'i')
|
||||
const isClosingTag = CLOSING_TAG_REGEX.test(content)
|
||||
|
||||
if (content) {
|
||||
// 插入到栈顶标签对应的 html_block 中
|
||||
const [, openIndex] = tagStack[tagStack.length - 1]
|
||||
const openToken = toks[openIndex] as Token & { content?: string, loading: boolean }
|
||||
openToken.content = `${openToken.content || ''}\n${content}`
|
||||
if (openToken.loading !== false)
|
||||
openToken.loading = !isClosingTag
|
||||
}
|
||||
if (isClosingTag) {
|
||||
tagStack.pop()
|
||||
}
|
||||
// 删除当前 token
|
||||
toks.splice(i, 1)
|
||||
i-- // 调整索引
|
||||
}
|
||||
else {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Some custom tags (e.g. <thinking>) can be tokenized by markdown-it into
|
||||
// multiple top-level inline tokens, with the closing tag arriving in a
|
||||
// later inline token. Our inline parser can only match closing tags within
|
||||
// the same inline token's children list, so we merge such sequences here.
|
||||
if (customTagSet.size > 0) {
|
||||
const openReCache = new Map<string, RegExp>()
|
||||
const closeReCache = new Map<string, RegExp>()
|
||||
const getOpenRe = (tag: string) => {
|
||||
let r = openReCache.get(tag)
|
||||
if (!r) {
|
||||
r = new RegExp(`<\\s*${tag}\\b`, 'i')
|
||||
openReCache.set(tag, r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
const getCloseRe = (tag: string) => {
|
||||
let r = closeReCache.get(tag)
|
||||
if (!r) {
|
||||
r = new RegExp(`<\\s*\\/\\s*${tag}\\s*>`, 'i')
|
||||
closeReCache.set(tag, r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
const stack: Array<{ tag: string, index: number }> = []
|
||||
for (let i = 0; i < toks.length; i++) {
|
||||
const tok = toks[i] as Token & { content?: string, children?: any[] }
|
||||
const content = String(tok.content ?? '')
|
||||
|
||||
// If we're inside an unclosed custom tag, we may need to close it even
|
||||
// if the closing tag is emitted as html_block (markdown-it can do this).
|
||||
if (stack.length > 0) {
|
||||
const top = stack[stack.length - 1]
|
||||
const openTok = toks[top.index] as Token & { content?: string, children?: any[] }
|
||||
|
||||
// Close via an html_block token like "</thinking>"
|
||||
if (tok.type === 'html_block' && getCloseRe(top.tag).test(content)) {
|
||||
openTok.content = `${String(openTok.content ?? '')}\n${content}`
|
||||
if (Array.isArray(openTok.children)) {
|
||||
openTok.children.push({
|
||||
type: 'html_inline',
|
||||
content: `</${top.tag}>`,
|
||||
raw: `</${top.tag}>`,
|
||||
} as any)
|
||||
}
|
||||
toks.splice(i, 1)
|
||||
i--
|
||||
stack.pop()
|
||||
continue
|
||||
}
|
||||
|
||||
// Only merge inline tokens; keep block structure intact.
|
||||
if (tok.type !== 'inline')
|
||||
continue
|
||||
|
||||
const children = Array.isArray(tok.children) ? tok.children : []
|
||||
const closeChildIndex = findMatchingCloseChildIndex(children, top.tag)
|
||||
|
||||
// If the closing tag is inside this inline token, merge up to it and
|
||||
// keep the trailing content as a new paragraph so it doesn't get
|
||||
// swallowed by the custom tag.
|
||||
if (closeChildIndex !== -1) {
|
||||
const beforeChildren = children.slice(0, closeChildIndex + 1)
|
||||
const afterChildren = children.slice(closeChildIndex + 1)
|
||||
|
||||
const beforeText = beforeChildren
|
||||
.map((c: any) => String(c?.content ?? c?.raw ?? ''))
|
||||
.join('')
|
||||
|
||||
// Only append the fragment up to and including the closing tag.
|
||||
openTok.content = `${String(openTok.content ?? '')}\n${beforeText}`
|
||||
if (Array.isArray(openTok.children))
|
||||
openTok.children.push(...beforeChildren)
|
||||
|
||||
// Replace current token with trailing content (if any)
|
||||
if (afterChildren.length) {
|
||||
const afterText = afterChildren.map((c: any) => String(c.content ?? c.raw ?? '')).join('')
|
||||
if (afterText.trim()) {
|
||||
const trimmed = afterText.replace(/^\s+/, '')
|
||||
if (trimmed.startsWith('<')) {
|
||||
toks.splice(i, 1, { type: 'html_block', content: trimmed } as any)
|
||||
}
|
||||
else {
|
||||
toks.splice(i, 1, { type: 'paragraph_open', tag: 'p', nesting: 1 } as any, { type: 'inline', tag: '', nesting: 0, content: afterText, children: [{ type: 'text', content: afterText, raw: afterText }] } as any, { type: 'paragraph_close', tag: 'p', nesting: -1 } as any)
|
||||
// current index now points at paragraph_open; move on
|
||||
}
|
||||
}
|
||||
else {
|
||||
toks.splice(i, 1)
|
||||
i--
|
||||
}
|
||||
}
|
||||
else {
|
||||
toks.splice(i, 1)
|
||||
i--
|
||||
}
|
||||
|
||||
stack.pop()
|
||||
continue
|
||||
}
|
||||
|
||||
// No closing tag: merge everything and remove current inline token.
|
||||
openTok.content = `${String(openTok.content ?? '')}\n${content}`
|
||||
if (Array.isArray(openTok.children))
|
||||
openTok.children.push(...children)
|
||||
toks.splice(i, 1)
|
||||
i--
|
||||
continue
|
||||
}
|
||||
|
||||
// Not inside: detect an opening custom tag that does not close within this token.
|
||||
if (tok.type !== 'inline')
|
||||
continue
|
||||
const children = Array.isArray(tok.children) ? tok.children : []
|
||||
for (const tag of customTagSet) {
|
||||
const trailingOpenDepth = children.length
|
||||
? getTrailingOpenDepth(children, tag)
|
||||
: (getOpenRe(tag).test(content) && !getCloseRe(tag).test(content) ? 1 : 0)
|
||||
if (trailingOpenDepth > 0) {
|
||||
stack.push({ tag, index: i })
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Defensive cleanup: some edge cases can end up with an orphan
|
||||
// paragraph_close token (without a matching paragraph_open) after
|
||||
// core-stage token mutations. Drop such invalid closes so downstream
|
||||
// consumers don't see stray paragraph_close.
|
||||
{
|
||||
let depth = 0
|
||||
for (let i = 0; i < toks.length; i++) {
|
||||
const t = toks[i] as Token
|
||||
if (t.type === 'paragraph_open') {
|
||||
depth++
|
||||
continue
|
||||
}
|
||||
if (t.type === 'paragraph_close') {
|
||||
if (depth > 0) {
|
||||
depth--
|
||||
}
|
||||
else {
|
||||
toks.splice(i, 1)
|
||||
i--
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < toks.length; i++) {
|
||||
const t = toks[i] as Token & { content?: string, children: any[], loading?: boolean }
|
||||
if (t.type === 'html_block') {
|
||||
const rawTag = t.content?.match(/<([^\s>/]+)/)?.[1] ?? ''
|
||||
const tag = rawTag.toLowerCase()
|
||||
|
||||
// Special tags: comments, doctypes, processing instructions
|
||||
// These should not be modified
|
||||
if (tag.startsWith('!') || tag.startsWith('?')) {
|
||||
t.loading = false
|
||||
continue
|
||||
}
|
||||
|
||||
// Custom tags (configured by user): keep as html_block
|
||||
// These are handled specially to support streaming and structured nodes
|
||||
if (customTagSet.has(tag)) {
|
||||
const raw = String(t.content ?? '')
|
||||
const openEnd = findTagCloseIndexOutsideQuotes(raw)
|
||||
const closeRange = openEnd === -1
|
||||
? null
|
||||
: findMatchingCloseRangeInHtml(raw, tag, openEnd + 1)
|
||||
const hasClose = !!closeRange
|
||||
t.loading = hasClose ? false : t.loading !== undefined ? t.loading : true
|
||||
|
||||
const endTagIndex = closeRange?.start ?? -1
|
||||
const closeLen = closeRange ? closeRange.end - closeRange.start : 0
|
||||
|
||||
if (endTagIndex !== -1) {
|
||||
// Found a closing tag - extract inner content and trim
|
||||
const rawForNode = raw.slice(0, endTagIndex + closeLen)
|
||||
let inner = ''
|
||||
if (openEnd !== -1 && openEnd < endTagIndex) {
|
||||
inner = raw.slice(openEnd + 1, endTagIndex)
|
||||
}
|
||||
|
||||
// Create structured node with inner content
|
||||
t.children = [
|
||||
{
|
||||
type: tag,
|
||||
content: inner,
|
||||
raw: rawForNode,
|
||||
attrs: [],
|
||||
tag,
|
||||
loading: false,
|
||||
},
|
||||
] as any[]
|
||||
|
||||
// Update token content
|
||||
t.content = rawForNode
|
||||
;(t as any).raw = rawForNode
|
||||
|
||||
// Insert trailing content as a new token if present
|
||||
const afterContent = raw.slice(endTagIndex + closeLen) || ''
|
||||
const afterTrimmed = afterContent.replace(/^\s+/, '')
|
||||
if (afterTrimmed) {
|
||||
toks.splice(i + 1, 0, afterTrimmed.startsWith('<')
|
||||
? ({ type: 'html_block', content: afterTrimmed } as any)
|
||||
: ({ type: 'text', content: afterTrimmed, raw: afterTrimmed } as any))
|
||||
}
|
||||
}
|
||||
else {
|
||||
// No closing tag yet (streaming mid-state)
|
||||
t.children = [
|
||||
{
|
||||
type: tag,
|
||||
content: '',
|
||||
raw,
|
||||
attrs: [],
|
||||
tag,
|
||||
loading: true,
|
||||
},
|
||||
] as any[]
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// Common block tags: skip conversion to inline
|
||||
if (['br', 'hr', 'img', 'input', 'link', 'meta', 'div', 'p', 'ul', 'li'].includes(tag))
|
||||
continue
|
||||
|
||||
// Other HTML tags: convert to inline type for better handling
|
||||
// Parse attributes and check for closing tag
|
||||
t.type = 'inline'
|
||||
const attrs: [string, string][] = []
|
||||
const attrRegex = /\s([\w:-]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g
|
||||
let match
|
||||
while ((match = attrRegex.exec(t.content || '')) !== null) {
|
||||
const attrName = match[1]
|
||||
const attrValue = match[2] || match[3] || match[4] || ''
|
||||
attrs.push([attrName, attrValue])
|
||||
}
|
||||
|
||||
const raw = String(t.content ?? '')
|
||||
const closeRe = new RegExp(`<\\/\\s*${tag}\\s*>`, 'i')
|
||||
const closeMatch = closeRe.exec(raw)
|
||||
const endTagIndex = closeMatch ? closeMatch.index : -1
|
||||
const closeLen = closeMatch ? closeMatch[0].length : 0
|
||||
|
||||
if (endTagIndex !== -1) {
|
||||
// Found a closing tag - split content
|
||||
const rawForNode = raw.slice(0, endTagIndex + closeLen)
|
||||
const afterContent = raw.slice(endTagIndex + closeLen) || ''
|
||||
const afterTrimmed = afterContent.replace(/^\s+/, '')
|
||||
|
||||
// Create html_block child node
|
||||
t.children = [
|
||||
{
|
||||
type: 'html_block',
|
||||
content: rawForNode,
|
||||
tag,
|
||||
loading: false,
|
||||
},
|
||||
] as any[]
|
||||
|
||||
// Update token content
|
||||
t.content = rawForNode
|
||||
;(t as any).raw = rawForNode
|
||||
|
||||
// Insert trailing content as a new token if present
|
||||
if (afterTrimmed) {
|
||||
toks.splice(i + 1, 0, afterTrimmed.startsWith('<')
|
||||
? ({ type: 'html_block', content: afterTrimmed } as any)
|
||||
: ({ type: 'text', content: afterTrimmed, raw: afterTrimmed } as any))
|
||||
}
|
||||
}
|
||||
else {
|
||||
// No closing tag - streaming mid-state
|
||||
t.children = [
|
||||
{
|
||||
type: 'html_block',
|
||||
content: t.content,
|
||||
tag,
|
||||
loading: true,
|
||||
},
|
||||
] as any[]
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (!t || t.type !== 'inline')
|
||||
continue
|
||||
|
||||
// 修复children 是单个 html_inline的场景
|
||||
if (t.children.length === 2 && t.children[0].type === 'html_inline') {
|
||||
// 补充一个闭合标签
|
||||
const rawTag = t.children[0].content?.match(/<([^\s>/]+)/)?.[1] ?? ''
|
||||
const tag = rawTag.toLowerCase()
|
||||
const second = t.children[1] as any
|
||||
const secondCloseTag = String(second?.content ?? '').match(/^<\s*\/\s*([^\s>]+)/)?.[1]?.toLowerCase() ?? ''
|
||||
// Already a complete open+close pair: don't append another closing tag.
|
||||
if (second?.type === 'html_inline' && secondCloseTag === tag)
|
||||
continue
|
||||
// 如果是常见的 inline标签（含用户自定义），则只追加结尾标签，否则转换成 html_block
|
||||
if (autoCloseInlineTagSet.has(tag)) {
|
||||
t.children[0].loading = true
|
||||
t.children[0].tag = tag
|
||||
t.children.push({
|
||||
type: 'html_inline',
|
||||
tag,
|
||||
loading: true,
|
||||
content: `</${tag}>`,
|
||||
} as any)
|
||||
}
|
||||
else {
|
||||
t.children = [
|
||||
{
|
||||
type: 'html_block',
|
||||
loading: true,
|
||||
tag,
|
||||
content: t.children[0].content + t.children[1].content,
|
||||
} as any,
|
||||
]
|
||||
}
|
||||
continue
|
||||
}
|
||||
else if (t.children.length === 3 && t.children[0].type === 'html_inline' && t.children[2].type === 'html_inline') {
|
||||
const rawTag = t.children[0].content?.match(/<([^\s>/]+)/)?.[1] ?? ''
|
||||
const tag = rawTag.toLowerCase()
|
||||
// 如果是常见的 inline标签（含用户自定义），则不处理，否则转换成 html_block
|
||||
if (autoCloseInlineTagSet.has(tag))
|
||||
continue
|
||||
t.children = [
|
||||
{
|
||||
type: 'html_block',
|
||||
loading: false,
|
||||
tag,
|
||||
content: t.children.map(ct => ct.content).join(''),
|
||||
} as any,
|
||||
]
|
||||
continue
|
||||
}
|
||||
// Only handle pathological cases where inline content is a single HTML-ish chunk
|
||||
if (!t.content?.startsWith('<') || (t as any).children?.length !== 1)
|
||||
continue
|
||||
|
||||
const raw = String(t.content)
|
||||
const htmlToken = t as unknown as { children: Array<{ type: string, content: string }> }
|
||||
const onlyChild = htmlToken.children[0] as { type?: string, content?: string } | undefined
|
||||
|
||||
// Keep literal text untouched (e.g. malformed "<robot=xxx>..."), but still
|
||||
// suppress dangling "<tag" mid-states to avoid streaming jitter.
|
||||
if (onlyChild?.type !== 'html_inline') {
|
||||
if (/^<\s*(?:\/\s*)?[A-Z][\w:-]*\s*$/i.test(raw))
|
||||
htmlToken.children.length = 0
|
||||
continue
|
||||
}
|
||||
|
||||
const strictTagName = String(onlyChild.content ?? raw).match(STRICT_OPEN_TAG_NAME_AT_START_RE)?.[1]?.toLowerCase() ?? ''
|
||||
if (!strictTagName)
|
||||
continue
|
||||
|
||||
const selfClosing = /\/\s*>\s*$/.test(raw)
|
||||
const isVoid = selfClosing || VOID_TAGS.has(strictTagName)
|
||||
|
||||
if (isVoid) {
|
||||
// For void/self-closing tags, keep a single html_inline token
|
||||
htmlToken.children = [
|
||||
{ type: 'html_inline', content: raw },
|
||||
] as any
|
||||
continue
|
||||
}
|
||||
htmlToken.children.length = 0
|
||||
}
|
||||
})
|
||||
}
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* Fix indented code blocks that should be paragraphs or inline HTML.
|
||||
*
|
||||
* In streaming scenarios, AI-generated content may have lines that are
|
||||
* indented with 4+ spaces (which markdown-it treats as code blocks per
|
||||
* CommonMark spec) but are actually plain text or HTML entities.
|
||||
*
|
||||
* This plugin runs at the core stage to convert single-line indented
|
||||
* code blocks that don't look like code into paragraphs.
|
||||
*/
|
||||
|
||||
import type { MarkdownIt, Token } from 'markdown-it-ts'
|
||||
|
||||
export interface FixIndentedCodeBlockOptions {
|
||||
/**
|
||||
* Whether to enable this fix. Default: true
|
||||
*/
|
||||
enabled?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a line looks like code (vs plain text or HTML entities).
|
||||
* Returns true if the line appears to be code.
|
||||
*/
|
||||
function looksLikeCode(line: string): boolean {
|
||||
const trimmed = line.trim()
|
||||
|
||||
// Empty line
|
||||
if (!trimmed)
|
||||
return false
|
||||
|
||||
// 1. HTML entities at start (e.g., ✖, , &)
|
||||
// These are typically plain text with character references, not code
|
||||
if (/^&[a-z0-9#]+;/i.test(trimmed))
|
||||
return false
|
||||
|
||||
// 2. Common programming keywords and JavaScript literals
|
||||
if (/^(?:const|let|var|function|class|import|export|if|for|while|return|await|async|yield|try|catch|throw|new|typeof|instanceof|switch|case|break|continue|def|ruby|perl|print|echo|true|false|null|undefined|NaN|Infinity|this)\b/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 3. Function/method calls with parentheses
|
||||
if (/[a-z_$][\w$]*(?:\.[a-z_$][\w$]*|\['[^']*'\]|\["[^"]*"\]|\[\d+\])*\s*\(/i.test(trimmed))
|
||||
return true
|
||||
|
||||
// 4. Property access with dot or bracket notation
|
||||
if (/[a-z_$][\w$]*(?:\.[a-z_$][\w$]*|\['[^']*'\]|\["[^"]*"\]|\[[\d+\]])+/i.test(trimmed))
|
||||
return true
|
||||
|
||||
// 5. Comparison and logical operators (but not just symbols alone)
|
||||
// Require identifiers around operators for better precision
|
||||
// Match: identifier operator identifier OR operator(s) identifier
|
||||
if (/\w+\s*(?:===?|!==?|<=?|>=?|\+\+|--|&&|\|\||\?\.)/.test(trimmed))
|
||||
return true
|
||||
// Double operators before identifier (like !!value) - but avoid matching list items
|
||||
if (/^(?:!!|\+\+|--)\s*\w/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 6. Assignment operators
|
||||
if (/[\w$]+\s*(?:\+=|-=|\*=|\/=|%=|\*\*=|=)/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 7. URLs, protocols
|
||||
if (/^(?:https?:\/\/|ftp:\/\/|file:\/\/|\/\/|www\.)/i.test(trimmed))
|
||||
return true
|
||||
|
||||
// 8. Template strings with interpolation
|
||||
if (/`[^`]*\$\{[^}]*\}[^`]*`/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 9. JSX/TSX tags (uppercase component names)
|
||||
if (/<\/?[A-Z][a-zA-Z0-9]*/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 10. HTML-like tags (lowercase with attributes)
|
||||
if (/<[a-z][a-z0-9]*\s[^>]+>/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 11. Quoted strings (single, double, backtick) - often code statements
|
||||
if (/^(["'`]).*\1\s*[;,]?$/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 12. Array/object/function patterns with brackets
|
||||
// Empty brackets [] {} () are common in code
|
||||
if (/^\[[\s\S]*\]$/.test(trimmed) || /^\{[\s\S]*\}$/.test(trimmed) || /^\(\s*\)$/.test(trimmed)) {
|
||||
return true
|
||||
}
|
||||
|
||||
// 13. Code-like expressions with identifiers and operators
|
||||
// e.g., a + b, x > y, count !== 0
|
||||
// This also catches property access like foo.bar
|
||||
// Must have at least one identifier (letter/digit/underscore) on each side of operator
|
||||
if (/[\w$]+(?:\s*[+\-*/%<>=!&|^~:]+\s*[\w$]+|\s*\.\s*[\w$]+)/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 14. Arrow functions, fat arrows
|
||||
if (/=>|->|::/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 15. Decorators and annotations
|
||||
if (/^@[\w.$]+$/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 16. Numbers (hex, binary, octal, decimal with units)
|
||||
if (/^(?:0x[0-9a-fA-F]+|0b[01]+|0o[0-7]+|\d+(?:\.\d*)?(?:px|em|rem|%|vh|vw|deg|s|ms)?)$/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 17. Variable declarations with $ (common in jQuery/shell)
|
||||
if (/^\$[\w$]+\s*[=:]/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 18. Pipes and special operators
|
||||
if (/\|\s*\w+|\w+\s*\|/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 19. Command-like patterns (git, npm, etc.)
|
||||
if (/^(?:git|npm|yarn|pnpm|bun|pip|cargo|go|rust|python|node|java|mvn|gradle|docker|kubectl)\s+/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 20. Console methods and common API calls
|
||||
if (/(?:console|window|document|Math|JSON|Date|Array|Object|String|Number|Boolean)\.[a-zA-Z]/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 21. Comment patterns
|
||||
if (/^(?:\/\/|#|\/\*|\*\/|<!--|-->)/.test(trimmed))
|
||||
return true
|
||||
|
||||
// 22. Heredoc patterns
|
||||
if (/^(?:<<<|<<\s*['"]?\w+['"]?)/.test(trimmed))
|
||||
return true
|
||||
|
||||
// Default: doesn't look like code
|
||||
return false
|
||||
}
|
||||
|
||||
export function applyFixIndentedCodeBlock(md: MarkdownIt, options: FixIndentedCodeBlockOptions = {}) {
|
||||
if (options.enabled === false)
|
||||
return
|
||||
|
||||
// Process at the core stage after tokenization is complete
|
||||
md.core.ruler.after('inline', 'fix_indented_code_block', (state: any) => {
|
||||
const tokens = state.tokens as Token[]
|
||||
|
||||
for (let i = 0; i < tokens.length; i++) {
|
||||
const token = tokens[i]
|
||||
|
||||
// Look for single-line code_block tokens
|
||||
if (token.type !== 'code_block')
|
||||
continue
|
||||
|
||||
const content = String((token as any).content ?? '').trim()
|
||||
if (!content)
|
||||
continue
|
||||
|
||||
const lines = content.split(/\r?\n/).filter(line => line.trim().length > 0)
|
||||
|
||||
// Only convert single-line code blocks that don't look like code
|
||||
if (lines.length === 1 && !looksLikeCode(lines[0] ?? '')) {
|
||||
// Convert code_block to paragraph
|
||||
const textContent = lines[0] ?? ''
|
||||
|
||||
// Replace code_block token with paragraph tokens
|
||||
tokens.splice(i, 1, { type: 'paragraph_open', tag: 'p', nesting: 1, level: token.level } as any, {
|
||||
type: 'inline',
|
||||
tag: '',
|
||||
nesting: 0,
|
||||
level: token.level,
|
||||
content: textContent,
|
||||
children: [{ type: 'text', content: textContent, level: token.level + 1 }] as any,
|
||||
block: true,
|
||||
} as any, { type: 'paragraph_close', tag: 'p', nesting: -1, level: token.level } as any)
|
||||
// Skip the newly inserted tokens
|
||||
i += 2
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,653 @@
|
||||
import type { MarkdownIt } from 'markdown-it-ts'
|
||||
import type { MarkdownToken } from '../types'
|
||||
import { shouldDemoteFilenameLikeLinkify } from '../parser/linkifyHeuristics'
|
||||
|
||||
// We hard-stop FULLWIDTH exclamation mark used as CJK punctuation.
|
||||
// ASCII `!` is valid in URLs (path/query/fragment), so do not stop on it.
|
||||
const LINKIFY_HARD_STOP_CHARS = ['！'] as const
|
||||
|
||||
// Small helpers to reduce repetition when building token fragments
|
||||
function textToken(content: string) {
|
||||
return {
|
||||
type: 'text',
|
||||
content,
|
||||
raw: content,
|
||||
}
|
||||
}
|
||||
|
||||
function pushEmOpen(arr: any[], type: number) {
|
||||
if (type === 1) {
|
||||
arr.push({ type: 'em_open', tag: 'em', nesting: 1 })
|
||||
}
|
||||
else if (type === 2) {
|
||||
arr.push({ type: 'strong_open', tag: 'strong', nesting: 1 })
|
||||
}
|
||||
else if (type === 3) {
|
||||
arr.push({ type: 'strong_open', tag: 'strong', nesting: 1 })
|
||||
arr.push({ type: 'em_open', tag: 'em', nesting: 1 })
|
||||
}
|
||||
}
|
||||
|
||||
function pushEmClose(arr: any[], type: number) {
|
||||
if (type === 1) {
|
||||
arr.push({ type: 'em_close', tag: 'em', nesting: -1 })
|
||||
}
|
||||
else if (type === 2) {
|
||||
arr.push({ type: 'strong_close', tag: 'strong', nesting: -1 })
|
||||
}
|
||||
else if (type === 3) {
|
||||
arr.push({ type: 'em_close', tag: 'em', nesting: -1 })
|
||||
arr.push({ type: 'strong_close', tag: 'strong', nesting: -1 })
|
||||
}
|
||||
}
|
||||
|
||||
function createLinkToken(text: string, href: string, loading: boolean) {
|
||||
let title = ''
|
||||
if (href.includes('"')) {
|
||||
const temps = href.split('"')
|
||||
href = temps[0].trim()
|
||||
title = temps[1].trim()
|
||||
}
|
||||
return {
|
||||
type: 'link',
|
||||
loading,
|
||||
href,
|
||||
title,
|
||||
text,
|
||||
children: [
|
||||
{
|
||||
type: 'text',
|
||||
content: text,
|
||||
raw: text,
|
||||
},
|
||||
],
|
||||
raw: String(`[${text}](${href})`),
|
||||
}
|
||||
}
|
||||
|
||||
function appendToLinkToken(link: any, suffix: string) {
|
||||
if (!link || !suffix)
|
||||
return
|
||||
link.href = String(link.href ?? '') + suffix
|
||||
link.text = String(link.text ?? '') + suffix
|
||||
link.raw = String(`[${link.text}](${link.href})`)
|
||||
if (Array.isArray(link.children) && link.children.length) {
|
||||
const last = link.children[link.children.length - 1]
|
||||
if (last?.type === 'text') {
|
||||
last.content = String(last.content ?? '') + suffix
|
||||
last.raw = String(last.raw ?? '') + suffix
|
||||
}
|
||||
else {
|
||||
link.children.push(textToken(suffix))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function firstIndexOfAny(input: string, chars: readonly string[]) {
|
||||
let first = -1
|
||||
for (const ch of chars) {
|
||||
const idx = input.indexOf(ch)
|
||||
if (idx !== -1 && (first === -1 || idx < first))
|
||||
first = idx
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
function getHrefFromLinkOpen(token: any) {
|
||||
const href = token?.attrs?.find((attr: any) => attr?.[0] === 'href')?.[1]
|
||||
return typeof href === 'string' ? href : ''
|
||||
}
|
||||
|
||||
function setHrefOnLinkOpen(token: any, href: string) {
|
||||
if (!token)
|
||||
return
|
||||
token.attrs = Array.isArray(token.attrs) ? token.attrs : []
|
||||
const idx = token.attrs.findIndex((attr: any) => attr?.[0] === 'href')
|
||||
if (idx >= 0)
|
||||
token.attrs[idx][1] = href
|
||||
else
|
||||
token.attrs.push(['href', href])
|
||||
}
|
||||
|
||||
function collectLinkifyText(tokens: MarkdownToken[], openIndex: number, closeIndex: number) {
|
||||
let text = ''
|
||||
for (let index = openIndex + 1; index < closeIndex; index++) {
|
||||
const token = tokens[index]
|
||||
if (token?.type !== 'text' || typeof token.content !== 'string')
|
||||
return null
|
||||
text += token.content
|
||||
}
|
||||
return text || null
|
||||
}
|
||||
|
||||
export function applyFixLinkTokens(md: MarkdownIt) {
|
||||
// Run after the inline rule so markdown-it has produced inline tokens
|
||||
// for block-level tokens; we then adjust each inline token's children
|
||||
// so downstream code receives corrected token arrays during the same
|
||||
// parsing pass.
|
||||
md.core.ruler.after('inline', 'fix_link_tokens', (state: unknown) => {
|
||||
const s = state as unknown as { tokens?: Array<{ type?: string, children?: any[] }> }
|
||||
const toks = s.tokens ?? []
|
||||
for (let i = 0; i < toks.length; i++) {
|
||||
const t = toks[i]
|
||||
if (t && t.type === 'inline' && Array.isArray(t.children)) {
|
||||
try {
|
||||
t.children = fixLinkToken(t.children)
|
||||
}
|
||||
catch (e) {
|
||||
// Swallow errors to avoid breaking parsing; keep original children
|
||||
// so parse still succeeds even if our fix fails for an unexpected shape.
|
||||
// Errors should be rare and indicate malformed token arrays.
|
||||
|
||||
console.error('[applyFixLinkTokens] failed to fix inline children', e)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function fixLinkToken(tokens: MarkdownToken[]): MarkdownToken[] {
|
||||
// Need at least `link_open + text + link_close` for linkify/autolink fixes.
|
||||
// Keep this low to allow fixing `<https://...！suffix>` cases where inline children length is 3.
|
||||
if (tokens.length < 3)
|
||||
return tokens
|
||||
|
||||
// 如果包含 code_inline 类型的 token，说明是包含内联代码的链接，直接返回原样，避免错误处理
|
||||
if (tokens.some(token => token.type === 'code_inline'))
|
||||
return tokens
|
||||
|
||||
for (let i = 0; i <= tokens.length - 1; i++) {
|
||||
if (i < 0) {
|
||||
i = 0
|
||||
}
|
||||
const curToken = tokens[i]
|
||||
if (!curToken)
|
||||
break
|
||||
// Hard-stop certain punctuation for linkify-generated bare URLs.
|
||||
// Example: `https://a.com/x.png！文字` should become link `https://a.com/x.png` + text `！文字`.
|
||||
if (curToken.type === 'link_open' && (curToken.markup === 'linkify' || curToken.markup === 'autolink')) {
|
||||
let closeIdx = -1
|
||||
for (let j = i + 1; j < tokens.length; j++) {
|
||||
if (tokens[j]?.type === 'link_close') {
|
||||
closeIdx = j
|
||||
break
|
||||
}
|
||||
}
|
||||
if (closeIdx !== -1) {
|
||||
const linkText = collectLinkifyText(tokens, i, closeIdx)
|
||||
if (
|
||||
curToken.markup === 'linkify'
|
||||
&& linkText
|
||||
&& shouldDemoteFilenameLikeLinkify(linkText)
|
||||
) {
|
||||
tokens.splice(i, closeIdx - i + 1, textToken(linkText) as any)
|
||||
continue
|
||||
}
|
||||
|
||||
const href = getHrefFromLinkOpen(curToken as any)
|
||||
const hrefStop = firstIndexOfAny(href, LINKIFY_HARD_STOP_CHARS)
|
||||
// Prefer splitting by the visible text token, but also trim href if it contains stop chars.
|
||||
for (let j = i + 1; j < closeIdx; j++) {
|
||||
const t = tokens[j]
|
||||
if (t?.type !== 'text' || typeof t.content !== 'string')
|
||||
continue
|
||||
const stopAt = firstIndexOfAny(t.content, LINKIFY_HARD_STOP_CHARS)
|
||||
if (stopAt === -1)
|
||||
continue
|
||||
|
||||
const stopChar = t.content[stopAt]
|
||||
const before = t.content.slice(0, stopAt)
|
||||
let tail = t.content.slice(stopAt)
|
||||
// Move any remaining text tokens that were inside the linkify span to the tail.
|
||||
for (let k = j + 1; k < closeIdx; k++) {
|
||||
const tk = tokens[k]
|
||||
if (tk?.type === 'text' && typeof tk.content === 'string')
|
||||
tail += tk.content
|
||||
}
|
||||
|
||||
t.content = before
|
||||
;(t as any).raw = before
|
||||
|
||||
const removeCount = closeIdx - (j + 1)
|
||||
if (removeCount > 0) {
|
||||
tokens.splice(j + 1, removeCount)
|
||||
closeIdx = j + 1
|
||||
}
|
||||
|
||||
let newHref = href
|
||||
if (hrefStop !== -1) {
|
||||
newHref = href.slice(0, hrefStop)
|
||||
}
|
||||
else if (tail) {
|
||||
// linkify-it may percent-encode non-ASCII, so use encoded tail / stop-char to locate split point.
|
||||
const encodedTail = encodeURI(tail)
|
||||
if (encodedTail && href.endsWith(encodedTail)) {
|
||||
newHref = href.slice(0, href.length - encodedTail.length)
|
||||
}
|
||||
else {
|
||||
const encodedStop = stopChar ? encodeURI(stopChar) : ''
|
||||
const idx = encodedStop ? href.indexOf(encodedStop) : -1
|
||||
if (idx !== -1)
|
||||
newHref = href.slice(0, idx)
|
||||
}
|
||||
}
|
||||
if (newHref !== href)
|
||||
setHrefOnLinkOpen(curToken as any, newHref)
|
||||
|
||||
if (tail) {
|
||||
tokens.splice(closeIdx + 1, 0, textToken(tail) as any)
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if (curToken?.type === 'em_open' && tokens[i - 1]?.type === 'text' && tokens[i - 1].content?.endsWith('*')) {
|
||||
const beforeText = tokens[i - 1].content?.replace(/(\*+)$/, '') || ''
|
||||
tokens[i - 1].content = beforeText
|
||||
// 修改当前 type 'em_open' -> 'strong_open'
|
||||
curToken.type = 'strong_open'
|
||||
curToken.tag = 'strong'
|
||||
curToken.markup = '**'
|
||||
// 还需要把对应的 em_close 也修改为 strong_close
|
||||
for (let j = i + 1; j < tokens.length; j++) {
|
||||
if (tokens[j]?.type === 'em_close') {
|
||||
tokens[j].type = 'strong_close'
|
||||
tokens[j].tag = 'strong'
|
||||
tokens[j].markup = '**'
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (curToken?.type === 'text' && curToken.content?.endsWith('(') && tokens[i + 1]?.type === 'link_open') {
|
||||
const match = curToken.content!.match(/\[([^\]]+)\]/)
|
||||
if (match) {
|
||||
let beforeText = curToken.content!.slice(0, match.index)
|
||||
const emphasisMatch = beforeText.match(/(\*+)$/)
|
||||
const replacerTokens = []
|
||||
if (emphasisMatch) {
|
||||
beforeText = beforeText.slice(0, emphasisMatch.index)
|
||||
if (beforeText)
|
||||
replacerTokens.push(textToken(beforeText))
|
||||
const text = match[1]
|
||||
const type = emphasisMatch[1].length
|
||||
pushEmOpen(replacerTokens, type)
|
||||
let href = tokens[i + 2]?.content || ''
|
||||
if (tokens[i + 4]?.type === 'text' && !tokens[i + 4].content?.startsWith(')')) {
|
||||
href += tokens[i + 4]?.content || ''
|
||||
tokens[i + 4].content = ''
|
||||
}
|
||||
replacerTokens.push(createLinkToken(text, href, !tokens[i + 4]?.content?.startsWith(')')))
|
||||
pushEmClose(replacerTokens, type)
|
||||
if (tokens[i + 4]?.type === 'text') {
|
||||
const afterText = tokens[i + 4].content?.replace(/^\)\**/, '')
|
||||
if (afterText)
|
||||
replacerTokens.push(textToken(afterText))
|
||||
tokens.splice(i, 5, ...replacerTokens)
|
||||
}
|
||||
else {
|
||||
tokens.splice(i, 4, ...replacerTokens)
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (beforeText)
|
||||
replacerTokens.push(textToken(beforeText))
|
||||
let text = match[1]
|
||||
const emphasisMatch = text.match(/^\*+/)
|
||||
if (emphasisMatch) {
|
||||
const type = emphasisMatch[0].length
|
||||
text = text.replace(/^\*+/, '').replace(/\*+$/, '')
|
||||
// Put emphasis inside the link children when the asterisks are inside the brackets
|
||||
// so link nodes will contain the emphasis node as a child.
|
||||
let href = tokens[i + 2]?.content || ''
|
||||
if (tokens[i + 4]?.type === 'text' && !tokens[i + 4].content?.startsWith(')')) {
|
||||
href += tokens[i + 4]?.content || ''
|
||||
tokens[i + 4].content = ''
|
||||
}
|
||||
// wrap the link with emphasis open/close tokens
|
||||
pushEmOpen(replacerTokens, type)
|
||||
replacerTokens.push(createLinkToken(text, href, !tokens[i + 4]?.content?.startsWith(')')))
|
||||
pushEmClose(replacerTokens, type)
|
||||
// we've already pushed the link, skip the standard push below
|
||||
if (tokens[i + 4]?.type === 'text') {
|
||||
const afterText = tokens[i + 4].content?.replace(/^\)/, '')
|
||||
if (afterText)
|
||||
replacerTokens.push(textToken(afterText))
|
||||
tokens.splice(i, 5, ...replacerTokens)
|
||||
}
|
||||
else {
|
||||
tokens.splice(i, 4, ...replacerTokens)
|
||||
}
|
||||
if (i === 0) {
|
||||
i = replacerTokens.length - 1
|
||||
}
|
||||
else {
|
||||
i -= (replacerTokens.length + 1)
|
||||
}
|
||||
continue
|
||||
}
|
||||
let href = tokens[i + 2]?.content || ''
|
||||
if (tokens[i + 4]?.type === 'text' && !tokens[i + 4].content?.startsWith(')')) {
|
||||
href += tokens[i + 4]?.content || ''
|
||||
tokens[i + 4].content = ''
|
||||
}
|
||||
replacerTokens.push(createLinkToken(text, href, !tokens[i + 4]?.content?.startsWith(')')))
|
||||
if (tokens[i + 4]?.type === 'text') {
|
||||
const afterText = tokens[i + 4].content?.replace(/^\)/, '')
|
||||
if (afterText)
|
||||
replacerTokens.push(textToken(afterText))
|
||||
tokens.splice(i, 5, ...replacerTokens)
|
||||
}
|
||||
else {
|
||||
tokens.splice(i, 4, ...replacerTokens)
|
||||
}
|
||||
}
|
||||
i -= (replacerTokens.length + 1)
|
||||
continue
|
||||
}
|
||||
}
|
||||
else if (curToken.type === 'link_open' && curToken.markup === 'linkify' && tokens[i - 1]?.type === 'text' && tokens[i - 1].content?.endsWith('(')) {
|
||||
if (tokens[i - 2]?.type === 'link_close') {
|
||||
// 合并link
|
||||
const replacerTokens = []
|
||||
const text = (tokens[i - 3].content || '')
|
||||
let href = curToken.attrs?.find(attr => attr[0] === 'href')?.[1] || ''
|
||||
|
||||
if (tokens[i + 3]?.type === 'text') {
|
||||
const m = (tokens[i + 3]?.content ?? '').indexOf(')')
|
||||
const loading = m === -1
|
||||
if (m === -1) {
|
||||
href += (tokens[i + 3]?.content?.slice(0, m) || '')
|
||||
tokens[i + 3].content = ''
|
||||
}
|
||||
|
||||
replacerTokens.push(createLinkToken(text, href, loading))
|
||||
const afterText = tokens[i + 3].content?.replace(/^\)\**/, '')
|
||||
if (afterText)
|
||||
replacerTokens.push(textToken(afterText))
|
||||
tokens.splice(i - 4, 8, ...replacerTokens)
|
||||
}
|
||||
else {
|
||||
replacerTokens.push({
|
||||
type: 'link',
|
||||
loading: true,
|
||||
href,
|
||||
title: '',
|
||||
text,
|
||||
children: [
|
||||
{
|
||||
type: 'text',
|
||||
content: href,
|
||||
raw: href,
|
||||
},
|
||||
],
|
||||
raw: String(`[${text}](${href})`),
|
||||
})
|
||||
tokens.splice(i - 4, 7, ...replacerTokens)
|
||||
}
|
||||
continue
|
||||
}
|
||||
else if (tokens[i - 1].content === '](' && tokens[i - 3]?.type === 'text' && tokens[i - 3].content?.endsWith(')')) {
|
||||
// 移除 i-3 的 link部分，合并到 link_open 下的 text中
|
||||
if (tokens[i - 2]?.type === 'strong_open') {
|
||||
const [beforeText, linText] = tokens[i - 3].content?.split('[**') || []
|
||||
tokens[i + 1].content = linText || ''
|
||||
tokens[i - 3].content = beforeText || ''
|
||||
tokens[i - 1].content = ''
|
||||
}
|
||||
else if (tokens[i - 2]?.type === 'em_open') {
|
||||
const [beforeText, linText] = tokens[i - 3].content?.split('[*') || []
|
||||
tokens[i + 1].content = linText || ''
|
||||
tokens[i - 3].content = beforeText || ''
|
||||
tokens[i - 1].content = ''
|
||||
}
|
||||
else {
|
||||
const [beforeText, linText] = tokens[i - 3].content?.split('[') || []
|
||||
tokens[i + 1].content = linText || ''
|
||||
tokens[i - 3].content = beforeText || ''
|
||||
tokens[i - 1].content = ''
|
||||
}
|
||||
}
|
||||
}
|
||||
if (
|
||||
curToken.type === 'link_close'
|
||||
&& curToken.nesting === -1
|
||||
&& tokens[i - 2]?.type === 'link_open'
|
||||
&& tokens[i + 1]?.type === 'text'
|
||||
&& tokens[i - 1]?.type === 'text'
|
||||
) {
|
||||
// 修复链接后多余文本被包含在链接内的问题
|
||||
const text = tokens[i - 1].content || ''
|
||||
const attrs = tokens[i - 2].attrs || []
|
||||
const href = attrs.find(a => a[0] === 'href')?.[1] || ''
|
||||
const title = attrs.find(a => a[0] === 'title')?.[1] || ''
|
||||
let count = 3
|
||||
let deleteCount = 2
|
||||
const beforeText = tokens[i - 3]?.content || ''
|
||||
const emphasisMatch = beforeText.match(/^(\*+)$/)
|
||||
const replacerTokens: any[] = []
|
||||
if (emphasisMatch) {
|
||||
deleteCount += 1
|
||||
const type = emphasisMatch[1].length
|
||||
pushEmOpen(replacerTokens, type)
|
||||
}
|
||||
|
||||
if (curToken.markup !== 'linkify' && tokens[i + 1].type === 'text' && tokens[i + 1]?.content?.startsWith('](')) {
|
||||
count += 1
|
||||
for (let j = i + 1; j < tokens.length; j++) {
|
||||
const type = emphasisMatch ? emphasisMatch[1].length : tokens[i - 3].markup!.length
|
||||
const t = tokens[j]
|
||||
if (type === 1 && t.type === 'em_close') {
|
||||
break
|
||||
}
|
||||
else if (type === 2 && t.type === 'strong_close') {
|
||||
break
|
||||
}
|
||||
else if (type === 3) {
|
||||
if (t.type === 'em_close' || t.type === 'strong_close') {
|
||||
break
|
||||
}
|
||||
}
|
||||
count += 1
|
||||
}
|
||||
}
|
||||
|
||||
replacerTokens.push({
|
||||
type: 'link',
|
||||
loading: false,
|
||||
href,
|
||||
title,
|
||||
text,
|
||||
children: [
|
||||
{
|
||||
type: 'text',
|
||||
content: text,
|
||||
raw: text,
|
||||
},
|
||||
],
|
||||
raw: String(`[${text}](${href})`),
|
||||
} as any)
|
||||
if (emphasisMatch) {
|
||||
const type = emphasisMatch[1].length
|
||||
pushEmClose(replacerTokens, type)
|
||||
}
|
||||
tokens.splice(i - deleteCount, count, ...replacerTokens)
|
||||
i -= (replacerTokens.length + 1)
|
||||
continue
|
||||
}
|
||||
else if (curToken.content?.startsWith('](') && tokens[i - 1].markup?.includes('*') && tokens[i - 4]?.type === 'text' && tokens[i - 4].content?.endsWith('[')) {
|
||||
const type = tokens[i - 1].markup!.length
|
||||
const replacerTokens = []
|
||||
const beforeText = tokens[i - 4].content!.slice(0, tokens[i - 4].content!.length - type)
|
||||
if (beforeText)
|
||||
replacerTokens.push(textToken(beforeText))
|
||||
pushEmOpen(replacerTokens, type)
|
||||
const text = tokens[i - 2].content || ''
|
||||
let href = curToken.content!.slice(2)
|
||||
let loading = true
|
||||
if (tokens[i + 1]?.type === 'text') {
|
||||
const m = (tokens[i + 1]?.content ?? '').indexOf(')')
|
||||
loading = m === -1
|
||||
if (m === -1) {
|
||||
href += (tokens[i + 1]?.content?.slice(0, m) || '')
|
||||
tokens[i + 1].content = ''
|
||||
}
|
||||
}
|
||||
replacerTokens.push(createLinkToken(text, href, loading))
|
||||
pushEmClose(replacerTokens, type)
|
||||
if (tokens[i + 1]?.type === 'text') {
|
||||
const afterText = tokens[i + 1].content?.replace(/^\)\**/, '')
|
||||
if (afterText)
|
||||
replacerTokens.push(textToken(afterText))
|
||||
tokens.splice(i - 4, 8, ...replacerTokens)
|
||||
}
|
||||
else if (tokens[i + 1]?.type === 'link_open') {
|
||||
// 特殊情况其实要把href也处理，这里可以直接跳过
|
||||
tokens.splice(i - 4, 10, ...replacerTokens)
|
||||
}
|
||||
else {
|
||||
tokens.splice(i - 4, 7, ...replacerTokens)
|
||||
}
|
||||
i -= (replacerTokens.length + 1)
|
||||
continue
|
||||
}
|
||||
else if (curToken.content?.startsWith('](') && tokens[i - 1].type === 'strong_close' && tokens[i - 4]?.type === 'text' && tokens[i - 4]?.content?.includes('**[')) {
|
||||
// 此时的场景是 link 被 strong 包裹，link 中又包含了强调符号
|
||||
const replacerTokens = []
|
||||
const beforeText = tokens[i - 4].content!.split('**[')[0]
|
||||
if (beforeText)
|
||||
replacerTokens.push(textToken(beforeText))
|
||||
pushEmOpen(replacerTokens, 2)
|
||||
const text = tokens[i - 2].content || ''
|
||||
let href = curToken.content!.slice(2)
|
||||
let loading = true
|
||||
if (tokens[i + 1]?.type === 'text') {
|
||||
const m = (tokens[i + 1]?.content ?? '').indexOf(')')
|
||||
loading = m === -1
|
||||
if (m === -1) {
|
||||
href += (tokens[i + 1]?.content?.slice(0, m) || '')
|
||||
tokens[i + 1].content = ''
|
||||
}
|
||||
}
|
||||
replacerTokens.push(createLinkToken(text, href, loading))
|
||||
pushEmClose(replacerTokens, 2)
|
||||
if (tokens[i + 1]?.type === 'text') {
|
||||
const afterText = tokens[i + 1].content?.replace(/^\)\**/, '')
|
||||
if (afterText)
|
||||
replacerTokens.push(textToken(afterText))
|
||||
tokens.splice(i - 4, 8, ...replacerTokens)
|
||||
}
|
||||
else if (tokens[i + 1]?.type === 'link_open') {
|
||||
// 特殊情况其实要把href也处理，这里可以直接跳过
|
||||
tokens.splice(i - 4, 10, ...replacerTokens)
|
||||
}
|
||||
else {
|
||||
tokens.splice(i - 4, 7, ...replacerTokens)
|
||||
}
|
||||
i -= (replacerTokens.length + 1)
|
||||
continue
|
||||
}
|
||||
else if (curToken.type === 'strong_close' && tokens[i + 1]?.type === 'text' && tokens[i + 1].content?.includes('](') && tokens[i - 1].type === 'text' && /\[.*$/.test(tokens[i - 1].content || '')) {
|
||||
const replacerTokens = []
|
||||
const [beforeText, afterText] = tokens[i - 1].content?.split('[') || ['', '']
|
||||
if (beforeText)
|
||||
replacerTokens.push(textToken(beforeText))
|
||||
pushEmOpen(replacerTokens, 2)
|
||||
let [text, href] = tokens[i + 1].content!.split('](')
|
||||
text = afterText + text
|
||||
let deleteCount = 4
|
||||
if (tokens[i + 2]?.type === 'link_open') {
|
||||
const _href = tokens[i + 2].attrs?.find(a => a[0] === 'href')?.[1]
|
||||
if (tokens[i + 5]?.type === 'text' && tokens[i + 5].content === '.') {
|
||||
href = (_href || href) + tokens[i + 5].content
|
||||
tokens[i + 5].content = ''
|
||||
}
|
||||
else {
|
||||
href = _href || href
|
||||
}
|
||||
|
||||
deleteCount += 3
|
||||
}
|
||||
let loading = true
|
||||
if (curToken.nesting === -1) {
|
||||
// 嵌套 strong_close，需要去掉尾部的 **
|
||||
text = text.replace(/\*+$/, '')
|
||||
}
|
||||
if (tokens[i + 2]?.type === 'text') {
|
||||
const m = (tokens[i + 2]?.content ?? '').indexOf(')')
|
||||
loading = m === -1
|
||||
if (m === -1) {
|
||||
href += (tokens[i + 2]?.content?.slice(0, m) || '')
|
||||
tokens[i + 2].content = ''
|
||||
}
|
||||
}
|
||||
replacerTokens.push(createLinkToken(text, href, loading))
|
||||
pushEmClose(replacerTokens, 2)
|
||||
tokens.splice(i - 2, deleteCount, ...replacerTokens)
|
||||
}
|
||||
// 处理强调 + 链接拆分：首个 text 含前导星号与 '['，后续出现独立的 "]("、link_open、href、link_close、")"、strong_close。
|
||||
// 期望合并为：strong_open + 复合 link(label 尾部补回星号) + strong_close。
|
||||
if (
|
||||
curToken.type === 'text'
|
||||
&& /\*+\[[^\]]*$/.test(curToken.content || '')
|
||||
&& tokens[i + 1]?.type === 'strong_open'
|
||||
&& tokens[i + 2]?.type === 'text' && tokens[i + 2].content === ']('
|
||||
&& tokens[i + 3]?.type === 'link_open'
|
||||
&& tokens[i + 5]?.type === 'link_close'
|
||||
&& tokens[i + 6]?.type === 'text' && tokens[i + 6].content === ')'
|
||||
&& tokens[i + 7]?.type === 'strong_close'
|
||||
) {
|
||||
const contentVal = curToken.content || ''
|
||||
const startMatch = contentVal.match(/^(\*+)\[(.*)$/)
|
||||
if (startMatch) {
|
||||
const innerLabel = startMatch[2] || ''
|
||||
// 将前导星号数量追加到标签末尾
|
||||
const finalLabel = innerLabel + startMatch[1]
|
||||
// 获取 href：优先使用 link_open 的属性，若为空则回退到后续文本 token 内容
|
||||
let href = tokens[i + 3]?.attrs?.find(a => a[0] === 'href')?.[1] || ''
|
||||
if (!href && tokens[i + 4]?.type === 'text') {
|
||||
href = tokens[i + 4].content || ''
|
||||
}
|
||||
const out: MarkdownToken[] = []
|
||||
pushEmOpen(out as any[], 2)
|
||||
out.push(createLinkToken(finalLabel, href, false))
|
||||
pushEmClose(out as any[], 2)
|
||||
tokens.splice(i, 9, ...out)
|
||||
i -= (out.length - 1)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Post-pass: linkify-it strips trailing ASCII punctuation like `!`, but in some URLs
|
||||
// it's meaningful (e.g. `?q=!`, `#!`). Merge a standalone leading `!` text token
|
||||
// back into the preceding autolink/linkify link when it ends with `=` or `#`.
|
||||
for (let i = 0; i < tokens.length - 1; i++) {
|
||||
const t = tokens[i] as any
|
||||
const next = tokens[i + 1] as any
|
||||
if (t?.type !== 'link' || next?.type !== 'text' || typeof next.content !== 'string')
|
||||
continue
|
||||
if (!next.content.startsWith('!'))
|
||||
continue
|
||||
const href = String(t.href ?? '')
|
||||
const text = String(t.text ?? '')
|
||||
// Only merge for bare URLs (where visible text equals href) to avoid
|
||||
// changing explicit Markdown links like `[label](https://a/#)!`.
|
||||
if (text !== href)
|
||||
continue
|
||||
if (!href.endsWith('=') && !href.endsWith('#'))
|
||||
continue
|
||||
|
||||
appendToLinkToken(t, '!')
|
||||
const rest = next.content.slice(1)
|
||||
if (rest) {
|
||||
next.content = rest
|
||||
next.raw = rest
|
||||
}
|
||||
else {
|
||||
tokens.splice(i + 1, 1)
|
||||
}
|
||||
}
|
||||
|
||||
return tokens
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import type { MarkdownIt } from 'markdown-it-ts'
|
||||
import type { MarkdownToken } from '../types'
|
||||
|
||||
export function applyFixListItem(md: MarkdownIt) {
|
||||
// Normalize list-item related inline tokens after inline tokenization
|
||||
// so downstream parsers see corrected children.
|
||||
md.core.ruler.after('inline', 'fix_list_item_tokens', (state: unknown) => {
|
||||
const s = state as unknown as { tokens?: Array<{ type?: string, children?: any[] }> }
|
||||
const toks = s.tokens ?? []
|
||||
for (let i = 0; i < toks.length; i++) {
|
||||
const t = toks[i]
|
||||
if (t && t.type === 'inline' && Array.isArray(t.children)) {
|
||||
try {
|
||||
t.children = fixListItem(t.children)
|
||||
}
|
||||
catch (e) {
|
||||
// Keep original children on error to avoid breaking parsing
|
||||
|
||||
console.error('[applyFixListItem] failed to fix inline children', e)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function fixListItem(tokens: MarkdownToken[]): MarkdownToken[] {
|
||||
const last = tokens[tokens.length - 1]
|
||||
const lastContent = String(last?.content ?? '')
|
||||
|
||||
if (last?.type === 'text' && (/^\s*\d+\.\s*$/.test(lastContent) && tokens[tokens.length - 2]?.tag === 'br')) {
|
||||
tokens.splice(tokens.length - 1, 1)
|
||||
}
|
||||
|
||||
return tokens
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
import type { MarkdownIt } from 'markdown-it-ts'
|
||||
import type { MarkdownToken } from 'stream-markdown-parser'
|
||||
|
||||
export function applyFixStrongTokens(md: MarkdownIt) {
|
||||
// Run after inline tokenization to normalize strong/em tokens in
|
||||
// each inline token's children. This ensures downstream inline
|
||||
// parsers receive a normalized token list.
|
||||
md.core.ruler.after('inline', 'fix_strong_tokens', (state: unknown) => {
|
||||
const s = state as unknown as { tokens?: Array<{ type?: string, children?: any[] }> }
|
||||
const toks = s.tokens ?? []
|
||||
for (let i = 0; i < toks.length; i++) {
|
||||
const t = toks[i]
|
||||
if (t && t.type === 'inline' && Array.isArray(t.children)) {
|
||||
try {
|
||||
t.children = fixStrongTokens(t.children)
|
||||
}
|
||||
catch (e) {
|
||||
// don't break parsing on plugin error
|
||||
|
||||
console.error('[applyFixStrongTokens] failed to fix inline children', e)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function fixStrongTokens(tokens: MarkdownToken[]): MarkdownToken[] {
|
||||
let strongIndex = 0
|
||||
const cleansStrong = new Set<number>()
|
||||
const cleansEm = new Set<number>()
|
||||
let emIndex = 0
|
||||
for (let i = 0; i < tokens.length; i++) {
|
||||
const t = tokens[i]
|
||||
const type = t.type
|
||||
if (type === 'strong_open') {
|
||||
strongIndex++
|
||||
const markup = String(t.markup ?? '')
|
||||
let j = i - 1
|
||||
while (j >= 0 && tokens[j].type === 'text' && tokens[j].content === '') {
|
||||
j--
|
||||
}
|
||||
const preToken = tokens[j]
|
||||
let k = i + 1
|
||||
while (k < tokens.length && tokens[k].type === 'text' && tokens[k].content === '') {
|
||||
k++
|
||||
}
|
||||
const postToken = tokens[k]
|
||||
|
||||
if (markup === '__' && (preToken?.content?.endsWith('_') || postToken?.content?.startsWith('_') || postToken?.markup?.includes('_'))) {
|
||||
t.type = 'text'
|
||||
t.tag = ''
|
||||
t.content = markup
|
||||
t.raw = markup
|
||||
t.markup = ''
|
||||
t.attrs = null
|
||||
t.map = null
|
||||
t.info = ''
|
||||
t.meta = null
|
||||
cleansStrong.add(strongIndex)
|
||||
}
|
||||
}
|
||||
else if (type === 'strong_close') {
|
||||
if (cleansStrong.has(strongIndex) && t.markup === '__') {
|
||||
t.type = 'text'
|
||||
t.content = t.markup
|
||||
t.raw = String(t.markup ?? '')
|
||||
t.tag = ''
|
||||
t.markup = ''
|
||||
t.attrs = null
|
||||
t.map = null
|
||||
t.info = ''
|
||||
t.meta = null
|
||||
}
|
||||
strongIndex--
|
||||
if (strongIndex < 0)
|
||||
strongIndex = 0
|
||||
}
|
||||
else if (type === 'em_open') {
|
||||
emIndex++
|
||||
const markup = String(t.markup ?? '')
|
||||
let j = i - 1
|
||||
while (j >= 0 && tokens[j].type === 'text' && tokens[j].content === '') {
|
||||
j--
|
||||
}
|
||||
const preToken = tokens[j]
|
||||
let k = i + 1
|
||||
while (k < tokens.length && tokens[k].type === 'text' && tokens[k].content === '') {
|
||||
k++
|
||||
}
|
||||
const postToken = tokens[k]
|
||||
if (markup === '_' && (preToken?.content?.endsWith('_') || postToken?.content?.startsWith('_') || postToken?.markup?.includes('_'))) {
|
||||
t.type = 'text'
|
||||
t.tag = ''
|
||||
t.content = markup
|
||||
t.raw = markup
|
||||
t.markup = ''
|
||||
t.attrs = null
|
||||
t.map = null
|
||||
t.info = ''
|
||||
t.meta = null
|
||||
cleansEm.add(emIndex)
|
||||
}
|
||||
}
|
||||
else if (type === 'em_close') {
|
||||
if (cleansEm.has(emIndex) && t.markup === '_') {
|
||||
t.type = 'text'
|
||||
t.content = t.markup
|
||||
t.raw = String(t.markup ?? '')
|
||||
t.tag = ''
|
||||
t.markup = ''
|
||||
t.attrs = null
|
||||
t.map = null
|
||||
t.info = ''
|
||||
t.meta = null
|
||||
}
|
||||
emIndex--
|
||||
if (emIndex < 0)
|
||||
emIndex = 0
|
||||
}
|
||||
}
|
||||
if (tokens.length < 5)
|
||||
return tokens
|
||||
const i = tokens.length - 4
|
||||
const token = tokens[i]
|
||||
let fixedTokens = [...tokens]
|
||||
const nextToken = tokens[i + 1]
|
||||
const tokenContent = String(token.content ?? '')
|
||||
if (token.type === 'link_open' && tokens[i - 1]?.type === 'em_open' && tokens[i - 2]?.type === 'text' && tokens[i - 2].content?.endsWith('*')) {
|
||||
const textContent = String(tokens[i - 2].content ?? '').slice(0, -1)
|
||||
|
||||
const replaceTokens = [
|
||||
{
|
||||
type: 'strong_open',
|
||||
tag: 'strong',
|
||||
attrs: null,
|
||||
map: null,
|
||||
children: null,
|
||||
content: '',
|
||||
markup: '**',
|
||||
info: '',
|
||||
meta: null,
|
||||
raw: '',
|
||||
},
|
||||
tokens[i],
|
||||
tokens[i + 1],
|
||||
tokens[i + 2],
|
||||
{
|
||||
type: 'strong_close',
|
||||
tag: 'strong',
|
||||
attrs: null,
|
||||
map: null,
|
||||
children: null,
|
||||
content: '',
|
||||
markup: '**',
|
||||
info: '',
|
||||
meta: null,
|
||||
raw: '',
|
||||
},
|
||||
]
|
||||
if (textContent) {
|
||||
replaceTokens.unshift({
|
||||
type: 'text',
|
||||
content: textContent,
|
||||
raw: textContent,
|
||||
})
|
||||
}
|
||||
fixedTokens.splice(i - 2, 6, ...replaceTokens)
|
||||
}
|
||||
else if (token.type === 'text' && tokenContent.endsWith('*') && nextToken.type === 'em_open') {
|
||||
// 解析有问题，要合并 emphasis 和 前面的 * 为 strong
|
||||
const _nextToken = tokens[i + 2]
|
||||
const count = _nextToken?.type === 'text' ? 4 : 3
|
||||
const insert = [
|
||||
{
|
||||
type: 'strong_open',
|
||||
tag: 'strong',
|
||||
attrs: null,
|
||||
map: null,
|
||||
children: null,
|
||||
content: '',
|
||||
markup: '**',
|
||||
info: '',
|
||||
meta: null,
|
||||
raw: '',
|
||||
},
|
||||
{
|
||||
type: 'text',
|
||||
content: _nextToken?.type === 'text' ? String(_nextToken.content ?? '') : '',
|
||||
raw: _nextToken?.type === 'text' ? String(_nextToken.content ?? '') : '',
|
||||
},
|
||||
{
|
||||
type: 'strong_close',
|
||||
tag: 'strong',
|
||||
attrs: null,
|
||||
map: null,
|
||||
children: null,
|
||||
content: '',
|
||||
markup: '**',
|
||||
info: '',
|
||||
meta: null,
|
||||
raw: '',
|
||||
},
|
||||
] as MarkdownToken[]
|
||||
const beforeText = tokenContent.slice(0, -1)
|
||||
if (beforeText) {
|
||||
insert.unshift({
|
||||
type: 'text',
|
||||
content: beforeText,
|
||||
raw: beforeText,
|
||||
})
|
||||
}
|
||||
fixedTokens.splice(i, count, ...insert)
|
||||
}
|
||||
|
||||
// Fix: markdown-it + math inline can incorrectly split a single strong span
|
||||
// into multiple strong nodes and leave the closing `**` inside a trailing text token.
|
||||
// Example:
|
||||
// **化简集合\\( P \\)并求补集**：...
|
||||
// Tokens may become:
|
||||
// strong_open, text('化简集合'), strong_close, strong_open, math_inline, strong_close, text('并求补集**：...')
|
||||
// Normalize into:
|
||||
// strong_open, text('化简集合'), math_inline, text('并求补集'), strong_close, text('：...')
|
||||
fixedTokens = mergeBrokenStrongAroundMathInline(fixedTokens)
|
||||
|
||||
return fixedTokens
|
||||
}
|
||||
|
||||
function mergeBrokenStrongAroundMathInline(tokens: MarkdownToken[]): MarkdownToken[] {
|
||||
if (tokens.length < 7)
|
||||
return tokens
|
||||
|
||||
const out: MarkdownToken[] = []
|
||||
for (let i = 0; i < tokens.length; i++) {
|
||||
const t0 = tokens[i]
|
||||
const t1 = tokens[i + 1]
|
||||
const t2 = tokens[i + 2]
|
||||
const t3 = tokens[i + 3]
|
||||
const t4 = tokens[i + 4]
|
||||
const t5 = tokens[i + 5]
|
||||
const t6 = tokens[i + 6]
|
||||
|
||||
if (
|
||||
t0?.type === 'strong_open'
|
||||
&& t1?.type === 'text'
|
||||
&& t2?.type === 'strong_close'
|
||||
&& t3?.type === 'strong_open'
|
||||
&& t4?.type === 'math_inline'
|
||||
&& t5?.type === 'strong_close'
|
||||
&& t6?.type === 'text'
|
||||
) {
|
||||
const textContent = String(t6.content ?? '')
|
||||
const closeIdx = textContent.indexOf('**')
|
||||
if (closeIdx !== -1) {
|
||||
const beforeClose = textContent.slice(0, closeIdx)
|
||||
const afterClose = textContent.slice(closeIdx + 2)
|
||||
|
||||
out.push(t0)
|
||||
out.push(t1)
|
||||
out.push(t4)
|
||||
|
||||
if (beforeClose) {
|
||||
out.push({
|
||||
...t6,
|
||||
type: 'text',
|
||||
content: beforeClose,
|
||||
raw: beforeClose,
|
||||
} as MarkdownToken)
|
||||
}
|
||||
|
||||
// Reuse one of the strong_close tokens to close the merged strong span.
|
||||
out.push(t5)
|
||||
|
||||
if (afterClose) {
|
||||
out.push({
|
||||
...t6,
|
||||
type: 'text',
|
||||
content: afterClose,
|
||||
raw: afterClose,
|
||||
} as MarkdownToken)
|
||||
}
|
||||
|
||||
i += 6
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
out.push(t0)
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
import type { MarkdownIt } from 'markdown-it-ts'
|
||||
import type { MarkdownToken } from '../types'
|
||||
|
||||
export function applyFixTableTokens(md: MarkdownIt) {
|
||||
// Run after block parsing so block-level tokens (including inline
|
||||
// children) are present. We replace the token array with the
|
||||
// fixed version returned by `fixTableTokens`.
|
||||
md.core.ruler.after('block', 'fix_table_tokens', (state: unknown) => {
|
||||
const s = state as unknown as { tokens?: any[] }
|
||||
try {
|
||||
const toks = s.tokens ?? []
|
||||
const fixed = fixTableTokens(toks)
|
||||
if (Array.isArray(fixed))
|
||||
s.tokens = fixed
|
||||
}
|
||||
catch (e) {
|
||||
// swallow errors to avoid breaking parsing; keep original tokens
|
||||
console.error('[applyFixTableTokens] failed to fix table tokens', e)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function createStart() {
|
||||
return [
|
||||
{
|
||||
type: 'table_open',
|
||||
tag: 'table',
|
||||
attrs: null,
|
||||
map: null,
|
||||
children: null,
|
||||
content: '',
|
||||
markup: '',
|
||||
info: '',
|
||||
level: 0,
|
||||
loading: true,
|
||||
meta: null,
|
||||
},
|
||||
{
|
||||
type: 'thead_open',
|
||||
tag: 'thead',
|
||||
attrs: null,
|
||||
block: true,
|
||||
level: 1,
|
||||
children: null,
|
||||
},
|
||||
{
|
||||
type: 'tr_open',
|
||||
tag: 'tr',
|
||||
attrs: null,
|
||||
block: true,
|
||||
level: 2,
|
||||
children: null,
|
||||
},
|
||||
|
||||
]
|
||||
}
|
||||
function createEnd() {
|
||||
return [
|
||||
{
|
||||
type: 'tr_close',
|
||||
tag: 'tr',
|
||||
attrs: null,
|
||||
block: true,
|
||||
level: 2,
|
||||
children: null,
|
||||
},
|
||||
{
|
||||
type: 'thead_close',
|
||||
tag: 'thead',
|
||||
attrs: null,
|
||||
block: true,
|
||||
level: 1,
|
||||
children: null,
|
||||
},
|
||||
{
|
||||
type: 'table_close',
|
||||
tag: 'table',
|
||||
attrs: null,
|
||||
map: null,
|
||||
children: null,
|
||||
content: '',
|
||||
markup: '',
|
||||
info: '',
|
||||
level: 0,
|
||||
meta: null,
|
||||
},
|
||||
]
|
||||
}
|
||||
function createTh(text: string) {
|
||||
return [{
|
||||
type: 'th_open',
|
||||
tag: 'th',
|
||||
attrs: null,
|
||||
block: true,
|
||||
level: 3,
|
||||
children: null,
|
||||
}, {
|
||||
type: 'inline',
|
||||
tag: '',
|
||||
children: null,
|
||||
content: text,
|
||||
level: 4,
|
||||
attrs: null,
|
||||
block: true,
|
||||
}, {
|
||||
type: 'th_close',
|
||||
tag: 'th',
|
||||
attrs: null,
|
||||
block: true,
|
||||
level: 3,
|
||||
children: null,
|
||||
}]
|
||||
}
|
||||
|
||||
function getPipeRowCells(line: string, requireTrailingPipe: boolean) {
|
||||
if (!line.startsWith('|') || line.includes('\n'))
|
||||
return null
|
||||
if (requireTrailingPipe && !line.endsWith('|'))
|
||||
return null
|
||||
|
||||
const cells = line.slice(1).split('|')
|
||||
if (cells.at(-1) === '')
|
||||
cells.pop()
|
||||
|
||||
return cells.length > 0 && cells.every(cell => cell.length > 0) ? cells : null
|
||||
}
|
||||
|
||||
function isSingleLineFallbackTable(line: string) {
|
||||
return getPipeRowCells(line, false) !== null
|
||||
}
|
||||
|
||||
function hasTrailingPipeHeaderRow(line: string) {
|
||||
return getPipeRowCells(line, true) !== null
|
||||
}
|
||||
|
||||
function isSeparatorCell(cell: string) {
|
||||
return /^:?-+:?$/.test(cell.trim())
|
||||
}
|
||||
|
||||
function isTableSeparatorRow(line: string) {
|
||||
if (!line.startsWith('|'))
|
||||
return false
|
||||
|
||||
const cells = line.slice(1).split('|')
|
||||
if (cells.at(-1) === '')
|
||||
cells.pop()
|
||||
|
||||
return cells.length > 0 && cells.every(isSeparatorCell)
|
||||
}
|
||||
|
||||
function isTruncatedSeparatorRow(line: string) {
|
||||
return line === '|' || line === '|:'
|
||||
}
|
||||
|
||||
function hasTrailingPipeHeaderRowWithoutColon(line: string) {
|
||||
const cells = getPipeRowCells(line, true)
|
||||
return cells !== null && cells.every(cell => !cell.includes(':'))
|
||||
}
|
||||
|
||||
export function fixTableTokens(tokens: MarkdownToken[]): MarkdownToken[] {
|
||||
const fixedTokens = [...tokens]
|
||||
if (tokens.length < 3)
|
||||
return fixedTokens
|
||||
const i = tokens.length - 2
|
||||
const token = tokens[i]
|
||||
if (token.type === 'inline') {
|
||||
const tcontent = String(token.content ?? '')
|
||||
const headerContent = tcontent.split('\n')[0] ?? ''
|
||||
const [headerLine = '', separatorLine = '', ...rest] = tcontent.split('\n')
|
||||
|
||||
if (!tcontent.includes('\n') && isSingleLineFallbackTable(tcontent)) {
|
||||
// 解析 table
|
||||
const body = headerContent.slice(1).split('|').map(i => i.trim()).filter(Boolean).flatMap(i => createTh(i))
|
||||
const insert = ([
|
||||
...createStart(),
|
||||
...body,
|
||||
...createEnd(),
|
||||
] as unknown) as MarkdownToken[]
|
||||
fixedTokens.splice(i - 1, 3, ...insert)
|
||||
}
|
||||
else if (
|
||||
tcontent.includes('\n')
|
||||
&& rest.length === 0
|
||||
&& hasTrailingPipeHeaderRow(headerLine)
|
||||
&& isTableSeparatorRow(separatorLine)
|
||||
) {
|
||||
// 解析 table
|
||||
const body = headerContent.slice(1, -1).split('|').map(i => i.trim()).flatMap(i => createTh(i))
|
||||
const insert = ([
|
||||
...createStart(),
|
||||
...body,
|
||||
...createEnd(),
|
||||
] as unknown) as MarkdownToken[]
|
||||
fixedTokens.splice(i - 1, 3, ...insert)
|
||||
}
|
||||
else if (
|
||||
tcontent.includes('\n')
|
||||
&& rest.length === 0
|
||||
&& hasTrailingPipeHeaderRowWithoutColon(headerLine)
|
||||
&& isTruncatedSeparatorRow(separatorLine)
|
||||
) {
|
||||
token.content = tcontent.slice(0, -2)
|
||||
token.children!.splice(2, 1)
|
||||
}
|
||||
}
|
||||
|
||||
return fixedTokens
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
export const TEX_BRACE_COMMANDS = [
|
||||
'boldsymbol',
|
||||
'mathbb',
|
||||
'mathcal',
|
||||
'mathfrak',
|
||||
'mathrm',
|
||||
'mathit',
|
||||
'mathsf',
|
||||
'vec',
|
||||
'hat',
|
||||
'bar',
|
||||
'tilde',
|
||||
'overline',
|
||||
'underline',
|
||||
'mathscr',
|
||||
'mathnormal',
|
||||
'operatorname',
|
||||
'mathbf*',
|
||||
]
|
||||
|
||||
export const ESCAPED_TEX_BRACE_COMMANDS = TEX_BRACE_COMMANDS.map(c => c.replace(/[.*+?^${}()|[\\]"\]/g, '\\$&')).join('|')
|
||||
|
||||
const TEX_CMD_RE = /\\[a-z]+/i
|
||||
const PREFIX_CLASS = '(?:\\\\|\\u0008)'
|
||||
const TEX_CMD_WITH_BRACES_RE = new RegExp(String.raw`${PREFIX_CLASS}(?:${ESCAPED_TEX_BRACE_COMMANDS})\s*\{[^}]+\}`, 'i')
|
||||
// Detect brace-taking TeX commands even when the leading backslash or the
|
||||
// closing brace/content is missing (e.g. "operatorname{" or "operatorname{span").
|
||||
// This helps the heuristic treat incomplete but clearly TeX-like fragments
|
||||
// as math-like instead of plain text.
|
||||
const TEX_BRACE_CMD_START_RE = new RegExp(String.raw`(?:${PREFIX_CLASS})?(?:${ESCAPED_TEX_BRACE_COMMANDS})\s*\{`, 'i')
|
||||
const TEX_SPECIFIC_RE = /\\(?:text|frac|left|right|times)/
|
||||
// Match common math operator symbols or named commands.
|
||||
// Avoid treating the C/C++ increment operator ("++") as a math operator by
|
||||
// ensuring a lone '+' isn't matched when it's part of a '++' sequence.
|
||||
// Avoid lookbehind for older iOS: use a non-capturing prefix instead
|
||||
const OPS_RE = /(?:^|[^+])\+(?!\+)|[=\-*/^<>]|\\times|\\pm|\\cdot|\\le|\\ge|\\neq/
|
||||
// Hyphenated multi-word (like "Quasi-Streaming") should not be treated
|
||||
// as a math operator. But single-letter-variable hyphens (e.g. "x-y") are
|
||||
// still math; so only ignore hyphens between multi-letter words.
|
||||
const HYPHENATED_MULTIWORD_RE = /\b[A-Z]{2,}-[A-Z]{2,}\b/i
|
||||
const FUNC_CALL_RE = /[A-Z]+\s*\([^)]+\)/i
|
||||
const WORDS_RE = /\b(?:sin|cos|tan|log|ln|exp|sqrt|frac|sum|lim|int|prod)\b/
|
||||
// Heuristic to detect common date/time patterns like 2025/9/30 21:37:24 and
|
||||
// avoid classifying them as math merely because they contain '/' or ':'
|
||||
const DATE_TIME_RE = /\b\d{4}\/\d{1,2}\/\d{1,2}(?:[ T]\d{1,2}:\d{2}(?::\d{2})?)?\b/
|
||||
const CONTROL_TEX_REPLACEMENTS: Record<string, string> = {
|
||||
[String.fromCharCode(8)]: '\\b',
|
||||
[String.fromCharCode(11)]: '\\v',
|
||||
[String.fromCharCode(12)]: '\\f',
|
||||
}
|
||||
|
||||
function normalizeMathControlChars(value: string) {
|
||||
let result = ''
|
||||
for (const ch of value)
|
||||
result += CONTROL_TEX_REPLACEMENTS[ch] ?? ch
|
||||
return result
|
||||
}
|
||||
|
||||
export function isMathLike(s: string) {
|
||||
if (!s)
|
||||
return false
|
||||
|
||||
// Normalize accidental control characters that may appear if a single
|
||||
// backslash sequence was interpreted in a JS string literal (for example
|
||||
// '\\frac' being typed as '\frac' which turns into a form-feed + "rac").
|
||||
// Convert the handful of control characters that collide with common TeX
|
||||
// commands (\b, \f, \v) back into their two-character escaped forms so our
|
||||
// regexes can match the intent reliably without flagging real newlines/tabs.
|
||||
const norm = normalizeMathControlChars(s)
|
||||
const stripped = norm.trim()
|
||||
|
||||
// quick bailouts
|
||||
// If the content looks like a timestamp or date, it's not math.
|
||||
if (DATE_TIME_RE.test(stripped))
|
||||
return false
|
||||
if (stripped.includes('**'))
|
||||
return false
|
||||
if (stripped.length > 2000)
|
||||
return true // very long blocks likely math
|
||||
|
||||
// TeX commands e.g. \frac, \alpha
|
||||
const texCmd = TEX_CMD_RE.test(norm)
|
||||
const texCmdWithBraces = TEX_CMD_WITH_BRACES_RE.test(norm)
|
||||
const texBraceStart = TEX_BRACE_CMD_START_RE.test(norm)
|
||||
|
||||
// Explicit common TeX tokens (keeps compatibility with previous heuristic)
|
||||
const texSpecific = TEX_SPECIFIC_RE.test(norm)
|
||||
const subscriptPattern = /(?:^|[^\w\\])(?:[A-Z]|\\[A-Z]+)_(?:\{[^}]+\}|[A-Z0-9\\])/i
|
||||
const superscriptPattern = /(?:^|[^\w\\])(?:[A-Z]|\\[A-Z]+)\^(?:\{[^}]+\}|[A-Z0-9\\])/i
|
||||
const superSub = subscriptPattern.test(norm) || superscriptPattern.test(norm)
|
||||
// common math operator symbols or named commands
|
||||
// Ignore operator-like hyphens that are clearly word compound separators
|
||||
// (e.g. "Quasi-Streaming"). Keep hyphens for math like "x-y".
|
||||
const ops = OPS_RE.test(norm) && !HYPHENATED_MULTIWORD_RE.test(norm)
|
||||
// function-like patterns: f(x), sin(x)
|
||||
const funcCall = FUNC_CALL_RE.test(norm)
|
||||
// common math words
|
||||
const words = WORDS_RE.test(norm)
|
||||
// 纯单个英文字母也渲染成数学公式（常见变量/元素符号）
|
||||
// e.g. (w) (x) (y) (z) 或 $H$, $x$ 等
|
||||
const pureWord = /^\([a-z]\)$/i.test(stripped) || /^(?:[a-z]|pi)$/i.test(stripped)
|
||||
// 简单的化学式/下标：如 H_2O, CO_2, CH_3CH_2OH, CH_3COOH
|
||||
// 收紧规则：
|
||||
// - 区分大小写（化学元素以大写或大写+小写开头）
|
||||
// - 下标/上标通常是数字（可选花括号），避免匹配 get_time 之类的普通下划线单词
|
||||
const chemicalLike = /^(?:[A-Z][a-z]?(?:_\{?\d+\}?|\^\{?\d+\}?)?)+$/.test(stripped)
|
||||
|
||||
return texCmd || texCmdWithBraces || texBraceStart || texSpecific || superSub || ops || funcCall || words || pureWord || chemicalLike
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user