{ "$schema": "https://ui.shadcn.com/schema/registry-item.json", "name": "markdown-joiner-transform", "dependencies": [ "ai@7" ], "files": [ { "path": "src/registry/lib/markdown-joiner-transform.ts", "content": "import type { TextStreamPart, ToolSet } from 'ai';\n\n/**\n * Transform chunks like [**,bold,**] to [**bold**] make the md deserializer\n * happy.\n *\n * @experimental\n */\nexport const markdownJoinerTransform =\n () =>\n () => {\n const joiner = new MarkdownJoiner();\n let lastTextDeltaId: string | undefined;\n let textStreamEnded = false;\n\n return new TransformStream, TextStreamPart>({\n async flush(controller) {\n // Only flush if we haven't seen text-end yet\n if (!textStreamEnded) {\n const remaining = joiner.flush();\n if (remaining && lastTextDeltaId) {\n controller.enqueue({\n id: lastTextDeltaId,\n text: remaining,\n type: 'text-delta',\n } as TextStreamPart);\n }\n }\n },\n async transform(chunk, controller) {\n if (chunk.type === 'text-delta') {\n lastTextDeltaId = chunk.id;\n const processedText = joiner.processText(chunk.text);\n if (processedText) {\n controller.enqueue({\n ...chunk,\n text: processedText,\n });\n await delay(joiner.delayInMs);\n }\n } else if (chunk.type === 'text-end') {\n // Flush any remaining buffer before text-end\n const remaining = joiner.flush();\n if (remaining && lastTextDeltaId) {\n controller.enqueue({\n id: lastTextDeltaId,\n text: remaining,\n type: 'text-delta',\n } as TextStreamPart);\n }\n textStreamEnded = true;\n controller.enqueue(chunk);\n } else {\n controller.enqueue(chunk);\n }\n },\n });\n };\n\nconst DEFAULT_DELAY_IN_MS = 10;\nconst NEST_BLOCK_DELAY_IN_MS = 100;\n\nconst BOLD_PATTERN = /\\*\\*.*?\\*\\*/;\nconst CODE_LINE_PATTERN = /```[^\\s]+/;\nconst LINK_PATTERN = /^\\[.*?\\]\\(.*?\\)$/;\nconst UNORDERED_LIST_PATTERN = /^[*-]\\s+.+/;\nconst TODO_LIST_PATTERN = /^[*-]\\s+\\[[ xX]\\]\\s+.+/;\nconst ORDERED_LIST_PATTERN = /^\\d+\\.\\s+.+/;\nconst MDX_TAG_PATTERN = /<([A-Za-z][A-Za-z0-9\\-_]*)>/;\nconst DIGIT_PATTERN = /^[0-9]$/;\n\nexport class MarkdownJoiner {\n delayInMs = DEFAULT_DELAY_IN_MS;\n\n private buffer = '';\n private documentCharacterCount = 0;\n private isBuffering = false;\n private streamingCodeBlock = false;\n private streamingLargeDocument = false;\n private streamingTable = false;\n\n private clearBuffer(): void {\n this.buffer = '';\n this.isBuffering = false;\n }\n private isCompleteBold(): boolean {\n return BOLD_PATTERN.test(this.buffer);\n }\n\n private isCompleteCodeBlockEnd(): boolean {\n return this.buffer.trimEnd() === '```';\n }\n\n private isCompleteCodeBlockStart(): boolean {\n return CODE_LINE_PATTERN.test(this.buffer);\n }\n\n private isCompleteLink(): boolean {\n return LINK_PATTERN.test(this.buffer);\n }\n\n private isCompleteList(): boolean {\n if (UNORDERED_LIST_PATTERN.test(this.buffer) && this.buffer.includes('['))\n return TODO_LIST_PATTERN.test(this.buffer);\n\n return (\n UNORDERED_LIST_PATTERN.test(this.buffer) ||\n ORDERED_LIST_PATTERN.test(this.buffer) ||\n TODO_LIST_PATTERN.test(this.buffer)\n );\n }\n\n private isCompleteMdxTag(): boolean {\n return MDX_TAG_PATTERN.test(this.buffer);\n }\n\n private isCompleteTableStart(): boolean {\n return this.buffer.startsWith('|') && this.buffer.endsWith('|');\n }\n\n private isFalsePositive(char: string): boolean {\n // when link is not complete, even if ths buffer is more than 30 characters, it is not a false positive\n if (this.buffer.startsWith('[') && this.buffer.includes('http')) {\n return false;\n }\n\n return char === '\\n' || this.buffer.length > 30;\n }\n\n private isLargeDocumentStart(): boolean {\n return this.documentCharacterCount > 2500;\n }\n\n private isListStartChar(char: string): boolean {\n return char === '-' || char === '*' || DIGIT_PATTERN.test(char);\n }\n\n private isTableExisted(): boolean {\n return this.buffer.length > 10 && !this.buffer.includes('|');\n }\n\n flush(): string {\n const remaining = this.buffer;\n this.clearBuffer();\n return remaining;\n }\n\n processText(text: string): string {\n let output = '';\n\n for (const char of text) {\n if (\n this.streamingCodeBlock ||\n this.streamingTable ||\n this.streamingLargeDocument\n ) {\n this.buffer += char;\n\n if (char === '\\n') {\n output += this.buffer;\n this.clearBuffer();\n }\n\n if (this.isCompleteCodeBlockEnd() && this.streamingCodeBlock) {\n this.streamingCodeBlock = false;\n this.delayInMs = DEFAULT_DELAY_IN_MS;\n\n output += this.buffer;\n this.clearBuffer();\n }\n\n if (this.isTableExisted() && this.streamingTable) {\n this.streamingTable = false;\n this.delayInMs = DEFAULT_DELAY_IN_MS;\n\n output += this.buffer;\n this.clearBuffer();\n }\n } else if (this.isBuffering) {\n this.buffer += char;\n\n if (this.isCompleteCodeBlockStart()) {\n this.delayInMs = NEST_BLOCK_DELAY_IN_MS;\n this.streamingCodeBlock = true;\n continue;\n }\n\n if (this.isCompleteTableStart()) {\n this.delayInMs = NEST_BLOCK_DELAY_IN_MS;\n this.streamingTable = true;\n continue;\n }\n\n if (this.isLargeDocumentStart()) {\n this.delayInMs = NEST_BLOCK_DELAY_IN_MS;\n this.streamingLargeDocument = true;\n continue;\n }\n\n if (\n this.isCompleteBold() ||\n this.isCompleteMdxTag() ||\n this.isCompleteList() ||\n this.isCompleteLink()\n ) {\n output += this.buffer;\n this.clearBuffer();\n } else if (this.isFalsePositive(char)) {\n // False positive - flush buffer as raw text\n output += this.buffer;\n this.clearBuffer();\n }\n // Check if we should start buffering\n } else if (\n char === '*' ||\n char === '<' ||\n char === '`' ||\n char === '|' ||\n char === '[' ||\n this.isListStartChar(char)\n ) {\n this.buffer = char;\n this.isBuffering = true;\n } else {\n // Pass through character directly\n output += char;\n }\n }\n\n this.documentCharacterCount += text.length;\n return output;\n }\n}\n\nasync function delay(delayInMs?: number | null): Promise {\n return delayInMs == null\n ? Promise.resolve()\n : new Promise((resolve) => setTimeout(resolve, delayInMs));\n}\n", "type": "registry:lib" } ], "type": "registry:hook" }