fix: fix multiline list item adds extra newline to raw (#3735)

* fix: fix multiline list item adds extra newline to raw

* update other raw

* remove only

* write test for code

* add def test

Author: UziTech

src/Lexer.ts

@@ -142,7 +142,7 @@ export class _Lexer<ParserOutput = string, RendererOutput = string> {
         const lastToken = tokens.at(-1);
         // An indented code block cannot interrupt a paragraph.
         if (lastToken?.type === 'paragraph' || lastToken?.type === 'text') {
-          lastToken.raw += '\n' + token.raw;
+          lastToken.raw += (lastToken.raw.endsWith('\n') ? '' : '\n') + token.raw;
           lastToken.text += '\n' + token.text;
           this.inlineQueue.at(-1)!.src = lastToken.text;
         } else {
@@ -198,7 +198,7 @@ export class _Lexer<ParserOutput = string, RendererOutput = string> {
         src = src.substring(token.raw.length);
         const lastToken = tokens.at(-1);
         if (lastToken?.type === 'paragraph' || lastToken?.type === 'text') {
-          lastToken.raw += '\n' + token.raw;
+          lastToken.raw += (lastToken.raw.endsWith('\n') ? '' : '\n') + token.raw;
           lastToken.text += '\n' + token.raw;
           this.inlineQueue.at(-1)!.src = lastToken.text;
         } else if (!this.tokens.links[token.tag]) {
@@ -244,7 +244,7 @@ export class _Lexer<ParserOutput = string, RendererOutput = string> {
       if (this.state.top && (token = this.tokenizer.paragraph(cutSrc))) {
         const lastToken = tokens.at(-1);
         if (lastParagraphClipped && lastToken?.type === 'paragraph') {
-          lastToken.raw += '\n' + token.raw;
+          lastToken.raw += (lastToken.raw.endsWith('\n') ? '' : '\n') + token.raw;
           lastToken.text += '\n' + token.text;
           this.inlineQueue.pop();
           this.inlineQueue.at(-1)!.src = lastToken.text;
@@ -261,7 +261,7 @@ export class _Lexer<ParserOutput = string, RendererOutput = string> {
         src = src.substring(token.raw.length);
         const lastToken = tokens.at(-1);
         if (lastToken?.type === 'text') {
-          lastToken.raw += '\n' + token.raw;
+          lastToken.raw += (lastToken.raw.endsWith('\n') ? '' : '\n') + token.raw;
           lastToken.text += '\n' + token.text;
           this.inlineQueue.pop();
           this.inlineQueue.at(-1)!.src = lastToken.text;

test/unit/Lexer.test.js

@@ -1173,6 +1173,123 @@ paragraph
         ],
       });
     });
+
+    it('multiline', () => {
+      expectTokens({
+        md: `
+- line 1
+  line 2
+`,
+        tokens: [
+          {
+            type: 'space',
+            raw: '\n',
+          }, {
+            type: 'list',
+            raw: '- line 1\n  line 2\n',
+            ordered: false,
+            start: '',
+            loose: false,
+            items: [
+              {
+                type: 'list_item',
+                raw: '- line 1\n  line 2',
+                task: false,
+                checked: undefined,
+                loose: false,
+                text: 'line 1\nline 2',
+                tokens: [{
+                  type: 'text',
+                  raw: 'line 1\nline 2',
+                  text: 'line 1\nline 2',
+                  tokens: [{ type: 'text', raw: 'line 1\nline 2', text: 'line 1\nline 2', escaped: false }],
+                }],
+              },
+            ],
+          },
+        ],
+      });
+    });
+
+    it('indented code after paragraph', () => {
+      expectTokens({
+        md: `
+- a
+      - b
+`,
+        tokens: [{
+          type: 'space',
+          raw: '\n',
+        },
+        {
+          type: 'list',
+          raw: '- a\n      - b\n',
+          ordered: false,
+          start: '',
+          loose: false,
+          items: [
+            {
+              type: 'list_item',
+              raw: '- a\n      - b',
+              task: false,
+              checked: undefined,
+              loose: false,
+              text: 'a\n    - b',
+              tokens: [
+                {
+                  type: 'text',
+                  raw: 'a\n    - b',
+                  text: 'a\n- b',
+                  tokens: [
+                    { type: 'text', raw: 'a\n- b', text: 'a\n- b', escaped: false },
+                  ],
+                },
+              ],
+            },
+          ],
+        },
+        ],
+      });
+    });
+
+    it('def after paragraph', () => {
+      expectTokens({
+        md: `
+- hello
+[1]: hello
+`,
+        tokens: [
+          { type: 'space', raw: '\n' },
+          {
+            type: 'list',
+            raw: '- hello\n[1]: hello\n',
+            ordered: false,
+            start: '',
+            loose: false,
+            items: [
+              {
+                type: 'list_item',
+                raw: '- hello\n[1]: hello',
+                task: false,
+                checked: undefined,
+                loose: false,
+                text: 'hello\n[1]: hello',
+                tokens: [
+                  {
+                    type: 'text',
+                    raw: 'hello\n[1]: hello',
+                    text: 'hello\n[1]: hello',
+                    tokens: [
+                      { type: 'text', raw: 'hello\n[1]: hello', text: 'hello\n[1]: hello', escaped: false },
+                    ],
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      });
+    });
   });
 
   describe('html', () => {