Skip to content

Commit 539d031

Browse files
authored
Remove whitespace tokenizing rule in markdown lexer (#1008)
fix #744 related issue: go-gitea/gitea#32136 current markdown lexer is tokenizing by whitespace, but in markdown sytanx this implementation can cause some errors.
1 parent 6622bd5 commit 539d031

File tree

3 files changed

+29
-2
lines changed

3 files changed

+29
-2
lines changed

lexers/markdown.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ func markdownRules() Rules {
4040
{"`[^`]+`", LiteralStringBacktick, nil},
4141
{`[@#][\w/:]+`, NameEntity, nil},
4242
{`(!?\[)([^]]+)(\])(\()([^)]+)(\))`, ByGroups(Text, NameTag, Text, Text, NameAttribute, Text), nil},
43-
{`[^\\\s]+`, Other, nil},
4443
{`.|\n`, Other, nil},
4544
},
4645
}

lexers/testdata/markdown.actual

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,3 +85,9 @@ end loop;
8585
end
8686
$$;
8787
```
88+
89+
## MarkdownLink Test
90+
[normal link](https://google.com)
91+
[whitespace before link](https://google.com)
92+
abc[non-whitespace before link](https://google.com)
93+
([first link](https://google.com) and [second link](https://google.com))

lexers/testdata/markdown.expected

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -441,5 +441,27 @@
441441
{"type":"Punctuation","value":";"},
442442
{"type":"Text","value":"\n"},
443443
{"type":"LiteralString","value":"```"},
444-
{"type":"Text","value":"\n"}
444+
{"type":"Text","value":"\n\n"},
445+
{"type":"GenericSubheading","value":"## MarkdownLink Test\n"},
446+
{"type":"Text","value":"["},
447+
{"type":"NameTag","value":"normal link"},
448+
{"type":"Text","value":"]("},
449+
{"type":"NameAttribute","value":"https://google.com"},
450+
{"type":"Text","value":") \n ["},
451+
{"type":"NameTag","value":"whitespace before link"},
452+
{"type":"Text","value":"]("},
453+
{"type":"NameAttribute","value":"https://google.com"},
454+
{"type":"Text","value":") \nabc["},
455+
{"type":"NameTag","value":"non-whitespace before link"},
456+
{"type":"Text","value":"]("},
457+
{"type":"NameAttribute","value":"https://google.com"},
458+
{"type":"Text","value":") \n(["},
459+
{"type":"NameTag","value":"first link"},
460+
{"type":"Text","value":"]("},
461+
{"type":"NameAttribute","value":"https://google.com"},
462+
{"type":"Text","value":") and ["},
463+
{"type":"NameTag","value":"second link"},
464+
{"type":"Text","value":"]("},
465+
{"type":"NameAttribute","value":"https://google.com"},
466+
{"type":"Text","value":")) \n"}
445467
]

0 commit comments

Comments
 (0)