using custom delimiters breaks recognition of comments #187
Description
After tinkering with Jet using custom delimiters I was able to reproduce the following problem directly on Jet test suite.
- modified file :
testData/custom_delimiters.jet - first line now reads:
{* comment *}[[ . ]] - test fails
c:\GitHub\jet>go test
--- FAIL: TestParseTemplateWithCustomDelimiters (0.00s)
parse_test.go:46: Unexpected tree on custom_delimiters.jet Got:
{* comment *}{{.}}
{{singleValue}}
{{nil}}
{{""}}
{{val.Field}}
{{url("")}}
{{url("", "") | pipe}}
{{url("") | pipe | pipe}}
{{url("", "").Field | pipe}}
{{url("", "").Method("") | pipe}}
Expected:
{{.}}
{{singleValue}}
{{nil}}
{{""}}
{{val.Field}}
{{url("")}}
{{url("", "") | pipe}}
{{url("") | pipe | pipe}}
{{url("", "").Field | pipe}}
{{url("", "").Method("") | pipe}}
FAIL
exit status 1
FAIL github.com/CloudyKit/jet/v6 1.775s
Might it be that this is the source of the problem? See below... It seems that methow WithDelims should accept full delimiter set to allow for correct parsing of comments.
// state functions
func lexText(l *lexer) stateFn {
for {
// look for first character of the custom delimiter. if not found, break
// this EXCLUDES comment defined as {* from the parse, and sends it on output as text ???
if i := strings.IndexByte(l.input[l.pos:], l.leftDelim[0]); i == -1 {
l.pos = Pos(len(l.input))
break
} else {
l.pos += Pos(i)
if strings.HasPrefix(l.input[l.pos:], l.leftDelim) {
ld := Pos(len(l.leftDelim))
trimLength := Pos(0)
if strings.HasPrefix(l.input[l.pos+ld:], leftTrimMarker) {
trimLength = rightTrimLength(l.input[l.start:l.pos])
}
l.pos -= trimLength
if l.pos > l.start {
l.emit(itemText)
}
l.pos += trimLength
l.ignore()
return lexLeftDelim
}
// with custom delimiters, this will NEVER be activated
if strings.HasPrefix(l.input[l.pos:], leftComment) {
if l.pos > l.start {
l.emit(itemText)
}
return lexComment
}
}
if l.next() == eof {
break
}
}
// Correctly reached EOF.
if l.pos > l.start {
l.emit(itemText)
}
l.emit(itemEOF)
return nil
}