go: goeval A new codegen for the Go language

Just like the existing `gen_go`, this new generator outputs everything
needed to parse an input string into a single file.  In fact, it
generates a parser with almost exactly the same API as the existing
mechanism.  The exception is the method `Parser.SetCaptureSpaces`,
which is currently not supported by the evaluator.

This change also uncommented the Grammar AST enrichment steps in the
`vm_test.go` file to do exactly what the `cmd/langlang/main.go` file
is doing.  At some point all that stuff is going to be moved into a
nice API that's going to hide that away from callers.

While outputting go code using `literalSanitizer`, it ended up failing
with the replacement of single quotes because apparently go doesn't
have an escape sequence for them.

Author: clarete

go/cmd/langlang/main.go

@@ -134,8 +134,6 @@ func main() {
 	// If it's interactive, it will open a lil REPL shell
 
 	if *a.interactive {
-		fmt.Println(ast.HighlightPrettyString())
-		fmt.Println(asm.HighlightPrettyString())
 		code := langlang.Encode(asm)
 
 		for {
@@ -165,9 +163,6 @@ func main() {
 	// if there's an input path, just run the match right away
 
 	if *a.inputPath != "" {
-		fmt.Println(ast.HighlightPrettyString())
-		fmt.Println(asm.HighlightPrettyString())
-
 		text, err := os.ReadFile(*a.inputPath)
 		if err != nil {
 			log.Fatalf("Can't open input file: %s", err.Error())
@@ -195,7 +190,11 @@ func main() {
 			PackageName: *a.goOptPackage,
 			RemoveLib:   *a.goOptRemoveLib,
 		})
-
+	case "goeval":
+		outputData, err = langlang.GenGoEval(asm, langlang.GenGoOptions{
+			PackageName: *a.goOptPackage,
+			RemoveLib:   *a.goOptRemoveLib,
+		})
 	// case "python":
 	// 	outputData, err = langlang.GenParserPython(ast)
 	default:

go/gen_go_eval.go

@@ -0,0 +1,165 @@
+package langlang
+
+import (
+	"bytes"
+	"embed"
+	"fmt"
+	"text/template"
+)
+
+//go:embed vm.go vm_stack.go tree_printer.go errors.go value.go
+var goEvalContent embed.FS
+
+func GenGoEval(asm *Program, opt GenGoOptions) (string, error) {
+	g := newGoEvalEmitter(opt)
+	g.writePrelude()
+	g.writeParserProgram(Encode(asm))
+	g.writeParserStruct()
+	g.writeParserConstructor()
+	g.writeParserMethods(asm)
+	g.writeDeps()
+	return g.output()
+}
+
+type goEvalEmitter struct {
+	options GenGoOptions
+	parser  *outputWriter
+}
+
+func newGoEvalEmitter(opt GenGoOptions) *goEvalEmitter {
+	return &goEvalEmitter{
+		options: opt,
+		parser:  newOutputWriter("\t"),
+	}
+}
+
+func (g *goEvalEmitter) writePrelude() {
+	g.parser.write("package ")
+	g.parser.write(g.options.PackageName)
+	g.parser.writel("\n")
+
+	g.parser.write("import (\n")
+	g.parser.indent()
+	g.parser.writeil(`"encoding/binary"`)
+	g.parser.writeil(`"fmt"`)
+	g.parser.writeil(`"strconv"`)
+	g.parser.writeil(`"strings"`)
+
+	if !g.options.RemoveLib {
+		g.parser.writeil(`"io"`)
+	}
+
+	g.parser.unindent()
+	g.parser.writel(")\n")
+}
+
+func (g *goEvalEmitter) writeParserProgram(bt *Bytecode) {
+	g.parser.writel("var parserProgram = &Bytecode{")
+	g.parser.indent()
+
+	g.parser.writeil("code: []byte{")
+	g.parser.indent()
+	g.parser.writei("")
+	for _, byte := range bt.code {
+		g.parser.write(fmt.Sprintf("%d, ", byte))
+	}
+	g.parser.writel("")
+	g.parser.unindent()
+	g.parser.writeil("},")
+
+	g.parser.writeil("strs: []string{")
+	g.parser.indent()
+	g.parser.writei("")
+	for _, s := range bt.strs {
+		g.parser.write(fmt.Sprintf(`"%s", `, escapeLiteral(s)))
+	}
+	g.parser.writel("")
+	g.parser.unindent()
+	g.parser.writeil("},")
+
+	g.parser.writeil("rxps: map[int]int{")
+	g.parser.indent()
+	for k, v := range bt.rxps {
+		g.parser.write(fmt.Sprintf("%d: %d,", k, v))
+	}
+	g.parser.unindent()
+	g.parser.writeil("},")
+
+	g.parser.unindent()
+	g.parser.writel("}")
+}
+
+func (g *goEvalEmitter) writeParserStruct() {
+	g.parser.writel("type Parser struct{")
+	g.parser.indent()
+	g.parser.writeil("input     string")
+	g.parser.writeil("errLabels map[string]string")
+	g.parser.unindent()
+	g.parser.writel("}")
+}
+
+func (g *goEvalEmitter) writeParserConstructor() {
+	g.parser.writel("func NewParser() *Parser { return &Parser{} }")
+}
+
+func (g *goEvalEmitter) writeParserMethods(asm *Program) {
+	var (
+		cursor  = 0
+		addrmap = make(map[int]int, len(asm.identifiers))
+	)
+	for i, instruction := range asm.code {
+		switch instruction.(type) {
+		case ILabel:
+			addrmap[i] = cursor
+		default:
+			cursor += instruction.SizeInBytes()
+		}
+	}
+	for addr, strID := range asm.identifiers {
+		name := asm.strings[strID]
+		g.parser.write(fmt.Sprintf("func (p *Parser) Parse%s() (Value, error) { ", name))
+		g.parser.write(fmt.Sprintf("return p.parseFn(%d)", addrmap[addr]))
+		g.parser.writel(" }")
+	}
+	g.parser.writel("func (p *Parser) Parser() (Value, error) { return p.parseFn(5) }")
+	g.parser.writel("func (p *Parser) SetInput(input string) { p.input = input }")
+	g.parser.writel("func (p *Parser) SetLabelMessages(el map[string]string) { p.errLabels = el }")
+	g.parser.writel("func (p *Parser) parseFn(addr uint16) (Value, error) {")
+	g.parser.indent()
+	g.parser.writeil("writeU16(parserProgram.code[1:], addr)")
+	g.parser.writeil("vm := newVirtualMachine(parserProgram, p.errLabels)")
+	g.parser.writeil("val, _, err := vm.Match(strings.NewReader(p.input))")
+	g.parser.writeil("return val, err")
+	g.parser.unindent()
+	g.parser.writel("}")
+}
+
+func (g *goEvalEmitter) writeDeps() {
+	if g.options.RemoveLib {
+		return
+	}
+	for _, file := range []string{
+		"value.go", "tree_printer.go", "errors.go", "vm_stack.go", "vm.go",
+	} {
+		s, err := cleanGoModule(goEvalContent, file)
+		if err != nil {
+			panic(err.Error())
+		}
+		g.parser.write(s)
+	}
+}
+
+func (g *goEvalEmitter) output() (string, error) {
+	parserTmpl, err := template.New("parser").Parse(g.parser.buffer.String())
+	if err != nil {
+		return "", err
+	}
+	var output bytes.Buffer
+	vv := tmplRenderOpts{
+		PackageName: g.options.PackageName,
+	}
+	if err = parserTmpl.Execute(&output, vv); err != nil {
+		return "", err
+	}
+	return output.String(), nil
+}

go/tree_printer.go

@@ -56,7 +56,7 @@ func (tp *treePrinter[T]) pwrite(s string) {
 
 var literalSanitizer = strings.NewReplacer(
 	`"`, `\"`,
-	`'`, `\'`,
+	// `'`, `\'`,
 	`\`, `\\`,
 	string('\n'), `\n`,
 	string('\r'), `\r`,

go/vm_test.go

@@ -1,7 +1,7 @@
 package langlang
 
 import (
-	"fmt"
+	// "fmt"
 	"strings"
 	"testing"
 
@@ -312,28 +312,28 @@ func exec(expr, input string, optimize int) (Value, int, error) {
 	if err != nil {
 		panic(err)
 	}
-	// ast, err = InjectWhitespaces(ast)
-	// if err != nil {
-	// 	panic(err)
-	// }
-	// ast, err = AddBuiltins(ast)
-	// if err != nil {
-	// 	panic(err)
-	// }
+	ast, err = InjectWhitespaces(ast)
+	if err != nil {
+		panic(err)
+	}
+	ast, err = AddBuiltins(ast)
+	if err != nil {
+		panic(err)
+	}
 	ast, err = AddCaptures(ast)
 	if err != nil {
 		panic(err)
 	}
-	fmt.Printf("ast\n%s\n", ast.HighlightPrettyString())
+	// fmt.Printf("ast\n%s\n", ast.HighlightPrettyString())
 	asm, err := Compile(ast, CompilerConfig{Optimize: optimize})
 	if err != nil {
 		panic(err)
 	}
-	fmt.Printf("asm\n%s\n", asm.HighlightPrettyString())
+	// fmt.Printf("asm\n%s\n", asm.HighlightPrettyString())
 
 	code := Encode(asm)
 
-	fmt.Printf("code\n%#v\n", code.code)
+	// fmt.Printf("code\n%#v\n", code.code)
 
 	return code.Match(strings.NewReader(input))
 }