Fix the Trie matching bug (#493)

Fixes #379

Author: apocelipes

examples/issue379/filename.py

@@ -0,0 +1,7 @@
+a = ''
+# equals to 'test'
+b = 'test'''
+
+# comment
+for i in range(10):
+    print(i)

processor/structs.go

@@ -183,11 +183,20 @@ func (root *Trie) Match(token []byte) (int, int, []byte) {
 	var c byte
 
 	node = root
+	var prevClosedNode *Trie
+	var prevClosedDepth int
 	for depth, c = range token {
 		if node.Table[c] == nil {
-			return node.Type, depth, node.Close
+			break
 		}
 		node = node.Table[c]
+		if len(node.Close) > 0 {
+			prevClosedNode = node
+			prevClosedDepth = depth
+		}
+	}
+	if len(node.Close) == 0 && prevClosedNode != nil {
+		return prevClosedNode.Type, prevClosedDepth, prevClosedNode.Close
 	}
 	return node.Type, depth, node.Close
 }

processor/structs_test.go

@@ -3,6 +3,7 @@
 package processor
 
 import (
+	"slices"
 	"testing"
 )
 
@@ -30,3 +31,66 @@ func TestCheckDuplicates(t *testing.T) {
 		t.Error("Expected no match")
 	}
 }
+
+func TestMatch(t *testing.T) {
+	trie := &Trie{}
+	trie.InsertClose(TString, []byte("'"), []byte("'"))
+	trie.InsertClose(TString, []byte("'''"), []byte("'''"))
+
+	testCases := []struct {
+		token        []byte
+		expectType   int
+		expectDepth  int
+		expectClosed []byte
+	}{
+		{
+			token:        []byte("'"),
+			expectType:   TString,
+			expectDepth:  0,
+			expectClosed: []byte("'"),
+		},
+		{
+			token:        []byte("-"),
+			expectType:   0,
+			expectDepth:  0,
+			expectClosed: []byte{},
+		},
+		{
+			token:        []byte("'-"),
+			expectType:   TString,
+			expectDepth:  1,
+			expectClosed: []byte("'"),
+		},
+		{
+			token:        []byte("''"),
+			expectType:   TString,
+			expectDepth:  0,
+			expectClosed: []byte("'"),
+		},
+		{
+			token:        []byte("'''"),
+			expectType:   1,
+			expectDepth:  2,
+			expectClosed: []byte("'''"),
+		},
+		{
+			token:        []byte("'''a'''"),
+			expectType:   TString,
+			expectDepth:  3,
+			expectClosed: []byte("'''"),
+		},
+	}
+
+	for _, tc := range testCases {
+		typ, depth, closed := trie.Match(tc.token)
+		if typ != tc.expectType {
+			t.Errorf("\"%v\" matched wrong type, want: %v, got: %v", string(tc.token), tc.expectType, typ)
+		}
+		if depth != tc.expectDepth {
+			t.Errorf("\"%v\" matched wrong depth, want: %v, got: %v", string(tc.token), tc.expectDepth, depth)
+		}
+		if !slices.Equal(tc.expectClosed, closed) {
+			t.Errorf("\"%v\" matched wrong closed, want: %v, got: %v", string(tc.token), tc.expectClosed, closed)
+		}
+	}
+}

test-all.sh

@@ -935,7 +935,7 @@ done
 a=$(./scc "examples/issue345/" -f csv | sed -n '2 p')
 b="C++,4,3,1,0,0,76,1,0"
 if [ "$a" == "$b" ]; then
-    echo -e "{GREEN}PASSED string termination check"
+    echo -e "${GREEN}PASSED string termination check"
 else
     echo -e "$a"
     echo -e "${RED}======================================================="
@@ -944,6 +944,41 @@ else
     exit
 fi
 
+# Regression issue https://github.com/boyter/scc/issues/379
+Issue379Line=$(./scc -f csv "examples/issue379/" | grep 'Python' | cut -d ',' -f 2)
+Issue379Code=$(./scc -f csv "examples/issue379/" | grep 'Python' | cut -d ',' -f 3)
+Issue379Comments=$(./scc -f csv "examples/issue379/" | grep 'Python' | cut -d ',' -f 4)
+Issue379Blanks=$(./scc -f csv "examples/issue379/" | grep 'Python' | cut -d ',' -f 5)
+Issue379Complexity=$(./scc -f csv "examples/issue379/" | grep 'Python' | cut -d ',' -f 6)
+if [ $Issue379Line -ne 7 ] ; then
+    echo -e "${RED}======================================================="
+    echo -e "FAILED Issue379 line counting"
+    echo -e "=======================================================${NC}"
+    exit
+elif [ $Issue379Code -ne 4 ] ; then
+    echo -e "${RED}======================================================="
+    echo -e "FAILED Issue379 code counting"
+    echo -e "=======================================================${NC}"
+    exit
+elif [ $Issue379Comments -ne 2 ] ; then
+    echo -e "${RED}======================================================="
+    echo -e "FAILED Issue379 comments counting"
+    echo -e "=======================================================${NC}"
+    exit
+elif [ $Issue379Blanks -ne 1 ] ; then
+    echo -e "${RED}======================================================="
+    echo -e "FAILED Issue379 blanks counting"
+    echo -e "=======================================================${NC}"
+    exit
+elif [ $Issue379Complexity -ne 1 ] ; then
+    echo -e "${RED}======================================================="
+    echo -e "FAILED Issue379 complexity counting"
+    echo -e "=======================================================${NC}"
+    exit
+else
+    echo -e "${GREEN}PASSED Issue379 Regression Check"
+fi
+
 # Extra case for longer languages that are normally truncated
 for i in 'CloudFormation (YAM' 'CloudFormation (JSO'
 do