feat: unescape any HTML entities

Any HTML entities of URL-decoded characters should be unescape before
perform the checks.

$ curl localhost:3000 -so /dev/null -w "%{http_code}\n" -A X -d "body=%22autofocus%20onFocUs=%27%26%2397%3blert()%27"
403

Author: dwisiswant0

README.md

@@ -194,22 +194,22 @@ goos: linux
 goarch: amd64
 pkg: github.com/kitabisa/teler-waf
 cpu: 11th Gen Intel(R) Core(TM) i9-11900H @ 2.50GHz
-BenchmarkTelerDefaultOptions-4               	    4530	    265197 ns/op	   35710 B/op	    1690 allocs/op
-BenchmarkTelerCommonWebAttackOnly-4          	   32484	     35325 ns/op	    5949 B/op	     118 allocs/op
-BenchmarkTelerCVEOnly-4                      	    6248	    187397 ns/op	   33402 B/op	    1647 allocs/op
-BenchmarkTelerBadIPAddressOnly-4             	   20649	     54890 ns/op	    5974 B/op	      86 allocs/op
-BenchmarkTelerBadReferrerOnly-4              	   48594	     22629 ns/op	    5548 B/op	      87 allocs/op
-BenchmarkTelerBadCrawlerOnly-4               	   41832	     26891 ns/op	    5634 B/op	      85 allocs/op
-BenchmarkTelerDirectoryBruteforceOnly-4      	   48087	     22008 ns/op	    5554 B/op	      84 allocs/op
-BenchmarkTelerCustomRule-4                   	   50428	     21523 ns/op	    5323 B/op	      84 allocs/op
-BenchmarkTelerWithoutCommonWebAttack-4       	    5133	    230608 ns/op	   34619 B/op	    1654 allocs/op
-BenchmarkTelerWithoutCVE-4                   	   15229	     75995 ns/op	    7169 B/op	     124 allocs/op
-BenchmarkTelerWithoutBadIPAddress-4          	    5677	    211478 ns/op	   34602 B/op	    1685 allocs/op
-BenchmarkTelerWithoutBadReferrer-4           	    4875	    240689 ns/op	   35127 B/op	    1684 allocs/op
-BenchmarkTelerWithoutBadCrawler-4            	    4922	    238995 ns/op	   35000 B/op	    1686 allocs/op
-BenchmarkTelerWithoutDirectoryBruteforce-4   	    4894	    242973 ns/op	   35241 B/op	    1687 allocs/op
+BenchmarkTelerDefaultOptions-4               	    4396	    266918 ns/op	   35944 B/op	    1696 allocs/op
+BenchmarkTelerCommonWebAttackOnly-4          	   30795	     35602 ns/op	    5990 B/op	     118 allocs/op
+BenchmarkTelerCVEOnly-4                      	    6171	    194193 ns/op	   33533 B/op	    1652 allocs/op
+BenchmarkTelerBadIPAddressOnly-4             	   20464	     55957 ns/op	    5986 B/op	      86 allocs/op
+BenchmarkTelerBadReferrerOnly-4              	   48403	     23128 ns/op	    5551 B/op	      87 allocs/op
+BenchmarkTelerBadCrawlerOnly-4               	   42002	     27165 ns/op	    5633 B/op	      85 allocs/op
+BenchmarkTelerDirectoryBruteforceOnly-4      	   50103	     23074 ns/op	    5535 B/op	      84 allocs/op
+BenchmarkTelerCustomRule-4                   	   49483	     22086 ns/op	    5332 B/op	      84 allocs/op
+BenchmarkTelerWithoutCommonWebAttack-4       	    5156	    228950 ns/op	   34683 B/op	    1658 allocs/op
+BenchmarkTelerWithoutCVE-4                   	   15295	     76501 ns/op	    7167 B/op	     124 allocs/op
+BenchmarkTelerWithoutBadIPAddress-4          	    5484	    216523 ns/op	   34820 B/op	    1691 allocs/op
+BenchmarkTelerWithoutBadReferrer-4           	    4894	    240202 ns/op	   35133 B/op	    1689 allocs/op
+BenchmarkTelerWithoutBadCrawler-4            	    5012	    239976 ns/op	   34995 B/op	    1691 allocs/op
+BenchmarkTelerWithoutDirectoryBruteforce-4   	    4736	    247549 ns/op	   35496 B/op	    1693 allocs/op
 PASS
-ok  	github.com/kitabisa/teler-waf	23.207s
+ok  	github.com/kitabisa/teler-waf	23.660s
 ```
 
 > **Note**: It's important to note that the benchmarking results may vary and may not be consistent. Those results were obtained when there were **>1.5k** CVE templates and the [teler-resources](https://github.com/kitabisa/teler-resources) dataset may have increased since then, which may impact the results.

analyze.go

@@ -110,8 +110,8 @@ func (t *Teler) checkCustomRules(r *http.Request) error {
 	// Converts map of headers to RAW string
 	headers := headersToRawString(r.Header)
 
-	// Decode the URL-encoded request URI of the URL
-	uri := toURLDecode(r.URL.RequestURI())
+	// Decode the URL-encoded and unescape HTML entities request URI of the URL
+	uri := stringDeUnescape(r.URL.RequestURI())
 
 	// Declare byte slice for request body.
 	var body string
@@ -130,8 +130,8 @@ func (t *Teler) checkCustomRules(r *http.Request) error {
 		body = buf.String()
 	}
 
-	// Decode the URL-encoded of body
-	body = toURLDecode(body)
+	// Decode the URL-encoded and unescape HTML entities of body
+	body = stringDeUnescape(body)
 
 	// Iterate over the Customs field of the Teler struct, which is a slice of custom rules
 	for _, rule := range t.opt.Customs {
@@ -201,8 +201,8 @@ func (t *Teler) checkCustomRules(r *http.Request) error {
 // If a match is found, it returns an error indicating a common web attack has been detected.
 // If no match is found, it returns nil.
 func (t *Teler) checkCommonWebAttack(r *http.Request) error {
-	// Decode the URL-encoded request URI of the URL
-	uri := toURLDecode(r.URL.RequestURI())
+	// Decode the URL-encoded and unescape HTML entities request URI of the URL
+	uri := stringDeUnescape(r.URL.RequestURI())
 
 	// Declare byte slice for request body.
 	var body string
@@ -221,8 +221,8 @@ func (t *Teler) checkCommonWebAttack(r *http.Request) error {
 		body = buf.String()
 	}
 
-	// Decode the URL-encoded of body
-	body = toURLDecode(body)
+	// Decode the URL-encoded and unescape HTML entities of body
+	body = stringDeUnescape(body)
 
 	// Iterate over the filters in the CommonWebAttack data stored in the t.threat.cwa.Filters field
 	for _, filter := range t.threat.cwa.Filters {

utils.go

@@ -2,6 +2,7 @@ package teler
 
 import (
 	"fmt"
+	"html"
 	"strings"
 
 	"net/http"
@@ -69,11 +70,24 @@ func headersToRawString(headers http.Header) string {
 	return h.String()
 }
 
-// toURLDecode decode URL-decoded characters string using mdurl package
+// unescapeHTML to unescapes any HTML entities, i.e. á"
+// unescapes to "á", as does "á" and "á".
+func unescapeHTML(s string) string {
+	return html.UnescapeString(s)
+}
+
+// toURLDecode decode URL-decoded characters string using mdurl
 func toURLDecode(s string) string {
 	return mdurl.Decode(s)
 }
 
+// stringDeUnescape to decode URL-decoded characters, and
+// unescapes any HTML entities
+func stringDeUnescape(s string) string {
+	s = toURLDecode(s)
+	return unescapeHTML(s)
+}
+
 // isValidMethod check if the given request.Method is valid
 func isValidMethod(method request.Method) bool {
 	switch method {