Add HTML parsing to discover WordPress plugins

Author: Chocapikk

go.mod

@@ -1,8 +1,8 @@
 module github.com/Chocapikk/wpprobe
 
-go 1.22
+go 1.23.0
 
-toolchain go1.22.12
+toolchain go1.24.3
 
 require (
 	github.com/Masterminds/semver v1.5.0
@@ -11,7 +11,8 @@ require (
 	github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213
 	github.com/schollz/progressbar/v3 v3.18.0
 	github.com/spf13/cobra v1.8.1
-	golang.org/x/text v0.21.0
+	golang.org/x/net v0.40.0
+	golang.org/x/text v0.25.0
 )
 
 require (
@@ -25,6 +26,6 @@ require (
 	github.com/muesli/termenv v0.15.2 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/sys v0.30.0 // indirect
-	golang.org/x/term v0.28.0 // indirect
+	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/term v0.32.0 // indirect
 )

go.sum

@@ -45,13 +45,15 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
+golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
-golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
-golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
+golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
+golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
+golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/scanner/html.go

@@ -0,0 +1,84 @@
+// Copyright (c) 2025 Valentin Lobstein (Chocapikk) <[email protected]>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+// the Software, and to permit persons to whom the Software is furnished to do so,
+// subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+package scanner
+
+import (
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/Chocapikk/wpprobe/internal/utils"
+	"golang.org/x/net/html"
+)
+
+func discoverPluginsFromHTML(target string, headers []string) ([]string, error) {
+	normalized := utils.NormalizeURL(target) + "/"
+
+	client := utils.NewHTTPClient(10*time.Second, headers)
+	htmlContent, err := client.Get(normalized)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch homepage %s: %w", normalized, err)
+	}
+
+	slugsSet := make(map[string]struct{})
+	if err := extractSlugsFromReader(strings.NewReader(htmlContent), slugsSet); err != nil {
+		return nil, fmt.Errorf("failed to parse HTML %s: %w", normalized, err)
+	}
+
+	var slugs []string
+	for slug := range slugsSet {
+		slugs = append(slugs, slug)
+	}
+	return slugs, nil
+}
+
+func extractSlugsFromReader(r io.Reader, dest map[string]struct{}) error {
+	z := html.NewTokenizer(r)
+
+	for {
+		tt := z.Next()
+		switch tt {
+		case html.ErrorToken:
+			if z.Err() == io.EOF {
+				return nil
+			}
+			return z.Err()
+
+		case html.StartTagToken, html.SelfClosingTagToken:
+			t := z.Token()
+			for _, attr := range t.Attr {
+				val := strings.TrimSpace(attr.Val)
+				if val == "" {
+					continue
+				}
+				if attr.Key == "href" || attr.Key == "src" {
+					if idx := strings.Index(val, "/wp-content/plugins/"); idx != -1 {
+						rest := val[idx+len("/wp-content/plugins/"):]
+						parts := strings.SplitN(rest, "/", 2)
+						if len(parts) > 0 && parts[0] != "" {
+							dest[parts[0]] = struct{}{}
+						}
+					}
+				}
+			}
+		}
+	}
+}

internal/scanner/html_test.go

@@ -0,0 +1,118 @@
+// html_test.go
+// Tests for HTML‐based plugin discovery functions.
+
+package scanner
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func sortedSlice(ss []string) []string {
+	s := append([]string(nil), ss...)
+	sort.Strings(s)
+	return s
+}
+
+func TestExtractSlugsFromReader(t *testing.T) {
+	tests := []struct {
+		name      string
+		html      string
+		wantSlugs []string
+	}{
+		{
+			name: "Single plugin in href",
+			html: `<html><head>
+				<link rel="stylesheet" href="https://example.com/wp-content/plugins/pluginA/style.css" />
+			</head><body></body></html>`,
+			wantSlugs: []string{"pluginA"},
+		},
+		{
+			name: "Multiple plugins in href and src",
+			html: `<html><body>
+				<img src="/wp-content/plugins/pluginB/images/img.png" />
+				<a href="http://foo/wp-content/plugins/pluginC/file.php"></a>
+			</body></html>`,
+			wantSlugs: []string{"pluginB", "pluginC"},
+		},
+		{
+			name: "Duplicate slugs and nested paths",
+			html: `<html><body>
+				<script src="/wp-content/plugins/pluginA/js/app.js"></script>
+				<link href="/wp-content/plugins/pluginA/css/app.css" rel="stylesheet">
+				<img src="/some/other/path/pluginA/wp-content/plugins/pluginD/img.jpg">
+			</body></html>`,
+			wantSlugs: []string{"pluginA", "pluginD"},
+		},
+		{
+			name:      "No plugin references",
+			html:      `<html><body><p>No plugins here</p></body></html>`,
+			wantSlugs: []string{},
+		},
+		{
+			name: "Malformed attributes",
+			html: `<html><body>
+				<a href="wp-content/plugins//style.css"></a>
+				<a href="/wp-content/plugins/"></a>
+			</body></html>`,
+			wantSlugs: []string{},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dest := make(map[string]struct{})
+			err := extractSlugsFromReader(strings.NewReader(tt.html), dest)
+			if err != nil {
+				t.Fatalf("extractSlugsFromReader returned error: %v", err)
+			}
+
+			var got []string
+			for slug := range dest {
+				got = append(got, slug)
+			}
+			got = sortedSlice(got)
+			want := sortedSlice(tt.wantSlugs)
+
+			if !reflect.DeepEqual(got, want) {
+				t.Errorf("extractSlugsFromReader = %v, want %v", got, want)
+			}
+		})
+	}
+}
+
+func TestDiscoverPluginsFromHTML(t *testing.T) {
+	const sampleHTML = `<!DOCTYPE html>
+<html>
+<head>
+	<link rel="stylesheet" href="/wp-content/plugins/pluginX/css/style.css">
+	<script src="http://host/wp-content/plugins/pluginY/js/app.js"></script>
+</head>
+<body>
+	<img src="/wp-content/plugins/pluginZ/images/pic.png" alt="image">
+	<a href="/some/other/path"></a>
+</body>
+</html>`
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(sampleHTML))
+	}))
+	defer ts.Close()
+
+	slugs, err := discoverPluginsFromHTML(ts.URL, nil)
+	if err != nil {
+		t.Fatalf("discoverPluginsFromHTML returned error: %v", err)
+	}
+
+	got := sortedSlice(slugs)
+	want := sortedSlice([]string{"pluginX", "pluginY", "pluginZ"})
+
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("discoverPluginsFromHTML = %v, want %v", got, want)
+	}
+}

internal/scanner/scan.go

@@ -20,6 +20,7 @@
 package scanner
 
 import (
+	"fmt"
 	"math"
 	"sync"
 
@@ -100,6 +101,15 @@ func performStealthyScan(
 	opts ScanOptions,
 	progress *utils.ProgressManager,
 ) ([]string, PluginDetectionResult) {
+	if progress != nil && opts.File == "" {
+		progress.SetMessage("🔎 Discovering plugins from HTML...")
+	}
+
+	htmlSlugs, err := discoverPluginsFromHTML(target, opts.Headers)
+	if err != nil {
+		utils.DefaultLogger.Warning(fmt.Sprintf("HTML discovery failed on %s: %v", target, err))
+	}
+
 	if progress != nil && opts.File == "" {
 		progress.SetMessage("🔎 Scanning REST API endpoints...")
 	}
@@ -116,14 +126,29 @@ func performStealthyScan(
 	}
 
 	endpoints := FetchEndpoints(target, opts.Headers)
-	if len(endpoints) == 0 {
-		if opts.File == "" {
-			utils.DefaultLogger.Warning("No REST endpoints found on " + target)
+
+	var result PluginDetectionResult
+	if len(endpoints) > 0 {
+		result = DetectPlugins(endpoints, endpointsData)
+	} else {
+		result = PluginDetectionResult{
+			Plugins:  make(map[string]*PluginData),
+			Detected: nil,
+		}
+	}
+
+	for _, slug := range htmlSlugs {
+		if _, exists := result.Plugins[slug]; !exists {
+			result.Plugins[slug] = &PluginData{
+				Score:      1,
+				Confidence: 50.0,
+				Ambiguous:  false,
+				Matches:    nil,
+			}
+			result.Detected = append(result.Detected, slug)
 		}
-		return nil, PluginDetectionResult{}
 	}
 
-	result := DetectPlugins(endpoints, endpointsData)
 	if len(result.Detected) == 0 {
 		return nil, result
 	}