Support enabling verbose client-go logging (#1806)

* Support enabling verbose client-go logging

- Can set via extraEnvVars in helm chart
- Allows debugging of k8s client things

* Fix linting error

Author: foot

cmd/clusters-service/app/server.go

@@ -11,6 +11,7 @@ import (
 	"crypto/x509/pkix"
 	"encoding/pem"
 	"errors"
+	"flag"
 	"fmt"
 	"math/big"
 	"net"
@@ -31,7 +32,7 @@ import (
 	"github.com/go-logr/logr"
 	grpc_runtime "github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/spf13/cobra"
-	flag "github.com/spf13/pflag"
+	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	gitopsv1alpha1 "github.com/weaveworks/cluster-controller/api/v1alpha1"
 	"github.com/weaveworks/go-checkpoint"
@@ -81,6 +82,7 @@ import (
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	k8scache "k8s.io/client-go/tools/cache"
+	"k8s.io/klog/v2"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -143,6 +145,7 @@ type Params struct {
 	CostEstimationFilters             string                    `mapstructure:"cost-estimation-filters"`
 	CostEstimationAPIRegion           string                    `mapstructure:"cost-estimation-api-region"`
 	CostEstimationFilename            string                    `mapstructure:"cost-estimation-csv-file"`
+	KLogVerbosity                     string                    `mapstructure:"klog-verbosity"`
 }
 
 type OIDCAuthenticationOptions struct {
@@ -227,12 +230,13 @@ func NewAPIServerCommand(log logr.Logger, tempDir string) *cobra.Command {
 	cmdFlags.Bool("use-k8s-cached-clients", true, "Enables the use of cached clients")
 	cmdFlags.String("ui-config", "", "UI configuration, JSON encoded")
 	cmdFlags.String("pipeline-controller-address", pipelines.DefaultPipelineControllerAddress, "Pipeline controller address")
+	cmdFlags.String("klog-verbosity", "0", "Set the logging of the klog library")
 
 	cmdFlags.String("cost-estimation-filters", "", "Cost estimation filters")
 	cmdFlags.String("cost-estimation-api-region", "", "API region for cost estimation queries")
 	cmdFlags.String("cost-estimation-csv-file", "", "Filename to parse as Cost Estimation data")
 
-	cmdFlags.VisitAll(func(fl *flag.Flag) {
+	cmdFlags.VisitAll(func(fl *pflag.Flag) {
 		if strings.HasPrefix(fl.Name, "cost-estimation") {
 			cobra.CheckErr(cmdFlags.MarkHidden(fl.Name))
 		}
@@ -306,10 +310,33 @@ func initializeConfig(cmd *cobra.Command) error {
 	return nil
 }
 
-func StartServer(ctx context.Context, log logr.Logger, tempDir string, p Params) error {
+// configureKlogVerbosity sets the klog verbosity level.
+// This log is used by the client-go k8s libraries and can be useful for debugging
+// client-go's network requests.
+//
+// v=5 - log CRD cache things?
+// v=6 - log requests (e.g. GET url)
+// v=7 - log req/res headers
+// v=8 - log res body
+func configureKLogVerbosity(v string) error {
+	klog.InitFlags(nil)
+	err := flag.Set("v", v)
+	if err != nil {
+		return fmt.Errorf("could not set klog verbosity: %w", err)
+	}
+	flag.Parse()
+	return nil
+}
 
+func StartServer(ctx context.Context, log logr.Logger, tempDir string, p Params) error {
 	featureflags.SetFromEnv(os.Environ())
 
+	log.Info("Setting klog verbosity", "verbosity", p.KLogVerbosity)
+	err := configureKLogVerbosity(p.KLogVerbosity)
+	if err != nil {
+		return fmt.Errorf("could not configure klog verbosity: %w", err)
+	}
+
 	if p.CAPITemplatesNamespace == "" {
 		return errors.New("CAPI templates namespace not set")
 	}
@@ -327,7 +354,7 @@ func StartServer(ctx context.Context, log logr.Logger, tempDir string, p Params)
 		schemeBuilder = append(schemeBuilder, capiv1.AddToScheme)
 	}
 
-	err := schemeBuilder.AddToScheme(scheme)
+	err = schemeBuilder.AddToScheme(scheme)
 	if err != nil {
 		return err
 	}

go.mod

@@ -326,7 +326,7 @@ require (
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/apiserver v0.25.4 // indirect
 	k8s.io/component-base v0.25.4 // indirect
-	k8s.io/klog/v2 v2.80.1 // indirect
+	k8s.io/klog/v2 v2.80.1
 	k8s.io/kube-openapi v0.0.0-20221110221610-a28e98eb7c70 // indirect
 	k8s.io/kubectl v0.25.4 // indirect
 	k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2 // indirect

tools/dev-values.yaml

@@ -20,6 +20,9 @@ extraEnvVars:
     value: ""
   - name: WEAVE_GITOPS_FEATURE_TELEMETRY
     value: "false"
+  # Can bump this up to 4+ to get more verbose logging
+  - name: KLOG_VERBOSITY
+    value: "0"
 
 extraEnvVarsSecret: ""