Merge pull request #654 from souvikinator/souvik/fix/linkerd-model-generation

Add support for Helm CRD template sanitization to fix YAML parsing for Linkerd

Author: leecalcote

generators/github/git_repo.go

@@ -39,14 +39,16 @@ func (gr GitRepo) GetContent() (models.Package, error) {
 	dirPath := filepath.Join(os.TempDir(), owner, repo, branch)
 	_ = os.MkdirAll(dirPath, 0755)
 	filePath := filepath.Join(dirPath, utils.GetRandomAlphabetsOfDigit(5))
-	fd, err := os.Create(filePath) // perform cleanup
+	fd, err := os.Create(filePath)
 	if err != nil {
+		os.RemoveAll(dirPath)
 		return nil, utils.ErrCreateFile(err, filePath)
 	}
 	br := bufio.NewWriter(fd)
 
 	defer func() {
-		_ = br.Flush()
+		br.Flush()
+		fd.Close()
 	}()
 	gw := gitWalker.
 		Owner(owner).

utils/helm/helm.go

@@ -145,29 +145,147 @@ func LoadHelmChart(path string, w io.Writer, extractOnlyCrds bool, kubeVersion s
 	if err != nil {
 		return ErrLoadHelmChart(err, path)
 	}
-	if extractOnlyCrds {
-		crds := chart.CRDObjects()
-		size := len(crds)
-		for index, crd := range crds {
-			_, err := w.Write(crd.File.Data)
-			if err != nil {
-				errs = append(errs, err)
-				continue
-			}
-			if index == size-1 {
-				break
-			}
-			_, _ = w.Write([]byte("\n---\n"))
-		}
-	} else {
+
+	if !extractOnlyCrds {
 		manifests, err := DryRunHelmChart(chart, kubeVersion)
 		if err != nil {
 			return ErrLoadHelmChart(err, path)
 		}
 		_, err = w.Write(manifests)
+		return err
+	}
+
+	// Look for all the yaml file in the helm dir that is a CRD
+	err = filepath.WalkDir(path, func(filePath string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return ErrLoadHelmChart(err, path)
 		}
+		if !d.IsDir() && (strings.HasSuffix(filePath, ".yaml") || strings.HasSuffix(filePath, ".yml")) {
+			data, err := os.ReadFile(filePath)
+			if err != nil {
+				return err
+			}
+
+			if isCRDFile(data) {
+				data = RemoveHelmPlaceholders(data)
+				if err := writeToWriter(w, data); err != nil {
+					errs = append(errs, err)
+				}
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		errs = append(errs, err)
 	}
+
 	return utils.CombineErrors(errs, "\n")
 }
+
+func writeToWriter(w io.Writer, data []byte) error {
+	trimmedData := bytes.TrimSpace(data)
+
+	if len(trimmedData) == 0 {
+		return nil
+	}
+
+	// Check if the document already starts with separators
+	startsWithSeparator := bytes.HasPrefix(trimmedData, []byte("---"))
+
+	// If it doesn't start with ---, add one
+	if !startsWithSeparator {
+		if _, err := w.Write([]byte("---\n")); err != nil {
+			return err
+		}
+	}
+
+	if _, err := w.Write(trimmedData); err != nil {
+		return err
+	}
+
+	_, err := w.Write([]byte("\n"))
+	return err
+}
+
+// checks if the content is a CRD
+// NOTE: kubernetes.IsCRD(manifest string) already exists however using that leads to cyclic dependency
+func isCRDFile(content []byte) bool {
+	str := string(content)
+	return strings.Contains(str, "kind: CustomResourceDefinition")
+}
+
+// RemoveHelmPlaceholders - replaces helm templates placeholder with YAML compatible empty value
+// since these templates cause YAML parsing error
+// NOTE: this is a quick fix
+func RemoveHelmPlaceholders(data []byte) []byte {
+	content := string(data)
+
+	// Regular expressions to match different Helm template patterns
+	// Match multiline template blocks that start with {{- and end with }}
+	multilineRegex := regexp.MustCompile(`(?s){{-?\s*.*?\s*}}`)
+
+	// Match single line template expressions
+	singleLineRegex := regexp.MustCompile(`{{-?\s*[^}]*}}`)
+
+	// Process the content line by line to maintain YAML structure
+	lines := strings.Split(content, "\n")
+	var processedLines []string
+
+	for _, line := range lines {
+		trimmedLine := strings.TrimSpace(line)
+		if trimmedLine == "" {
+			processedLines = append(processedLines, line)
+			continue
+		}
+
+		// Handle multiline template blocks first
+		if multilineRegex.MatchString(line) {
+			// If line starts with indentation + list marker
+			if listMatch := regexp.MustCompile(`^(\s*)- `).FindStringSubmatch(line); listMatch != nil {
+				// Convert list item to empty map to maintain structure
+				processedLines = append(processedLines, listMatch[1]+"- {}")
+				continue
+			}
+
+			// If it's a value assignment with multiline template
+			if valueMatch := regexp.MustCompile(`^(\s*)(\w+):\s*{{`).FindStringSubmatch(line); valueMatch != nil {
+				// Preserve the key with empty map value
+				processedLines = append(processedLines, valueMatch[1]+valueMatch[2]+": {}")
+				continue
+			}
+
+			// For other multiline templates, replace with empty line
+			processedLines = append(processedLines, "")
+			continue
+		}
+
+		// Handle single line template expressions
+		if singleLineRegex.MatchString(line) {
+			// If line contains a key-value pair
+			if keyMatch := regexp.MustCompile(`^(\s*)(\w+):\s*{{`).FindStringSubmatch(line); keyMatch != nil {
+				// Preserve the key with empty string value
+				processedLines = append(processedLines, keyMatch[1]+keyMatch[2]+": ")
+				continue
+			}
+
+			// If line is a list item
+			if listMatch := regexp.MustCompile(`^(\s*)- `).FindStringSubmatch(line); listMatch != nil {
+				// Convert to empty map to maintain list structure
+				processedLines = append(processedLines, listMatch[1]+"- {}")
+				continue
+			}
+
+			// For standalone template expressions, remove them (includes, control statements)
+			line = singleLineRegex.ReplaceAllString(line, "")
+			if strings.TrimSpace(line) != "" {
+				processedLines = append(processedLines, line)
+			}
+			continue
+		}
+
+		processedLines = append(processedLines, line)
+	}
+
+	return []byte(strings.Join(processedLines, "\n"))
+}

utils/utils.go

@@ -841,33 +841,27 @@ func ParseKubeStatusErr(err *kubeerror.StatusError) (shortDescription, longDescr
 
 	status := err.Status()
 
-	// Create base description including status code
-	baseDesc := fmt.Sprintf("[Status Code: %d] %s", status.Code, status.Reason)
-	if status.Details != nil {
-		baseDesc = fmt.Sprintf("%s: %s '%s'", baseDesc, status.Details.Kind, status.Details.Name)
-	}
-	shortDescription = append(shortDescription, baseDesc)
-
-	// Add the main error message
-	longDescription = append(longDescription, status.Message)
+	// Add the high-level error message with status code to longDescription
+	longDescription = append(longDescription, fmt.Sprintf("[Status Code: %d] %s", status.Code, status.Message))
 
-	// Add reason-based guidance
 	pc, rem := handleStatusReason(status.Reason)
 	probableCause = append(probableCause, pc)
 	remedy = append(remedy, rem)
 
-	// Process detailed causes if available
+	// Add specific field validation errors
 	if status.Details != nil && len(status.Details.Causes) > 0 {
 		for _, cause := range status.Details.Causes {
-			// Add detailed cause description
-			longDescription = append(longDescription,
-				fmt.Sprintf("Field '%s': %s", cause.Field, cause.Message))
+			longDescription = append(longDescription, fmt.Sprintf("Field '%s': %s", cause.Field, cause.Message))
 
-			// Get cause-specific messages
 			pc, rem := handleStatusCause(cause, status.Details.Kind)
 			probableCause = append(probableCause, pc)
 			remedy = append(remedy, rem)
 		}
+	} else {
+		// If no specific causes are provided, add the general reason-based guidance
+		pc, rem := handleStatusReason(status.Reason)
+		probableCause = append(probableCause, pc)
+		remedy = append(remedy, rem)
 	}
 
 	return