Merge pull request #1 from autero1/auto_retry_implementation

Implement configuration init.

Author: autero1

cli/args.go

@@ -99,6 +99,7 @@ func parseTerragruntOptionsFromArgs(args []string, writer, errWriter io.Writer)
 
 	opts.TerraformPath = filepath.ToSlash(terraformPath)
 	opts.AutoInit = !parseBooleanArg(args, OPT_TERRAGRUNT_NO_AUTO_INIT, os.Getenv("TERRAGRUNT_AUTO_INIT") == "false")
+	opts.AutoRetry = !parseBooleanArg(args, OPT_TERRAGRUNT_NO_AUTO_RETRY, os.Getenv("TERRAGRUNT_AUTO_RETRY") == "false")
 	opts.NonInteractive = parseBooleanArg(args, OPT_NON_INTERACTIVE, os.Getenv("TF_INPUT") == "false" || os.Getenv("TF_INPUT") == "0")
 	opts.TerraformCliArgs = filterTerragruntArgs(args)
 	opts.TerraformCommand = util.FirstArg(opts.TerraformCliArgs)

cli/cli_app.go

@@ -25,6 +25,7 @@ import (
 const OPT_TERRAGRUNT_CONFIG = "terragrunt-config"
 const OPT_TERRAGRUNT_TFPATH = "terragrunt-tfpath"
 const OPT_TERRAGRUNT_NO_AUTO_INIT = "terragrunt-no-auto-init"
+const OPT_TERRAGRUNT_NO_AUTO_RETRY = "terragrunt-no-auto-retry"
 const OPT_NON_INTERACTIVE = "terragrunt-non-interactive"
 const OPT_WORKING_DIR = "terragrunt-working-dir"
 const OPT_DOWNLOAD_DIR = "terragrunt-download-dir"
@@ -34,7 +35,7 @@ const OPT_TERRAGRUNT_IAM_ROLE = "terragrunt-iam-role"
 const OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS = "terragrunt-ignore-dependency-errors"
 const OPT_TERRAGRUNT_EXCLUDE_DIR = "terragrunt-exclude-dir"
 
-var ALL_TERRAGRUNT_BOOLEAN_OPTS = []string{OPT_NON_INTERACTIVE, OPT_TERRAGRUNT_SOURCE_UPDATE, OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS, OPT_TERRAGRUNT_NO_AUTO_INIT}
+var ALL_TERRAGRUNT_BOOLEAN_OPTS = []string{OPT_NON_INTERACTIVE, OPT_TERRAGRUNT_SOURCE_UPDATE, OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS, OPT_TERRAGRUNT_NO_AUTO_INIT, OPT_TERRAGRUNT_NO_AUTO_RETRY}
 var ALL_TERRAGRUNT_STRING_OPTS = []string{OPT_TERRAGRUNT_CONFIG, OPT_TERRAGRUNT_TFPATH, OPT_WORKING_DIR, OPT_DOWNLOAD_DIR, OPT_TERRAGRUNT_SOURCE, OPT_TERRAGRUNT_IAM_ROLE, OPT_TERRAGRUNT_EXCLUDE_DIR}
 
 const CMD_PLAN_ALL = "plan-all"
@@ -108,6 +109,7 @@ GLOBAL OPTIONS:
    terragrunt-config                    Path to the Terragrunt config file. Default is terraform.tfvars.
    terragrunt-tfpath                    Path to the Terraform binary. Default is terraform (on PATH).
    terragrunt-no-auto-init              Don't automatically run 'terraform init' during other terragrunt commands. You must run 'terragrunt init' manually.
+   terragrunt-no-auto-retry             Don't automatically re-run command or in case of transient errors.
    terragrunt-non-interactive           Assume "yes" for all prompts.
    terragrunt-working-dir               The path to the Terraform templates. Default is current directory.
    terragrunt-download-dir              The path where to download Terraform code. Default is .terragrunt-cache in the working directory.

options/auto_retry_options.go

@@ -0,0 +1,18 @@
+package options
+
+const DEFAULT_MAX_RETRY_ATTEMPTS = 3
+const DEFAULT_SLEEP = 5
+
+var RETRYABLE_ERRORS = []string{
+	".*Failed to load state.*tcp.*timeout.*",
+	".*Failed to load backend.*TLS handshake timeout.*",
+	".*Creating metric alarm failed.*request to update this alarm is in progress.*",
+	".*Error installing provider.*TLS handshake timeout.*",
+}
+
+var ERRORS_REQUIRING_INIT = []string{
+	".*Failed to load state.*tcp.*timeout.*",
+	".*Failed to load backend.*TLS handshake timeout.*",
+	".*Creating metric alarm failed.*request to update this alarm is in progress.*",
+	".*Error installing provider.*TLS handshake timeout.*",
+}

options/options.go

@@ -80,6 +80,21 @@ type TerragruntOptions struct {
 	// exposed here primarily so we can set it to a low value at test time.
 	MaxFoldersToCheck int
 
+	// Whether we should automatically run terraform init if necessary when executing other commands
+	AutoRetry bool
+
+	// Maximum number of times to retry errors matching RetryableErrors
+	MaxRetryAttempts int
+
+	// Sleep is the duration in seconds to wait before retrying
+	Sleep int
+
+	// RetryableErrors is an array of regular expressions with RE2 syntax (https://github.com/google/re2/wiki/Syntax) that qualify for retrying
+	RetryableErrors []string
+
+	// ErrorsRequiringInit is an array of regular expressions with RE2 syntax (https://github.com/google/re2/wiki/Syntax) that qualify for re-running init
+	ErrorsRequiringInit []string
+
 	// Unix-style glob of directories to exclude when running *-all commands
 	ExcludeDirs []string
 
@@ -117,6 +132,11 @@ func NewTerragruntOptions(terragruntConfigPath string) (*TerragruntOptions, erro
 		Writer:                 os.Stdout,
 		ErrWriter:              os.Stderr,
 		MaxFoldersToCheck:      DEFAULT_MAX_FOLDERS_TO_CHECK,
+		AutoRetry:              true,
+		MaxRetryAttempts:       DEFAULT_MAX_RETRY_ATTEMPTS,
+		Sleep:                  DEFAULT_SLEEP,
+		RetryableErrors:        util.CloneStringList(RETRYABLE_ERRORS),
+		ErrorsRequiringInit:    util.CloneStringList(ERRORS_REQUIRING_INIT),
 		ExcludeDirs:            []string{},
 		RunTerragrunt: func(terragruntOptions *TerragruntOptions) error {
 			return errors.WithStackTrace(RunTerragruntCommandNotSet)
@@ -178,6 +198,11 @@ func (terragruntOptions *TerragruntOptions) Clone(terragruntConfigPath string) *
 		Writer:                 terragruntOptions.Writer,
 		ErrWriter:              terragruntOptions.ErrWriter,
 		MaxFoldersToCheck:      terragruntOptions.MaxFoldersToCheck,
+		AutoRetry:              terragruntOptions.AutoRetry,
+		MaxRetryAttempts:       terragruntOptions.MaxRetryAttempts,
+		Sleep:                  terragruntOptions.Sleep,
+		RetryableErrors:        util.CloneStringList(terragruntOptions.RetryableErrors),
+		ErrorsRequiringInit:    util.CloneStringList(terragruntOptions.ErrorsRequiringInit),
 		ExcludeDirs:            terragruntOptions.ExcludeDirs,
 		RunTerragrunt:          terragruntOptions.RunTerragrunt,
 	}

test/fixture-auto-retry/re-run/main.tf

@@ -0,0 +1,6 @@
+resource "null_resource" "tf_retryable_error" {
+  provisioner "local-exec" {
+    command = "if [[ -f terraform.tfstate.backup ]]; then exit 0; else echo 'Error: Error loading modules: module foo: not found, mayx need to run 'terraform init'' && exit 1; fi"
+    interpreter = ["/bin/bash", "-c"]
+  }
+}

test/fixture-auto-retry/re-run/terraform.tfvars

@@ -0,0 +1 @@
+terragrunt = {}

test/integration_test.go

@@ -73,6 +73,7 @@ const (
 	TEST_FIXTURE_HOOKS_INIT_ONCE_WITH_SOURCE_WITH_BACKEND   = "fixture-hooks/init-once/with-source-with-backend"
 	TEST_FIXTURE_FAILED_TERRAFORM                           = "fixture-failure"
 	TEST_FIXTURE_EXIT_CODE                                  = "fixture-exit-code"
+	TEST_FIXTURE_AUTO_RETRY_RERUN                           = "fixture-auto-retry/re-run"
 	TERRAFORM_FOLDER                                        = ".terraform"
 	TERRAFORM_STATE                                         = "terraform.tfstate"
 	TERRAFORM_STATE_BACKUP                                  = "terraform.tfstate.backup"
@@ -839,6 +840,19 @@ func TestPriorityOrderOfArgument(t *testing.T) {
 	assert.Contains(t, out.String(), fmt.Sprintf("test = %s", injectedValue))
 }
 
+//func TestAutoRetryRerun(t *testing.T) {
+//	t.Parallel()
+//
+//	out := new(bytes.Buffer)
+//	rootPath := copyEnvironment(t, TEST_FIXTURE_AUTO_RETRY_RERUN)
+//	modulePath := util.JoinPath(rootPath, TEST_FIXTURE_AUTO_RETRY_RERUN)
+//	err := runTerragruntCommand(t, fmt.Sprintf("terragrunt apply --auto-approve --terragrunt-non-interactive --terragrunt-working-dir %s", modulePath), out, os.Stderr)
+//
+//	assert.Nil(t, err)
+//	// t.Log(out)
+//	assert.NotContains(t, out.String(), "may need to run 'terraform init'")
+//}
+
 // This tests terragrunt properly passes through terraform commands and any number of specified args
 func TestTerraformCommandCliArgs(t *testing.T) {
 	t.Parallel()