feat: Add ability to override offline store bucket name.

.gitignore

@@ -2,3 +2,5 @@
 *.iml
 **/.terraform
 
+scripts/.venv/
+*.pyc

README.md

@@ -62,3 +62,7 @@ module "tecton" {
 ```
 
 Please refer to the specific `README.md` within each module's directory for detailed instructions and the full list of variables for that module.
+
+
+### Validation Script
+There is a validation script ([details here](./scripts/README.md)) that can be run after applying one of the above modules, to check that the expected resources are in lace.

deployment/README.md

@@ -12,6 +12,7 @@
 | <a name="input_account_id"></a> [account\_id](#input\_account\_id) | Data plane (customer) AWS account ID. | `string` | n/a | yes |
 | <a name="input_additional_offline_storage_tags"></a> [additional\_offline\_storage\_tags](#input\_additional\_offline\_storage\_tags) | **(Optional)** Additional tags for offline storage (S3 bucket) | `map(string)` | `{}` | no |
 | <a name="input_additional_s3_read_only_principals"></a> [additional\_s3\_read\_only\_principals](#input\_additional\_s3\_read\_only\_principals) | n/a | `list(string)` | `[]` | no |
+| <a name="input_bucket_name_override"></a> [bucket\_name\_override](#input\_bucket\_name\_override) | Name of the offline store bucket. | `string` | `null` | no |
 | <a name="input_bucket_sse_algorithm"></a> [bucket\_sse\_algorithm](#input\_bucket\_sse\_algorithm) | Server-side encryption algorithm to use. Valid values are AES256 and aws:kms.<br/> Note: (1) All resources should also be granted permission to decrypt with the KMS key if using KMS.<br/>       (2) If athena retrieval is used, the kms\_key option must also be set on the athena session. | `string` | `"AES256"` | no |
 | <a name="input_bucket_sse_key_enabled"></a> [bucket\_sse\_key\_enabled](#input\_bucket\_sse\_key\_enabled) | Whether or not to use Amazon S3 Bucket Keys for SSE-KMS. | `bool` | `null` | no |
 | <a name="input_controlplane_access_only"></a> [controlplane\_access\_only](#input\_controlplane\_access\_only) | Whether to only grant control-plane account access to the cross-account role | `bool` | `false` | no |

deployment/buckets.tf

@@ -1,5 +1,9 @@
+locals {
+  bucket_name = var.bucket_name_override != null ? var.bucket_name_override : "tecton-${var.deployment_name}"
+}
+
 resource "aws_s3_bucket" "tecton" {
-  bucket = "tecton-${var.deployment_name}"
+  bucket = local.bucket_name
   tags   = merge(local.tags, var.additional_offline_storage_tags)
   lifecycle {
     ignore_changes = [lifecycle_rule]
@@ -66,7 +70,7 @@ data "aws_iam_policy_document" "s3_bucket_policy" {
         identifiers = var.s3_read_write_principals
         type        = "AWS"
       }
-      resources = ["arn:aws:s3:::tecton-${var.deployment_name}"]
+      resources = ["arn:aws:s3:::${local.bucket_name}"]
     }
   }
 
@@ -84,7 +88,7 @@ data "aws_iam_policy_document" "s3_bucket_policy" {
         identifiers = var.s3_read_write_principals
         type        = "AWS"
       }
-      resources = ["arn:aws:s3:::tecton-${var.deployment_name}/*"]
+      resources = ["arn:aws:s3:::${local.bucket_name}/*"]
     }
   }
 }

deployment/roles.tf

@@ -95,6 +95,7 @@ resource "aws_iam_policy" "cross_account_policy_spark" {
     DEPLOYMENT_NAME = var.deployment_name
     REGION          = var.region
     SPARK_ROLE      = local.spark_role_name
+    BUCKET_NAME     = local.bucket_name
   })
   tags = local.tags
 }
@@ -108,6 +109,7 @@ resource "aws_iam_policy" "cross_account_policy_rift" {
     ACCOUNT_ID      = var.account_id
     DEPLOYMENT_NAME = var.deployment_name
     REGION          = var.region
+    BUCKET_NAME     = local.bucket_name
   })
   tags = local.tags
 }
@@ -146,6 +148,7 @@ resource "aws_iam_policy" "common_spark_policy" {
     ACCOUNT_ID      = var.account_id
     DEPLOYMENT_NAME = var.deployment_name
     REGION          = var.region
+    BUCKET_NAME     = local.bucket_name
   })
   tags = local.tags
 }

deployment/variables.tf

@@ -130,4 +130,10 @@ variable "include_crossaccount_bucket_access" {
   type        = bool
   description = "Whether to grant direct cross-account bucket access"
   default     = true
+}
+
+variable "bucket_name_override" {
+  type        = string
+  description = "Name of the offline store bucket."
+  default     = null
 }

modules/controlplane_rift/README.md

@@ -72,6 +72,7 @@ This module provisions:
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_account_id"></a> [account\_id](#input\_account\_id) | ID of the AWS account where Tecton will be deployed. | `string` | n/a | yes |
+| <a name="input_bucket_name_override"></a> [bucket\_name\_override](#input\_bucket\_name\_override) | Name of the offline store bucket. | `string` | `null` | no |
 | <a name="input_cross_account_external_id"></a> [cross\_account\_external\_id](#input\_cross\_account\_external\_id) | The external ID for cross-account access. Obtain this from your Tecton representative. | `string` | n/a | yes |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name of the Tecton deployment. Must be less than 22 characters due to AWS limitations. | `string` | n/a | yes |
 | <a name="input_kms_key_id"></a> [kms\_key\_id](#input\_kms\_key\_id) | (Optional) The customer-managed key (ID) for encrypting data at rest. | `string` | `null` | no |

modules/controlplane_rift/infrastructure.tf

@@ -23,6 +23,7 @@ module "tecton" {
   use_spark_compute                 = false
   use_rift_cross_account_policy     = true
   kms_key_id                        = var.kms_key_id
+  bucket_name_override              = var.bucket_name_override
 }
 
 # S3 module to store outputs

modules/controlplane_rift/variables.tf

@@ -29,6 +29,12 @@ variable "kms_key_id" {
   default     = null
 }
 
+variable "bucket_name_override" {
+  description = "Name of the offline store bucket."
+  type        = string
+  default     = null
+}
+
 variable "outputs_location_config" {
   description = "Configuration for where to store the outputs. Defaults to creating a dedicated bucket."
   type = object({

modules/controlplane_rift_with_emr/README.md

@@ -86,6 +86,7 @@ This module provisions:
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_account_id"></a> [account\_id](#input\_account\_id) | The AWS account ID where Tecton will be deployed. | `string` | n/a | yes |
+| <a name="input_bucket_name_override"></a> [bucket\_name\_override](#input\_bucket\_name\_override) | Name of the offline store bucket. | `string` | `null` | no |
 | <a name="input_cross_account_external_id"></a> [cross\_account\_external\_id](#input\_cross\_account\_external\_id) | The external ID for cross-account access. Obtain this from your Tecton representative. | `string` | n/a | yes |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name of the Tecton deployment. Must be less than 22 characters due to AWS limitations. | `string` | n/a | yes |
 | <a name="input_emr_debugging_count"></a> [emr\_debugging\_count](#input\_emr\_debugging\_count) | Set to 1 to allow Tecton to debug EMR clusters. Set to 0 to disable. Requires Tecton deployment. | `number` | `0` | no |

modules/controlplane_rift_with_emr/infrastructure.tf

@@ -23,7 +23,8 @@ module "tecton" {
   use_spark_compute                 = true
   use_rift_cross_account_policy     = true
   kms_key_id                        = var.kms_key_id
-  create_emr_roles = true
+  create_emr_roles                  = true
+  bucket_name_override              = var.bucket_name_override
 }
 
 ## EMR Resources

modules/controlplane_rift_with_emr/variables.tf

@@ -68,6 +68,12 @@ variable "notebook_glue_account_id" {
   default     = null # Will be dynamically set to var.account_id if null
 }
 
+variable "bucket_name_override" {
+  description = "Name of the offline store bucket."
+  type        = string
+  default     = null
+}
+
 variable "outputs_location_config" {
   description = "Configuration for where to store the outputs. Defaults to creating a dedicated bucket."
   type = object({

modules/databricks/README.md

@@ -85,6 +85,7 @@ This module provisions:
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_account_id"></a> [account\_id](#input\_account\_id) | The AWS account ID where Tecton and Databricks are deployed. | `string` | n/a | yes |
+| <a name="input_bucket_name_override"></a> [bucket\_name\_override](#input\_bucket\_name\_override) | Name of the offline store bucket. | `string` | `null` | no |
 | <a name="input_cross_account_external_id"></a> [cross\_account\_external\_id](#input\_cross\_account\_external\_id) | The external ID for cross-account access by Tecton. Obtain this from your Tecton representative. | `string` | n/a | yes |
 | <a name="input_databricks_workspace_url"></a> [databricks\_workspace\_url](#input\_databricks\_workspace\_url) | The URL of your Databricks workspace (e.g., mycompany.cloud.databricks.com). | `string` | n/a | yes |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name for your Tecton deployment. Must be less than 22 characters due to AWS S3 bucket naming limitations. | `string` | n/a | yes |

modules/databricks/infrastructure.tf

@@ -21,6 +21,7 @@ module "tecton" {
   kms_key_id                 = var.kms_key_id
   databricks_spark_role_name = var.spark_role_name
   s3_read_write_principals   = [format("arn:aws:iam::%s:root", var.tecton_control_plane_account_id)]
+  bucket_name_override       = var.bucket_name_override
 }
 
 # S3 module to store outputs

modules/databricks/variables.tf

@@ -44,6 +44,12 @@ variable "kms_key_id" {
   default     = null
 }
 
+variable "bucket_name_override" {
+  description = "Name of the offline store bucket."
+  type        = string
+  default     = null
+}
+
 variable "outputs_location_config" {
   description = "Configuration for where to store the outputs. Defaults to creating a dedicated bucket."
   type = object({

modules/dataplane_rift/README.md

@@ -79,6 +79,7 @@ output "tecton" {
 | <a name="input_account_id"></a> [account\_id](#input\_account\_id) | The AWS account ID where Tecton will be deployed. | `string` | n/a | yes |
 | <a name="input_additional_allowed_egress_domains"></a> [additional\_allowed\_egress\_domains](#input\_additional\_allowed\_egress\_domains) | (Optional) List of additional domains to allow for egress if use\_network\_firewall is true. Only works if using VPC managed by this module (i.e. existing\_vpc is not provided). | `list(string)` | `null` | no |
 | <a name="input_additional_s3_read_access_buckets"></a> [additional\_s3\_read\_access\_buckets](#input\_additional\_s3\_read\_access\_buckets) | (Optional) List of additional S3 bucket names in the dataplane account that the rift compute role should have read access to. | `list(string)` | `[]` | no |
+| <a name="input_bucket_name_override"></a> [bucket\_name\_override](#input\_bucket\_name\_override) | Name of the offline store bucket. | `string` | `null` | no |
 | <a name="input_controlplane_access_only"></a> [controlplane\_access\_only](#input\_controlplane\_access\_only) | Whether to only grant control-plane account access to the cross-account role | `bool` | `true` | no |
 | <a name="input_cross_account_external_id"></a> [cross\_account\_external\_id](#input\_cross\_account\_external\_id) | The external ID for cross-account access. Obtain this from your Tecton representative. | `string` | n/a | yes |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name of the Tecton deployment. Must be less than 22 characters due to AWS limitations. | `string` | n/a | yes |

modules/dataplane_rift/infrastructure.tf

@@ -25,6 +25,7 @@ module "tecton" {
   kms_key_id                         = var.kms_key_id
   controlplane_access_only           = var.controlplane_access_only
   include_crossaccount_bucket_access = var.include_crossaccount_bucket_access
+  bucket_name_override               = var.bucket_name_override
 }
 
 

modules/dataplane_rift/variables.tf

@@ -114,6 +114,12 @@ variable "additional_s3_read_access_buckets" {
   default     = []
 }
 
+variable "bucket_name_override" {
+  description = "Name of the offline store bucket."
+  type        = string
+  default     = null
+}
+
 variable "outputs_location_config" {
   description = "Configuration for where to store the outputs. Defaults to creating a dedicated bucket."
   type = object({

modules/dataplane_rift_with_emr/README.md

@@ -38,7 +38,6 @@ module "tecton" {
   tecton_control_plane_account_id    = "987654321098"     # Replace with Tecton's Control Plane Account ID
   cross_account_external_id          = "your-external-id" # Replace with the External ID from Tecton
   tecton_control_plane_role_name     = "TectonControlPlaneRole" # Role name from Tecton
-  include_crossaccount_bucket_access = false
 
   # Get outputs destination URL from Tecton
   outputs_location_config = {
@@ -95,6 +94,7 @@ This module provisions:
 | <a name="input_account_id"></a> [account\_id](#input\_account\_id) | The AWS account ID where Tecton will be deployed. | `string` | n/a | yes |
 | <a name="input_additional_allowed_egress_domains"></a> [additional\_allowed\_egress\_domains](#input\_additional\_allowed\_egress\_domains) | (Optional) List of additional domains to allow for egress if use\_network\_firewall is true. Only works if using VPC managed by this module (i.e. existing\_vpc is not provided). | `list(string)` | `null` | no |
 | <a name="input_additional_s3_read_access_buckets"></a> [additional\_s3\_read\_access\_buckets](#input\_additional\_s3\_read\_access\_buckets) | (Optional) List of additional S3 bucket names in the dataplane account that the rift compute role should have read access to. | `list(string)` | `[]` | no |
+| <a name="input_bucket_name_override"></a> [bucket\_name\_override](#input\_bucket\_name\_override) | Name of the offline store bucket. | `string` | `null` | no |
 | <a name="input_controlplane_access_only"></a> [controlplane\_access\_only](#input\_controlplane\_access\_only) | Whether to only grant control-plane account access to the cross-account role | `bool` | `true` | no |
 | <a name="input_cross_account_external_id"></a> [cross\_account\_external\_id](#input\_cross\_account\_external\_id) | The external ID for cross-account access. Obtain this from your Tecton representative. | `string` | n/a | yes |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name of the Tecton deployment. Must be less than 22 characters due to AWS limitations. | `string` | n/a | yes |

modules/dataplane_rift_with_emr/infrastructure.tf

@@ -26,6 +26,7 @@ module "tecton" {
   create_emr_roles                   = true
   controlplane_access_only           = var.controlplane_access_only
   include_crossaccount_bucket_access = var.include_crossaccount_bucket_access
+  bucket_name_override               = var.bucket_name_override
 }
 
 module "rift" {

modules/dataplane_rift_with_emr/variables.tf

@@ -153,6 +153,12 @@ variable "additional_s3_read_access_buckets" {
   default     = []
 }
 
+variable "bucket_name_override" {
+  description = "Name of the offline store bucket."
+  type        = string
+  default     = null
+}
+
 variable "outputs_location_config" {
   description = "Configuration for where to store the outputs. Defaults to creating a dedicated bucket."
   type = object({

modules/emr/README.md

@@ -92,6 +92,7 @@ Before using this module, ensure you have:
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_account_id"></a> [account\_id](#input\_account\_id) | The AWS account ID where Tecton and EMR resources will be deployed. | `string` | n/a | yes |
+| <a name="input_bucket_name_override"></a> [bucket\_name\_override](#input\_bucket\_name\_override) | Name of the offline store bucket. | `string` | `null` | no |
 | <a name="input_cross_account_external_id"></a> [cross\_account\_external\_id](#input\_cross\_account\_external\_id) | The external ID for cross-account access by Tecton. Obtain this from your Tecton representative. | `string` | n/a | yes |
 | <a name="input_cross_account_principal_arn_for_s3_policy"></a> [cross\_account\_principal\_arn\_for\_s3\_policy](#input\_cross\_account\_principal\_arn\_for\_s3\_policy) | (Optional) The ARN of the principal in another account that should get read-only access to the Tecton S3 bucket. Used if setting up cross-account EMR notebooks manually or extending this module. | `string` | `null` | no |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name for your Tecton deployment. Must be less than 22 characters due to AWS S3 bucket naming limitations. | `string` | n/a | yes |

modules/emr/variables.tf

@@ -104,6 +104,12 @@ variable "cross_account_principal_arn_for_s3_policy" {
   default     = null
 }
 
+variable "bucket_name_override" {
+  description = "Name of the offline store bucket."
+  type        = string
+  default     = null
+}
+
 variable "outputs_location_config" {
   description = "Configuration for where to store the outputs. Defaults to creating a dedicated bucket."
   type = object({