sassoftware · riragh · Nov 21, 2024 · Oct 11, 2024 · Oct 14, 2024 · Oct 15, 2024
diff --git a/README.md b/README.md
@@ -8,7 +8,7 @@ This project contains Terraform scripts to provision Google Cloud infrastructure
   >- Managed Google Kubernetes Engine (GKE) cluster
   >- System and User GKE Node pools with required Labels and Taints
   >- Infrastructure to deploy SAS Viya platform CAS in SMP or MPP mode
-  >- Shared Storage options for SAS Viya platform -  Google Filestore (ha) or NFS Server (standard)
+  >- Shared Storage options for SAS Viya platform -  Google Filestore (ha), Google NetApp Volumes (ha) or NFS Server (standard)
   >- Google Cloud SQL for PostgreSQL instance, optional
 
 [<img src="./docs/images/viya4-iac-gcp-diag.png" alt="Architecture Diagram" width="750"/>](./docs/images/viya4-iac-gcp-diag.png?raw=true)

diff --git a/docs/CONFIG-VARS.md b/docs/CONFIG-VARS.md
@@ -17,7 +17,8 @@ Supported configuration variables are listed in the table below.  All variables
     - [Additional Nodepools](#additional-nodepools)
   - [Storage](#storage)
     - [For `storage_type=standard` only (NFS server VM)](#for-storage_typestandard-only-nfs-server-vm)
-    - [For `storage_type=ha` only (Google Filestore)](#for-storage_typeha-only-google-filestore)
+    - [For `storage_type=ha` with Google Filestore](#for-storage_typeha-with-google-filestore)
+    - [For `storage_type=ha` with Google NetApp Volumes](#for-storage_typeha-with-google-netapp-volumes)
   - [Google Artifact Registry (GAR) and Google Container Registry (GCR)](#google-artifact-registry-gar-and-google-container-registry-gcr)
   - [Postgres Servers](#postgres-servers)
   - [Monitoring](#monitoring)
@@ -69,6 +70,7 @@ You can use `default_public_access_cidrs` to set a default range for all created
 | misc_subnet_cidr | Address space for the the auxiliary resources (Jump VM and optionally NFS VM) subnet | string | "192.168.2.0/24" | This variable is ignored when `subnet_names` is set (aka bring your own subnet) |
 | filestore_subnet_cidr | Address space for Google Filestore subnet | string | "192.168.3.0/29" | Needs to be at least a /29 range. Only used when `storage_type="ha"` |
 | database_subnet_cidr | Address space for Google Cloud SQL Postgres subnet | string | "192.168.4.0/23" | Only used with external postgres |
+| netapp_subnet_cidr | Address space for Google Cloud NetApp Volumes subnet | string | "192.168.5.0/24" | Needs to be at least a /24 range. Only used when `storage_type="ha"` and `storage_type_backend="netapp"` |
 
 ### Use Existing
 
@@ -212,6 +214,7 @@ stateful = {
 | Name | Description | Type | Default | Notes |
 | :--- | ---: | ---: | ---: | ---: |
 | storage_type | Type of Storage. Valid Values: "standard", "ha"  | string | "standard" |  "standard" creates NFS server VM, "ha" Google Filestore instance |
+| storage_type_backend | The storage backend for the chosen `storage_type`. | string | If `storage_type=standard` the default is "nfs";<br>If `storage_type=ha` the default is "filestore" | Valid Values: "nfs" if `storage_type=standard`; "filestore" or "netapp" if `storage_type=ha` |
 
 ### For `storage_type=standard` only (NFS server VM)
 
@@ -221,13 +224,27 @@ stateful = {
 | nfs_vm_admin | OS Admin User for the NFS server VM | string | "nfsuser" | The NFS server VM is only created when storage_type="standard" |
 | nfs_raid_disk_size | Size in Gb for each disk of the RAID5 cluster on the NFS server VM | number | 1000 | The NFS server VM is only created when storage_type="standard" |
 
-### For `storage_type=ha` only (Google Filestore)
+### For `storage_type=ha` with Google Filestore
 
 | Name | Description | Type | Default | Notes |
 | :--- | ---: | ---: | ---: | ---: |
 | filestore_tier | The service tier for the Google Filestore Instance | string | "BASIC_HDD" | Valid Values: "BASIC_HDD", "BASIC_SSD" (previously called "STANDARD" and "PREMIUM" respectively.)  |
 | filestore_size_in_gb | Size in GB of Filesystem in the Google Filestore Instance | number | 1024 for BASIC_HDD, 2560 for BASIC_SDD | 2560 GB is the minimum size for the BASIC_SSD tier. The BASIC_HDD tier allows a minimum size of 1024 GB. |
 
+### For `storage_type=ha` with Google NetApp Volumes
+
+When `storage_type=ha` and `storage_type_backend=netapp` are specified, [Google NetApp Volumes](https://cloud.google.com/netapp/volumes/docs/discover/overview) service is created. Before using this storage option,
+- Enable the Google Cloud NetApp Volumes API for your project, see how to enable [here](https://cloud.google.com/netapp/volumes/docs/get-started/configure-access/initiate-console-settings#enable_the_api).
+- Grant access to NetApp Volumes operations by granting IAM roles to users. The two predefined roles are `roles/netapp.admin` and `roles/netapp.viewer`. You can assign these roles to specific users or service accounts.
+- NetApp Volumes is available in several regions. For details about region availability, see [NetApp Volumes locations](https://cloud.google.com/netapp/volumes/docs/locations).
+
+| Name | Description | Type | Default | Notes |
+| :--- | ---: | ---: | ---: | ---: |
+| netapp_service_level | The service level of the storage pool. | string | "PREMIUM" | Valid Values are: PREMIUM, EXTREME, STANDARD, FLEX. |
+| netapp_protocols | The target volume protocol expressed as a list. | list(string) | ["NFSV3"] | Each value may be one of: NFSV3, NFSV4, SMB. Currently, only NFSV3 is supported by SAS Viya Platform. |
+| netapp_capacity_gib | Capacity of the storage pool (in GiB). Storage Pool capacity specified must be between 2048 GiB and 10485760 GiB. | string | "2048" | |
+| netapp_volume_path | A unique file path for the volume. Used when creating mount targets. Needs to be unique per location.| string | | |
+
 ## Google Artifact Registry (GAR) and Google Container Registry (GCR)
 
 | Name | Description | Type | Default | Notes |

diff --git a/docs/user/TerraformGCPAuthentication.md b/docs/user/TerraformGCPAuthentication.md
@@ -39,7 +39,9 @@ The Service Account will need the following [IAM roles](https://cloud.google.com
 | `roles/container.admin` | Kubernetes Engine Admin | Cluster creation |
 | `roles/container.clusterAdmin` | Kubernetes Engine Cluster Admin | Terraform Kubernetes Engine Module |
 | `roles/container.developer` | Kubernetes Engine Developer | Cluster creation |
-| `roles/file.editor` | Cloud Filestore Editor | Needed for [`storage_type=="HA"`](../CONFIG-VARS.md#storage) |
+| `roles/file.editor` | Cloud Filestore Editor | Needed for [`storage_type=="ha" && storage_type_backend = "filestore"`](../CONFIG-VARS.md#storage) |
+| `roles/netapp.admin` | NetApp Admin | Needed for [`storage_type=="ha" && storage_type_backend = "netapp"`](../CONFIG-VARS.md#storage) |
+| `roles/netapp.viewer` | NetApp Viewer | Needed for [`storage_type=="ha" && storage_type_backend = "netapp"`](../CONFIG-VARS.md#storage) |
 | `roles/iam.serviceAccountAdmin` | Service Account Admin | Terraform Kubernetes Engine Module |
 | `roles/iam.serviceAccountUser` | Service Account User | Terraform Kubernetes Engine Module |
 | `roles/resourcemanager.projectIamAdmin` | Project IAM Admin | Terraform Kubernetes Engine Module |
@@ -59,6 +61,8 @@ gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAM
 gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/container.clusterAdmin
 gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/container.developer
 gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/file.editor
+gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/netapp.admin
+gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/netapp.viewer
 gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/iam.serviceAccountAdmin
 gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/iam.serviceAccountUser
 gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/resourcemanager.projectIamAdmin

diff --git a/examples/sample-input-ha.tfvars b/examples/sample-input-ha.tfvars
@@ -94,5 +94,6 @@ jump_vm_admin         = "jumpuser"
 # Storage for Viya Compute Services
 # Supported storage_type values
 #    "standard" - Custom managed NFS Server VM and disks
-#    "ha"       - Google Filestore  
+#    "ha"       - Google Filestore  or Google NetApp Volumes
 storage_type = "ha"
+storage_type_backend = "filestore"  # "filestore" is the default, use "netapp" to create Google NetApp Volumes
diff --git a/locals.tf b/locals.tf
@@ -25,6 +25,12 @@ locals {
     : null
   )
 
+  # Storage
+  storage_type_backend = (var.storage_type == "none" ? "none"
+    : var.storage_type == "standard" ? "nfs"
+    : var.storage_type == "ha" && var.storage_type_backend == "netapp" ? "netapp"
+  : var.storage_type == "ha" ? "filestore" : "none")
+
   # Kubernetes
   kubeconfig_path = var.iac_tooling == "docker" ? "/workspace/${var.prefix}-gke-kubeconfig.conf" : "${var.prefix}-gke-kubeconfig.conf"
 

diff --git a/main.tf b/main.tf
@@ -66,7 +66,7 @@ EOT
 
 resource "google_filestore_instance" "rwx" {
   name     = "${var.prefix}-rwx-filestore"
-  count    = var.storage_type == "ha" ? 1 : 0
+  count    = var.storage_type == "ha" && local.storage_type_backend == "filestore" ? 1 : 0
   tier     = upper(var.filestore_tier)
   location = local.zone
   labels   = var.tags
@@ -301,3 +301,19 @@ module "sql_proxy_sa" {
   project_roles = ["${var.project}=>roles/cloudsql.admin"]
   display_name  = "IAC-managed service account for cluster ${var.prefix} and sql-proxy integration."
 }
+
+module "google_netapp" {
+  source = "./modules/google_netapp"
+
+  count = var.storage_type == "ha" && local.storage_type_backend == "netapp" ? 1 : 0
+
+  prefix             = var.prefix
+  region             = local.region
+  network            = module.vpc.network_name
+  netapp_subnet_cidr = var.netapp_subnet_cidr
+  service_level      = var.netapp_service_level
+  capacity_gib       = var.netapp_capacity_gib
+  protocols          = var.netapp_protocols
+  volume_path        = "${var.prefix}-${var.netapp_volume_path}"
+  allowed_clients    = join(",", [local.gke_subnet_cidr, local.misc_subnet_cidr])
+}
diff --git a/modules/google_netapp/main.tf b/modules/google_netapp/main.tf
@@ -0,0 +1,70 @@
+# Copyright © 2021-2024, SAS Institute Inc., Cary, NC, USA. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Terraform Registry : https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/netapp_volume
+# GitHub Repository  : https://github.com/terraform-google-modules
+#
+
+# Reserve compute address CIDR for NetApp Volumes to use
+resource "google_compute_global_address" "private_ip_alloc" {
+  name          = "${var.network}-ip-range"
+  purpose       = "VPC_PEERING"
+  address_type  = "INTERNAL"
+  address       = split("/", var.netapp_subnet_cidr)[0]
+  prefix_length = split("/", var.netapp_subnet_cidr)[1]
+  network       = var.network
+}
+
+# Create the PSA peering
+resource "google_service_networking_connection" "default" {
+  network                 = var.network
+  service                 = "netapp.servicenetworking.goog"
+  reserved_peering_ranges = [google_compute_global_address.private_ip_alloc.name]
+
+  deletion_policy = "ABANDON"
+}
+
+# Modify the PSA Connection to allow import/export of custom routes
+resource "google_compute_network_peering_routes_config" "route_updates" {
+  peering = google_service_networking_connection.default.peering
+  network = var.network
+
+  import_custom_routes = true
+  export_custom_routes = true
+}
+
+resource "google_netapp_storage_pool" "netapp-tf-pool" {
+  name          = "${var.prefix}-netapp-storage-pool"
+  location      = var.region
+  service_level = var.service_level
+  capacity_gib  = var.capacity_gib
+  network       = var.network
+
+  lifecycle {
+    ignore_changes = [network]
+  }
+}
+
+resource "google_netapp_volume" "netapp-nfs-volume" {
+  location         = var.region
+  name             = "${var.prefix}-netapp-volume"
+  capacity_gib     = var.capacity_gib # Size can be up to space available in pool
+  share_name       = var.volume_path
+  storage_pool     = google_netapp_storage_pool.netapp-tf-pool.name
+  protocols        = var.protocols
+  unix_permissions = "0777"
+  export_policy {
+    rules {
+      access_type     = "READ_WRITE"
+      allowed_clients = var.allowed_clients
+      has_root_access = true
+      nfsv3           = contains(var.protocols, "NFSV3") ? true : false
+      nfsv4           = contains(var.protocols, "NFSV4") ? true : false
+    }
+  }
+
+  depends_on = [
+    google_netapp_storage_pool.netapp-tf-pool,
+    google_service_networking_connection.default
+  ]
+}
diff --git a/modules/google_netapp/outputs.tf b/modules/google_netapp/outputs.tf
@@ -0,0 +1,7 @@
+output "mountpath" {
+  value = google_netapp_volume.netapp-nfs-volume.mount_options[0].export
+}
+
+output "export_ip" {
+  value = split(":", google_netapp_volume.netapp-nfs-volume.mount_options[0].export_full)[0]
+}
diff --git a/modules/google_netapp/variables.tf b/modules/google_netapp/variables.tf
@@ -0,0 +1,54 @@
+# Copyright © 2021-2024, SAS Institute Inc., Cary, NC, USA. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+variable "prefix" {
+  description = "A prefix used in the name for all cloud resources created by this script. The prefix string must start with lowercase letter and contain only lowercase alphanumeric characters and hyphen or dash(-), but can not start or end with '-'."
+  type        = string
+}
+
+variable "region" {
+  description = "The region to create the VM in"
+  type        = string
+}
+
+variable "service_level" {
+  description = "Service level of the storage pool. Possible values are: PREMIUM, EXTREME, STANDARD, FLEX."
+  type        = string
+  default     = "PREMIUM"
+}
+
+variable "protocols" {
+  description = "The target volume protocol expressed as a list. Allowed combinations are ['NFSV3'], ['NFSV4'], ['SMB'], ['NFSV3', 'NFSV4'], ['SMB', 'NFSV3'] and ['SMB', 'NFSV4']. Each value may be one of: NFSV3, NFSV4, SMB."
+  type        = list(string)
+  default     = ["NFSV3"]
+}
+
+variable "capacity_gib" {
+  description = "Capacity of the storage pool (in GiB)."
+  type        = string
+  default     = 2048
+}
+
+variable "volume_path" {
+  description = "A unique file path for the volume. Used when creating mount targets. Needs to be unique per location."
+  type        = string
+  default     = "export"
+}
+
+variable "network" {
+  description = "VPC network name with format: `projects/{{project}}/global/networks/{{network}}`"
+  type        = string
+}
+
+
+variable "allowed_clients" {
+  description = "CIDR blocks allowed to mount nfs exports"
+  type        = string
+  default     = "0.0.0.0/0"
+}
+
+variable "netapp_subnet_cidr" {
+  description = "Address space for Google Cloud NetApp Volumes subnet"
+  type        = string
+  default     = "192.168.5.0/24"
+}
diff --git a/network.tf b/network.tf
@@ -72,7 +72,7 @@ resource "google_service_networking_connection" "private_vpc_connection" {
   # required as of hashicorp/google v5.12.0 when using google_service_networking_connection in
   # conjunction with CloudSQL instances in order to cleanly delete resources
   # https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_networking_connection
-  deletion_policy         = "ABANDON"
+  deletion_policy = "ABANDON"
 }
 
 resource "google_compute_firewall" "nfs_vm_cluster_firewall" {

diff --git a/outputs.tf b/outputs.tf
@@ -27,15 +27,17 @@ output "rwx_filestore_endpoint" {
   description = "Shared Storage private IP"
   value = (var.storage_type == "none"
     ? null
-    : var.storage_type == "ha" ? google_filestore_instance.rwx[0].networks[0].ip_addresses[0] : module.nfs_server[0].private_ip
+    : var.storage_type == "ha" && local.storage_type_backend == "filestore" ? google_filestore_instance.rwx[0].networks[0].ip_addresses[0]
+    : var.storage_type == "ha" && local.storage_type_backend == "netapp" ? module.google_netapp[0].export_ip : module.nfs_server[0].private_ip
   )
 }
 
 output "rwx_filestore_path" {
   description = "Shared Storage mount path"
   value = (var.storage_type == "none"
     ? null
-    : var.storage_type == "ha" ? "/${google_filestore_instance.rwx[0].file_shares[0].name}" : "/export"
+    : var.storage_type == "ha" && local.storage_type_backend == "filestore" ? "/${google_filestore_instance.rwx[0].file_shares[0].name}"
+    : var.storage_type == "ha" && local.storage_type_backend == "netapp" ? module.google_netapp[0].mountpath : "/export"
   )
 }
 

diff --git a/variables.tf b/variables.tf
@@ -184,6 +184,18 @@ variable "storage_type" {
   }
 }
 
+variable "storage_type_backend" {
+  description = "The storage backend used for the chosen storage type. Defaults to 'nfs' for storage_type='standard'. Defaults to 'filestore for storage_type='ha'. 'filestore' and 'netapp' are valid choices for storage_type='ha'."
+  type        = string
+  default     = "nfs"
+  # If storage_type is standard, this will be set to "nfs"
+
+  validation {
+    condition     = contains(["nfs", "filestore", "netapp", "none"], lower(var.storage_type_backend))
+    error_message = "ERROR: Supported values for `storage_type_backend` are nfs, filestore, netapp or none."
+  }
+}
+
 variable "minimum_initial_nodes" {
   description = "Number of initial nodes to aim for to overcome the Ingress quota limit of 100"
   type        = number
@@ -426,6 +438,41 @@ variable "enable_registry_access" {
   default     = true
 }
 
+## Google NetApp Volumes
+variable "netapp_service_level" {
+  description = "Service level of the storage pool. Possible values are: PREMIUM, EXTREME, STANDARD, FLEX."
+  type        = string
+  default     = "PREMIUM"
+
+  validation {
+    condition     = var.netapp_service_level != null ? contains(["PREMIUM", "EXTREME", "STANDARD", "FLEX"], var.netapp_service_level) : null
+    error_message = "ERROR: netapp_service_level - Valid values include - PREMIUM, EXTREME, STANDARD, FLEX."
+  }
+}
+
+variable "netapp_protocols" {
+  description = "The target volume protocol expressed as a list. Each value may be one of: NFSV3, NFSV4, SMB. Currently, only NFS is supported."
+  type        = list(string)
+  default     = ["NFSV3"]
+
+  validation {
+    condition     = var.netapp_protocols != null ? startswith(var.netapp_protocols[0], "NFS") : null
+    error_message = "ERROR: Currently, only NFS protocol is supported."
+  }
+}
+
+variable "netapp_capacity_gib" {
+  description = "Capacity of the storage pool (in GiB). Storage Pool capacity specified must be between 2048 GiB and 10485760 GiB."
+  type        = string
+  default     = 2048
+}
+
+variable "netapp_volume_path" {
+  description = "A unique file path for the volume. Used when creating mount targets. Needs to be unique per location."
+  type        = string
+  default     = "export"
+}
+
 # GKE Monitoring
 variable "create_gke_monitoring_service" {
   description = "Enable GKE metrics from pods in the cluster to the Google Cloud Monitoring API."
@@ -519,6 +566,11 @@ variable "database_subnet_cidr" {
   default     = "192.168.4.0/23"
 }
 
+variable "netapp_subnet_cidr" {
+  description = "Address space for Google Cloud NetApp Volumes subnet"
+  type        = string
+  default     = "192.168.5.0/24"
+}
 
 variable "gke_network_policy" {
   description = "Sets up network policy to be used with GKE CNI. Network policy allows us to control the traffic flow between pods. Currently supported values are true (calico) and false (kubenet). Changing this forces a new resource to be created."