Use Filestore over NFS to speed up and share GCS Object Storage cached slabs

.github/actions/start-services/action.yml

@@ -88,7 +88,9 @@ runs:
         ENVIRONMENT: "local"
         OTEL_COLLECTOR_GRPC_ENDPOINT: "localhost:4317"
         MAX_PARALLEL_MEMFILE_SNAPSHOTTING: "2"
+        LOCAL_TEMPLATE_CACHE_PATH: "./.e2b-slab-cache"
       run: |
+        mkdir -p $LOCAL_TEMPLATE_CACHE_PATH
         mkdir -p ~/logs
 
         # Start otel-collector

.gitignore

@@ -5,6 +5,7 @@
 !.env.template
 .last_used_env
 .tfplan.*
+terraform.tfvars
 terraform.tfstate
 terraform.tfstate.backup
 **/.tfplan.*

.mockery.yaml

@@ -7,3 +7,11 @@ packages:
         config:
           dir: packages/envd/internal/services/legacy
           pkgname: legacy
+
+  github.com/e2b-dev/infra/packages/shared/pkg/storage:
+    interfaces:
+      StorageObjectProvider:
+        config:
+          dir: packages/shared/pkg/storage
+          filename: storage_test.go
+          pkgname: storage

Makefile

@@ -51,7 +51,8 @@ tf_vars := 	TF_VAR_environment=$(TERRAFORM_ENVIRONMENT) \
 	$(call tfvar, TEMPLATE_BUCKET_LOCATION) \
 	$(call tfvar, ENVD_TIMEOUT) \
 	$(call tfvar, REDIS_MANAGED) \
-	$(call tfvar, GRAFANA_MANAGED)
+	$(call tfvar, GRAFANA_MANAGED) \
+	$(call tfvar, USE_FILESTORE_CACHE) \
 
 # Login for Packer and Docker (uses gcloud user creds)
 # Login for Terraform (uses application default creds)
@@ -180,10 +181,14 @@ build/%:
 build-and-upload:build-and-upload/api
 build-and-upload:build-and-upload/client-proxy
 build-and-upload:build-and-upload/docker-reverse-proxy
+build-and-upload:build-and-upload/clean-nfs-cache
 build-and-upload:build-and-upload/orchestrator
 build-and-upload:build-and-upload/template-manager
 build-and-upload:build-and-upload/envd
 build-and-upload:build-and-upload/clickhouse-migrator
+build-and-upload/clean-nfs-cache:
+	./scripts/confirm.sh $(TERRAFORM_ENVIRONMENT)
+	GCP_PROJECT_ID=$(GCP_PROJECT_ID) $(MAKE) -C packages/orchestrator build-and-upload/clean-nfs-cache
 build-and-upload/template-manager:
 	./scripts/confirm.sh $(TERRAFORM_ENVIRONMENT)
 	GCP_PROJECT_ID=$(GCP_PROJECT_ID) $(MAKE) -C packages/orchestrator build-and-upload/template-manager
@@ -238,7 +243,7 @@ switch-env:
 import:
 	@ printf "Importing resources for env: `tput setaf 2``tput bold`$(ENV)`tput sgr0`\n\n"
 	./scripts/confirm.sh $(TERRAFORM_ENVIRONMENT)
-	$(tf_vars) $(TF) import $(TARGET) $(ID)
+	$(tf_vars) $(TF) import "$(TARGET)" "$(ID)" -no-color
 
 .PHONY: setup-ssh
 setup-ssh:

main.tf

@@ -22,7 +22,7 @@ terraform {
     }
     nomad = {
       source  = "hashicorp/nomad"
-      version = "2.1.0"
+      version = "~> 2.4.0"
     }
     random = {
       source  = "hashicorp/random"
@@ -137,6 +137,8 @@ module "cluster" {
   consul_acl_token_secret = module.init.consul_acl_token_secret
   nomad_acl_token_secret  = module.init.nomad_acl_token_secret
 
+  filestore_cache = var.filestore_cache
+
   labels = var.labels
   prefix = var.prefix
 }
@@ -255,6 +257,10 @@ module "nomad" {
   redis_port = var.redis_port
 
   launch_darkly_api_key_secret_name = module.init.launch_darkly_api_key_secret_version.secret
+
+  # Filestore
+  filestore_cache = var.filestore_cache
+  slab_cache_path = module.cluster.nfs_slab_cache_path
 }
 
 module "redis" {

packages/cluster-disk-image/main.pkr.hcl

@@ -80,6 +80,13 @@ build {
     ]
   }
 
+  provisioner "shell" {
+    inline = [
+      "sudo apt-get -y update",
+      "sudo apt-get install -y nfs-common",
+    ]
+  }
+
   provisioner "shell" {
     inline = [
       "sudo snap install go --classic"

packages/cluster/filestore/main.tf

@@ -0,0 +1,132 @@
+resource "google_filestore_instance" "slab-cache" {
+  name        = var.name
+  description = "High performance slab cache"
+  tier        = var.tier
+  protocol    = "NFS_V4_1"
+
+  deletion_protection_enabled = true
+  deletion_protection_reason  = "If this gets removed, the orchestrator will throw tons of errors"
+
+  file_shares {
+    capacity_gb = var.capacity_gb
+    name        = "slabs"
+  }
+
+  networks {
+    modes = [
+      "MODE_IPV4",
+    ]
+    network = var.network_name
+  }
+}
+
+data "google_monitoring_notification_channel" "notification" {
+  count        = var.notification_display_name == null ? 0 : 1
+  display_name = var.notification_display_name
+}
+
+resource "google_monitoring_alert_policy" "warning" {
+  count = var.free_space_warning_threshold == 0 ? 0 : 1
+
+  combiner = "OR"
+
+  display_name = "memory-cache-disk-usage-low"
+
+  notification_channels = var.notification_display_name == null ? [] : [
+    data.google_monitoring_notification_channel.notification[0].id
+  ]
+
+  severity = "WARNING"
+
+  alert_strategy {
+    notification_prompts = [
+      "OPENED",
+      "CLOSED",
+    ]
+  }
+
+  conditions {
+    display_name = "Over ${var.free_space_warning_threshold}% of the memory cache disk has been used"
+
+    condition_threshold {
+      comparison      = "COMPARISON_GT"
+      duration        = "0s"
+      filter          = <<EOT
+resource.type = "filestore_instance"
+AND metric.type = "file.googleapis.com/nfs/server/used_bytes_percent"
+AND metric.labels.file_share = "${google_filestore_instance.slab-cache.file_shares[0].name}"
+EOT
+      threshold_value = var.free_space_warning_threshold
+
+      aggregations {
+        alignment_period   = "300s"
+        group_by_fields    = []
+        per_series_aligner = "ALIGN_MAX"
+      }
+
+      trigger {
+        count   = 1
+        percent = 0
+      }
+    }
+  }
+
+  documentation {
+    mime_type = "text/markdown"
+    subject   = "Memory cache disk usage has gone over ${var.free_space_warning_threshold}%"
+    content   = "Your memory cache filestore instance disk usage has gone over ${var.free_space_warning_threshold}%. "
+  }
+}
+
+resource "google_monitoring_alert_policy" "error" {
+  count = var.free_space_error_threshold == 0 ? 0 : 1
+
+  combiner = "OR"
+
+  display_name = "memory-cache-disk-usage-very-low"
+
+  notification_channels = var.notification_display_name == null ? [] : [
+    data.google_monitoring_notification_channel.notification[0].id
+  ]
+
+  severity = "ERROR"
+
+  alert_strategy {
+    notification_prompts = [
+      "OPENED",
+      "CLOSED",
+    ]
+  }
+
+  conditions {
+    display_name = "Over ${var.free_space_error_threshold}% of the memory cache disk has been used"
+
+    condition_threshold {
+      comparison      = "COMPARISON_GT"
+      duration        = "0s"
+      filter          = <<EOT
+resource.type = "filestore_instance"
+AND metric.type = "file.googleapis.com/nfs/server/used_bytes_percent"
+AND metric.labels.file_share = "${google_filestore_instance.slab-cache.file_shares[0].name}"
+EOT
+      threshold_value = var.free_space_error_threshold
+
+      aggregations {
+        alignment_period   = "300s"
+        group_by_fields    = []
+        per_series_aligner = "ALIGN_MAX"
+      }
+
+      trigger {
+        count   = 1
+        percent = 0
+      }
+    }
+  }
+
+  documentation {
+    mime_type = "text/markdown"
+    subject   = "Memory cache disk usage has gone over ${var.free_space_error_threshold}%"
+    content   = "Your memory cache filestore instance disk usage has gone over ${var.free_space_error_threshold}%. "
+  }
+}

packages/cluster/filestore/output.tf

@@ -0,0 +1,3 @@
+output "nfs_ip_addresses" {
+  value = google_filestore_instance.slab-cache.networks[0].ip_addresses
+}

packages/cluster/filestore/variables.tf

@@ -0,0 +1,29 @@
+variable "name" {
+  description = "The name of the Nomad cluster (e.g. nomad-stage). This variable is used to namespace all resources created by this module."
+  type        = string
+}
+
+variable "network_name" {
+  description = "The name of the VPC Network where all resources should be created."
+  type        = string
+}
+
+variable "tier" {
+  type = string
+}
+
+variable "capacity_gb" {
+  type = number
+}
+
+variable "notification_display_name" {
+  type = string
+}
+
+variable "free_space_warning_threshold" {
+  type = number
+}
+
+variable "free_space_error_threshold" {
+  type = number
+}