Clenaup

Author: ValentaTomas

.github/actions/start-services/action.yml

@@ -88,9 +88,9 @@ runs:
         ENVIRONMENT: "local"
         OTEL_COLLECTOR_GRPC_ENDPOINT: "localhost:4317"
         MAX_PARALLEL_MEMFILE_SNAPSHOTTING: "2"
-        LOCAL_TEMPLATE_CACHE_PATH: "./.e2b-slab-cache"
+        SHARED_TEMPLATE_CACHE_PATH: "./.e2b-slab-cache"
       run: |
-        mkdir -p $LOCAL_TEMPLATE_CACHE_PATH
+        mkdir -p $SHARED_TEMPLATE_CACHE_PATH
         mkdir -p ~/logs
 
         # Start otel-collector

main.tf

@@ -261,9 +261,7 @@ module "nomad" {
   launch_darkly_api_key_secret_name = module.init.launch_darkly_api_key_secret_version.secret
 
   # Filestore
-  filestore_cache_enabled = var.filestore_cache_enabled
-
-  slab_cache_path         = module.cluster.nfs_slab_cache_path
+  shared_template_cache_path = module.cluster.shared_template_cache_path
 }
 
 module "redis" {

packages/cluster/main.tf

@@ -137,8 +137,7 @@ module "client_cluster" {
     NFS_IP_ADDRESS               = var.filestore_cache_enabled ? join(",", module.filestore[0].nfs_ip_addresses) : ""
     NFS_MOUNT_PATH               = local.nfs_mount_path
     NFS_MOUNT_SUBDIR             = local.nfs_mount_subdir
-
-    use_filestore_cache = var.filestore_cache_enabled
+    USE_FILESTORE_CACHE          = var.filestore_cache_enabled
   })
 
   environment = var.environment
@@ -285,9 +284,7 @@ module "build_cluster" {
     NFS_IP_ADDRESS               = var.filestore_cache_enabled ? join(",", module.filestore[0].nfs_ip_addresses) : ""
     NFS_MOUNT_PATH               = local.nfs_mount_path
     NFS_MOUNT_SUBDIR             = local.nfs_mount_subdir
-
-    // We don't use the filestore cache for the build cluster for now.
-    use_filestore_cache = false
+    USE_FILESTORE_CACHE          = var.filestore_cache_enabled
   })
 
   environment = var.environment

packages/cluster/outputs.tf

@@ -2,6 +2,6 @@ output "logs_proxy_ip" {
   value = module.network.logs_proxy_ip
 }
 
-output "nfs_slab_cache_path" {
-  value = "${local.nfs_mount_path}/${local.nfs_mount_subdir}"
+output "shared_template_cache_path" {
+  value = var.filestore_cache_enabled ? "${local.nfs_mount_path}/${local.nfs_mount_subdir}" : ""
 }

packages/cluster/scripts/start-client.sh

@@ -47,7 +47,7 @@ echo "$SWAPFILE none swap sw 0 0" | sudo tee -a /etc/fstab
 sudo sysctl vm.swappiness=10
 sudo sysctl vm.vfs_cache_pressure=50
 
-%{ if use_filestore_cache }
+%{ if USE_FILESTORE_CACHE }
 # Mount NFS
 sudo mkdir -p "${NFS_MOUNT_PATH}"
 echo "${NFS_IP_ADDRESS}:/slabs ${NFS_MOUNT_PATH} nfs defaults,tcp,nconnect=2,sec=sys,_netdev 0 0" | sudo tee -a /etc/fstab

packages/nomad/filestore-cleanup.tf

@@ -12,12 +12,13 @@ data "external" "filestore_cleanup_checksum" {
 }
 
 resource "nomad_job" "filestore_cleanup" {
-  count = var.filestore_cache_enabled ? 1 : 0
+  count = var.shared_template_cache_path != "" ? 1 : 0
+
   jobspec = templatefile("${path.module}/filestore-cleanup.hcl", {
     bucket_name              = var.fc_env_pipeline_bucket_name
     environment              = var.environment
     clean_nfs_cache_checksum = data.external.filestore_cleanup_checksum.result.hex
-    nfs_cache_mount_path     = var.slab_cache_path
+    nfs_cache_mount_path     = var.shared_template_cache_path
     max_disk_usage_target    = var.filestore_cache_max_disk_usage_target
   })
 }

packages/nomad/main.tf

@@ -407,7 +407,7 @@ locals {
     otel_collector_grpc_endpoint = "localhost:${var.otel_collector_grpc_port}"
     allow_sandbox_internet       = var.allow_sandbox_internet
     launch_darkly_api_key        = trimspace(data.google_secret_manager_secret_version.launch_darkly_api_key.secret_data)
-    nfs_cache_mount_path         = var.slab_cache_path
+    nfs_cache_mount_path         = var.shared_template_cache_path
     clickhouse_connection_string = var.clickhouse_server_count > 0 ? "clickhouse://${var.clickhouse_username}:${random_password.clickhouse_password.result}@clickhouse.service.consul:${var.clickhouse_server_port.port}/${var.clickhouse_database}" : ""
   }
 
@@ -492,8 +492,10 @@ resource "nomad_job" "template_manager" {
     logs_collector_public_ip     = var.logs_proxy_address
     orchestrator_services        = "template-manager"
     allow_sandbox_internet       = var.allow_sandbox_internet
-    nfs_cache_mount_path         = var.slab_cache_path
     clickhouse_connection_string = local.clickhouse_connection_string
+
+    # For now we disable the shared template cache in the template manager
+    nfs_cache_mount_path = ""
   })
 }
 resource "nomad_job" "loki" {

packages/nomad/orchestrator.hcl

@@ -71,7 +71,7 @@ EOT
         TEMPLATE_BUCKET_NAME         = "${template_bucket_name}"
         OTEL_COLLECTOR_GRPC_ENDPOINT = "${otel_collector_grpc_endpoint}"
         ALLOW_SANDBOX_INTERNET       = "${allow_sandbox_internet}"
-        LOCAL_TEMPLATE_CACHE_PATH    = "${nfs_cache_mount_path}"
+        SHARED_TEMPLATE_CACHE_PATH    = "${nfs_cache_mount_path}"
         CLICKHOUSE_CONNECTION_STRING = "${clickhouse_connection_string}"
 
 %{ if launch_darkly_api_key != "" }

packages/nomad/template-manager.hcl

@@ -65,7 +65,7 @@ job "template-manager" {
         ORCHESTRATOR_SERVICES         = "${orchestrator_services}"
         LOGS_COLLECTOR_PUBLIC_IP      = "${logs_collector_public_ip}"
         ALLOW_SANDBOX_INTERNET        = "${allow_sandbox_internet}"
-        LOCAL_TEMPLATE_CACHE_PATH    = "${nfs_cache_mount_path}"
+        SHARED_TEMPLATE_CACHE_PATH    = "${nfs_cache_mount_path}"
         CLICKHOUSE_CONNECTION_STRING  = "${clickhouse_connection_string}"
 %{ if !update_stanza }
         FORCE_STOP                    = "true"

packages/nomad/variables.tf

@@ -301,12 +301,9 @@ variable "clickhouse_node_pool" {
   type        = string
 }
 
-variable "slab_cache_path" {
-  type = string
-}
-
-variable "filestore_cache_enabled" {
-  type = bool
+variable "shared_template_cache_path" {
+  type    = string
+  default = ""
 }
 
 variable "filestore_cache_max_disk_usage_target" {