chore: prepare Realtime for Marketplaces (#49)

* chore: update Packer & Ansible files

Update Packer & Ansible settings to prepare for AWS/DO Marketplaces

* chore: set aws.json's `source_ami` to be user var

`source_ami`'s default is set to x64 Ubuntu 18.04 EBS HVM in
ap-southeast-1 (Singpore), but ami codes change between regions, so this
should be a user variable along with `region`.

* fix: get SECRET_KEY_BASE from env

* chore: proper cleanup of Ansible tmp files

* docs: better error messages for SECRET_KEY_BASE

* ci: fix artifact naming of release assets

Author: soedirgo

.github/workflows/release.yml

@@ -3,7 +3,7 @@ name: Release
 on:
   push:
     tags:
-      - '*'
+      - "[0-9]+.[0-9]+.[0-9]+*"
 
 jobs:
   release:
@@ -22,12 +22,16 @@ jobs:
           elixir-version: 1.10.x
           otp-version: 22.x
 
+      - name: Get the version
+        id: get_version
+        run: echo ::set-output name=version::${GITHUB_REF#refs/tags/}
+
       - name: Prepare release
         run: |
           mix deps.get
           mix compile
           mix release
-          tar -czvf realtime-ubuntu-latest.tar.gz -C ./_build/prod/rel/realtime/bin realtime
+          tar -czf realtime-${{ steps.get_version.outputs.version }}-x86_64-linux-gnu.tar.gz -C ./_build/prod/rel realtime
         env:
           MIX_ENV: prod
 
@@ -39,8 +43,6 @@ jobs:
         with:
           tag_name: ${{ github.ref }}
           release_name: ${{ github.ref }}
-          draft: false
-          prerelease: false
 
       - name: Upload release assets
         id: upload-release-asset
@@ -49,8 +51,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./server/realtime-ubuntu-latest.tar.gz
-          asset_name: realtime-ubuntu-latest.tar.gz
+          asset_path: ./server/realtime-${{ steps.get_version.outputs.version }}-x86_64-linux-gnu.tar.gz
+          asset_name: realtime-${{ steps.get_version.outputs.version }}-x86_64-linux-gnu.tar.gz
           asset_content_type: application/gzip
 
       - name: Upload build to Docker Hub

ansible/files/apt_periodic

@@ -0,0 +1,4 @@
+APT::Periodic::Update-Package-Lists "1";
+APT::Periodic::Download-Upgradeable-Packages "1";
+APT::Periodic::AutocleanInterval "7";
+APT::Periodic::Unattended-Upgrade "1";

ansible/files/realtime.env

@@ -1,8 +1,8 @@
 HOSTNAME=0.0.0.0
 PORT=4000
 DB_USER=postgres
-DB_HOST=localhost
+DB_HOST=
 DB_PORT=5432
 DB_NAME=postgres
-DB_PASSWORD=postgres
-SECRET_KEY_BASE=SOMETHING_SUPER_SECRET
+DB_PASSWORD=
+SECRET_KEY_BASE=

ansible/files/realtime.service.j2

@@ -3,13 +3,13 @@ Description=Supabase Realtime server
 
 [Service]
 Type=simple
-ExecStart=/opt/realtime/server/_build/prod/rel/realtime/bin/realtime start
+ExecStart=/opt/realtime/bin/realtime start
 Restart=always
 RestartSec=3
 
 # User for the build, and service
 User=realtime
-EnvironmentFile=/etc/realtime.env
+EnvironmentFile=/etc/realtime/realtime.env
 
 # Not specified in the supabase server docs but startup will fail if the HOME environmental
 # variable is not set.

ansible/playbook.yml

@@ -3,15 +3,13 @@
   become: true
 
   vars:
-    supabase_commit: 951ef2350465d42eb6f741f4659ed5b6fda4cd7b
-    supabase_commit_checksum: sha1:eaf75edba248db39c881dc0dce5b129296afaecb
-
-    erlang_solutions_deb: erlang-solutions_2.0_all.deb
-    erlang_solutions_deb_checksum: sha1:1968ec2ae81a5e1f56d2f173144926ec90a5e7c7
+    realtime_version: 0.7.4
+    realtime_checksum: sha1:9971212a8d39ada4385b97b44486e30230223116
 
   tasks:
     - include_tasks: tasks/setup-system.yml
 
-    - include_tasks: tasks/setup-elixir.yml
-
     - include_tasks: tasks/setup-realtime.yml
+
+    - name: Remove temp dir at $HOME
+      shell: rm -rf ~/.ansible

ansible/tasks/setup-elixir.yml

@@ -2,38 +2,29 @@
 - name: System user
   user: name=realtime
 
-- name: Install hex dependecy
-  shell: mix local.hex --force
-  become_user: realtime
-  args:
-    chdir: /opt/realtime/server
-
-- name: Install rebar dependency
-  shell: mix local.rebar --force
-  become_user: realtime
-  args:
-    chdir: /opt/realtime/server
-
-- name: Install other dependencies
-  shell: mix deps.get
-  become_user: realtime
-  args:
-    chdir: /opt/realtime/server
+- name: Download release
+  get_url:
+    url: "https://github.com/supabase/realtime/releases/download/{{ realtime_version }}/realtime-{{ realtime_version }}-x86_64-linux-gnu.tar.gz"
+    dest: /tmp/realtime.tar.gz
+    checksum: "{{ realtime_checksum }}"
+
+- name: Unpack archive to /opt/realtime
+  unarchive:
+    remote_src: yes
+    src: /tmp/realtime.tar.gz
+    dest: /opt
+    owner: realtime
 
-- name: Build release
-  # IF A BUILD EXISTS the following message is shown
-  # Release realtime-0.7.1 already exists. Overwrite? [Yn]
-  #
-  # There is no flag to stop if an existing build is available, thus we pipe 'n' to stop
-  shell: echo n | MIX_ENV=prod mix release
-  become_user: realtime
-  args:
-    chdir: /opt/realtime/server
+- name: Create /etc/realtime
+  file:
+    path: /etc/realtime
+    state: directory
+    owner: realtime
 
-- name: Create /etc/realtime.env
+- name: Dump /etc/realtime/realtime.env
   copy:
     src: files/realtime.env
-    dest: /etc/realtime.env
+    dest: /etc/realtime/realtime.env
     owner: realtime
 
 - name: Create service file
@@ -46,6 +37,3 @@
     daemon_reload: yes
     name: realtime
     enabled: yes
-
-- name: Restart service
-  service: name=realtime state=restarted

ansible/tasks/setup-realtime.yml

@@ -1,7 +1,28 @@
 # DigitalOcean's ubuntu droplet isn't up to date with installed packages, and on
 # a fresh install I see 71 security upgrades available.
 - name: System - apt update and apt upgrade
-  apt: update_cache=yes upgrade=yes # SEE http://archive.vn/DKJjs#parameter-upgrade
+  apt:
+    update_cache=yes upgrade=yes
+    # SEE http://archive.vn/DKJjs#parameter-upgrade
+
+- name: add universe repository for bionic
+  apt_repository:
+    repo: deb http://archive.ubuntu.com/ubuntu bionic universe
+    state: present
+
+- name: Install essentials
+  apt:
+    pkg:
+      - ufw
+      - fail2ban
+      - unattended-upgrades
+    update_cache: yes
+    cache_valid_time: 3600
+
+- name: Adjust APT update intervals
+  copy:
+    src: files/apt_periodic
+    dest: /etc/apt/apt.conf.d/10periodic
 
 - name: System - Create services.slice
   template:
@@ -10,3 +31,26 @@
 
 - name: System - systemd reload
   systemd: daemon_reload=yes
+
+- name: UFW - Deny incoming traffics by default
+  ufw:
+    state: enabled
+    default: deny
+    direction: incoming
+
+- name: UFW - Allow SSH
+  ufw:
+    rule: allow
+    name: OpenSSH
+
+- name: UFW - Allow Postgres
+  ufw:
+    rule: allow
+    port: "5432"
+    proto: tcp
+
+- name: UFW - Allow realtime
+  ufw:
+    rule: allow
+    port: "4000"
+    proto: tcp

ansible/tasks/setup-system.yml

@@ -1,32 +1,40 @@
 {
   "variables": {
     "aws_access_key": "{{env `AWS_ACCESS_KEY`}}",
-    "aws_secret_key": "{{env `AWS_SECRET_KEY`}}"
+    "aws_secret_key": "{{env `AWS_SECRET_KEY`}}",
+    "region": "ap-southeast-1",
+    "source_ami": "ami-0e763a959ec839f5e",
+    "instance_type": "t2.micro"
   },
   "builders": [
     {
       "type": "amazon-ebs",
       "access_key": "{{user `aws_access_key`}}",
       "secret_key": "{{user `aws_secret_key`}}",
-      "region": "ap-southeast-1",
-      "source_ami_filter": {
-        "filters": {
-          "virtualization-type": "hvm",
-          "name": "ubuntu/images/*ubuntu-bionic-18.04-amd64-server-*",
-          "root-device-type": "ebs"
-        },
-        "owners": ["099720109477"],
-        "most_recent": true
-      },
-      "instance_type": "t2.micro",
+      "region": "{{user `region`}}",
+      "source_ami": "{{user `source_ami`}}",
+      "instance_type": "{{user `instance_type`}}",
       "ssh_username": "ubuntu",
       "ami_name": "supabase-realtime-0.7.4"
     }
   ],
   "provisioners": [
     {
       "type": "ansible",
-      "playbook_file": "ansible/playbook.yml"
+      "playbook_file": "ansible/playbook.yml",
+      "ansible_env_vars": ["ANSIBLE_SSH_ARGS='-o IdentitiesOnly=yes'"],
+      "user": "ubuntu"
+    },
+    {
+      "execute_command": "echo 'packer' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'",
+      "type": "shell",
+      "scripts": [
+        "scripts/01-test",
+        "scripts/02-credentials_cleanup.sh",
+        "scripts/90-cleanup.sh",
+        "scripts/91-log_cleanup.sh",
+        "scripts/99-img_check.sh"
+      ]
     }
   ]
 }

aws.json

@@ -1,14 +1,16 @@
 {
   "variables": {
-    "do_api_token": "{{env `DO_API_TOKEN`}}"
+    "do_api_token": "{{env `DO_API_TOKEN`}}",
+    "region": "sgp1",
+    "size": "512mb"
   },
   "builders": [
     {
       "type": "digitalocean",
       "api_token": "{{user `do_api_token`}}",
       "image": "ubuntu-18-04-x64",
-      "region": "sgp1",
-      "size": "512mb",
+      "region": "{{user `region`}}",
+      "size": "{{user `size`}}",
       "ssh_username": "root",
       "snapshot_name": "supabase-realtime-0.7.4"
     }
@@ -17,7 +19,17 @@
     {
       "type": "ansible",
       "playbook_file": "ansible/playbook.yml",
-      "ansible_env_vars": ["ANSIBLE_SSH_ARGS='-o IdentitiesOnly=yes'"]
+      "ansible_env_vars": ["ANSIBLE_SSH_ARGS='-o IdentitiesOnly=yes'"],
+      "user": "root"
+    },
+    {
+      "type": "shell",
+      "scripts": [
+        "scripts/01-test",
+        "scripts/90-cleanup.sh",
+        "scripts/91-log_cleanup.sh",
+        "scripts/99-img_check.sh"
+      ]
     }
   ]
 }