Merge pull request #106 from sherifabdlnaby/8.10

Upgrade to 8.10.2

- Fixes #82 
- Fixes #76 
- Fixes #102

Author: sherifabdlnaby

.env

@@ -1,5 +1,5 @@
 COMPOSE_PROJECT_NAME=elastic
-ELK_VERSION=8.8.0
+ELK_VERSION=8.10.2
 
 #----------- Resources --------------------------#
 ELASTICSEARCH_HEAP=1024m
@@ -15,7 +15,6 @@ KIBANA_HOST=kibana
 KIBANA_PORT=5601
 
 LOGSTASH_HOST=logstash
-LOGSTASH_PORT=8080
 
 APMSERVER_HOST=apm-server
 APMSERVER_PORT=8200

Makefile

@@ -1,16 +1,14 @@
 .DEFAULT_GOAL:=help
 
-COMPOSE_ALL_FILES := -f docker-compose.yml -f docker-compose.monitor.yml -f docker-compose.tools.yml -f docker-compose.nodes.yml -f docker-compose.logs.yml
+COMPOSE_ALL_FILES := -f docker-compose.yml -f docker-compose.monitor.yml -f docker-compose.nodes.yml -f docker-compose.logs.yml
 COMPOSE_MONITORING := -f docker-compose.yml -f docker-compose.monitor.yml
 COMPOSE_LOGGING := -f docker-compose.yml -f docker-compose.logs.yml
-COMPOSE_TOOLS := -f docker-compose.yml -f docker-compose.tools.yml
 COMPOSE_NODES := -f docker-compose.yml -f docker-compose.nodes.yml
 ELK_SERVICES   := elasticsearch logstash kibana apm-server
 ELK_LOG_COLLECTION := filebeat
 ELK_MONITORING := elasticsearch-exporter logstash-exporter filebeat-cluster-logs
-ELK_TOOLS  := rubban
 ELK_NODES := elasticsearch-1 elasticsearch-2
-ELK_MAIN_SERVICES := ${ELK_SERVICES} ${ELK_MONITORING} ${ELK_TOOLS}
+ELK_MAIN_SERVICES := ${ELK_SERVICES} ${ELK_MONITORING}
 ELK_ALL_SERVICES := ${ELK_MAIN_SERVICES} ${ELK_NODES} ${ELK_LOG_COLLECTION}
 
 compose_v2_not_supported = $(shell command docker compose 2> /dev/null)
@@ -21,7 +19,7 @@ else
 endif
 
 # --------------------------
-.PHONY: setup keystore certs all elk monitoring tools build down stop restart rm logs
+.PHONY: setup keystore certs all elk monitoring build down stop restart rm logs
 
 keystore:		## Setup Elasticsearch Keystore, by initializing passwords, and add credentials defined in `keystore.sh`.
 	$(DOCKER_COMPOSE_COMMAND) -f docker-compose.setup.yml run --rm keystore
@@ -41,17 +39,14 @@ elk:		    ## Start ELK.
 
 up:
 	@make elk
-	@echo "Visit Kibana: https://localhost:5601"
+	@echo "Visit Kibana: https://localhost:5601 (user: elastic, password: changeme) [Unless you changed values in .env]"
 
 monitoring:		## Start ELK Monitoring.
 	$(DOCKER_COMPOSE_COMMAND) ${COMPOSE_MONITORING} up -d --build ${ELK_MONITORING}
 
-collect-docker-logs:		## Start Filebeat that collects all Host Docker Logs and ship it to ELK
+collect-docker-logs: 		## Start Filebeat that collects all Host Docker Logs and ship it to ELK
 	$(DOCKER_COMPOSE_COMMAND) ${COMPOSE_LOGGING} up -d --build ${ELK_LOG_COLLECTION}
 
-tools:		    ## Start ELK Tools (ElastAlert, Curator).
-	$(DOCKER_COMPOSE_COMMAND) ${COMPOSE_TOOLS} up -d --build ${ELK_TOOLS}
-
 nodes:		    ## Start Two Extra Elasticsearch Nodes
 	$(DOCKER_COMPOSE_COMMAND) ${COMPOSE_NODES} up -d --build ${ELK_NODES}
 

README.md

@@ -6,7 +6,7 @@
 <h4 align="center">Configured to be ready to be used for Log, Metrics, APM, Alerting, Machine Learning, and Security (SIEM) usecases.</h4>
 <p align="center">
    <a>
-      <img src="https://img.shields.io/badge/Elastic%20Stack-8.8.0-blue?style=flat&logo=elasticsearch" alt="Elastic Stack Version 7^^">
+      <img src="https://img.shields.io/badge/Elastic%20Stack-8.10.2-blue?style=flat&logo=elasticsearch" alt="Elastic Stack Version 7^^">
    </a>
    <a>
       <img src="https://img.shields.io/github/v/tag/sherifabdlnaby/elastdocker?label=release&amp;sort=semver">
@@ -36,7 +36,7 @@ Elastic Stack (**ELK**) Docker Composition, preconfigured with **Security**, **M
 
 Suitable for Demoing, MVPs and small production deployments.
 
-Stack Version: [8.8.0](https://www.elastic.co/blog/whats-new-elastic-8-8-0) 🎉  - Based on [Official Elastic Docker Images](https://www.docker.elastic.co/)
+Stack Version: [8.10.2](https://www.elastic.co/blog/whats-new-elastic-8-10-0) 🎉  - Based on [Official Elastic Docker Images](https://www.docker.elastic.co/)
 > You can change Elastic Stack version by setting `ELK_VERSION` in `.env` file and rebuild your images. Any version >= 8.0.0 is compatible with this template.
 
 ### Main Features 📜
@@ -45,18 +45,18 @@ Stack Version: [8.8.0](https://www.elastic.co/blog/whats-new-elastic-8-8-0) 🎉
 - Security Enabled By Default.
 - Configured to Enable:
   - Logging & Metrics Ingestion
+    - Option to collect logs of all Docker Containers running on the host. via `make collect-docker-logs`.
   - APM
   - Alerting
   - Machine Learning
-  - SIEM
+  - Anomaly Detection
+  - SIEM (Security information and event management).
   - Enabling Trial License
 - Use Docker-Compose and `.env` to configure your entire stack parameters.
 - Persist Elasticsearch's Keystore and SSL Certifications.
 - Self-Monitoring Metrics Enabled.
 - Prometheus Exporters for Stack Metrics.
-- Collect Docker Host Logs to ELK via `make collect-docker-logs`.
 - Embedded Container Healthchecks for Stack Images.
-- [Rubban](https://github.com/sherifabdlnaby/rubban) for Kibana curating tasks.
 
 #### More points
 And comparing Elastdocker and the popular [deviantony/docker-elk](https://github.com/deviantony/docker-elk)
@@ -85,7 +85,7 @@ Elastdocker differs from `deviantony/docker-elk` in the following points.
 
 - Configuring the Self-Monitoring and the Filebeat agent that ship ELK logs to ELK itself. (as a step to shipping it to a monitoring cluster in the future).
 
-- Configured tools and Prometheus Exporters.
+- Configured Prometheus Exporters.
 
 - The Makefile that simplifies everything into some simple commands.
 
@@ -135,10 +135,6 @@ Elastdocker differs from `deviantony/docker-elk` in the following points.
 ```shell
 $ make monitoring
 ```
-#### To Start Tools
-```shell
-$ make tools
-```
 #### To Ship Docker Container Logs to ELK 
 ```shell
 $ make collect-docker-logs
@@ -172,15 +168,14 @@ $ make prune
 
 * Some Configuration are parameterized in the `.env` file.
   * `ELASTIC_PASSWORD`, user `elastic`'s password (default: `changeme` _pls_).
-  * `ELK_VERSION` Elastic Stack Version (default: `8.8.0`)
+  * `ELK_VERSION` Elastic Stack Version (default: `8.10.2`)
   * `ELASTICSEARCH_HEAP`, how much Elasticsearch allocate from memory (default: 1GB -good for development only-)
   * `LOGSTASH_HEAP`, how much Logstash allocate from memory.
   * Other configurations which their such as cluster name, and node name, etc.
 * Elasticsearch Configuration in `elasticsearch.yml` at `./elasticsearch/config`.
 * Logstash Configuration in `logstash.yml` at `./logstash/config/logstash.yml`.
 * Logstash Pipeline in `main.conf` at `./logstash/pipeline/main.conf`.
 * Kibana Configuration in `kibana.yml` at `./kibana/config`.
-* Rubban Configuration using Docker-Compose passed Environment Variables.
 
 ### Setting Up Keystore
 

docker-compose.nodes.yml

@@ -20,6 +20,7 @@ services:
       ELASTIC_NODE_NAME: ${ELASTIC_NODE_NAME_1}
       ELASTIC_INIT_MASTER_NODE: ${ELASTIC_INIT_MASTER_NODE}
       ELASTIC_DISCOVERY_SEEDS: ${ELASTIC_DISCOVERY_SEEDS}
+      ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT}
       ES_JAVA_OPTS: -Xmx${ELASTICSEARCH_HEAP} -Xms${ELASTICSEARCH_HEAP} -Des.enforce.bootstrap.checks=true
       bootstrap.memory_lock: "true"
     volumes:
@@ -56,6 +57,7 @@ services:
       ELASTIC_NODE_NAME: ${ELASTIC_NODE_NAME_2}
       ELASTIC_INIT_MASTER_NODE: ${ELASTIC_INIT_MASTER_NODE}
       ELASTIC_DISCOVERY_SEEDS: ${ELASTIC_DISCOVERY_SEEDS}
+      ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT}
       ES_JAVA_OPTS: -Xmx${ELASTICSEARCH_HEAP} -Xms${ELASTICSEARCH_HEAP} -Des.enforce.bootstrap.checks=true
       bootstrap.memory_lock: "true"
     volumes:

docker-compose.tools.yml

@@ -45,6 +45,7 @@ services:
       ELASTIC_NODE_NAME: ${ELASTIC_NODE_NAME}
       ELASTIC_INIT_MASTER_NODE: ${ELASTIC_INIT_MASTER_NODE}
       ELASTIC_DISCOVERY_SEEDS: ${ELASTIC_DISCOVERY_SEEDS}
+      ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT}
       ES_JAVA_OPTS: "-Xmx${ELASTICSEARCH_HEAP} -Xms${ELASTICSEARCH_HEAP} -Des.enforce.bootstrap.checks=true -Dlog4j2.formatMsgNoLookups=true"
       bootstrap.memory_lock: "true"
     volumes:
@@ -63,7 +64,7 @@ services:
       - source: elasticsearch.key
         target: /usr/share/elasticsearch/config/certs/elasticsearch.key
     ports:
-      - "9200:9200"
+      - "${ELASTICSEARCH_PORT}:${ELASTICSEARCH_PORT}"
       - "9300:9300"
     ulimits:
       memlock:
@@ -73,7 +74,7 @@ services:
         soft: 200000
         hard: 200000
     healthcheck:
-      test: ["CMD", "sh", "-c", "curl -sf --insecure https://$ELASTIC_USERNAME:$ELASTIC_PASSWORD@localhost:9200/_cat/health | grep -ioE 'green|yellow' || echo 'not green/yellow cluster status'"]
+      test: ["CMD", "sh", "-c", "curl -sf --insecure https://$ELASTIC_USERNAME:$ELASTIC_PASSWORD@localhost:$ELASTICSEARCH_PORT/_cat/health | grep -ioE 'green|yellow' || echo 'not green/yellow cluster status'"]
 
   logstash:
     image: elastdocker/logstash:${ELK_VERSION}
@@ -113,6 +114,7 @@ services:
       ELASTIC_USERNAME: ${ELASTIC_USERNAME}
       ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
       ELASTICSEARCH_HOST_PORT: https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}
+      KIBANA_PORT: ${KIBANA_PORT}
     env_file:
       - ./secrets/.env.kibana.token
     secrets:
@@ -123,7 +125,7 @@ services:
       - source: kibana.key
         target: /certs/kibana.key
     ports:
-      - "5601:5601"
+      - "${KIBANA_PORT}:${KIBANA_PORT}"
 
   apm-server:
     image: elastdocker/apm-server:${ELK_VERSION}

docker-compose.yml

@@ -5,6 +5,7 @@ cluster.name: ${ELASTIC_CLUSTER_NAME}
 node.name:    ${ELASTIC_NODE_NAME}
 network.host: 0.0.0.0
 transport.host: 0.0.0.0
+http.port: ${ELASTICSEARCH_PORT}
 
 ## Cluster Settings
 discovery.seed_hosts: ${ELASTIC_DISCOVERY_SEEDS}

elasticsearch/config/elasticsearch.yml

@@ -3,7 +3,7 @@ set -eo pipefail
 
 host="$(hostname --ip-address || echo '127.0.0.1')"
 
-if health="$(curl -fsSL "http://$ELASTIC_USERNAME:$ELASTIC_PASSWORD@$host:9200/_cat/health?h=status")"; then
+if health="$(curl -fsSL "https://$ELASTIC_USERNAME:$ELASTIC_PASSWORD@$host:$ELASTICSEARCH_PORT/_cat/health?h=status" --insecure")"; then
 	health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" # trim whitespace (otherwise we'll have "green ")
 	if [ "$health" = 'green' ] || [ "$health" = "yellow" ]; then
 		exit 0

elasticsearch/scripts/docker-healthcheck

@@ -4,6 +4,7 @@
 #
 server.name: kibana
 server.host: "0.0.0.0"
+server.port: ${KIBANA_PORT}
 
 # Elasticsearch Connection
 elasticsearch.hosts: [ "${ELASTICSEARCH_HOST_PORT}" ]