Merge pull request #3 from kpacha/mesos-dns

Mesos dns

Author: kpacha

.travis.yml

@@ -9,6 +9,5 @@ go:
   - 1.4
 
 install:
-  - go get ./...
-  - go fmt
+  - make
   - go vet

Dockerfile

@@ -1,19 +1,15 @@
-FROM golang:1.5.1-onbuild
+FROM golang:1.5.1
 MAINTAINER kpacha
 
-ENV MESOS_MASTER_HOST=leader.mesos
-ENV MESOS_MASTER_PORT=5050
-ENV MESOS_SLAVE_HOST=slave.mesos
-ENV MESOS_SLAVE_PORT=5051
-
-ENV MARATHON_HOST=marathon.mesos
-ENV MARATHON_PORT=8080
-
-ENV INFLUXDB_HOST=influxdb
-ENV INFLUXDB_PORT=8086
-ENV INFLUXDB_DB=mesos
 ENV INFLUXDB_USER=root
 ENV INFLUXDB_PWD=root
 
-ENV COLLECTOR_LAPSE=1
-ENV COLLECTOR_LIFETIME=1800
+RUN mkdir -p /go/src/github.com/kpacha/mesos-influxdb-collector
+COPY . /go/src/github.com/kpacha/mesos-influxdb-collector
+
+WORKDIR /go/src/github.com/kpacha/mesos-influxdb-collector
+RUN make install
+
+ENTRYPOINT ["/go/bin/mesos-influxdb-collector"]
+
+CMD ["-c", "conf.hcl"]

Dockerfile-min

@@ -0,0 +1,12 @@
+FROM busybox:1.24.1
+MAINTAINER kpacha
+
+ENV INFLUXDB_USER=root
+ENV INFLUXDB_PWD=root
+
+COPY ./mesos-influxdb-collector /mesos-influxdb-collector
+COPY ./conf.hcl /conf.hcl
+
+ENTRYPOINT ["/./mesos-influxdb-collector"]
+
+CMD ["-c", "/conf.hcl"]

Makefile

@@ -0,0 +1,23 @@
+all: deps build test
+
+deps:
+	go get github.com/hashicorp/hcl
+	go get github.com/influxdb/influxdb/client
+
+gen:
+	go fmt ./...
+
+build:
+	sh -c 'CGO_ENABLED=0 go build -a --installsuffix cgo --ldflags="-s" -o mesos-influxdb-collector'
+
+install: all do_install
+
+do_install:
+	go install
+
+test:
+	go test -cover ./...
+
+docker:
+	docker run --rm -it -e "GOPATH=/go" -v "${PWD}:/go/src/github.com/kpacha/mesos-influxdb-collector" -w /go/src/github.com/kpacha/mesos-influxdb-collector golang:1.5.1 make
+	docker build -f Dockerfile-min -t kpacha/mesos-influxdb-collector:mesos-dns-min .

README.md

@@ -3,7 +3,16 @@ mesos-influxdb-collector
 
 Lightweight mesos stats collector for influxdb
 
-Since this collector is intended to be deployed as a [marathon](https://mesosphere.github.io/marathon) app, it comes with a *lifetime* param. This defines how long the collector will run until it dies, so marathon will re-launch it, allowing easy allocation optimizations. Check the [marathon/](https://github.com/kpacha/mesos-influxdb-collector/tree/master/marathon) folder for more details on how to launch it.
+Since this collector is intended to be deployed as a [marathon](https://mesosphere.github.io/marathon) app, it comes with a *lifetime* param. This defines how long the collector will run until it dies, so marathon will re-launch it, allowing easy allocation optimizations. Check the [fixtures/marathon/](https://github.com/kpacha/mesos-influxdb-collector/tree/master/fixtures/marathon) folder for more details on how to launch it.
+
+# Goals
+
++ Discover the mesos cluster through `mesos-dns`
++ Collect the mesos leader stats
++ Collect the mesos master stats
++ Collect the mesos slave stats
++ Collect the marathon master stats
++ Collect the chronos task stats (TODO)
 
 # Installing
 
@@ -19,27 +28,93 @@ Alternatively, if you have Go installed:
 $ go get github.com/kpacha/mesos-influxdb-collector
 ```
 
-# Running
+# Integration with `mesos-dns`
+
+The `mesos-influxdb-collector` is able to discover all your mesos nodes (masters and slaves) and the marathon master using the REST API exposed by the [mesos-dns](http://mesosphere.github.io/mesos-dns/) service. Check the next section for details.
+
+# Configuration
 
 The collector use these environmental vars:
 
-+ `INFLUXDB_DB`
-+ `INFLUXDB_HOST`
-+ `INFLUXDB_PORT`
 + `INFLUXDB_USER`
 + `INFLUXDB_PWD`
-+ `MESOS_MASTER_HOST`
-+ `MESOS_MASTER_PORT`
-+ `MESOS_SLAVE_HOST`
-+ `MESOS_SLAVE_PORT`
-+ `MARATHON_HOST`
-+ `MARATHON_PORT`
-+ `COLLECTOR_LAPSE`
-+ `COLLECTOR_LIFETIME`
+
+It also requires a config file with the list of nodes to monitor or the details about the `mesos-dns` service among these other params:
+
++ *Lapse*: time between consecutive collections. Default: 30 seconds
++ *DieAfter*: duration of the running instance. Default: 1 hour
+
+### MesosDNS
+
+Optional. Add it if you have a `mesos-dns` service running in your mesos cluster.
+
+```
+mesosDNS {
+  domain = "mesos" // the domain used by the mesos-dns service
+  marathon = true // resolve marathon master
+  host = "master.mesos" // host of the mesos-dns service
+  port = 8123 // port of the REST API
+}
+```
+
+### InfluxDB
+
+Required.
+
+```
+influxdb {
+  host = "influxdb.marathon.mesos" // host of the influxdb instance
+  port = 8086 // port of the REST API
+  db = "mesos" // name of the database to use
+  checkLapse = 30 // ping frequency
+}
+```
+
+### Mesos masters
+
+For manual definition of some (or all) mesos masters, use the `Master` struct:
+
+```
+Master {
+  host = "localhost"
+  port = 5051
+  leader = true // optional
+}
+Master {
+  host = "localhost"
+  port = 5052
+}
+```
+
+### Mesos slaves
+
+For manual definition of some (or all) mesos slave, use the `Slave` struct:
+
+```
+Slave {
+  host = "localhost"
+  port = 5051
+}
+```
+
+### Marathon instances
+
+For manual definition of some (or all) marathon instances, use the `Marathon` struct:
+
+```
+Marathon {
+  host = "localhost"
+  port = 5052
+}
+```
+
+Check [`config/configuration_test.go`](https://github.com/kpacha/mesos-influxdb-collector/blob/master/config/configuration_test.go) and [`conf.hcl`](https://github.com/kpacha/mesos-influxdb-collector/blob/master/conf.hcl) for examples.
+
+# Running
 
 ## Dockerized version
 
-Run the container with the default params (check the Dockerfile and overwrite whatever you need):
+Run the container with the default params:
 
 ```
 $ docker pull --name mesos-influxdb-collector \
@@ -48,77 +123,27 @@ $ docker pull --name mesos-influxdb-collector \
     -it --rm kpacha/mesos-influxdb-collector
 ```
 
-Since the default value for `INFLUXDB_HOST` is `ìnfluxb`, you can link the collector to the influxdb container, dependeing on your environment.
+If you need to customize something, copy the `conf.hcl`, make your changes and link it as a volume:
 
 ```
-$ docker run --name mesos-influxdb-collector \
-    --link influxdb \
-    -it --rm kpacha/mesos-influxdb-collector
+$ docker pull --name mesos-influxdb-collector \
+    -v /path/to/my/custom/conf.hcl:/tmp/conf.hcl \
+    -it --rm kpacha/mesos-influxdb-collector -c /tmp/conf.hcl
 ```
 
+Tip: if you link your config file to `/go/src/github.com/kpacha/mesos-influxdb-collector/conf.hcl` you don't need to worry about that flag!
+
 ## Binary version
 
 ```
 $ ./mesos-influxdb-collector -h
 Usage of ./mesos-influxdb-collector:
-  -Id string
-      influxdb database (default "mesos")
-  -Ih string
-      influxdb host (default "localhost")
-  -Ip int
-      influxdb port (default 8086)
-  -Mmh string
-      mesos master host (default "localhost")
-  -Mmp int
-      mesos master port (default 5050)
-  -Msh string
-      mesos slave host (default "localhost")
-  -Msp int
-      mesos slave port (default 5051)
-  -d int
-      die after N seconds (default 300)
-  -l int
-      sleep time between collections in seconds (default 1)
-  -mh string
-      marathon host (default "localhost")
-  -mp int
-      marathon port (default 8080)
-```
-
-This is the relation between those params and the environmnetal variables listed above.
-
-Flag  | EnvVar
-----  | ------
-`Id`  | `INFLUXDB_DB`
-`Ih`  | `INFLUXDB_HOST`
-`Ip`  | `INFLUXDB_PORT`
-`Mmh` | `MESOS_MASTER_HOST`
-`Mmp` | `MESOS_MASTER_PORT`
-`Msh` | `MESOS_SLAVE_HOST`
-`Msp` | `MESOS_SLAVE_PORT`
-`mh`  | `MARATHON_HOST`
-`mp`  | `MARATHON_PORT`
-`d`   | `COLLECTOR_LAPSE`
-`l`   | `COLLECTOR_LIFETIME`
+  -c string
+      path to the config file (default "conf.hcl")
+```
 
 The credentials for the influxdb database are accepted just as env_var (`INFLUXDB_USER` & `INFLUXDB_PWD`)
 
-## Testing environment
-
-In order to do a quick test of the collector, you can use one of the available mesos test environments: [playa-mesos](https://github.com/mesosphere/playa-mesos) & [mesoscope](https://github.com/schibsted/mesoscope). The other components can be deployed with public containers. Replace the `$DOCKER_IP` and `$MESOS_HOST` with the correct values. If you are running the mesoscope env, `MESOS_HOST=$DOCKER_IP`. For the playa-mesos option, `MESOS_HOST=10.141.141.10`.
-
-```
-$ docker run --name influxdb -p 8083:8083 -p 8086:8086 \
-    --expose 8090 --expose 8099 \
-    -d tutum/influxdb
-$ docker run --name grafana -p 3000:3000 \
-    -e GF_SERVER_ROOT_URL="http://$DOCKER_IP" \
-    -e GF_SECURITY_ADMIN_PASSWORD=secret \
-    -d grafana/grafana
-$ docker run --name mesos-influxdb-collector \
-    --link influxdb \
-    -e MESOS_HOST=$MESOS_HOST \
-    -it --rm kpacha/mesos-influxdb-collector
-```
+# Grafana dashboards
 
-The `grafana` folder contains several grafana dashboard definitions. Go to the grafana website (`http://$DOCKER_IP:3000/) and, after configuring the influxdb datasource, import them and start monitoring your mesos cluster.
+The `fixtures/grafana` folder contains several grafana dashboard definitions. Go to the grafana website and, after configuring the influxdb datasource, import them and start monitoring your mesos cluster.

collector.go

@@ -15,10 +15,6 @@ type MultiCollector struct {
 	Collectors []Collector
 }
 
-func NewMultiCollector(collectors []Collector) Collector {
-	return MultiCollector{collectors}
-}
-
 func (mc MultiCollector) Collect() ([]store.Point, error) {
 	var data []store.Point
 	for _, c := range mc.Collectors {

collector/collector.go

@@ -0,0 +1,60 @@
+package collector
+
+import (
+	"fmt"
+	"github.com/kpacha/mesos-influxdb-collector/config"
+	"github.com/kpacha/mesos-influxdb-collector/parser/marathon"
+	"github.com/kpacha/mesos-influxdb-collector/parser/mesos"
+	"log"
+	"net/url"
+)
+
+func NewCollectorFromConfig(configuration *config.Config) Collector {
+	var collectors []Collector
+
+	for _, master := range configuration.Master {
+		collectors = append(collectors, NewMesosMasterCollector(master.Host, master.Port, master.Leader))
+	}
+	for _, slave := range configuration.Slave {
+		collectors = append(collectors, NewMesosSlaveCollector(slave.Host, slave.Port))
+	}
+	for _, marathonInstance := range configuration.Marathon {
+		collectors = append(collectors, NewMarathonCollector(marathonInstance.Host, marathonInstance.Port))
+	}
+
+	log.Println("Total collectors created:", len(collectors))
+
+	return NewMultiCollector(collectors)
+}
+
+func NewMultiCollector(collectors []Collector) Collector {
+	return MultiCollector{collectors}
+}
+
+func NewMesosLeaderCollector(host string, port int) Collector {
+	return NewMesosMasterCollector(host, port, true)
+}
+
+func NewMesosMasterCollector(host string, port int, leader bool) Collector {
+	u, err := url.Parse(fmt.Sprintf("http://%s:%d/metrics/snapshot", host, port))
+	if err != nil {
+		log.Fatal("Error building the mesos master collector:", err)
+	}
+	return UrlCollector{Url: u.String(), Parser: mesos.MasterParser{Node: host, Leader: leader}}
+}
+
+func NewMesosSlaveCollector(host string, port int) Collector {
+	u, err := url.Parse(fmt.Sprintf("http://%s:%d/metrics/snapshot", host, port))
+	if err != nil {
+		log.Fatal("Error building the mesos slave collector:", err)
+	}
+	return UrlCollector{Url: u.String(), Parser: mesos.SlaveParser{Node: host}}
+}
+
+func NewMarathonCollector(host string, port int) Collector {
+	u, err := url.Parse(fmt.Sprintf("http://%s:%d/metrics", host, port))
+	if err != nil {
+		log.Fatal("Error building the marathon collector:", err)
+	}
+	return UrlCollector{Url: u.String(), Parser: marathon.MarathonParser{Node: host}}
+}