From 1a0fdef48979422bd2c9c77796d670ac9611f5d8 Mon Sep 17 00:00:00 2001 From: haniffm Date: Wed, 20 Mar 2019 11:08:54 +0100 Subject: [PATCH 1/2] Add single docker-compose to start all services at once --- .env | 4 + docker-compose.yml | 137 ++++++++++++++++++ docker/elasticsearch/Dockerfile | 8 + docker/elasticsearch/config/elasticsearch.yml | 16 ++ docker/kibana/Dockerfile | 7 + docker/kibana/config/kibana.yml | 7 + docker/logstash/Dockerfile | 7 + docker/logstash/config/logstash.yml | 6 + docker/logstash/pipeline/logstash.conf | 29 ++++ 9 files changed, 221 insertions(+) create mode 100644 .env create mode 100644 docker-compose.yml create mode 100644 docker/elasticsearch/Dockerfile create mode 100644 docker/elasticsearch/config/elasticsearch.yml create mode 100644 docker/kibana/Dockerfile create mode 100644 docker/kibana/config/kibana.yml create mode 100644 docker/logstash/Dockerfile create mode 100644 docker/logstash/config/logstash.yml create mode 100644 docker/logstash/pipeline/logstash.conf diff --git a/.env b/.env new file mode 100644 index 000000000..6526089ce --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +ELK_VERSION=6.5.4 +RABBITMQ_VERSION=3.7.9 +MARIADB_VERSION=10.2.16 +REDIS_VERSION=5.0.4 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..cfd348319 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,137 @@ +version: '3' + +services: + + + ###### EPP ####### + epp_db: + image: "mariadb:$MARIADB_VERSION" + ports: + - 3003:3306 + environment: + MYSQL_DATABASE: epp + MYSQL_USER: arkiv + MYSQL_PASSWORD: password + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + + epp_rabbitmq: + image: "rabbitmq:$RABBITMQ_VERSION" + ports: + - 15003:5672 + environment: + - RABBITMQ_DEFAULT_USER=rabbitmq + - RABBITMQ_DEFAULT_PASS=rabbitmq + + epp_redis: + image: "redis:$REDIS_VERSION" + ports: + - 6003:6379 + + + ###### ETA ####### + eta_db: + image: "mariadb:$MARIADB_VERSION" + ports: + - 3002:3306 + environment: + MYSQL_DATABASE: eta + MYSQL_USER: arkiv + MYSQL_PASSWORD: password + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + eta_rabbitmq: + image: "rabbitmq:$RABBITMQ_VERSION" + ports: + - 15002:5672 + environment: + - RABBITMQ_DEFAULT_USER=rabbitmq + - RABBITMQ_DEFAULT_PASS=rabbitmq + eta_redis: + image: "redis:$REDIS_VERSION" + ports: + - 6002:6379 + + + ###### ETP ####### + etp_db: + image: "mariadb:$MARIADB_VERSION" + ports: + - 3001:3306 + environment: + MYSQL_DATABASE: etp + MYSQL_USER: arkiv + MYSQL_PASSWORD: password + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + etp_rabbitmq: + image: "rabbitmq:$RABBITMQ_VERSION" + ports: + - 15001:5672 + environment: + - RABBITMQ_DEFAULT_USER=rabbitmq + - RABBITMQ_DEFAULT_PASS=rabbitmq + etp_redis: + image: "redis:$REDIS_VERSION" + ports: + - 6001:6379 + + + ###### ELK Stack ####### + elk_elasticsearch: + build: + context: docker/elasticsearch + args: + ELK_VERSION: $ELK_VERSION + volumes: + - ./docker/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro + ports: + - "9200:9200" + - "9300:9300" + - "9202:9200" + - "9302:9300" + - "9203:9200" + - "9303:9300" + environment: + ES_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + + elk_logstash: + build: + context: docker/logstash + args: + ELK_VERSION: $ELK_VERSION + volumes: + - ./docker/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro + - ./docker/logstash/pipeline:/usr/share/logstash/pipeline:ro + ports: + - "5000:5000" + - "9600:9600" + - "5002:5000" + - "9602:9600" + - "5003:5000" + - "9603:9600" + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + depends_on: + - elk_elasticsearch + + elk_kibana: + build: + context: docker/kibana + args: + ELK_VERSION: $ELK_VERSION + volumes: + - ./docker/kibana/config/:/usr/share/kibana/config:ro + ports: + - "5601:5601" + - "5602:5601" + - "5603:5601" + networks: + - elk + depends_on: + - elk_elasticsearch + +networks: + elk: + driver: bridge diff --git a/docker/elasticsearch/Dockerfile b/docker/elasticsearch/Dockerfile new file mode 100644 index 000000000..1f4604ed6 --- /dev/null +++ b/docker/elasticsearch/Dockerfile @@ -0,0 +1,8 @@ +ARG ELK_VERSION + +# https://github.com/elastic/elasticsearch-docker +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION} + +# Add your elasticsearch plugins setup here + +RUN elasticsearch-plugin install --batch ingest-attachment diff --git a/docker/elasticsearch/config/elasticsearch.yml b/docker/elasticsearch/config/elasticsearch.yml new file mode 100644 index 000000000..e97577084 --- /dev/null +++ b/docker/elasticsearch/config/elasticsearch.yml @@ -0,0 +1,16 @@ +--- +## Default Elasticsearch configuration from elasticsearch-docker. +## from https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml +# +cluster.name: "docker-cluster" +network.host: 0.0.0.0 + +# minimum_master_nodes need to be explicitly set when bound on a public IP +# set to 1 to allow single node clusters +# Details: https://github.com/elastic/elasticsearch/pull/17288 +discovery.zen.minimum_master_nodes: 1 + +## Use single node discovery in order to disable production mode and avoid bootstrap checks +## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html +# +discovery.type: single-node diff --git a/docker/kibana/Dockerfile b/docker/kibana/Dockerfile new file mode 100644 index 000000000..202855cc9 --- /dev/null +++ b/docker/kibana/Dockerfile @@ -0,0 +1,7 @@ +ARG ELK_VERSION + +# https://github.com/elastic/kibana-docker +FROM docker.elastic.co/kibana/kibana:${ELK_VERSION} + +# Add your kibana plugins setup here +# Example: RUN kibana-plugin install diff --git a/docker/kibana/config/kibana.yml b/docker/kibana/config/kibana.yml new file mode 100644 index 000000000..4b34a1606 --- /dev/null +++ b/docker/kibana/config/kibana.yml @@ -0,0 +1,7 @@ +--- +## Default Kibana configuration from kibana-docker. +## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml +# +server.name: kibana +server.host: "0" +elasticsearch.url: http://elasticsearch:9200 diff --git a/docker/logstash/Dockerfile b/docker/logstash/Dockerfile new file mode 100644 index 000000000..6cc863745 --- /dev/null +++ b/docker/logstash/Dockerfile @@ -0,0 +1,7 @@ +ARG ELK_VERSION + +# https://github.com/elastic/logstash-docker +FROM docker.elastic.co/logstash/logstash:${ELK_VERSION} + +# Add your logstash plugins setup here +# Example: RUN logstash-plugin install logstash-filter-json diff --git a/docker/logstash/config/logstash.yml b/docker/logstash/config/logstash.yml new file mode 100644 index 000000000..e7e260f22 --- /dev/null +++ b/docker/logstash/config/logstash.yml @@ -0,0 +1,6 @@ +--- +## Default Logstash configuration from logstash-docker. +## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash-oss.yml +# +http.host: "0.0.0.0" +path.config: /usr/share/logstash/pipeline diff --git a/docker/logstash/pipeline/logstash.conf b/docker/logstash/pipeline/logstash.conf new file mode 100644 index 000000000..875320e7c --- /dev/null +++ b/docker/logstash/pipeline/logstash.conf @@ -0,0 +1,29 @@ +input { + tcp { + port => 5000 + codec => json + } +} + +## Add your filters / logstash plugins configuration here +filter { + json { + source => "[message][raw]" + } + + # Workaround for HTTP logs created from "django.channels.server" that have an extra "\u001b[m" at start and "\u001b[0m" at end. + if [type] == "django_http" { + grok { + match => { "message" => "m%{URIPROTO:protocol} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:status_code} \[%{NUMBER:duration}, %{HOSTPORT:host}\]" } + } + mutate { + remove_field => [ "message" ] + } + } +} + +output { + elasticsearch { + hosts => "elasticsearch:9200" + } +} From b21e59d70ffee20102dbbe80db64413feaa9cb6b Mon Sep 17 00:00:00 2001 From: haniffm Date: Wed, 20 Mar 2019 17:30:27 +0100 Subject: [PATCH 2/2] Changed the DBs from mysql to postgres --- .env | 2 +- docker-compose.yml | 47 +++++++++++++++++++++++----------------------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/.env b/.env index 6526089ce..6a4ff6e52 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ ELK_VERSION=6.5.4 RABBITMQ_VERSION=3.7.9 -MARIADB_VERSION=10.2.16 +POSTGRES_VERSION=11.2-alpine REDIS_VERSION=5.0.4 diff --git a/docker-compose.yml b/docker-compose.yml index cfd348319..b2c9f5b4b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,14 +5,13 @@ services: ###### EPP ####### epp_db: - image: "mariadb:$MARIADB_VERSION" + image: "postgres:$POSTGRES_VERSION" ports: - - 3003:3306 + - 3003:5432 environment: - MYSQL_DATABASE: epp - MYSQL_USER: arkiv - MYSQL_PASSWORD: password - MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + POSTGRES_DB: epp + POSTGRES_USER: arkiv + POSTGRES_PASSWORD: password epp_rabbitmq: image: "rabbitmq:$RABBITMQ_VERSION" @@ -30,14 +29,14 @@ services: ###### ETA ####### eta_db: - image: "mariadb:$MARIADB_VERSION" + image: "postgres:$POSTGRES_VERSION" ports: - - 3002:3306 + - 3002:5432 environment: - MYSQL_DATABASE: eta - MYSQL_USER: arkiv - MYSQL_PASSWORD: password - MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + POSTGRES_DB: eta + POSTGRES_USER: arkiv + POSTGRES_PASSWORD: password + eta_rabbitmq: image: "rabbitmq:$RABBITMQ_VERSION" ports: @@ -45,6 +44,7 @@ services: environment: - RABBITMQ_DEFAULT_USER=rabbitmq - RABBITMQ_DEFAULT_PASS=rabbitmq + eta_redis: image: "redis:$REDIS_VERSION" ports: @@ -53,14 +53,14 @@ services: ###### ETP ####### etp_db: - image: "mariadb:$MARIADB_VERSION" + image: "postgres:$POSTGRES_VERSION" ports: - - 3001:3306 + - 3001:5432 environment: - MYSQL_DATABASE: etp - MYSQL_USER: arkiv - MYSQL_PASSWORD: password - MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + POSTGRES_DB: etp + POSTGRES_USER: arkiv + POSTGRES_PASSWORD: password + etp_rabbitmq: image: "rabbitmq:$RABBITMQ_VERSION" ports: @@ -68,6 +68,7 @@ services: environment: - RABBITMQ_DEFAULT_USER=rabbitmq - RABBITMQ_DEFAULT_PASS=rabbitmq + etp_redis: image: "redis:$REDIS_VERSION" ports: @@ -75,7 +76,7 @@ services: ###### ELK Stack ####### - elk_elasticsearch: + elasticsearch: build: context: docker/elasticsearch args: @@ -94,7 +95,7 @@ services: networks: - elk - elk_logstash: + logstash: build: context: docker/logstash args: @@ -114,9 +115,9 @@ services: networks: - elk depends_on: - - elk_elasticsearch + - elasticsearch - elk_kibana: + kibana: build: context: docker/kibana args: @@ -130,7 +131,7 @@ services: networks: - elk depends_on: - - elk_elasticsearch + - elasticsearch networks: elk: