08: Apache Spark standalone cluster on Docker to read from AWS S3 bucket & write to MySQL database

Posted on June 4, 2018 by

This extends Apache Spark local mode read from AWS S3 bucket with Docker. Step 1: The “docker-compose.yml” with minio to emulate AWS S3, MySQL DB, Spark master and Spark worker to form a cluster. “mysql” is the hostname.

version: "2" 

services:
  s3:
    container_name: s3
    image: minio/minio:RELEASE.2018-05-04T23-13-12Z
    environment:
      MINIO_ACCESS_KEY: user
      MINIO_SECRET_KEY: password
    ports:
      - 9000:9000
    volumes:
      - ./docker/s3/data:/data
      - ./docker/s3/config:/root/.minio
    networks:
      - infra
    command: server /data

  mysql:
    container_name: mysql
    image: mysql:5.7.21
    ports:
      - 3306:3306
    volumes:
      - ./docker/mysql/data:/var/lib/mysql
    environment:
      MYSQL_ROOT_PASSWORD: password
    networks:
      - infra

  spark-master:
      build:
        dockerfile: spark.dockerfile
        context: ./docker/spark
      command: bin/spark-class org.apache.spark.deploy.master.Master -h spark-master
      hostname: spark-master
      environment:
        MASTER: spark://spark-master:7077
        SPARK_CONF_DIR: /conf
        SPARK_PUBLIC_DNS: 127.0.0.1
      expose:
        - 7001
        - 7002
        - 7003
        - 7004
        - 7005
        - 7006
        - 7077
        - 6066
        - 5005
      ports:
        - 5005:5005
        - 6066:6066
        - 7077:7077
        - 8080:8080
      volumes:
        - ./docker/spark/spark-master:/conf
        - ./data:/tmp/data 
      networks:
        - infra 
  
  spark-worker-1:
      build:
        dockerfile: spark.dockerfile
        context: ./docker/spark
      command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
      hostname: spark-worker-1
      environment:
        SPARK_CONF_DIR: /conf
        SPARK_PUBLIC_DNS: 127.0.0.1
        SPARK_WORKER_CORES: 2
        SPARK_WORKER_MEMORY: 2g
        SPARK_WORKER_PORT: 8881
        SPARK_WORKER_WEBUI_PORT: 8081
      links:
        - spark-master
      expose:
        - 7012
        - 7013
        - 7014
        - 7015
        - 7016
        - 8881
      ports:
        - 8081:8081
      volumes:
        - ./conf/spark-worker-1:/conf
        - ./data:/tmp/data 
      networks:
        - infra

networks:
  infra:
    external:
      name: docker_test_infra

version: "2"

services:

s3:

container_name: s3

image: minio/minio:RELEASE.2018-05-04T23-13-12Z

environment:

MINIO_ACCESS_KEY: user

MINIO_SECRET_KEY: password

ports:

- 9000:9000

volumes:

- ./docker/s3/data:/data

- ./docker/s3/config:/root/.minio

networks:

- infra

command: server /data

mysql:

container_name: mysql

image: mysql:5.7.21

ports:

- 3306:3306

volumes:

- ./docker/mysql/data:/var/lib/mysql

environment:

MYSQL_ROOT_PASSWORD: password

networks:

- infra

spark-master:

build:

dockerfile: spark.dockerfile

context: ./docker/spark

command: bin/spark-class org.apache.spark.deploy.master.Master -h spark-master

hostname: spark-master

environment:

MASTER: spark://spark-master:7077

SPARK_CONF_DIR: /conf

SPARK_PUBLIC_DNS: 127.0.0.1

expose:

- 7001

- 7002

- 7003

- 7004

- 7005

- 7006

- 7077

- 6066

- 5005

ports:

- 5005:5005

- 6066:6066

- 7077:7077

- 8080:8080

volumes:

- ./docker/spark/spark-master:/conf

- ./data:/tmp/data

networks:

- infra

spark-worker-1:

build:

dockerfile: spark.dockerfile

context: ./docker/spark

command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077

hostname: spark-worker-1

environment:

SPARK_CONF_DIR: /conf

SPARK_PUBLIC_DNS: 127.0.0.1

SPARK_WORKER_CORES: 2

SPARK_WORKER_MEMORY: 2g

SPARK_WORKER_PORT: 8881

SPARK_WORKER_WEBUI_PORT: 8081

links:

- spark-master

expose:

- 7012

- 7013

- 7014

- 7015

- 7016

- 8881

ports:

- 8081:8081

volumes:

- ./conf/spark-worker-1:/conf

- ./data:/tmp/data

networks:

- infra

networks:

infra:

external:

name: docker_test_infra

Step…

Members Only Content

Thanks for your support in extending the membership. I am able to clear the tech interviews and started new job with a bank. I can definitely say that Java success played a major role in my success. -- Balakrishna D

I am Naidu, I have 4 years of experience in java/Java EE. Few months back i started attending interviews and brought you book "Core Java Career essentials" and gone through most of the questions. I have cracked more than 10 companies and now i have joined Product based company. I felt very happy and thought of writing a mail to you and you help in the form of blog and book. -- Naidu M

More reviews

The contents in this Java-Success are copyrighted and from EmpoweringTech pty ltd. The EmpoweringTech pty ltd has the right to correct or enhance the current content without any prior notice. These are general advice only, and one needs to take his/her own circumstances into consideration. The EmpoweringTech pty ltd will not be held liable for any damages caused or alleged to be caused either directly or indirectly by these materials and resources. Any trademarked names or labels used in this blog remain the property of their respective trademark owners. Links to external sites do not imply endorsement of the linked-to sites. Privacy Policy

Responsive WordPress Theme powered by CyberChimps

Top

08: Apache Spark standalone cluster on Docker to read from AWS S3 bucket & write to MySQL database

Members Only Content

Categories Menu - Q&As, FAQs & Tutorials

Your Career Blueprint

Reviews

More reviews

Disclaimer