DolphinScheduler系列（一）DolphinScheduler安装

一、docker-compose方式安装DolphinScheduler

1.1环境准备

1.1.1 环境准备

------------------关闭selinux-----------
--查看状态
shell>getenforce
------------关闭selinux
shell> vi /etc/selinux/config
selinux配置有三项：缺省是强制策略。
enforcing - 强制打开,拒绝违反安全策略
permissive - 遇到违反安全策略仍正常执行，但输出警告
disabled - 关闭安全策略
修改SELINUX=disabled
------------------关闭Firewalld
shell>systemctl disable firewalld.service
shell>systemctl stop firewalld.service
shell>systemctl status firewalld.service
---------------关闭iptalbes
shell>service iptables stop
shell>chkconfig iptables off

1.1.2 安装docker

1.1.3 安装docker-compose

1.2 下载apache-dolphinscheduler

apache-dolphinscheduler-2.0.6-src.tar.gz下载链接

[root@master ~]# mkdir -p /data/app
[root@master ~]# cd -p /data/app
[root@master app]# wget https://dlcdn.apache.org/dolphinscheduler/2.0.6/apache-dolphinscheduler-2.0.6-src.tar.gz --no-check-certificate
[root@master app]# tar -zxvf apache-dolphinscheduler-2.0.6-src.tar.gz
[root@master app]# mv apache-dolphinscheduler-2.0.6-src dolphinscheduler
[root@master app]# cd dolphinscheduler/docker/docker-swarm
################拉取镜像并重命名
[root@master docker-swarm]# docker pull dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.6
2.0.6: Pulling from apache/dolphinscheduler
[root@master docker-swarm]# docker tag dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.6 apache/dolphinscheduler:latest

1.3 封装mysql和pyhton的dolphinscheduler 镜像

1.3.1 下载mysql链接jar包

[root@master docker-swarm]# wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar

1.3.2 创建 Dockerfile

[root@master docker-swarm]# vim Dockerfile
#####content######
FROM apache/dolphinscheduler:latest
COPY mysql-connector-java-8.0.16.jar /opt/dolphinscheduler/lib
RUN apt-get update && \
    apt-get install -y --no-install-recommends python3 && \
    rm -rf /var/lib/apt/lists/*

1.3.3 构建mysql和pyhton的镜像

[root@master docker-swarm]# docker build -t apache/dolphinscheduler:sql-driver .

1.4 修改docker-compose.yml

[root@master docker-swarm]# cat docker-compose.yml 
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

version: "3.1"

services:
  dolphinscheduler-mysql:
    image: bitnami/mysql:latest
    environment:
      TZ: Asia/Shanghai
      MYSQL_USERNAME: root
      MYSQL_ROOT_PASSWORD: root
      MYSQL_DATABASE: dolphinscheduler
    ports:
    - 3306:3306
    volumes:
    - dolphinscheduler-mysql:/bitnami/mysql
    restart: unless-stopped
    networks:
    - dolphinscheduler
#  dolphinscheduler-postgresql:
#    image: bitnami/postgresql:11.11.0
#    environment:
#      TZ: Asia/Shanghai
#      POSTGRESQL_USERNAME: root
#      POSTGRESQL_PASSWORD: root
#      POSTGRESQL_DATABASE: dolphinscheduler
#    volumes:
#    - dolphinscheduler-postgresql:/bitnami/postgresql
#    restart: unless-stopped
#    networks:
#    - dolphinscheduler

  dolphinscheduler-zookeeper:
    image: bitnami/zookeeper:3.6.2
    environment:
      TZ: Asia/Shanghai
      ALLOW_ANONYMOUS_LOGIN: "yes"
      ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
    volumes:
    - dolphinscheduler-zookeeper:/bitnami/zookeeper
    restart: unless-stopped
    networks:
    - dolphinscheduler

  dolphinscheduler-api:
    image: apache/dolphinscheduler:sql-driver
    command: api-server
    ports:
    - 12345:12345
    environment:
      TZ: Asia/Shanghai
    env_file: config.env.sh
    healthcheck:
      test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
    - dolphinscheduler-postgresql
    - dolphinscheduler-zookeeper
    volumes:
    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    - dolphinscheduler-shared-local:/opt/soft
    - dolphinscheduler-resource-local:/dolphinscheduler
    restart: unless-stopped
    networks:
    - dolphinscheduler

  dolphinscheduler-alert:
    image: apache/dolphinscheduler:sql-driver
    command: alert-server
    environment:
      TZ: Asia/Shanghai
    env_file: config.env.sh
    healthcheck:
      test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
    - dolphinscheduler-postgresql
    volumes:
    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    restart: unless-stopped
    networks:
    - dolphinscheduler

  dolphinscheduler-master:
    image: apache/dolphinscheduler:sql-driver
    command: master-server
    environment:
      TZ: Asia/Shanghai
    env_file: config.env.sh
    healthcheck:
      test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
    - dolphinscheduler-postgresql
    - dolphinscheduler-zookeeper
    volumes:
    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    - dolphinscheduler-shared-local:/opt/soft
    restart: unless-stopped
    networks:
    - dolphinscheduler

  dolphinscheduler-worker:
    image: apache/dolphinscheduler:sql-driver
    command: worker-server
    environment:
      TZ: Asia/Shanghai
    env_file: config.env.sh
    healthcheck:
      test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
    - dolphinscheduler-postgresql
    - dolphinscheduler-zookeeper
    volumes:
    - dolphinscheduler-worker-data:/tmp/dolphinscheduler
    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    - dolphinscheduler-shared-local:/opt/soft
    - dolphinscheduler-resource-local:/dolphinscheduler
    restart: unless-stopped
    networks:
    - dolphinscheduler

networks:
  dolphinscheduler:
    driver: bridge

volumes:
  dolphinscheduler-mysql:
#  dolphinscheduler-postgresql:
  dolphinscheduler-zookeeper:
  dolphinscheduler-worker-data:
  dolphinscheduler-logs:
  dolphinscheduler-shared-local:
  dolphinscheduler-resource-local:

1.5 修改config.env.sh

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

#============================================================================
# Database
#============================================================================
# postgresql
#DATABASE_TYPE=postgresql
#DATABASE_DRIVER=org.postgresql.Driver
#DATABASE_HOST=dolphinscheduler-postgresql
#DATABASE_PORT=5432
#DATABASE_USERNAME=root
#DATABASE_PASSWORD=root
#DATABASE_DATABASE=dolphinscheduler
#DATABASE_PARAMS=characterEncoding=utf8
# mysql
 DATABASE_TYPE=mysql
 DATABASE_DRIVER=com.mysql.jdbc.Driver
 DATABASE_HOST=dolphinscheduler-mysql
 DATABASE_PORT=3306
 DATABASE_USERNAME=root
 DATABASE_PASSWORD=root
 DATABASE_DATABASE=dolphinscheduler
 DATABASE_PARAMS=useUnicode=true&characterEncoding=UTF-8

#============================================================================
# Registry
#============================================================================
REGISTRY_PLUGIN_NAME=zookeeper
REGISTRY_SERVERS=dolphinscheduler-zookeeper:2181

#============================================================================
# Common
#============================================================================
# common opts
DOLPHINSCHEDULER_OPTS=
# common env
DATA_BASEDIR_PATH=/tmp/dolphinscheduler
RESOURCE_STORAGE_TYPE=HDFS
RESOURCE_UPLOAD_PATH=/dolphinscheduler
FS_DEFAULT_FS=file:///
FS_S3A_ENDPOINT=s3.xxx.amazonaws.com
FS_S3A_ACCESS_KEY=xxxxxxx
FS_S3A_SECRET_KEY=xxxxxxx
HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE=false
JAVA_SECURITY_KRB5_CONF_PATH=/opt/krb5.conf
LOGIN_USER_KEYTAB_USERNAME=hdfs@HADOOP.COM
LOGIN_USER_KEYTAB_PATH=/opt/hdfs.keytab
KERBEROS_EXPIRE_TIME=2
HDFS_ROOT_USER=hdfs
RESOURCE_MANAGER_HTTPADDRESS_PORT=8088
YARN_RESOURCEMANAGER_HA_RM_IDS=
YARN_APPLICATION_STATUS_ADDRESS=http://ds1:%s/ws/v1/cluster/apps/%s
YARN_JOB_HISTORY_STATUS_ADDRESS=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
DATASOURCE_ENCRYPTION_ENABLE=false
DATASOURCE_ENCRYPTION_SALT=!@#$%^&*
SUDO_ENABLE=true
# dolphinscheduler env
HADOOP_HOME=/opt/soft/hadoop
HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
SPARK_HOME1=/opt/soft/spark1
SPARK_HOME2=/opt/soft/spark2
PYTHON_HOME=/usr/bin/python
JAVA_HOME=/usr/local/openjdk-8
HIVE_HOME=/opt/soft/hive
FLINK_HOME=/opt/soft/flink
DATAX_HOME=/opt/soft/datax

#============================================================================
# Master Server
#============================================================================
#MASTER_SERVER_OPTS=-Xms1g -Xmx1g -Xmn512m
MASTER_EXEC_THREADS=100
MASTER_EXEC_TASK_NUM=20
MASTER_DISPATCH_TASK_NUM=3
MASTER_HOST_SELECTOR=LowerWeight
MASTER_HEARTBEAT_INTERVAL=10
MASTER_TASK_COMMIT_RETRYTIMES=5
MASTER_TASK_COMMIT_INTERVAL=1000
MASTER_MAX_CPULOAD_AVG=-1
MASTER_RESERVED_MEMORY=0.3
MASTER_FAILOVER_INTERVAL=10
MASTER_KILL_YARN_JOB_WHEN_HANDLE_FAILOVER=true
MASTER_PERSIST_EVENT_STATE_THREADS=10

#============================================================================
# Worker Server
#============================================================================
#WORKER_SERVER_OPTS=-Xms1g -Xmx1g -Xmn512m
WORKER_EXEC_THREADS=100
WORKER_HEARTBEAT_INTERVAL=10
WORKER_HOST_WEIGHT=100
WORKER_MAX_CPULOAD_AVG=-1
WORKER_RESERVED_MEMORY=0.3
WORKER_GROUPS=default
WORKER_RETRY_REPORT_TASK_STATUES_INTERVAL=10
ALERT_LISTEN_HOST=dolphinscheduler-alert

#============================================================================
# Alert Server
#============================================================================
#ALERT_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m

#============================================================================
# Api Server
#============================================================================
#API_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m

#============================================================================
# Logger Server
#============================================================================
#LOGGER_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m

1.6 执行docker-compose

急记：切换到/data/app/dolphinscheduler/docker/docker-swarm

shell>docker-compose -f docker-compose.yml up -d