Docker部署大数据集群

2019-05-29 13:14:49 来源: 晴天小雨 0

摘要：本文主要是介绍使用Docker部署大数据集群的方法。

安装Docker和配置加速服务

详见：CentOS 7安装 Docker

下载CentOS镜像

docker pull centos

生成带有SSH功能的CentOS镜像

创建Dockerfile文件

vim Dockerfile

添加以下内容：

#基于CentOS7镜像库创建
FROM centos
MAINTAINER rainy

#安装ssh
RUN yum install -y openssh-server sudo
RUN sed -i 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config
RUN yum  install -y openssh-clients

#安装vim和which
RUN yum install -y vim sudo
RUN yum install -y which sudo

#配置root名
RUN echo "root:root123456" | chpasswd
RUN echo "root   ALL=(ALL)       ALL" >> /etc/sudoers

#添加hadoop账号并加入管理员
RUN useradd hadoop
RUN echo "hadoop:hadoop123456" | chpasswd
RUN echo "hadoop   ALL=(ALL)       ALL" >> /etc/sudoers

#生成ssh key
RUN ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key
RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key

#配置sshd服务
RUN mkdir /var/run/sshd
EXPOSE 22
CMD ["/usr/sbin/sshd", "-D"]

生成镜像(注意尾部的.号)

docker build -t="centos7-ssh" .

生成集群镜像文件

所有相关压缩包与Dockerfile需放在同一个文件夹

创建Dockerfile文件

vim Dockerfile

添加以下内容：

#基于centos7-ssh构建
FROM centos7-ssh

#创建目录
RUN mkdir /home/hadoop/hadoop_spark/

#安装java
ADD jdk-7u80-linux-x64.tar.gz /home/hadoop/hadoop_spark/
RUN mv /home/hadoop/hadoop_spark/jdk1.7.0_80 /home/hadoop/hadoop_spark/jdk1.7.0

#配置java环境变量
ENV JAVA_HOME /home/hadoop/hadoop_spark/jdk1.7.0
ENV PATH ${JAVA_HOME}/bin:$PATH

#安装zookeeper
ADD zookeeper-3.4.5-cdh5.9.3.tar.gz /home/hadoop/hadoop_spark/
RUN mv /home/hadoop/hadoop_spark/zookeeper-3.4.5-cdh5.9.3 /home/hadoop/hadoop_spark/zookeeper3.4.5

#配置zookeeper
ENV ZOOKEEPER_HOME  /home/hadoop/hadoop_spark/zookeeper3.4.5
ENV PATH=$PATH:$ZOOKEEPER_HOME/bin

#安装hadoop
ADD hadoop-2.6.0-cdh5.9.3.tar.gz /home/hadoop/hadoop_spark/
RUN mv /home/hadoop/hadoop_spark/hadoop-2.6.0-cdh5.9.3 /home/hadoop/hadoop_spark/hadoop2.6.0

#配置hadoop环境变量
ENV HADOOP_HOME /home/hadoop/hadoop_spark/hadoop2.6.0
ENV PATH $HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

#安装scala  Spark2.0对于Scala的版本要求是2.11.x
ADD scala-2.12.8.tgz /home/hadoop/hadoop_spark/
RUN mv /home/hadoop/hadoop_spark/scala-2.12.8 /home/hadoop/hadoop_spark/scala2.12.8

#配置scala环境变量
ENV SCALA_HOME /home/hadoop/hadoop_spark/scala2.12.8
ENV PATH $SCALA_HOME/bin:$PATH

#安装spark
ADD spark-1.6.0-cdh5.9.3.tar.gz /home/hadoop/hadoop_spark/
RUN mv /home/hadoop/hadoop_spark/spark-1.6.0-cdh5.9.3 /home/hadoop/hadoop_spark/spark1.6.0

#配置spark环境变量
ENV SPARK_HOME /home/hadoop/hadoop_spark/spark1.6.0
ENV PATH $PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin

#更改目录所有者
RUN chown -R hadoop:hadoop /home/hadoop/hadoop_spark/

#安装R语言
RUN rpm -ivh http://mirrors.yun-idc.com/epel//epel-release-latest-7.noarch.rpm
RUN yum -y install R

生成镜像(注意尾部的.号)

docker build -t="spark" .

配置集群节点(3个)

以下是shell脚本文件，也可以手动一条条执行

#!/bin/bash
# 镜像名称
read -p "请输入镜像名称：" image_name
#sudo docker build -t=$image_name .

#容器映设
sudo docker run   -d -P -p 50070:50070 -p 8088:8088 -p 8900:8900 --name master -h master --add-host slave01:172.17.0.3 --add-host slave02:172.17.0.4 $image_name

#创建slave01容器,在容器host文件，添加hostname，并配置其他节点主机名称和IP地址
sudo docker run   -d -P --name slave01 -h slave01 --add-host master:172.17.0.2 --add-host slave02:172.17.0.4  $image_name

#创建slave02容器
sudo docker run   -d -P --name slave02 -h slave02 --add-host master:172.17.0.2 --add-host slave01:172.17.0.3  $image_name

开启容器并配置ssh免密登陆

#master生成秘钥并拷贝
sudo docker exec -it master /bin/bash
ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa
ssh-copy-id -i /home/hadoop/.ssh/id_rsa -p 22 hadoop@slave01
ssh-copy-id -i /home/hadoop/.ssh/id_rsa -p 22 hadoop@slave02

#slave01生成秘钥并拷贝
sudo docker exec -it slave01 /bin/bash
ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa
ssh-copy-id -i /home/hadoop/.ssh/id_rsa -p 22 hadoop@master
ssh-copy-id -i /home/hadoop/.ssh/id_rsa -p 22 hadoop@slave02


#slave02生成秘钥并拷贝
sudo docker exec -it slave02 /bin/bash
ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa
ssh-copy-id -i /home/hadoop/.ssh/id_rsa -p 22 hadoop@slave01
ssh-copy-id -i /home/hadoop/.ssh/id_rsa -p 22 hadoop@master

配置Zookeeper

修改Zookeeper配置文件

cp $ZOOKEEPER_HOME/conf/zoo_sample.cfg $ZOOKEEPER_HOME/conf/zoo.cfg

#数据存储目录
dataDir=/home/hadoop/zookeeper/tmp
#添加Zkserver配置信息：
server.1=master:2888:3888
server.2=slave01:2888:3888
server.3=slave02:2888:3888

修改~/zookeeper/tmp/myid文件中保存的数字代表本机的Zkserver编号

master：echo 1 > ~/zookeeper/tmp/myid

slave01：echo 2 > ~/zookeeper/tmp/myid

slave03：echo 3 > ~/zookeeper/tmp/myid

配置Hadoop

修改hadoop-env.sh

export JAVA_HOME=/home/hadoop/hadoop_spark/jdk1.7.0

修改slaves

slave01
slave02

修改core-site.xm

<configuration></configuration>之间添加内容

io.file.buffer.size:文件IO缓存大小，默认4k，可以设置为64k(65536byte)、128k(131072byte)

fs.defaultFS:默认的文件系统

hadoop.tmp.dir:临时文件目录

ha.zookeeper.quorum:Zkserver信息

<property>
<name>hadoop.tmp.dir</name>
<value>file:/home/hadoop/hadoop_spark/hadoop2.6.0/hdfs/tmp</value>
<description>A base for other temporary directories.</description>
</property>
<property>
<name>io.file.buffer.size</name>
<value>131072</value>
</property>
<property>
<name>fs.defaultFS</name>
<value>hdfs://master:9000</value>
</property>
<property>
<name>ha.zookeeper.quorum</name> 
<value>master:2181,slave01:2181,slave02:2181</value>
</property>

修改hdfs-site.xml

<configuration></configuration>之间添加内容

dfs.ha.namenodes.<nameservie> :配置名称服务下有哪些NameNode

dfs.namenode.rpc-address.<nameservice>.<namenode> :配置NameNode远程调用地址

dfs.namenode.http-address.<nameservice>.<namenode>: 配置NameNode浏览器访问地址

dfs.namenode.shared.edits.dir :配置名称服务对应的JournalNode

dfs.journalnode.edits.dir JournalNode:存储数据的路径

# 基础配置
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/home/hadoop/hadoop_spark/hadoop2.6.0/hdfs/name</value>
<final>true</final>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/home/hadoop/hadoop_spark/hadoop2.6.0/hdfs/data</value>
<final>true</final>
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>master:9001</value>
</property>
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.permissions</name>
<value>false</value>
</property
# zookeeper相关部分
<property>
<name>dfs.nameservices</name>
<value>ns1</value>
</property>
<property>
<name>dfs.ha.namenodes.ns1</name>
<value>nn1,nn2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.ns1.nn1</name>
<value>master:9002</value>
</property>
<property>
<name>dfs.namenode.http-address.ns1.nn1</name>
<value>master:50070</value>
</property>
<property>
<name>dfs.namenode.rpc-address.ns1.nn2</name>
<value>slave01:9000</value>
</property>
<property>
<name>dfs.namenode.http-address.ns1.nn2</name>
<value>slave01:50070</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name> 
<value>qjournal://master:8485;slave:8485;slave8485/ns1</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/home/hadoop/journal</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.ns1</name>
<value>
org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>
sshfence
shell(/bin/true)
</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/home/hadoop/.ssh/id_rsa</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.connect-timeout</name>
<value>30000</value>
</property>

修改mapred-site.xml

拷贝并重命名：cp mapred-site.xml.template mapred-site.xml

<configuration></configuration>之间添加内容

<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>

修改yarn-site.xml

<configuration></configuration>之间添加内容

<property>
<name>yarn.resourcemanager.address</name>
<value>master:18040</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>master:18030</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>master:8088</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>master:18025</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>master:18141</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>

分发hadoop目录给各个节点

$HADOOP_HOME为hadoop包所在的目录，前边已经配置，可直接使用

scp -r $HADOOP_HOME slave01:/home/hadoop/hadoop_spark/

scp -r $HADOOP_HOME slave02:/home/hadoop/hadoop_spark/

启动Zookeepe

# 启动Zookeeper集群
master>runRemoteCmd.sh “zkServer.sh start” zookeeper
# 分别在三台机器上启动
master>zkServer.sh start
slave01>zkServer.sh start
slave02>zkServer.sh start

# 初始化zkfc
hdfs zkfc -formatZK

启动Hadoop

#初始化namenode
hdfs namenode -format

#启动dfs 
start-dfs.sh      

# 在 master上执行jps，查看是否存在namenode
jps

# 在 slave上分别执行jps，查看是否存在datanode 
jps

#开启yarn
start-yarn.sh

浏览器输入http://本机ip地址:50070 ，可以浏览hadoop node管理界面

浏览器输入http://本机ip地址:8088/cluster 可以浏览节点

配置spark集群

修改spark-env.sh

cp spark-env.sh.template spark-env.sh

添加以下内容

#Scala家目录
export SCALA_HOME=/home/hadoop/hadoop_spark/scala2.12.8

#Java家目录
export JAVA_HOME=/home/hadoop/hadoop_spark/jdk1.7.0

#Hadoop家目录及配置文件目录
export HADOOP_HOME=/home/hadoop/hadoop_spark/hadoop2.6.0
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop

#Spark主节点IP及端口
export SPARK_MASTER_IP=master
export SPARK_MASTER_PORT=7077

#Spark家目录
SPARK_LOCAL_DIRS=/home/hadoop/hadoop_spark/spark1.6.0

#Spark master主节点内存大小设置，默认512Mb，使用于yarn部署模式
export SPARK_DRIVER_MEMORY=1G

#只支持standalone部署模式，设置每个worker内存大小、核数及产生实例个数
export SPARK_WORKER_MEMORY=128m
export SPARK_WORKER_CORES=1 
export SPARK_WORKER_INSTANCES=1

#执行器内存大小设置，默认1G，1个Worker可以执行多个执行器，使用于yarn部署模式
#export SPARK_EXECUTOR_MEMORY=1G

修改slaves

slave01
slave02

启动Spark集群

start-all.sh

浏览Spark的Web管理页面： http://本机ip地址:8900

集群启动和停止顺序

启动顺序

# 启动Zookeeper集群
master>runRemoteCmd.sh “zkServer.sh start” zookeeper
# 分别在三台机器上启动
master>zkServer.sh start
slave1>zkServer.sh start
slave2>zkServer.sh start

# 启动hadoop集群
master>start-dfs.sh
# 启动yarn集群
master>start-yarn.sh
# slave机器上单独启动resourcemanager服务
ssh slave1
yarn-daemon.sh start resourcemanager
ssh slave2
yarn-daemon.sh start resourcemanager
ssh master

#启动hbase集群
master>start-hbase.sh
#或者分别启动hbase集群
#runRemoteCmd.sh “hbase-daemon.sh start regionserver” all
#runRemoteCmd.sh “hbase-daemon.sh start master” master

#启动spark集群
#启动master节点：
sbin/start-master.sh
#启动worker节点：
sbin/start-slaves.sh
#或者：sbin/start-all.sh

停止顺序

#停止spark集群
master>spark/sbin/stop-slaves.sh
master>spark/sbin/stop-master.sh

#停止hbase集群
master>stop-hbase.sh

#停止yarn集群
master>stop-yarn.sh

#停止hadoop集群
master>stop-dfs.sh

#停止zookeeper集群
master>runRemoteCmd.sh “zkServer.sh stop” zookeeper