生产环境实战spark （9）分布式集群 5台设备 SPARK集群安装
1, 上传spark到master，检查[root@master rhzf_spark_setupTools]# ls
hadoop-2.6.5.tar.gz jdk-8u121-linux-x64.tar.gz scala-2.11.8.zip spark-2.1.0-bin-hadoop2.6.tgz
[root@master rhzf_spark_setupTools]# [root@master rhzf_spark_setupTools]# tar -zxvf spark-2.1.0-bin-hadoop2.6.tgz
[root@master rhzf_spark_setupTools]# ls
hadoop-2.6.5.tar.gz jdk-8u121-linux-x64.tar.gz scala-2.11.8.zip spark-2.1.0-bin-hadoop2.6 spark-2.1.0-bin-hadoop2.6.tgz
[root@master rhzf_spark_setupTools]# mv spark-2.1.0-bin-hadoop2.6 /usr/local
[root@master rhzf_spark_setupTools]# cd /usr/local
[root@master local]# ls
bin games include lib libexec rhzf_spark_setupTools scala-2.11.8 spark-2.1.0-bin-hadoop2.6
etc hadoop-2.6.5 jdk1.8.0_121 lib64 rhzf_setup_scripts sbin share src
[root@master local]# export JAVA_HOME=/usr/local/jdk1.8.0_121
export SCALA_HOME=/usr/local/scala-2.11.8
export HADOOP_HOME=/usr/local/hadoop-2.6.5
export SPARK_HOME=/usr/local/spark-2.1.0-bin-hadoop2.6
export PATH=.:$PATH:$JAVA_HOME/bin:$SCALA_HOME/bin:$HADOOP_HOME/bin:$SPARK_HOME/bin
刷新生效
[root@master spark-2.1.0-bin-hadoop2.6]# source /etc/profile[root@master spark-2.1.0-bin-hadoop2.6]# cd ..
[root@master local]# ls
bin games include lib libexec rhzf_spark_setupTools scala-2.11.8 spark-2.1.0-bin-hadoop2.6
etc hadoop-2.6.5 jdk1.8.0_121 lib64 rhzf_setup_scripts sbin share src
[root@master local]# cd spark-2.1.0-bin-hadoop2.6
[root@master spark-2.1.0-bin-hadoop2.6]# ls
bin conf data examples jars LICENSE licenses NOTICE python R README.md RELEASE sbin yarn
[root@master spark-2.1.0-bin-hadoop2.6]# cd conf
[root@master conf]# ls
docker.properties.template log4j.properties.template slaves.template spark-env.sh.template
fairscheduler.xml.template metrics.properties.template spark-defaults.conf.template
[root@master conf]# mv spark-env.sh.template spark-env.sh
[root@master conf]# ls
docker.properties.template log4j.properties.template slaves.template spark-env.sh
fairscheduler.xml.template metrics.properties.template spark-defaults.conf.template
[root@master conf]# vi spark-env.sh
export JAVA_HOME=/usr/local/jdk1.8.0_121
export SCALA_HOME=/usr/local/scala-2.11.8
export SPARK_MASTER_IP=10. 0.237
export SPARK_WORKER_MEMORY=2g
export HADOOP_CONF_DIR=/usr/local/hadoop-2.6.5/etc/hadoop
"spark-env.sh" 82L, 4180C written
[root@master conf]# [root@master conf]# ls
docker.properties.template log4j.properties.template slaves.template spark-env.sh
fairscheduler.xml.template metrics.properties.template spark-defaults.conf.template
[root@master conf]# mv slaves.template slaves
[root@master conf]# ls
docker.properties.template fairscheduler.xml.template log4j.properties.template metrics.properties.template slaves spark-defaults.conf.template spark-env.sh
[root@master conf]# vi slaves
worker01
worker02
worker03
worker04[root@master rhzf_setup_scripts]# ls
rhzf_hadoop.sh rhzf_hosts_scp.sh rhzf_jdk.sh rhzf_scala.sh rhzf_ssh.sh
[root@master rhzf_setup_scripts]# vi rhzf_spark.sh
#!/bin/sh
for i in 238 239 240 241
do
scp -rq /usr/local/spark-2.1.0-bin-hadoop2.6 root@10 .$i:/usr/local/spark-2.1.0-bin-hadoop2.6
scp -rq /etc/profile root@10 .$i:/etc/profile
ssh root@10. 0.$i source /etc/profile
done
[root@master rhzf_setup_scripts]# ls
rhzf_hadoop.sh rhzf_hosts_scp.sh rhzf_jdk.sh rhzf_scala.sh rhzf_spark.sh rhzf_ssh.sh
[root@master rhzf_setup_scripts]# chmod u+x rhzf_spark.sh
[root@master rhzf_setup_scripts]# ./rhzf_spark.sh
[root@master rhzf_setup_scripts]# [root@master bin]# pwd
/usr/local/spark-2.1.0-bin-hadoop2.6/bin
[root@master bin]# cd ..
[root@master spark-2.1.0-bin-hadoop2.6]# cd sbin
[root@master sbin]# ls
slaves.sh start-all.sh start-mesos-shuffle-service.sh start-thriftserver.sh stop-mesos-dispatcher.sh stop-slaves.sh
spark-config.sh start-history-server.sh start-shuffle-service.sh stop-all.sh stop-mesos-shuffle-service.sh stop-thriftserver.sh
spark-daemon.sh start-master.sh start-slave.sh stop-history-server.sh stop-shuffle-service.sh
spark-daemons.sh start-mesos-dispatcher.sh start-slaves.sh stop-master.sh stop-slave.sh
[root@master sbin]# start-all.sh
starting org.apache.spark.deploy.master.Master, logging to /usr/local/spark-2.1.0-bin-hadoop2.6/logs/spark-root-org.apache.spark.deploy.master.Master-1-master.out
worker03: starting org.apache.spark.deploy.worker.Worker, logging to /usr/local/spark-2.1.0-bin-hadoop2.6/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-worker03.out
worker04: starting org.apache.spark.deploy.worker.Worker, logging to /usr/local/spark-2.1.0-bin-hadoop2.6/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-worker04.out
worker01: starting org.apache.spark.deploy.worker.Worker, logging to /usr/local/spark-2.1.0-bin-hadoop2.6/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-worker01.out
worker02: starting org.apache.spark.deploy.worker.Worker, logging to /usr/local/spark-2.1.0-bin-hadoop2.6/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-worker02.out
