Centos7上Hadoop 3.3.1的分布式集群安装过程

1. 集群规划

• 每台服务器相互设置ssh无密码登录，注意authorized_keys权限为600。可以用ssh-copy-id hostname将本机的id_rsa.pub内容复制到远程服务器的.ssh/authorized_key中

服务名安装服务器java8bigdata001/2/3NameNodebigdata001DataNodebigdata002/3ResourceManagerbigdata001NodeManagerbigdata002/3 2. 下载(在bigdata001操作)

执行下面的命令进行下载和解压

curl -O https://ftp.nluug.nl/internet/apache/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz

tar -zxvf hadoop-3.3.1.tar.gz

进入hadoop目录

[root@bigdata001 opt]# 
[root@bigdata001 opt]# cd hadoop-3.3.1
[root@bigdata001 hadoop-3.3.1]# 
[root@bigdata001 hadoop-3.3.1]# pwd
/opt/hadoop-3.3.1
[root@bigdata001 hadoop-3.3.1]# 

3. 配置文件修改(在bigdata001操作) 3.1 hadoop-env.sh

创建pids和logs文件

[root@bigdata001 hadoop-3.3.1]# 
[root@bigdata001 hadoop-3.3.1]# mkdir pids				
[root@bigdata001 hadoop-3.3.1]# 
[root@bigdata001 hadoop-3.3.1]# mkdir logs				
[root@bigdata001 hadoop-3.3.1]# 
[root@bigdata001 hadoop-3.3.1]# ls
bin  etc  include  lib  libexec  LICENSE-binary  licenses-binary  LICENSE.txt  logs  NOTICE-binary  NOTICE.txt  pids  README.txt  sbin  share
[root@bigdata001 hadoop-3.3.1]#
[root@bigdata001 hadoop-3.3.1]# pwd
/opt/hadoop-3.3.1
[root@bigdata001 hadoop-3.3.1]#

修改etc/hadoop/hadoop-env.sh文件

修改部分：

export JAVA_HOME=/opt/jdk1.8.0_201       
export HADOOP_PID_DIR=/opt/hadoop-3.3.1/pids				
export HADOOP_LOG_DIR=/opt/hadoop-3.3.1/logs				

export HDFS_NAMENODE_USER=root

添加部分

export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root

3.2 core-site.xml

修改etc/hadoop/core-site.xml

添加部分:

    
        fs.defaultFS		
        hdfs://bigdata001:9000
    
    
    
        io.file.buffer.size				
        131072
    

    
        hadoop.proxyuser.root.hosts
        *
        Hadoop的超级用户root能代理的节点
    

    
        hadoop.proxyuser.root.groups
        *
        Hadoop的超级用户root能代理的用户组
    

3.3 hdfs-site.xml

修改etc/hadoop/hdfs-site.xml

添加namenode和datanode文件夹

[root@bigdata001 hadoop-3.3.1]# 
[root@bigdata001 hadoop-3.3.1]# mkdir namenode
[root@bigdata001 hadoop-3.3.1]# 
[root@bigdata001 hadoop-3.3.1]# mkdir datanode
[root@bigdata001 hadoop-3.3.1]# 
[root@bigdata001 hadoop-3.3.1]# pwd
/opt/hadoop-3.3.1
[root@bigdata001 hadoop-3.3.1]# 

添加部分:

    
        dfs.replication			
        1
    

    
        dfs.namenode.name.dir				
        /opt/hadoop-3.3.1/namenode
    
	
    
        dfs.blocksize					
        268435456
    

    
        dfs.namenode.handler.count			
        100
    
    
    
        dfs.datanode.data.dir					
        /opt/hadoop-3.3.1/datanode
    

3.4 mapred-site.xml

修改etc/hadoop/mapred-site.xml

添加部分：

    
        mapreduce.framework.name		
        yarn
    

3.5 yarn-site.xml

修改etc/hadoop/yarn-site.xml

[root@bigdata001 hadoop-3.3.1]# 
[root@bigdata001 hadoop-3.3.1]# pwd
/opt/hadoop-3.3.1
[root@bigdata001 hadoop-3.3.1]# 
[root@bigdata001 hadoop-3.3.1]# mkdir nm-local-dir
[root@bigdata001 hadoop-3.3.1]# mkdir nm-log-dir
[root@bigdata001 hadoop-3.3.1]# mkdir nm-remote-app-log-dir
[root@bigdata001 hadoop-3.3.1]# 

添加部分：

    
        yarn.acl.enable				
        false
    

    
        yarn.log-aggregation-enable			
        false
    

    
        yarn.resourcemanager.address					
        ${yarn.resourcemanager.hostname}:8032
    
	
	
    
        yarn.resourcemanager.scheduler.address			
        ${yarn.resourcemanager.hostname}:8030
    
	
    
        yarn.resourcemanager.resource-tracker.address			
        ${yarn.resourcemanager.hostname}:8031
    
	
    
        yarn.resourcemanager.admin.address			
        ${yarn.resourcemanager.hostname}:8033
    

    
        yarn.resourcemanager.webapp.address		
        ${yarn.resourcemanager.hostname}:8088
    
	
    
        yarn.resourcemanager.hostname   			
        bigdata001
    
	
    
        yarn.resourcemanager.scheduler.class			
        org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
    
	
    
        yarn.scheduler.minimum-allocation-mb			
        1024
    
	
    
        yarn.scheduler.maximum-allocation-mb			
        8192
    
	
    
        yarn.resourcemanager.nodes.include-path			
        
    
	
    
        yarn.nodemanager.resource.memory-mb			
        8192
    

    
        yarn.nodemanager.vmem-pmem-ratio				
        2.1
    
	
    
        yarn.nodemanager.local-dirs				
        /opt/hadoop-3.3.1/nm-local-dir
    
	
    
        yarn.nodemanager.log-dirs				
        /opt/hadoop-3.3.1/nm-log-dir
    
	
    
        yarn.nodemanager.log.retain-seconds				
        10800
    
	
    
        yarn.nodemanager.remote-app-log-dir			
        /opt/hadoop-3.3.1/nm-remote-app-log-dir
    
	
    
        yarn.nodemanager.remote-app-log-dir-suffix			
        logs
    

    
        yarn.nodemanager.aux-services			
        mapreduce_shuffle
    

    
        yarn.nodemanager.env-whitelist				
        JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME
    

3.6 修改workers文件

bigdata002			
bigdata003			

4. hadoop目录分发（在bigdata001操作)

将bigdata001上配置的hadoop目录分发到其余两台服务器

[root@bigdata001 opt]# scp -r /opt/hadoop-3.3.1 root@bigdata002:/opt
[root@bigdata001 opt]# scp -r /opt/hadoop-3.3.1 root@bigdata003:/opt

5. 初始化和启动（在bigdata001操作) 5.1 添加环境变量

1. /etc/profile添加内容如下：

export HADOOP_HOME=/opt/hadoop-3.3.1

export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

2. 使环境变量生效

[root@bigdata001 ~]# 
[root@bigdata001 ~]# source /etc/profile
[root@bigdata001 ~]#

5.2 HDFS

hdfs初始化

[root@bigdata001 opt]#
[root@bigdata001 opt]# hdfs namenode -format		
[root@bigdata001 opt]#

启动hdfs

[root@bigdata001 opt]#
[root@bigdata001 opt]# start-dfs.sh										
[root@bigdata001 opt]#

8. 访问http://bigdata001:9870

5.3 YARN

启动yarn

[root@bigdata001 opt]# 
[root@bigdata001 opt]# start-yarn.sh													
[root@bigdata001 opt]#

访问http://bigdata001:8088