HDFS开发环境搭建

准备工作（必须）

1. 启动CentOS端的Hadoop系统
2. windows安装Hadoop环境，请参考win10安装hadoop3.1.2

第一步：创建Maven形式的Java项目 第二步：添加Maven依赖

  
        junit
        junit
        4.12
    
    
        org.apache.hadoop
        hadoop-client
        3.1.2
    
    
        org.apache.hadoop
        hadoop-common
        3.1.2
    
    
        org.apache.hadoop
        hadoop-hdfs
        3.1.2
    

    
        org.apache.hadoop
        hadoop-yarn-common
        3.1.2
    
    
        org.apache.hadoop
        hadoop-yarn-client
        3.1.2
    
    
        org.apache.hadoop
        hadoop-yarn-server-resourcemanager
        3.1.2
    

    
        org.apache.hadoop
        hadoop-mapreduce-client-core
        3.1.2
    
    
        org.apache.hadoop
        hadoop-mapreduce-client-jobclient
        3.1.2
    
    
        org.apache.hadoop
        hadoop-mapreduce-client-common
        3.1.2
    

    
        org.anarres.lzo
        lzo-hadoop
        1.0.6
    

第三步：搭建项目框架并测试 1.将要访问的Hadoop的配置信息写在Java代码中 实现一

public class HDFSAppFrame {
    private final String HDFS_PATH = "hdfs://hcmaster:9000";

    private FileSystem fs = null;
    private Configuration cfg = null;

    @Before
    public void setUp() throws Exception {
        //src目录下不需要core-site.xml文件
        cfg = new Configuration();
        //返回指定服务器（HDFS）的文件系统
        fs = FileSystem.get(new URI(HDFS_PATH), cfg,"hc");//此时不再需要在windows中配置环境变量HADOOP_USER_NAME了
        System.out.println("HDFS APP 开启");
    }

    @Test
    public void frameTest() throws IOException {
        System.out.println(cfg);
        System.out.println(fs);
    }

    @After
    public void tearDown() throws Exception {
        fs.close();
        System.out.println("HDFS APP 关闭");
    }
}

实现二

public class HDFSAppFrame {
    private final String HDFS_PATH = "hdfs://hcmaster:9000";
    private FileSystem fs = null;
    private Configuration cfg = null;

    @Before
    public void setUp() throws Exception {
        //src目录下不需要core-site.xml文件
        cfg = new Configuration();
        fs = FileSystem.get(new URI(HDFS_PATH), cfg);//返回指定服务器（HDFS）的文件系统
        System.out.println("HDFS APP 开启");
    }

    @Test
    public void frameTest(){
        System.out.println(cfg);
        System.out.println(fs);
    }
    
    @After
    public void tearDown() throws Exception {
        fs.close();
        System.out.println("HDFS APP 关闭");
    }
}

实现三

public class HDFSAppFrame {
    private FileSystem fs = null;
    private Configuration cfg = null;

    @Before
    public void setUp() throws Exception {
        cfg = new Configuration();//src目录下不需要core-site.xml文件
        cfg.set("fs.defaultFS","hdfs://hcmaster:9000");
        fs = FileSystem.get(cfg);//返回指定服务器（HDFS）的文件系统
        System.out.println("HDFS APP 开启");
    }

    @Test
    public void frameTest(){
        System.out.println(cfg);
        System.out.println(fs);
    }

    @After
    public void tearDown() throws Exception {
        fs.close();
        System.out.println("HDFS APP 关闭");
    }
}

2.将要访问的Hadoop的配置信息写在xml文件中

• 在项目类根路径下，创建core-site.xml：

  
  
  	
  		fs.defaultFS
  		hdfs://hcmaster:9000
  	
  	
  		hadoop.tmp.dir
  		/usr/local/hadoop3.1.2/hadoop-standalone/data/tmp/
  	
  
  

• 测试代码：

  public class HDFSAppFrame {
      private FileSystem fs = null;
      private Configuration cfg = null;
      @Before
      public void setUp() throws Exception {
          cfg = new Configuration();
          fs = FileSystem.get(cfg); //返回指定服务器的文件系统，使用resources/core-site.xml文件
          System.out.println("HDFS APP 开启");
      }
  
      @Test
      public  void frameTest() throws IOException {
          System.out.println(cfg);
          System.out.println(fs);
      }
  
      @After
      public void tearDown() throws Exception {
          fs.close();
          System.out.println("HDFS APP 关闭");
      }
  }