三、Hadoop 的 API

1.环境搭建

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-hdfs</artifactId>
    <version>2.6.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-common</artifactId>
    <version>2.6.0</version>
</dependency>

2.问题解决

Windows开发Hadoop应用环境配置

• 解压hadoop安装包到C:/
• 将winutils.exe和hadoop.dll拷贝到hadoop的bin目录下
• 在windows配置HADOOP_HOME环境变量
• 重启开发工具idea,否则开发工具无法识别HADOOP_HOME
• 在Windows主机配置CentOS的主机名和IP的映射关系

C:\Windows\System32\drivers\etc\hosts

192.168.169.139 CentOS

HDFS权限不足导致写失败?

org.apache.hadoop.security.AccessControlException: Permission denied: user=HIAPAD, access=WRITE, inode="/":root:supergroup:drwxr-xr-x
	at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkFsPermission(FSPermissionChecker.java:271)
	at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:257)
...

解决方案

方案1

etc/hadoop/hdfs-site.xml

<property>
        <name>dfs.permissions.enabled</name>
        <value>false</value>
</property>

关闭HDFS文件权限检查,修改完成后,重启HDFS服务

方案2

-DHADOOP_USER_NAME=root

设置JAVA虚拟机启动参数java XXX -Dxx=xxx

3.HDFS

package com.baizhi.hdfs;/**

- @Author:luoht
- @Description：
- @Date：Create in 16:21 2019/1/3
  */

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.*;
import java.io.*;

/**

- @program: hadoop_01
- @description:
- @author: luoht
- @create: 2019-01-03 16:21
  **/

public class TestHDFS {
    private FileSystem fileSystem;
    private Configuration conf;

    @Before
    public void before() throws IOException {
        conf = new Configuration();
        conf.addResource("core-site.xml");
        conf.addResource("hdfs-site.xml");
        fileSystem=FileSystem.newInstance(conf);
    }

    @Test
    public void testConfig(){
        String value = conf.get("dfs.replication");
        System.out.println(value);
    }

    @Test
    public void testUpload01() throws IOException {
        String file ="C:\\Users\\Administrator\\Desktop\\郁金香開了麽？.txt";
        Path dst = new Path("/demo/access/springBoot.pdf");
        FileInputStream inputStream = new FileInputStream(file);
        FSDataOutputStream outputStream = fileSystem.create(dst, new Progressable() {
            @Override
            public void progress() {
                System.out.println("..");
            }
        });
        IOUtils.copyBytes(inputStream,outputStream,1024,true);
    }

    @Test
    public void testUpload02() throws IOException {
        Path src = new Path("C:\\Users\\Administrator\\Desktop\\郁金香開了麽？.txt");
        Path dst = new Path("/郁金香开了么？.txt");
        fileSystem.copyFromLocalFile(src,dst);
    }

    @Test
    public void testDownload01() throws IOException {
        String file="C:\\Users\\Administrator\\Desktop\\郁金香不开了.txt";
        Path dst = new Path("/郁金香开了么？.txt");
        FileOutputStream outputStream = new FileOutputStream(file);
        InputStream inputStream = fileSystem.open(dst);
        IOUtils.copyBytes(inputStream,outputStream,1024,true);
    }

    @Test
    public void testDownload02() throws IOException {
        Path dst = new Path("C:\\Users\\Administrator\\Desktop\\郁金香不开了.txt");
        Path src = new Path("/demo/access/springBoot.pdf");
        fileSystem.copyToLocalFile(false,src,dst,true);
    }

    @Test
    public void testDelete() throws IOException {
        Path src = new Path("/demo");
        /*true 代表递归删除子文件*/
        fileSystem.delete(src,true);
    }

    @Test
    public void testExits() throws IOException {
        Path src = new Path("/郁金香开了么？.txt");
        boolean exists = fileSystem.exists(src);
        //Junit之 断言 -静态导包
        assertTrue(exists);
    }

    @Test
    public void testMkdir() throws IOException {
        Path src = new Path("/demo.assess");
        boolean exists = fileSystem.exists(src);
        if (!exists){
            fileSystem.mkdirs(src);
        }
    }

    @Test
    public void testListFiles() throws IOException {
        Path src = new Path("/tt");
        RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(src, true);
        while (files.hasNext()){
            LocatedFileStatus fileStatus = files.next();
            System.out.println(fileStatus.getPath()+"路径  "+fileStatus.isFile()+"是否文件  "+fileStatus.getLen()+"文件大小");
            System.out.println("------------------------------");
            BlockLocation[] locations = fileStatus.getBlockLocations();
            for (BlockLocation location : locations) {
                System.out.println("offset"+location.getOffset()+"length"+location.getLength());
            }
            System.out.println("================================");
        }
    }

    @Test
    public void testDeleteWithTrash() throws IOException {
        Trash trash = new Trash(fileSystem, conf);
        Path dst = new Path("/123.txt");
        trash.moveToTrash(dst);
    }

    @After
    public void after() throws IOException{
        fileSystem.close();
    }

}