HDFS文件系统操作

HDFS文件系统操作

1、文件的新增

2、文件删除

3、文件修改

4、文件查询

package com.skymesh.hadoop.hdfs;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.permission.FsPermission;

import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Iterator;
import java.util.Map;

/**
 * 1、按照hadoop-2.6.4搭建手册安装好hadoop集群
 * window10 安装hadoop：https://blog.csdn.net/songhaifengshuaige/article/details/79575308
 * window安装hadoop,hadoop找不到JAVA_HOME,java不能放在有空格的路径中，eg C:Program Files  C:Program Files (x86),重新复制java安装到到没有空格的目录下
 * window10 hadoop工具：https://github.com/steveloughran/winutils
 * window10 用户名称和hadoop集群用户权限问题在hadoop集群配置
 */
public class HdfsClient {

    static FileSystem fileSystem=null;

    public static void main(String[] args) throws IOException, URISyntaxException, InterruptedException {
        HdfsClient hdfsClient = new HdfsClient();
        hdfsClient.init2();
        hdfsClient.uploadFileToHadoop();
        //hdfsClient.deletFile();
        hdfsClient.queryFile();
        hdfsClient.downLoadFile();
    }

    public void init() throws IOException {
        Configuration conf = new Configuration();
        conf.set("fs.default.name", "hdfs://192.168.136.128:9000");
        fileSystem = FileSystem.get(conf);
        System.out.println("连接"+"hdfs://192.168.136.128:9000"+"成功");
    }

    public void init2() throws URISyntaxException, IOException, InterruptedException {
        Configuration conf = new Configuration();
        conf.set("fs.replication","3");
        //设置用户
        fileSystem = FileSystem.get(new URI("hdfs://192.168.136.128:9000"),conf,"root");
    }

    /**
     * 上传本地文件到Hadoop系统
     * 如果系统中存在再次上传同样的文件名称会覆盖原来文件
     * @throws IOException
     */
    public void uploadFileToHadoop() throws IOException {
        Path from = new Path("C:\Users\admin\Desktop\word2.txt");
        Path to = new Path("/wordcount/input/word2.txt");
        fileSystem.copyFromLocalFile(from,to);
        System.out.println("上传本地文件到hadoop成功");
    }

    /**
     * 删除文件
     * @throws IOException
     */
    public void deletFile() throws IOException {
        Path filePath = new Path("/readme22.txt");
        fileSystem.delete(filePath,true);
        System.out.println("删除文件："+"/readme22.txt");
    }

    /**
     * 查询文件系统文件
     * @throws IOException
     */
    public void queryFile() throws IOException {
        Path filePath = new Path("/file/我的文件.txt");
        Path homeDirectory = fileSystem.getHomeDirectory();
        System.out.println(homeDirectory.getName());
        fileSystem.createNewFile(filePath);
        System.out.println("创建文件");
        boolean exists = fileSystem.exists(filePath);
        System.out.println("文件/file/我的文件.txt是否存在:"+exists);
        Path rootPath = new Path("/");
        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fileSystem.listFiles(rootPath, true);
        while (locatedFileStatusRemoteIterator.hasNext()){
            LocatedFileStatus next = locatedFileStatusRemoteIterator.next();
            long blockSize = next.getBlockSize();
            String group = next.getGroup();
            long len = next.getLen();
            String owner = next.getOwner();
            short replication = next.getReplication();
            FsPermission permission = next.getPermission();
            boolean isfile = next.isFile();
            Path path = next.getPath();
            System.out.println("是否文件："+isfile+"t 副本数量："+replication+"t 拥有人："
                    +owner+"t 文件大小："+len+"t 文件所属组："+group+"t blockSize:"+blockSize+"t 文件名称："+path.getName());
        }
    }

    public void queryConfigKey(){
        Configuration conf = new Configuration();
        conf.set("fs.default.name", "hdfs://192.168.136.128:9000");
        Iterator<Map.Entry<String, String>> iterator = conf.iterator();
        while (iterator.hasNext()){
            Map.Entry<String, String> entry = iterator.next();
            System.out.println(entry.getKey()+"="+entry.getValue());
        }
    }

    /**
     * 从hadoop上拷贝数据到本地
     * @throws IOException
     */
    public void downLoadFile() throws IOException {
        FSDataInputStream dataInput = fileSystem.open(new Path("/readme.txt"));
        DataOutputStream out = new DataOutputStream(new FileOutputStream(new File("C:\Users\admin\Desktop\readme77.txt")));
        IOUtils.copy(dataInput,out);
        System.out.println("下载文件：");
    }

}

最后

以上就是霸气爆米花为你收集整理的HDFS文件系统操作的全部内容，希望文章能够帮你解决HDFS文件系统操作所遇到的程序开发问题。

如果觉得靠谱客网站的内容还不错，欢迎将靠谱客网站推荐给程序员好友。