Hadoop之API

71 阅读 0 评论 47 点赞

我是靠谱客的博主激昂爆米花，最近开发中收集的这篇文章主要介绍Hadoop之API，觉得挺不错的，现在分享给大家，希望可以做个参考。

概述

读取文件

@Test
public void testConnentNamenode() throws Exception{
	Configuration cf = new Configuration();
	FileSystem fs = FileSystem.get(new URI("hdfs://192.168.000.000:9000"), cf, "root");
	InputStream in = fs.open(new Path("/test/demo.txt"));
	OutputStream out = new FileOutputStream("demo.txt");
	IOUtils.copyBytes(in, out, cf);
	in.close();
	out.close();
}

上传文件

@Test
public void testPut() throws Exception{
	Configuration cf = new Configuration();
	cf.set("dfs.replication", "2");//表示设置两个副本
	FileSystem fs = FileSystem.get(new URI("hdfs://192.168.000.000:9000"), cf, "root");
	OutputStream out = fs.create(new Path("/test.txt"));
	FileInputStream in = new FileInputStream("D:\test.txt");
	IOUtils.copyBytes(in, out, cf);
	in.close();
	out.close();
}

删除文件

@Test
public void testDelete() throws Exception{
	Configuration cf = new Configuration();
	FileSystem fs = FileSystem.get(new URI("hdfs://192.168.000.000:9000"), cf, "root");
	//true表示无论是否为空目录都删除
	fs.delete(new Path("/test"), true);
	//false表示只能删除不为空的目录
	fs.delete(new Path("/test"), false);
	fs.close();
}

在hdfs上创建文件夹

@Test
public void testMkdir()throws Exception{
	Configuration cf = new Configuration();
	FileSystem fs = FileSystem.get(new URI("hdfs://192.168.000.000:9000"), cf, "root");
	fs.mkdirs(new Path("/test"));
}

查询hdfs指定目录下的文件

@Test
public void testLs()throws Exception{
	Configuration cf = new Configuration();
	FileSystem fs = FileSystem.get(new URI("hdfs://192.168.000.000:9000", cf, "root");
	FileStatus[] ls = fs.listStatus(new Path("/"));
	for(FileStatus status: ls){
		System.out.println(status);
	}	
}

递归查看指定目录下的文件

@Test
public void testLs()throws Exception{
    Configuration cf=new Configuration();
    FileSystem fs=FileSystem.get(new URI("hdfs://192.168.000.000:9000"),cf,"root");
    RemoteIterator<LocatedFileStatus> rt=fs.listFiles(new Path("/"), true);
    while(rt.hasNext()){
        System.out.println(rt.next());
    }
}

重命名

@Test
public void testCreateNewFile() throws Exception{
    Configuration cf=new Configuration();
    FileSystem fs=FileSystem.get(new URI("hdfs://192.168.000.000:9000"),cf,"root");
    fs.rename(new Path("/test"), new Path("/test"));
}

获取文件块的信息

@Test
public void testCopyFromLoaclFileSystem() throws Exception{
    Configuration cf=new Configuration();
    FileSystem fs=FileSystem.get(new URI("hdfs://192.168.000.000:9000"),cf,"root");
    BlockLocation[] data=fs.getFileBlockLocations(new Path("/test/test.txt"),0,Integer.MaxValue);
    for(BlockLocation bl:data){
        System.out.println(bl);
    }
}