Java访问kerberos认证的HDFS文件

81 阅读 0 评论 54 点赞

我是靠谱客的博主典雅大地，这篇文章主要介绍Java访问kerberos认证的HDFS文件，现在分享给大家，希望可以做个参考。

Kerberos是一种计算机网络授权协议，用来在非安全网络中，对个人通信以安全的手段进行身份认证。

具体HADOOP的访问HDFS使用Kerberos的作用和原理请自己查阅相关文档。

之前做项目时第一次使用Kbs访问HDFS,当时不了解，翻阅资料搞了好久，也入了不少坑，现分享出来，方便大家。

下面代码在项目亲测过，可用

代码如下：

复制代码

package zqmKerberos;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.security.UserGroupInformation;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.UUID;
import org.apache.hadoop.fs.Path;
public class BjKerberos {
public final String USER_KEY = "usergrp@zqm"; //用户key
public final String KEY_TAB_PATH = "/home/usergrp.user-app_yxkj.keytab"; //keytab文件
public final String HDFS_PATH = "/user/finalRes"; //要访问的HDFS路径
public HashMap<String,String> map = new HashMap<String,String>();
public HashMap<String,String> Kerberos() throws IOException {
SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd");
String currentDay = df.format(new Date());
//
HDFS的Kerberos认证
System.setProperty("java.security.krb5.conf", "/home/krb5.conf");
Configuration conf = new Configuration();
//
必须加，不然会报找不到文件系统
conf.addResource(new Path("/home/hdfs-site.xml"));
conf.addResource(new Path("/home/core-site.xml"));
//
设置conf信息
conf.setBoolean("hadoop.security.authorization", true);
conf.set("hadoop.security.authentication", "kerberos");
try {
UserGroupInformation.setConfiguration(conf);
UserGroupInformation.loginUserFromKeytab(USER_KEY, KEY_TAB_PATH);
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("Kerberos Checked Finsh!
Get HDFS Data ! n");
//
下面是自己的业务逻辑，不用看
FileSystem fs = FileSystem.get(conf);
FileStatus dir[] = fs.listStatus(new Path(HDFS_PATH));
for (int i = 0; i < dir.length; i++) {
FileStatus dir_two[] = fs.listStatus(dir[i].getPath());
for (int j = 0; j < dir_two.length; j++) {
if(dir_two[j].getPath().toString().contains(currentDay)){
FileStatus files[] = fs.listStatus(dir_two[j].getPath());
for(int n = 0; n < files.length; n++){
//
结果数据文件
System.out.println(files[n].getPath());
InputStream in = fs.open(files[n].getPath());
BufferedReader br = new BufferedReader(new InputStreamReader(in, "utf-8"));
String line = null;
while ((line = br.readLine()) != null) {
String str[] = line.split("t");
//
因为一个标识会被多人呼，所以加UUID使其唯一
map.put(str[0].trim() + "_" + UUID.randomUUID(), str[1].trim());
}
}
}
}
}
System.out.printf("HDFS Data Total number:%d (tiao)n", map.size());
for (String key : map.keySet()) {
System.out.printf("Show first sample data: " + key + "t" + map.get(key) + "n");
break;
}
return map;
}
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
package zqmKerberos;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.security.UserGroupInformation;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.UUID;
import org.apache.hadoop.fs.Path;
public class BjKerberos {
public final String USER_KEY = "usergrp@zqm"; //用户key
public final String KEY_TAB_PATH = "/home/usergrp.user-app_yxkj.keytab"; //keytab文件
public final String HDFS_PATH = "/user/finalRes"; //要访问的HDFS路径
public HashMap<String,String> map = new HashMap<String,String>();
public HashMap<String,String> Kerberos() throws IOException {
SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd");
String currentDay = df.format(new Date());
//
HDFS的Kerberos认证
System.setProperty("java.security.krb5.conf", "/home/krb5.conf");
Configuration conf = new Configuration();
//
必须加，不然会报找不到文件系统
conf.addResource(new Path("/home/hdfs-site.xml"));
conf.addResource(new Path("/home/core-site.xml"));
//
设置conf信息
conf.setBoolean("hadoop.security.authorization", true);
conf.set("hadoop.security.authentication", "kerberos");
try {
UserGroupInformation.setConfiguration(conf);
UserGroupInformation.loginUserFromKeytab(USER_KEY, KEY_TAB_PATH);
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("Kerberos Checked Finsh!
Get HDFS Data ! n");
//
下面是自己的业务逻辑，不用看
FileSystem fs = FileSystem.get(conf);
FileStatus dir[] = fs.listStatus(new Path(HDFS_PATH));
for (int i = 0; i < dir.length; i++) {
FileStatus dir_two[] = fs.listStatus(dir[i].getPath());
for (int j = 0; j < dir_two.length; j++) {
if(dir_two[j].getPath().toString().contains(currentDay)){
FileStatus files[] = fs.listStatus(dir_two[j].getPath());
for(int n = 0; n < files.length; n++){
//
结果数据文件
System.out.println(files[n].getPath());
InputStream in = fs.open(files[n].getPath());
BufferedReader br = new BufferedReader(new InputStreamReader(in, "utf-8"));
String line = null;
while ((line = br.readLine()) != null) {
String str[] = line.split("t");
//
因为一个标识会被多人呼，所以加UUID使其唯一
map.put(str[0].trim() + "_" + UUID.randomUUID(), str[1].trim());
}
}
}
}
}
System.out.printf("HDFS Data Total number:%d (tiao)n", map.size());
for (String key : map.keySet()) {
System.out.printf("Show first sample data: " + key + "t" + map.get(key) + "n");
break;
}
return map;
}
}