spark之从外部文件获取广播变量
public static Map<String,String> loadDataFromFile() {
Map<String,String> map = new ConcurrentHashMap<>();
try {
File file = new File("filename.txt");
if (file.isFile() && file.exists()) {
InputStreamReader read = new InputStreamReader(new FileInputStream(file), "UTF-8");
BufferedReader bufferedReader = new BufferedReader(read);
String line = null;
while ((line = bufferedReader.readLine()) != null) {
String[] split = line.split(",");
if (split.length==2) {
map.put(split[0],split[1]);
}
}
bufferedReader.close();
read.close();
}
} catch (Exception e) {
e.printStackTrace();
}
return map;
}//获取文件内容并广播
Map<String,String> pidMap = loadDataFromFile();
Broadcast<Map<String,String>> broadcast = sc.broadcast(pidMap);
//在worker中使用
broadcast.getValue(); 相关推荐
Lzs 2020-10-23
聚合室 2020-11-16
零 2020-09-18
Justhavefun 2020-10-22
ChaITSimpleLove 2020-10-06
周游列国之仕子 2020-09-15
afanti 2020-09-16
88234852 2020-09-15
YClimb 2020-09-15
风雨断肠人 2020-09-04
卖口粥湛蓝的天空 2020-09-15
stulen 2020-09-15
pythonxuexi 2020-09-06
abfdada 2020-08-26
梦的天空 2020-08-25