spark之从外部文件获取广播变量
public static Map<String,String> loadDataFromFile() { Map<String,String> map = new ConcurrentHashMap<>(); try { File file = new File("filename.txt"); if (file.isFile() && file.exists()) { InputStreamReader read = new InputStreamReader(new FileInputStream(file), "UTF-8"); BufferedReader bufferedReader = new BufferedReader(read); String line = null; while ((line = bufferedReader.readLine()) != null) { String[] split = line.split(","); if (split.length==2) { map.put(split[0],split[1]); } } bufferedReader.close(); read.close(); } } catch (Exception e) { e.printStackTrace(); } return map; }
//获取文件内容并广播 Map<String,String> pidMap = loadDataFromFile(); Broadcast<Map<String,String>> broadcast = sc.broadcast(pidMap); //在worker中使用 broadcast.getValue();
相关推荐
Lzs 2020-10-23
聚合室 2020-11-16
零 2020-09-18
Justhavefun 2020-10-22
jacktangj 2020-10-14
ChaITSimpleLove 2020-10-06
Andrea0 2020-09-18
周游列国之仕子 2020-09-15
afanti 2020-09-16
88234852 2020-09-15
YClimb 2020-09-15
风雨断肠人 2020-09-04
卖口粥湛蓝的天空 2020-09-15
stulen 2020-09-15
pythonxuexi 2020-09-06
abfdada 2020-08-26
梦的天空 2020-08-25