Spark2.x写Hbase1-2.x
import org.apache.hadoop.hbase.io.ImmutableBytesWritable import org.apache.hadoop.hbase.mapreduce.TableOutputFormat import org.apache.hadoop.hbase.client.Result import org.apache.hadoop.hbase.client.Put import org.apache.hadoop.mapreduce.Job import org.apache.hadoop.hbase.util.Bytes import org.apache.spark.{SparkConf, SparkContext} /** * Spark写HBase */ object SparkWriteHbase { def main(args: Array[String]): Unit = { val conf = new SparkConf().setAppName("SparkWriteHBase").setMaster("local") val sc = new SparkContext(conf) val tableName = "student" sc.hadoopConfiguration.set(TableOutputFormat.OUTPUT_TABLE, tableName) val job = new Job(sc.hadoopConfiguration) job.setOutputKeyClass(classOf[ImmutableBytesWritable]) job.setOutputValueClass(classOf[Result]) job.setOutputFormatClass(classOf[TableOutputFormat[ImmutableBytesWritable]]) val inDataRDD = sc.makeRDD(Array("3,Rongcheng,M,26","4,Guanhua,M,27")) val rdd = inDataRDD.map(_.split(",")).map(arr=>{ val put = new Put(Bytes.toBytes(arr(0))) put.addColumn(Bytes.toBytes("info"),Bytes.toBytes("name"),Bytes.toBytes(arr(1))) put.addColumn(Bytes.toBytes("info"),Bytes.toBytes("gender"),Bytes.toBytes(arr(2))) put.addColumn(Bytes.toBytes("info"),Bytes.toBytes("age"),Bytes.toBytes(arr(3))) (new ImmutableBytesWritable(),put) }) rdd.saveAsNewAPIHadoopDataset(job.getConfiguration) } }
相关推荐
hjr 2020-09-15
FightFourEggs 2020-08-16
踩风火轮的乌龟 2020-07-26
xiyoukeke 2020-07-19
xwb 2020-07-19
拿什么来拯救自己 2020-07-07
yjsflxiang 2020-07-04
luobotoutou 2020-06-16
JF0 2020-06-13
柠檬班 2020-06-11
adwen00 2020-06-09
hitxueliang 2020-06-05
PlayerL 2020-06-03
bluet00 2020-05-31
bianruifeng 2020-05-31
zhangchaoming 2020-05-17
wennuanwarm 2020-05-11
plusz 2020-05-09
nan00zzu 2020-05-11