hbase 出现的问题

log4j:WARN No appenders could be found for logger (org.apache.hadoop.util.Shell).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
Exception in thread "main" org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=36, exceptions:
Fri Jun 12 12:00:57 CST 2015, null, java.net.SocketTimeoutException: callTimeout=60000, callDuration=71425: row 'mytown:blog,,' on table 'hbase:meta' at region=hbase:meta,,1.1588230740, hostname=zjenterprise06,60020,1434081500945, seqNum=0

    at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:264)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:215)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:56)
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200)
    at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:288)
    at org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:267)
    at org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:139)
    at org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:134)
    at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:825)
    at org.apache.hadoop.hbase.MetaTableAccessor.fullScan(MetaTableAccessor.java:601)
    at org.apache.hadoop.hbase.MetaTableAccessor.tableExists(MetaTableAccessor.java:365)
    at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:281)
    at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:291)
    at org.zjenterprise.tizr.hbase.HBaseOperation.createTable(HBaseOperation.java:39)
    at org.zjenterprise.tizr.hbase.HBaseTestb.main(HBaseTestb.java:33)
Caused by: java.net.SocketTimeoutException: callTimeout=60000, callDuration=71425: row 'mytown:blog,,' on table 'hbase:meta' at region=hbase:meta,,1.1588230740, hostname=zjenterprise06,60020,1434081500945, seqNum=0
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:159)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:310)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:291)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.hadoop.net.ConnectTimeoutException: 10000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=zjenterprise06/60.191.124.236:60020]
    at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:534)
    at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:495)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupConnection(RpcClientImpl.java:403)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupIOstreams(RpcClientImpl.java:709)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:880)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:849)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1173)
    at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:216)
    at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:300)
    at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:31889)
    at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:344)
    at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:188)
    at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:62)
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126)
    ... 6 more
//////////////////////////////////////////////////////////////////////////////////////////////////////////

start create table ......
log4j:WARN No appenders could be found for logger (org.apache.hadoop.metrics2.lib.MutableMetricsFactory).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=36, exceptions:
Mon Jun 01 08:29:21 CST 2015, null, java.net.SocketTimeoutException: callTimeout=60000, callDuration=68287: row 'mytown:myhome,,' on table 'hbase:meta' at region=hbase:meta,,1.1588230740, hostname=zjenterprise05,60021,1432399066976, seqNum=0

    at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:264)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:199)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:56)
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200)
    at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:287)
    at org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:267)
    at org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:139)
    at org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:134)
    at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:825)
    at org.apache.hadoop.hbase.MetaTableAccessor.fullScan(MetaTableAccessor.java:601)
    at org.apache.hadoop.hbase.MetaTableAccessor.tableExists(MetaTableAccessor.java:365)
    at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:281)
    at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:291)
    at org.zjenterprise.framework.hbase.HbaseTest.createTable(HbaseTest.java:60)
    at org.zjenterprise.framework.hbase.HbaseTest.main(HbaseTest.java:46)
Caused by: java.net.SocketTimeoutException: callTimeout=60000, callDuration=68287: row 'mytown:myhome,,' on table 'hbase:meta' at region=hbase:meta,,1.1588230740, hostname=zjenterprise05,60021,1432399066976, seqNum=0
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:159)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:294)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:275)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.hadoop.net.ConnectTimeoutException: 10000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=zjenterprise05/60.191.124.236:60021]
    at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:533)
    at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:494)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupConnection(RpcClientImpl.java:403)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupIOstreams(RpcClientImpl.java:709)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:880)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:849)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1173)
    at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:216)
    at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:300)
    at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:31751)
    at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:332)
    at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:187)
    at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:62)
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126)
    ... 6 more
end create table ......

2

－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－－

org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=36, exceptions:
Mon Jun 01 08:42:10 CST 2015, null, java.net.SocketTimeoutException: callTimeout=60000, callDuration=68563: row 'mytown:myhome,,' on table 'hbase:meta' at region=hbase:meta,,1.1588230740, hostname=zjenterprise01,60021,1433119375125, seqNum=0

    at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:264)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:199)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:56)
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200)
    at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:287)
    at org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:267)
    at org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:139)
    at org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:134)
    at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:825)
    at org.apache.hadoop.hbase.MetaTableAccessor.fullScan(MetaTableAccessor.java:601)
    at org.apache.hadoop.hbase.MetaTableAccessor.tableExists(MetaTableAccessor.java:365)
    at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:281)
    at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:291)
    at org.zjenterprise.framework.hbase.HbaseTest.createTable(HbaseTest.java:62)
    at org.zjenterprise.framework.hbase.HbaseTest.main(HbaseTest.java:47)
Caused by: java.net.SocketTimeoutException: callTimeout=60000, callDuration=68563: row 'mytown:myhome,,' on table 'hbase:meta' at region=hbase:meta,,1.1588230740, hostname=zjenterprise01,60021,1433119375125, seqNum=0
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:159)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:294)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:275)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.hadoop.net.ConnectTimeoutException: 10000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=zjenterprise01/60.191.124.236:60021]
    at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:533)
    at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:494)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupConnection(RpcClientImpl.java:403)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupIOstreams(RpcClientImpl.java:709)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:880)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:849)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1173)
    at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:216)
    at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:300)
    at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:31751)
    at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:332)
    at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:187)
    at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:62)
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126)
    ... 6 more
end create table ......

2015-06-11 15:42:13,062 INFO  [main] zookeeper.ZooKeeper: Initiating client connection, connectString=192.168.3.140:2181,192.168.3.130:2181,192.168.3.142:2181 sessionTimeout=120000 watcher=master:600000x0, quorum=192.168.3.140:2181,192.168.3.130:2181,192.168.3.142:2181, baseZNode=/hbase
2015-06-11 15:42:13,072 INFO  [main-SendThread(192.168.3.142:2181)] zookeeper.ClientCnxn: Opening socket connection to server 192.168.3.142/192.168.3.142:2181. Will not attempt to authenticate using SASL (unknown error)
2015-06-11 15:42:13,106 INFO  [main-SendThread(192.168.3.142:2181)] zookeeper.ClientCnxn: Socket connection established to 192.168.3.142/192.168.3.142:2181, initiating session
2015-06-11 15:42:13,122 INFO  [main-SendThread(192.168.3.142:2181)] zookeeper.ClientCnxn: Session establishment complete on server 192.168.3.142/192.168.3.142:2181, sessionid = 0x34de190d9cf0001, negotiated timeout = 40000
2015-06-11 15:42:13,175 INFO  [RpcServer.responder] ipc.RpcServer: RpcServer.responder: starting
2015-06-11 15:42:13,181 INFO  [RpcServer.listener,port=60000] ipc.RpcServer: RpcServer.listener,port=60000: starting
2015-06-11 15:42:13,238 INFO  [main] mortbay.log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
2015-06-11 15:42:13,244 INFO  [main] http.HttpRequestLog: Http request log for http.requests.master is not defined
2015-06-11 15:42:13,251 INFO  [main] http.HttpServer: Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter)
2015-06-11 15:42:13,255 INFO  [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master
2015-06-11 15:42:13,255 INFO  [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs
2015-06-11 15:42:13,255 INFO  [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static
2015-06-11 15:42:13,265 INFO  [main] http.HttpServer: Jetty bound to port 16010
2015-06-11 15:42:13,265 INFO  [main] mortbay.log: jetty-6.1.26
2015-06-11 15:42:13,612 INFO  [main] mortbay.log: Started [email protected]:16010
2015-06-11 15:42:13,613 INFO  [main] master.HMaster: hbase.rootdir=hdfs://192.168.3.130:9000/hbase, hbase.cluster.distributed=true
2015-06-11 15:42:13,621 INFO  [main] master.HMaster: Adding backup master ZNode /hbase/backup-masters/zjenterprise01,60000,1434008532400
2015-06-11 15:42:13,734 INFO  [zjenterprise01:60000.activeMasterManager] master.ActiveMasterManager: Deleting ZNode for /hbase/backup-masters/zjenterprise01,60000,1434008532400 from backup master directory
2015-06-11 15:42:13,755 INFO  [zjenterprise01:60000.activeMasterManager] master.ActiveMasterManager: Registered Active Master=zjenterprise01,60000,1434008532400
2015-06-11 15:42:13,772 INFO  [master/zjenterprise01/192.168.3.130:60000] zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x2bba5623 connecting to ZooKeeper ensemble=192.168.3.140:2181,192.168.3.130:2181,192.168.3.142:2181
2015-06-11 15:42:13,772 INFO  [master/zjenterprise01/192.168.3.130:60000] zookeeper.ZooKeeper: Initiating client connection, connectString=192.168.3.140:2181,192.168.3.130:2181,192.168.3.142:2181 sessionTimeout=120000 watcher=hconnection-0x2bba56230x0, quorum=192.168.3.140:2181,192.168.3.130:2181,192.168.3.142:2181, baseZNode=/hbase
2015-06-11 15:42:13,773 INFO  [master/zjenterprise01/192.168.3.130:60000-SendThread(192.168.3.130:2181)] zookeeper.ClientCnxn: Opening socket connection to server 192.168.3.130/192.168.3.130:2181. Will not attempt to authenticate using SASL (unknown error)
2015-06-11 15:42:13,773 INFO  [master/zjenterprise01/192.168.3.130:60000-SendThread(192.168.3.130:2181)] zookeeper.ClientCnxn: Socket connection established to 192.168.3.130/192.168.3.130:2181, initiating session
2015-06-11 15:42:13,784 INFO  [master/zjenterprise01/192.168.3.130:60000-SendThread(192.168.3.130:2181)] zookeeper.ClientCnxn: Session establishment complete on server 192.168.3.130/192.168.3.130:2181, sessionid = 0x14de190d61e0001, negotiated timeout = 40000
2015-06-11 15:42:13,801 INFO  [master/zjenterprise01/192.168.3.130:60000] regionserver.HRegionServer: ClusterId : a1299f43-a1d1-4189-95cd-963fc5e220b6
2015-06-11 15:42:13,852 INFO  [zjenterprise01:60000.activeMasterManager] util.FSUtils: Waiting for dfs to exit safe mode...
2015-06-11 15:42:24,491 INFO  [zjenterprise01:60000.activeMasterManager] fs.HFileSystem: Added intercepting call to namenode#getBlockLocations so can do block reordering using class class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2015-06-11 15:42:24,500 INFO  [zjenterprise01:60000.activeMasterManager] coordination.SplitLogManagerCoordination: Found 0 orphan tasks and 0 rescan nodes
2015-06-11 15:42:24,522 INFO  [zjenterprise01:60000.activeMasterManager] zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x4de50506 connecting to ZooKeeper ensemble=192.168.3.140:2181,192.168.3.130:2181,192.168.3.142:2181
2015-06-11 15:42:24,522 INFO  [zjenterprise01:60000.activeMasterManager] zookeeper.ZooKeeper: Initiating client connection, connectString=192.168.3.140:2181,192.168.3.130:2181,192.168.3.142:2181 sessionTimeout=120000 watcher=hconnection-0x4de505060x0, quorum=192.168.3.140:2181,192.168.3.130:2181,192.168.3.142:2181, baseZNode=/hbase
2015-06-11 15:42:24,523 INFO  [zjenterprise01:60000.activeMasterManager-SendThread(192.168.3.142:2181)] zookeeper.ClientCnxn: Opening socket connection to server 192.168.3.142/192.168.3.142:2181. Will not attempt to authenticate using SASL (unknown error)
2015-06-11 15:42:24,524 INFO  [zjenterprise01:60000.activeMasterManager-SendThread(192.168.3.142:2181)] zookeeper.ClientCnxn: Socket connection established to 192.168.3.142/192.168.3.142:2181, initiating session
2015-06-11 15:42:24,536 INFO  [zjenterprise01:60000.activeMasterManager-SendThread(192.168.3.142:2181)] zookeeper.ClientCnxn: Session establishment complete on server 192.168.3.142/192.168.3.142:2181, sessionid = 0x34de190d9cf0002, negotiated timeout = 40000
2015-06-11 15:42:24,555 INFO  [zjenterprise01:60000.activeMasterManager] balancer.StochasticLoadBalancer: loading config
2015-06-11 15:42:24,609 INFO  [zjenterprise01:60000.activeMasterManager] master.HMaster: Server active/primary master=zjenterprise01,60000,1434008532400, sessionid=0x34de190d9cf0001, setting cluster-up flag (Was=true)
2015-06-11 15:42:24,629 INFO  [zjenterprise01:60000.activeMasterManager] procedure.ZKProcedureUtil: Clearing all procedure znodes: /hbase/flush-table-proc/acquired /hbase/flush-table-proc/reached /hbase/flush-table-proc/abort
2015-06-11 15:42:24,645 INFO  [zjenterprise01:60000.activeMasterManager] procedure.ZKProcedureUtil: Clearing all procedure znodes: /hbase/online-snapshot/acquired /hbase/online-snapshot/reached /hbase/online-snapshot/abort
2015-06-11 15:42:24,663 INFO  [zjenterprise01:60000.activeMasterManager] master.MasterCoprocessorHost: System coprocessor loading is enabled
2015-06-11 15:42:24,666 INFO  [zjenterprise01:60000.activeMasterManager] zookeeper.RecoverableZooKeeper: Process identifier=replicationLogCleaner connecting to ZooKeeper ensemble=192.168.3.140:2181,192.168.3.130:2181,192.168.3.142:2181
2015-06-11 15:42:24,666 INFO  [zjenterprise01:60000.activeMasterManager] zookeeper.ZooKeeper: Initiating client connection, connectString=192.168.3.140:2181,192.168.3.130:2181,192.168.3.142:2181 sessionTimeout=120000 watcher=replicationLogCleaner0x0, quorum=192.168.3.140:2181,192.168.3.130:2181,192.168.3.142:2181, baseZNode=/hbase
2015-06-11 15:42:24,667 INFO  [zjenterprise01:60000.activeMasterManager-SendThread(192.168.3.140:2181)] zookeeper.ClientCnxn: Opening socket connection to server 192.168.3.140/192.168.3.140:2181. Will not attempt to authenticate using SASL (unknown error)
2015-06-11 15:42:24,667 INFO  [zjenterprise01:60000.activeMasterManager-SendThread(192.168.3.140:2181)] zookeeper.ClientCnxn: Socket connection established to 192.168.3.140/192.168.3.140:2181, initiating session
2015-06-11 15:42:24,679 INFO  [zjenterprise01:60000.activeMasterManager-SendThread(192.168.3.140:2181)] zookeeper.ClientCnxn: Session establishment complete on server 192.168.3.140/192.168.3.140:2181, sessionid = 0x24de190d5be0006, negotiated timeout = 40000
2015-06-11 15:42:24,698 INFO  [zjenterprise01:60000.activeMasterManager] master.ServerManager: Waiting for region servers count to settle; currently checked in 0, slept for 0 ms, expecting minimum of 1, maximum of 2147483647, timeout of 4500 ms, interval of 1500 ms.
2015-06-11 15:42:24,752 INFO  [B.defaultRpcServer.handler=13,queue=1,port=60000] master.ServerManager: Registering server=zjenterprise06,60020,1434008534587
2015-06-11 15:42:24,798 INFO  [zjenterprise01:60000.activeMasterManager] master.ServerManager: Waiting for region servers count to settle; currently checked in 1, slept for 100 ms, expecting minimum of 1, maximum of 2147483647, timeout of 4500 ms, interval of 1500 ms.
2015-06-11 15:42:25,026 INFO  [B.defaultRpcServer.handler=16,queue=1,port=60000] master.ServerManager: Registering server=zjenterprise07,60020,1434008536683
2015-06-11 15:42:25,049 INFO  [zjenterprise01:60000.activeMasterManager] master.ServerManager: Waiting for region servers count to settle; currently checked in 2, slept for 351 ms, expecting minimum of 1, maximum of 2147483647, timeout of 4500 ms, interval of 1500 ms.
2015-06-11 15:42:26,554 INFO  [zjenterprise01:60000.activeMasterManager] master.ServerManager: Waiting for region servers count to settle; currently checked in 2, slept for 1856 ms, expecting minimum of 1, maximum of 2147483647, timeout of 4500 ms, interval of 1500 ms.
2015-06-11 15:42:27,107 INFO  [B.defaultRpcServer.handler=19,queue=1,port=60000] master.ServerManager: Registering server=zjenterprise01,60020,1434008533879
2015-06-11 15:42:27,156 INFO  [zjenterprise01:60000.activeMasterManager] master.ServerManager: Waiting for region servers count to settle; currently checked in 3, slept for 2458 ms, expecting minimum of 1, maximum of 2147483647, timeout of 4500 ms, interval of 1500 ms.
2015-06-11 15:42:28,661 INFO  [zjenterprise01:60000.activeMasterManager] master.ServerManager: Waiting for region servers count to settle; currently checked in 3, slept for 3962 ms, expecting minimum of 1, maximum of 2147483647, timeout of 4500 ms, interval of 1500 ms.
2015-06-11 15:42:29,213 INFO  [zjenterprise01:60000.activeMasterManager] master.ServerManager: Finished waiting for region servers count to settle; checked in 3, slept for 4515 ms, expecting minimum of 1, maximum of 2147483647, master is running
2015-06-11 15:42:29,223 INFO  [zjenterprise01:60000.activeMasterManager] master.MasterFileSystem: Log folder hdfs://192.168.3.130:9000/hbase/WALs/zjenterprise01,60020,1434008533879 belongs to an existing region server
2015-06-11 15:42:29,225 INFO  [zjenterprise01:60000.activeMasterManager] master.MasterFileSystem: Log folder hdfs://192.168.3.130:9000/hbase/WALs/zjenterprise06,60020,1434008534587 belongs to an existing region server
2015-06-11 15:42:29,226 INFO  [zjenterprise01:60000.activeMasterManager] master.MasterFileSystem: Log folder hdfs://192.168.3.130:9000/hbase/WALs/zjenterprise07,60020,1434008536683 belongs to an existing region server
2015-06-11 15:42:29,314 INFO  [zjenterprise01:60000.activeMasterManager] zookeeper.MetaTableLocator: Failed verification of hbase:meta,,1 at address=zjenterprise07,60020,1433932864870, exception=org.apache.hadoop.hbase.NotServingRegionException: Region hbase:meta,,1 is not online on zjenterprise07,60020,1434008536683
    at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2749)
    at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:863)
    at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegionInfo(RSRpcServices.java:1141)
    at org.apache.hadoop.hbase.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:20862)
    at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031)
    at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
    at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
    at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
    at java.lang.Thread.run(Thread.java:745)

2015-06-11 15:42:29,319 INFO  [zjenterprise01:60000.activeMasterManager] master.MasterFileSystem: Log dir for server zjenterprise07,60020,1433932864870 does not exist
2015-06-11 15:42:29,319 INFO  [zjenterprise01:60000.activeMasterManager] master.SplitLogManager: dead splitlog workers [zjenterprise07,60020,1433932864870]
2015-06-11 15:42:29,321 INFO  [zjenterprise01:60000.activeMasterManager] master.SplitLogManager: started splitting 0 logs in [] for [zjenterprise07,60020,1433932864870]
2015-06-11 15:42:29,322 INFO  [zjenterprise01:60000.activeMasterManager] master.SplitLogManager: finished splitting (more than or equal to) 0 bytes in 0 log files in [] in 1ms
2015-06-11 15:42:29,323 INFO  [zjenterprise01:60000.activeMasterManager] zookeeper.MetaTableLocator: Deleting hbase:meta region location in ZooKeeper
2015-06-11 15:42:29,375 INFO  [zjenterprise01:60000.activeMasterManager] master.AssignmentManager: Assigning hbase:meta,,1.1588230740 to zjenterprise07,60020,1434008536683
2015-06-11 15:42:29,375 INFO  [zjenterprise01:60000.activeMasterManager] master.RegionStates: Transition {1588230740 state=OFFLINE, ts=1434008549358, server=null} to {1588230740 state=PENDING_OPEN, ts=1434008549375, server=zjenterprise07,60020,1434008536683}
2015-06-11 15:42:29,418 INFO  [zjenterprise01:60000.activeMasterManager] master.ServerManager: AssignmentManager hasn't finished failover cleanup; waiting
2015-06-11 15:42:29,460 INFO  [AM.ZK.Worker-pool2-t1] master.RegionStates: Transition {1588230740 state=PENDING_OPEN, ts=1434008549375, server=zjenterprise07,60020,1434008536683} to {1588230740 state=OPENING, ts=1434008549460, server=zjenterprise07,60020,1434008536683}
2015-06-11 15:42:30,026 INFO  [AM.ZK.Worker-pool2-t2] master.RegionStates: Transition {1588230740 state=OPENING, ts=1434008549460, server=zjenterprise07,60020,1434008536683} to {1588230740 state=OPEN, ts=1434008550025, server=zjenterprise07,60020,1434008536683}
2015-06-11 15:42:30,029 INFO  [AM.ZK.Worker-pool2-t2] coordination.ZkOpenRegionCoordination: Handling OPENED of 1588230740 from zjenterprise01,60000,1434008532400; deleting unassigned node
2015-06-11 15:42:30,041 INFO  [AM.ZK.Worker-pool2-t3] master.RegionStates: Onlined 1588230740 on zjenterprise07,60020,1434008536683
2015-06-11 15:42:30,041 INFO  [zjenterprise01:60000.activeMasterManager] master.HMaster: hbase:meta assigned=1, rit=false, location=zjenterprise07,60020,1434008536683
2015-06-11 15:42:30,108 INFO  [zjenterprise01:60000.activeMasterManager] hbase.MetaMigrationConvertingToPB: hbase:meta doesn't have any entries to update.
2015-06-11 15:42:30,108 INFO  [zjenterprise01:60000.activeMasterManager] hbase.MetaMigrationConvertingToPB: META already up-to date with PB serialization
2015-06-11 15:42:30,149 INFO  [zjenterprise01:60000.activeMasterManager] master.AssignmentManager: Clean cluster startup. Assigning user regions
2015-06-11 15:42:30,172 INFO  [zjenterprise01:60000.activeMasterManager] master.AssignmentManager: Joined the cluster in 64ms, failover=false
2015-06-11 15:42:30,191 INFO  [zjenterprise01:60000.activeMasterManager] master.TableNamespaceManager: Namespace table not found. Creating...
2015-06-11 15:42:30,315 FATAL [zjenterprise01:60000.activeMasterManager] master.HMaster: Failed to become active master
org.apache.hadoop.hbase.TableExistsException: hbase:namespace
    at org.apache.hadoop.hbase.master.handler.CreateTableHandler.checkAndSetEnablingTable(CreateTableHandler.java:151)
    at org.apache.hadoop.hbase.master.handler.CreateTableHandler.prepare(CreateTableHandler.java:124)
    at org.apache.hadoop.hbase.master.TableNamespaceManager.createNamespaceTable(TableNamespaceManager.java:233)
    at org.apache.hadoop.hbase.master.TableNamespaceManager.start(TableNamespaceManager.java:86)
    at org.apache.hadoop.hbase.master.HMaster.initNamespace(HMaster.java:871)
    at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:722)
    at org.apache.hadoop.hbase.master.HMaster.access$500(HMaster.java:165)
    at org.apache.hadoop.hbase.master.HMaster$1.run(HMaster.java:1428)
    at java.lang.Thread.run(Thread.java:745)
2015-06-11 15:42:30,318 FATAL [zjenterprise01:60000.activeMasterManager] master.HMaster: Master server abort: loaded coprocessors are: []
2015-06-11 15:42:30,318 FATAL [zjenterprise01:60000.activeMasterManager] master.HMaster: Unhandled exception. Starting shutdown.
org.apache.hadoop.hbase.TableExistsException: hbase:namespace
    at org.apache.hadoop.hbase.master.handler.CreateTableHandler.checkAndSetEnablingTable(CreateTableHandler.java:151)
    at org.apache.hadoop.hbase.master.handler.CreateTableHandler.prepare(CreateTableHandler.java:124)
    at org.apache.hadoop.hbase.master.TableNamespaceManager.createNamespaceTable(TableNamespaceManager.java:233)
    at org.apache.hadoop.hbase.master.TableNamespaceManager.start(TableNamespaceManager.java:86)
    at org.apache.hadoop.hbase.master.HMaster.initNamespace(HMaster.java:871)
    at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:722)
    at org.apache.hadoop.hbase.master.HMaster.access$500(HMaster.java:165)
    at org.apache.hadoop.hbase.master.HMaster$1.run(HMaster.java:1428)
    at java.lang.Thread.run(Thread.java:745)
2015-06-11 15:42:30,318 INFO  [zjenterprise01:60000.activeMasterManager] regionserver.HRegionServer: STOPPED: Unhandled exception. Starting shutdown.
2015-06-11 15:42:30,318 INFO  [master/zjenterprise01/192.168.3.130:60000] regionserver.HRegionServer: Stopping infoServer
2015-06-11 15:42:30,321 INFO  [master/zjenterprise01/192.168.3.130:60000] mortbay.log: Stopped [email protected]:16010
2015-06-11 15:42:30,422 INFO  [master/zjenterprise01/192.168.3.130:60000] regionserver.HRegionServer: stopping server zjenterprise01,60000,1434008532400
2015-06-11 15:42:30,429 INFO  [master/zjenterprise01/192.168.3.130:60000] client.ConnectionManager$HConnectionImplementation: Closing zookeeper sessionid=0x14de190d61e0001
2015-06-11 15:42:30,447 INFO  [master/zjenterprise01/192.168.3.130:60000] zookeeper.ZooKeeper: Session: 0x14de190d61e0001 closed
2015-06-11 15:42:30,447 INFO  [master/zjenterprise01/192.168.3.130:60000-EventThread] zookeeper.ClientCnxn: EventThread shut down
2015-06-11 15:42:30,502 INFO  [zjenterprise01,60000,1434008532400.splitLogManagerTimeoutMonitor] master.SplitLogManager$TimeoutMonitor: zjenterprise01,60000,1434008532400.splitLogManagerTimeoutMonitor exiting
2015-06-11 15:42:30,548 INFO  [master/zjenterprise01/192.168.3.130:60000] regionserver.HRegionServer: stopping server zjenterprise01,60000,1434008532400; all regions closed.
2015-06-11 15:42:30,549 INFO  [zjenterprise01,60000,1434008532400-BalancerChore] balancer.BalancerChore: zjenterprise01,60000,1434008532400-BalancerChore exiting
2015-06-11 15:42:30,549 INFO  [zjenterprise01:60000.oldLogCleaner] cleaner.LogCleaner: zjenterprise01:60000.oldLogCleaner exiting
2015-06-11 15:42:30,549 INFO  [CatalogJanitor-zjenterprise01:60000] master.CatalogJanitor: CatalogJanitor-zjenterprise01:60000 exiting
2015-06-11 15:42:30,549 INFO  [zjenterprise01,60000,1434008532400-ClusterStatusChore] balancer.ClusterStatusChore: zjenterprise01,60000,1434008532400-ClusterStatusChore exiting
2015-06-11 15:42:30,549 INFO  [zjenterprise01:60000.oldLogCleaner] master.ReplicationLogCleaner: Stopping replicationLogCleaner-0x24de190d5be0006, quorum=192.168.3.140:2181,192.168.3.130:2181,192.168.3.142:2181, baseZNode=/hbase
2015-06-11 15:42:30,550 INFO  [zjenterprise01:60000.archivedHFileCleaner] cleaner.HFileCleaner: zjenterprise01:60000.archivedHFileCleaner exiting
2015-06-11 15:42:30,557 INFO  [zjenterprise01:60000.oldLogCleaner] zookeeper.ZooKeeper: Session: 0x24de190d5be0006 closed
2015-06-11 15:42:30,557 INFO  [zjenterprise01:60000.activeMasterManager-EventThread] zookeeper.ClientCnxn: EventThread shut down
2015-06-11 15:42:30,565 INFO  [master/zjenterprise01/192.168.3.130:60000] client.ConnectionManager$HConnectionImplementation: Closing zookeeper sessionid=0x34de190d9cf0002
2015-06-11 15:42:30,606 INFO  [master/zjenterprise01/192.168.3.130:60000] zookeeper.ZooKeeper: Session: 0x34de190d9cf0002 closed
2015-06-11 15:42:30,606 INFO  [zjenterprise01:60000.activeMasterManager-EventThread] zookeeper.ClientCnxn: EventThread shut down
2015-06-11 15:42:30,707 INFO  [master/zjenterprise01/192.168.3.130:60000] flush.MasterFlushTableProcedureManager: stop: server shutting down.
2015-06-11 15:42:30,708 INFO  [master/zjenterprise01/192.168.3.130:60000] ipc.RpcServer: Stopping server on 60000
2015-06-11 15:42:30,708 INFO  [RpcServer.listener,port=60000] ipc.RpcServer: RpcServer.listener,port=60000: stopping
2015-06-11 15:42:30,709 INFO  [RpcServer.responder] ipc.RpcServer: RpcServer.responder: stopped
2015-06-11 15:42:30,709 INFO  [RpcServer.responder] ipc.RpcServer: RpcServer.responder: stopping
2015-06-11 15:42:30,731 INFO  [master/zjenterprise01/192.168.3.130:60000] zookeeper.RecoverableZooKeeper: Node /hbase/rs/zjenterprise01,60000,1434008532400 already deleted, retry=false
2015-06-11 15:42:30,739 INFO  [master/zjenterprise01/192.168.3.130:60000] zookeeper.ZooKeeper: Session: 0x34de190d9cf0001 closed
2015-06-11 15:42:30,739 INFO  [main-EventThread] zookeeper.ClientCnxn: EventThread shut down
2015-06-11 15:42:30,739 INFO  [master/zjenterprise01/192.168.3.130:60000] regionserver.HRegionServer: stopping server zjenterprise01,60000,1434008532400; zookeeper connection closed.
2015-06-11 15:42:30,745 INFO  [master/zjenterprise01/192.168.3.130:60000] regionserver.HRegionServer: master/zjenterprise01/192.168.3.130:60000 exiting