hadoop hbase 集群的安装(未整理,先记录在这)
ssh免登陆的配置
ssh-keygen-trsa
cd~/.ssh
[[email protected]]#ll
总用量8
-rw-------1rootroot16791月221:04id_rsa
-rw-r--r--1rootroot3931月221:04id_rsa.pub
slave1
scp~/.ssh/id_rsa.pubroot@master:~/.ssh/id_rsa.pub_slave1
slave2
scp~/.ssh/id_rsa.pubroot@master:~/.ssh/id_rsa.pub_slave2
[[email protected]]#ll
总用量20
-rw-------1rootroot16791月221:04id_rsa
-rw-r--r--1rootroot3931月221:04id_rsa.pub
-rw-r--r--1rootroot3931月221:24id_rsa.pub_slave1
-rw-r--r--1rootroot3931月221:24id_rsa.pub_slave2
-rw-r--r--1rootroot8081月221:11known_hosts
cd~/.ssh
touchauthorized_keys
cat~/.ssh/id_rsa.pub>>authorized_keys
cat~/.ssh/id_rsa.pub_slave1>>authorized_keys
cat~/.ssh/id_rsa.pub_slave2>>authorized_keys
catauthorized_keys
[[email protected]]#catauthorized_keys
ssh-rsaAAAAB3NzaC1yc2EAAAABIwAAAQEA29JCTGZjQBQUubsyMeL0rCRsRLlDWi7FqQTyp5/u+3WtS2Sg8psi/k3B+okQ6rVx6fYaUvF7TsBzjZv51ru+3Utlh56XuxL+gtrVr4KV1St/1DLuOpAvHImEbRIQxzAedaxY8PLSScVnQCoU5T9XYXdjb/z3AcvXC8Kr5GtTwGEgxndmbdmKKw0+VBboJNjNZ1chCfCpdJdZv5DK7a6uUhElXC/60+/OkM0C6lKi1/UVYdwN+A3Lch8OjfcX3iABWLjf8g2Z+tuHyDJ74XGSkkTRmgUBG1zFobnYgfnBk3LmRRtMK6yY6gsI9sYAEJs9LvP72PsXv08EcHG9qGmUNQ==root@master
ssh-rsaAAAAB3NzaC1yc2EAAAABIwAAAQEAkV6o6cLTfUclAQpC1oJRJi+0xtX6Bx3rLlEi/nZtw6CkpGkh3JIq5MJ9A7lSfVIcLTjv6i1MovrPc9wEbVao89E5l1gUBf8ZXcoDIg6y1UknRpdkfft348/NmIg858QAENXT3XVCDN4HviIM2kpdli5r+PLbis0XFdv5Coetx/bSHkak8v0rkQSIjMrQI6ClVI3sjOfZSYwT9C3TfNK0DI7hCyqHZnAuRS52v7ntti39KOnDmCPoBf+Aat4uv5zm3KmWG43qY+fFqsBiwCnT+7omKrdQSdhhosbZsPrc7oPJ0f2snyJDWalnZx9TFYOMoXlktiZZadFo41JEGJufiQ==root@slave1
ssh-rsaAAAAB3NzaC1yc2EAAAABIwAAAQEAxQswKavM4rPd5GLwKNpOzNhjKiw8JOd0x2EPJJ0SayV5C+/99htVQ7e6tWNaMoHhO4JiWQXnkMK7PHd8J0EyI+IzfPTcjJOMlp2oduivPknp8THMjjYdVq+NhkybNeaF9/ZCZc7/S6x/hU+Q5nPxev9glBNeZtEFiOrSV0wPeKwykBUDPJItNZNwSFJedfxtNz+LC7TPkK+EwuEnLxZz03EEWYGv8105LROdHzuPnwSXooyXHEfNGmrGkeAeUC8ghKxOiUBXvAngjXZMtQIJlmiho3b5YD1Nfawl4/RIvFz+3R+KRFbVm/AULN5z375ekLk/w6SGjgmDsY71QrE2RQ==root@slave2
rm-rf~/.ssh/id_rsa.pub_slave1
rm-rf~/.ssh/id_rsa.pub_slave2
scp~/.ssh/authorized_keysroot@slave1:~/.ssh/authorized_keys
scp~/.ssh/authorized_keysroot@slave2:~/.ssh/authorized_keys
zookeeper的安装
cd/usr/local/software
tar-zxvfzookeeper-3.4.7.tar.gz-C/usr/local
cd/usr/local/zookeeper-3.4.7/conf
cpzoo_sample.cfgzoo.cfg
chmod777zoo.cfg
mkdir/usr/local/zookeeper-3.4.7/dataDir
cd/usr/local/zookeeper-3.4.7/dataDir
touchmyid
echo1>>myid
//修改dataDir路径
dataDir=/usr/local/zookeeper-3.4.7/dataDir
server.1=master:2888:3888
server.2=slave1:2888:3888
server.3=slave2:2888:3888
scp-r/usr/local/zookeeper-3.4.7root@slave1:/usr/local/
scp-r/usr/local/zookeeper-3.4.7root@slave2:/usr/local/
/usr/local/zookeeper-3.4.7/bin/zkServer.shstart
Usage:/usr/local/zookeeper-3.4.7/bin/zkServer.sh{start|start-foreground|stop|restart|status|upgrade|print-cmd}
/usr/local/zookeeper-3.4.7/bin/zkServer.shstatus
配置文件如下
#Thenumberofmillisecondsofeachtick
tickTime=2000
#Thenumberofticksthattheinitial
#synchronizationphasecantake
initLimit=10
#Thenumberofticksthatcanpassbetween
#sendingarequestandgettinganacknowledgement
syncLimit=5
#thedirectorywherethesnapshotisstored.
#donotuse/tmpforstorage,/tmphereisjust
#examplesakes.
dataDir=/usr/local/zookeeper-3.4.7/dataDir
#theportatwhichtheclientswillconnect
clientPort=2181
#themaximumnumberofclientconnections.
#increasethisifyouneedtohandlemoreclients
#maxClientCnxns=60
#
#Besuretoreadthemaintenancesectionofthe
#administratorguidebeforeturningonautopurge.
#
#http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
#ThenumberofsnapshotstoretainindataDir
#autopurge.snapRetainCount=3
#Purgetaskintervalinhours
#Setto"0"todisableautopurgefeature
#autopurge.purgeInterval=1
server.1=0.0.0.0:2888:3888
server.2=slave1:2888:3888
server.3=slave2:2888:3888
hadoop集群的安装
tar-zxvf/usr/local/software/hadoop-2.6.0.tar.gz-C/usr/local
vi/etc/profile
HADOOP_HOME=/usr/local/hadoop-2.6.0
exportHADOOP_HOME
source/etc/profile
cp/usr/local/hadoop-2.6.0/etc/hadoop/mapred-site.xml.template/usr/local/hadoop-2.6.0/etc/hadoop/mapred-site.xml
/usr/local/hadoop-2.6.0/etc/hadoop/hadoop-env.sh这个文件的第一行
exportJAVA_HOME=/usr/local/jdk1.8.0_65
/usr/local/hadoop-2.6.0/etc/hadoop/yarn-env.sh这个文件的第一行
exportJAVA_HOME=/usr/local/jdk1.8.0_65
/usr/local/hadoop-2.6.0/bin/hdfsnamenode-format
/usr/local/hadoop-2.6.0/bin/hdfsdfsadmin-report
scp-r/usr/local/hadoop-2.6.0root@slave1:/usr/local
scp-r/usr/local/hadoop-2.6.0root@slave2:/usr/local
cat/usr/local/hadoop-2.6.0/etc/hadoop/slaves
[root@master~]#cat/usr/local/hadoop-2.6.0/etc/hadoop/slaves
slave1
slave2
scp-r/usr/local/jdk1.8.0_65root@slave1:/usr/local
scp-r/usr/local/jdk1.8.0_65root@slave2:/usr/local
scp/etc/profileroot@slave1:/etc/profile
scp/etc/profileroot@slave2:/etc/profile
/usr/local/hadoop-2.6.0/sbin/start-all.sh
/usr/local/hadoop-2.6.0/sbin/stop-all.sh
http://192.168.140.128:50070
/usr/local/hadoop-2.6.0/bin/hadoopfs-copyFromLocal/usr/local/jdk1.8.0_65/src.ziphdfs://master:9000/test/src.zip
/usr/local/hadoop-2.6.0/bin/hadoopfs-mkdirhdfs://master:9000/test
/usr/local/hadoop-2.6.0/bin/hadoopfs-ls/
/usr/local/hadoop-2.6.0/bin/hadoopfs-ls/test
//报错的解决
rm-rf/usr/local/hadoop-2.6.0/etc/hadoop/slaves
touch/usr/local/hadoop-2.6.0/etc/hadoop/slaves
chmod777/usr/local/hadoop-2.6.0/etc/hadoop/slaves
scp/usr/local/hadoop-2.6.0/etc/hadoop/slavesroot@slave1:/usr/local/hadoop-2.6.0/etc/hadoop/slaves
scp/usr/local/hadoop-2.6.0/etc/hadoop/slavesroot@slave2:/usr/local/hadoop-2.6.0/etc/hadoop/slaves
[root@masterhadoop]#cat/etc/hosts
192.168.140.128master
192.168.140.129slave1
192.168.140.130slave2
关闭ipv6
vi/etc/sysconfig/network-scripts/ifcfg-eth0
vi/etc/sysconfig/network
vi/etc/modprobe.d/dist.conf
hbase的安装
cd/usr/local/software
tar-zxvfhbase-1.1.2-bin.tar.gz-C/usr/local
cd/usr/local/hbase-1.1.2/lib
ls-l|grephadoop
[root@masterlib]#ls-l|grephadoop
-rw-r--r--1rootroot170418月2710:57hadoop-annotations-2.5.1.jar
-rw-r--r--1rootroot524498月2710:57hadoop-auth-2.5.1.jar
-rw-r--r--1rootroot25578月2710:59hadoop-client-2.5.1.jar
-rw-r--r--1rootroot29626858月2710:57hadoop-common-2.5.1.jar
-rw-r--r--1rootroot70952308月2710:59hadoop-hdfs-2.5.1.jar
-rw-r--r--1rootroot4914098月2710:59hadoop-mapreduce-client-app-2.5.1.jar
-rw-r--r--1rootroot6628928月2710:59hadoop-mapreduce-client-common-2.5.1.jar
-rw-r--r--1rootroot14983688月2710:57hadoop-mapreduce-client-core-2.5.1.jar
-rw-r--r--1rootroot357338月2710:59hadoop-mapreduce-client-jobclient-2.5.1.jar
-rw-r--r--1rootroot436428月2710:59hadoop-mapreduce-client-shuffle-2.5.1.jar
-rw-r--r--1rootroot16498528月2710:57hadoop-yarn-api-2.5.1.jar
-rw-r--r--1rootroot1179828月2710:59hadoop-yarn-client-2.5.1.jar
-rw-r--r--1rootroot14164278月2710:57hadoop-yarn-common-2.5.1.jar
-rw-r--r--1rootroot2423818月2710:59hadoop-yarn-server-common-2.5.1.jar
-rw-r--r--1rootroot876628月2711:11hbase-hadoop2-compat-1.1.2.jar
-rw-r--r--1rootroot359448月2711:11hbase-hadoop-compat-1.1.2.jar
hadoop-annotations-2.5.1.jar
find-name'hadoop-yarn-server-common*.jar'
cp/usr/local/hadoop-2.6.0/share/hadoop/common/lib/hadoop-annotations-2.6.0.jar/usr/local/hbase-1.1.2/lib/
rm-rf/usr/local/hbase-1.1.2/lib/hadoop-annotations-2.5.1.jar
cp/usr/local/hadoop-2.6.0/share/hadoop/common/lib/hadoop-auth-2.6.0.jar/usr/local/hbase-1.1.2/lib/
rm-rf/usr/local/hbase-1.1.2/lib/hadoop-auth-2.5.1.jar
cp/usr/local/hadoop-2.6.0/share/hadoop/common/hadoop-common-2.6.0.jar/usr/local/hbase-1.1.2/lib/
rm-rf/usr/local/hbase-1.1.2/lib/hadoop-common-2.5.1.jar
cp/usr/local/hadoop-2.6.0/share/hadoop/hdfs/hadoop-hdfs-2.6.0.jar/usr/local/hbase-1.1.2/lib/
rm-rf/usr/local/hbase-1.1.2/lib/hadoop-hdfs-2.5.1.jar
cp/usr/local/hadoop-2.6.0/share/hadoop/mapreduce/hadoop-mapreduce-client-app-2.6.0.jar/usr/local/hbase-1.1.2/lib/
rm-rf/usr/local/hbase-1.1.2/lib/hadoop-mapreduce-client-app-2.5.1.jar
cp/usr/local/hadoop-2.6.0/share/hadoop/mapreduce/hadoop-mapreduce-client-common-2.6.0.jar/usr/local/hbase-1.1.2/lib/
rm-rf/usr/local/hbase-1.1.2/lib/hadoop-mapreduce-client-common-2.5.1.jar
cp/usr/local/hadoop-2.6.0/share/hadoop/mapreduce/hadoop-mapreduce-client-core-2.6.0.jar/usr/local/hbase-1.1.2/lib/
rm-rf/usr/local/hbase-1.1.2/lib/hadoop-mapreduce-client-core-2.5.1.jar
cp/usr/local/hadoop-2.6.0/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-2.6.0.jar/usr/local/hbase-1.1.2/lib/
rm-rf/usr/local/hbase-1.1.2/lib/hadoop-mapreduce-client-jobclient-2.5.1.jar
cp/usr/local/hadoop-2.6.0/share/hadoop/mapreduce/hadoop-mapreduce-client-shuffle-2.6.0.jar/usr/local/hbase-1.1.2/lib/
rm-rf/usr/local/hbase-1.1.2/lib/hadoop-mapreduce-client-shuffle-2.5.1.jar
cp/usr/local/hadoop-2.6.0/share/hadoop/yarn/hadoop-yarn-api-2.6.0.jar/usr/local/hbase-1.1.2/lib/
rm-rf/usr/local/hbase-1.1.2/lib/hadoop-yarn-api-2.5.1.jar
cp/usr/local/hadoop-2.6.0/share/hadoop/yarn/hadoop-yarn-client-2.6.0.jar/usr/local/hbase-1.1.2/lib/
rm-rf/usr/local/hbase-1.1.2/lib/hadoop-yarn-client-2.5.1.jar
cp/usr/local/hadoop-2.6.0/share/hadoop/yarn/hadoop-yarn-common-2.6.0.jar/usr/local/hbase-1.1.2/lib/
rm-rf/usr/local/hbase-1.1.2/lib/hadoop-yarn-common-2.5.1.jar
cp/usr/local/hadoop-2.6.0/share/hadoop/yarn/hadoop-yarn-server-common-2.6.0.jar/usr/local/hbase-1.1.2/lib/
rm-rf/usr/local/hbase-1.1.2/lib/hadoop-yarn-server-common-2.5.1.jar
hbase安装
mkdir/usr/local/hbase-1.1.2/pids
/usr/local/hadoop-2.6.0/bin/hadoopfs-mkdirhdfs://master:9000/hbase
<configuration>
<property>
<name>hbase.rootdir</name>
<value>hdfs://master:9000/hbase</value>
</property>
<property>
<name>hbase.tmp.dir</name>
<value>/usr/local/hbase-1.1.2/tmp</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>master,slave1,slave2</value>
</property>
</configuration>
scp-r/usr/local/hbase-1.1.2root@slave1:/usr/local
scp-r/usr/local/hbase-1.1.2root@slave2:/usr/local
exportHBASE_HOME=/usr/local/hbase-1.1.2
exportPATH=$PATH:$HBASE_HOME/bin
scp/etc/profileroot@slave1:/etc/profile
scp/etc/profileroot@slave2:/etc/profile
scp/usr/local/hbase-1.1.2/conf/hbase-site.xmlroot@slave1:/usr/local/hbase-1.1.2/conf/hbase-site.xml
scp/usr/local/hbase-1.1.2/conf/hbase-site.xmlroot@slave2:/usr/local/hbase-1.1.2/conf/hbase-site.xml
/usr/local/hbase-1.1.2/bin/start-hbase.sh
/usr/local/hbase-1.1.2/bin/stop-hbase.sh
date-R
yuminstall-yntpdate
ntpdatetime.nist.gov
cp/usr/share/zoneinfo/Asia/Shanghai/etc/localtime
ntpdateus.pool.ntp.org
cp/usr/local/hadoop-2.6.0/share/hadoop/common/lib/htrace-core-3.0.4.jar/usr/local/hbase-1.1.2/lib
/usr/local/hbase-1.1.2/bin/hbaseshell
create'emp','emp_no','emp_name'
list
describe'emp'
put'emp','row_7369','emp_no','7369'
put'emp','row_7369','emp_name','Jay'
get'emp','row_7369'
启动
/usr/local/zookeeper-3.4.7/bin/zkServer.shstart
/usr/local/hadoop-2.6.0/sbin/start-all.sh
/usr/local/hbase-1.1.2/bin/start-hbase.sh
停止
/usr/local/hbase-1.1.2/bin/stop-hbase.sh
/usr/local/hadoop-2.6.0/sbin/stop-all.sh
/usr/local/zookeeper-3.4.7/bin/zkServer.shstop