centos7安装hadoop、hbase、zookeeper

#system config
#systemctl stop firewalld & systemctl disable firewalld
#hostnamectl set-hostname server1 (must reconnect ssh for SecureCRT)
#chmod +x /etc/rc.d/rc.local

#configure hosts
echo '200.200.200.51 server1' >> /etc/hosts
echo '200.200.200.52 server2' >> /etc/hosts
echo '200.200.200.53 server3' >> /etc/hosts

#configure server1 no passwd ssh to server1,server2,server3
ssh-keygen -t rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
ssh server2 'mkdir -m 700 ~/.ssh'
cat ~/.ssh/id_rsa.pub | ssh server2 'cat - >> ~/.ssh/authorized_keys'
ssh server3 'mkdir -m 700 ~/.ssh'
cat ~/.ssh/id_rsa.pub | ssh server3 'cat - >> ~/.ssh/authorized_keys'

echo '' >> /etc/security/limits.conf
echo '* soft nofile 1048576' >> /etc/security/limits.conf
echo '* hard nofile 1048576' >> /etc/security/limits.conf

#install jdk
tar -zxvf /opt/software/jdk-8u131-linux-x64.tar.gz -C /usr/local/
echo '' >> /etc/profile
echo 'export JAVA_HOME=/usr/local/jdk1.8.0_131' >> /etc/profile
echo 'export PATH=/usr/local/hadoop/bin:/usr/local/hbase/bin:/usr/local/zookeeper/bin:$JAVA_HOME/bin:$PATH' >> /etc/profile
source /etc/profile

#install hadoop
tar -zxvf /opt/software/hadoop-2.7.3.tar.gz -C /usr/local/
mv /usr/local/hadoop-2.7.3 /usr/local/hadoop

#core-site:xml

<configuration>
<property>
    <name>fs.defaultFS</name>
    <value>hdfs://server1:9000</value>
</property>
<property>
    <name>hadoop.tmp.dir</name>
    <value>/usr/local/hadoop/tmp</value>
</property>
<property>
    <name>io.file.buffer.size</name>
    <value>131702</value>
</property>
</configuration>

#hdfs-site.xml

<configuration>
<property>
    <name>dfs.replication</name>
    <value>3</value>
</property>
<property>
    <name>dfs.namenode.name.dir</name>
    <value>/usr/local/hadoop/dfs/name</value>
</property>
<property>
    <name>dfs.datanode.data.dir</name>
    <value>/usr/local/hadoop/dfs/data</value>
</property>
<property>
    <name>dfs.namenode.secondary.http-address</name>
    <value>server1:9001</value>
</property>
<property>
    <name>dfs.webhdfs.enabled</name>
    <value>true</value>
</property>
</configuration>

#mapred-site.xml

cp mapred-site.xml.template mapred-site.xml
<configuration>
<property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
</property>
<property>
    <name>mapreduce.jobhistory.address</name>
    <value>server1:10020</value>
</property>
<property>
    <name>mapreduce.jobhistory.webapp.address</name>
    <value>server1:19888</value>
</property>
</configuration>

#yarn-site.xml

<configuration>
<property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
</property>
<property>
    <name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
    <name>yarn.resourcemanager.address</name>
    <value>server1:8032</value>
</property>
<property>
    <name>yarn.resourcemanager.scheduler.address</name>
    <value>server1:8030</value>
</property>
<property>
    <name>yarn.resourcemanager.resource-tracker.address</name>
    <value>server1:8031</value>
</property>
<property>
    <name>yarn.resourcemanager.admin.address</name>
    <value>server1:8033</value>
</property>
<property>
    <name>yarn.resourcemanager.webapp.address</name>
    <value>server1:8088</value>
</property>
<property>
    <name>yarn.nodemanager.resource.memory-mb</name>
    <value>32768</value>
</property>
</configuration>

#hadoop-env.sh
export JAVA_HOME=/usr/local/jdk1.8.0_131
export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/local/hbase/lib/*

#yarn-env.sh
export JAVA_HOME=/usr/local/jdk1.8.0_131

#slaves
server1
server2
server3

#copy hadoop to server2,server3
scp -r /usr/local/hadoop server2:/usr/local/
scp -r /usr/local/hadoop server3:/usr/local/

#format namenode
/usr/local/hadoop/bin/hdfs namenode -format

#start hadoop(hdfs,yarn)
/usr/local/hadoop/sbin/start-all.sh

#stop hadoop(hdfs,yarn)
#/usr/local/hadoop/sbin/stop-all.sh

#install zookeeper
tar -zxvf /opt/software/zookeeper-3.4.10.tar.gz -C /usr/local/
mv /usr/local/zookeeper-3.4.10 /usr/local/zookeeper
mkdir /usr/local/zookeeper/data
cp zoo_sample.cfg zoo.cfg

#conf/zoo.cfg
dataDir=/usr/local/zookeeper/data
server.1=server1:7000:7001
server.2=server2:7000:7001
server.3=server3:7000:7001

#bin/zkEnv.sh
ZOO_LOG_DIR="/usr/local/zookeeper/logs"

#copy zookeeper to server2,server3
scp -r /usr/local/zookeeper server2:/usr/local/
scp -r /usr/local/zookeeper server3:/usr/local/

#myid
echo 1 > /usr/local/zookeeper/data/myid
ssh server2 'echo 2 > /usr/local/zookeeper/data/myid'
ssh server3 'echo 3 > /usr/local/zookeeper/data/myid'

#start zookeeper,must ssh to server2,server3 execute this cmd
/usr/local/zookeeper/bin/zkServer.sh start

#stop zookeeper,must ssh to server2,server3 execute this cmd
#/usr/local/zookeeper/bin/zkServer.sh stop

#install hbase
tar -zxvf /opt/software/hbase-1.2.5-bin.tar.gz -C /usr/local/
mv /usr/local/hbase-1.2.5 /usr/local/hbase

#conf/hbase-env.sh
export JAVA_HOME=/usr/local/jdk1.8.0_131
export HBASE_MANAGES_ZK=false

#conf/hbase-site.xml

<configuration>
<property>
    <name>hbase.rootdir</name>
    <value>hdfs://server1:9000/hbase</value>
</property>
<property>
    <name>hbase.cluster.distributed</name>
    <value>true</value>
</property>
<property>
    <name>hbase.tmp.dir</name>
    <value>/usr/local/hbase/tmp</value>
</property>
<property>
    <name>hbase.zookeeper.quorum</name>
    <value>server1,server2,server3</value>
</property>
<property>
    <name>hbase.zookeeper.property.dataDir</name>
    <value>/usr/local/hbase/tmp/zookeeper</value>
</property>
<property>
    <name>zookeeper.session.timeout</name>
    <value>90000</value>
</property>
<property>
    <name>hbase.regionserver.restart.on.zk.expire</name>
    <value>true</value>
</property>
</configuration>

#conf/regionservers
server1
server2
server3

#copy hbase to server2,server3
scp -r /usr/local/hbase server2:/usr/local/
scp -r /usr/local/hbase server3:/usr/local/

#start hbase
/usr/local/hbase/bin/start-hbase.sh

#stop hbase
#/usr/local/hbase/bin/stop-hbase.sh

标签: centos7安装hadoop、hbase、zookeeper

添加新评论