HWADDR=00:0C:29:B3:AE:0E
TYPE=Ethernet
BOOTPROTO=static
DEFROUTE=yes
PEERDNS=yes
PEERROUTES=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_PEERDNS=yes
IPV6_PEERROUTES=yes
IPV6_FAILURE_FATAL=no
NAME=eno16777736
UUID=2cb8e76d-0626-4f8e-87e5-7e0743e4555f
ONBOOT=yes
IPADDR=192.168.10.186
NETMASK=255.255.255.0
GATEWAY=192.168.10.1
DNS1=192.168.10.1

修改主机映射:vi /etc/hostname
pc.apachevi /etc/hosts
192.168.1.186 pc.apache
192.168.1.187 pc.apache2
...
reboot #即可生效

配置免秘钥登入：很多公司都修改了ssh端口，使用 vi ~/.ssh/config 修改，  注意： config的最大使用权限600， chmod -R 600 config。 其配置为：
Host *
Port 你的端口0.安装lrzsz工具：
yum install -y lrzsz1.首先以root用户身份，修改：vim /etc/ssh/sshd_config
StrictModes no
RSAAuthentication yes
PubkeyAuthentication yes
AuthorizedKeysFile      .ssh/authorized_keys2.创建用户hadoop:
useradd hadoop
passwd hadoop3. 切换到hadoop用户：
su - hadoop4. 三台机器都生成证书：
ssh-keygen -t rsa5 每台机器的证书，通过如下命令导入到一个相同的文件。这样，authorized_keys文件中追加了三台机器各自生成的证书：
cat  id_rsa.pub >>  authorized_keys6.将包含三台机器证书的文件authorized_keys分发到三台机器的~/.ssh/authorized_keys目录下:
rz上传，sz下载7 然后把三台机器 .ssh/ 文件夹权限改为700，authorized_keys文件权限改为644：
chmod 700 ~/.ssh
chmod 644 ~/.ssh/authorized_keys

vi /etc/profile
export JAVA_HOME=/opt/software/jdk1.8.0_191
export PATH=$JAVA_HOME/bin:$PATH

source /etc/profile

java安装好后, 可使用jps。

zookeeper安装：进入目录 mkdir zkDatacp conf/zoo_sample.cfg conf/zoo.cfg$vim conf/zoo.cfgdataDir=/data/software/zookeeper-3.4.5/zkDataserver.1=pc1.hadoop:2888:3888server.2=pc2.hadoop:2888:3888server.3=pc3.hadoop:2888:3888#在三台机器上同样敲入 #pc1 : $vim zkData/myid 1         1 对应 server.1

Zookeeper + Hadoop HA:rm -rf /data/software/zookeeper-3.4.5/zkData/*
vim /data/software/zookeeper-3.4.5/zkData/myid
rm -rf /data/dataware/*zkServer.sh start
zkServer.sh status
zkServer.sh stop先清空其他节点的配置： rm -rf /data/software/hadoop-2.7.3/etc/*
scp -r hadoop/ hadoop@app-003:/data/software/hadoop-2.7.3/etc/第一次初始化：
先在各节点启动：  hadoop-daemon.sh start journalnode
在主节点启动：    hdfs namenode -format
将core-site.xml 中hadoop.tmp.dir的目录， 在主节点复制到第二个namenode节点：scp -r /data/dataware/hadoop/tmp/  hadoop@app-002:/data/dataware/hadoop/
在主节点启动：    hdfs zkfc -formatZK主节点启动： start-dfs.sh
#主节点启动： yarn-daemon.sh start resourcemanager
主节点启动： start-yarn.shhadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.3.jar pi 1 3第二次及以后启动hadoop时， 可直接启动dfs不需要再先启动 journalnode， 但是这就遇到一个问题， journalnode启动需要时间，只有其稳定后， namenode才能稳定。
否则会出现一直连不到app:8485的情况。解决： ipc.client.connect.max.retrie  20 ；   ipc.client.connect.retry.interval  5000主节点启动： stop-yarn.sh   stop-dfs.sh
各节点启动： zkServer.sh stop

#关闭防火墙
service iptables stop#关闭防火墙开机启动
chkconfig iptable　　s off#关闭linux安全模式
/etc/sysconfig/selinux#关闭centos7防火墙
systemctl stop firewalld.service # 关闭firewall
systemctl disable firewalld.service # 禁止firewall开机启动报错:
The authenticity of host 'XXXX' can't be established错误解决
vim /etc/ssh/ssh_config
最后面添加
StrictHostKeyChecking no
UserKnownHostsFile /dev/null无法连接MKS，  在任务管理器中打开所有VM服务进程bin/hdfs namenode -format
sbin/start-dfs.sh
sbin/start-yarn.shpc.apache:50070
pc.apache:8088hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.3.jar pi 1 3关闭方式:  除了使用 stop命令外， 实在没办法了可以使用:
killall java相关配置如下:vim hadoop-env.shexport JAVA_HOME=${JAVA_HOME}
export JAVA_HOME=/opt/software/jdk1.8.0_191vim core-site.xml<configuration><property><name>fs.defaultFS</name><value>hdfs://pc.apache:8020</value></property><property><name>hadoop.tmp.dir</name><value>/opt/software/hadoop-2.7.3/data</value></property>
</configuration>vim hdfs-site.xml<configuration><property><name>dfs.replication</name><value>1</value></property><property><name>dfs.permissions</name><value>false</value></property><property><name>dfs.namenode.name.dir</name><value>/opt/software/hadoop-2.7.3/data/name</value></property><property><name>dfs.webhdfs.enable</name><value>true</value></property><property><name>dfs.permissions.enable</name><value>false</value></property></configuration>vim mapred-site.xml<configuration><property><name>mapreduce.framework.name</name><value>yarn</value></property><!--指定jobhistory服务的主机及RPC端口号--><property><name>mapreduce.jobhistory.address</name><!--配置实际的主机名和端口--><value>pc.apache:10020</value></property><!--指定jobhistory服务的web访问的主机及RPC端口号--><property><name>mapreduce.jobhistory.webapp.address</name><value>pc.apache:19888</value></property></configuration>vim slavespc.apachevim yarn-site.xml<configuration><property><name>yarn.nodemanager.aux-services</name><value>mapreduce_shuffle</value></property><!-- 指定ResorceManager所在服务器的主机名--><property><name>yarn.resourcemanager.hostname</name><value>pc.apache</value></property><!--启用日志聚合功能--><property><name>yarn.log-aggregation-enable</name><value>true</value></property><!--日志保存时间--><property><name>yarn.log-aggregation.retain-seconds</name><value>86400</value></property></configuration>

完全卸载mysql：yum remove mysql-community mysql-community-server mysql-community-libs mysql-community-common -yyum -y remove mysql57-community-release-el7-10.noarchrpm -qa |grep -i mysql
rpm -ev MySQL-client-5.5.60-1.el7.x86_64  --nodepsfind / -name mysql   全部删除rpm -qa|grep mysql
rpm -ev mysql57-community-release-el7-8.noarch

安装MySQL:wget http://dev.mysql.com/get/mysql57-community-release-el7-8.noarch.rpmyum localinstall mysql57-community-release-el7-8.noarch.rpmyum repolist enabled | grep "mysql.*-community.*"yum install mysql-community-serversystemctl start mysqld
systemctl status mysqld
systemctl enable mysqld
systemctl daemon-reload首次安装，获取临时密码:
grep 'temporary password' /var/log/mysqld.log
mysql -uroot -p    #即可登入但是，在登入之前最好设置密码规则， 便于修改
vim /etc/my.cnf
validate_password = offsystemctl restart mysqld   重启服务
ALTER USER 'root'@'localhost' IDENTIFIED BY '111111';   #修改密码grant all privileges on *.* to 'root'@'%' identified by '111111';  #给其他用户权限
flush privileges;在/etc/my.cnf 配置编码
[mysqld]
character_set_server=utf8
init_connect='SET NAMES utf8'

create user 'hive'@'localhost' identified by 'hive';
create database hive;
alter database hive character set latin1;grant all on hive.* to hive@'%'  identified by 'hive';
grant all on hive.* to hive@'localhost'  identified by 'hive';
grant all on metastore.* to hive@'localhost'  identified by 'hive';
grant all on metastore.* to hive@'%'  identified by 'hive';show grants for hive@'localhost';
flush privileges; 如果重装HIVE:  删除Hive在MySQL的元数据， 如下。drop database metastore;select * from metastore.SDS;
select * from metastore.DBS;
delete from `metastore`.`TABLE_PARAMS`
drop table  `metastore`.`TABLE_PARAMS`
delete from `metastore`.`TBLS`
drop table `metastore`.`TBLS`delete from  metastore.SDS
delete from  metastore.DBS
drop table  metastore.SDS
drop table metastore.DBS

下载并解压好hive项目cp hive-default.xml.template hive-site.xml
cp hive-env.sh.template hive-env.sh
cp hive-log4j2.properties.template hive-log4j2.propertiesvim hive-site.xml<configuration>
<property><name>hive.cli.print.header</name><value>true</value><description>Whether to print the names of the columns in query output.</description>
</property><property><name>hive.cli.print.current.db</name><value>true</value><description>Whether to include the current database in the Hive prompt.</description>
</property><property><name>javax.jdo.option.ConnectionURL</name><value>jdbc:mysql://pc1.hadoop:3306/metastore?createDatabaseIfNotExist=true</value><description>JDBC connect string for a JDBC metastore</description>
</property><property><name>javax.jdo.option.ConnectionDriverName</name><value>com.mysql.jdbc.Driver</value><description>Driver class name for a JDBC metastore</description>
</property><property><name>javax.jdo.option.ConnectionUserName</name><value>hive</value><description>username to use against metastore database</description>
</property><property><name>javax.jdo.option.ConnectionPassword</name><value>hive</value><description>password to use against metastore database</description>
</property><property>
<name>hive.server2.long.polling.timeout</name>
<value>5000</value>
</property><property>
<name>hive.server2.thrift.port</name>
<value>10001</value><!-- 因为Spark-sql服务端口是10000， 避免冲突，这里改成10001-->
</property><property>
<name>hive.server2.thrift.bind.host</name>
<value>pc1.hadoop</value>
</property></configuration>vim hive-env.shexport JAVA_HOME=/opt/software/jdk1.8.0_191
export HADOOP_HOME=/opt/software/hadoop-2.7.3/etc/hadoop
export HIVE_CONF_DIR=/opt/software/hive-2.1.1/confvim hive-log4j2.propertieshive.log.dir=/                # 配置一下目录地址

在Hive目录下， 初始化数据库。报错:  Host is not allowed to connect to this MySQL server
use mysql;
update user set host = '%' where user = 'root';
FLUSH PRIVILEGES;schematool -dbType mysql -initSchema

将HIVE_HOME添加到环境变量， 执行hive后， 报错： Relative path in absolute URI在hive-site.xml中 找到所有${system:java.io.tmpdir}   替换成 ./hive/logs/iotemp

再次运行hive

启动hive服务hive --service metastore &hive --service hiveserver2 &

第一次启动spark:hadoop fs -put /data/software/spark-2.1.1/jars/* /user/spark/libs/start-master.sh
start-slaves.sh

第一步 下载一个scala:
https://www.scala-lang.org/download/2.11.8.html      #将scala添加进环境变量即可;第二步 解压spark-2.2.0-bin-hadoop2.7.tgz,  并将spark添加环境变量;vim spark-env.shexport JAVA_HOME=/opt/software/jdk1.8.0_191export HADOOP_CONF_DIR=/opt/software/hadoop-2.7.3export HIVE_CONF_DIR=/opt/software/hive-2.1.1export SCALA_HOME=/opt/software/scala-2.11.8export SPARK_WORK_MEMORY=1gexport MASTER=spark://pc.apache:7077由于Spark-SQL需要用到Hive数据源, 因此需要修改Hive中的hive-site.xml<property><name>hive.metastore.uris</name><value>thrift://pc.apache:9083</value></property>修改好后， 将其复制到spark/conf 目录下
cp hive-site.xml /opt/software/spark-2.2.0-bin-hadoop2.7/conf/复制依赖的jars:
cp $HIVE_HOME/lib/hive-hbase-handler-2.1.1.jar $SPARK_HOME/jars/mkdir $SPARK_HOME/lib
cp $HIVE_HOME/lib/mysql-connector-java-5.1.34.jar $SPARK_HOME/lib/
cp $HIVE_HOME/lib/metrics-core-2.2.0.jar $SPARK_HOME/libcp $HBASE_HOME/lib/guava-12.0.1.jar $SPARK_HOME/lib/
cp $HBASE_HOME/lib/hbase-common-1.2.5-tests.jar $SPARK_HOME/lib/
cp $HBASE_HOME/lib/hbase-client-1.2.5.jar $SPARK_HOME/lib/
cp $HBASE_HOME/lib/hbase-protocol-1.2.5.jar $SPARK_HOME/lib/
cp $HBASE_HOME/lib/htrace-core-3.1.0-incubating.jar $SPARK_HOME/lib/
cp $HBASE_HOME/lib/hbase-common-1.2.5.jar $SPARK_HOME/lib/
cp $HBASE_HOME/lib/hbase-server-1.2.5.jar $SPARK_HOME/lib/将上述环境变量添加进 vim $SPARK_HOME/conf/spark-env.shexport SPARK_CLASSPATH=$SPARK_CLASSPATH:$SPARK_HOME/lib/guava-12.0.1.jarexport SPARK_CLASSPATH=$SPARK_CLASSPATH:$SPARK_HOME/lib/hbase-client-1.2.5.jarexport SPARK_CLASSPATH=$SPARK_CLASSPATH:$SPARK_HOME/lib/hbase-common-1.2.5.jarexport SPARK_CLASSPATH=$SPARK_CLASSPATH:$SPARK_HOME/lib/hbase-common-1.2.5-tests.jarexport SPARK_CLASSPATH=$SPARK_CLASSPATH:$SPARK_HOME/lib/hbase-protocol-1.2.5.jarexport SPARK_CLASSPATH=$SPARK_CLASSPATH:$SPARK_HOME/lib/hbase-server-1.2.5.jarexport SPARK_CLASSPATH=$SPARK_CLASSPATH:$SPARK_HOME/lib/htrace-core-3.1.0-incubating.jarexport SPARK_CLASSPATH=$SPARK_CLASSPATH:$SPARK_HOME/lib/mysql-connector-java-5.1.34.jarexport SPARK_CLASSPATH=$SPARK_CLASSPATH:$SPARK_HOME/lib/metrics-core-2.2.0.jar启动hive数据源， 测试spark-sql
nohup hive --service metastore >/opt/software/metastore.log 2>&1 &

启动spark/opt/software/spark-2.2.0-bin-hadoop2.7/sbin/start-master.sh
/opt/software/spark-2.2.0-bin-hadoop2.7/sbin/start-slaves.shweb: http://192.168.1.186:8080/

启动spark-sql时报错:  Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient将hive.site.xml文件中的<property>  <name>hive.metastore.schema.verification</name>  <value>true</value></property>改为false即可

但是spark-sql服务却起不来, 执行$SPARK_HOME/sbin/start-thriftserver.sh 时报错: Could not create ServerSocket on address pc.apache/192.168.1.186:10001使用 jps -ml查看java进程详情;

原因是 HIVE的服务与SPARK-SQL服务只能起一个？  我把HIVE服务注释掉就可以了，  但是这样是不妥的， 可能要把HIVE服务设置成什么其他的端口避免10001？ 有待进一步测试！

spark-shell --master yarn-client启动后会发现报错: Error initializing SparkContextvim yarn-site.xml<property><name>yarn.nodemanager.vmem-check-enabled</name><value>false</value><description>Whether virtual memory limits will be enforced for containers</description></property><property><name>yarn.nodemanager.vmem-pmem-ratio</name><value>4</value><description>Ratio between virtual memory to physical memory when setting memory limits for containers</description></property>

node1     node2   node3
nn1       nn2     dn3
dn1       dn2     nm3
rm1       rm2     zk3
nm1       nm2     mysql
zk1       zk2
hivestat  hivserv hivemeta主节点启动： start-dfs.sh
#主节点启动： yarn-daemon.sh start resourcemanager
主节点启动： start-yarn.sh
stop-yarn.sh   stop-dfs.shhive --service metastore > /home/hadoop/hive.meta  &
hive --service hiveserver2 > /home/hadoop/hive.log &#hadoop fs -mkdir -p /user/spark/libs/
#hadoop fs -put /data/software/spark-2.1.1/jars/* /user/spark/libs/
hadoop fs -mkdir -p /tmp/spark/logs/start-master.sh
start-slaves.shzkCli.sh
rm -rf /data/software/spark-2.1.1/conf/
scp -r /data/software/spark-2.1.1/conf/ hadoop@app-002:/data/software/spark-2.1.1/Yarn运行日志：
/tmp/logs/hadoop/logs提交任务做测试：hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.3.jar pi 1 3spark-shell --master yarn --deploy-mode clientspark-submit --class org.apache.spark.examples.SparkPi \
--master yarn \
--deploy-mode cluster \
--driver-memory 1000m \
--executor-memory 1000m \
--executor-cores 1 \
/data/software/spark-2.1.1/examples/jars/spark-examples_2.11-2.1.1.jar \
3

debug:nohup java -jar sent-mail.jar > log.txt &查看端口是否占用：netstat -ntulp |grep 8020
查看一个服务有多少端口：ps -ef |grep mysqldrm -rf /data/software/hadoop-2.7.3/logs/*单独启动各个组件， 查看bug产生原因。
hadoop-daemon.sh start namenode
hadoop-daemon.sh start datanodejps -ml
kill -9

vim hbase-env.shexport JAVA_HOME=/opt/software/jdk1.8.0_191
export HBASE_MANAGES_ZK=true
export HADOOP_HOME=/opt/software/hadoop-2.7.3
export HBASE_CLASSPATH=/opt/software/hadoop-2.7.3/etc/hadoop
export HBASE_PID_DIR=/opt/software/hbase-1.2.5/pidsvim hbase-site.xml

<property>
<name>hbase.rootdir</name>
<value>hdfs://pc.apache:8020/hbase</value>
</property>

<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>

<property>
<name>hbase.zookeeper.quorum</name>
<value>pc.apache</value>
</property>
<property>
<name>hbase.master</name>
<value>hdfs://pc.apache:60000</value>
</property>

<property>
<name>hbase.tmp.dir</name>
<value>/opt/software/hbase-1.2.5/tmp</value>
</property>

<property>
<name>hbase.zookeeper.property.dataDir</name>
<value>/opt/software/hbase-1.2.5/zooData</value>
</property>

vim regionserverspc.apache

Hbase window最简单安装版本下载：hbase-1.2.3。（http://apache.fayea.com/hbase/stable/）必须： Java_Home,  Hadoop_Home1.
修改 hbase-1.0.2\conf\hbase-env.cmd 文件set JAVA_HOME=C:\Program Files\Java\jdk1.8.0_05
set HBASE_MANAGES_ZK=flase 2.
修改 hbase-1.0.2\conf\hbase-env.sh 文件export HBASE_MANAGES_ZK=false3.
修改hbase-1.0.2\conf\hbase-site.xml 文件 路径自己改为自己的实际路径<configuration><property><name>hbase.rootdir</name><value>file:///E:/software/hbase-1.4.10/root</value></property><property>
<name>hbase.tmp.dir</name><value>E:/software/hbase-1.4.10/tmp</value></property><property>
<name>hbase.zookeeper.quorum</name>
<value>127.0.0.1</value></property><property>
<name>hbase.zookeeper.property.dataDir</name>
<value>E:/software/hbase-1.4.10/zoo</value></property><property>
<name>hbase.cluster.distributed</name>
<value>false</value></property>
</configuration>4.进入到bin目录
点击 start-hbase.cmd在该目录下执行命令窗口 ：
hbase shellcreate 'test','cf'
scan 'test'