实验0-Hive配置与启动

实验目的

掌握Hive配置与启动的方法。

实验内容

预置环境：1.1.1 HDFS、1.1.2. zookeeper和1.1.3 yarn均正常启动。

将文件上传hdfs

hdfs dfs -mkdir -p /user/root/tez/
hdfs dfs -put /usr/hdp/3.0.1.0-187/tez/lib/tez.tar.gz /user/root/tez/

复制jar包

cp -r /usr/hdp/3.0.1.0-187/tez/.jar /usr/hdp/3.0.1.0-187/hadoop/lib/
cp -r /usr/hdp/3.0.1.0-187/tez/.jar /usr/hdp/3.0.1.0-187/hive/lib/
cp -r /usr/hdp/3.0.1.0-187/tez/lib/* /usr/hdp/3.0.1.0-187/hive/lib/
cp -r /usr/hdp/3.0.1.0-187/tez/lib/* /usr/hdp/3.0.1.0-187/hadoop/lib/

删除重复的guava包

rm -f /usr/hdp/3.0.1.0-187/hive/lib/guava-11.0.2.jar
rm -f /usr/hdp/3.0.1.0-187/hadoop/lib/guava-11.0.2.jar

编辑tez-site.xml文件

vim /etc/tez/conf/tez-site.xml

注意下面的IP需要更改为主节点IP

<?xml version="1.0"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> tez.lib.uris hdfs://10.244.11.74:8020/user/root/tez/tez.tar.gz tez.use.cluster.hadoop-libs true tez.runtime.compress false tez.runtime.compress.codec org.apache.hadoop.io.compress.SnappyCodec

<property><name>tez.am.am-rm.heartbeat.interval-ms.max</name><value>250</value>
</property><property><name>tez.am.container.idle.release-timeout-max.millis</name><value>20000</value>
</property><property><name>tez.am.container.idle.release-timeout-min.millis</name><value>10000</value>
</property><property><name>tez.am.container.reuse.enabled</name><value>true</value>
</property><property><name>tez.am.container.reuse.locality.delay-allocation-millis</name><value>250</value>
</property><property><name>tez.am.container.reuse.non-local-fallback.enabled</name><value>false</value>
</property><property><name>tez.am.container.reuse.rack-fallback.enabled</name><value>true</value>
</property><property><name>tez.am.java.opts</name><value>-server -Xmx819m -Djava.net.preferIPv4Stack=true</value>
</property><property><name>tez.am.max.app.attempts</name><value>2</value>
</property><property><name>tez.am.maxtaskfailures.per.node</name><value>10</value>
</property><property><name>tez.am.resource.memory.mb</name><value>2048</value>
</property><property><name>tez.counters.max</name><value>10000</value>
</property><property><name>tez.counters.max.groups</name><value>3000</value>
</property><property><name>tez.generate.debug.artifacts</name><value>false</value>
</property><property><name>tez.grouping.max-size</name><value>1073741824</value>
</property><property><name>tez.grouping.min-size</name><value>16777216</value>
</property><property><name>tez.grouping.split-waves</name><value>1.7</value>
</property><property><name>tez.runtime.io.sort.mb</name><value>270</value>
</property><property><name>tez.runtime.optimize.local.fetch</name><value>true</value>
</property><property><name>tez.runtime.pipelined.sorter.sort.threads</name><value>2</value>
</property><property><name>tez.runtime.shuffle.fetch.buffer.percent</name><value>0.6</value>
</property><property><name>tez.runtime.shuffle.memory.limit.percent</name><value>0.25</value>
</property><property><name>tez.runtime.sorter.class</name><value>PIPELINED</value>
</property><property><name>tez.runtime.unordered.output.buffer.size-mb</name><value>76</value>
</property><property><name>tez.session.am.dag.submit.timeout.secs</name><value>600</value>
</property><property><name>tez.session.client.timeout.secs</name><value>-1</value>
</property><property><name>tez.shuffle-vertex-manager.max-src-fraction</name><value>0.4</value>
</property><property><name>tez.shuffle-vertex-manager.min-src-fraction</name><value>0.2</value>
</property>
<property><name>tez.task.max-events-per-heartbeat</name><value>500</value>
</property><property><name>tez.task.resource.memory.mb</name><value>1024</value>
</property>

编辑hive-site.xml文件

先备份原来的配置文件

mv /etc/hive/conf/hive-site.xml /etc/hive/conf/hive-site.xml.bak
vim /etc/hive/conf/hive-site.xml

<?xml version="1.0"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> javax.jdo.option.ConnectionURL jdbc:mysql://127.0.0.1:3306/metastore?createDatabaseIfNotExist=true JDBC connect string for a JDBC metastore javax.jdo.option.ConnectionDriverName com.mysql.jdbc.Driver Driver class name for a JDBC metastore javax.jdo.option.ConnectionUserName root username to use against metastore database javax.jdo.option.ConnectionPassword 123456 password to use against metastore database hive.metastore.schema.verification false hive.metastore.warehouse.dir /user/hive/warehouse hive.execution.engine tez datanucleus.autoStartMechanism SchemaTable hive.tez.container.size 2048

    <property><name>hive.tez.java.opts</name><value>-server -Xmx2048m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC</value></property><property><name>hive.server2.tez.initialize.default.sessions</name><value>false</value></property><property><name>hive.server2.tez.default.queues</name><value>default</value></property><property><name>hive.tez.input.format</name><value>org.apache.hadoop.hive.ql.io.HiveInputFormat</value></property><property><name>hive.server2.tez.sessions.per.default.queue</name><value>1</value></property>

启动hive

在主节点上启动Hive

nohup hive --service metastore &
nohup hive --service hiveserver2 &

进入

beeline

连接hive server2的jdbc地址

!connect jdbc:hive2://127.0.0.1:10000
#输入用户名root
#输入密码:123456

CREATE TABLE students (name VARCHAR(64), age INT, gpa DECIMAL(3,2));
INSERT INTO TABLE students VALUES (‘tom’, 35, 1.28);

INSERT INTO TABLE students VALUES (‘cat’, 32, 2.32);
#时间稍微有点长，请慢慢等待，或者先学其他内容

select * from students ;
#确认插入记录成功

!quit
#退出