tonglin0325的个人主页

Hive学习笔记——metastore listener

除了使用hive hook来记录hive上用户的操作之外,还可以使用hive metastore listener来进行记录,参考:

1
2
https://towardsdatascience.com/apache-hive-hooks-and-metastore-listeners-a-tale-of-your-metadata-903b751ee99f

hive metastore的接口有3种,分别是

 

Property Abstract class
hive.metastore.pre.event.listeners org.apache.hadoop.hive.metastore.MetaStorePreEventListener
hive.metastore.end.function.listeners org.apache.hadoop.hive.metastore.MetaStoreEndFunctionListener
hive.metastore.event.listeners org.apache.hadoop.hive.metastore.MetaStoreEventListener

下面的代码中继承了MetaStoreEventListener,实现的功能是在建表和修改表的时候,日志输出一下table的元数据信息

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
package com.bigdata.hive;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;

public class MyListener extends MetaStoreEventListener {

private static final Logger LOGGER = LoggerFactory.getLogger(MyListener.class);
private static final ObjectMapper objMapper = new ObjectMapper();

public MyListener(Configuration config) {
super(config);
logWithHeader(" created ");
}

@Override
public void onCreateTable(CreateTableEvent event) {
logWithHeader(event.getTable());
}

@Override
public void onAlterTable(AlterTableEvent event) {
logWithHeader(event.getOldTable());
logWithHeader(event.getNewTable());
}

private void logWithHeader(Object obj) {
LOGGER.info("[CustomListener][Thread: " + Thread.currentThread().getName() + "] | " + objToStr(obj));
}

private String objToStr(Object obj) {
try {
return objMapper.writeValueAsString(obj);
} catch (IOException e) {
LOGGER.error("Error on conversion", e);
}
return null;
}

}

进行打包

1
2
mvn clean package

将jar包放到/var/lib/hive目录下,并修改用户组

1
2
3
lintong@master:/var/lib/hive$ ls -al| grep jar
-rw-r--r-- 1 hive hive 363002613 12月 26 16:30 bigdata-1.0-SNAPSHOT-jar-with-dependencies.jar

在CDH界面上配置metastore listener,注意是在metastore的配置项中,而不是hiveserver2的

 

 

然后重启hive

之后在HUE上运行建表语句

1
2
create table test_table2(id int,name string)

查看hive metastore的日志,可以看到日志中打印了所创建的hive表的元数据信息

1
2
3
4
lintong@master:/var/log/hive$ tail -f hadoop-cmf-hive-HIVEMETASTORE-master.log.out | grep Listen
2021-12-26 21:43:53,737 INFO com.bigdata.hive.MyListener: [main]: [CustomListener][Thread: main] | " created "
2021-12-26 21:45:15,224 INFO com.bigdata.hive.MyListener: [pool-5-thread-14]: [CustomListener][Thread: pool-5-thread-14] | {"tableName":"CM_TEST_TABLE","dbName":"cloudera_manager_metastore_canary_test_db_hive_hivemetastore_0a5c4e82140edb166b3b3c29b6817024","owner":null,"createTime":1640526315,"lastAccessTime":0,"retention":0,"sd":{"cols":[{"name":"s","type":"string","comment":"test string","setType":true,"setComment":true,"setName":true},{"name":"f","type":"float","comment":"test float","setType":true,"setComment":true,"setName":true},{"name":"a","type":"array<map<string,struct<p1:int,p2:int>>>","comment":"test complex type","setType":true,"setComment":true,"setName":true}],"location":"hdfs://master:8020/user/hue/.cloudera_manager_hive_metastore_canary/hive_HIVEMETASTORE_0a5c4e82140edb166b3b3c29b6817024/cm_test_table","inputFormat":null,"outputFormat":null,"compressed":false,"numBuckets":1,"serdeInfo":{"name":"CM_TEST_TABLE","serializationLib":null,"parameters":{"serialization.format":"1"},"setSerializationLib":false,"setParameters":true,"parametersSize":1,"setName":true},"bucketCols":[],"sortCols":[],"parameters":{},"skewedInfo":null,"storedAsSubDirectories":false,"setParameters":true,"setSkewedInfo":false,"colsSize":3,"colsIterator":[{"name":"s","type":"string","comment":"test string","setType":true,"setComment":true,"setName":true},{"name":"f","type":"float","comment":"test float","setType":true,"setComment":true,"setName":true},{"name":"a","type":"array<map<string,struct<p1:int,p2:int>>>","comment":"test complex type","setType":true,"setComment":true,"setName":true}],"setCols":true,"setLocation":true,"setInputFormat":false,"setOutputFormat":false,"setCompressed":true,"setNumBuckets":true,"setSerdeInfo":true,"bucketColsSize":0,"bucketColsIterator":[],"setBucketCols":true,"sortColsSize":0,"sortColsIterator":[],"setSortCols":true,"setStoredAsSubDirectories":false,"parametersSize":0},"partitionKeys":[{"name":"p1","type":"string","comment":"partition-key-1","setType":true,"setComment":true,"setName":true},{"name":"p2","type":"int","comment":"partition-key-2","setType":true,"setComment":true,"setName":true}],"parameters":{"transient_lastDdlTime":"1640526315"},"viewOriginalText":null,"viewExpandedText":null,"tableType":null,"privileges":null,"temporary":false,"ownerType":"USER","partitionKeysSize":2,"setParameters":true,"setTableType":false,"setTableName":true,"setOwner":false,"setRetention":true,"partitionKeysIterator":[{"name":"p1","type":"string","comment":"partition-key-1","setType":true,"setComment":true,"setName":true},{"name":"p2","type":"int","comment":"partition-key-2","setType":true,"setComment":true,"setName":true}],"setPartitionKeys":true,"setViewOriginalText":false,"setViewExpandedText":false,"setPrivileges":false,"setTemporary":false,"setOwnerType":true,"setDbName":true,"setCreateTime":true,"setLastAccessTime":true,"setSd":true,"parametersSize":1}