[Java知识库]java 访问hivemeta及hivemeta安全防护

简单记录下java 访问hivemeta

pom.xml配置

无需hadoop依赖，配置hive相关依赖即可。

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<parent>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-parent</artifactId>
		<version>2.5.4</version>
	</parent>
	<groupId>hivemeta</groupId>
	<artifactId>hivemetaapi</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<name>hivemetaapi</name>
	<description>Demo project for Spring Boot and hivemeta api</description>
	<properties>
		<java.version>1.8</java.version>
		<hive.version>3.1.0</hive.version>
		<jackson.version>2.9.9</jackson.version>
		<commons-collections.version>3.2.2</commons-collections.version>

		<jackson.databind.version>2.10.0</jackson.databind.version>

	</properties>
	<dependencies>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter</artifactId>



		</dependency>

		<!-- <dependency> -->
		<!-- <groupId>org.springframework.boot</groupId> -->
		<!-- <artifactId>spring-boot-starter-jersey</artifactId> -->
		<!-- </dependency> -->

		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-test</artifactId>
			<scope>test</scope>
		</dependency>


		<dependency>
			<groupId>org.apache.hive</groupId>
			<artifactId>hive-metastore</artifactId>
			<version>${hive.version}</version>
			<!-- <scope>provided</scope> -->
			<exclusions>
				<exclusion>
					<groupId>org.mortbay.jetty</groupId>
					<artifactId>*</artifactId>
				</exclusion>
				<exclusion>
					<groupId>org.eclipse.jetty</groupId>
					<artifactId>*</artifactId>
				</exclusion>
			</exclusions>

		</dependency>

		<dependency>
			<groupId>org.apache.hive</groupId>
			<artifactId>hive-exec</artifactId>
			<version>${hive.version}</version>
			<!-- <scope>provided</scope> -->
			<exclusions>
				<exclusion>
					<groupId>javax.servlet</groupId>
					<artifactId>*</artifactId>
				</exclusion>
			</exclusions>
		</dependency>

		<dependency>
			<groupId>org.apache.hive</groupId>
			<artifactId>hive-jdbc</artifactId>
			<version>${hive.version}</version>
			<!-- <scope>test</scope> -->
			<exclusions>
				<exclusion>
					<groupId>javax.servlet</groupId>
					<artifactId>*</artifactId>
				</exclusion>
				<exclusion>
					<groupId>javax.ws.rs</groupId>
					<artifactId>*</artifactId>
				</exclusion>
				<exclusion>
					<groupId>org.eclipse.jetty</groupId>
					<artifactId>*</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
 

		<!-- 添加对应的 jackson-core -->
		<dependency>
			<groupId>com.fasterxml.jackson.core</groupId>
			<artifactId>jackson-core</artifactId>
		</dependency>

		<dependency>
			<groupId>com.fasterxml.jackson.core</groupId>
			<artifactId>jackson-databind</artifactId>
			<!-- <version>${jackson.databind.version}</version> -->
		</dependency>

		<dependency>
			<groupId>commons-collections</groupId>
			<artifactId>commons-collections</artifactId>
			<version>${commons-collections.version}</version>
		</dependency>


	</dependencies>

	<build>
		<plugins>
			<plugin>
				<groupId>org.springframework.boot</groupId>
				<artifactId>spring-boot-maven-plugin</artifactId>
			</plugin>
		</plugins>
	</build>

</project>

hive-site.xml

可以使用hive-site.xml来配置访问信息，这样更加灵活，下面的示意包括了sasl模式（未测试）。

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?><!--
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
   this work for additional information regarding copyright ownership.
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
   https://github.com/apache/hive/blob/master/data/conf/hive-site.xml
--><configuration>
<property><name>hive.metastore.uris</name><value>thrift://XXX.XXX.XX.XXX:9083</value></property>
<!--  begin of metaserver use kerberos -->

<!--  https://querysurge.zendesk.com/hc/en-us/articles/115001218863-Setting-Up-a-Hive-Connection-with-Kerberos-using-Apache-JDBC-Drivers-Windows-
	A principal has three parts: the primary, the instance, 
	and the realm. The format of a typical Kerberos V5 principal 
	is: primary/instance@REALM. Your principal for your gss-jaas.conf 
	may need both the primary and the instance: -->
 
<!-- <property><name>hive.metastore.kerberos.principal</name><value>hive-metastore/_HOST@EXAMPLE.COM</value></property> -->

<!-- <property><name>hive.metastore.sasl.enabled</name><value>true</value></property> -->
<!-- <property><name>hive.security.authorization.enabled</name><value>false</value></property> -->
<!-- <property><name>hive.metastore.execute.setugi</name><value>true</value></property> -->

 <!--  end of metaserver use kerberos -->
 
        	
        	 
 
<property><name>hive.metastore.schema.verification</name><value>false</value></property>


<property><name>hive.security.authorization.createtable.owner.grants</name><value>ALL</value></property>
</configuration>

java代码

核心入口代码： Hive hiveClient = Hive.get(hiveconf);

 try {
			Hive hiveClient = Hive.get(hiveconf);
			 
 			List<String> databaseNames = hiveClient.getAllDatabases();
			
			if (!CollectionUtils.isEmpty(databaseNames)) {
				LOG.info("Found {} databases", databaseNames.size());

				for (String databaseName : databaseNames) {
					  List<String> tableNames;

//					if (StringUtils.isEmpty(tblName)) {
 						tableNames = hiveClient.getAllTables(databaseName);
//					} else {
//						tableNames = hiveClient.getTablesByPattern(databaseName, tblName);
					//}
 						if (!CollectionUtils.isEmpty(tableNames)) {
 							for (String tableName : tableNames) {
 									Table table = hiveClient.getTable(databaseName, tableName);
 									
 									System.out.println("tablename:"+table.getCompleteName());
 							}
 						}
 			}
//				printTableInfo(hiveClient,"default","part_test1");
			 
			}	 
		} catch (HiveException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

判断分区是否存在，不存在则创建

Partition p =hiveClient.getPartition(table, partSpec, false);
		if (p==null)
			p=hiveClient.createPartition(table, partSpec);
			//.getPartition(table, partSpec, true);

partition 文件路径及分区变量值

Set<Partition> partitions = hiveClient.getAllPartitionsOf(table);
		partitions.forEach( partition->{
			
			List<String> values = partition.getValues();
			String va="";
			for(int i=0;i<values.size();i++)
				va+=values.get(i)+",";
			System.out.println("partition:"+partition.getDataLocation()+",values=" +va);
		});

openLDAP安全不生效

openLDAP安全对hivemeta server不肾虚， 不需要用户和密码访问，因此需要注意管控安全，避免端口暴露

metaserver端hive-site.xml有如下配置也没有用。

<property><name>hive.server2.authentication</name><value>LDAP</value></property>
<property><name>hive.server2.authentication.ldap.baseDN</name><value>ou=users,dc=xxxxx,dc=com</value></property>
<property><name>hive.server2.authentication.ldap.url</name><value>ldap://openldap</value></property>
<property><name>hive.server2.authentication.ldap.userDNPattern</name><value>cn=%s,ou=users,dc=xxxxx,dc=com</value></property>

关注博主即可阅读全文