|
今天试了一下hbase的 Scan 和 FilterList, 看多个filter 之间是and还是or逻辑, 目前发现是and逻辑, 虽然网上多种资料说scan+FilterList会很慢, 但是也不妨碍我们利用 关系型数据库的box过滤方式, 实验证明是可行的, BOX Filter 过滤得写法如下(下面例子是查询得包含关系):
Scan scan1 = new Scan();
FilterList filters = new FilterList();
filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("GEOMETRY"), Bytes.toBytes("XMIN"),
CompareOperator.GREATER, Bytes.toBytes(xmin)));
filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("GEOMETRY"), Bytes.toBytes("YMIN"),
CompareOperator.GREATER, Bytes.toBytes(ymin)));
filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("GEOMETRY"), Bytes.toBytes("XMAX"),
CompareOperator.LESS, Bytes.toBytes(xmax)));
filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("GEOMETRY"), Bytes.toBytes("YMAX"),
CompareOperator.LESS, Bytes.toBytes(ymax)));
scan1.setFilter(filters);
//在行上获取遍历器
//int caching = 100;
//scan1.setCaching(caching );
//scan1.setBatch(5);
ResultScanner scanner1 = table.getScanner(scan1);
box过滤之后然后可以使用jts做或者内核做几何精确过滤,? 可以达到同样空间数据库得基本空间查询效果.? 这里下一步试试这样得效率,? 如果不行(大概率是大数据就会慢), 然后准备设计一种rowkey, 查询得时候scan给startrowkey和endrowkey,? 然后再加上filter .? 这里提前说明一下startrowkey和endrowkey一定会过多得返回数据, 就看box filter 能不能在有start和endkey得时候加速效果. 以下是完整得代码
package hbasedatabase;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.crypto.Mac;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.Cell.Type;
import org.apache.hadoop.hbase.CellBuilder;
import org.apache.hadoop.hbase.CellBuilderFactory;
import org.apache.hadoop.hbase.CellBuilderType;
import org.apache.hadoop.hbase.CompareOperator;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.util.Bytes;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryFactory;
import org.locationtech.jts.io.ParseException;
import org.locationtech.jts.io.WKBReader;
import org.locationtech.jts.io.WKBWriter;
import org.locationtech.jts.io.WKTReader;
import org.locationtech.jts.io.WKTWriter;
import jnr.ffi.Struct.int16_t;
public class testSpatialQuery {
private static final String TABLE_NAME = "ASDFG";
static Configuration conf = null;
static TableName tableName = null;
static Table table = null;
static Connection connection = null;
static Admin admin = null;
//测试数据
static String familygeos[] = {"GEOMETRY","INFO"};
static String family1[] = {"GEOMETRY","XMIN","YMIN","XMAX","YMAX"};
static String family1FieldsType[] = {"geometry","double","double","double","double"};
static String family2[] = { "OID","NAME","LENGTH"};
static String family2FieldsType[] = { "long","string","double"};
static HashMap<String,HashMap<String,String>> map = new HashMap<String,HashMap<String,String>>();
static String polygon1 = "POLYGON((113.4854749254902 23.41916473613878,113.4854365755001 23.41911314486015,113.485408828651 23.4190774788114,113.485424543505 23.4190619093373,113.485437580626 23.41905590278759,113.485467041062 23.41905302022758,113.4854947123682 23.41905762592069,113.4855127324191 23.41905317072354,113.485522569163 23.41903956918361,113.485519427731 23.41901698699501,113.485513058565 23.41896884002222,113.48549513176 23.41894918873771,113.4854789845281 23.41892502695794,113.4854545713212 23.4188872952044,113.4854433523681 23.41882847882044,113.485446765259 23.418805917317,113.485468060442 23.41878946421219,113.4854909273421 23.41877139194617,113.4855138857942 23.4187549443131,113.4855334478511 23.41875654278465,113.485538299341 23.41876703341984,113.4855366755932 23.41878210576323,113.485525187459 23.41879263287558,113.4855169165271 23.41880614837947,113.485520208168 23.41881518831894,113.4855348373772 23.41882733549164,113.485543056915 23.41882736256224,113.4855544462291 23.418816836026,113.4855659050261 23.41881389506329,113.48557410711 23.41881843572094,113.485574066609 23.41882890934147,113.485569121258 23.41884243670313,113.4855641765311 23.41885605526005,113.485569039556 23.41886356452403,113.485578801144 23.4188696461664,113.485586944281 23.41886362348697,113.485614800673 23.41884565702208,113.4856361192332 23.41882315447521,113.4856492242071 23.41882464228223,113.485655657276 23.41883071386314,113.485655622366 23.41883974284661,113.4856539996091 23.41885481519478,113.485647421913 23.41886084393514,113.485624474005 23.41887431111695,113.485617878225 23.41888476315194,113.485612945372 23.41889531185609,113.485619477268 23.41890138195799,113.4856292553351 23.41890295000559,113.4856424017911 23.41889396419595,113.485657136101 23.41887893321473,113.4856702516191 23.41887744235538,113.4856816804711 23.41888199453445,113.485686537566 23.41889103962461,113.4856831421712 23.41890908573907,113.4856732649561 23.41893316181336,113.4856714794992 23.41896475687594,113.485681255294 23.41899196755162,113.485695890825 23.41900248858968,113.485710521049 23.41901463574959,113.4857170124572 23.41903118127403,113.485716954169 23.41904625877311,113.485712021321 23.41905680657749,113.4857054196 23.41906879444582,113.4857086015521 23.41908090391354,113.4857200488541 23.41908094160164,113.485731553472 23.4190658999807,113.485743019203 23.41906142317885,113.4857527857691 23.41906596988031,113.4857510866041 23.41907499338493,113.485749463848 23.41909006753929,113.485750987479 23.41910063569719,113.4857640990451 23.41910067886107,113.485775546349 23.41910071654525,113.485786924826 23.41911881310114,113.4857982268991 23.4191308581033,113.4857966041511 23.41914593135545,113.485791700618 23.41914889571933,113.485775342868 23.41915335642045,113.4857621739923 23.41916839075992,113.4857654248191 23.41918799550729,113.485780089345 23.41919102248164,113.4857979763121 23.41919568710591,113.485806148478 23.4192077227072,113.4858141645242 23.41923483638812,113.4858091908482 23.41925594720254,113.4857847017961 23.41926336164369,113.4857471021451 23.4192707311139,113.4856834370621 23.41926302825902,113.485660571405 23.41925545791713,113.4856411209191 23.4192252375878,113.4856055152551 23.41914683715942,113.485579566833 23.41910151683078,113.485550092563 23.41910746835663,113.485509195003 23.4191329762985,113.4854749254902 23.41916473613878))";
static HashMap<String,byte[]> valuesArrayList= new HashMap<String,byte[]> ();
static void InitConf() throws IOException, ParseException
{
// System.loadLibrary("gsjavaport");
// com.geostar.kernel.GsKernel.Initialize();
// System.out.println("GsKernel loadlibrary succeed");
Configuration cnf = new Configuration();
cnf.set("hbase.zookeeper.quorum", "127.0.0.1:2181");
cnf.set("hbase.zookeeper.property.clientPort", "2181");
conf = HBaseConfiguration.create(cnf);
connection = ConnectionFactory.createConnection(conf);
admin = connection.getAdmin();
tableName = TableName.valueOf(TABLE_NAME);
table = connection.getTable(tableName);
InitFeatureData();
}
static void InitFeatureData() throws ParseException
{
GeometryFactory fac = new GeometryFactory();
WKTReader reader = new WKTReader( );
Geometry jtspolygon1 = reader.read(polygon1);
WKBWriter writer = new WKBWriter();
byte[] b1 = writer.write(jtspolygon1);
valuesArrayList.put("GEOMETRY", b1);
valuesArrayList.put("XMIN",Bytes.toBytes(jtspolygon1.getEnvelopeInternal().getMinX()) );
valuesArrayList.put("YMIN",Bytes.toBytes(jtspolygon1.getEnvelopeInternal().getMinY()) );
valuesArrayList.put("XMAX",Bytes.toBytes(jtspolygon1.getEnvelopeInternal().getMaxX()) );
valuesArrayList.put("YMAX",Bytes.toBytes(jtspolygon1.getEnvelopeInternal().getMaxY()) );
valuesArrayList.put("OID",Bytes.toBytes((long)1));
valuesArrayList.put("NAME",Bytes.toBytes("名称1"));
valuesArrayList.put("LENGTH",Bytes.toBytes(0.001234567));
HashMap<String,String> family1t = new HashMap<String,String>();
HashMap<String,String> family2t = new HashMap<String,String>();
for(int i = 0; i < family1.length; i++)
{
family1t.put(family1[i], family1FieldsType[i]);
}
for(int i = 0; i < family2.length; i++)
{
family2t.put(family2[i], family2FieldsType[i]);
}
map.put(familygeos[0], family1t);
map.put(familygeos[1], family2t);
}
static void Exit() throws IOException
{
//admin.disableTable(tableName);
//admin.close();
//table.close();
connection.close();
}
public static void main(String... args) throws IOException, ParseException {
InitConf();
NewVersionTestCreateTable();
Inserttest();
long begintime1 = System.currentTimeMillis();
SearchData(108,20,115,26);
System.out.println("SearchData(108,20,115,26); "+ (System.currentTimeMillis() - begintime1));
begintime1 = System.currentTimeMillis();
SearchData(108,25,115,27);
System.out.println(" SearchData(108,25,115,27); "+ (System.currentTimeMillis() - begintime1));
Exit() ;
}
public static void NewVersionTestCreateTable() throws IOException
{
if (admin.tableExists(tableName))
{
if(!admin.isTableDisabled(tableName))
admin.disableTable(tableName);
admin.deleteTable(tableName);
System.out.println("Table is exist. delete and create it");
//return;
}
//创建表
TableDescriptorBuilder tbuilder = TableDescriptorBuilder.newBuilder(tableName);
ColumnFamilyDescriptor familygeo = ColumnFamilyDescriptorBuilder.newBuilder(familygeos[0].getBytes()).build();
ColumnFamilyDescriptor familyinfo = ColumnFamilyDescriptorBuilder.newBuilder(familygeos[1].getBytes()).build();
List<ColumnFamilyDescriptor> families = new ArrayList<>();
families.add(familygeo);
families.add(familyinfo);
TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName).setColumnFamilies(families).build();
admin.createTable(tableDescriptor);
}
public static void Inserttest() throws IOException {
if (!admin.tableExists(tableName)) {
System.out.println("Table does not exist.");
System.exit(-1);
}
table = connection.getTable(tableName);
CellBuilder cellbuilder = CellBuilderFactory.create(CellBuilderType.DEEP_COPY);
//table.setAutoFlushTo(false);
//table.setWriteBufferSize(534534534);
ArrayList<Put> arrayList = new ArrayList<Put>();
for (int i = 0; i < 50; i++) {
Put put = new Put(Bytes.toBytes(i));
int j = 0;
for (Map.Entry<String, HashMap<String, String>> kvEntry : map.entrySet()) {
{
HashMap<String, String> vvkv = kvEntry.getValue();
String keyString = kvEntry.getKey();
for(Map.Entry<String,String> kss:vvkv.entrySet() )
{
//String typeString = kss.getValue();
String colmunString = kss.getKey();
cellbuilder.setRow(Bytes.toBytes(i));
cellbuilder.setFamily(Bytes.toBytes(keyString));
cellbuilder.setQualifier(Bytes.toBytes(colmunString));
cellbuilder.setType(Type.Put);
cellbuilder.setValue(valuesArrayList.get(colmunString));
Cell cl = cellbuilder.build();
put.add(cl);
j++;
}
}
arrayList.add(put);
}
}
//插入数据
table.put(arrayList);
System.out.println("Insert end");
}
public static void SearchData(double xmin, double ymin, double xmax, double ymax) throws IOException, ParseException
{
if (!admin.tableExists(tableName)) {
System.out.println("Table does not exist.");
System.exit(-1);
}
java.util.concurrent.ExecutorService pool;
table = connection.getTable(tableName);
Scan scan1 = new Scan();
FilterList filters = new FilterList();
filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("GEOMETRY"), Bytes.toBytes("XMIN"),
CompareOperator.GREATER, Bytes.toBytes(xmin)));
filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("GEOMETRY"), Bytes.toBytes("YMIN"),
CompareOperator.GREATER, Bytes.toBytes(ymin)));
filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("GEOMETRY"), Bytes.toBytes("XMAX"),
CompareOperator.LESS, Bytes.toBytes(xmax)));
filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("GEOMETRY"), Bytes.toBytes("YMAX"),
CompareOperator.LESS, Bytes.toBytes(ymax)));
scan1.setFilter(filters);
//在行上获取遍历器
//int caching = 100;
//scan1.setCaching(caching );
//scan1.setBatch(5);
ResultScanner scanner1 = table.getScanner(scan1);
//打印行的值
int ncount =0;
for (org.apache.hadoop.hbase.client.Result res : scanner1) {
printrow(res,false);
ncount++;
}
//关闭释放资源
scanner1.close();
System.out.println("search end ,get"+ncount +" features");
}
static WKBReader wkbReader = new WKBReader();
static WKTWriter wktWriter = new WKTWriter();
static void printrow(org.apache.hadoop.hbase.client.Result rtResult ,boolean print) throws ParseException, UnsupportedEncodingException
{
System.out.println(rtResult);
System.out.println( new String(rtResult.getRow()).toString());
for (Map.Entry<String, HashMap<String, String>> kvEntry : map.entrySet()) {
{
HashMap<String, String> vvkv = kvEntry.getValue();
String keyString = kvEntry.getKey();
for(Map.Entry<String,String> kss:vvkv.entrySet() )
{
String typeString = kss.getValue();
String colmunString = kss.getKey();
Cell lengthCells = rtResult.getColumnLatestCell(Bytes.toBytes(keyString), Bytes.toBytes(colmunString));
if(null == lengthCells )
continue;
switch (typeString)
{
case "geometry":
byte[] vv = Arrays.copyOfRange(lengthCells.getValueArray(), lengthCells.getValueOffset(), lengthCells.getValueLength());
if(print)
{
Geometry geometry = wkbReader.read(vv);
System.out.println( wktWriter.write(geometry));
}
break;
case "double":
byte[] v = lengthCells.getValueArray();
if(print)
System.out.println( (Bytes.toDouble(v,lengthCells.getValueOffset())));
break;
case "long":
byte[] v1 = lengthCells.getValueArray();
System.out.println( (Bytes.toLong(v1,lengthCells.getValueOffset(),lengthCells.getValueLength())));
break;
case "string":
byte[] v11 = lengthCells.getValueArray();
if(print)
System.out.println( new String(Bytes.toString(v11,lengthCells.getValueOffset(),lengthCells.getValueLength())));
break;
case "int":
byte[] v111 = lengthCells.getValueArray();
if(print)
System.out.println( (Bytes.toInt(v111,lengthCells.getValueOffset())));
break;
case "float":
byte[] v1111 = lengthCells.getValueArray();
if(print)
System.out.println( (Bytes.toFloat(v1111,lengthCells.getValueOffset())));
break;
default:
break;
}
}
}
}
}
}
|