描述
本文记录了基于sharding-jdbc mybatis-plus 实现的时序数据库
本文将实现
- sharding-jdbc 主从读写分离配置
- sharding-jdbc 分库(水平) 分表(水平)配置
- sharding-jdbc 分库(水平) 分表(水平)算法
- sharding-jdbc mybatis-plus的集成
- sharding-jdbc mybatis-plus的分页查询
shardingjdbc,mybatis-plus版本
注意其官网上的配置跟版本相关,每个版本的配置都略有改动,最新5.x的配置因为使用的人较少,不利于初学者。所以本文采用4.x版本。
maven引入
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.5.3</version>
<relativePath/>
</parent>
<groupId>com.jisen</groupId>
<artifactId>springboot-rws</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>springboot-mycat</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
<version>3.4.3.1</version>
<exclusions>
<exclusion>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-generator</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.shardingsphere</groupId>
<artifactId>sharding-jdbc-spring-boot-starter</artifactId>
<version>4.1.1</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.9.9</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.56</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<excludes>
<exclude>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</project>
sharding jdbc 数据源配置
spring.shardingsphere.datasource.names=masterwrite0,masterwrite1,masterread0,masterread1,slaveread0,slaveread1
spring.shardingsphere.datasource.masterwrite0.jdbc-url=jdbc:mysql://192.168.43.201:3306/readwritesplit0?serverTimezone=Asia/Shanghai&useSSL=false&useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true
spring.shardingsphere.datasource.masterwrite0.type=com.zaxxer.hikari.HikariDataSource
spring.shardingsphere.datasource.masterwrite0.driver-class-name=com.mysql.cj.jdbc.Driver
spring.shardingsphere.datasource.masterwrite0.username=root
spring.shardingsphere.datasource.masterwrite0.password=123456
spring.shardingsphere.datasource.masterwrite1.jdbc-url=jdbc:mysql://192.168.43.201:3306/readwritesplit1?serverTimezone=Asia/Shanghai&useSSL=false&useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true
spring.shardingsphere.datasource.masterwrite1.type=com.zaxxer.hikari.HikariDataSource
spring.shardingsphere.datasource.masterwrite1.driver-class-name=com.mysql.cj.jdbc.Driver
spring.shardingsphere.datasource.masterwrite1.username=root
spring.shardingsphere.datasource.masterwrite1.password=123456
spring.shardingsphere.datasource.masterread0.jdbc-url=jdbc:mysql://192.168.43.201:3306/readwritesplit0?serverTimezone=Asia/Shanghai&useSSL=false&useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true
spring.shardingsphere.datasource.masterread0.type=com.zaxxer.hikari.HikariDataSource
spring.shardingsphere.datasource.masterread0.driver-class-name=com.mysql.cj.jdbc.Driver
spring.shardingsphere.datasource.masterread0.username=root
spring.shardingsphere.datasource.masterread0.password=123456
spring.shardingsphere.datasource.masterread1.jdbc-url=jdbc:mysql://192.168.43.201:3306/readwritesplit1?serverTimezone=Asia/Shanghai&useSSL=false&useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true
spring.shardingsphere.datasource.masterread1.type=com.zaxxer.hikari.HikariDataSource
spring.shardingsphere.datasource.masterread1.driver-class-name=com.mysql.cj.jdbc.Driver
spring.shardingsphere.datasource.masterread1.username=root
spring.shardingsphere.datasource.masterread1.password=123456
spring.shardingsphere.datasource.slaveread0.jdbc-url=jdbc:mysql://192.168.43.202:3306/readwritesplit0?serverTimezone=Asia/Shanghai&useSSL=false&useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true
spring.shardingsphere.datasource.slaveread0.type=com.zaxxer.hikari.HikariDataSource
spring.shardingsphere.datasource.slaveread0.driver-class-name=com.mysql.cj.jdbc.Driver
spring.shardingsphere.datasource.slaveread0.username=root
spring.shardingsphere.datasource.slaveread0.password=123456
spring.shardingsphere.datasource.slaveread1.jdbc-url=jdbc:mysql://192.168.43.202:3306/readwritesplit1?serverTimezone=Asia/Shanghai&useSSL=false&useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true
spring.shardingsphere.datasource.slaveread1.type=com.zaxxer.hikari.HikariDataSource
spring.shardingsphere.datasource.slaveread1.driver-class-name=com.mysql.cj.jdbc.Driver
spring.shardingsphere.datasource.slaveread1.username=root
spring.shardingsphere.datasource.slaveread1.password=123456
spring.shardingsphere.sharding.binding-tables=tb_keyvalue
spring.shardingsphere.props.sql.show=true
#读写分离配置库1
spring.shardingsphere.sharding.master-slave-rules.ds0.name=ds0datasource
spring.shardingsphere.sharding.master-slave-rules.ds0.load-balance-algorithm-type=round_robin
spring.shardingsphere.sharding.master-slave-rules.ds0.master-data-source-name=masterwrite0
spring.shardingsphere.sharding.master-slave-rules.ds0.slave-data-source-names=masterread0,slaveread0
#读写分离配置库2
spring.shardingsphere.sharding.master-slave-rules.ds1.name=ds1datasource
spring.shardingsphere.sharding.master-slave-rules.ds1.load-balance-algorithm-type=round_robin
spring.shardingsphere.sharding.master-slave-rules.ds1.master-data-source-name=masterwrite1
spring.shardingsphere.sharding.master-slave-rules.ds1.slave-data-source-names=masterread1, slaveread1
########################主键生成策略
spring.shardingsphere.sharding.tables.tb_keyvalue.key-generator.column=id
#spring.shardingsphere.sharding.tables.tb_keyvalue.key-generator.type=SNOWFLAKE
spring.shardingsphere.sharding.tables.tb_keyvalue.key-generator.type=NanoTimeShardingKeyGenerator
####################################################标准分片策略########################################################
#标准分片策略
spring.shardingsphere.sharding.tables.tb_keyvalue.actual-data-nodes=ds$->{0..1}.tb_keyvalue_$->{0..2}
spring.shardingsphere.sharding.tables.tb_keyvalue.database-strategy.standard.precise-algorithm-class-name=com.jisen.rws.common.MyDBPreciseShardingAlgorithm
spring.shardingsphere.sharding.tables.tb_keyvalue.database-strategy.standard.range-algorithm-class-name=com.jisen.rws.common.MyDBRangeShardingAlgorithm
spring.shardingsphere.sharding.tables.tb_keyvalue.database-strategy.standard.sharding-column=id
#
spring.shardingsphere.sharding.tables.tb_keyvalue.table-strategy.standard.precise-algorithm-class-name=com.jisen.rws.common.MyTablePreciseShardingAlgorithm
spring.shardingsphere.sharding.tables.tb_keyvalue.table-strategy.standard.range-algorithm-class-name=com.jisen.rws.common.MyTableRangeShardingAlgorithm
spring.shardingsphere.sharding.tables.tb_keyvalue.table-strategy.standard.sharding-column=id
#行分片策略,只能支持 = 或者 in() 的精准查询,不支持范围查询
###################################################以下为行分片策略############################################
#spring.shardingsphere.sharding.tables.tb_keyvalue.actual-data-nodes=ds$->{0..1}.tb_keyvalue_$->{0..2}
#
#spring.shardingsphere.sharding.tables.tb_keyvalue.database-strategy.inline.sharding-column=id
#spring.shardingsphere.sharding.tables.tb_keyvalue.database-strategy.inline.algorithm-expression=ds$->{(id / 10).toBigInteger() % 2}
#
#spring.shardingsphere.sharding.tables.tb_keyvalue.table-strategy.inline.sharding-column=id
#spring.shardingsphere.sharding.tables.tb_keyvalue.table-strategy.inline.algorithm-expression=tb_keyvalue_$->{id % 3}
主从库和表
- 主从两个机器192.168.43.201(master),192.168.43.202(slave),且主从库表结构完全一致,主用于读写,从只用于读
- master创建两个库 readwritesplit0,readwritesplit1;每个库三个表 tb_keyvalue_0,tb_keyvalue_1,tb_keyvalue_2,从库会根据binlog执行master 语句。
数据源
主从请需要配置6个数据源,详情查看配置文件
masterwrite0,masterwrite1 用于写 masterread0,masterread1 用于读 slaveread0,slaveread1 用于读
主从读写分离
如下配置,读写面对两个库的读写分离,分别是masterwrite0(master和slave上的readwritesplit0),readwritesplit1(master和slave上的readwritesplit1).其中master可读可写。采用轮询法执行均衡读
#读写分离配置库1
spring.shardingsphere.sharding.master-slave-rules.ds0.name=ds0datasource
spring.shardingsphere.sharding.master-slave-rules.ds0.load-balance-algorithm-type=round_robin
spring.shardingsphere.sharding.master-slave-rules.ds0.master-data-source-name=masterwrite0
spring.shardingsphere.sharding.master-slave-rules.ds0.slave-data-source-names=masterread0,slaveread0
#读写分离配置库2
spring.shardingsphere.sharding.master-slave-rules.ds1.name=ds1datasource
spring.shardingsphere.sharding.master-slave-rules.ds1.load-balance-algorithm-type=round_robin
spring.shardingsphere.sharding.master-slave-rules.ds1.master-data-source-name=masterwrite1
spring.shardingsphere.sharding.master-slave-rules.ds1.slave-data-source-names=masterread1, slaveread1
主键以及分片算法选择
分库分表不能采用自增id,水平分布式存储需要避免主键id重复的问题。 一般采用uuid或者snowflake。 其中uuid是不带中横线的32位字符串:e.g.0f2a207c89da40b5adfc1fbc1fb2ec82 snowflake是long类型长整型:e.g.627088648474460160
主键决定了如何均衡的将记录插入水平分库分表的6个表中。 可插入的库表 master.readwrirtesplite0.tb_keyvalue_0 master.readwrirtesplite0.tb_keyvalue_1 master.readwrirtesplite0.tb_keyvalue_2 master.readwrirtesplite1.tb_keyvalue_0 master.readwrirtesplite1.tb_keyvalue_1 master.readwrirtesplite1.tb_keyvalue_2
如果是uuid可以将字符串转化位hashCode再取模的方式 Math.abs(“uuid”.hashCode())%3 则表的分片算法可以用Math.abs(“uuid”.hashCode())%3来计算
那么库的分片算法如何 可采用十位来计算 即Math.abs(“uuid”.hashCode())/10%2
snowflake是长整形 库分片算法 (627088648474460160/10).toBigInteger()%2 表分片算法 627088648474460160%2
时序数据库
时序数据库的特点是数据量大,数据类型比较单一,牺牲掉了关系型数据库事务的特性,且跟采集时间密切相关
当前时序数据库influxdb或者tdengin时序数据库,将时间作为主键以提高数据库性能,重复的时间主键将会被丢弃。因此需要非常精确的时间主键。
纳秒时间戳,毫秒级的基础下有一百万的数据容量。这个时间戳完全适用大规模数据情况,并且可以当作时间字段
nanoTime是系统纳秒级别的测量方法api,只记录系统经过时间。cpu一个周期也是纳秒级别,因此在单机情况(同一虚拟机实例)下可以不必考虑多线程导致Nanotime重复的问题。但是如果是分布式部署,不同的系统中其基准不一致,因此如果需要完全不重复的时间戳可采用redis获取微妙时间(类似于分布式锁)。
public class NanoTimeUtils {
public static Long getNanoTime() {
long currentTimeMillis = System.currentTimeMillis();
long nanoTime = System.nanoTime();
System.out.println("获取纳秒时间戳:"+(currentTimeMillis * 1000000L + nanoTime/100 % 1000000L));
return currentTimeMillis * 1000000L + nanoTime/100 % 1000000L;
}
}
时序数据库中,有库,超级表,表的概念。将同种设备的数据归于超级表,每个设备的数据放置于单个表中。这类似于关系型数据库的分片功能
mysql实现的分库分表中要实现时序数据的特点,就需要将设备分片,同个设备中了减轻单设备数据量过大的问题又可以根据时间(天,月)水平分表。 device_mac_time device_mac0_time0 device_mac0_time1 device_mac2_time0 device_mac2_time1
shardingjdbc自定义主键
########################主键生成策略
spring.shardingsphere.sharding.tables.tb_keyvalue.key-generator.column=id
#spring.shardingsphere.sharding.tables.tb_keyvalue.key-generator.type=SNOWFLAKE
spring.shardingsphere.sharding.tables.tb_keyvalue.key-generator.type=NanoTimeShardingKeyGenerator
- 自定义主键生成,实现ShardingKeyGenerator
package com.jisen.rws.common;
import com.jisen.rws.utils.NanoTimeUtils;
import org.apache.shardingsphere.spi.keygen.ShardingKeyGenerator;
import java.util.Properties;
public class NanoTimeShardingKeyGenerator implements ShardingKeyGenerator {
@Override
public Comparable<?> generateKey() {
return NanoTimeUtils.getNanoTime();
}
@Override
public String getType() {
return "NanoTimeShardingKeyGenerator";
}
@Override
public Properties getProperties() {
return null;
}
@Override
public void setProperties(Properties properties) {
}
}
- classpath下创建META-INF.services包目录,
- 添加org.apache.shardingsphere.spi.keygen.ShardingKeyGenerator文件
- 内容为实现的主键生成类的全限定类名e.g. com.jisen.rws.common.NanoTimeShardingKeyGenerator
分库分表配置
- inline 分片策略
只能支持精确查找 = 或者 in() 的精准查询,不支持> < between and 范围查询
#行分片策略,只能支持 = 或者 in() 的精准查询,不支持范围查询
###################################################以下为行分片策略############################################
#spring.shardingsphere.sharding.tables.tb_keyvalue.actual-data-nodes=ds$->{0..1}.tb_keyvalue_$->{0..2}
#
#spring.shardingsphere.sharding.tables.tb_keyvalue.database-strategy.inline.sharding-column=id
#spring.shardingsphere.sharding.tables.tb_keyvalue.database-strategy.inline.algorithm-expression=ds$->{(id / 10).toBigInteger() % 2}
#
#spring.shardingsphere.sharding.tables.tb_keyvalue.table-strategy.inline.sharding-column=id
#spring.shardingsphere.sharding.tables.tb_keyvalue.table-strategy.inline.algorithm-expression=tb_keyvalue_$->{id % 3}
库和表的分片配置如下,需要指定precise和range两种分片算法的实现,
sql中精确查找 = ,in()将会走precise算法 sql中范围查找 >= ,<,between and 将会走range算法
####################################################标准分片策略########################################################
#标准分片策略
spring.shardingsphere.sharding.tables.tb_keyvalue.actual-data-nodes=ds$->{0..1}.tb_keyvalue_$->{0..2}
spring.shardingsphere.sharding.tables.tb_keyvalue.database-strategy.standard.precise-algorithm-class-name=com.jisen.rws.common.MyDBPreciseShardingAlgorithm
spring.shardingsphere.sharding.tables.tb_keyvalue.database-strategy.standard.range-algorithm-class-name=com.jisen.rws.common.MyDBRangeShardingAlgorithm
spring.shardingsphere.sharding.tables.tb_keyvalue.database-strategy.standard.sharding-column=id
#
spring.shardingsphere.sharding.tables.tb_keyvalue.table-strategy.standard.precise-algorithm-class-name=com.jisen.rws.common.MyTablePreciseShardingAlgorithm
spring.shardingsphere.sharding.tables.tb_keyvalue.table-strategy.standard.range-algorithm-class-name=com.jisen.rws.common.MyTableRangeShardingAlgorithm
spring.shardingsphere.sharding.tables.tb_keyvalue.table-strategy.standard.sharding-column=id
- MyDBPreciseShardingAlgorithm
package com.jisen.rws.common;
import org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm;
import org.apache.shardingsphere.api.sharding.standard.PreciseShardingValue;
import java.util.Collection;
public class MyDBPreciseShardingAlgorithm implements PreciseShardingAlgorithm<Long> {
@Override
public String doSharding(Collection<String> availableTargetNames, PreciseShardingValue<Long> shardingValue) {
for (String tableName : availableTargetNames) {
if (tableName.endsWith((shardingValue.getValue()/10) % 2 + "")) {
return tableName;
}
}
throw new IllegalArgumentException();
}
}
- MyDBRangeShardingAlgorithm
package com.jisen.rws.common;
import org.apache.shardingsphere.api.sharding.standard.RangeShardingAlgorithm;
import org.apache.shardingsphere.api.sharding.standard.RangeShardingValue;
import java.util.Collection;
public class MyDBRangeShardingAlgorithm implements RangeShardingAlgorithm<Long> {
@Override
public Collection<String> doSharding(Collection<String> collection, RangeShardingValue<Long> rangeShardingValue) {
return collection;
}
}
- MyTablePreciseShardingAlgorithm
package com.jisen.rws.common;
import org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm;
import org.apache.shardingsphere.api.sharding.standard.PreciseShardingValue;
import java.util.Collection;
public class MyTablePreciseShardingAlgorithm implements PreciseShardingAlgorithm<Long> {
@Override
public String doSharding(Collection<String> availableTargetNames, PreciseShardingValue<Long> shardingValue) {
for (String tableName : availableTargetNames) {
if (tableName.endsWith(shardingValue.getValue() % 3 + "")) {
return tableName;
}
}
throw new IllegalArgumentException();
}
}
- MyTableRangeShardingAlgorithm
package com.jisen.rws.common;
import org.apache.shardingsphere.api.sharding.standard.RangeShardingAlgorithm;
import org.apache.shardingsphere.api.sharding.standard.RangeShardingValue;
import java.util.Collection;
public class MyTableRangeShardingAlgorithm implements RangeShardingAlgorithm<Long> {
@Override
public Collection<String> doSharding(Collection<String> collection, RangeShardingValue<Long> rangeShardingValue) {
return collection;
}
}
mybatis-plus集成
分页查询,需要开启分页插件
@Bean
public MybatisPlusInterceptor mybatisPlusInterceptor() {
MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor();
interceptor.addInnerInterceptor(new PaginationInnerInterceptor(DbType.MYSQL));
return interceptor;
}
源码,至此主从读写分离,水平分库分表,已经配置完成,代码如下
https://gitee.com/jisen_zhong/springboot-rws.git
|