方式一:采用easyexcel导入
1.需要导入的依赖
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId>
<version>2.2.8</version>
</dependency>
实体类代码:
@Data
@Entity
@Table(name = "tb_user")
@ApiModel(description = "用户表")
public class User{
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
@ApiModelProperty(hidden = true)
private Integer id;
@ApiModelProperty(value = "用户名")
@ExcelProperty(index = 1)
@Column(name = "user_name", unique = true, nullable = false, length = 50)
private String userName;
@ApiModelProperty(value = "用户工号")
@ExcelProperty(index = 2)
@Column(name = "login_name", length = 20)
private String loginName;
@ApiModelProperty(value = "密码")
@Column(name = "password", length = 50)
private String password;
@ApiModelProperty(value = "头像")
@Column(name = "avatar", length = 100)
private String avatar;
@ApiModelProperty(value = "性别")
@ExcelProperty(index = 3)
@Column(name = "gender", nullable = false, length = 1)
private String gender;
@ApiModelProperty(value = "邮箱")
@ExcelProperty(index = 4)
@Column(name = "email", length = 50)
private String email;
@ApiModelProperty(value = "描述")
@ExcelProperty(index = 7,value = "描述")
@Column(name = "description")
private String description;
@ApiModelProperty(value = "所属公司")
@ExcelProperty(index = 5,value = "所属公司")
@Column(name = "company", length = 50)
private String company;
@ApiModelProperty(value = "职位")
@ExcelProperty(index = 6,value ="职位")
@Column(name = "position", length = 20)
private String position;
@ApiModelProperty(hidden = true)
private Date createTime;
@ApiModelProperty(hidden = true)
private Date updateTime;
}
实现监听回调
public class PrtcpntInfoListener extends AnalysisEventListener<User> {
private static final Logger log = LoggerFactory.getLogger(PrtcpntInfoListener.class);
private static int TOTAL_COUNT;
private static final int BATCH_COUNT = 110000;
List<User> list = new ArrayList<>();
private BatchLead batchLead;
public PrtcpntInfoListener(BatchLead batchLead) {
this.batchLead = batchLead;
}
@Override
public void invoke(User data, AnalysisContext context) {
log.info("解析到一条数据:{}", JSON.toJSONString(data));
list.add(data);
if (list.size() >= BATCH_COUNT) {
ThreadOperation.readExcel(list);
TOTAL_COUNT += list.size();
list.clear();
}
}
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
batchLead.save(list);
TOTAL_COUNT += list.size();
log.info("所有数据已入库完成,总计:{}条", TOTAL_COUNT);
}
public static void main(String[] args) {
Long startTime = System.currentTimeMillis();
ExcelReader excelReader = null;
try {
String filePath = "C:/Users/86135/Desktop/用户信息.xlsx";
excelReader = EasyExcel.read(filePath, User.class, new PrtcpntInfoListener(new BatchLead())).build();
ReadSheet readSheet = EasyExcel.readSheet(0).headRowNumber(2).build();
excelReader.read(readSheet);
} finally {
if (excelReader != null) {
excelReader.finish();
}
}
log.info("总耗时:" + (System.currentTimeMillis() - startTime) / 1000 + "(s)");
}
}
保存数据 JDBC直连方式
public class BatchLead {
private static final Logger log = LoggerFactory.getLogger(BatchLead.class);
int i = 0;
public void save(List<User> list) {
System.out.println("保存="+list.size());
Long startTime = System.currentTimeMillis();
Connection conn = null;
PreparedStatement pstmt = null;
String sql = "insert into tb_user(user_name,login_name,password,avatar,gender,email,description,company,position)values(?,?,?,?,?,?,?,?,?)";
conn = getConnection();
try {
conn.setAutoCommit(false);
pstmt = conn.prepareStatement(sql);
} catch (SQLException e1) {
e1.printStackTrace();
}
for (User dataInfo : list) {
try {
pstmt.setString(1, dataInfo.getUserName());
pstmt.setString(2, dataInfo.getLoginName());
pstmt.setString(3, dataInfo.getPassword());
pstmt.setString(4, dataInfo.getAvatar());
pstmt.setString(5, dataInfo.getGender());
pstmt.setString(6, dataInfo.getEmail());
pstmt.setString(7, dataInfo.getDescription());
pstmt.setString(8, dataInfo.getCompany());
pstmt.setString(9, dataInfo.getPosition());
pstmt.addBatch();
i++;
} catch (SQLException e) {
e.printStackTrace();
}
}
try {
pstmt.executeBatch();
conn.commit();
conn.close();
pstmt.close();
} catch (SQLException e) {
e.printStackTrace();
}
log.info("批量入库,本次入库数据:{}条,耗时:{}(s)", list.size(), (System.currentTimeMillis() - startTime) / 1000);
}
public static Connection getConnection() {
Connection connection = null;
try {
Class.forName("com.mysql.jdbc.Driver");
connection = DriverManager.getConnection(
"jdbc:mysql://127.0.0.1:3306/localtest?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC",
"root", "root");
} catch (ClassNotFoundException | SQLException e) {
e.printStackTrace();
}
return connection;
}
}
或者使用mybatis,在覆盖doAfterAllAnalysed方法中调用
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
UserMapper userMapper = ApplicationContextProvider.getBean(UserMapper.class);
userMapper.batchAddUser(list);
TOTAL_COUNT += list.size();
log.info("所有数据已入库完成,总计:{}条", TOTAL_COUNT);
}
controller层调用:
@PostMapping("/users/add")
@ApiOperation(value = "批量导入")
public ResultDTO saveBatchUser(MultipartFile file) {
String fileName = file.getOriginalFilename();
if (!StrUtil.endWithAny(fileName, EXCEL_XLS, EXCEL_XLSX)) {
return ResultDTO.error("只支持.xlsx与.xls格式的文件导入");
}
return myUserService.saveBatchUser(file,fileName);
}
service层代码:
public ResultDTO saveBatchUser(MultipartFile multipartFile, String fileName) {
Long startTime = System.currentTimeMillis();
ExcelReader excelReader = null;
File file = null;
try {
String[] filename = fileName.split("\\.");
file=File.createTempFile(filename[0], ".xlsx");
multipartFile.transferTo(file);
excelReader = EasyExcel.read(file, User.class, new PrtcpntInfoListener(new BatchLead())).build();
ReadSheet readSheet = EasyExcel.readSheet(0).headRowNumber(1).build();
excelReader.read(readSheet);
}catch (Exception e){
e.printStackTrace();
} finally {
if (excelReader != null) {
excelReader.finish();
}
}
log.info("总耗时:" + (System.currentTimeMillis() - startTime) / 1000 + "(s)");
return ResultDTO.isSuccess();
}
上面是单线程版 多线程保存数据库方式 先定义线程资源类
public class ImportThread implements Runnable {
BatchLead batchLead;
private List<User> list;
private CountDownLatch end;
private MyUserMapper myUserMapper;
public ImportThread(List<User> list, CountDownLatch begin, CountDownLatch end, BatchLead batchLead) {
this.list = list;
this.end = end;
this.batchLead = batchLead;
}
@Override
public void run() {
UserMapper userMapper = ApplicationContextProvider.getBean(UserMapper.class);
userMapper.batchAddUser(list);
}
}
上面连接数据库如果使用mybatis方式,sql语句如下
<insert id="batchAddUser">
insert into tb_user(user_name,login_name,password,avatar,gender,email,description,company,position) values
<foreach collection="list" separator="," item="item">
(#{item.userName},#{item.loginName},#{item.password},#{item.avatar},#{item.gender},#{item.email},#{item.description},#{item.company},#{item.position})
</foreach>
</insert>
使用线程池
public class ThreadOperation {
public static void readExcel(List<User> list) {
int count = 11000;
int runSize = 10;
List<User> newlist = null;
ThreadPoolExecutor executor = new ThreadPoolExecutor(
runSize, runSize, 1, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5),
new ThreadPoolExecutor.DiscardOldestPolicy());
CountDownLatch end = new CountDownLatch(runSize);
for (int i = 0; i < runSize; i++) {
if ((i+1)==runSize){
newlist=list.subList((i*count),list.size());
}else {
newlist=list.subList((i*count),(i+1)*count);
}
ImportThread mythead = new ImportThread(newlist, end,new BatchLead());
executor.execute(mythead);
}
try {
end.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
executor.shutdown();
}
}
方式二:采用POI导入
依赖准备
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>3.9</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>3.9</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.4</version>
</dependency>
自定义注解
@Target(ElementType.TYPE)
@Retention(RetentionPolicy.RUNTIME)
public @interface EnableExport {
String fileName();
}
@Target(ElementType.FIELD)
@Retention(RetentionPolicy.RUNTIME)
public @interface EnableExportField {
int colWidth() default 100;
String colName();
}
@Target(ElementType.FIELD)
@Retention(RetentionPolicy.RUNTIME)
public @interface ImportIndex {
int index() ;
}
@EnableExport(fileName = "XXX功能")
public class DataCheckExport {
@EnableExportField(colName = "编码",colWidth = 90)
@ImportIndex(index = 0)
private String code;
....
}
导入 、导出、 自定义表格颜色工具类
public class ExcelUtils {
public static final Map<Integer,Map<String,String>> ALL_SELECT_LIST_MAP = new HashMap<Integer,Map<String,String>> ();
public static List<?> parseExcelToList(File excel,Class clazz){
List<Object> res = new ArrayList<Object>();
InputStream is = null;
Sheet sheet = null;
try {
is = new FileInputStream(excel.getAbsolutePath());
if (is != null) {
Workbook workbook = WorkbookFactory.create(is);
sheet = workbook.getSheetAt(0);
if (sheet != null) {
int i = 2;
String values[] ;
Row row = sheet.getRow(i);
while (row != null) {
int cellNum = row.getPhysicalNumberOfCells();
values = new String[cellNum];
for (int j = 0; j <= cellNum; j++) {
Cell cell = row.getCell(j);
if (cell != null) {
cell.setCellType(Cell.CELL_TYPE_STRING );
String value = cell.getStringCellValue() == null ? null : cell.getStringCellValue();
values[j]=value;
}
}
Field[] fields = clazz.getDeclaredFields();
Object obj = clazz.newInstance();
for(Field f : fields){
if(f.isAnnotationPresent(ImportIndex.class)){
ImportIndex annotation = f.getAnnotation(ImportIndex.class);
int index = annotation.index();
String useSetMethodName = annotation.useSetMethodName();
if(!"".equals(useSetMethodName)){
Object val = TypeUtils.cast(values[index],f.getType(),null);
f.setAccessible(true);
Method method = clazz.getMethod(useSetMethodName, new Class[]{f.getType(),Object.class});
method.setAccessible(true);
method.invoke(obj, new Object[]{f.get(obj),val});
}else{
f.setAccessible(true);
Object val =TypeUtils.cast(values[index],f.getType(),null);
f.set(obj,val);
}
}
}
res.add(obj);
i++;
row=sheet.getRow(i);
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
return res;
}
public static List<?> parseExcelToList(InputStream excel,Class clazz) throws IOException,
InvalidFormatException,
InstantiationException,
IllegalAccessException,
NoSuchMethodException,
InvocationTargetException {
List<Object> res = new ArrayList<Object>();
InputStream is = null;
Sheet sheet = null;
is = excel;
if (is != null) {
Workbook workbook = WorkbookFactory.create(is);
sheet = workbook.getSheetAt(0);
if (sheet != null) {
int i = 2;
String values[] ;
Row row = sheet.getRow(i);
while (row != null) {
int cellNum = row.getPhysicalNumberOfCells();
values = new String[cellNum];
for (int j = 0; j <= cellNum; j++) {
Cell cell = row.getCell(j);
if (cell != null) {
cell.setCellType(Cell.CELL_TYPE_STRING );
String value = cell.getStringCellValue() == null ? null : cell.getStringCellValue();
values[j]=value;
}
}
Field[] fields = clazz.getDeclaredFields();
Object obj = clazz.newInstance();
for(Field f : fields){
if(f.isAnnotationPresent(ImportIndex.class)){
ImportIndex annotation = f.getAnnotation(ImportIndex.class);
int index = annotation.index();
Object value = values[index];
if(f.isAnnotationPresent(EnableSelectList.class)){
value = getKeyByValue(ALL_SELECT_LIST_MAP.get(index),String.valueOf(value ) );
}
String useSetMethodName = annotation.useSetMethodName();
if(!"".equals(useSetMethodName)){
Object val =TypeUtils.cast(value,f.getType(),null);
f.setAccessible(true);
Method method = clazz.getMethod(useSetMethodName, new Class[]{f.getType(),Object.class});
method.setAccessible(true);
method.invoke(obj, new Object[]{f.get(obj),val});
}else{
f.setAccessible(true);
Object val =TypeUtils.cast(value,f.getType(),null);
f.set(obj,val);
}
}
}
res.add(obj);
i++;
row=sheet.getRow(i);
}
}
}
return res;
}
public static void exportExcel(OutputStream outputStream, List dataList, Class clazz, Map<Integer,Map<String,String>> selectListMap,String exportTitle){
HSSFWorkbook workbook = new HSSFWorkbook();
HSSFSheet hssfsheet = workbook.createSheet();
hssfsheet.setDefaultRowHeight( ( short )(20*20) );
if(clazz.isAnnotationPresent(EnableExport.class)) {
EnableExport export = (EnableExport) clazz.getAnnotation(EnableExport.class);
List<String> colNames =new ArrayList<String>();
List<ColorEnum> colors =new ArrayList<ColorEnum>();
List<Field> fieldList = new ArrayList<Field>();
for(Field field : clazz.getDeclaredFields()){
if(field.isAnnotationPresent(EnableExportField.class)){
EnableExportField enableExportField = field.getAnnotation(EnableExportField.class);
colNames.add(enableExportField.colName());
colors.add(enableExportField.cellColor());
fieldList.add(field);
}
}
for(int i=0;i<fieldList.size();i++){
Field field = fieldList.get(i);
hssfsheet.setColumnWidth(i,field.getAnnotation(EnableExportField.class).colWidth()*20);
}
HSSFRow hssfRow = null;
HSSFCell hssfcell = null;
String fileName =export.fileName();
if(exportTitle!=null){
fileName = exportTitle;
}
createTitle(workbook,hssfRow,hssfcell,hssfsheet, colNames.size()-1 ,fileName,export.cellColor());
createHeadRow(workbook,hssfRow,hssfcell,hssfsheet,colNames,colors);
try {
HSSFCellStyle cellStyle= getBasicCellStyle(workbook);
int i=0;
for (Object obj : dataList) {
hssfRow = hssfsheet.createRow(i + 2);
for(int j=0;j<fieldList.size();j++){
Field field = fieldList.get(j);
field.setAccessible(true);
Object value = field.get(obj);
EnableExportField enableExportField = field.getAnnotation(EnableExportField.class);
String getMethodName = enableExportField.useGetMethod();
if(!"".equals( getMethodName )){
Method method = clazz.getMethod(getMethodName, new Class[]{field.getType()});
method.setAccessible(true);
value= method.invoke(obj, new Object[]{value});
}
if(field.isAnnotationPresent(EnableSelectList.class)){
if(selectListMap!=null&& selectListMap.get(j)!=null)
value =selectListMap.get(j).get(value);
}
setCellValue(value, hssfcell, hssfRow, cellStyle, j);
}
i++;
}
createDataValidation(hssfsheet,selectListMap);
workbook.write(outputStream);
outputStream.flush();
outputStream.close();
} catch (IllegalAccessException e ) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}catch (NoSuchMethodException e) {
e.printStackTrace();
}catch (InvocationTargetException e) {
e.printStackTrace();
}
}
}
private static HSSFCellStyle getBasicCellStyle(HSSFWorkbook workbook){
HSSFCellStyle hssfcellstyle = workbook.createCellStyle();
hssfcellstyle.setBorderLeft(HSSFCellStyle.BORDER_THIN);
hssfcellstyle.setBorderBottom(HSSFCellStyle.BORDER_THIN);
hssfcellstyle.setBorderRight(HSSFCellStyle.BORDER_THIN);
hssfcellstyle.setBorderTop(HSSFCellStyle.BORDER_THIN);
hssfcellstyle.setAlignment(HSSFCellStyle.ALIGN_CENTER);
hssfcellstyle.setVerticalAlignment(HSSFCellStyle.VERTICAL_CENTER);
hssfcellstyle.setWrapText(true);
return hssfcellstyle;
}
private static HSSFCellStyle getTitleCellStyle(HSSFWorkbook workbook,ColorEnum color){
HSSFCellStyle hssfcellstyle = getBasicCellStyle(workbook);
hssfcellstyle.setFillForegroundColor(color.getIndex());
hssfcellstyle.setFillPattern(HSSFCellStyle.SOLID_FOREGROUND);
return hssfcellstyle;
}
private static void createTitle(HSSFWorkbook workbook, HSSFRow hssfRow , HSSFCell hssfcell, HSSFSheet hssfsheet,int allColNum,String title,ColorEnum color){
CellRangeAddress cra = new CellRangeAddress(0, 0, 0, allColNum);
hssfsheet.addMergedRegion(cra);
RegionUtil.setBorderBottom(1, cra, hssfsheet, workbook);
RegionUtil.setBorderLeft(1, cra, hssfsheet, workbook);
RegionUtil.setBorderRight(1, cra, hssfsheet, workbook);
RegionUtil.setBorderTop(1, cra, hssfsheet, workbook);
hssfRow = hssfsheet.getRow(0);
hssfcell = hssfRow.getCell(0);
hssfcell.setCellStyle( getTitleCellStyle(workbook,color));
hssfcell.setCellType(HSSFCell.CELL_TYPE_STRING);
hssfcell.setCellValue(title);
}
private static void createHeadRow(HSSFWorkbook workbook,HSSFRow hssfRow , HSSFCell hssfcell,HSSFSheet hssfsheet,List<String> colNames,List<ColorEnum> colors){
hssfRow = hssfsheet.createRow(1);
for (int i = 0; i < colNames.size(); i++) {
hssfcell = hssfRow.createCell(i);
hssfcell.setCellStyle(getTitleCellStyle(workbook,colors.get(i)));
hssfcell.setCellType(HSSFCell.CELL_TYPE_STRING);
hssfcell.setCellValue(colNames.get(i));
}
}
public static void createDataValidation(Sheet sheet, Map<Integer,Map<String,String>> selectListMap) {
if(selectListMap!=null) {
for(Map.Entry<Integer,Map<String,String>> entry:selectListMap.entrySet() ){
Integer key = entry.getKey();
Map<String,String> value = entry.getValue();
if(value.size()>0) {
int i=0;
String[] valueArr = new String[value.size()];
for(Map.Entry<String,String> ent :value.entrySet()){
valueArr[i] = ent.getValue();
i++;
}
CellRangeAddressList cellRangeAddressList = new CellRangeAddressList(2, 65535, key, key);
DataValidationHelper helper = sheet.getDataValidationHelper();
DataValidationConstraint constraint = helper.createExplicitListConstraint(valueArr);
DataValidation dataValidation = helper.createValidation(constraint, cellRangeAddressList);
if (dataValidation instanceof XSSFDataValidation) {
dataValidation.setSuppressDropDownArrow(true);
dataValidation.setShowErrorBox(true);
} else {
dataValidation.setSuppressDropDownArrow(false);
}
dataValidation.setEmptyCellAllowed(true);
dataValidation.setShowPromptBox(true);
dataValidation.createPromptBox("提示", "只能选择下拉框里面的数据");
sheet.addValidationData(dataValidation);
}
}
}
}
private static String getKeyByValue(Map<String,String> selectMap,String value){
if(selectMap!=null){
for(Map.Entry<String,String> ent :selectMap.entrySet()){
if(value!=null&&value.equals(ent.getValue()))
return ent.getKey();
}
}else{
return value;
}
return null;
}
private static boolean isNumeric(String str) {
Pattern pattren =
Pattern.compile("[-+]?(([0-9]+)([.]([0-9]+))?|([.]([0-9]+))?)$");
if (str != null && !"".equals(str.trim())) {
Matcher matcher = pattren.matcher(str);
if (matcher.matches()) {
if (!str.contains(".") && str.startsWith("0")) {
return false;
}
return true;
}
}
return false;
}
private static void setCellValue(Object value,HSSFCell hssfcell,HSSFRow hssfRow,CellStyle cellStyle,int cellIndex) {
String valueStr = String.valueOf(value);
hssfcell =hssfRow.createCell(cellIndex );
if (isNumeric(valueStr)) {
hssfcell.setCellStyle(cellStyle);
hssfcell.setCellType(HSSFCell.CELL_TYPE_NUMERIC);
hssfcell.setCellValue(Double.valueOf(valueStr));
} else {
hssfcell.setCellStyle(cellStyle);
hssfcell.setCellType(HSSFCell.CELL_TYPE_STRING);
hssfcell.setCellValue(valueStr);
}
}
}
还可以自定义表格颜色,根据业务需要使用,定义一个枚举类存储需要的颜色。
public enum ColorEnum {
RED("红色", HSSFColor.RED.index),
GREEN("绿色", HSSFColor.GREEN.index),
BLANK("白色", HSSFColor.WHITE.index),
YELLOW("黄色", HSSFColor.YELLOW.index),
BLUE("蓝色", HSSFColor.CORNFLOWER_BLUE.index);
private String name;
private short index;
private ColorEnum( String name, short index) {
this.name = name;
this.index = index;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public short getIndex() {
return index;
}
public void setIndex(short index) {
this.index = index;
}
}
|