[Java知识库]java实现文件切片上传百度云+断点续传

前言：

本文代码通过dubbo进行远程调用的接口，如果不使用dubbo，直接将service放到你的service，并稍作修改，redis替换成自己封装的工具即可。下方代码有点多，但基本上都加注释了。

切片上传百度云和上传到本地有所不同，上传到本地是需要前端给个唯一标识，通过自定义唯一标识保存到本地，最后在合并文件。而上传百度云则需要直接把切片传到百度云，唯一标识uploadId是百度云提供的，上传及合并都是直接调用百度云提供的接口。下方为百度云的使用文档。

对象存储BOS - 文件管理 | 百度智能云文档 (baidu.com)

1.定义一个实体类，用于接收前台传过来的信息，信息应该包括总切片数，当前切片数，文件名称，文件总大小。

import com.geely.tcm.common.annotation.Excel;
import lombok.Data;

import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
import java.io.Serializable;


@Data
public class MutilFileInfo implements Serializable {
    private static final long serialVersionUID = 1L;

    @NotNull(message = "车型不能为空")
    @Excel(name = "车型")
    private String model;

    @Excel(name = "文件名称")
    private String name;

    @Excel(name = "文件总大小")
    private Long totalSize;

    @Min(value = 1, message = "不能小于1")
    @Max(value = 10000, message = "不能大于10000")
    @Excel(name = "当前分片序号")
    private Integer chunk;

    @Min(value = 1, message = "不能小于1")
    @Max(value = 10000, message = "不能大于10000")
    @Excel(name = "分片总数")
    private Integer chunks;
}

2.创建controller，用于处理前端过来的文件及信息

import com.alibaba.fastjson.JSON;
import com.baidubce.services.bos.model.PartETag;
import com.geely.tcm.common.annotation.Log;
import com.geely.tcm.common.bean.MutilFileInfo;
import com.geely.tcm.common.bean.baidu.CustomPartETag;
import com.geely.tcm.common.bean.measure.Cmm3dModel;
import com.geely.tcm.common.configuration.RedisUtils;
import com.geely.tcm.common.enums.BusinessType;
import com.geely.tcm.common.util.StringUtils;
import com.geely.tcom.remote.api.baidu.BaiduCloudBosService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.apache.dubbo.config.annotation.DubboReference;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;

import java.util.*;
import java.util.concurrent.Semaphore;
import java.util.stream.Collectors;

@Api(tags = "文件上传下载服务")
@RequestMapping("/threeModel")
@RestController
@Slf4j
public class Model3DController1  {
    //DubboReference这个是远程调用，如果没有远程调用，直接Autowired就可以
    @DubboReference(version = "${sheet.service.version}")
    private BaiduCloudBosService baiduCloudBosService;
    //通过加锁保证线程安全
    private Semaphore lock = new Semaphore(1);

    //api开头的这些是swagger用到的，如果没使用swagger可以删到PostMapping上边
    @ApiOperation(value = "obj模型上传")
    @ApiImplicitParams({
            @ApiImplicitParam(name = "totalSize", value = "文件总大小",dataTypeClass = Long.class),
            @ApiImplicitParam(name = "chunks", value = "总切片数",dataTypeClass = Integer.class),
            @ApiImplicitParam(name = "chunk", value = "当前切片数",dataTypeClass = Integer.class),
            @ApiImplicitParam(name = "model", value = "车型",dataTypeClass = String.class),
            @ApiImplicitParam(name = "file", value = "切片文件",dataTypeClass = MultipartFile.class)
    })
    @Log(title = "obj模型上传", businessType = BusinessType.INSERT)
    @PostMapping("/upload3DToBos")
    public Object upload3DToBos( MutilFileInfo fileinfo, @RequestParam(required=false,value="file") MultipartFile multipartFile) {
        String model=fileinfo.getModel().trim();
        Map<String,Object> m=new HashMap();
        String fileName=fileinfo.getName();
        if (StringUtils.isEmpty(fileName)) {//判断文件名称是否为空
            m.put("key","error");
            m.put("value","文件名不能为空");
            return m;
        }
        if (multipartFile == null) {//判断上传过来的切片是否为空
            m.put("key","error");
            m.put("value","请上传文件");
            return m;
        }
        try{
            String filePath="/test/"+model+"/"+fileName;//拼接百度云地址
            String redisKey="partETags-"+model+"-"+fileName;//缓存的键
            Integer current =fileinfo.getChunk();//当前切片数
            Integer total = fileinfo.getChunks();//总切片数
            Long size=multipartFile.getSize();//文件大小
            String uploadId = getUploadId(redisKey, filePath);//该方法获取百度云的唯一标识uploadId
            byte[] file = multipartFile.getBytes();//将切片过来的文件转成字节数组
            if(current==1){//这块进行断点续传，如果是第一个切片，查询一下缓存中是否有数据了，如果有返回给前台，如果没有进行上传
                if(getCurrentEqualsOne(redisKey).size()>0) {
                    //给前台提供是否缓存中已有数据
                    return getCurrentEqualsOne(redisKey);
                }
            }
            //切片上传百度云
            List<CustomPartETag> customPartETags = getPartEtags(redisKey, uploadId, filePath, file, size, current);
            //如果缓存中成功的切片数等于总数，那么进行合并
            if (customPartETags.size() == total) {
                customPartETags = customPartETags.stream().sorted(Comparator.comparing(PartETag::getPartNumber)).collect(Collectors.toList());//进行排序
                baiduCloudBosService.completePart(uploadId, customPartETags, filePath);//执行合并
                RedisUtils.hdel(redisKey,"uploadId","customPartETags");//如果合并成功，删除缓存中的数据
                // 入库做记录
                m.put("key","success");
                m.put("value","上传成功");
                return m;
            }
            m.put("key","continue");
            m.put("value",current+1);
            return m;
        }catch (Exception e){
            e.printStackTrace();
            m.put("key","error");
            m.put("value","上传失败");
            return m;
        }
    }

    /**
     * 获取 part
     *
     * @param redisPartKey redisKey
     * @return
     */
    private List<CustomPartETag> getPartEtags(String redisPartKey, String uploadId, String filePath,  byte[] file, long size, int current) {
        List<CustomPartETag> customPartETags = null;
        try {
            lock.acquire();
            //因为每次上传成功后都会把成功的切片存进百度云，如注解下方第四行代码，所以每次上传之前需要先取出来，将成功的切片放进去
            customPartETags = RedisUtils.hget(redisPartKey, "customPartETags") == null ? new ArrayList<>() : JSON.parseArray((String) RedisUtils.hget(redisPartKey, "customPartETags"), CustomPartETag.class);
            //上传百度云成功之后会返回一个PartETagg对象，由于项目中存在dubbo，顾将CustomPartETag类继承PartETagg进行序列化
            CustomPartETag customPartETag = baiduCloudBosService.uploadCustomPartETag(uploadId, filePath, file, size, current);
            customPartETags.add(customPartETag);
            setPartEtagsToRedis(redisPartKey, customPartETags);

        } catch (InterruptedException e) {
            e.printStackTrace();
        } finally {
            lock.release();
        }
        return customPartETags;
    }

    /**
     * 获取 uploadId
     *
     * @param redisPartKey redisKey
     * @param filePath     filePath
     * @return
     */
    private String getUploadId(String redisPartKey, String filePath) {
        String uploadId = null;
        try {
            lock.acquire();//线程锁
            uploadId = (String) RedisUtils.hget(redisPartKey, "uploadId");//先去redis中找，是否已经有uploadId
            if (StringUtils.isEmpty(uploadId)) {//判断是否为空，如果为空，说明之前没上传过，为首次上传，如果已经存在，返回该uploadId
                uploadId = baiduCloudBosService.initMultipartUpload(filePath);//进行初始化，返回一个唯一标识uploadId
                RedisUtils.hset(redisPartKey, "uploadId", uploadId, 60 * 60);//将返回来的uploadId放到缓存中，下个切片上传直接使用这个uploadId
            }
        } catch (InterruptedException e) {
            e.printStackTrace();
        } finally {
            lock.release();
        }
        return uploadId;
    }

    /**
     * 薄墩 partKey到redis
     *
     * @param redisPartKey
     * @param customPartETags
     */
    private void setPartEtagsToRedis(String redisPartKey, List<CustomPartETag> customPartETags) {
        RedisUtils.hset(redisPartKey, "customPartETags", JSON.toJSONString(customPartETags), 60 * 60);
    }
    public Map<String,Object> getCurrentEqualsOne(String redisKey){
        Map<String,Object> m=new HashMap<>();
        List<CustomPartETag>  customPartETags = RedisUtils.hget(redisKey, "customPartETags") == null ? new ArrayList<>() : JSON.parseArray((String) RedisUtils.hget(redisKey, "customPartETags"), CustomPartETag.class);
        //给前台提供是否缓存中已有数据
        if(customPartETags.size()==0){
            return m;
        }else{
                //缓存中有数据,返回缓存中数据
                int t=  customPartETags.get(customPartETags.size()-1).getPartNumber();
                m.put("key","hasValue");
                m.put("value",t);
                return  m;
            }
    }

}

3.远程的service,(如果不用远程调用，直接创建一个service就可以)

package com.geely.tcm.sheet.service;

import com.baidubce.services.bos.BosClient;
import com.baidubce.services.bos.model.*;
import com.geely.tcm.common.bean.baidu.BosResult;
import com.geely.tcm.common.bean.baidu.CustomPartETag;
import com.geely.tcm.common.util.StringUtils;
import com.geely.tcom.remote.api.baidu.BaiduCloudBosService;
import lombok.extern.slf4j.Slf4j;
import org.apache.dubbo.config.annotation.DubboService;
import org.springframework.beans.factory.annotation.Autowired;

import java.io.*;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

/**
 * 百度私有云服务操作
 *
 * @author : wangbin
 * @date : 2021/9/24
 * @since : 1.0.0
 */
@Slf4j
@DubboService(version = "${sheet.service.version}")
public class BaiduCloudBosServiceImpl{

    @Autowired
    private BosClient bosClient;

   
    public String initMultipartUpload(String filePath) {
        // 开始Multipart Upload
        InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(null, filePath);
        InitiateMultipartUploadResponse initiateMultipartUploadResponse = bosClient.initiateMultipartUpload(initiateMultipartUploadRequest);
        // 打印UploadId，它是区分分块上传事件的唯一标识
        String uploadId = initiateMultipartUploadResponse.getUploadId();
        log.info("filePath= {},uploadId = {}", filePath, uploadId);
        return uploadId;
    }


    public CustomPartETag uploadCustomPartETag(String uploadId, String filePath, byte[] file, long partSize, int partNumber) {
        UploadPartRequest uploadPartRequest = new UploadPartRequest();
        uploadPartRequest.setBucketName(null);
        uploadPartRequest.setKey(filePath);
        uploadPartRequest.setUploadId(uploadId);
        InputStream input= new ByteArrayInputStream(file);
        uploadPartRequest.setInputStream(input);
        uploadPartRequest.setPartSize(partSize);
        uploadPartRequest.setPartNumber(partNumber );
        uploadPartRequest.setMd5Digest("");
        UploadPartResponse uploadPartResponse = bosClient.uploadPart(uploadPartRequest);
        PartETag partETag = uploadPartResponse.getPartETag();
        CustomPartETag  customPartETag = new CustomPartETag();
        customPartETag.setETag(partETag.getETag());
        customPartETag.setPartNumber(partETag.getPartNumber());
        return customPartETag;
    }

    /**
     * 分块上传结束之后，当所有的数据Part验证通过后，BOS将把这些数据part组合成一个完整的Object
     *
     * @param uploadId
     * @param customPartETags
     * @param filePath
     */
    public void completePart(String uploadId, List<CustomPartETag> customPartETags, String filePath) {
        //for循环转一下 CustomPartETag 转PartETag
        if(customPartETags.size()>0){
            List<PartETag> partETags=new ArrayList<>();
            for(CustomPartETag c:customPartETags){
                PartETag partETag=new PartETag();
                partETag.setETag(c.getETag());
                partETag.setPartNumber(c.getPartNumber());
                partETags.add(partETag);
            }
            CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(null, filePath, uploadId, partETags);
            CompleteMultipartUploadResponse completeMultipartUploadResponse = bosClient.completeMultipartUpload(completeMultipartUploadRequest);
            log.info("完成分片上传 ， filePath = {},uploadId = {}, etag = {}", filePath, uploadId,completeMultipartUploadResponse.getETag());
        }
    }

   
}

4.定义一个实体类，继承，实现序列化，（dubbo不序列化就会传不过去）

import com.baidubce.services.bos.model.PartETag;
import java.io.Serializable;

public class CustomPartETag extends PartETag implements Serializable {


}

5.这里提供一个临时测试的前端代码。

<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>upload</title>
    <script src="https://cdn.bootcss.com/jquery/3.2.1/jquery.min.js"></script>
</head>
<body>
    <input type="file" name="file" id="file">
    <button id="upload" onClick="upload()">upload</button>
    <script type="text/javascript">
        var bytesPerPiece = 1024 * 1024; // 每个文件切片大小定为1MB .
        var totalPieces;
        
        function upload() {
            var blob = document.getElementById("file").files[0];
            var start = 0;
            var end;
            var index = 1;
            var filesize = blob.size;
            var filename = blob.name;

            //计算文件切片总数
            totalPieces = Math.ceil(filesize / bytesPerPiece);
            while(start < filesize) {
                end = start + bytesPerPiece;
                if(end > filesize) {
                    end = filesize;
                }

                var chunk = blob.slice(start,end);//切割文件    
                var sliceIndex= blob.name + index;
                var formData = new FormData();
                formData.append("file", chunk, filename);
				formData.append("totalSize",filesize);
				formData.append("chunks",totalPieces);
				formData.append("chunk",index);
				formData.append("model",'kx11');
                $.ajax({
//路径换成你的路径
                    url: 'http://127.0.0.1:6300/threeModel/upload3DToBos',
                    type: 'POST',
					dataType:'json',
                    cache: false,
                    data: formData,
                    processData: false,
                    contentType: false,
					mimeType: "multipart/form-data",
					success: function (result){
						console.log(result.value);
					}
				});
				
				
                start = end;
                index++;
            }
        }
    </script>
</body>
</html>