springboot2.0结合webuploader实现文件分片上传

[toc]

1. 上传页面代码

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>webuploader文件上传</title>
    <!--引入CSS-->
    <link rel="stylesheet" type="text/css" href="https://my.oschina.net//u/4308120/blog/3321016/webuploader/webuploader.css">
    <!--引入JS-->
    <script type="text/javascript" src="https://cdn.bootcss.com/jquery/3.4.1/jquery.js"></script>
    <script type="text/javascript" src="https://my.oschina.net//u/4308120/blog/3321016/webuploader/webuploader.js"></script>
</head>
<body>
<div id="uploader" class="wu-example">
    <!--用来存放文件信息-->
    <div id="thelist" class="uploader-list"></div>
    <div class="btns">
        <div id="picker">选择文件</div>
        <button id="ctlBtn" class="btn btn-default" onclick="upload()">开始上传</button>
    </div>
</div>

</body>
<script>
    //
    WebUploader.Uploader.register({
            "before-send-file": "beforeSendFile",
            "before-send": "beforeSend",
            "after-send-file": "afterSendFile"
        }, {
            //在上传文件前执行，检查待文件是否存在
            beforeSendFile: function (file) {
                // 创建一个deffered,用于通知是否完成操作
                var deferred = WebUploader.Deferred();
                // 计算文件的唯一标识，用于断点续传
                (new WebUploader.Uploader()).md5File(file)
                    .then(function (val) {

                        this.fileMd5 = val;
                        this.uploadFile = file;
//                alert(this.fileMd5 )
                        //向服务端请求注册上传文件
                        $.ajax(
                            {
                                type: "POST",
                                url: "http://127.0.0.1:31400/media/upload/register",
                                data: {
                                    // 文件唯一表示
                                    fileMd5: this.fileMd5,
                                    fileName: file.name,
                                    fileSize: file.size,
                                    mimetype: file.type,
                                    fileExt: file.ext
                                },
                                dataType: "json",
                                success: function (response) {
                                    if (response.success) {
                                        //alert('上传文件注册成功开始上传');
                                        deferred.resolve();
                                    } else {
                                        alert(response.message);
                                        deferred.reject();
                                    }
                                }
                            }
                        );
                    }.bind(this));

                return deferred.promise();
            }.bind(this),
            //在上传分片前执行，检查分片是否已经存在
            beforeSend: function (block) {
                var deferred = WebUploader.Deferred();
                // 每次上传分块前校验分块，如果已存在分块则不再上传，达到断点续传的目的
                $.ajax(
                    {
                        type: "POST",
                        url: "http://127.0.0.1:31400/media/upload/checkchunk",
                        data: {
                            // 文件唯一表示
                            fileMd5: this.fileMd5,
                            // 当前分块下标
                            chunk: block.chunk,
                            // 当前分块大小
                            chunkSize: block.end - block.start
                        },
                        dataType: "json",
                        success: function (response) {
                            if (response.fileExist) {
                                // 分块存在，跳过该分块
                                deferred.reject();
                            } else {
                                // 分块不存在或不完整，重新发送
                                deferred.resolve();
                            }
                        }
                    }
                );
                //构建fileMd5参数，上传分块时带上fileMd5
                this.uploader.options.formData.fileMd5 = this.fileMd5;
                this.uploader.options.formData.chunk = block.chunk;
                return deferred.promise();
            }.bind(this),
            // 在上传分片完成后触发，用于后台处理合成分片，判断上传文件是否成功以及可以携带返回上传文件成功的url
            afterSendFile: function (file) {
                // 合并分块
                $.ajax(
                    {
                        type: "POST",
                        url: "http://127.0.0.1:31400/media/upload/mergechunks",
                        data: {
                            fileMd5: this.fileMd5,
                            fileName: file.name,
                            fileSize: file.size,
                            mimetype: file.type,
                            fileExt: file.ext
                        },
                        success: function (response) {
                            //在这里解析合并成功结果
                            if (response && response.success) {
                                alert("上传成功")
                            } else {
                                alert("上传失败")
                            }
                        }
                    }
                );
            }.bind(this)
        }
    );
    //创建webuploader实例
    var uploader = WebUploader.create(
        {
            swf: "./webuploader/Uploader.swf",//上传文件的flash文件，浏览器不支持h5时启动flash
            server: "http://127.0.0.1:31400/media/upload/uploadchunk",//上传分块的服务端地址，注意跨域问题
            fileVal: "file",//文件上传域的name
            pick: "#picker",//指定选择文件的按钮容器
            auto: false,//手动触发上传
            disableGlobalDnd: true,//禁掉整个页面的拖拽功能
            chunked: true,// 是否分块上传
            chunkSize: 1 * 1024 * 1024, // 分块大小（默认5M）
            threads: 3, // 开启多个线程（默认3个）
            prepareNextFile: true// 允许在文件传输时提前把下一个文件准备好
        }
    );
    // 将文件添加到队列
    uploader.on("fileQueued", function (file) {
            this.uploadFile = file;
            this.percentage = 0;

        }.bind(this)
    );
    //选择文件后触发
    uploader.on("beforeFileQueued", function (file) {
//     this.uploader.removeFile(file)
        //重置uploader
        this.uploader.reset()
        this.percentage = 0;
    }.bind(this));

    // 监控上传进度
    // percentage:代表上传文件的百分比
    uploader.on("uploadProgress", function (file, percentage) {
        this.percentage = Math.ceil(percentage * 100);
        console.log(percentage)
    }.bind(this));
    //上传失败触发
    uploader.on("uploadError", function (file, reason) {
        console.log(reason)
        alert("上传文件失败");
    });
    //上传成功触发
    uploader.on("uploadSuccess", function (file, response) {
        console.log(response)
//        alert("上传文件成功！");
    });
    //每个分块上传请求后触发
    uploader.on('uploadAccept', function (file, response) {
        if (!(response && response.success)) {//分块上传失败，返回false
            return false;
        }
    });

    //触发执行上传
    function upload(){
        if(this.uploadFile && this.uploadFile.id){
            this.uploader.upload(this.uploadFile.id);
        }else{
            alert("请选择文件");
        }
    }

</script>
</html>

2. nginx配置

worker_processes  1;


events {
    worker_connections  1024;
}

http {
    include       mime.types;
    default_type  application/octet-stream;
    sendfile        on;
    keepalive_timeout  65;

    server {
        listen 80;
        server_name video.xuecheng.com;

        location / {
            alias  D:/test/webuploader-demo/;
        }    
    }
}

3. 后台主要代码

3.1 application.yml

server:
  port: 31400
spring:
  application:
    name: xc-service-manage-media
  data:
    mongodb:
      uri:  mongodb://root:root@localhost:27017
      database: xc_media
xc-service-manage-media:
  upload-location: D:/test/video/

3.2 跨域处理

package com.xuecheng.manage_media.config;

import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.cors.CorsConfiguration;
import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
import org.springframework.web.filter.CorsFilter;

/**
 * @author john
 * @date 2020/1/1 - 15:07
 */
@Configuration
public class CorsConfig {
    @Bean
    public CorsFilter corsFilter() {
        //1.添加CORS配置信息
        CorsConfiguration config = new CorsConfiguration();
        //1) 允许的域,不要写*，否则cookie就无法使用了
        config.addAllowedOrigin("http://video.xuecheng.com");
        //2) 是否发送Cookie信息
        config.setAllowCredentials(true);
        //3) 允许的请求方式
        config.addAllowedMethod("OPTIONS");
        config.addAllowedMethod("HEAD");
        config.addAllowedMethod("GET");
        config.addAllowedMethod("PUT");
        config.addAllowedMethod("POST");
        config.addAllowedMethod("DELETE");
        config.addAllowedMethod("PATCH");
        // 4）允许的头信息
        config.addAllowedHeader("*");

        //2.添加映射路径，我们拦截一切请求
        UrlBasedCorsConfigurationSource configSource = new UrlBasedCorsConfigurationSource();
        configSource.registerCorsConfiguration("/**", config);

        //3.返回新的CorsFilter.
        return new CorsFilter(configSource);
    }
}

3.3 控制器代码

package com.xuecheng.manage_media.controller;

import com.xuecheng.framework.domain.media.response.CheckChunkResult;
import com.xuecheng.framework.model.response.ResponseResult;
import com.xuecheng.manage_media.service.MediaUploadService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;

/**
 * @author john
 * @date 2020/1/1 - 13:13
 */
@RestController
@RequestMapping("/media/upload")
public class MediaUploadController {
    @Autowired
    MediaUploadService mediaUploadService;

    //检查文件是否已经存在
    @PostMapping("/register")
    public ResponseResult register(@RequestParam("fileMd5") String fileMd5,
                                   @RequestParam("fileName") String fileName,
                                   @RequestParam("fileSize") Long fileSize,
                                   @RequestParam("mimetype") String mimetype,
                                   @RequestParam("fileExt") String fileExt) {
        return mediaUploadService.register(fileMd5, fileName, fileSize, mimetype, fileExt);
    }

    //校验文件块
    @PostMapping("/checkchunk")
    public CheckChunkResult checkchunk(@RequestParam("fileMd5") String fileMd5,
                                       @RequestParam("chunk") Integer chunk,
                                       @RequestParam("chunkSize") Integer chunkSize) {
        return mediaUploadService.checkChunk(fileMd5, chunk, chunkSize);
    }

    //上传文件块
    @PostMapping("/uploadchunk")
    public ResponseResult uploadchunk(@RequestParam("file") MultipartFile file,
                                      @RequestParam("fileMd5") String fileMd5,
                                      @RequestParam("chunk") Integer chunk) {
        return mediaUploadService.uploadChunk(file, fileMd5, chunk);
    }

    //合并文件块
    @PostMapping("/mergechunks")
    public ResponseResult mergechunks(@RequestParam("fileMd5") String fileMd5,
                                      @RequestParam("fileName") String fileName,
                                      @RequestParam("fileSize") Long fileSize,
                                      @RequestParam("mimetype") String mimetype,
                                      @RequestParam("fileExt") String fileExt) {
        return mediaUploadService.mergeChunks(fileMd5, fileName, fileSize, mimetype, fileExt);
    }
}

3.4 service代码

package com.xuecheng.manage_media.service;

import com.xuecheng.framework.domain.media.MediaFile;
import com.xuecheng.framework.domain.media.response.CheckChunkResult;
import com.xuecheng.framework.domain.media.response.MediaCode;
import com.xuecheng.framework.exception.ExceptionCast;
import com.xuecheng.framework.model.response.CommonCode;
import com.xuecheng.framework.model.response.ResponseResult;
import com.xuecheng.manage_media.dao.MediaFileRepository;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;

import java.io.*;
import java.util.*;

/**
 * @author john
 * @date 2020/1/1   -   12:07
 */
@Service
@Slf4j
public class MediaUploadService {
    @Autowired
    MediaFileRepository mediaFileRepository;

    //上传文件的根目录
    @Value("${xc-service-manage-media.upload-location}")
    String uploadPath;

    /**
     * 根据文件md5得到文件路径
     * 规则：
     * 一级目录：md5的第一个字符
     * 二级目录：md5的第二个字符
     * 三级目录：md5
     * 文件名：md5+文件扩展名
     *
     * @param fileMd5 文件md5值
     * @param fileExt 文件扩展名
     * @return 文件路径
     */
    private String getFilePath(String fileMd5, String fileExt) {
        return getFileFolderPath(fileMd5) + fileMd5 + "." + fileExt;
    }

    //得到文件目录相对路径，路径中去掉根目录
    private String getFileFolderRelativePath(String fileMd5) {
        return fileMd5.substring(0, 1) + "/"
                + fileMd5.substring(1, 2) + "/"
                + fileMd5 + "/";
    }

    //得到文件所在目录
    private String getFileFolderPath(String fileMd5) {
        return uploadPath + getFileFolderRelativePath(fileMd5);
    }

    //创建文件目录
    private boolean createFileFold(String fileMd5) {
        //创建上传文件，目录
        String fileFolderPath = getFileFolderPath(fileMd5);
        File fileFolder = new File(fileFolderPath);
        if (!fileFolder.exists()) {
            //创建文件夹
            return fileFolder.mkdirs();
        }
        return true;
    }

    //检查待上传文件是否存在
    public ResponseResult register(String fileMd5, String fileName, Long fileSize, String mimetype, String fileExt) {
        //   检查文件是否上传
        //   1.   获取文件的路径
        String filePath = getFilePath(fileMd5, fileExt);
        File file = new File(filePath);

        //   2.   查询数据库文件是否存在
        Optional<MediaFile> optional = mediaFileRepository.findById(fileMd5);
        //   文件存在直接返回
        if (file.exists() && optional.isPresent()) {
            ExceptionCast.cast(MediaCode.UPLOAD_FILE_REGISTER_EXIST);
        }
        //下面说明文件并未上传
        boolean fileFold = createFileFold(fileMd5);
        if (!fileFold) {
            //上传文件目录创建失败
            ExceptionCast.cast(MediaCode.UPLOAD_FILE_REGISTER_CREATEFOLDER_FAIL);
        }
        return new ResponseResult(CommonCode.SUCCESS);
    }

    //   得到块文件所在目录
    private String getChunkFileFolderPath(String fileMd5) {
        return getFileFolderPath(fileMd5) + "/" + "chunks" + "/";
    }

    //检查块文件
    public CheckChunkResult checkChunk(String fileMd5, Integer chunk, Integer chunkSize) {
        //得到块文件所在路径
        String chunkFileFolderPath = getChunkFileFolderPath(fileMd5);
        //块文件的文件名称以1,2,3..序号命名，没有扩展名
        File chunkFile = new File(chunkFileFolderPath + chunk);
        if (chunkFile.exists()) {
            return new CheckChunkResult(MediaCode.CHUNK_FILE_EXIST_CHECK, true);
        } else {
            return new CheckChunkResult(MediaCode.CHUNK_FILE_EXIST_CHECK, false);
        }
    }

    private boolean createChunkFileFolder(String fileMd5) {
        //创建上传文件目录
        String chunkFileFolderPath = getChunkFileFolderPath(fileMd5);
        File chunkFileFolder = new File(chunkFileFolderPath);
        if (!chunkFileFolder.exists()) {
            //创建文件夹
            return chunkFileFolder.mkdirs();
        }
        return true;
    }

    //块文件上传
    public ResponseResult uploadChunk(MultipartFile file, String fileMd5, Integer chunk) {
        if (file == null) {
            ExceptionCast.cast(MediaCode.UPLOAD_FILE_REGISTER_ISNULL);
        }
        //创建块文件目录
        boolean fileFold = createChunkFileFolder(fileMd5);
        //块文件
        File chunkfile = new File(getChunkFileFolderPath(fileMd5) + chunk);
        //上传的块文件
        InputStream inputStream = null;
        FileOutputStream outputStream = null;
        try {
            inputStream = file.getInputStream();
            outputStream = new FileOutputStream(chunkfile);
            IOUtils.copy(inputStream, outputStream);
        } catch (Exception e) {
            e.printStackTrace();
            log.error("upload         chunk         file         fail:{}", e.getMessage());
            ExceptionCast.cast(MediaCode.CHUNK_FILE_UPLOAD_FAIL);
        } finally {
            try {
                inputStream.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
            try {
                outputStream.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return new ResponseResult(CommonCode.SUCCESS);
    }

    //合并块文件
    public ResponseResult mergeChunks(String fileMd5, String fileName, Long fileSize, String
            mimeType, String fileExt) {
        //获取块文件的路径
        String chunkFileFolderPath = getChunkFileFolderPath(fileMd5);
        File chunkFileFolder = new File(chunkFileFolderPath);
        if (!chunkFileFolder.exists()) {
            chunkFileFolder.mkdirs();
        }
        //合并文件路径
        File mergeFile = new File(getFilePath(fileMd5, fileExt));
        //创建合并文件
        //合并文件存在先删除再创建
        if (mergeFile.exists()) {
            mergeFile.delete();
        }
        boolean newFile = false;
        try {
            newFile = mergeFile.createNewFile();
        } catch (IOException e) {
            e.printStackTrace();
            log.error("mergechunks..create mergeFile fail:{}", e.getMessage());
        }
        if (!newFile) {
            ExceptionCast.cast(MediaCode.MERGE_FILE_CREATEFAIL);
        }
        //获取块文件，此列表是已经排好序的列表
        List<File> chunkFiles = getChunkFiles(chunkFileFolder);
        //合并文件
        mergeFile = mergeFile(mergeFile, chunkFiles);
        if (mergeFile == null) {
            ExceptionCast.cast(MediaCode.MERGE_FILE_FAIL);
        }
        //校验文件
        boolean checkResult = this.checkFileMd5(mergeFile, fileMd5);
        if (!checkResult) {
            ExceptionCast.cast(MediaCode.MERGE_FILE_CHECKFAIL);
        }
        //将文件信息保存到数据库
        MediaFile mediaFile = new MediaFile();
        mediaFile.setFileId(fileMd5);
        mediaFile.setFileName(fileMd5 + "." + fileExt);
        mediaFile.setFileOriginalName(fileName);
        //文件路径保存相对路径
        mediaFile.setFilePath(getFileFolderRelativePath(fileMd5));
        mediaFile.setFileSize(fileSize);
        mediaFile.setUploadTime(new Date());
        mediaFile.setMimeType(mimeType);
        mediaFile.setFileType(fileExt);
        //状态为上传成功
        mediaFile.setFileStatus("301002");
        MediaFile save = mediaFileRepository.save(mediaFile);
        return new ResponseResult(CommonCode.SUCCESS);
    }

    //校验文件的md5值
    private boolean checkFileMd5(File mergeFile, String md5) {
        if (mergeFile == null || StringUtils.isEmpty(md5)) {
            return false;
        }
        //进行md5校验
        FileInputStream mergeFileInputstream = null;
        try {
            mergeFileInputstream = new FileInputStream(mergeFile);
            //得到文件的md5
            String mergeFileMd5 = DigestUtils.md5Hex(mergeFileInputstream);
            //比较md5
            if (md5.equalsIgnoreCase(mergeFileMd5)) {
                return true;
            }
        } catch (Exception e) {
            e.printStackTrace();
            log.error("checkFileMd5 error,file is:{},md5 is: {} ", mergeFile.getAbsoluteFile(), md5);
        } finally {
            try {
                mergeFileInputstream.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return false;
    }

    //获取所有块文件
    private List<File> getChunkFiles(File chunkFileFolder) {
        //获取路径下的所有块文件
        File[] chunkFiles = chunkFileFolder.listFiles();
        //将文件数组转成list，并排序
        List<File> chunkFileList = new ArrayList<File>();
        chunkFileList.addAll(Arrays.asList(chunkFiles));
        //排序
        Collections.sort(chunkFileList, new Comparator<File>() {
            @Override
            public int compare(File o1, File o2) {
                if (Integer.parseInt(o1.getName()) > Integer.parseInt(o2.getName())) {
                    return 1;
                }
                return -1;
            }
        });
        return chunkFileList;
    }

    //合并文件
    private File mergeFile(File mergeFile, List<File> chunkFiles) {
        try {
            //创建写文件对象
            RandomAccessFile raf_write = new RandomAccessFile(mergeFile, "rw");
            //遍历分块文件开始合并
            //读取文件缓冲区
            byte[] b = new byte[1024];
            for (File chunkFile : chunkFiles) {
                RandomAccessFile raf_read = new RandomAccessFile(chunkFile, "r");
                int len = -1;
                //读取分块文件
                while ((len = raf_read.read(b)) != -1) {
                    //向合并文件中写数据
                    raf_write.write(b, 0, len);
                }
                raf_read.close();
            }
            raf_write.close();
        } catch (Exception e) {
            e.printStackTrace();
            log.error("merge file error:{}", e.getMessage());
            return null;
        }
        return mergeFile;
    }
}