nodejs文件的分片与合并上传介绍

功能介绍：nodejs文件的分片与合并上传介绍！在JavaScript中，FIle对象是' Blob '对象的子类，该对象包含一个重要的方法slice，通过该方法我们可以这样分割二进制文件。

代码如下：

<!DOCTYPE html>
<html>
<head>
  <meta charset="UTF-8">
  <meta http-equiv="X-UA-Compatible" content="IE=edge">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>Document</title>
  <script src="https://cdn.bootcdn.net/ajax/libs/axios/0.24.0/axios.min.js"></script>
</head>
<body>
  <input type="file" multiple="multiple" id="fileInput" />
  <button onclick="SliceUpload()">上传</button>  
  <script>
    function SliceUpload() {
      const file = document.getElementById('fileInput').files[0]
      if (!file) return
	  
      // 文件分片
      let size = 1024 * 50; //50KB 50KB Section size
      let fileChunks = [];
      let index = 0;        //Section num
      for (let cur = 0; cur < file.size; cur += size) {
          fileChunks.push({
              hash: index++,
              chunk: file.slice(cur, cur + size),
          });
      }
	  
      // 上传分片
      const uploadList = fileChunks.map((item, index) => {
          let formData = new FormData();
          formData.append("filename", file.name);
          formData.append("hash", item.hash);
          formData.append("chunk", item.chunk);
          return axios({
              method: "post",
              url: "/upload",
              data: formData,
          });
      });
      await Promise.all(uploadList);
	  
      // 所有分片上传完成，通知服务器合并分片
      await axios({
          method: "get",
          url: "/merge",
          params: {
              filename: file.name,
          },
      });
      console.log("Upload to complete");
    }
  </script>
</body>
</html>

结合Promise.race()控制并发

// 加入并发控制
async function SliceUpload() {
  const file = document.getElementById('fileInput').files[0]
  if (!file) return
  
  // 文件分片
  let size = 1024 * 50; //50KB 50KB Section size
  let fileChunks = [];
  let index = 0;        //Section num
  for (let cur = 0; cur < file.size; cur += size) {
    fileChunks.push({
        hash: index++,
        chunk: file.slice(cur, cur + size),
    });
  }
  
  let pool = []; //Concurrent pool
  let max = 3; //Maximum concurrency
  for (let i = 0; i < fileChunks.length; i++) {
    let item = fileChunks[i];
    let formData = new FormData();
    formData.append("filename", file.name);
    formData.append("hash", item.hash);
    formData.append("chunk", item.chunk);
  
    // 上传分片
    let task = axios({
        method: "post",
        url: "/upload",
        data: formData,
    });
    task.then(() => {
    // 从并发池中移除已经完成的请求
    let index = pool.findIndex((t) => t === task);
        pool.splice(index);
    });
  
    // 把请求放入并发池中，如果已经达到最大并发量
    pool.push(task);
    if (pool.length === max) {
        //All requests are requested complete
        await Promise.race(pool);
    }
  }
  
  // 所有分片上传完成，通知服务器合并分片
  await axios({
    method: "get",
    url: "/merge",
    params: {
        filename: file.name,
    },
  });
  console.log("Upload to complete");
}

使代码可复用

function SliceUpload() {
  const file = document.getElementById('fileInput').files[0]
  if (!file) return
  
  // 文件分片
  let size = 1024 * 50; // 分片大小设置
  let fileChunks = [];
  let index = 0;        // 分片序号
  for (let cur = 0; cur < file.size; cur += size) {
    fileChunks.push({
      hash: index++,
      chunk: file.slice(cur, cur + size),
    });
  }
  
  const uploadFileChunks = async function(list){
    if(list.length === 0){
      // 所有分片上传完成，通知如无
      await axios({
        method: 'get',
        url: '/merge',
        params: {
            filename: file.name
        }
      });
      console.log('Upload to complete')
      return
    }
  
    let pool = []       // 并发池
    let max = 3         // 最大并发数
    let finish = 0      // 完成数量
    let failList = []   // 失败列表
    for(let i=0;i{
        // 从并发池中移除已经完成的请求
        let index = pool.findIndex(t=> t===task)
        pool.splice(index)
      }).catch(()=>{
          failList.push(item)
      }).finally(()=>{
        finish++
        // 如果有失败的重新上传
        if(finish===list.length){
            uploadFileChunks(failList)
        }
      })
      pool.push(task)
      if(pool.length === max){
        await Promise.race(pool)
      }
    }
  }
  
  uploadFileChunks(fileChunks)
}

服务端接口实现

const express = require('express')
const multiparty = require('multiparty')
const fs = require('fs')
const path = require('path')
const { Buffer } = require('buffer')
// file path
const STATIC_FILES = path.join(__dirname, './static/files')
// Temporary path to upload files
const STATIC_TEMPORARY = path.join(__dirname, './static/temporary')
const server = express()
// Static file hosting
server.use(express.static(path.join(__dirname, './dist')))
// Interface for uploading slices
server.post('/upload', (req, res) => {
  const form = new multiparty.Form();
  form.parse(req, function(err, fields, files) {
    let filename = fields.filename[0]
    let hash = fields.hash[0]
    let chunk = files.chunk[0]
    let dir = `${STATIC_TEMPORARY}/${filename}`
    // console.log(filename, hash, chunk)
    try {
      if (!fs.existsSync(dir)) fs.mkdirSync(dir)
      const buffer = fs.readFileSync(chunk.path)
      const ws = fs.createWriteStream(`${dir}/${hash}`)
      ws.write(buffer)
      ws.close()
      res.send(`${filename}-${hash} Section uploaded successfully`)
    } catch (error) {
      console.error(error)
      res.status(500).send(`${filename}-${hash} Section uploading failed`)
    }
  })
})

//Merged slice interface
server.get('/merge', async (req, res) => {
  const { filename } = req.query
  try {
    let len = 0
    const bufferList = fs.readdirSync(`${STATIC_TEMPORARY}/${filename}`).map((hash,index) => {
      const buffer = fs.readFileSync(`${STATIC_TEMPORARY}/${filename}/${index}`)
      len += buffer.length
      return buffer
    });
    //Merge files
    const buffer = Buffer.concat(bufferList, len);
    const ws = fs.createWriteStream(`${STATIC_FILES}/${filename}`)
    ws.write(buffer);
    ws.close();
    res.send(`Section merge completed`);
  } catch (error) {
      console.error(error);
  }
})

server.listen(3000, _ => {
  console.log('http://localhost:3000/')
})

html网页上如何把文件压缩成zip代码示列（前端HTML/JavaScript和后端Node.js + Express）

js根据相同key合并生成子数组children（兼容ie10）

javascript根据相同id合并形成child子数组（支持低版本浏览器）

nodejs爬取数据并生成Excel文件实现代码介绍

js数组里的对象相同key合并，其余多个相同key值相加

标签： express, 合并

上面是“nodejs文件的分片与合并上传介绍”的全面内容，想了解更多关于 nodejs 内容，请继续关注web建站教程。

当前网址：https://ipkd.cn/webs_2269.html