Implementing Parallel Large File Download in JavaScript Using async‑pool and HTTP Range Requests

In this tutorial we explore how to download large files in parallel on the client side by leveraging the async-pool library, which internally uses Promise.all and Promise.race for concurrency control.

We first review HTTP range requests, which allow a server to send only a portion of a resource. When the response includes an Accept-Ranges header (value other than "none"), the client can request specific byte ranges using the Range header. Valid responses are 206 Partial Content for successful range requests and 416 Range Not Satisfiable for invalid ranges.

Next we define several helper functions needed for the download process.

function getContentLength(url) {
  return new Promise((resolve, reject) => {
    let xhr = new XMLHttpRequest();
    xhr.open("HEAD", url);
    xhr.send();
    xhr.onload = function () {
      resolve(~~xhr.getResponseHeader("Content-Length"));
    };
    xhr.onerror = reject;
  });
}

async function asyncPool(poolLimit, array, iteratorFn) {
  const ret = [];
  const executing = [];
  for (const item of array) {
    const p = Promise.resolve().then(() => iteratorFn(item, array));
    ret.push(p);
    if (poolLimit <= array.length) {
      const e = p.then(() => executing.splice(executing.indexOf(e), 1));
      executing.push(e);
      if (executing.length >= poolLimit) {
        await Promise.race(executing);
      }
    }
  }
  return Promise.all(ret);
}

function getBinaryContent(url, start, end, i) {
  return new Promise((resolve, reject) => {
    try {
      let xhr = new XMLHttpRequest();
      xhr.open("GET", url, true);
      xhr.setRequestHeader("range", `bytes=${start}-${end}`);
      xhr.responseType = "arraybuffer";
      xhr.onload = function () {
        resolve({ index: i, buffer: xhr.response });
      };
      xhr.send();
    } catch (err) {
      reject(new Error(err));
    }
  });
}

function concatenate(arrays) {
  if (!arrays.length) return null;
  let totalLength = arrays.reduce((acc, value) => acc + value.length, 0);
  let result = new Uint8Array(totalLength);
  let length = 0;
  for (let array of arrays) {
    result.set(array, length);
    length += array.length;
  }
  return result;
}

function saveAs({ name, buffers, mime = "application/octet-stream" }) {
  const blob = new Blob([buffers], { type: mime });
  const blobUrl = URL.createObjectURL(blob);
  const a = document.createElement("a");
  a.download = name || Math.random();
  a.href = blobUrl;
  a.click();
  URL.revokeObjectURL(blob);
}

async function download({ url, chunkSize, poolLimit = 1 }) {
  const contentLength = await getContentLength(url);
  const chunks = typeof chunkSize === "number" ? Math.ceil(contentLength / chunkSize) : 1;
  const results = await asyncPool(
    poolLimit,
    [...new Array(chunks).keys()],
    i => {
      let start = i * chunkSize;
      let end = i + 1 == chunks ? contentLength - 1 : (i + 1) * chunkSize - 1;
      return getBinaryContent(url, start, end, i);
    }
  );
  const sortedBuffers = results.map(item => new Uint8Array(item.buffer));
  return concatenate(sortedBuffers);
}

Using the above utilities we can perform a multi‑threaded download:

function multiThreadedDownload() {
  const url = document.querySelector("#fileUrl").value;
  if (!url || !/https?/.test(url)) return;
  console.log("Download start: " + new Date());
  download({
    url,
    chunkSize: 0.1 * 1024 * 1024,
    poolLimit: 6,
  }).then(buffers => {
    console.log("Download end: " + new Date());
    saveAs({ buffers, name: "my-archive", mime: "application/zip" });
  });
}

The article concludes that the asyncPool function not only enables parallel downloading but can also be adapted for parallel uploading, encouraging readers to experiment further.

References include articles on Blob handling, MDN documentation for ArrayBuffer and HTTP range requests, and the original tutorial on JavaScript concurrency control.