Effective Ways to Use cURL in JavaScript for Web Scraping

Every second, billions of data packets zoom across the internet. If you want to tap into that flow — whether for scraping, API integrations, or automated testing — mastering cURL in JavaScript is a game-changer.
cURL isn’t just a command-line relic. When paired with JavaScript, it becomes a powerhouse for handling complex network requests efficiently and reliably. Let’s show you how to wield that power, step-by-step.

Why Use cURL in JavaScript

Control over network requests beyond basic fetch or XMLHttpRequest.
Support for multiple protocols (HTTP, HTTPS, FTP, and more).
Ideal for web scraping, downloading files, and stress-testing APIs.
And the best part? You’re not locked into a single way to use cURL in JavaScript. Let’s explore four practical methods, plus pro tips to keep your requests bulletproof.

What You Need

Before diving in, make sure you have:

Node.js (version 12+ recommended)
npm (to manage packages)
Basic understanding of asynchronous JavaScript (async/await or Promises)
cURL installed on your machine (for shell command approaches)

Set up your project folder:

mkdir curl-js-project
cd curl-js-project
npm init -y

Method 1: Use Node.js to Run cURL

This is the quickest way to run cURL commands inside JavaScript using Node’s built-in tools.

const { exec } = require('child_process');

function curlRequest(url, options = {}) {
  return new Promise((resolve, reject) => {
    let curlCommand = `curl -s "${url}"`;

    if (options.headers) {
      Object.entries(options.headers).forEach(([key, value]) => {
        curlCommand += ` -H "${key}: ${value}"`;
      });
    }

    if (options.method) {
      curlCommand += ` -X ${options.method}`;
    }

    if (options.data) {
      curlCommand += ` -d '${JSON.stringify(options.data)}'`;
    }

    exec(curlCommand, (error, stdout, stderr) => {
      if (error) return reject(new Error(error.message));
      if (stderr) return reject(new Error(stderr));

      try {
        resolve(JSON.parse(stdout));
      } catch {
        resolve(stdout);
      }
    });
  });
}

Heads up:

This requires cURL on your system.
Building commands by string concatenation can be risky — watch out for injection vulnerabilities.
Error handling is basic.
Still, it’s a neat way to start.

Method 2: Use the node-libcurl Package for Direct Libcurl Access

Want more control and better performance? node-libcurl hooks directly into libcurl’s powerful API.

Install it:

npm install node-libcurl

Example:

const { Curl } = require('node-libcurl');

function performRequest(url, options = {}) {
  return new Promise((resolve, reject) => {
    const curl = new Curl();

    curl.setOpt(Curl.option.URL, url);
    if (options.headers) {
      const headerList = Object.entries(options.headers).map(([k, v]) => `${k}: ${v}`);
      curl.setOpt(Curl.option.HTTPHEADER, headerList);
    }

    if (options.method === 'POST' && options.data) {
      curl.setOpt(Curl.option.POST, true);
      curl.setOpt(Curl.option.POSTFIELDS, JSON.stringify(options.data));
    }

    curl.setOpt(Curl.option.FOLLOWLOCATION, true);
    if (options.timeout) curl.setOpt(Curl.option.TIMEOUT, options.timeout);

    curl.on('end', (statusCode, data, headers) => {
      try {
        resolve({ statusCode, data: JSON.parse(data), headers });
      } catch {
        resolve({ statusCode, data, headers });
      }
      curl.close();
    });

    curl.on('error', (error) => {
      reject(error);
      curl.close();
    });

    curl.perform();
  });
}

Why choose this?

Faster and more efficient than shell commands.
Precise control over request options.
Solid error handling.

Method 3: Use request-promise for Simpler Syntax

If you want a straightforward interface with cURL-like capabilities, request-promise delivers.

Install it:

npm install request request-promise

Here’s how:

const rp = require('request-promise');

async function curlLikeRequest(url, options = {}) {
  const requestOptions = {
    uri: url,
    method: options.method || 'GET',
    headers: options.headers || {},
    json: true,
    resolveWithFullResponse: true,
  };

  if (options.data && ['POST', 'PUT'].includes(options.method)) {
    requestOptions.body = options.data;
  }

  if (options.params) {
    requestOptions.qs = options.params;
  }

  try {
    const response = await rp(requestOptions);
    return {
      statusCode: response.statusCode,
      data: response.body,
      headers: response.headers,
    };
  } catch (error) {
    if (error.response) {
      throw {
        statusCode: error.statusCode,
        data: error.response.body,
        headers: error.response.headers,
      };
    }
    throw error;
  }
}

Method 4: Axios — The Modern HTTP Client

Axios is now a staple for many developers. It’s promise-based, works in browsers and Node, and makes HTTP requests slick and simple.

Install Axios:

npm install axios

Example usage:

const axios = require('axios');

async function curlWithAxios(url, options = {}) {
  const config = {
    url,
    method: options.method || 'GET',
    headers: options.headers || {},
    timeout: options.timeout || 10000,
    data: options.data || undefined,
    params: options.params || undefined,
  };

  try {
    const response = await axios(config);
    return {
      statusCode: response.status,
      data: response.data,
      headers: response.headers,
    };
  } catch (error) {
    if (error.response) {
      throw {
        statusCode: error.response.status,
        data: error.response.data,
        headers: error.response.headers,
      };
    } else if (error.request) {
      throw new Error('No response received');
    }
    throw error;
  }
}

Deal with Errors Professionally

Network requests fail. That’s a fact. Be ready.
Implement retries with exponential backoff. Detect status codes and handle them accordingly. Here’s a quick snippet for retries:

async function retryRequest(url, retries = 3, delay = 1000) {
  try {
    return await curlWithAxios(url);
  } catch (error) {
    if (retries > 0) {
      await new Promise(r => setTimeout(r, delay));
      return retryRequest(url, retries - 1, delay * 2);
    }
    throw error;
  }
}

Don’t ignore:

4xx errors: Often mean bad request or auth issues.
5xx errors: Server problems—retry might help.
Network timeouts: Adjust timeout settings per request.

Prevent Common Mistakes

Not handling failures: Crashes happen. Prepare for them.
Ignoring rate limits: Respect API limits or get banned.
Poor timeout settings: Too short or too long can cripple your app.
Skipping logging: Track errors with detail. You’ll thank yourself later.

Using Proxies with cURL in JavaScript

Proxies are essential when scraping or avoiding geo-blocks.

With child_process:

function curlThroughProxy(url, { host, port, username, password }) {
  return new Promise((resolve, reject) => {
    let proxyAuth = username && password ? `${username}:${password}` : '';
    let command = `curl -s -x ${host}:${port}`;

    if (proxyAuth) command += ` -U "${proxyAuth}"`;
    command += ` "${url}"`;

    exec(command, (error, stdout, stderr) => {
      if (error) return reject(error);
      if (stderr) return reject(stderr);

      try {
        resolve(JSON.parse(stdout));
      } catch {
        resolve(stdout);
      }
    });
  });
}

With node-libcurl:

const { Curl } = require('node-libcurl');

function requestViaProxy(url, proxyConfig) {
  return new Promise((resolve, reject) => {
    const curl = new Curl();
    curl.setOpt(Curl.option.URL, url);
    curl.setOpt(Curl.option.PROXY, `${proxyConfig.host}:${proxyConfig.port}`);
    curl.setOpt(Curl.option.PROXYTYPE, Curl.proxyType.HTTP);

    if (proxyConfig.username && proxyConfig.password) {
      curl.setOpt(Curl.option.PROXYUSERPWD, `${proxyConfig.username}:${proxyConfig.password}`);
    }

    curl.on('end', (statusCode, data, headers) => {
      try {
        resolve({ statusCode, data: JSON.parse(data), headers });
      } catch {
        resolve({ statusCode, data, headers });
      }
      curl.close();
    });

    curl.on('error', (error) => {
      reject(error);
      curl.close();
    });

    curl.perform();
  });
}

With Axios:

const axios = require('axios');
const HttpsProxyAgent = require('https-proxy-agent');

async function axiosWithProxy(url, proxyConfig) {
  const proxyUrl = proxyConfig.username
    ? `http://${proxyConfig.username}:${proxyConfig.password}@${proxyConfig.host}:${proxyConfig.port}`
    : `http://${proxyConfig.host}:${proxyConfig.port}`;

  const httpsAgent = new HttpsProxyAgent(proxyUrl);

  try {
    const response = await axios.get(url, { httpsAgent });
    return {
      statusCode: response.status,
      data: response.data,
      headers: response.headers,
    };
  } catch (error) {
    throw error;
  }
}

Wrapping Up

cURL in JavaScript is more than running commands; it’s about making reliable, flexible network requests for real applications. If you need a quick solution, use child_process. For more control, node-libcurl works well. request-promise offers simplicity, while Axios provides a modern, versatile client. Proper error handling and proxies are key to building stable, effective integrations.

Swiftproxy - Residential Proxies @swiftproxy_residential