Read a large file line-by-line in Node.js using streams + readline (memory efficient, backpressure-friendly).

node-read-large-file-line-by-line.js

#!/usr/bin/env node
/**
 * Read a large file line-by-line (streams) in Node.js.
 *
 * Usage:
 *   node node-read-large-file-line-by-line.js ./bigfile.txt
 *
 * Notes:
 * - Uses streams to avoid loading the entire file into memory.
 * - readline provides a clean async iterator interface.
 * - Handles Windows line endings via crlfDelay: Infinity
 */

const fs = require("fs");
const readline = require("readline");
const path = require("path");

async function readLines(filePath, onLine) {
  const absolute = path.resolve(filePath);

  const stream = fs.createReadStream(absolute, {
    encoding: "utf8",
    highWaterMark: 1024 * 1024, // 1MB chunks (tune if needed)
  });

  stream.on("error", (err) => {
    throw err;
  });

  const rl = readline.createInterface({
    input: stream,
    crlfDelay: Infinity,
  });

  let lineNum = 0;

  try {
    for await (const line of rl) {
      lineNum += 1;
      await onLine(line, lineNum);
    }
  } finally {
    rl.close();
  }
}

async function main() {
  const filePath = process.argv[2];
  if (!filePath || filePath === "-h" || filePath === "--help") {
    console.error("Usage: node node-read-large-file-line-by-line.js <file>");
    process.exit(2);
  }

  // Example: count lines + print the first 5 lines
  let total = 0;

  await readLines(filePath, async (line, lineNum) => {
    total += 1;
    if (lineNum <= 5) {
      console.log(`${lineNum}: ${line}`);
    }

    // Example: simulate async processing
    // if (lineNum % 10000 === 0) await new Promise(r => setTimeout(r, 1));
  });

  console.error(`Done. Total lines: ${total}`);
}

main().catch((err) => {
  console.error("ERROR:", err && err.stack ? err.stack : err);
  process.exit(1);
});