Release v2.1.0

CHANGELOG.md

@@ -1,5 +1,15 @@
 # check-http-status changelog
 
+## 2.1.0 - Oct 2, 2026
+
+### New
+
+- `onPageLimit` option: when `maxPages` is reached, ask whether to keep crawling instead of stopping (for apps built on the library)
+
+### Changed
+
+- `maxPages` (`--max-pages`) now defaults to 5000 instead of 1000
+
 ## 2.0.1 - Oct 2, 2026
 
 ### Fixed

README.md

@@ -159,7 +159,7 @@ checkHttpStatus({
 | `sitemaps` | Array | | *Optional.* Sitemap / sitemap index URLs. With `crawl`, their pages are crawled too and orphans are flagged; without it, their URLs are checked. |
 | `include` | Array / String | | Only crawl URLs matching one of these rules. |
 | `exclude` | Array / String | | Don't crawl URLs matching these rules. They are still listed and checked. |
-| `maxPages` | Number | `1000` | Maximum pages to crawl. Later pages are listed as "Skipped (max pages)". |
+| `maxPages` | Number | `5000` | Maximum pages to crawl. Later pages are listed as "Skipped (max pages)". |
 | `maxDepth` | Number | unlimited | Maximum link depth from the start URL. |
 | `checkExternal` | Boolean | `true` | Check the status of external links. Excluded pages and assets are always checked. |
 | `checkAssets` | Boolean | `false` | Also check images, scripts, stylesheets, icons and iframes. |
@@ -183,6 +183,7 @@ checkHttpStatus({
 | `signal` | AbortSignal | | Stop the crawl. Everything checked so far is returned; unfinished URLs are "Not checked". |
 | `onResult` | Function | | Called with each result as soon as its URL is checked (live results). |
 | `onProgress` | Function | | Called with progress stats after each URL. |
+| `onPageLimit` | Function | | Called when everything up to `maxPages` is checked but more pages are waiting, instead of stopping. Return (or resolve) `true` to crawl up to `maxPages` more, a number to crawl that many more (`Infinity` for all), or `false` to finish. Receives the progress stats plus `waitingPages`. |
 
 Provide at least one of `crawl`, `urls` or `sitemaps`. For a full site audit, `crawl` on its own is enough.
 

bin/check-http-status.js

@@ -25,7 +25,7 @@ Crawl options
   -e, --exclude <text>     Don't crawl URLs containing <text>; they are still listed
                            and their status is checked (repeatable)
                            Rules may use * wildcards or /regex/
-      --max-pages <n>      Max pages to crawl (default 1000)
+      --max-pages <n>      Max pages to crawl (default 5000)
       --max-depth <n>      Max link depth from the start URL (default unlimited)
       --no-external        Don't check the status of external links (excluded
                            pages and assets are still checked)

docs/index.html

@@ -228,7 +228,7 @@ <h2>Options</h2>
           <tr><td><code>-s, --sitemap</code></td><td><code>sitemaps</code></td><td></td><td><em>Optional.</em> Sitemap or sitemap index URL(s). Not needed for crawling. Gzip supported.</td></tr>
           <tr><td><code>-i, --include</code></td><td><code>include</code></td><td></td><td>Only crawl URLs matching one of these rules.</td></tr>
           <tr><td><code>-e, --exclude</code></td><td><code>exclude</code></td><td></td><td>Don't crawl matching URLs. They're still listed and checked.</td></tr>
-          <tr><td><code>--max-pages</code></td><td><code>maxPages</code></td><td>1000</td><td>Maximum pages to crawl.</td></tr>
+          <tr><td><code>--max-pages</code></td><td><code>maxPages</code></td><td>5000</td><td>Maximum pages to crawl.</td></tr>
           <tr><td><code>--max-depth</code></td><td><code>maxDepth</code></td><td>∞</td><td>Maximum link depth from the start URL.</td></tr>
           <tr><td><code>--no-external</code></td><td><code>checkExternal: false</code></td><td>checked</td><td>Don't check external links. Excluded pages and assets are still checked.</td></tr>
           <tr><td><code>--assets</code></td><td><code>checkAssets</code></td><td>off</td><td>Also check images, scripts, CSS and iframes.</td></tr>

index.d.ts

@@ -96,7 +96,7 @@ declare namespace checkHttpStatus {
     include?: string | string[];
     /** Don't crawl matching URLs; they're still listed and checked. */
     exclude?: string | string[];
-    /** Default 1000. */
+    /** Default 5000. */
     maxPages?: number;
     /** Maximum link depth from the start URL. Default unlimited. */
     maxDepth?: number | null;
@@ -139,6 +139,8 @@ declare namespace checkHttpStatus {
     onResult?: (result: Result) => void;
     /** Called with progress after each URL. */
     onProgress?: (progress: Progress) => void;
+    /** At `maxPages`: `true` crawls `maxPages` more, a number that many (`Infinity` all), `false` stops. */
+    onPageLimit?: (progress: PageLimitProgress) => boolean | number | Promise<boolean | number>;
   }
 
   interface FoundOn {
@@ -203,6 +205,11 @@ declare namespace checkHttpStatus {
     pageLimitReached: boolean;
   }
 
+  interface PageLimitProgress extends Progress {
+    /** Pages found after `maxPages` was reached. */
+    waitingPages: number;
+  }
+
   interface Summary extends Progress {
     crawl: string[];
     sitemaps: string[];

index.js

@@ -67,8 +67,9 @@ function normalizeConfig(config) {
     'ignoreQuery': !!config.ignoreQuery,
     'include': rules(config.include),
     'maxDepth': config.maxDepth === undefined || config.maxDepth === null || config.maxDepth === '' ? null : toInt(config.maxDepth, null, 0, 10000),
-    'maxPages': toInt(config.maxPages, 1000, 1, 1000000),
+    'maxPages': toInt(config.maxPages, 5000, 1, 1000000),
     'maxRedirects': toInt(config.maxRedirects, 10, 0, 50),
+    'onPageLimit': typeof config.onPageLimit === 'function' ? config.onPageLimit : null,
     'onProgress': typeof config.onProgress === 'function' ? config.onProgress : null,
     'onResult': typeof config.onResult === 'function' ? config.onResult : null,
     // CLI: print the URL list even when reports are saved.

lib/checker.js

@@ -52,6 +52,7 @@ class Checker {
     this.include = compilePatterns(config.include);
     this.exclude = compilePatterns(config.exclude);
     this.crawlMode = config.crawl.length > 0;
+    this.initialMaxPages = config.maxPages;
     // In list mode "internal" means the hosts of the URLs the user supplied.
     this.rootHosts = new Set((this.crawlMode ? config.crawl : config.urls).map((url) => stripWww(new URL(url).host)));
 
@@ -63,6 +64,7 @@ class Checker {
     this.pagesCrawled = 0;
     this.checked = 0;
     this.stopped = false;
+    this.askingPageLimit = false;
     this.state = 'idle';
     this.startedAt = null;
     this.finishedAt = null;
@@ -306,11 +308,67 @@ class Checker {
       return;
     }
 
-    if (this.stopped || (this.pageQueue.length === 0 && this.checkQueue.length === 0)) {
-      this.state = this.stopped ? 'stopped' : 'done';
-      this.finishedAt = Date.now();
-      this.resolveDone();
+    if (this.stopped) {
+      this.finish();
+    } else if (this.pageQueue.length === 0 && this.checkQueue.length === 0 && !this.askingPageLimit) {
+      if (this.config.onPageLimit && this.waitingPages() > 0) {
+        this.askPageLimit();
+      } else {
+        this.finish();
+      }
+    }
+  }
+
+  finish() {
+    this.askingPageLimit = false;
+    this.state = this.stopped ? 'stopped' : 'done';
+    this.finishedAt = Date.now();
+    this.resolveDone();
+  }
+
+  waitingPages() {
+    let count = 0;
+    for (const record of this.records.values()) {
+      if (record.scope === 'limit') {
+        count++;
+      }
     }
+
+    return count;
+  }
+
+  // `true`: `maxPages` more, a number: that many more, `Infinity`: all.
+  askPageLimit() {
+    this.askingPageLimit = true;
+    const progress = { ...this.progress(), 'waitingPages': this.waitingPages() };
+
+    Promise.resolve()
+      .then(() => this.config.onPageLimit(progress))
+      .then((answer) => answer, () => false)
+      .then((answer) => {
+        if (this.state !== 'running') {
+          return;
+        }
+
+        this.askingPageLimit = false;
+        const more = answer === true ? this.initialMaxPages : Number(answer);
+        if (this.stopped || !answer || !(more > 0)) {
+          this.finish();
+          return;
+        }
+
+        this.config.maxPages += more;
+        for (const record of this.records.values()) {
+          if (record.scope === 'limit' && this.crawlQueued < this.config.maxPages) {
+            record.scope = 'crawl';
+            this.crawlQueued++;
+            this.enqueue(record);
+          }
+        }
+
+        this.pump();
+        this.maybeFinish();
+      });
   }
 
   async process(record) {

package.json

@@ -1,7 +1,7 @@
 {
 	"name": "check-http-status",
 	"description": "Crawl a website or check a list of URLs: HTTP status codes, redirect chains, broken links and the pages they were found on. Export to Excel, CSV, JSON or an HTML report.",
-	"version": "2.0.1",
+	"version": "2.1.0",
 	"main": "index.js",
 	"types": "index.d.ts",
 	"bin": {

schema.json

@@ -29,7 +29,7 @@
     "sitemaps": { "$ref": "#/definitions/stringList", "description": "Optional sitemap / sitemap index URLs." },
     "include": { "$ref": "#/definitions/stringList", "description": "Only crawl URLs matching one of these rules (substring, * wildcard or /regex/)." },
     "exclude": { "$ref": "#/definitions/stringList", "description": "Don't crawl URLs matching these rules. They are still listed and checked." },
-    "maxPages": { "type": "integer", "minimum": 1, "default": 1000 },
+    "maxPages": { "type": "integer", "minimum": 1, "default": 5000 },
     "maxDepth": { "type": ["integer", "null"], "minimum": 0, "description": "Maximum link depth from the start URL. Unlimited when omitted." },
     "maxRedirects": { "type": "integer", "minimum": 0, "maximum": 50, "default": 10 },
     "checkExternal": { "type": "boolean", "default": true, "description": "Check the status of external links. Excluded pages and assets are always checked." },

test-d/usage.ts

@@ -26,7 +26,8 @@ async function main(): Promise<void> {
     onProgress: (progress) => {
       const state: 'idle' | 'running' | 'done' | 'stopped' = progress.state;
       void state;
-    }
+    },
+    onPageLimit: async (progress) => progress.waitingPages > 100 ? 1000 : true
   });
 
   const notFound = report.results.filter((result) => result.category === 'Not Found');

test/checker.test.js

@@ -139,6 +139,53 @@ describe('crawl limits and rules', () => {
     assert.strictEqual(report.summary.pageLimitReached, true);
   });
 
+  test('onPageLimit can keep crawling past maxPages, in chunks or all at once', async () => {
+    const asked = [];
+    const full = await checkHttpStatus({ 'crawl': site.origin, 'silent': true });
+    const fullPages = full.results.filter((result) => result.scope === 'crawl').length;
+
+    const chunked = await checkHttpStatus({
+      'crawl': site.origin,
+      'maxPages': 3,
+      'onPageLimit': (progress) => {
+        asked.push(progress);
+        return asked.length === 1;
+      },
+      'silent': true
+    });
+    assert.strictEqual(asked.length, 2);
+    assert.strictEqual(asked[0].pagesCrawled, 3);
+    assert.ok(asked[0].waitingPages > 0);
+    assert.strictEqual(chunked.results.filter((result) => result.scope === 'crawl').length, 6);
+    assert.ok(chunked.results.some((result) => result.scope === 'limit'));
+    assert.strictEqual(chunked.summary.state, 'done');
+
+    const all = await checkHttpStatus({ 'crawl': site.origin, 'maxPages': 3, 'onPageLimit': async () => Infinity, 'silent': true });
+    assert.strictEqual(all.results.filter((result) => result.scope === 'crawl').length, fullPages);
+    assert.ok(!all.results.some((result) => result.scope === 'limit'));
+  });
+
+  test('onPageLimit answering false, or aborting while asked, keeps the v2 result', async () => {
+    const declined = await checkHttpStatus({ 'crawl': site.origin, 'maxPages': 3, 'onPageLimit': () => false, 'silent': true });
+    assert.strictEqual(declined.results.filter((result) => result.scope === 'crawl').length, 3);
+    assert.strictEqual(declined.summary.pageLimitReached, true);
+
+    const controller = new AbortController();
+    const aborted = await checkHttpStatus({
+      'crawl': site.origin,
+      'maxPages': 3,
+      // Never answers: the abort has to end the crawl.
+      'onPageLimit': () => {
+        setTimeout(() => controller.abort(), 20);
+        return new Promise(() => {});
+      },
+      'signal': controller.signal,
+      'silent': true
+    });
+    assert.strictEqual(aborted.summary.state, 'stopped');
+    assert.strictEqual(aborted.results.filter((result) => result.scope === 'crawl').length, 3);
+  });
+
   test('checkExternal: false skips external links but still checks excluded pages and assets', async () => {
     const report = await checkHttpStatus({ 'checkAssets': true, 'checkExternal': false, 'crawl': site.origin, 'exclude': ['/docs'], 'silent': true });
     assert.strictEqual(find(report, '/partners', site.externalOrigin).category, 'Not checked');