Bot and AI crawler tracking

// proxy.ts (middleware.ts before Next.js 16)
import { NextResponse, type NextFetchEvent, type NextRequest } from "next/server"

const CRAWLER = /bot|crawl|spider|slurp|-user\b|claude|anthropic|cohere|meta-external|facebookexternalhit|google|bingpreview/i

export function proxy(request: NextRequest, event: NextFetchEvent) {
  const userAgent = request.headers.get("user-agent") ?? ""
  if (request.method === "GET" && CRAWLER.test(userAgent)) {
    const host = request.headers.get("x-forwarded-host") ?? request.headers.get("host")
    // Report in the background; never delay or fail the page.
    event.waitUntil(
      fetch("https://usedatasense.com/api/crawls", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({
          websiteId: "YOUR_SITE_ID",
          href: `https://${host}${request.nextUrl.pathname}`,
          ai: {
            userAgent,
            // Only trust these headers behind a proxy that sets them (Vercel, Cloudflare, nginx).
            ip: request.headers.get("cf-connecting-ip") ?? request.headers.get("x-real-ip") ??
              request.headers.get("x-forwarded-for")?.split(",")[0]?.trim(),
            source: "nextjs",
          },
        }),
        signal: AbortSignal.timeout(2000),
      }).catch(() => {}),
    )
  }
  return NextResponse.next()
}

export const config = {
  // Skip assets, but keep robots.txt, llms.txt and sitemaps: crawlers read those first.
  matcher: ["/((?!api|_next/static|_next/image|favicon.ico).*)"],
}

const CRAWLER = /bot|crawl|spider|slurp|-user\b|claude|anthropic|cohere|meta-external|facebookexternalhit|google|bingpreview/i

// Before your routes. Reports after the response is sent, with its status code.
app.use((req, res, next) => {
  const userAgent = req.get("user-agent") ?? ""
  if (req.method === "GET" && CRAWLER.test(userAgent)) {
    res.on("finish", () => {
      fetch("https://usedatasense.com/api/crawls", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({
          websiteId: "YOUR_SITE_ID",
          href: `${req.protocol}://${req.get("host")}${req.path}`,
          // Behind a proxy, set app.set("trust proxy", 1) so req.ip is the crawler's address.
          ai: { userAgent, ip: req.ip, statusCode: res.statusCode, source: "express" },
        }),
        signal: AbortSignal.timeout(2000),
      }).catch(() => {})
    })
  }
  next()
})

const CRAWLER = /bot|crawl|spider|slurp|-user\b|claude|anthropic|cohere|meta-external|facebookexternalhit|google|bingpreview/i

export default {
  async fetch(request, env, ctx) {
    const response = await fetch(request) // or your existing handler
    const userAgent = request.headers.get("user-agent") ?? ""
    if (request.method === "GET" && CRAWLER.test(userAgent)) {
      const url = new URL(request.url)
      ctx.waitUntil(
        fetch("https://usedatasense.com/api/crawls", {
          method: "POST",
          headers: { "Content-Type": "application/json" },
          body: JSON.stringify({
            websiteId: "YOUR_SITE_ID",
            href: url.origin + url.pathname,
            ai: {
              userAgent,
              ip: request.headers.get("cf-connecting-ip"),
              statusCode: response.status,
              source: "cloudflare",
            },
          }),
        }).catch(() => {}),
      )
    }
    return response
  },
}

# From your server, for each request whose user agent looks like a crawler.
# Use a short timeout and ignore failures. Send the path without its query string.
curl -X POST https://usedatasense.com/api/crawls \
  -H "Content-Type: application/json" \
  -d '{
    "websiteId": "YOUR_SITE_ID",
    "href": "https://example.com/docs/get-started",
    "ai": {
      "userAgent": "<the request's full User-Agent>",
      "ip": "<the crawler's IP>",
      "statusCode": 200
    }
  }'

# 200 {"success":true,"tracked":true}   recorded
# 200 {"success":true,"tracked":false}  not a known crawler, ignored