Logflare Cloudflare Worker

logflare-cf-worker.js

const sleep = ms => {
  return new Promise(resolve => {
    setTimeout(resolve, ms)
  })
}

const makeid = length => {
  let text = ""
  const possible = "ABCDEFGHIJKLMNPQRSTUVWXYZ0123456789"
  for (let i = 0; i < length; i += 1) {
    text += possible.charAt(Math.floor(Math.random() * possible.length))
  }
  return text
}


const buildMetadataFromHeaders = headers => {
  const responseMetadata = {}
  Array.from(headers).forEach(([key, value]) => {
    const fixedKey = key.replace(/-/g, "_")
    if (![
      "x_b3_flags",
      "x_b3_parentspanid",
      "x_b3_sampled",
      "x_b3_spanid",
      "x_b3_traceid",
      "sentry_trace",
      "newrelic",
      "x_mi_xprotocol",
      "x_mi_xversion",
      "x_bluecoat_via",
    ].includes(fixedKey)) {
      responseMetadata[fixedKey] = value
    }
  })
  return responseMetadata
}


// Batching
const BATCH_INTERVAL_MS = 500
const MAX_REQUESTS_PER_BATCH = 100
const WORKER_ID = makeid(6)


let batchTimeoutReached = true
let logEventsBatch = []

// Backoff

const BACKOFF_INTERVAL = 10000
let backoff = 0

// Logflare API
const sourceKey = "" // Change This
const apiKey = "" // Change This
const logflareApiURL = "https://api.logflare.app/logs/cloudflare"



function addToBatch(body, event) {
  logEventsBatch.push(body)
  if (logEventsBatch.length >= MAX_REQUESTS_PER_BATCH) {
    event.waitUntil(postBatch(event))
  }

  return true
}

async function handleRequest(event) {
  const { request } = event

  const requestMetadata = buildMetadataFromHeaders(request.headers)

  const t1 = Date.now()
  const response = await fetch(request)
  const originTimeMs = Date.now() - t1

  const responseMetadata = buildMetadataFromHeaders(response.headers)

  const rMeth = request.method
  const rUrl = request.url
  const uAgent = request.headers.get("user-agent")
  const cfRay = request.headers.get("cf-ray")
  const cIP = request.headers.get("cf-connecting-ip")

  const rCf = {

    asOrganization: request.cf.asOrganization || null,
    asn: request.cf.asn || null,
    city: request.cf.city || null,
    clientAcceptEncoding: request.cf.clientAcceptEncoding || null,
    colo: request.cf.colo || null,
    continent: request.cf.continent || null,
    country: request.cf.country || null,
    httpProtocol: request.cf.httpProtocol || null,
    isEUCountry: request.cf.isEUCountry || null,
    region: request.cf.region || null,
    regionCode: request.cf.regionCode || null,
    metroCode: request.cf.metroCode || null,
    tlsVersion: request.cf.tlsVersion || null,
    verifiedBotCategory: request.cf.verifiedBotCategory || null

  }
  const statusCode = response.status

  let country = null
  let asOrganization = null

  if (rCf && rCf.country) {
    country = rCf.country
  }
  if (rCf && rCf.city) {
    country = rCf.city + ', ' + country
  }
  if (rCf && rCf.asOrganization) {
    asOrganization = rCf.asOrganization
  }


  const buildLogMessage = `${rMeth} | ${statusCode} | ${cIP} | ${cfRay} | ${rUrl} | ${uAgent} | ${originTimeMs}${country ? ` | ${country}` : ''}${asOrganization ? ` | ${asOrganization}` : ''}`

  const logflareEventBody = {
    source: sourceKey,
    message: buildLogMessage,
    timestamp: new Date().toISOString(),
    metadata: {
      response: {
        headers: responseMetadata,
        origin_time: originTimeMs,
        status_code: response.status,
      },
      request: {
        url: rUrl,
        method: rMeth,
        headers: requestMetadata,
        cf: rCf,
      },
      logflare_worker: {
        worker_id: WORKER_ID,
      },
    },
  }
  event.waitUntil(
    addToBatch(logflareEventBody, requestMetadata.cf_connecting_ip, event, asOrganization),
  )

  return response
}

const fetchAndSetBackOff = async (lfRequest, event) => {
  if (backoff <= Date.now()) {
    const resp = await fetch(logflareApiURL, lfRequest)
    if (resp.status === 403 || resp.status === 429) {
      backoff = Date.now() + BACKOFF_INTERVAL
    }
  }

  event.waitUntil(scheduleBatch(event))

  return true
}

const postBatch = async event => {
  const batchInFlight = [...logEventsBatch]
  logEventsBatch = []
  const rHost = batchInFlight[0].metadata.request.headers.host
  const body = JSON.stringify({ batch: batchInFlight, source: sourceKey })
  const request = {
    method: "POST",
    headers: {
      "X-API-KEY": apiKey,
      "Content-Type": "application/json",
      "User-Agent": `Cloudflare Worker via ${rHost}`,
    },
    body,
  }
  event.waitUntil(fetchAndSetBackOff(request, event))
}

const scheduleBatch = async event => {
  if (batchTimeoutReached) {
    batchTimeoutReached = false
    await sleep(BATCH_INTERVAL_MS)
    if (logEventsBatch.length > 0) {
      event.waitUntil(postBatch(event))
    }
    batchTimeoutReached = true
  }
  return true
}

addEventListener("fetch", event => {
  event.passThroughOnException()

  event.waitUntil(scheduleBatch(event))
  event.respondWith(handleRequest(event))
})