Cloudflare
Collect logs using a Cloudflare Worker. Set up everything in 5 minutes or less.
  1. 1.
    Go to the Cloudflare dashboard β†’ Select your domain β†’ Workers
  2. 2.
    Click on Manage Workers β†’ Create a Worker
  3. 3.
    Replace the script with the following code and click on Save and Deploy
    1
    const CF_APP_VERSION = '1.0.0'
    2
    ​
    3
    const logtailApiURL = "https://in.logtail.com/";
    4
    const apiKey = "YOUR_LOGTAIL_SOURCE_TOKEN";
    5
    ​
    6
    const headers = [
    7
    "rMeth",
    8
    "rUrl",
    9
    "uAgent",
    10
    "cfRay",
    11
    "cIP",
    12
    "statusCode",
    13
    "contentLength",
    14
    "cfCacheStatus",
    15
    "contentType",
    16
    "responseConnection",
    17
    "requestConnection",
    18
    "cacheControl",
    19
    "acceptRanges",
    20
    "expectCt",
    21
    "expires",
    22
    "lastModified",
    23
    "vary",
    24
    "server",
    25
    "etag",
    26
    "date",
    27
    "transferEncoding",
    28
    ]
    29
    ​
    30
    const options = {
    31
    metadata: headers.map(value => ({ field: value })),
    32
    }
    33
    ​
    34
    const sleep = ms => {
    35
    return new Promise(resolve => {
    36
    setTimeout(resolve, ms)
    37
    })
    38
    }
    39
    ​
    40
    const makeid = length => {
    41
    let text = ""
    42
    const possible = "ABCDEFGHIJKLMNPQRSTUVWXYZ0123456789"
    43
    for (let i = 0; i < length; i += 1) {
    44
    text += possible.charAt(Math.floor(Math.random() * possible.length))
    45
    }
    46
    return text
    47
    }
    48
    ​
    49
    const buildLogMessage = (request, response) => {
    50
    const logDefs = {
    51
    rMeth: request.method,
    52
    rUrl: request.url,
    53
    uAgent: request.headers.get("user-agent"),
    54
    cfRay: request.headers.get("cf-ray"),
    55
    cIP: request.headers.get("cf-connecting-ip"),
    56
    statusCode: response.status,
    57
    contentLength: response.headers.get("content-length"),
    58
    cfCacheStatus: response.headers.get("cf-cache-status"),
    59
    contentType: response.headers.get("content-type"),
    60
    responseConnection: response.headers.get("connection"),
    61
    requestConnection: request.headers.get("connection"),
    62
    cacheControl: response.headers.get("cache-control"),
    63
    acceptRanges: response.headers.get("accept-ranges"),
    64
    expectCt: response.headers.get("expect-ct"),
    65
    expires: response.headers.get("expires"),
    66
    lastModified: response.headers.get("last-modified"),
    67
    vary: response.headers.get("vary"),
    68
    server: response.headers.get("server"),
    69
    etag: response.headers.get("etag"),
    70
    date: response.headers.get("date"),
    71
    transferEncoding: response.headers.get("transfer-encoding"),
    72
    }
    73
    ​
    74
    const logArray = []
    75
    options.metadata.forEach(entry => logArray.push(logDefs[entry.field]))
    76
    return logArray.join(" | ")
    77
    }
    78
    ​
    79
    const buildMetadataFromHeaders = headers => {
    80
    const responseMetadata = {}
    81
    Array.from(headers).forEach(([key, value]) => {
    82
    responseMetadata[key.replace(/-/g, "_")] = value
    83
    })
    84
    return responseMetadata
    85
    }
    86
    ​
    87
    // Batching
    88
    const BATCH_INTERVAL_MS = 500
    89
    const MAX_REQUESTS_PER_BATCH = 100
    90
    const WORKER_ID = makeid(6)
    91
    ​
    92
    let workerTimestamp
    93
    ​
    94
    let batchTimeoutReached = true
    95
    let logEventsBatch = []
    96
    ​
    97
    // Backoff
    98
    const BACKOFF_INTERVAL = 10000
    99
    let backoff = 0
    100
    ​
    101
    async function addToBatch(body, connectingIp, event) {
    102
    logEventsBatch.push(body)
    103
    ​
    104
    if (logEventsBatch.length >= MAX_REQUESTS_PER_BATCH) {
    105
    event.waitUntil(postBatch(event))
    106
    }
    107
    ​
    108
    return true
    109
    }
    110
    ​
    111
    async function handleRequest(event) {
    112
    const { request } = event
    113
    ​
    114
    const requestMetadata = buildMetadataFromHeaders(request.headers)
    115
    ​
    116
    const t1 = Date.now()
    117
    const response = await fetch(request)
    118
    const originTimeMs = Date.now() - t1
    119
    ​
    120
    const rUrl = request.url
    121
    const rMeth = request.method
    122
    const rCf = request.cf
    123
    delete rCf.tlsClientAuth
    124
    delete rCf.tlsExportedAuthenticator
    125
    ​
    126
    const responseMetadata = buildMetadataFromHeaders(response.headers)
    127
    ​
    128
    const eventBody = {
    129
    message: buildLogMessage(request, response),
    130
    dt: new Date().toISOString(),
    131
    metadata: {
    132
    response: {
    133
    headers: responseMetadata,
    134
    origin_time: originTimeMs,
    135
    status_code: response.status,
    136
    },
    137
    request: {
    138
    url: rUrl,
    139
    method: rMeth,
    140
    headers: requestMetadata,
    141
    cf: rCf,
    142
    },
    143
    cloudflare_worker: {
    144
    version: CF_APP_VERSION,
    145
    worker_id: WORKER_ID,
    146
    worker_started: workerTimestamp,
    147
    },
    148
    },
    149
    }
    150
    event.waitUntil(
    151
    addToBatch(eventBody, requestMetadata.cf_connecting_ip, event),
    152
    )
    153
    ​
    154
    return response
    155
    }
    156
    ​
    157
    const fetchAndSetBackOff = async (lfRequest, event) => {
    158
    if (backoff <= Date.now()) {
    159
    const resp = await fetch(logtailApiURL, lfRequest)
    160
    if (resp.status === 403 || resp.status === 429) {
    161
    backoff = Date.now() + BACKOFF_INTERVAL
    162
    }
    163
    }
    164
    ​
    165
    event.waitUntil(scheduleBatch(event))
    166
    ​
    167
    return true
    168
    }
    169
    ​
    170
    const postBatch = async event => {
    171
    const batchInFlight = [...logEventsBatch]
    172
    logEventsBatch = []
    173
    const rHost = batchInFlight[0].metadata.request.headers.host
    174
    const body = JSON.stringify(batchInFlight)
    175
    const request = {
    176
    method: "POST",
    177
    headers: {
    178
    "Authorization": `Bearer ${apiKey}`,
    179
    "Content-Type": "application/json",
    180
    "User-Agent": `Cloudflare Worker via ${rHost}`,
    181
    },
    182
    body,
    183
    }
    184
    event.waitUntil(fetchAndSetBackOff(request, event))
    185
    }
    186
    ​
    187
    const scheduleBatch = async event => {
    188
    if (batchTimeoutReached) {
    189
    batchTimeoutReached = false
    190
    await sleep(BATCH_INTERVAL_MS)
    191
    if (logEventsBatch.length > 0) {
    192
    event.waitUntil(postBatch(event))
    193
    }
    194
    batchTimeoutReached = true
    195
    }
    196
    return true
    197
    }
    198
    ​
    199
    addEventListener("fetch", event => {
    200
    event.passThroughOnException()
    201
    ​
    202
    if (!workerTimestamp) {
    203
    workerTimestamp = new Date().toISOString()
    204
    }
    205
    ​
    206
    event.waitUntil(scheduleBatch(event))
    207
    event.respondWith(handleRequest(event))
    208
    })
    Copied!
  4. 4.
    Go back to Workers β†’ Click on Add route
  5. 5.
    Set the route pattern to *.your-domain.com/* and select the new Logtail worker
  6. 6.
    Click on Save and you're all set πŸŽ‰
Make sure to replaceYOUR_LOGTAIL_SOURCE_TOKEN at the beginning of the script with your own source token from Logtail.com.
Last modified 6mo ago
Copy link