MVP, working sentry scraper

app/controllers/replays_controller.ts

@@ -0,0 +1,80 @@
+import Replay from '#models/replay'
+import env from '#start/env'
+import type { HttpContext } from '@adonisjs/core/http'
+const SENTRY_TOKEN = env.get('SENTRY_TOKEN')
+const SENTRY_ORG = env.get('SENTRY_ORG')
+let recordsUpdated = 0
+export default class ReplaysController {
+    
+
+    async index({ request, response }: HttpContext) {
+      const {statsPeriod, start, end} = request.qs()
+      recordsUpdated = 0
+
+        let queryString: string = '?statsPeriod=24h'// Default in case none is provided
+        if (statsPeriod) {
+          queryString = `?statsPeriod=${statsPeriod}`
+        } else if (start && end) {
+          queryString = `?start=${start}&end=${end}`
+        } 
+        const replays = await fetchBatch(`https://sentry.io/api/0/organizations/${SENTRY_ORG}/replays/${queryString}`)
+        return response.json(replays)
+    }
+
+}
+
+
+async function fetchBatch(url: string) {
+    const options: RequestInit = {
+        headers: {
+         Authorization: `Bearer ${SENTRY_TOKEN}`
+        }
+     }
+    const req = await fetch(url, options)
+    const resp = await req.json() as unknown
+    const replays = await resp.data as unknown
+    const headers = await req.headers
+
+    const cleanedData = replays.map(record => sanitizeInput(record, Replay.allowedFields))
+
+    let updated = await Replay.updateOrCreateMany('id', cleanedData )
+    recordsUpdated = recordsUpdated + updated.length
+    const pagination = parseSentryLinkHeader(headers.get('link'))
+
+    if (pagination.hasNextResults == true) {
+        console.log('fetching', pagination.next)
+        await fetchBatch(pagination.next)
+    }
+    console.log('no more results')
+    return {recordsUpdated}
+
+}
+function parseSentryLinkHeader(header:string) {
+    const links = header.split(',').map(part => part.trim())
+  
+    const result = {}
+  
+    for (const link of links) {
+      const match = link.match(/<([^>]+)>;\s*rel="([^"]+)";\s*results="([^"]+)";\s*cursor="([^"]+)"/)
+      if (!match) continue
+  
+      const [, url, rel, results] = match
+  
+      if (rel === 'previous') {
+        result.previous = url
+        result.hasPreviousResults = results === 'true'
+      } else if (rel === 'next') {
+        result.next = url
+        result.hasNextResults = results === 'true'
+      }
+    }
+  
+    return result
+  }
+
+  function sanitizeInput(data: Record<string, any>, allowedFields: string[]) {
+    return allowedFields.reduce((acc, key) => {
+      if (key in data) acc[key] = data[key]
+      return acc
+    }, {} as Record<string, any>)
+  }