Add healthcheck and update node version

.env.example

@@ -1,12 +1,13 @@
 TZ=UTC
 PORT=3333
-HOST=localhost
+HOST=0.0.0.0
 LOG_LEVEL=info
-APP_KEY=
+APP_KEY=sMoYEqixvC3sgJO4WM9ej9ctlcVtAdCE
 NODE_ENV=development
 SESSION_DRIVER=cookie
+PG_USER=postgres
 PG_PORT=5432
-PG_HOST=localhost
+PG_HOST=db
 PG_PASSWORD=password
 SENTRY_TOKEN=
 SENTRY_ORG=
@@ -14,4 +15,4 @@ REDIS_HOST=sentry-redis-1
 REDIS_PORT=6379
 REDIS_PASSWORD=
 WEBHOOK_URL=
-QUERY_FILTER='!user.email:*@mailinator.com !user.email:*@example.com'
+QUERY_FILTER='!user.email:*@mailinator.com !user.email:*@example.com'

Dockerfile

@@ -1,4 +1,6 @@
-FROM node:22-alpine AS base
+FROM node:lts-alpine3.22 AS base
+HEALTHCHECK --interval=5s --timeout=10s --start-period=5s --retries=5 \
+  CMD sh -c 'wget --no-verbose --tries=1 --spider http://127.0.0.1:3333 || exit 1'
 
 # All deps stage
 FROM base AS deps
@@ -32,4 +34,5 @@ WORKDIR /app
 COPY --from=production-deps /app/node_modules /app/node_modules
 COPY --from=build /app/build /app
 EXPOSE 8080
+
 CMD ["node", "./bin/server.js"]

README.md

@@ -0,0 +1,32 @@
+# Sentry Toolkit
+This project was born out of a simple marketing request.  Basically along the lines of "how can we track user engagement in our CRM?", to which I answered "We already use Sentry for Session recording, we can pull that data from the API, aggregate it and push it to the CRM."  Hence this project.  It is currently pretty simple and includes an API as well as basic web ui.
+
+
+## Tech Stack
+- [AdonisJS](https://adonisjs.com): I decided to use the wonderful AdonisJS framework for this project.  Overkill? probably but it has a lot of nicecities built in and I didn't want to reinvent the wheel for this simple project.  I also wanted to play around with InertiaJS which comes included.
+- [Docker](https://docker.com) - All services have been containerized for convience of developing, testing and deploying.  A `compose.yml` and `compose.override.yml` are included for testing and developing locally.
+- Redis - Some basic caching because why not?
+- Postgresql - Useful for storing historical session data.
+- Traefik - Reverse Proxy/Ingress controller Provided for convienent development and local testing.
+- Grafana - (Optional) For building pretty dashboards.
+
+
+## Getting started
+```shell
+$ cp .env.example .env.develop
+# Add/edit values in .env.develop as needed
+# The WEBHOOK_URL is not strictly necessary for basic functionality.
+
+# Tested on Linux, I have not had the pleasure of setting up Traefik on Windows/Mac
+# recently so suggestions welcome.  Also you may need `sudo` depending on how your
+# Docker environment is set up.
+$ docker compose up -d
+```
+
+Once all of the containers come up, you should be able to access the UI/API on [http://sentry.docker.localhost]() (Docker compose magic.)  The database migrations should automatically run when you start with `docker compose` but if you are running the backend with node instead you will need to run `node ace migration:run` after starting the app for the first time.
+
+The main page will list any Replay sessions stored in the database.
+![](./docs/assets/homepage.jpg)
+
+
+[http://sentry.docker.localhost/replays]() will fetch session data from Sentry and store it in the database.  It will also return the results as JSON.

app/controllers/replays_controller.ts

@@ -5,8 +5,19 @@ const SENTRY_ORG = env.get('SENTRY_ORG')
 import redis from '@adonisjs/redis/services/main'
 import { fetchBatch } from '../Helpers/Replays.js'
 import { sendDataToWebhook } from '../Helpers/Webhook.js'
+import { faker } from '@faker-js/faker'
 
 export default class ReplaysController {
+  public async faker({ request, response }: HttpContext) {
+    const { page } = await request.qs()
+    const sessions = Array.from({ length: 100 }, generateFakeSession)
+    const nextPage = +page + 1
+    await response.safeHeader(
+      'link',
+      `<http://localhost:3333/faker/?page=${page}>; rel="previous"; results="true"; cursor="0:1100:1", <http://localhost:3333/faker/?page=${nextPage}>; rel="next"; results="${page == 10 ? 'false' : 'true'}"; cursor="0:${page * 100}:0"`
+    )
+    return { data: sessions, count: sessions.length, page: page }
+  }
   public async stats({ request, response }: HttpContext) {
     const { sendToWebhook } = request.qs()
     const latestVersion = await redis.get(`replays:stats:latest_version`)
@@ -34,7 +45,7 @@ export default class ReplaysController {
     return response.json(responseData)
   }
 
-  public async list({ request, inertia }: HttpContext) {
+  public async home({ request, inertia }: HttpContext) {
     const page = request.input('page', 1)
     const perPage = 20
     const cacheKey = `replays:page:${page}`
@@ -46,7 +57,7 @@ export default class ReplaysController {
       ;({ paginated, meta, replays } = JSON.parse(data))
     } else {
       paginated = await Replay.query().paginate(page, perPage)
-      paginated.baseUrl('/list')
+      paginated.baseUrl('/')
 
       const json = paginated.toJSON()
 
@@ -77,9 +88,12 @@ export default class ReplaysController {
       queryString = `?start=${start}&end=${end}`
     }
     const queryFilter = env.get('QUERY_FILTER')
-    await fetchBatch(
-      `https://sentry.io/api/0/organizations/${SENTRY_ORG}/replays/${queryString}&field=id&field=user&field=duration&field=started_at&field=finished_at&query=${encodeURIComponent(queryFilter)}`
-    )
+    const baseUrl =
+      env.get('NODE_ENV') == 'production'
+        ? `https://sentry.io/api/0/organizations/${SENTRY_ORG}/replays/${queryString}&field=id&field=user&field=duration&field=started_at&field=finished_at&query=${encodeURIComponent(queryFilter)}`
+        : 'http://localhost:3333/faker?page=1'
+    console.log('base', baseUrl)
+    await fetchBatch(baseUrl)
 
     let queryResults = await Replay.updateReplayStats()
 
@@ -104,7 +118,7 @@ function buildPaginationLinks(meta: {
 
   for (let page = 1; page <= meta.lastPage; page++) {
     links.push({
-      url: `/list?page=${page}`,
+      url: `/?page=${page}`,
       label: page.toString(),
       active: page === meta.currentPage,
     })
@@ -119,3 +133,63 @@ function buildPaginationLinks(meta: {
 
   return links
 }
+
+function generateFakeSession() {
+  const uuid = faker.string.uuid()
+  const browserName = faker.helpers.arrayElement(['Chrome', 'Firefox', 'Safari', 'Edge', 'Brave'])
+  const deviceBrand = faker.helpers.arrayElement(['Apple', 'Samsung', 'Google'])
+  const osName = faker.helpers.arrayElement(['iOS', 'Android', 'Windows', 'macOS'])
+  const platform = faker.helpers.arrayElement(['Sentry', 'Datadog', 'New Relic', 'Rollbar'])
+  const finishedAt = new Date(Date.now() - faker.number.int({ min: 0, max: 60 * 60 * 1000 }))
+  const displayName = faker.internet.email()
+  return {
+    activity: faker.number.int({ min: 1, max: 10 }),
+    browser: {
+      name: browserName,
+      version: faker.system.semver(),
+    },
+    count_dead_clicks: faker.number.int({ min: 0, max: 10 }),
+    count_rage_clicks: faker.number.int({ min: 0, max: 5 }),
+    count_errors: faker.number.int({ min: 0, max: 5 }),
+    count_segments: faker.number.int({ min: 0, max: 3 }),
+    count_urls: faker.number.int({ min: 1, max: 3 }),
+    device: {
+      brand: deviceBrand,
+      family: deviceBrand === 'Apple' ? 'iPhone' : deviceBrand,
+      model: faker.string.numeric({ length: 2 }),
+      name: `${deviceBrand} ${faker.string.alphanumeric({ length: 3 })}`,
+    },
+    dist: null,
+    duration: faker.number.int({ min: 100, max: 1000 }),
+    environment: faker.helpers.arrayElement(['production', 'staging', 'development']),
+    error_ids: [uuid],
+    finished_at: faker.date.between({ from: finishedAt, to: new Date() }).toISOString(),
+    has_viewed: faker.datatype.boolean(),
+    id: uuid,
+    is_archived: faker.datatype.boolean() ? null : false,
+    os: {
+      name: osName,
+      version: `${faker.number.int({ min: 10, max: 17 })}.${faker.number.int({ min: 0, max: 5 })}`,
+    },
+    platform: platform,
+    project_id: faker.string.numeric({ length: 6 }),
+    releases: [`version@${faker.system.semver()}`],
+    sdk: {
+      name: faker.hacker.noun(),
+      version: faker.system.semver(),
+    },
+    started_at: faker.date.recent().toISOString(),
+    tags: {
+      hello: ['world', faker.person.fullName()],
+    },
+    trace_ids: [uuid],
+    urls: [faker.internet.url()],
+    user: {
+      display_name: displayName,
+      email: displayName,
+      id: faker.string.numeric({ length: 8 }),
+      ip: faker.internet.ip(),
+      username: faker.internet.username(),
+    },
+  }
+}

app/models/replay.ts

@@ -14,7 +14,9 @@ export default class Replay extends BaseModel {
               u.average_session_time_readable,
               u.average_time_seconds,
               r.id AS last_session_id,
-              r.finished_at AS last_session_time
+              r.finished_at AS last_session_time,
+              o.id AS oldest_session_id,
+              o.finished_at AS oldest_session_time
 
             FROM (
               -- Aggregate sessions in the last 30 days
@@ -38,6 +40,8 @@ export default class Replay extends BaseModel {
               WHERE
                 finished_at >= NOW() - INTERVAL '30 days'
                 AND "user" ->> 'display_name' LIKE '%@%'
+                AND "user" ->> 'display_name' !~ 'e2etesting|@paragontruss.com'
+                
               GROUP BY
                 "user" ->> 'display_name'
             ) u
@@ -47,14 +51,29 @@ export default class Replay extends BaseModel {
               SELECT id, finished_at
               FROM replays
               WHERE "user" ->> 'display_name' = u.display_name
+                AND "user" ->> 'display_name' LIKE '%@%'
+                AND "user" ->> 'display_name' !~ 'e2etesting|@paragontruss.com'
               ORDER BY
                 CASE WHEN finished_at >= NOW() - INTERVAL '30 days' THEN 0 ELSE 1 END,
                 finished_at DESC
               LIMIT 1
             ) r ON true
 
+              -- LATERAL JOIN to get the oldest session
+              JOIN LATERAL (
+                SELECT id, finished_at
+                FROM replays
+                WHERE "user" ->> 'display_name' = u.display_name
+                  AND "user" ->> 'display_name' LIKE '%@%'
+                  AND "user" ->> 'display_name' !~ 'e2etesting|@paragontruss.com'
+                ORDER BY finished_at ASC
+                LIMIT 1
+              ) o ON true
+
             ORDER BY
-              u.total_time_seconds DESC;`)
+              u.total_time_seconds DESC;
+              
+ `)
     const updatedVersion = await redis.incr('replays:stats:latest_version')
     results.version = updatedVersion
     results.updatedAt = Date.now()

package-lock.json

@@ -34,6 +34,7 @@
         "@adonisjs/eslint-config": "^2.0.0",
         "@adonisjs/prettier-config": "^1.4.4",
         "@adonisjs/tsconfig": "^1.4.0",
+        "@faker-js/faker": "^9.8.0",
         "@japa/assert": "^4.0.1",
         "@japa/plugin-adonisjs": "^4.0.0",
         "@japa/runner": "^4.2.0",

package.json

@@ -36,6 +36,7 @@
     "@adonisjs/eslint-config": "^2.0.0",
     "@adonisjs/prettier-config": "^1.4.4",
     "@adonisjs/tsconfig": "^1.4.0",
+    "@faker-js/faker": "^9.8.0",
     "@japa/assert": "^4.0.1",
     "@japa/plugin-adonisjs": "^4.0.0",
     "@japa/runner": "^4.2.0",

resources/views/inertia_layout.edge

@@ -6,7 +6,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1" />
 
     <title inertia>
-      AdonisJS x Inertia x VueJS
+      Sentry Toolkit
     </title>
 
     <link rel="preconnect" href="https://fonts.bunny.net" />

start/env.ts

@@ -34,7 +34,7 @@ export default await Env.create(new URL('../', import.meta.url), {
   PG_USER: Env.schema.string(),
   PG_PASSWORD: Env.schema.string(),
 
-  WEBHOOK_URL: Env.schema.string(),
+  WEBHOOK_URL: Env.schema.string.optional(),
 
   QUERY_FILTER: Env.schema.string(),
 })

start/routes.ts

@@ -9,7 +9,7 @@
 
 import ReplaysController from '#controllers/replays_controller'
 import router from '@adonisjs/core/services/router'
-router.on('/').renderInertia('home')
+router.get('/', [ReplaysController, 'home'])
 router.get('/replays', [ReplaysController, 'index'])
-router.get('/list', [ReplaysController, 'list'])
 router.get('/stats', [ReplaysController, 'stats'])
+router.get('/faker', [ReplaysController, 'faker'])