first commit

api/app/services/site_monitor.ts

@@ -0,0 +1,277 @@
+import tls from 'node:tls'
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import Site from '#models/site'
+import caddyService from '#services/caddy_service'
+
+const CHECK_EVERY_MS = 60_000
+const REQUEST_TIMEOUT_MS = 10_000
+const CERT_CHECK_EVERY_S = 6 * 3600
+const RETENTION_SECONDS = 31 * 86_400
+const CONCURRENCY = 5
+
+export interface SiteCheckResult {
+  ok: boolean
+  status: number | null
+  responseMs: number | null
+  error: string | null
+}
+
+/**
+ * Checks every minute that each domain of the Caddyfile answers, and
+ * twice a day when its HTTPS certificate expires.
+ *
+ * The checks run from the VPS itself: they spot a stopped container or a
+ * broken configuration, not the VPS being down.
+ */
+class SiteMonitor {
+  #timers: NodeJS.Timeout[] = []
+  #running = false
+
+  start() {
+    const tick = () =>
+      this.checkAll().catch((error) => logger.warn({ err: error }, 'site checks failed'))
+    setTimeout(tick, 3000)
+    this.#timers.push(setInterval(tick, CHECK_EVERY_MS))
+    this.#timers.push(setInterval(() => this.#prune(), 6 * 3600_000))
+  }
+
+  stop() {
+    this.#timers.forEach(clearInterval)
+  }
+
+  /**
+   * Adds the new domains of the Caddyfile and flags the removed ones.
+   */
+  async sync() {
+    const caddySites = await caddyService.sites()
+    const seen = new Set<string>()
+
+    for (const caddySite of caddySites) {
+      for (const domain of caddySite.domains) {
+        if (seen.has(domain)) continue
+        seen.add(domain)
+        await Site.updateOrCreate(
+          { domain },
+          { active: true, kind: caddySite.kind, upstream: caddySite.upstream }
+        )
+      }
+    }
+
+    const query = Site.query().where('active', true)
+    if (seen.size) query.whereNotIn('domain', [...seen])
+    await query.update({ active: false })
+  }
+
+  async checkAll() {
+    if (this.#running) return
+    this.#running = true
+    try {
+      try {
+        await this.sync()
+      } catch (error) {
+        logger.warn({ err: error }, 'cannot read the Caddyfile')
+      }
+
+      const sites = await Site.query().where('active', true)
+      const queue = [...sites]
+      await Promise.all(
+        Array.from({ length: CONCURRENCY }, async () => {
+          let site: Site | undefined
+          while ((site = queue.shift())) await this.check(site)
+        })
+      )
+    } finally {
+      this.#running = false
+    }
+  }
+
+  async check(site: Site) {
+    const result = await probe(site.domain)
+    const now = Math.floor(Date.now() / 1000)
+
+    await db.table('site_checks').insert({
+      site_id: site.id,
+      ts: now,
+      ok: result.ok,
+      status: result.status,
+      response_ms: result.responseMs,
+      error: result.error,
+    })
+
+    site.lastOk = result.ok
+    site.lastStatus = result.status
+    site.lastResponseMs = result.responseMs
+    site.lastError = result.error
+    site.lastCheckedAt = now
+
+    if (!site.certCheckedAt || now - site.certCheckedAt > CERT_CHECK_EVERY_S) {
+      site.certExpiresAt = await certificateExpiry(site.domain)
+      site.certCheckedAt = now
+    }
+
+    await site.save()
+    return site
+  }
+
+  /**
+   * Uptime (%) and average response time of every site over a period.
+   */
+  async stats(sinceSeconds: number) {
+    const since = Math.floor(Date.now() / 1000) - sinceSeconds
+    const rows = await db
+      .from('site_checks')
+      .select('site_id')
+      .select(
+        db.raw('avg(ok) * 100 as uptime, avg(response_ms) as avgResponseMs, count(*) as checks')
+      )
+      .where('ts', '>=', since)
+      .groupBy('site_id')
+
+    return new Map<number, { uptime: number; avgResponseMs: number | null; checks: number }>(
+      rows.map((row) => [
+        row.site_id,
+        {
+          uptime: Number(row.uptime),
+          avgResponseMs: row.avgResponseMs,
+          checks: Number(row.checks),
+        },
+      ])
+    )
+  }
+
+  /**
+   * Response times over a period (averaged per bucket) and the incidents
+   * (consecutive failed checks).
+   */
+  async history(siteId: number, sinceSeconds: number, bucket: number) {
+    const since = Math.floor(Date.now() / 1000) - sinceSeconds
+
+    const points = await db
+      .from('site_checks')
+      .select(db.raw('(ts / ?) * ? as t', [bucket, bucket]))
+      .select(db.raw('avg(response_ms) as responseMs, min(ok) as ok, count(*) as checks'))
+      .where('site_id', siteId)
+      .where('ts', '>=', since)
+      .groupBy('t')
+      .orderBy('t')
+
+    const checks = await db
+      .from('site_checks')
+      .select('ts', 'ok', 'status', 'error')
+      .where('site_id', siteId)
+      .where('ts', '>=', since)
+      .orderBy('ts')
+
+    const incidents: { start: number; end: number | null; error: string | null; checks: number }[] =
+      []
+    let open: (typeof incidents)[number] | null = null
+    for (const check of checks) {
+      if (!check.ok) {
+        if (!open) {
+          open = {
+            start: check.ts,
+            end: null,
+            error: check.error ?? `HTTP ${check.status}`,
+            checks: 0,
+          }
+          incidents.push(open)
+        }
+        open.checks++
+      } else if (open) {
+        open.end = check.ts
+        open = null
+      }
+    }
+
+    return {
+      points: points.map((point) => ({ ...point, ok: Boolean(point.ok) })),
+      incidents: incidents.reverse(),
+    }
+  }
+
+  async #prune() {
+    const limit = Math.floor(Date.now() / 1000) - RETENTION_SECONDS
+    await db
+      .from('site_checks')
+      .where('ts', '<', limit)
+      .delete()
+      .catch(() => {})
+  }
+}
+
+/**
+ * A site is "up" when it answers with a status below 500: an API without
+ * route on "/" answers 404 but works, while a stopped container gives 502.
+ */
+async function probe(domain: string): Promise<SiteCheckResult> {
+  const started = performance.now()
+  try {
+    const response = await fetch(`https://${domain}/`, {
+      redirect: 'manual',
+      signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+      headers: { 'user-agent': 'big-brother-monitor' },
+    })
+    const responseMs = Math.round(performance.now() - started)
+    await response.body?.cancel()
+    return {
+      ok: response.status < 500,
+      status: response.status,
+      responseMs,
+      error: response.status < 500 ? null : `HTTP ${response.status}`,
+    }
+  } catch (error: any) {
+    return { ok: false, status: null, responseMs: null, error: describeFetchError(error) }
+  }
+}
+
+function describeFetchError(error: any): string {
+  if (error?.name === 'TimeoutError') return `Pas de réponse en ${REQUEST_TIMEOUT_MS / 1000} s`
+  const code = error?.cause?.code ?? error?.code
+  switch (code) {
+    case 'ENOTFOUND':
+      return 'Domaine introuvable (DNS)'
+    case 'ECONNREFUSED':
+      return 'Connexion refusée'
+    case 'ECONNRESET':
+      return 'Connexion interrompue'
+    case 'CERT_HAS_EXPIRED':
+      return 'Certificat HTTPS expiré'
+    case 'ERR_TLS_CERT_ALTNAME_INVALID':
+    case 'DEPTH_ZERO_SELF_SIGNED_CERT':
+    case 'UNABLE_TO_VERIFY_LEAF_SIGNATURE':
+      return 'Certificat HTTPS invalide'
+    case 'UND_ERR_CONNECT_TIMEOUT':
+      return 'Délai de connexion dépassé'
+    default:
+      return error?.cause?.message ?? error?.message ?? 'Erreur inconnue'
+  }
+}
+
+/**
+ * Expiry date (unix seconds) of the certificate served for the domain.
+ */
+function certificateExpiry(domain: string): Promise<number | null> {
+  return new Promise((resolve) => {
+    const socket = tls.connect({
+      host: domain,
+      port: 443,
+      servername: domain,
+      rejectUnauthorized: false,
+      timeout: REQUEST_TIMEOUT_MS,
+    })
+    const done = (value: number | null) => {
+      socket.destroy()
+      resolve(value)
+    }
+    socket.once('secureConnect', () => {
+      const validTo = socket.getPeerCertificate()?.valid_to
+      done(validTo ? Math.floor(Date.parse(validTo) / 1000) : null)
+    })
+    socket.once('timeout', () => done(null))
+    socket.once('error', () => done(null))
+  })
+}
+
+const siteMonitor = new SiteMonitor()
+export default siteMonitor