import tls from 'node:tls' import db from '@adonisjs/lucid/services/db' import logger from '@adonisjs/core/services/logger' import Site from '#models/site' import caddyService from '#services/caddy_service' const CHECK_EVERY_MS = 60_000 const REQUEST_TIMEOUT_MS = 10_000 const CERT_CHECK_EVERY_S = 6 * 3600 const RETENTION_SECONDS = 31 * 86_400 const CONCURRENCY = 5 export interface SiteCheckResult { ok: boolean status: number | null responseMs: number | null error: string | null } /** * Checks every minute that each domain of the Caddyfile answers, and * twice a day when its HTTPS certificate expires. * * The checks run from the VPS itself: they spot a stopped container or a * broken configuration, not the VPS being down. */ class SiteMonitor { #timers: NodeJS.Timeout[] = [] #running = false start() { const tick = () => this.checkAll().catch((error) => logger.warn({ err: error }, 'site checks failed')) setTimeout(tick, 3000) this.#timers.push(setInterval(tick, CHECK_EVERY_MS)) this.#timers.push(setInterval(() => this.#prune(), 6 * 3600_000)) } stop() { this.#timers.forEach(clearInterval) } /** * Adds the new domains of the Caddyfile and flags the removed ones. */ async sync() { const caddySites = await caddyService.sites() const seen = new Set() for (const caddySite of caddySites) { for (const domain of caddySite.domains) { if (seen.has(domain)) continue seen.add(domain) await Site.updateOrCreate( { domain }, { active: true, kind: caddySite.kind, upstream: caddySite.upstream } ) } } const query = Site.query().where('active', true) if (seen.size) query.whereNotIn('domain', [...seen]) await query.update({ active: false }) } async checkAll() { if (this.#running) return this.#running = true try { try { await this.sync() } catch (error) { logger.warn({ err: error }, 'cannot read the Caddyfile') } const sites = await Site.query().where('active', true) const queue = [...sites] await Promise.all( Array.from({ length: CONCURRENCY }, async () => { let site: Site | undefined while ((site = queue.shift())) await this.check(site) }) ) } finally { this.#running = false } } async check(site: Site) { const result = await probe(site.domain) const now = Math.floor(Date.now() / 1000) await db.table('site_checks').insert({ site_id: site.id, ts: now, ok: result.ok, status: result.status, response_ms: result.responseMs, error: result.error, }) site.lastOk = result.ok site.lastStatus = result.status site.lastResponseMs = result.responseMs site.lastError = result.error site.lastCheckedAt = now if (!site.certCheckedAt || now - site.certCheckedAt > CERT_CHECK_EVERY_S) { site.certExpiresAt = await certificateExpiry(site.domain) site.certCheckedAt = now } await site.save() return site } /** * Uptime (%) and average response time of every site over a period. */ async stats(sinceSeconds: number) { const since = Math.floor(Date.now() / 1000) - sinceSeconds const rows = await db .from('site_checks') .select('site_id') .select( db.raw('avg(ok) * 100 as uptime, avg(response_ms) as avgResponseMs, count(*) as checks') ) .where('ts', '>=', since) .groupBy('site_id') return new Map( rows.map((row) => [ row.site_id, { uptime: Number(row.uptime), avgResponseMs: row.avgResponseMs, checks: Number(row.checks), }, ]) ) } /** * Response times over a period (averaged per bucket) and the incidents * (consecutive failed checks). */ async history(siteId: number, sinceSeconds: number, bucket: number) { const since = Math.floor(Date.now() / 1000) - sinceSeconds const points = await db .from('site_checks') .select(db.raw('(ts / ?) * ? as t', [bucket, bucket])) .select(db.raw('avg(response_ms) as responseMs, min(ok) as ok, count(*) as checks')) .where('site_id', siteId) .where('ts', '>=', since) .groupBy('t') .orderBy('t') const checks = await db .from('site_checks') .select('ts', 'ok', 'status', 'error') .where('site_id', siteId) .where('ts', '>=', since) .orderBy('ts') const incidents: { start: number; end: number | null; error: string | null; checks: number }[] = [] let open: (typeof incidents)[number] | null = null for (const check of checks) { if (!check.ok) { if (!open) { open = { start: check.ts, end: null, error: check.error ?? `HTTP ${check.status}`, checks: 0, } incidents.push(open) } open.checks++ } else if (open) { open.end = check.ts open = null } } return { points: points.map((point) => ({ ...point, ok: Boolean(point.ok) })), incidents: incidents.reverse(), } } async #prune() { const limit = Math.floor(Date.now() / 1000) - RETENTION_SECONDS await db .from('site_checks') .where('ts', '<', limit) .delete() .catch(() => {}) } } /** * A site is "up" when it answers with a status below 500: an API without * route on "/" answers 404 but works, while a stopped container gives 502. */ async function probe(domain: string): Promise { const started = performance.now() try { const response = await fetch(`https://${domain}/`, { redirect: 'manual', signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), headers: { 'user-agent': 'big-brother-monitor' }, }) const responseMs = Math.round(performance.now() - started) await response.body?.cancel() return { ok: response.status < 500, status: response.status, responseMs, error: response.status < 500 ? null : `HTTP ${response.status}`, } } catch (error: any) { return { ok: false, status: null, responseMs: null, error: describeFetchError(error) } } } function describeFetchError(error: any): string { if (error?.name === 'TimeoutError') return `Pas de réponse en ${REQUEST_TIMEOUT_MS / 1000} s` const code = error?.cause?.code ?? error?.code switch (code) { case 'ENOTFOUND': return 'Domaine introuvable (DNS)' case 'ECONNREFUSED': return 'Connexion refusée' case 'ECONNRESET': return 'Connexion interrompue' case 'CERT_HAS_EXPIRED': return 'Certificat HTTPS expiré' case 'ERR_TLS_CERT_ALTNAME_INVALID': case 'DEPTH_ZERO_SELF_SIGNED_CERT': case 'UNABLE_TO_VERIFY_LEAF_SIGNATURE': return 'Certificat HTTPS invalide' case 'UND_ERR_CONNECT_TIMEOUT': return 'Délai de connexion dépassé' default: return error?.cause?.message ?? error?.message ?? 'Erreur inconnue' } } /** * Expiry date (unix seconds) of the certificate served for the domain. */ function certificateExpiry(domain: string): Promise { return new Promise((resolve) => { const socket = tls.connect({ host: domain, port: 443, servername: domain, rejectUnauthorized: false, timeout: REQUEST_TIMEOUT_MS, }) const done = (value: number | null) => { socket.destroy() resolve(value) } socket.once('secureConnect', () => { const validTo = socket.getPeerCertificate()?.valid_to done(validTo ? Math.floor(Date.parse(validTo) / 1000) : null) }) socket.once('timeout', () => done(null)) socket.once('error', () => done(null)) }) } const siteMonitor = new SiteMonitor() export default siteMonitor