278 lines
7.8 KiB
TypeScript
278 lines
7.8 KiB
TypeScript
import tls from 'node:tls'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import logger from '@adonisjs/core/services/logger'
|
|
import Site from '#models/site'
|
|
import caddyService from '#services/caddy_service'
|
|
|
|
const CHECK_EVERY_MS = 60_000
|
|
const REQUEST_TIMEOUT_MS = 10_000
|
|
const CERT_CHECK_EVERY_S = 6 * 3600
|
|
const RETENTION_SECONDS = 31 * 86_400
|
|
const CONCURRENCY = 5
|
|
|
|
export interface SiteCheckResult {
|
|
ok: boolean
|
|
status: number | null
|
|
responseMs: number | null
|
|
error: string | null
|
|
}
|
|
|
|
/**
|
|
* Checks every minute that each domain of the Caddyfile answers, and
|
|
* twice a day when its HTTPS certificate expires.
|
|
*
|
|
* The checks run from the VPS itself: they spot a stopped container or a
|
|
* broken configuration, not the VPS being down.
|
|
*/
|
|
class SiteMonitor {
|
|
#timers: NodeJS.Timeout[] = []
|
|
#running = false
|
|
|
|
start() {
|
|
const tick = () =>
|
|
this.checkAll().catch((error) => logger.warn({ err: error }, 'site checks failed'))
|
|
setTimeout(tick, 3000)
|
|
this.#timers.push(setInterval(tick, CHECK_EVERY_MS))
|
|
this.#timers.push(setInterval(() => this.#prune(), 6 * 3600_000))
|
|
}
|
|
|
|
stop() {
|
|
this.#timers.forEach(clearInterval)
|
|
}
|
|
|
|
/**
|
|
* Adds the new domains of the Caddyfile and flags the removed ones.
|
|
*/
|
|
async sync() {
|
|
const caddySites = await caddyService.sites()
|
|
const seen = new Set<string>()
|
|
|
|
for (const caddySite of caddySites) {
|
|
for (const domain of caddySite.domains) {
|
|
if (seen.has(domain)) continue
|
|
seen.add(domain)
|
|
await Site.updateOrCreate(
|
|
{ domain },
|
|
{ active: true, kind: caddySite.kind, upstream: caddySite.upstream }
|
|
)
|
|
}
|
|
}
|
|
|
|
const query = Site.query().where('active', true)
|
|
if (seen.size) query.whereNotIn('domain', [...seen])
|
|
await query.update({ active: false })
|
|
}
|
|
|
|
async checkAll() {
|
|
if (this.#running) return
|
|
this.#running = true
|
|
try {
|
|
try {
|
|
await this.sync()
|
|
} catch (error) {
|
|
logger.warn({ err: error }, 'cannot read the Caddyfile')
|
|
}
|
|
|
|
const sites = await Site.query().where('active', true)
|
|
const queue = [...sites]
|
|
await Promise.all(
|
|
Array.from({ length: CONCURRENCY }, async () => {
|
|
let site: Site | undefined
|
|
while ((site = queue.shift())) await this.check(site)
|
|
})
|
|
)
|
|
} finally {
|
|
this.#running = false
|
|
}
|
|
}
|
|
|
|
async check(site: Site) {
|
|
const result = await probe(site.domain)
|
|
const now = Math.floor(Date.now() / 1000)
|
|
|
|
await db.table('site_checks').insert({
|
|
site_id: site.id,
|
|
ts: now,
|
|
ok: result.ok,
|
|
status: result.status,
|
|
response_ms: result.responseMs,
|
|
error: result.error,
|
|
})
|
|
|
|
site.lastOk = result.ok
|
|
site.lastStatus = result.status
|
|
site.lastResponseMs = result.responseMs
|
|
site.lastError = result.error
|
|
site.lastCheckedAt = now
|
|
|
|
if (!site.certCheckedAt || now - site.certCheckedAt > CERT_CHECK_EVERY_S) {
|
|
site.certExpiresAt = await certificateExpiry(site.domain)
|
|
site.certCheckedAt = now
|
|
}
|
|
|
|
await site.save()
|
|
return site
|
|
}
|
|
|
|
/**
|
|
* Uptime (%) and average response time of every site over a period.
|
|
*/
|
|
async stats(sinceSeconds: number) {
|
|
const since = Math.floor(Date.now() / 1000) - sinceSeconds
|
|
const rows = await db
|
|
.from('site_checks')
|
|
.select('site_id')
|
|
.select(
|
|
db.raw('avg(ok) * 100 as uptime, avg(response_ms) as avgResponseMs, count(*) as checks')
|
|
)
|
|
.where('ts', '>=', since)
|
|
.groupBy('site_id')
|
|
|
|
return new Map<number, { uptime: number; avgResponseMs: number | null; checks: number }>(
|
|
rows.map((row) => [
|
|
row.site_id,
|
|
{
|
|
uptime: Number(row.uptime),
|
|
avgResponseMs: row.avgResponseMs,
|
|
checks: Number(row.checks),
|
|
},
|
|
])
|
|
)
|
|
}
|
|
|
|
/**
|
|
* Response times over a period (averaged per bucket) and the incidents
|
|
* (consecutive failed checks).
|
|
*/
|
|
async history(siteId: number, sinceSeconds: number, bucket: number) {
|
|
const since = Math.floor(Date.now() / 1000) - sinceSeconds
|
|
|
|
const points = await db
|
|
.from('site_checks')
|
|
.select(db.raw('(ts / ?) * ? as t', [bucket, bucket]))
|
|
.select(db.raw('avg(response_ms) as responseMs, min(ok) as ok, count(*) as checks'))
|
|
.where('site_id', siteId)
|
|
.where('ts', '>=', since)
|
|
.groupBy('t')
|
|
.orderBy('t')
|
|
|
|
const checks = await db
|
|
.from('site_checks')
|
|
.select('ts', 'ok', 'status', 'error')
|
|
.where('site_id', siteId)
|
|
.where('ts', '>=', since)
|
|
.orderBy('ts')
|
|
|
|
const incidents: { start: number; end: number | null; error: string | null; checks: number }[] =
|
|
[]
|
|
let open: (typeof incidents)[number] | null = null
|
|
for (const check of checks) {
|
|
if (!check.ok) {
|
|
if (!open) {
|
|
open = {
|
|
start: check.ts,
|
|
end: null,
|
|
error: check.error ?? `HTTP ${check.status}`,
|
|
checks: 0,
|
|
}
|
|
incidents.push(open)
|
|
}
|
|
open.checks++
|
|
} else if (open) {
|
|
open.end = check.ts
|
|
open = null
|
|
}
|
|
}
|
|
|
|
return {
|
|
points: points.map((point) => ({ ...point, ok: Boolean(point.ok) })),
|
|
incidents: incidents.reverse(),
|
|
}
|
|
}
|
|
|
|
async #prune() {
|
|
const limit = Math.floor(Date.now() / 1000) - RETENTION_SECONDS
|
|
await db
|
|
.from('site_checks')
|
|
.where('ts', '<', limit)
|
|
.delete()
|
|
.catch(() => {})
|
|
}
|
|
}
|
|
|
|
/**
|
|
* A site is "up" when it answers with a status below 500: an API without
|
|
* route on "/" answers 404 but works, while a stopped container gives 502.
|
|
*/
|
|
async function probe(domain: string): Promise<SiteCheckResult> {
|
|
const started = performance.now()
|
|
try {
|
|
const response = await fetch(`https://${domain}/`, {
|
|
redirect: 'manual',
|
|
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
|
|
headers: { 'user-agent': 'big-brother-monitor' },
|
|
})
|
|
const responseMs = Math.round(performance.now() - started)
|
|
await response.body?.cancel()
|
|
return {
|
|
ok: response.status < 500,
|
|
status: response.status,
|
|
responseMs,
|
|
error: response.status < 500 ? null : `HTTP ${response.status}`,
|
|
}
|
|
} catch (error: any) {
|
|
return { ok: false, status: null, responseMs: null, error: describeFetchError(error) }
|
|
}
|
|
}
|
|
|
|
function describeFetchError(error: any): string {
|
|
if (error?.name === 'TimeoutError') return `Pas de réponse en ${REQUEST_TIMEOUT_MS / 1000} s`
|
|
const code = error?.cause?.code ?? error?.code
|
|
switch (code) {
|
|
case 'ENOTFOUND':
|
|
return 'Domaine introuvable (DNS)'
|
|
case 'ECONNREFUSED':
|
|
return 'Connexion refusée'
|
|
case 'ECONNRESET':
|
|
return 'Connexion interrompue'
|
|
case 'CERT_HAS_EXPIRED':
|
|
return 'Certificat HTTPS expiré'
|
|
case 'ERR_TLS_CERT_ALTNAME_INVALID':
|
|
case 'DEPTH_ZERO_SELF_SIGNED_CERT':
|
|
case 'UNABLE_TO_VERIFY_LEAF_SIGNATURE':
|
|
return 'Certificat HTTPS invalide'
|
|
case 'UND_ERR_CONNECT_TIMEOUT':
|
|
return 'Délai de connexion dépassé'
|
|
default:
|
|
return error?.cause?.message ?? error?.message ?? 'Erreur inconnue'
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Expiry date (unix seconds) of the certificate served for the domain.
|
|
*/
|
|
function certificateExpiry(domain: string): Promise<number | null> {
|
|
return new Promise((resolve) => {
|
|
const socket = tls.connect({
|
|
host: domain,
|
|
port: 443,
|
|
servername: domain,
|
|
rejectUnauthorized: false,
|
|
timeout: REQUEST_TIMEOUT_MS,
|
|
})
|
|
const done = (value: number | null) => {
|
|
socket.destroy()
|
|
resolve(value)
|
|
}
|
|
socket.once('secureConnect', () => {
|
|
const validTo = socket.getPeerCertificate()?.valid_to
|
|
done(validTo ? Math.floor(Date.parse(validTo) / 1000) : null)
|
|
})
|
|
socket.once('timeout', () => done(null))
|
|
socket.once('error', () => done(null))
|
|
})
|
|
}
|
|
|
|
const siteMonitor = new SiteMonitor()
|
|
export default siteMonitor
|