263 lines
8.0 KiB
TypeScript
263 lines
8.0 KiB
TypeScript
import os from 'node:os'
|
|
import { EventEmitter } from 'node:events'
|
|
import { readFile, statfs } from 'node:fs/promises'
|
|
import { join } from 'node:path'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import logger from '@adonisjs/core/services/logger'
|
|
import vpsConfig from '#config/vps'
|
|
|
|
export interface HostSample {
|
|
ts: number
|
|
cpu: number
|
|
load1: number | null
|
|
memUsed: number
|
|
memTotal: number
|
|
diskUsed: number
|
|
diskTotal: number
|
|
/**
|
|
* Bytes per second (null when not available, e.g. on Windows).
|
|
*/
|
|
netRx: number | null
|
|
netTx: number | null
|
|
}
|
|
|
|
export type HistoryRange = '1h' | '24h' | '7d' | '30d'
|
|
|
|
/**
|
|
* Size of the buckets used to average the history, per range.
|
|
*/
|
|
const BUCKETS: Record<HistoryRange, { seconds: number; bucket: number }> = {
|
|
'1h': { seconds: 3600, bucket: 60 },
|
|
'24h': { seconds: 86_400, bucket: 300 },
|
|
'7d': { seconds: 7 * 86_400, bucket: 1800 },
|
|
'30d': { seconds: 30 * 86_400, bucket: 7200 },
|
|
}
|
|
|
|
const SAMPLE_EVERY_MS = 5000
|
|
const LIVE_SAMPLES = 120
|
|
const RETENTION_SECONDS = 31 * 86_400
|
|
|
|
/**
|
|
* Virtual interfaces (Docker bridges, veth pairs, loopback) are left out of
|
|
* the network traffic so that only the real traffic of the VPS is counted.
|
|
*/
|
|
const VIRTUAL_INTERFACE = /^(lo|docker\d*|br-|veth|virbr|tun|tap|wg|tailscale)/
|
|
|
|
/**
|
|
* Samples the host every 5 seconds, keeps the last 10 minutes in memory
|
|
* for the live charts and stores one averaged row per minute.
|
|
*/
|
|
class MetricsService extends EventEmitter {
|
|
#live: HostSample[] = []
|
|
#minute: HostSample[] = []
|
|
#previousCpu: { idle: number; total: number } | null = null
|
|
#previousNet: { rx: number; tx: number; ts: number } | null = null
|
|
#timers: NodeJS.Timeout[] = []
|
|
|
|
constructor() {
|
|
super()
|
|
this.setMaxListeners(100)
|
|
}
|
|
|
|
get latest(): HostSample | null {
|
|
return this.#live.at(-1) ?? null
|
|
}
|
|
|
|
get live() {
|
|
return this.#live
|
|
}
|
|
|
|
start() {
|
|
const tick = () =>
|
|
this.#sample().catch((error) => logger.warn({ err: error }, 'cannot sample host metrics'))
|
|
tick()
|
|
this.#timers.push(setInterval(tick, SAMPLE_EVERY_MS))
|
|
this.#timers.push(setInterval(() => this.#persist(), 60_000))
|
|
this.#timers.push(setInterval(() => this.#prune(), 6 * 3600_000))
|
|
}
|
|
|
|
stop() {
|
|
this.#timers.forEach(clearInterval)
|
|
}
|
|
|
|
async uptime(): Promise<number> {
|
|
const content = await this.#readProc('uptime')
|
|
return content ? Number.parseFloat(content) : os.uptime()
|
|
}
|
|
|
|
async history(range: HistoryRange) {
|
|
const { seconds, bucket } = BUCKETS[range]
|
|
const since = Math.floor(Date.now() / 1000) - seconds
|
|
const rows = await db
|
|
.from('metrics')
|
|
.select(db.raw('(ts / ?) * ? as t', [bucket, bucket]))
|
|
.select(
|
|
db.raw(`avg(cpu) as cpu, avg(load1) as load1,
|
|
avg(mem_used) as memUsed, max(mem_total) as memTotal,
|
|
avg(disk_used) as diskUsed, max(disk_total) as diskTotal,
|
|
avg(net_rx) as netRx, avg(net_tx) as netTx`)
|
|
)
|
|
.where('ts', '>=', since)
|
|
.groupBy('t')
|
|
.orderBy('t')
|
|
|
|
return { range, bucket, points: rows }
|
|
}
|
|
|
|
async #sample() {
|
|
const [cpu, memory, disk, net, load1] = await Promise.all([
|
|
this.#cpu(),
|
|
this.#memory(),
|
|
this.#disk(),
|
|
this.#network(),
|
|
this.#load(),
|
|
])
|
|
|
|
const sample: HostSample = { ts: Date.now(), cpu, load1, ...memory, ...disk, ...net }
|
|
this.#live.push(sample)
|
|
if (this.#live.length > LIVE_SAMPLES) this.#live.shift()
|
|
this.#minute.push(sample)
|
|
this.emit('sample', sample)
|
|
}
|
|
|
|
async #persist() {
|
|
const samples = this.#minute
|
|
this.#minute = []
|
|
if (!samples.length) return
|
|
|
|
const average = (pick: (sample: HostSample) => number | null) => {
|
|
const values = samples.map(pick).filter((value): value is number => value !== null)
|
|
return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : null
|
|
}
|
|
const last = samples.at(-1)!
|
|
|
|
try {
|
|
await db.table('metrics').insert({
|
|
ts: Math.floor(last.ts / 1000),
|
|
cpu: average((sample) => sample.cpu) ?? 0,
|
|
load1: average((sample) => sample.load1),
|
|
mem_used: Math.round(average((sample) => sample.memUsed) ?? 0),
|
|
mem_total: last.memTotal,
|
|
disk_used: last.diskUsed,
|
|
disk_total: last.diskTotal,
|
|
net_rx: average((sample) => sample.netRx),
|
|
net_tx: average((sample) => sample.netTx),
|
|
})
|
|
} catch (error) {
|
|
logger.error({ err: error }, 'cannot store host metrics')
|
|
}
|
|
}
|
|
|
|
async #prune() {
|
|
const limit = Math.floor(Date.now() / 1000) - RETENTION_SECONDS
|
|
await db
|
|
.from('metrics')
|
|
.where('ts', '<', limit)
|
|
.delete()
|
|
.catch(() => {})
|
|
}
|
|
|
|
async #readProc(file: string) {
|
|
const root = vpsConfig.hostProc ?? (process.platform === 'linux' ? '/proc' : null)
|
|
if (!root) return null
|
|
try {
|
|
return await readFile(join(root, file), 'utf8')
|
|
} catch {
|
|
return null
|
|
}
|
|
}
|
|
|
|
/**
|
|
* CPU usage between two samples, from /proc/stat (or os.cpus()).
|
|
*/
|
|
async #cpu() {
|
|
let idle = 0
|
|
let total = 0
|
|
const stat = await this.#readProc('stat')
|
|
if (stat) {
|
|
const fields = stat.split('\n')[0].trim().split(/\s+/).slice(1).map(Number)
|
|
/**
|
|
* user nice system idle iowait irq softirq steal (guest is already in user)
|
|
*/
|
|
const [user, nice, system, idleTime, iowait, irq, softirq, steal] = fields
|
|
idle = idleTime + (iowait ?? 0)
|
|
total =
|
|
user + nice + system + idleTime + (iowait ?? 0) + (irq ?? 0) + (softirq ?? 0) + (steal ?? 0)
|
|
} else {
|
|
for (const cpu of os.cpus()) {
|
|
idle += cpu.times.idle
|
|
total += cpu.times.user + cpu.times.nice + cpu.times.sys + cpu.times.idle + cpu.times.irq
|
|
}
|
|
}
|
|
|
|
const previous = this.#previousCpu
|
|
this.#previousCpu = { idle, total }
|
|
if (!previous || total <= previous.total) return 0
|
|
const usage = (1 - (idle - previous.idle) / (total - previous.total)) * 100
|
|
return Math.max(0, Math.min(100, usage))
|
|
}
|
|
|
|
async #memory() {
|
|
const meminfo = await this.#readProc('meminfo')
|
|
if (meminfo) {
|
|
const read = (key: string) => {
|
|
const match = new RegExp(`^${key}:\\s+(\\d+)`, 'm').exec(meminfo)
|
|
return match ? Number(match[1]) * 1024 : 0
|
|
}
|
|
const memTotal = read('MemTotal')
|
|
return { memTotal, memUsed: memTotal - read('MemAvailable') }
|
|
}
|
|
return { memTotal: os.totalmem(), memUsed: os.totalmem() - os.freemem() }
|
|
}
|
|
|
|
async #disk() {
|
|
try {
|
|
const stats = await statfs(vpsConfig.diskPath)
|
|
const used = (stats.blocks - stats.bfree) * stats.bsize
|
|
const available = stats.bavail * stats.bsize
|
|
/**
|
|
* Same computation as "df": the space reserved for root is left out.
|
|
*/
|
|
return { diskUsed: used, diskTotal: used + available }
|
|
} catch {
|
|
return { diskUsed: 0, diskTotal: 0 }
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Network traffic of the host. With HOST_PROC, /proc/1/net/dev is the
|
|
* network namespace of the host (not the one of the container).
|
|
*/
|
|
async #network(): Promise<{ netRx: number | null; netTx: number | null }> {
|
|
const content = await this.#readProc(vpsConfig.hostProc ? '1/net/dev' : 'net/dev')
|
|
if (!content) return { netRx: null, netTx: null }
|
|
|
|
let rx = 0
|
|
let tx = 0
|
|
for (const line of content.split('\n').slice(2)) {
|
|
const [name, data] = line.split(':')
|
|
if (!data || VIRTUAL_INTERFACE.test(name.trim())) continue
|
|
const fields = data.trim().split(/\s+/).map(Number)
|
|
rx += fields[0]
|
|
tx += fields[8]
|
|
}
|
|
|
|
const now = Date.now()
|
|
const previous = this.#previousNet
|
|
this.#previousNet = { rx, tx, ts: now }
|
|
if (!previous || rx < previous.rx || tx < previous.tx) return { netRx: null, netTx: null }
|
|
|
|
const seconds = (now - previous.ts) / 1000
|
|
return { netRx: (rx - previous.rx) / seconds, netTx: (tx - previous.tx) / seconds }
|
|
}
|
|
|
|
async #load() {
|
|
const content = await this.#readProc('loadavg')
|
|
if (content) return Number.parseFloat(content)
|
|
return process.platform === 'win32' ? null : os.loadavg()[0]
|
|
}
|
|
}
|
|
|
|
const metricsService = new MetricsService()
|
|
export default metricsService
|