Files
kener/src/lib/server/controllers/monitorsController.ts
T

730 lines
23 KiB
TypeScript

import {
GetMinuteStartNowTimestampUTC,
GetMinuteStartTimestampUTC,
GetNowTimestampUTC,
GetNowTimestampUTCInMs,
UnparsePercentage,
UptimeCalculator,
} from "../tool.js";
import { getCache, setCache } from "../cache/cache.js";
import type {
MonitorRecordInsert,
MonitorAlertInsert,
MonitorRecordTyped,
MonitorRecord,
MonitorAlert,
MonitoringData,
TimestampStatusCount,
UptimeCalculatorResult,
TimestampStatusCountByMonitor,
} from "../types/db.js";
import type { MonitorFilter } from "../db/repositories/base.js";
import db from "../db/db.js";
import type { PaginationInput } from "../../types/common.js";
import type { DayWiseStatus, NumberWithChange } from "../../types/monitor.js";
import GC, { getBadgeStyle, type BadgeStyle } from "../../global-constants.js";
import { makeBadge } from "badge-maker";
import { ErrorSvg } from "../../anywhere.js";
import { GetLastMonitoringValue, SetLastHeartbeat, DeleteMonitorCaches } from "../cache/setGet.js";
import type { HeartbeatMonitor, GroupMonitorTypeData } from "../types/monitor.js";
interface GroupUpdateData {
monitor_tag: string;
timestamp: number;
status: string;
latency: number;
}
interface MonitorInput extends MonitorRecordInsert {
id?: number;
}
/**
* Validates that a monitor tag is URL-friendly: lowercase alphanumeric, hyphens, and underscores only.
* Must start and end with an alphanumeric character.
*/
const VALID_TAG_REGEX = /^[a-z0-9][a-z0-9_-]*[a-z0-9]$/;
const VALID_TAG_SINGLE_CHAR_REGEX = /^[a-z0-9]$/;
function isValidMonitorTag(tag: string): boolean {
if (!tag || tag.length === 0) return false;
if (tag.length === 1) return VALID_TAG_SINGLE_CHAR_REGEX.test(tag);
return VALID_TAG_REGEX.test(tag);
}
function validateMonitorTag(tag: string): void {
const trimmed = tag?.trim();
if (!trimmed) {
throw new Error("Monitor tag is required");
}
if (!isValidMonitorTag(trimmed)) {
throw new Error(
"Monitor tag must be URL-friendly: only lowercase letters, numbers, hyphens, and underscores. Must start and end with a letter or number.",
);
}
}
interface DayGroupData {
timestamp: number;
total: number;
UP: number;
DOWN: number;
DEGRADED: number;
MAINTENANCE: number;
NO_DATA: number;
[key: string]: number;
}
interface UpdateMonitoringDataInput {
monitor_tag: string;
start: number;
end: number;
newStatus: string;
type: string;
latency?: number;
deviation?: number;
}
interface MonitoringDataInput {
monitor_tag: string;
timestamp: number;
status: string;
latency?: number;
type: string;
error_message?: string | null;
}
interface InterpolatedDataEntry {
timestamp: number;
status: string;
}
export const InsertMonitoringData = async (data: MonitoringDataInput): Promise<MonitoringData | null> => {
//do validation if present all fields below
if (!data.monitor_tag || !data.timestamp || !data.status || !data.type) {
throw new Error("Invalid data");
}
return await db.insertMonitoringData({
monitor_tag: data.monitor_tag,
timestamp: data.timestamp,
status: data.status,
latency: data.latency || 0,
type: data.type,
error_message: data.error_message,
});
};
export const UpdateMonitoringData = async (data: UpdateMonitoringDataInput): Promise<unknown[]> => {
let queryData = { ...data };
return await db.updateMonitoringData(
queryData.monitor_tag,
GetMinuteStartTimestampUTC(queryData.start),
GetMinuteStartTimestampUTC(queryData.end),
queryData.newStatus,
queryData.type,
queryData.latency ?? 0,
queryData.deviation ?? 0,
);
};
export const AggregateData = (
rawData: InterpolatedDataEntry[],
): { total: number; UPs: number; DOWNs: number; DEGRADEDs: number; NO_DATAs: number } => {
//data like [{ timestamp: 1732435920, status: 'NO_DATA' }]
let rawDataWithStatus = rawData.filter((data) => data.status !== GC.NO_DATA);
const total = rawDataWithStatus.length;
const UPs = rawDataWithStatus.filter((data) => data.status === GC.UP).length;
const DOWNs = rawDataWithStatus.filter((data) => data.status === GC.DOWN).length;
const DEGRADEDs = rawDataWithStatus.filter((data) => data.status === GC.DEGRADED).length;
const NO_DATAs = total - (UPs + DOWNs + DEGRADEDs);
return { total, UPs, DOWNs, DEGRADEDs, NO_DATAs };
};
export const GetMonitorsParsed = async (query: MonitorFilter): Promise<Array<MonitorRecordTyped>> => {
// Retrieve monitors from the database based on the provided query
const rawMonitors = await db.getMonitors(query);
// Parse type_data if available for each monitor
const parsedMonitors = rawMonitors.map((monitor) => {
const monitorData: MonitorRecord = { ...monitor };
const monitorTyped: MonitorRecordTyped = {
...monitorData,
type_data: {},
monitor_settings_json: {},
};
if (monitorData.type_data) {
try {
monitorTyped.type_data = JSON.parse(monitorData.type_data);
} catch (error) {
// Fallback to an empty object if JSON parsing fails
monitorTyped.type_data = {};
}
} else {
monitorTyped.type_data = {};
}
if (monitorData.monitor_settings_json) {
try {
monitorTyped.monitor_settings_json = JSON.parse(monitorData.monitor_settings_json);
} catch (error) {
// Fallback to default settings if JSON parsing fails
monitorTyped.monitor_settings_json = {
uptime_formula_numerator: GC.defaultNumeratorStr,
uptime_formula_denominator: GC.defaultDenominatorStr,
};
}
} else {
monitorTyped.monitor_settings_json = {
uptime_formula_numerator: GC.defaultNumeratorStr,
uptime_formula_denominator: GC.defaultDenominatorStr,
};
}
return monitorTyped;
});
return parsedMonitors;
};
export const CreateUpdateMonitor = async (monitor: MonitorInput): Promise<number | number[]> => {
let monitorData = { ...monitor };
if (monitorData.id) {
return await db.updateMonitor(monitorData as MonitorRecord);
} else {
validateMonitorTag(monitorData.tag);
return await db.insertMonitor(monitorData);
}
};
export const CreateMonitor = async (monitor: MonitorInput): Promise<number[]> => {
let monitorData = { ...monitor };
if (monitorData.id) {
throw new Error("monitor id must be empty or 0");
}
validateMonitorTag(monitorData.tag);
return await db.insertMonitor(monitorData);
};
interface CloneMonitorInput {
sourceTag: string;
newTag: string;
newName: string;
}
export const CloneMonitor = async ({ sourceTag, newTag, newName }: CloneMonitorInput): Promise<number[]> => {
const sourceTagTrimmed = sourceTag?.trim();
const newTagTrimmed = newTag?.trim();
const newNameTrimmed = newName?.trim();
if (!sourceTagTrimmed) {
throw new Error("Source monitor tag is required");
}
if (!newTagTrimmed) {
throw new Error("Tag is required");
}
validateMonitorTag(newTagTrimmed);
if (!newNameTrimmed) {
throw new Error("Name is required");
}
if (sourceTagTrimmed === newTagTrimmed) {
throw new Error("New tag must be different from source tag");
}
const source = await db.getMonitorsByTag(sourceTagTrimmed);
if (!source) {
throw new Error("Source monitor not found");
}
const existingTag = await db.getMonitorsByTag(newTagTrimmed);
if (existingTag) {
throw new Error("Monitor tag already exists");
}
const allMonitors = await db.getMonitors({});
if (allMonitors.some((m) => m.name === newNameTrimmed)) {
throw new Error("Monitor name already exists");
}
return await db.insertMonitor({
tag: newTagTrimmed,
name: newNameTrimmed,
description: source.description,
image: source.image,
cron: source.cron,
default_status: source.default_status,
status: source.status,
category_name: source.category_name,
monitor_type: source.monitor_type,
down_trigger: source.down_trigger,
degraded_trigger: source.degraded_trigger,
type_data: source.type_data,
day_degraded_minimum_count: source.day_degraded_minimum_count,
day_down_minimum_count: source.day_down_minimum_count,
include_degraded_in_downtime: source.include_degraded_in_downtime,
is_hidden: source.is_hidden,
monitor_settings_json: source.monitor_settings_json,
external_url: source.external_url,
});
};
export const UpdateMonitor = async (monitor: MonitorInput): Promise<number> => {
let monitorData = { ...monitor };
if (!!!monitorData.id || monitorData.id === 0) {
throw new Error("monitor id cannot be empty or 0");
}
return await db.updateMonitor(monitorData as MonitorRecord);
};
export const GetMonitors = async (data: MonitorFilter): Promise<MonitorRecord[]> => {
return await db.getMonitors(data);
};
export const GetLatestMonitoringData = async (monitor_tag: string): Promise<MonitoringData | undefined> => {
let latestData = await db.getLatestMonitoringData(monitor_tag);
return latestData;
};
export const GetLatestStatusActiveAll = async (): Promise<{ status: string }> => {
//get all the active not hidden monitor tags
const monitors = await db.getMonitors({ status: "ACTIVE", is_hidden: "NO" });
const monitor_tags = monitors.map((m) => m.tag);
const latestData: MonitoringData[] = [];
for (let i = 0; i < monitor_tags.length; i++) {
const tag = monitor_tags[i];
const lastObj = await GetLastMonitoringValue(tag, () => GetLatestMonitoringData(tag));
if (lastObj) {
latestData.push(lastObj);
}
}
let status: string = GC.NO_DATA;
for (let i = 0; i < latestData.length; i++) {
//if any status is down then status = down, if any is degraded then status = degraded, down > degraded > up
if (latestData[i].status === GC.DOWN) {
status = GC.DOWN;
} else if (latestData[i].status === GC.DEGRADED && status !== GC.DOWN) {
status = GC.DEGRADED;
} else if (latestData[i].status === GC.UP && status !== GC.DOWN && status !== GC.DEGRADED) {
status = GC.UP;
}
}
return {
status: status,
};
};
//getLatestMonitoringDataAllActive
export const GetLatestMonitoringDataAllActive = async (monitor_tags: string[]): Promise<MonitoringData[]> => {
let latestData = await db.getLatestMonitoringDataAllActive(monitor_tags);
return latestData;
};
export const GetLastHeartbeat = async (monitor_tag: string): Promise<MonitoringData | undefined> => {
return await db.getLastHeartbeat(monitor_tag);
};
export const RegisterHeartbeat = async (tag: string, secret: string): Promise<string> => {
let monitor = (await GetMonitorsParsed({ tag, status: "ACTIVE", monitor_type: "HEARTBEAT" }).then((monitors) =>
monitors.length > 0 ? monitors[0] : null,
)) as HeartbeatMonitor | null;
if (!monitor) {
throw new Error("Monitor not found");
}
let typeData = monitor.type_data;
if (!typeData) {
throw new Error("Monitor type data not found");
}
try {
let heartbeatConfig = typeData;
let heartbeatSecret = heartbeatConfig.secretString;
if (heartbeatSecret === secret) {
// Store last heartbeat in seconds (DB uses seconds). Heartbeat evaluator tolerates older ms values.
let nowSec = GetNowTimestampUTC();
// Avoid rare collisions with minute-rounded monitoring timestamps (which can overwrite due to PK constraints).
// Minute-rounded timestamps always end with :00 seconds.
if (nowSec % 60 === 0) {
nowSec += 1;
}
await SetLastHeartbeat(tag, nowSec);
// Best-effort persist a heartbeat SIGNAL for restart recovery.
// Failure here should not break heartbeat reception.
try {
await InsertMonitoringData({
monitor_tag: tag,
timestamp: nowSec,
status: GC.UP,
latency: 0,
type: GC.SIGNAL,
error_message: null,
});
} catch (e) {
console.error("Error persisting heartbeat signal:", e);
}
return "OK";
}
} catch (e) {
console.error("Error registering heartbeat:", e);
}
throw new Error("Invalid heartbeat secret");
};
/**
* Removes a monitor tag from all GROUP monitors that reference it,
* rebalances weights equally, and deactivates groups left with < 2 members.
*/
async function removeTagFromGroupMonitors(tag: string): Promise<void> {
const groupMonitors = await GetMonitorsParsed({ monitor_type: "GROUP" });
for (const group of groupMonitors) {
const typeData = group.type_data as GroupMonitorTypeData;
if (!typeData.monitors || !Array.isArray(typeData.monitors)) continue;
const hasMember = typeData.monitors.some((m) => m.tag === tag);
if (!hasMember) continue;
// Remove the deleted tag
const remaining = typeData.monitors.filter((m) => m.tag !== tag);
// Rebalance weights equally across remaining monitors
if (remaining.length > 0) {
const weight = Math.round((1 / remaining.length) * 1000) / 1000;
for (let i = 0; i < remaining.length; i++) {
remaining[i].weight =
i === remaining.length - 1
? Math.round((1 - weight * (remaining.length - 1)) * 1000) / 1000
: weight;
}
}
typeData.monitors = remaining;
const updateData: Record<string, unknown> = {
id: group.id,
tag: group.tag,
name: group.name,
description: group.description,
image: group.image,
cron: group.cron,
default_status: group.default_status,
status: remaining.length < 2 ? "INACTIVE" : group.status,
category_name: group.category_name,
monitor_type: group.monitor_type,
type_data: JSON.stringify(typeData),
day_degraded_minimum_count: group.day_degraded_minimum_count,
day_down_minimum_count: group.day_down_minimum_count,
include_degraded_in_downtime: group.include_degraded_in_downtime,
is_hidden: group.is_hidden,
monitor_settings_json:
typeof group.monitor_settings_json === "string"
? group.monitor_settings_json
: JSON.stringify(group.monitor_settings_json),
external_url: group.external_url,
};
await db.updateMonitor(updateData as unknown as MonitorRecord);
}
}
export const DeleteMonitorCompletelyUsingTag = async (tag: string): Promise<number> => {
await db.deleteMonitorDataByTag(tag);
await db.deleteIncidentMonitorsByTag(tag);
await db.deleteMonitorAlertsByTag(tag);
await db.deletePageMonitorsByTag(tag);
await db.deleteMaintenanceMonitorsByTag(tag);
await removeTagFromGroupMonitors(tag);
await DeleteMonitorCaches(tag);
return await db.deleteMonitorsByTag(tag);
};
//getMonitorsByTag
export const GetMonitorsByTag = async (tag: string): Promise<MonitorRecord | undefined> => {
return await db.getMonitorsByTag(tag);
};
export const GetAllAlertsPaginated = async (
data: PaginationInput,
): Promise<{ alerts: MonitorAlert[]; total: number }> => {
const countResult = await db.getMonitorAlertsCount();
return {
alerts: await db.getMonitorAlertsPaginated(data.page, data.limit),
total: countResult ? Number(countResult.count) : 0,
};
};
export const GetMonitoringData = async (tag: string, since: number, now: number): Promise<MonitoringData[]> => {
return await db.getMonitoringData(tag, since, now);
};
export const GetMonitoringDataAll = async (tags: string[], since: number, now: number): Promise<MonitoringData[]> => {
return await db.getMonitoringDataAll(tags, since, now);
};
export const InsertNewAlert = async (data: MonitorAlertInsert): Promise<MonitorAlert | undefined> => {
if (await db.alertExists(data.monitor_tag, data.monitor_status, data.alert_status)) {
return;
}
await db.insertAlert(data);
return await db.getActiveAlert(data.monitor_tag, data.monitor_status, data.alert_status);
};
//getStatusCountsByInterval
export const GetStatusCountsByInterval = async (
monitor_tag: string | string[],
start: number,
interval: number,
numIntervals: number,
): Promise<TimestampStatusCount[]> => {
return await db.getStatusCountsByInterval(monitor_tag, start, interval, numIntervals);
};
//getMonitoringDataPaginated
export const GetMonitoringDataPaginated = async (
page: number,
limit: number,
filter?: { monitor_tag?: string; start_time?: number; end_time?: number },
): Promise<{ data: MonitoringData[]; total: number }> => {
const data = await db.getMonitoringDataPaginated(page, limit, filter);
const countResult = await db.getMonitoringDataCount(filter);
return { data, total: countResult.count };
};
export type BadgeType = "status" | "uptime" | "latency";
export interface BadgeParams {
tag: string;
sinceLast?: string | null;
hideDuration?: string | null;
label?: string | null;
labelColor?: string | null;
color?: string | null;
style?: string | null;
metric?: string | null;
}
function formatDuration(rangeInSeconds: number): string {
const days = Math.floor(rangeInSeconds / 86400);
const hours = Math.floor((rangeInSeconds % 86400) / 3600);
const minutes = Math.floor((rangeInSeconds % 3600) / 60);
if (days > 0 || minutes < 1) {
return `${days}d`;
} else if (hours > 0) {
return `${hours}h`;
} else if (minutes > 0) {
return `${minutes}m`;
}
return "";
}
export const GetBadge = async (badgeType: BadgeType, params: BadgeParams): Promise<Response> => {
const { tag } = params;
if (!tag) {
return new Response(ErrorSvg, {
headers: { "Content-Type": "image/svg+xml" },
});
}
let name: string;
let message: string;
let badgeColor: string = params.color || "#0079FF";
// For status badge, we get real-time status
if (badgeType === "status") {
let lastObj: { status: string } | undefined;
if (tag === "_") {
// All monitors status
const siteData = await db.getSiteDataByKey("siteName");
name = (siteData?.value as string) || "All Monitors";
lastObj = await GetLatestStatusActiveAll();
} else {
// Single monitor status
const monitors = await GetMonitorsParsed({ tag, status: "ACTIVE", is_hidden: "NO" });
if (monitors.length === 0) {
return new Response(ErrorSvg, {
headers: { "Content-Type": "image/svg+xml" },
});
}
const m = monitors[0];
name = m.name;
lastObj = (await GetLastMonitoringValue(m.tag, () => GetLatestMonitoringData(m.tag))) as { status: string };
}
const status = (lastObj?.status as string) || GC.NO_DATA;
message = status;
// Use status-specific color if no custom color provided
if (!params.color) {
//get site colors
let myColors = {} as Record<string, string>;
const siteColorsData = await db.getSiteDataByKey("colors");
if (siteColorsData && siteColorsData.value) {
try {
myColors = JSON.parse(siteColorsData.value);
} catch (e) {
myColors = {};
}
}
const statusColors: Record<string, string> = {
UP: myColors.UP || "#00dfa2",
DEGRADED: myColors.DEGRADED || "#e6ca61",
DOWN: myColors.DOWN || "#ca3038",
MAINTENANCE: myColors.MAINTENANCE || "#6679cc",
NO_DATA: myColors.NO_DATA || "#9ca3af",
};
badgeColor = statusColors[status] || statusColors.NO_DATA;
}
} else {
// For uptime/latency badges, we calculate over a time period
let sinceLast: number;
const sinceLastParam = params.sinceLast;
if (sinceLastParam == undefined || isNaN(Number(sinceLastParam)) || Number(sinceLastParam) < 60) {
sinceLast = 90 * 24 * 60 * 60;
} else {
sinceLast = Number(sinceLastParam);
}
const rangeInSeconds = sinceLast;
const now = Math.floor(Date.now() / 1000);
const since = GetMinuteStartNowTimestampUTC() - rangeInSeconds;
const hideDuration = params.hideDuration === "true";
const formatted = formatDuration(rangeInSeconds);
let stats: TimestampStatusCount[] = [];
let uptimeData: UptimeCalculatorResult = {
uptime: "-",
avgLatency: "-",
maxLatency: "-",
minLatency: "-",
};
if (tag === "_") {
// All monitors badge
const siteData = await db.getSiteDataByKey("siteName");
const siteName = siteData?.value as string | undefined;
name = siteName || "All Monitors";
const goodMonitors = await GetMonitorsParsed({ status: "ACTIVE", is_hidden: "NO" });
const activeTags = goodMonitors.map((monitor) => monitor.tag);
stats = await db.getStatusCountsByInterval(activeTags, since, now - since, 1);
uptimeData = UptimeCalculator(stats);
} else {
// Single monitor badge
const monitors = await GetMonitorsParsed({ tag });
if (monitors.length === 0) {
return new Response(ErrorSvg, {
headers: { "Content-Type": "image/svg+xml" },
});
}
const m = monitors[0];
name = m.name;
stats = await db.getStatusCountsByInterval(m.tag, since, now - since, 1);
uptimeData = UptimeCalculator(
stats,
m.monitor_settings_json?.uptime_formula_numerator,
m.monitor_settings_json?.uptime_formula_denominator,
);
}
// Determine message based on badge type
if (badgeType === "uptime") {
message = uptimeData.uptime;
} else {
// latency badge - support metric param (average, maximum, minimum)
const metric = params.metric || "average";
if (metric === "maximum") {
message = uptimeData.maxLatency;
} else if (metric === "minimum") {
message = uptimeData.minLatency;
} else {
message = uptimeData.avgLatency;
}
}
// Build label with duration suffix for uptime/latency
name = name + (hideDuration ? "" : ` ${formatted}`);
}
// Build final label
let label: string = params.label || name;
label = label.trim();
const format = {
label,
message,
color: badgeColor,
labelColor: params.labelColor || "#333",
style: getBadgeStyle(params.style ?? null),
};
const svg = makeBadge(format);
return new Response(svg, {
headers: { "Content-Type": "image/svg+xml" },
});
};
//calculate uptime for last N rows
export const CalculateUptimeForLastNRows = async (
tag: string | string[],
lastX: number,
numeratorStr: string,
denominatorStr: string,
): Promise<number> => {
const statusCounts = await db.getStatusCountsForLastN(tag, lastX);
const uptime = UptimeCalculator([statusCounts], numeratorStr, denominatorStr);
return UnparsePercentage(uptime.uptime);
};
export const IsUptimeGreaterThanXPercent = async (
tag: string | string[],
lastX: number,
threshold: number,
numeratorStr: string,
denominatorStr: string,
): Promise<boolean> => {
const uptimePercent = await CalculateUptimeForLastNRows(tag, lastX, numeratorStr, denominatorStr);
return uptimePercent > threshold;
};
export const IsUptimeLessThanXPercent = async (
tag: string | string[],
lastX: number,
threshold: number,
numeratorStr: string,
denominatorStr: string,
): Promise<boolean> => {
const uptimePercent = await CalculateUptimeForLastNRows(tag, lastX, numeratorStr, denominatorStr);
return uptimePercent < threshold;
};
export const GetStatusCountsByIntervalGroupedByMonitor = async (
monitorTags: string[],
startTimestamp: number,
intervalInSeconds: number,
numberOfPoints: number,
): Promise<Array<TimestampStatusCountByMonitor>> => {
const sortedTags = [...monitorTags].sort();
const cacheKey = `status_counts_grouped:${sortedTags.join(",")}:${startTimestamp}:${intervalInSeconds}:${numberOfPoints}`;
const cached = await getCache<Array<TimestampStatusCountByMonitor>>(cacheKey);
if (cached) {
return cached;
}
const result = await db.getStatusCountsByIntervalGroupedByMonitor(
monitorTags,
startTimestamp,
intervalInSeconds,
numberOfPoints,
);
await setCache(cacheKey, result, 60);
return result;
};