From dc2aa554d01e9016625694ec3684f3fa63d55fbe Mon Sep 17 00:00:00 2001 From: Natalie Rose Date: Mon, 5 Oct 2026 18:31:39 +1000 Subject: [PATCH] Add metrics --- bin/lacuna.psgi | 8 ++ cpanfile | 1 + cpanfile.snapshot | 246 +++++++----------------------------------- lib/Lacuna/Metrics.pm | 138 ++++++++++++++++++++++++ t/004_Metrics.t | 145 +++++++++++++++++++++++++ 5 files changed, 333 insertions(+), 205 deletions(-) create mode 100644 lib/Lacuna/Metrics.pm create mode 100644 t/004_Metrics.t diff --git a/bin/lacuna.psgi b/bin/lacuna.psgi index b10f4d7c..e4e31fc5 100644 --- a/bin/lacuna.psgi +++ b/bin/lacuna.psgi @@ -6,6 +6,7 @@ use Plack::App::URLMap; use Log::Log4perl; use Log::Any::Adapter; use Lacuna; +use Lacuna::Metrics; use Plack::Builder; use JSON qw(encode_json); @@ -42,6 +43,9 @@ my $gameover = [ 500, ]; my $app = builder { + # Outermost, so it also sees the maintenance and game-over responses below. + enable sub { Lacuna::Metrics->middleware(shift) }; + if ($^O ne 'darwin' && not defined $ENV{'TLE_NO_MIDDLEWARE'} ) { ##Wrapper to fully enable size limiting. The psgix.harakiri has to be set for it to work. enable sub { @@ -80,6 +84,10 @@ my $app = builder { mount '/starman_ping' => sub { [200, [ 'Content-Type' => 'text/plain'], [ 'pong' ]] }; + # Prometheus scrape target. Caddy returns 404 for /metrics publicly (see + # deployment/caddy/Caddyfile), so this is reachable on the internal network only. + mount '/metrics' => Lacuna::Metrics->psgi; + mount "/map" => Lacuna::RPC::Map->new->to_app; mount "/body" => Lacuna::RPC::Body->new->to_app; mount "/empire" => Lacuna::RPC::Empire->new->to_app; diff --git a/cpanfile b/cpanfile index 48768275..4c2b24e4 100644 --- a/cpanfile +++ b/cpanfile @@ -53,6 +53,7 @@ requires 'Plack::Middleware::Headers'; requires 'Plack::Middleware::SizeLimit'; requires 'Plack::Middleware::LightProfile'; requires 'JSON::RPC::Dispatcher'; +requires 'Prometheus::Tiny::Shared', '0.027'; requires 'JSON::RPC::Dispatcher::App'; # --- Runtime: DB / ORM ----------------------------------------------------- diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 7c26c860..f96af625 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -1742,26 +1742,6 @@ DISTRIBUTIONS Canary::Stability 0 ExtUtils::MakeMaker 6.52 common::sense 0 - Email-Abstract-3.010 - pathname: R/RJ/RJBS/Email-Abstract-3.010.tar.gz - provides: - Email::Abstract 3.010 - Email::Abstract::EmailMIME 3.010 - Email::Abstract::EmailSimple 3.010 - Email::Abstract::MIMEEntity 3.010 - Email::Abstract::MailInternet 3.010 - Email::Abstract::MailMessage 3.010 - Email::Abstract::Plugin 3.010 - requirements: - Carp 0 - Email::Simple 1.998 - ExtUtils::MakeMaker 6.78 - MRO::Compat 0 - Module::Pluggable 1.5 - Scalar::Util 0 - perl 5.006 - strict 0 - warnings 0 Email-Address-1.913 pathname: R/RJ/RJBS/Email-Address-1.913.tar.gz provides: @@ -1772,143 +1752,6 @@ DISTRIBUTIONS perl 5.012 strict 0 warnings 0 - Email-Address-XS-1.05 - pathname: P/PA/PALI/Email-Address-XS-1.05.tar.gz - provides: - Email::Address::XS 1.05 - requirements: - Carp 0 - Exporter 0 - ExtUtils::MakeMaker 0 - XSLoader 0 - base 0 - overload 0 - perl 5.006000 - strict 0 - warnings 0 - Email-Date-Format-1.008 - pathname: R/RJ/RJBS/Email-Date-Format-1.008.tar.gz - provides: - Email::Date::Format 1.008 - requirements: - Exporter 5.57 - ExtUtils::MakeMaker 6.78 - Time::Local 1.27 - perl 5.012 - warnings 0 - Email-MIME-1.954 - pathname: R/RJ/RJBS/Email-MIME-1.954.tar.gz - provides: - Email::MIME 1.954 - Email::MIME::Creator 1.954 - Email::MIME::Encode 1.954 - Email::MIME::Header 1.954 - Email::MIME::Header::AddressList 1.954 - Email::MIME::Modifier 1.954 - requirements: - Carp 0 - Email::Address::XS 0 - Email::MIME::ContentType 1.023 - Email::MIME::Encodings 1.314 - Email::MessageID 0 - Email::Simple 2.212 - Email::Simple::Creator 0 - Email::Simple::Header 0 - Encode 1.9801 - ExtUtils::MakeMaker 6.78 - MIME::Base64 0 - MIME::Types 1.13 - Module::Runtime 0 - Scalar::Util 0 - parent 0 - perl 5.012 - strict 0 - warnings 0 - Email-MIME-ContentType-1.028 - pathname: R/RJ/RJBS/Email-MIME-ContentType-1.028.tar.gz - provides: - Email::MIME::ContentType 1.028 - requirements: - Carp 0 - Encode 2.87 - Exporter 5.57 - ExtUtils::MakeMaker 6.78 - Text::Unidecode 0 - perl 5.012 - warnings 0 - Email-MIME-Encodings-1.317 - pathname: R/RJ/RJBS/Email-MIME-Encodings-1.317.tar.gz - provides: - Email::MIME::Encodings 1.317 - requirements: - Carp 0 - ExtUtils::MakeMaker 6.78 - MIME::Base64 3.05 - MIME::QuotedPrint 3.05 - perl 5.012 - strict 0 - warnings 0 - Email-MessageID-1.408 - pathname: R/RJ/RJBS/Email-MessageID-1.408.tar.gz - provides: - Email::MessageID 1.408 - requirements: - ExtUtils::MakeMaker 6.78 - Sys::Hostname 0 - overload 0 - perl 5.012 - strict 0 - warnings 0 - Email-Send-2.202 - pathname: R/RJ/RJBS/Email-Send-2.202.tar.gz - provides: - Email::Send 2.202 - Email::Send::NNTP 2.202 - Email::Send::Qmail 2.202 - Email::Send::SMTP 2.202 - Email::Send::Sendmail 2.202 - Email::Send::Test 2.202 - requirements: - Email::Abstract 0 - Email::Address 1.80 - Email::Simple 1.92 - ExtUtils::MakeMaker 0 - File::Spec 0 - Module::Pluggable 2.97 - Net::SMTP 0 - Return::Value 0 - Scalar::Util 1.02 - Symbol 0 - perl 5.014000 - strict 0 - vars 0 - Email-Simple-2.218 - pathname: R/RJ/RJBS/Email-Simple-2.218.tar.gz - provides: - Email::Simple 2.218 - Email::Simple::Creator 2.218 - Email::Simple::Header 2.218 - requirements: - Carp 0 - Email::Date::Format 0 - ExtUtils::MakeMaker 6.78 - perl 5.012 - strict 0 - warnings 0 - Email-Stuff-2.105 - pathname: R/RJ/RJBS/Email-Stuff-2.105.tar.gz - provides: - Email::Stuff 2.105 - requirements: - Carp 0 - Email::MIME 1.901 - Email::Send 2.185 - Email::Simple 1.998 - ExtUtils::MakeMaker 6.30 - File::Basename 0 - File::Type 0.22 - Params::Util 0.23 - prefork 1.01 Email-Valid-1.204 pathname: R/RJ/RJBS/Email-Valid-1.204.tar.gz provides: @@ -2127,14 +1970,6 @@ DISTRIBUTIONS perl 5.006 strict 0 warnings 0 - File-Type-0.22 - pathname: P/PM/PMISON/File-Type-0.22.tar.gz - provides: - File::Type 0.22 - File::Type::Builder 0.11 - requirements: - IO::File 0.01 - Test::More 0.01 File-Which-1.27 pathname: P/PL/PLICEASE/File-Which-1.27.tar.gz provides: @@ -2390,6 +2225,21 @@ DISTRIBUTIONS requirements: ExtUtils::MakeMaker 0 perl 5.008001 + Hash-SharedMem-0.007 + pathname: K/KS/KSTAR/Hash-SharedMem-0.007.tar.gz + provides: + Hash::SharedMem 0.007 + Hash::SharedMem::Handle 0.007 + requirements: + Exporter 0 + ExtUtils::CBuilder 0.15 + ExtUtils::MakeMaker 0 + ExtUtils::ParseXS 3.30 + XSLoader 0 + parent 0 + perl 5.006 + strict 0 + warnings 0 IO-HTML-1.004 pathname: C/CJ/CJM/IO-HTML-1.004.tar.gz provides: @@ -2773,17 +2623,6 @@ DISTRIBUTIONS ExtUtils::MakeMaker 0 perl 5.008001 utf8 0 - MIME-Types-2.30 - pathname: M/MA/MARKOV/MIME-Types-2.30.tar.gz - provides: - MIME::Type 2.30 - MIME::Types 2.30 - MojoX::MIME::Types 2.30 - requirements: - ExtUtils::MakeMaker 0 - File::Basename 0 - File::Spec 0 - List::Util 0 MRO-Compat-0.15 pathname: H/HA/HAARG/MRO-Compat-0.15.tar.gz provides: @@ -4545,6 +4384,32 @@ DISTRIBUTIONS Linux::Pid 0 Linux::Smaps 0 perl 5.005 + Prometheus-Tiny-0.011 + pathname: R/RO/ROBN/Prometheus-Tiny-0.011.tar.gz + provides: + Prometheus::Tiny 0.011 + requirements: + Carp 0 + ExtUtils::MakeMaker 0 + Scalar::Util 0 + strict 0 + warnings 0 + Prometheus-Tiny-Shared-0.027 + pathname: R/RO/ROBN/Prometheus-Tiny-Shared-0.027.tar.gz + provides: + Prometheus::Tiny::Shared 0.027 + requirements: + Carp 0 + ExtUtils::MakeMaker 0 + File::Path 0 + File::Temp 0 + Hash::SharedMem 0 + JSON::XS 0 + Prometheus::Tiny 0.011 + Scalar::Util 0 + parent 0 + strict 0 + warnings 0 Protocol-WebSocket-0.26 pathname: V/VT/VTI/Protocol-WebSocket-0.26.tar.gz provides: @@ -4612,17 +4477,6 @@ DISTRIBUTIONS strict 0 vars 0 warnings 0 - Return-Value-1.666005 - pathname: R/RJ/RJBS/Return-Value-1.666005.tar.gz - provides: - Return::Value 1.666005 - requirements: - Carp 0 - Exporter 5.57 - ExtUtils::MakeMaker 0 - overload 0 - strict 0 - warnings 0 Role-Tiny-2.002005 pathname: H/HA/HAARG/Role-Tiny-2.002005.tar.gz provides: @@ -5814,13 +5668,6 @@ DISTRIBUTIONS requirements: ExtUtils::MakeMaker 0 perl 5.006 - Text-Unidecode-1.30 - pathname: S/SB/SBURKE/Text-Unidecode-1.30.tar.gz - provides: - Text::Unidecode 1.30 - requirements: - ExtUtils::MakeMaker 0 - perl 5.008 Text-WagnerFischer-0.04 pathname: D/DA/DAVIDEBE/Text-WagnerFischer-0.04.tar.gz provides: @@ -6597,17 +6444,6 @@ DISTRIBUTIONS ExtUtils::MakeMaker 0 Package::Stash 0.23 perl 5.008001 - prefork-1.05 - pathname: E/ET/ETHER/prefork-1.05.tar.gz - provides: - prefork 1.05 - requirements: - Carp 0 - ExtUtils::MakeMaker 0 - List::Util 0.18 - Scalar::Util 0.18 - perl 5.006 - strict 0 rlib-0.02 pathname: G/GB/GBARR/rlib-0.02.tar.gz provides: diff --git a/lib/Lacuna/Metrics.pm b/lib/Lacuna/Metrics.pm new file mode 100644 index 00000000..c5cd3b70 --- /dev/null +++ b/lib/Lacuna/Metrics.pm @@ -0,0 +1,138 @@ +package Lacuna::Metrics; + +# +# Prometheus metrics for the Perl app: request rate, response time and error +# counts per RPC method. Served at /metrics (see bin/lacuna.psgi), scraped by +# Prometheus on the internal network only; Caddy returns 404 for /metrics +# publicly. +# +# Starman runs several forked workers, so the counters live in a shared store +# (Prometheus::Tiny::Shared) rather than in each process. Call middleware() +# from bin/lacuna.psgi at load time, so the store is created once in the +# master process before the workers fork (see --preload-app in +# bin/start_lacuna.sh). +# +# Label values are bounded so a client can't create unlimited series by +# inventing method or path names: unrecognised methods and mounts are reported +# as 'unknown'. +# + +use strict; +use warnings; +use utf8; +use Time::HiRes qw(time); +use JSON qw(decode_json); +use Prometheus::Tiny::Shared; + +# Keep in step with the mount table in bin/lacuna.psgi. +my $STORE_FILE = $ENV{LACUNA_METRICS_FILE} || '/tmp/lacuna-metrics.shm'; + +my $prom; + +sub prom { + return $prom if $prom; + # init_file wipes the store on each container start. Counters restart at + # zero, which Prometheus handles as a counter reset. + $prom = Prometheus::Tiny::Shared->new(filename => $STORE_FILE, init_file => 1); + $prom->declare('tle_rpc_requests_total', + help => 'Requests handled by the Perl app, by mount and RPC method.', + type => 'counter'); + $prom->declare('tle_rpc_request_duration_seconds', + help => 'Time the Perl app spent handling a request, by mount and RPC method.', + type => 'histogram', + buckets => [0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]); + $prom->declare('tle_rpc_errors_total', + help => 'Requests that returned an error, by JSON-RPC error code. -32603 is a Perl exception inside a method; other codes are game errors. http_ is used when the body is not JSON-RPC.', + type => 'counter'); + return $prom; +} + +sub psgi { + return prom()->psgi; +} + +# Paths that are not game traffic and would only add noise. +my %SKIP = map { $_ => 1 } qw(/metrics /starman_ping); + +sub middleware { + my ($class, $app) = @_; + prom(); # create the store now, in the master process + + return sub { + my ($env) = @_; + + my $path = $env->{PATH_INFO} // '/'; + return $app->($env) if $SKIP{$path}; + + # The JSON-RPC body holds the method name, but the app still needs to + # read it. Buffer it and hand back an in-memory input handle. + my $method_name; + if ($env->{REQUEST_METHOD} eq 'POST' && $env->{'psgi.input'}) { + my $input = $env->{'psgi.input'}; + my $buffer = ''; + while ($input->read(my $chunk, 8192)) { + $buffer .= $chunk; + } + open my $fh, '<:raw', \$buffer or die "cannot reopen request body: $!"; + $env->{'psgi.input'} = $fh; + + my $request = eval { decode_json($buffer) }; + $method_name = $request->{method} if ref $request eq 'HASH'; + } + + my $start = time; + my $res = eval { $app->($env) }; + my $error = $@; + my $elapsed = time - $start; + + if ($error) { + eval { + prom()->inc('tle_rpc_requests_total', { mount => 'unknown', method => 'unknown' }); + prom()->inc('tle_rpc_errors_total', { code => 'unhandled' }); + 1; + } or warn "Lacuna::Metrics: could not record exception: $@"; + die $error; + } + + # Only a real (array) response carries a body we can inspect. + return $res unless ref $res eq 'ARRAY'; + + my ($status, $headers, $body) = @$res; + my $text = ref $body eq 'ARRAY' ? join('', grep { defined } @$body) : ''; + + my $response = length $text ? eval { decode_json($text) } : undef; + my $is_rpc = ref $response eq 'HASH' && ref $response->{error} eq 'HASH'; + + # A JSON-RPC error has the JSON-RPC code, which the dispatcher maps to HTTP status. + my $rpc_error_code = $is_rpc ? $response->{error}{code} : undef; + my $error_code = $is_rpc ? ($rpc_error_code // 'rpc_error') + : $status >= 400 ? 'http_' . $status + : undef; + + # A 404 that isn't JSON-RPC is a URL the app doesn't serve, so no mount or method is real. + # -32601 (method not found) keeps the real mount but reports the method as unknown. + my ($mount) = $path =~ m{^/([^/]+)}; + $mount = 'root' unless defined $mount; + $mount = 'unknown' if $status == 404 && !$is_rpc; + + my $method = 'unknown'; + if (defined $method_name && length $method_name + && !($status == 404 && !$is_rpc) + && !(defined $rpc_error_code && $rpc_error_code == -32601)) { + $method = $method_name; + } + + my $labels = { mount => $mount, method => $method }; + # A failed metric write must never fail the request it describes. + eval { + prom()->inc('tle_rpc_requests_total', $labels); + prom()->histogram_observe('tle_rpc_request_duration_seconds', $elapsed, $labels); + prom()->inc('tle_rpc_errors_total', { code => $error_code }) if defined $error_code; + 1; + } or warn "Lacuna::Metrics: could not record request: $@"; + + return $res; + }; +} + +1; diff --git a/t/004_Metrics.t b/t/004_Metrics.t new file mode 100644 index 00000000..ec69736f --- /dev/null +++ b/t/004_Metrics.t @@ -0,0 +1,145 @@ +#!/usr/bin/perl +# +# Unit tests for Lacuna::Metrics. Standalone: uses a fake PSGI app that +# mimics the JSON-RPC dispatcher's status codes, so no database, memcached or +# running server is needed. +# + +use strict; +use warnings; +use utf8; + +BEGIN { + # Keep the test store out of the real /tmp used by the server. + $ENV{LACUNA_METRICS_FILE} = "/tmp/lacuna-metrics-test-$$.shm"; +} + +use Test::More; +use POSIX (); +use JSON qw(decode_json encode_json); +use Lacuna::Metrics; + +my %KNOWN = map { $_ => 1 } qw(view_profile ping fail boom); + +# Fake app. Mirrors JSON::RPC::Dispatcher: an unknown method is -32601 with +# HTTP 404, and a game error is a JSON-RPC error with HTTP 500. +my $app = sub { + my ($env) = @_; + return [200, ['Content-Type' => 'text/plain'], ['metrics']] if $env->{PATH_INFO} eq '/metrics'; + return [404, ['Content-Type' => 'text/plain'], ['Not Found']] if $env->{PATH_INFO} eq '/missing'; + + my $body = do { local $/; my $in = $env->{'psgi.input'}; <$in> }; + my $req = decode_json($body); + my $id = $req->{id}; + my $json = ['Content-Type' => 'application/json-rpc']; + + return [404, $json, [encode_json({ jsonrpc => '2.0', id => $id, error => { code => -32601, message => 'Method not found.' } })]] + unless $KNOWN{$req->{method}}; + + die "kaboom\n" if $req->{method} eq 'boom'; + if ($req->{method} eq 'fail') { + return [500, $json, [encode_json({ jsonrpc => '2.0', id => $id, error => { code => 1002, message => 'Empire does not exist.' } })]]; + } + return [200, $json, [encode_json({ jsonrpc => '2.0', id => $id, result => { ok => 1 } })]]; +}; + +my $mw = Lacuna::Metrics->middleware($app); + +sub rpc_env { + my ($path, $method) = @_; + my $body = encode_json({ jsonrpc => '2.0', id => 1, method => $method, params => [] }); + open my $in, '<:raw', \$body or die $!; + return { REQUEST_METHOD => 'POST', PATH_INFO => $path, 'psgi.input' => $in }; +} + +# Value of one series, matched on metric name and exact label set. Prometheus +# output sorts labels, so order is not compared. +sub metric_value { + my ($name, $labels) = @_; + $labels ||= {}; + for my $line (split /\n/, Lacuna::Metrics::prom()->format) { + next if $line =~ /^#/; + my ($metric, $label_str, $value) = $line =~ /^(\w+)(?:\{(.*)\})? (\S+)$/ or next; + next unless $metric eq $name; + my %got = map { /^(\w+)="([^"]*)"$/ ? ($1, $2) : () } split /,(?=\w+=")/, ($label_str // ''); + next unless keys %got == keys %$labels; + next if grep { !exists $got{$_} || $got{$_} ne $labels->{$_} } keys %$labels; + return $value; + } + return 0; +} + +# A normal call: the app still receives the body, and the method is recorded. +{ + my $res = $mw->(rpc_env('/empire', 'view_profile')); + is($res->[0], 200, 'successful call passes through'); + like(join('', @{ $res->[2] }), qr/"ok":1/, 'response body is unchanged'); + is(metric_value('tle_rpc_requests_total', { mount => 'empire', method => 'view_profile' }), 1, + 'request counted by mount and method'); + is(metric_value('tle_rpc_request_duration_seconds_count', { mount => 'empire', method => 'view_profile' }), 1, + 'latency observed'); +} + +# A game error is an HTTP 500 with a JSON-RPC body: counted by its code. +{ + $mw->(rpc_env('/empire', 'fail')); + is(metric_value('tle_rpc_errors_total', { code => '1002' }), 1, 'game error counted by JSON-RPC code'); + is(metric_value('tle_rpc_requests_total', { mount => 'empire', method => 'fail' }), 1, + 'failing call still counted as a request'); +} + +# Method names the dispatcher doesn't know are 'unknown', so clients can't create series by inventing names. +{ + my $before = metric_value('tle_rpc_requests_total', { mount => 'empire', method => 'unknown' }); + $mw->(rpc_env('/empire', 'totally_made_up_name_123')); + is(metric_value('tle_rpc_requests_total', { mount => 'empire', method => 'unknown' }), $before + 1, + 'unknown method is bucketed under "unknown"'); + is(metric_value('tle_rpc_requests_total', { mount => 'empire', method => 'totally_made_up_name_123' }), 0, + 'invented method name is never stored as a label'); +} + +# An unmatched path is 'unknown' mount and method, even though the body names a method. +{ + my $before = metric_value('tle_rpc_requests_total', { mount => 'unknown', method => 'unknown' }); + $mw->(rpc_env('/missing', 'ping')); + is(metric_value('tle_rpc_requests_total', { mount => 'unknown', method => 'unknown' }), $before + 1, + 'unmatched mount is bucketed under "unknown"'); + is(metric_value('tle_rpc_requests_total', { mount => 'missing', method => 'ping' }), 0, + 'unmatched path is never stored as a mount label'); +} + +# A Perl exception that escapes the app is counted as unhandled and re-thrown. +{ + my $died = eval { $mw->(rpc_env('/body', 'boom')); 1 } ? 0 : 1; + ok($died, 'exception is re-thrown'); + is(metric_value('tle_rpc_errors_total', { code => 'unhandled' }), 1, 'exception counted as unhandled'); +} + +# /metrics and /starman_ping are not game traffic and are not counted. +{ + my $before = metric_value('tle_rpc_requests_total', { mount => 'metrics', method => 'unknown' }); + $mw->({ REQUEST_METHOD => 'GET', PATH_INFO => '/metrics' }); + is(metric_value('tle_rpc_requests_total', { mount => 'metrics', method => 'unknown' }), $before, + '/metrics is not counted'); +} + +# Forked workers share one store: counts from children appear in the parent. +{ + my $before = metric_value('tle_rpc_requests_total', { mount => 'empire', method => 'ping' }); + my @pids; + for (1 .. 3) { + my $pid = fork(); + die "fork failed: $!" unless defined $pid; + if ($pid == 0) { + $mw->(rpc_env('/empire', 'ping')) for 1 .. 5; + POSIX::_exit(0); # skip Test::More's END block in the child + } + push @pids, $pid; + } + waitpid($_, 0) for @pids; + is(metric_value('tle_rpc_requests_total', { mount => 'empire', method => 'ping' }) - $before, 15, + 'counts from 3 forked workers are summed in the shared store'); +} + +unlink $ENV{LACUNA_METRICS_FILE}; +done_testing; -- knotmirror 0.0.1