From: Dan Fuhry Date: Tue, 23 Apr 2024 16:17:43 +0000 (-0400) Subject: Add mTLS exporter X-Git-Url: https://go.fuhry.dev/?a=commitdiff_plain;h=30047746897970a402bdeee545a011b62843a852;p=runtime.git Add mTLS exporter --- diff --git a/metrics/mtls/stats.go b/metrics/mtls/stats.go new file mode 100644 index 0000000..6216ec9 --- /dev/null +++ b/metrics/mtls/stats.go @@ -0,0 +1,198 @@ +package mtls + +// NOTE: this is not part of the mtls package because mbclient depends on mtls +import ( + "context" + "crypto/x509" + "os" + "path" + "time" + + "go.fuhry.dev/runtime/metrics/metricbus/mbclient" + "go.fuhry.dev/runtime/mtls" + "go.fuhry.dev/runtime/mtls/certutil" + "go.fuhry.dev/runtime/mtls/fsnotify" + "go.fuhry.dev/runtime/utils/hashset" + "go.fuhry.dev/runtime/utils/log" +) + +type mtlsMetrics struct { + log *log.Logger + + svc *mbclient.MetricBusService + + secondsUntilExpiration mbclient.GaugeMetric + rotationsObserved mbclient.CounterMetric + certificatesOnHost mbclient.GaugeMetric + + certs map[string]*mtlsMetricCertificate +} + +type mtlsMetricCertificate struct { + baseDir string +} + +func NewMtlsMetricsService(ctx context.Context) *mtlsMetrics { + svc := mbclient.NewService(ctx) + + return &mtlsMetrics{ + log: log.WithPrefix("mtls_monitor"), + svc: svc, + + secondsUntilExpiration: svc.DefineGauge( + "mtls_certificate_seconds_until_expire", + "Time until an mTLS certificate expires", + "cert_name"), + rotationsObserved: svc.DefineCounter( + "mtls_certificate_rotations", + "Number of times the rotation of a given certificate has been observed", + "cert_name"), + certificatesOnHost: svc.DefineGauge( + "mtls_host_certificates_count", + "Total number of certificates present", + "status"), + + certs: make(map[string]*mtlsMetricCertificate), + } +} + +func (m *mtlsMetrics) Tick() { + allCerts := m.discoverMtlsCertificates() + + for _, baseDir := range allCerts { + if _, ok := m.certs[baseDir]; !ok { + m.log.Noticef("found a new certificate at %s", baseDir) + certWatcher := &mtlsMetricCertificate{ + baseDir: baseDir, + } + + err := fsnotify.NotifyPath(certWatcher.LeafPath(), m.fsNotifyEvent) + if err != nil { + m.log.Warnf("failed to add watcher on %s, not watching this cert: %v", + certWatcher.LeafPath(), err) + continue + } + + m.certs[baseDir] = certWatcher + } + } + + removedCerts := make([]string, 0) + certsHash := hashset.FromSlice(allCerts) + for baseDir, _ := range m.certs { + if !certsHash.Contains(baseDir) { + m.log.Noticef("certificate removed: %s", baseDir) + removedCerts = append(removedCerts, baseDir) + } + } + + for _, removedCert := range removedCerts { + delete(m.certs, removedCert) + } + + var countActive, countExpired float64 + + for _, cert := range m.certs { + expiresIn := cert.TimeUntilExpiration() + if expiresIn > 0 { + m.log.V(2).Infof("cert %s: counted as active", cert.Name()) + countActive += 1 + } else { + m.log.V(2).Infof("cert %s: counted as expired", cert.Name()) + countExpired += 1 + } + + m.log.V(2).Infof("cert %s: %.0f seconds until expiration", cert.Name(), expiresIn.Seconds()) + m.secondsUntilExpiration.WithLabelValues(mbclient.KV{"cert_name": cert.Name()}).Set(expiresIn.Seconds()) + } + + m.log.V(1).Infof("total %.0f active/%.0f expired certificates", countActive, countExpired) + m.certificatesOnHost.WithLabelValues(mbclient.KV{"status": "active"}).Set(countActive) + m.certificatesOnHost.WithLabelValues(mbclient.KV{"status": "expired"}).Set(countExpired) +} + +func (m *mtlsMetrics) FlushAndWait() { + m.svc.FlushAndWait() +} + +func (m *mtlsMetrics) fsNotifyEvent(eventPath string, op fsnotify.Op) { + if op != fsnotify.Create && op != fsnotify.Close { + return + } + + dir := path.Dir(eventPath) + cert, ok := m.certs[eventPath] + if !ok { + cert, ok = m.certs[dir] + } + if ok { + m.log.V(1).Infof("certificate %s was just rotated", cert.Name()) + m.rotationsObserved.WithLabelValues(mbclient.KV{"cert_name": cert.Name()}).Add(1) + } else { + m.log.V(1).Warnf("observed rotation of an unaccounted cert at path: %s", eventPath) + } +} + +func (m *mtlsMetrics) discoverMtlsCertificates() []string { + certDirs := make([]string, 0) + + for _, dir := range mtls.RootPaths() { + if entries, err := os.ReadDir(dir); err == nil { + for _, mtlsEntry := range entries { + if !mtlsEntry.IsDir() { + continue + } + + leafPath := path.Join(dir, mtlsEntry.Name(), "cert.pem") + leafStat, err := os.Stat(leafPath) + if err != nil { + continue + } + if !leafStat.Mode().IsRegular() { + continue + } + + certDirs = append(certDirs, path.Join(dir, mtlsEntry.Name())) + } + } + } + + return certDirs +} + +func (c *mtlsMetricCertificate) LeafPath() string { + return path.Join(c.baseDir, "cert.pem") +} + +func (c *mtlsMetricCertificate) Leaf() *x509.Certificate { + leafPath := c.LeafPath() + certsInLeaf, err := certutil.LoadCertificatesFromPEM(leafPath) + if err != nil { + return nil + } + + if len(certsInLeaf) != 1 { + return nil + } + + return certsInLeaf[0] +} + +func (c mtlsMetricCertificate) Name() string { + return path.Base(c.baseDir) +} + +func (c *mtlsMetricCertificate) TimeUntilExpiration() time.Duration { + cert := c.Leaf() + if cert == nil { + return 0 + } + + now := time.Now() + + if now.After(cert.NotAfter) { + return 0 + } + + return cert.NotAfter.Sub(now) +} diff --git a/mtls/mtls_exporter/main.go b/mtls/mtls_exporter/main.go new file mode 100644 index 0000000..636dbad --- /dev/null +++ b/mtls/mtls_exporter/main.go @@ -0,0 +1,36 @@ +package main + +import ( + "context" + "flag" + "os/signal" + "syscall" + "time" + + "github.com/coreos/go-systemd/daemon" + "go.fuhry.dev/runtime/metrics/mtls" +) + +func main() { + ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + flag.Parse() + + svc := mtls.NewMtlsMetricsService(ctx) + + ticker := time.NewTicker(10 * time.Second) + + defer svc.FlushAndWait() + + svc.Tick() + + daemon.SdNotify(false, daemon.SdNotifyReady) + + for { + select { + case <-ticker.C: + svc.Tick() + case <-ctx.Done(): + return + } + } +} diff --git a/mtls/mtls_exporter/mtls_exporter b/mtls/mtls_exporter/mtls_exporter new file mode 100755 index 0000000..eceedde Binary files /dev/null and b/mtls/mtls_exporter/mtls_exporter differ diff --git a/mtls/mtls_exporter/systemd/mtls-exporter.service b/mtls/mtls_exporter/systemd/mtls-exporter.service new file mode 100644 index 0000000..36a20d0 --- /dev/null +++ b/mtls/mtls_exporter/systemd/mtls-exporter.service @@ -0,0 +1,11 @@ +[Unit] +Description=Export mTLS certificate status +Requires=metric-collector.service + +[Service] +Type=notify +User=daemon +ExecStart=/usr/bin/mtls-exporter + +[Install] +WantedBy=default.target diff --git a/mtls/provider_file.go b/mtls/provider_file.go index fa4b638..6d5e591 100644 --- a/mtls/provider_file.go +++ b/mtls/provider_file.go @@ -479,6 +479,14 @@ func appendMtlsCertificateDir(path string) error { return nil } +// RootPaths returns the list of directories that are searched for mTLS certificates. +func RootPaths() []string { + rootPathsCopy := make([]string, len(mtlsRootPaths)) + copy(rootPathsCopy, mtlsRootPaths) + + return rootPathsCopy +} + func init() { defaultRootCAFile = fmt.Sprintf("%s/rootca.pem", defaultMtlsRootPath) defaultIntermediateCAFile = fmt.Sprintf("%s/ca.pem", defaultMtlsRootPath)