Add mTLS exporter

diff --git a/metrics/mtls/stats.go b/metrics/mtls/stats.go
new file mode 100644 (file)
index 0000000..6216ec9
--- /dev/null
+++ b/metrics/mtls/stats.go
@@ -0,0 +1,198 @@
+package mtls
+
+// NOTE: this is not part of the mtls package because mbclient depends on mtls
+import (
+       "context"
+       "crypto/x509"
+       "os"
+       "path"
+       "time"
+
+       "go.fuhry.dev/runtime/metrics/metricbus/mbclient"
+       "go.fuhry.dev/runtime/mtls"
+       "go.fuhry.dev/runtime/mtls/certutil"
+       "go.fuhry.dev/runtime/mtls/fsnotify"
+       "go.fuhry.dev/runtime/utils/hashset"
+       "go.fuhry.dev/runtime/utils/log"
+)
+
+type mtlsMetrics struct {
+       log *log.Logger
+
+       svc *mbclient.MetricBusService
+
+       secondsUntilExpiration mbclient.GaugeMetric
+       rotationsObserved      mbclient.CounterMetric
+       certificatesOnHost     mbclient.GaugeMetric
+
+       certs map[string]*mtlsMetricCertificate
+}
+
+type mtlsMetricCertificate struct {
+       baseDir string
+}
+
+func NewMtlsMetricsService(ctx context.Context) *mtlsMetrics {
+       svc := mbclient.NewService(ctx)
+
+       return &mtlsMetrics{
+               log: log.WithPrefix("mtls_monitor"),
+               svc: svc,
+
+               secondsUntilExpiration: svc.DefineGauge(
+                       "mtls_certificate_seconds_until_expire",
+                       "Time until an mTLS certificate expires",
+                       "cert_name"),
+               rotationsObserved: svc.DefineCounter(
+                       "mtls_certificate_rotations",
+                       "Number of times the rotation of a given certificate has been observed",
+                       "cert_name"),
+               certificatesOnHost: svc.DefineGauge(
+                       "mtls_host_certificates_count",
+                       "Total number of certificates present",
+                       "status"),
+
+               certs: make(map[string]*mtlsMetricCertificate),
+       }
+}
+
+func (m *mtlsMetrics) Tick() {
+       allCerts := m.discoverMtlsCertificates()
+
+       for _, baseDir := range allCerts {
+               if _, ok := m.certs[baseDir]; !ok {
+                       m.log.Noticef("found a new certificate at %s", baseDir)
+                       certWatcher := &mtlsMetricCertificate{
+                               baseDir: baseDir,
+                       }
+
+                       err := fsnotify.NotifyPath(certWatcher.LeafPath(), m.fsNotifyEvent)
+                       if err != nil {
+                               m.log.Warnf("failed to add watcher on %s, not watching this cert: %v",
+                                       certWatcher.LeafPath(), err)
+                               continue
+                       }
+
+                       m.certs[baseDir] = certWatcher
+               }
+       }
+
+       removedCerts := make([]string, 0)
+       certsHash := hashset.FromSlice(allCerts)
+       for baseDir, _ := range m.certs {
+               if !certsHash.Contains(baseDir) {
+                       m.log.Noticef("certificate removed: %s", baseDir)
+                       removedCerts = append(removedCerts, baseDir)
+               }
+       }
+
+       for _, removedCert := range removedCerts {
+               delete(m.certs, removedCert)
+       }
+
+       var countActive, countExpired float64
+
+       for _, cert := range m.certs {
+               expiresIn := cert.TimeUntilExpiration()
+               if expiresIn > 0 {
+                       m.log.V(2).Infof("cert %s: counted as active", cert.Name())
+                       countActive += 1
+               } else {
+                       m.log.V(2).Infof("cert %s: counted as expired", cert.Name())
+                       countExpired += 1
+               }
+
+               m.log.V(2).Infof("cert %s: %.0f seconds until expiration", cert.Name(), expiresIn.Seconds())
+               m.secondsUntilExpiration.WithLabelValues(mbclient.KV{"cert_name": cert.Name()}).Set(expiresIn.Seconds())
+       }
+
+       m.log.V(1).Infof("total %.0f active/%.0f expired certificates", countActive, countExpired)
+       m.certificatesOnHost.WithLabelValues(mbclient.KV{"status": "active"}).Set(countActive)
+       m.certificatesOnHost.WithLabelValues(mbclient.KV{"status": "expired"}).Set(countExpired)
+}
+
+func (m *mtlsMetrics) FlushAndWait() {
+       m.svc.FlushAndWait()
+}
+
+func (m *mtlsMetrics) fsNotifyEvent(eventPath string, op fsnotify.Op) {
+       if op != fsnotify.Create && op != fsnotify.Close {
+               return
+       }
+
+       dir := path.Dir(eventPath)
+       cert, ok := m.certs[eventPath]
+       if !ok {
+               cert, ok = m.certs[dir]
+       }
+       if ok {
+               m.log.V(1).Infof("certificate %s was just rotated", cert.Name())
+               m.rotationsObserved.WithLabelValues(mbclient.KV{"cert_name": cert.Name()}).Add(1)
+       } else {
+               m.log.V(1).Warnf("observed rotation of an unaccounted cert at path: %s", eventPath)
+       }
+}
+
+func (m *mtlsMetrics) discoverMtlsCertificates() []string {
+       certDirs := make([]string, 0)
+
+       for _, dir := range mtls.RootPaths() {
+               if entries, err := os.ReadDir(dir); err == nil {
+                       for _, mtlsEntry := range entries {
+                               if !mtlsEntry.IsDir() {
+                                       continue
+                               }
+
+                               leafPath := path.Join(dir, mtlsEntry.Name(), "cert.pem")
+                               leafStat, err := os.Stat(leafPath)
+                               if err != nil {
+                                       continue
+                               }
+                               if !leafStat.Mode().IsRegular() {
+                                       continue
+                               }
+
+                               certDirs = append(certDirs, path.Join(dir, mtlsEntry.Name()))
+                       }
+               }
+       }
+
+       return certDirs
+}
+
+func (c *mtlsMetricCertificate) LeafPath() string {
+       return path.Join(c.baseDir, "cert.pem")
+}
+
+func (c *mtlsMetricCertificate) Leaf() *x509.Certificate {
+       leafPath := c.LeafPath()
+       certsInLeaf, err := certutil.LoadCertificatesFromPEM(leafPath)
+       if err != nil {
+               return nil
+       }
+
+       if len(certsInLeaf) != 1 {
+               return nil
+       }
+
+       return certsInLeaf[0]
+}
+
+func (c mtlsMetricCertificate) Name() string {
+       return path.Base(c.baseDir)
+}
+
+func (c *mtlsMetricCertificate) TimeUntilExpiration() time.Duration {
+       cert := c.Leaf()
+       if cert == nil {
+               return 0
+       }
+
+       now := time.Now()
+
+       if now.After(cert.NotAfter) {
+               return 0
+       }
+
+       return cert.NotAfter.Sub(now)
+}

diff --git a/mtls/mtls_exporter/main.go b/mtls/mtls_exporter/main.go
new file mode 100644 (file)
index 0000000..636dbad
--- /dev/null
+++ b/mtls/mtls_exporter/main.go
@@ -0,0 +1,36 @@
+package main
+
+import (
+       "context"
+       "flag"
+       "os/signal"
+       "syscall"
+       "time"
+
+       "github.com/coreos/go-systemd/daemon"
+       "go.fuhry.dev/runtime/metrics/mtls"
+)
+
+func main() {
+       ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+       flag.Parse()
+
+       svc := mtls.NewMtlsMetricsService(ctx)
+
+       ticker := time.NewTicker(10 * time.Second)
+
+       defer svc.FlushAndWait()
+
+       svc.Tick()
+
+       daemon.SdNotify(false, daemon.SdNotifyReady)
+
+       for {
+               select {
+               case <-ticker.C:
+                       svc.Tick()
+               case <-ctx.Done():
+                       return
+               }
+       }
+}

diff --git a/mtls/mtls_exporter/mtls_exporter b/mtls/mtls_exporter/mtls_exporter
new file mode 100755 (executable)
index 0000000..eceedde
Binary files /dev/null and b/mtls/mtls_exporter/mtls_exporter differ

diff --git a/mtls/mtls_exporter/systemd/mtls-exporter.service b/mtls/mtls_exporter/systemd/mtls-exporter.service
new file mode 100644 (file)
index 0000000..36a20d0
--- /dev/null
+++ b/mtls/mtls_exporter/systemd/mtls-exporter.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Export mTLS certificate status
+Requires=metric-collector.service
+
+[Service]
+Type=notify
+User=daemon
+ExecStart=/usr/bin/mtls-exporter
+
+[Install]
+WantedBy=default.target

diff --git a/mtls/provider_file.go b/mtls/provider_file.go
index fa4b6383700fad2b8eb5eed0bbcb8b1d4df7d950..6d5e5911f4a871c9c6aa6698c818346cf59bbd5d 100644 (file)
--- a/mtls/provider_file.go
+++ b/mtls/provider_file.go
@@ -479,6 +479,14 @@ func appendMtlsCertificateDir(path string) error {
        return nil
 }
 
+// RootPaths returns the list of directories that are searched for mTLS certificates.
+func RootPaths() []string {
+       rootPathsCopy := make([]string, len(mtlsRootPaths))
+       copy(rootPathsCopy, mtlsRootPaths)
+
+       return rootPathsCopy
+}
+
 func init() {
        defaultRootCAFile = fmt.Sprintf("%s/rootca.pem", defaultMtlsRootPath)
        defaultIntermediateCAFile = fmt.Sprintf("%s/ca.pem", defaultMtlsRootPath)