From e3a54876ec85702b16dc64598ca08c0dce30b60a Mon Sep 17 00:00:00 2001
From: Alejandro Angulo <iam@alejandr0angul0.dev>
Date: Thu, 7 Sep 2023 22:51:38 -0700
Subject: [PATCH] Get loki, prometheus, and promtail configured

First iteration
---
 modules/services/loki/default.nix       | 122 ++++++++++++++++++++++++
 modules/services/prometheus/default.nix |  22 ++++-
 modules/services/promtail/default.nix   |  77 +++++++++++++++
 systems/x86_64-linux/node/default.nix   |   8 ++
 4 files changed, 228 insertions(+), 1 deletion(-)
 create mode 100644 modules/services/loki/default.nix
 create mode 100644 modules/services/promtail/default.nix

diff --git a/modules/services/loki/default.nix b/modules/services/loki/default.nix
new file mode 100644
index 0000000..fc503a9
--- /dev/null
+++ b/modules/services/loki/default.nix
@@ -0,0 +1,122 @@
+{
+  options,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+with lib; let
+  cfg = config.aa.services.loki;
+  loki = config.services.loki;
+in {
+  options.aa.services.loki = with types; {
+    enable = mkEnableOption "loki";
+    acmeCertName = mkOption {
+      type = str;
+      default = "";
+      description = ''
+        If set to a non-empty string, forces SSL with the supplied acme
+        certificate.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.loki = {
+      enable = true;
+      configuration = {
+        server.http_listen_port = 3030;
+        auth_enabled = false;
+
+        ingester = {
+          lifecycler = {
+            address = "127.0.0.1";
+            ring = {
+              kvstore = {
+                store = "inmemory";
+              };
+              replication_factor = 1;
+            };
+          };
+          # Any chunk not receiving new logs in this time will be flushed
+          chunk_idle_period = "1h";
+          # All chunks will be flushed when they hit this age
+          max_chunk_age = "1h";
+          # Loki will attempt to build chunks up to this size, flushing first
+          # if chunk_idle_period or max_chunk_age is reached first
+          chunk_target_size = 999999;
+          chunk_retain_period = "30s";
+          max_transfer_retries = 0;
+        };
+
+        schema_config = {
+          configs = [
+            {
+              from = "2022-06-06";
+              store = "boltdb-shipper";
+              object_store = "filesystem";
+              schema = "v11";
+              index = {
+                prefix = "index_";
+                period = "24h";
+              };
+            }
+          ];
+        };
+
+        storage_config = {
+          boltdb_shipper = {
+            active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+            cache_location = "/var/lib/loki/boltdb-shipper-cache";
+            cache_ttl = "24h";
+            shared_store = "filesystem";
+          };
+
+          filesystem = {
+            directory = "/var/lib/loki/chunks";
+          };
+        };
+
+        limits_config = {
+          reject_old_samples = true;
+          reject_old_samples_max_age = "168h";
+        };
+
+        chunk_store_config = {
+          max_look_back_period = "0s";
+        };
+
+        table_manager = {
+          retention_deletes_enabled = false;
+          retention_period = "0s";
+        };
+
+        compactor = {
+          working_directory = "/var/lib/loki";
+          shared_store = "filesystem";
+          compactor_ring = {
+            kvstore = {
+              store = "inmemory";
+            };
+          };
+        };
+      };
+    };
+
+    services.nginx = mkIf (cfg.acmeCertName != "") {
+      enable = true;
+      # Confirm with /loki/api/v1/status/buildinfo
+      virtualHosts."loki.${cfg.acmeCertName}" = {
+        locations."/" = {
+          proxyPass = "http://localhost:${toString loki.configuration.server.http_listen_port}";
+        };
+        forceSSL = true;
+        useACMEHost = cfg.acmeCertName;
+      };
+    };
+
+    networking.firewall = {
+      allowedTCPPorts = [80 443];
+    };
+  };
+}
diff --git a/modules/services/prometheus/default.nix b/modules/services/prometheus/default.nix
index 45de0f5..db10352 100644
--- a/modules/services/prometheus/default.nix
+++ b/modules/services/prometheus/default.nix
@@ -7,6 +7,7 @@
 }:
 with lib; let
   cfg = config.aa.services.prometheus;
+  exporters = config.services.prometheus.exporters;
 in {
   options.aa.services.prometheus = with types; {
     enable = mkEnableOption "prometheus";
@@ -21,7 +22,26 @@ in {
   };
 
   config = mkIf cfg.enable {
-    services.prometheus.enable = true;
+    services.prometheus = {
+      enable = true;
+      exporters = {
+        node = {
+          enable = true;
+          enabledCollectors = ["systemd"];
+          port = 9002;
+        };
+      };
+      scrapeConfigs = [
+        {
+          job_name = "foo";
+          static_configs = [
+            {
+              targets = ["127.0.0.1:${toString exporters.node.port}"];
+            }
+          ];
+        }
+      ];
+    };
 
     services.nginx = {
       enable = true;
diff --git a/modules/services/promtail/default.nix b/modules/services/promtail/default.nix
new file mode 100644
index 0000000..81839a3
--- /dev/null
+++ b/modules/services/promtail/default.nix
@@ -0,0 +1,77 @@
+{
+  options,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+with lib; let
+  cfg = config.aa.services.promtail;
+  loki = config.services.loki;
+in {
+  options.aa.services.promtail = with types; {
+    enable = mkEnableOption "promtail";
+    acmeCertName = mkOption {
+      type = str;
+      default = "";
+      description = ''
+        If set to a non-empty string, forces SSL with the supplied acme
+        certificate.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.promtail = {
+      enable = true;
+      configuration = {
+        server = {
+          http_listen_port = 28183;
+          grpc_listen_port = 0;
+        };
+        positions = {
+          filename = "/tmp/positions.yaml";
+        };
+        clients = [
+          {
+            url = "http://127.0.0.1:${toString loki.configuration.server.http_listen_port}/loki/api/v1/push";
+          }
+        ];
+        scrape_configs = [
+          {
+            job_name = "journal";
+            journal = {
+              max_age = "12h";
+              labels = {
+                job = "systemd-journal";
+                host = "node";
+              };
+            };
+            relabel_configs = [
+              {
+                source_labels = ["__journal__systemd_unit"];
+                target_label = "unit";
+              }
+            ];
+          }
+        ];
+      };
+    };
+
+    services.nginx = mkIf (cfg.acmeCertName != "") {
+      enable = true;
+      # Confirm with /loki/api/v1/status/buildinfo
+      virtualHosts."promtail.${cfg.acmeCertName}" = {
+        locations."/" = {
+          proxyPass = "http://localhost:${toString config.services.promtail.configuration.server.http_listen_port}";
+        };
+        forceSSL = true;
+        useACMEHost = cfg.acmeCertName;
+      };
+    };
+
+    networking.firewall = {
+      allowedTCPPorts = [80 443];
+    };
+  };
+}
diff --git a/systems/x86_64-linux/node/default.nix b/systems/x86_64-linux/node/default.nix
index a8c138a..99a96cf 100644
--- a/systems/x86_64-linux/node/default.nix
+++ b/systems/x86_64-linux/node/default.nix
@@ -38,6 +38,14 @@
       enable = true;
       acmeCertName = "kilonull.com";
     };
+    services.loki = {
+      enable = true;
+      acmeCertName = "kilonull.com";
+    };
+    services.promtail = {
+      enable = true;
+      acmeCertName = "kilonull.com";
+    };
     services.syncoid = {
       #  sudo -u backups zfs create -o mountpoint=/tank/backups/gospel tank/backups/gospel
       enable = true;