Added private s3 binary cache

modules/nixos/services/hydra/default.nix

@@ -25,9 +25,44 @@ in {
         certificate.
       '';
     };
+
+    secretKeyPath = mkOption {
+      type = str;
+      description = ''
+        The secret key used to sign builds uploaded to s3.
+      '';
+    };
+
+    s3Bucket = mkOption {
+      type = str;
+      description = ''
+        The s3 bucket name where build artifacts will be uploaded.
+      '';
+    };
+
+    s3Scheme = mkOption {
+      type = str;
+      default = "https";
+      description = ''
+        The scheme to use when connecting to s3.
+      '';
+    };
+
+    s3Endpoint = mkOption {
+      type = str;
+      description = ''
+        The s3 server endpoint.
+
+        Should use `amazonaws.com` if using amazon AWS.
+      '';
+    };
   };
 
   config = mkIf cfg.enable {
+    age.secrets = {
+      hydra-aws-creds.file = ../../../../secrets/hydra-aws-creds.age;
+    };
+
     # NOTE: Need to create user to allow web configuration
     # sudo -u hydra hydra-create-user alice \
     #   --full-name 'Alice Q. User' \
@@ -41,6 +76,13 @@ in {
       notificationSender = "hydra@localhost";
       buildMachinesFiles = [];
       useSubstitutes = true;
+      extraConfig = ''
+        store_uri = s3://${cfg.s3Bucket}?compression=zstd&parallel-compression=true&write-nar-listing=1&ls-compression=br&log-compression=br&scheme=${cfg.s3Scheme}&endpoint=${cfg.s3Endpoint}&secret-key=${cfg.secretKeyPath}
+      '';
+    };
+
+    systemd.services."hydra-queue-runner".serviceConfig = {
+      EnvironmentFile = config.age.secrets.hydra-aws-creds.path;
     };
 
     services.nginx = {

modules/nixos/services/minio/default.nix

@@ -0,0 +1,97 @@
+{
+  options,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+with lib; let
+  cfg = config.aa.services.minio;
+  minio_cfg = config.services.minio;
+in {
+  options.aa.services.minio = with types; {
+    enable = mkEnableOption "minio";
+    acmeCertName = mkOption {
+      type = str;
+      default = "";
+      description = ''
+        If set to a non-empty string, forces SSL with the supplied acme
+        certificate.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.minio = {
+      enable = true;
+    };
+
+    systemd.services.minio.environment = {
+      MINIO_SERVER_URL = "https://minio.kilonull.com";
+      MINIO_BROWSER_REDIRECT_URL = "https://minio.kilonull.com/ui";
+    };
+
+    services.nginx = {
+      enable = true;
+      virtualHosts = {
+        "minio.kilonull.com" =
+          {
+            extraConfig = ''
+              # Allow special characters in headers
+              ignore_invalid_headers off;
+              # Allow any size file to be uploaded.
+              # Set to a value such as 1000m; to restrict file size to a specific value
+              client_max_body_size 0;
+              # Disable buffering
+              proxy_buffering off;
+              proxy_request_buffering off;
+            '';
+
+            locations."/".extraConfig = ''
+              proxy_set_header Host $host;
+              proxy_set_header X-Real-IP $remote_addr;
+              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+              proxy_set_header X-Forwarded-Proto $scheme;
+
+              proxy_connect_timeout 300;
+              # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+              proxy_http_version 1.1;
+              proxy_set_header Connection "";
+              chunked_transfer_encoding off;
+
+              proxy_pass http://localhost:9000;
+            '';
+            locations."/ui".extraConfig = ''
+              rewrite ^/ui/(.*) /$1 break;
+              proxy_set_header Host $host;
+              proxy_set_header X-Real-IP $remote_addr;
+              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+              proxy_set_header X-Forwarded-Proto $scheme;
+              proxy_set_header X-NginX-Proxy true;
+
+              # This is necessary to pass the correct IP to be hashed
+              real_ip_header X-Real-IP;
+
+              proxy_connect_timeout 300;
+
+               # To support websockets in MinIO versions released after January 2023
+              proxy_http_version 1.1;
+              proxy_set_header Upgrade $http_upgrade;
+              proxy_set_header Connection "upgrade";
+              # Some environments may encounter CORS errors (Kubernetes + Nginx Ingress)
+              # Uncomment the following line to set the Origin request to an empty string
+              proxy_set_header Origin "";
+
+              chunked_transfer_encoding off;
+
+              proxy_pass http://localhost:9001;
+            '';
+          }
+          // lib.optionalAttrs (cfg.acmeCertName != "") {
+            forceSSL = true;
+            useACMEHost = cfg.acmeCertName;
+          };
+      };
+    };
+  };
+}

modules/nixos/suites/development/default.nix

@@ -30,6 +30,8 @@ in {
 
     environment.systemPackages = with pkgs; [
       pre-commit
+      minio-client
+      awscli2
     ];
   };
 }

secrets/hydra-aws-creds.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> piv-p256 UIEGzg AhvdNL4Di3uS5eK+hOQ9C09IQmhmWn1pD4fPRdnBi9+g
+4ylyFsajkX9IEzEDYGYLu31g0VKd+BwembvEdElJnJU
+-> ssh-ed25519 SYNSNQ yA6rQ73vj2N/GG7wtAkFeLwLLQGRct4trP6UWuw9oGk
+pp2LGZiog2IfEFSRLd0ks21MkekbVvQkHct82Ie7MGk
+--- A1jbB6cxmv8/JMFvCGRMD9VzK09N+w4PxAcz6r6dRd8
+žGþK<<19>Í‹ëo>ÞŸÇÉûï‚À\u‘rª(¼6B~<7E>íZ£qíô¬T£Š§¿è<C2BF>o}@|W,€Y†KÅ`†ÊpÍTVpmäl°ª¢ÎßGß`E]óp'E•)í·ð<<}¹¥L›âCÄÎqÈ=ûôþBA†swF^¤à¯ð0è

secrets/secrets.nix

@@ -15,4 +15,5 @@ in {
   "teslamate_db.age".publicKeys = [users.me machines.node machines.gospel];
   "teslamate_mqtt.age".publicKeys = [users.me machines.pi4 machines.node machines.gospel];
   "teslamate_encryption.age".publicKeys = [users.me machines.node machines.gospel];
+  "hydra-aws-creds.age".publicKeys = [users.me machines.gospel];
 }

systems/x86_64-linux/gospel/default.nix

@@ -58,6 +58,9 @@
     services.hydra = {
       enable = true;
       acmeCertName = "kilonull.com";
+      secretKeyPath = "/var/gospelCache";
+      s3Bucket = "nix-store";
+      s3Endpoint = "minio.kilonull.com";
     };
 
     hardware.audio.enable = true;

systems/x86_64-linux/node/default.nix

@@ -69,6 +69,11 @@
       acmeCertName = "kilonull.com";
     };
 
+    services.minio = {
+      enable = true;
+      acmeCertName = "kilonull.com";
+    };
+
     security.acme = {
       enable = true;
       domainName = "kilonull.com";