From 4bd2c41976d8d98b94fca40717ef8821c2bf2895 Mon Sep 17 00:00:00 2001 From: Alejandro Angulo Date: Sun, 10 Mar 2024 14:28:44 -0700 Subject: [PATCH] Added private s3 binary cache --- modules/nixos/services/hydra/default.nix | 42 +++++++++ modules/nixos/services/minio/default.nix | 97 ++++++++++++++++++++ modules/nixos/suites/development/default.nix | 2 + secrets/hydra-aws-creds.age | 7 ++ secrets/secrets.nix | 1 + systems/x86_64-linux/gospel/default.nix | 3 + systems/x86_64-linux/node/default.nix | 5 + 7 files changed, 157 insertions(+) create mode 100644 modules/nixos/services/minio/default.nix create mode 100644 secrets/hydra-aws-creds.age diff --git a/modules/nixos/services/hydra/default.nix b/modules/nixos/services/hydra/default.nix index 2543900..afc0b59 100644 --- a/modules/nixos/services/hydra/default.nix +++ b/modules/nixos/services/hydra/default.nix @@ -25,9 +25,44 @@ in { certificate. ''; }; + + secretKeyPath = mkOption { + type = str; + description = '' + The secret key used to sign builds uploaded to s3. + ''; + }; + + s3Bucket = mkOption { + type = str; + description = '' + The s3 bucket name where build artifacts will be uploaded. + ''; + }; + + s3Scheme = mkOption { + type = str; + default = "https"; + description = '' + The scheme to use when connecting to s3. + ''; + }; + + s3Endpoint = mkOption { + type = str; + description = '' + The s3 server endpoint. + + Should use `amazonaws.com` if using amazon AWS. + ''; + }; }; config = mkIf cfg.enable { + age.secrets = { + hydra-aws-creds.file = ../../../../secrets/hydra-aws-creds.age; + }; + # NOTE: Need to create user to allow web configuration # sudo -u hydra hydra-create-user alice \ # --full-name 'Alice Q. User' \ @@ -41,6 +76,13 @@ in { notificationSender = "hydra@localhost"; buildMachinesFiles = []; useSubstitutes = true; + extraConfig = '' + store_uri = s3://${cfg.s3Bucket}?compression=zstd¶llel-compression=true&write-nar-listing=1&ls-compression=br&log-compression=br&scheme=${cfg.s3Scheme}&endpoint=${cfg.s3Endpoint}&secret-key=${cfg.secretKeyPath} + ''; + }; + + systemd.services."hydra-queue-runner".serviceConfig = { + EnvironmentFile = config.age.secrets.hydra-aws-creds.path; }; services.nginx = { diff --git a/modules/nixos/services/minio/default.nix b/modules/nixos/services/minio/default.nix new file mode 100644 index 0000000..c203115 --- /dev/null +++ b/modules/nixos/services/minio/default.nix @@ -0,0 +1,97 @@ +{ + options, + config, + lib, + pkgs, + ... +}: +with lib; let + cfg = config.aa.services.minio; + minio_cfg = config.services.minio; +in { + options.aa.services.minio = with types; { + enable = mkEnableOption "minio"; + acmeCertName = mkOption { + type = str; + default = ""; + description = '' + If set to a non-empty string, forces SSL with the supplied acme + certificate. + ''; + }; + }; + + config = mkIf cfg.enable { + services.minio = { + enable = true; + }; + + systemd.services.minio.environment = { + MINIO_SERVER_URL = "https://minio.kilonull.com"; + MINIO_BROWSER_REDIRECT_URL = "https://minio.kilonull.com/ui"; + }; + + services.nginx = { + enable = true; + virtualHosts = { + "minio.kilonull.com" = + { + extraConfig = '' + # Allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # Disable buffering + proxy_buffering off; + proxy_request_buffering off; + ''; + + locations."/".extraConfig = '' + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://localhost:9000; + ''; + locations."/ui".extraConfig = '' + rewrite ^/ui/(.*) /$1 break; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; + + proxy_connect_timeout 300; + + # To support websockets in MinIO versions released after January 2023 + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + # Some environments may encounter CORS errors (Kubernetes + Nginx Ingress) + # Uncomment the following line to set the Origin request to an empty string + proxy_set_header Origin ""; + + chunked_transfer_encoding off; + + proxy_pass http://localhost:9001; + ''; + } + // lib.optionalAttrs (cfg.acmeCertName != "") { + forceSSL = true; + useACMEHost = cfg.acmeCertName; + }; + }; + }; + }; +} diff --git a/modules/nixos/suites/development/default.nix b/modules/nixos/suites/development/default.nix index 8cbfd4c..4d0cb0b 100644 --- a/modules/nixos/suites/development/default.nix +++ b/modules/nixos/suites/development/default.nix @@ -30,6 +30,8 @@ in { environment.systemPackages = with pkgs; [ pre-commit + minio-client + awscli2 ]; }; } diff --git a/secrets/hydra-aws-creds.age b/secrets/hydra-aws-creds.age new file mode 100644 index 0000000..07a1e6e --- /dev/null +++ b/secrets/hydra-aws-creds.age @@ -0,0 +1,7 @@ +age-encryption.org/v1 +-> piv-p256 UIEGzg AhvdNL4Di3uS5eK+hOQ9C09IQmhmWn1pD4fPRdnBi9+g +4ylyFsajkX9IEzEDYGYLu31g0VKd+BwembvEdElJnJU +-> ssh-ed25519 SYNSNQ yA6rQ73vj2N/GG7wtAkFeLwLLQGRct4trP6UWuw9oGk +pp2LGZiog2IfEFSRLd0ks21MkekbVvQkHct82Ie7MGk +--- A1jbB6cxmv8/JMFvCGRMD9VzK09N+w4PxAcz6r6dRd8 +GK<͋o> ޟ\ur(6B~ZqTo}@|W,YK`pTVpmlG`E]p'E)<<}LCq=BAswF^0 \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index c7188ad..f144354 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -15,4 +15,5 @@ in { "teslamate_db.age".publicKeys = [users.me machines.node machines.gospel]; "teslamate_mqtt.age".publicKeys = [users.me machines.pi4 machines.node machines.gospel]; "teslamate_encryption.age".publicKeys = [users.me machines.node machines.gospel]; + "hydra-aws-creds.age".publicKeys = [users.me machines.gospel]; } diff --git a/systems/x86_64-linux/gospel/default.nix b/systems/x86_64-linux/gospel/default.nix index 42042ef..d2f0c46 100644 --- a/systems/x86_64-linux/gospel/default.nix +++ b/systems/x86_64-linux/gospel/default.nix @@ -58,6 +58,9 @@ services.hydra = { enable = true; acmeCertName = "kilonull.com"; + secretKeyPath = "/var/gospelCache"; + s3Bucket = "nix-store"; + s3Endpoint = "minio.kilonull.com"; }; hardware.audio.enable = true; diff --git a/systems/x86_64-linux/node/default.nix b/systems/x86_64-linux/node/default.nix index 6ce3113..da70462 100644 --- a/systems/x86_64-linux/node/default.nix +++ b/systems/x86_64-linux/node/default.nix @@ -69,6 +69,11 @@ acmeCertName = "kilonull.com"; }; + services.minio = { + enable = true; + acmeCertName = "kilonull.com"; + }; + security.acme = { enable = true; domainName = "kilonull.com";