Merge pull request #501 from NixOS/mimas

build: init mimas

build/common.nix

@@ -71,6 +71,7 @@ with lib;
   networking.firewall.rejectPackets = true;
   networking.firewall.allowPing = true;
   networking.firewall.allowedTCPPorts = [ 10050 ];
+  networking.firewall.logRefusedConnections = false;
 
   services.resolved = {
     enable = true;
@@ -114,4 +115,6 @@ with lib;
 
   security.acme.acceptTerms = true;
   security.acme.defaults.email = "[email protected]";
+
+  services.zfs.autoScrub.enable = true;
 }

build/flake.nix

@@ -88,6 +88,15 @@
         ];
       };
 
+      nixosConfigurations.mimas = nixpkgs.lib.nixosSystem {
+        system = "x86_64-linux";
+
+        modules = [
+          flakesModule
+          ./mimas
+        ];
+      };
+
       colmena =
         {
           meta = {

build/haumea/postgresql.nix

@@ -89,8 +89,7 @@
 
     # FIXME: don't use 'trust'.
     authentication = ''
-      host hydra all 10.254.1.3/32 trust
-      host hydra all 10.254.1.5/32 trust
+      host hydra all 10.254.1.1/32 trust
       local all root peer map=prometheus
     '';
 

build/hydra-proxy.nix

@@ -1,6 +1,11 @@
 { config, ... }:
 
 {
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+
   services.nginx = {
     enable = true;
     enableReload = true;

build/hydra.nix

@@ -7,8 +7,57 @@ let
 in
 
 {
+  networking.firewall.allowedTCPPorts = [
+    9198 # queue-runnner metrics
+    9199 # hydra-notify metrics
+  ];
+
+  # garbage collection
+  nix.gc = {
+    automatic = true;
+    options = ''--max-freed "$((400 * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"'';
+    dates = "03,09,15,21:15";
+  };
+
+  # gc outputs as well, since they are served from the cache
+  nix.settings.gc-keep-outputs = false;
+
+  systemd.services.hydra-prune-build-logs = {
+    description = "Clean up old build logs";
+    startAt = "weekly";
+    serviceConfig = {
+      User = "hydra-queue-runner";
+      Group = "hydra";
+      ExecStart = lib.concatStringsSep " " [
+        (lib.getExe pkgs.findutils)
+        "/var/lib/hydra/build-logs/"
+        "-type"
+        "f"
+        "-mtime"
+        "+${toString (3 * 365)}"
+        "-delete"
+      ];
+    };
+  };
+
+  # Don't rate-limit the journal.
+  services.journald.rateLimitBurst = 0;
+
+  systemd.services.hydra-queue-runner = {
+    serviceConfig.ManagedOOMPreference = "avoid";
+  };
+
+  age.secrets.hydra-aws-credentials = {
+    file = ./secrets/hydra-aws-credentials.age;
+    path = "/var/lib/hydra/queue-runner/.aws/credentials";
+    owner = "hydra-queue-runner";
+    group = "hydra";
+  };
+
   services.hydra-dev.enable = true;
   services.hydra-dev.package = pkgs.hydra;
+  services.hydra-dev.buildMachinesFiles = [ "/etc/nix/machines" ];
+  services.hydra-dev.dbi = "dbi:Pg:dbname=hydra;host=10.254.1.9;user=hydra;";
   services.hydra-dev.logo = ./hydra-logo.png;
   services.hydra-dev.hydraURL = "https://hydra.nixos.org";
   services.hydra-dev.notificationSender = "[email protected]";
@@ -44,6 +93,9 @@ in
 
     max_concurrent_evals = 1
 
+    # increase the number of active compress slots (CPU is 48*2 on mimas)
+    max_local_worker_threads = 144
+
     max_unsupported_time = 86400
 
     allow_import_from_derivation = false

build/mimas/boot.nix

@@ -0,0 +1,29 @@
+{
+  boot = {
+    initrd.availableKernelModules = [
+      "ahci"
+      "xhci_pci"
+      "nvme"
+      "usbhid"
+    ];
+    supportedFilesystems.zfs = true;
+    loader = {
+      efi.canTouchEfiVariables = false;
+      grub = {
+        enable = true;
+        efiSupport = true;
+        efiInstallAsRemovable = true;
+        mirroredBoots = [
+          {
+            devices = [ "nodev" ];
+            path = "/efi/a";
+          }
+          {
+            devices = [ "nodev" ];
+            path = "/efi/b";
+          }
+        ];
+      };
+    };
+  };
+}

build/mimas/default.nix

@@ -0,0 +1,43 @@
+{
+  pkgs,
+  ...
+}:
+
+{
+  imports = [
+    ../common.nix
+    ../hydra.nix
+    ../hydra-proxy.nix
+    ../hydra-scaler.nix
+    ../packet-importer.nix
+    ./boot.nix
+    ./network.nix
+  ];
+
+  disko.devices = import ./disko.nix;
+
+  networking = {
+    hostName = "mimas";
+    domain = "nixos.org";
+    hostId = "aba92093";
+  };
+
+  zramSwap = {
+    enable = true;
+    memoryPercent = 50;
+  };
+
+  # garbage collection
+  nix.gc = {
+    automatic = true;
+    options = ''--max-freed "$((400 * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"'';
+    dates = "03,09,15,21:15";
+  };
+
+  # gc outputs as well, since they are served from the cache
+  nix.settings.gc-keep-outputs = false;
+
+  nixpkgs.hostPlatform = "x86_64-linux";
+
+  system.stateVersion = "24.11";
+}

build/mimas/disko.nix

@@ -0,0 +1,81 @@
+let
+  layout = id: {
+    type = "gpt";
+    partitions = {
+      esp = {
+        type = "EF00";
+        size = "512M";
+        content = {
+          type = "filesystem";
+          format = "vfat";
+          mountpoint = "/efi/${id}";
+        };
+      };
+      zfs = {
+        size = "100%";
+        content = {
+          type = "zfs";
+          pool = "zroot";
+        };
+      };
+    };
+  };
+in
+{
+  disk = {
+    nvme0n1 = {
+      type = "disk";
+      device = "/dev/disk/by-id/nvme-SAMSUNG_MZQL21T9HCJR-00A07_S64GNNFX604905";
+      content = layout "a";
+    };
+    nvme1n1 = {
+      type = "disk";
+      device = "/dev/disk/by-id/nvme-SAMSUNG_MZQL21T9HCJR-00A07_S64GNNFX604919";
+      content = layout "b";
+    };
+  };
+
+  zpool.zroot = {
+    type = "zpool";
+    mode = "mirror";
+    options.ashift = "12";
+
+    rootFsOptions = {
+      acltype = "posixacl";
+      atime = "off";
+      compression = "on";
+      mountpoint = "none";
+      xattr = "sa";
+    };
+
+    datasets = {
+      "root" = {
+        type = "zfs_fs";
+        mountpoint = "/";
+      };
+      "nix/store" = {
+        type = "zfs_fs";
+        mountpoint = "/nix";
+      };
+      "nix/db" = {
+        type = "zfs_fs";
+        mountpoint = "/nix/var/nix/db";
+      };
+      "hydra/cache" = {
+        type = "zfs_fs";
+        mountpoint = "/var/cache/hydra";
+      };
+      "hydra/state" = {
+        type = "zfs_fs";
+        mountpoint = "/var/lib/hydra";
+      };
+      "reserved" = {
+        type = "zfs_fs";
+        options = {
+          canmount = "off";
+          refreservation = "16G"; # roughly one system closure
+        };
+      };
+    };
+  };
+}

build/mimas/network.nix

@@ -0,0 +1,25 @@
+{
+  networking.useDHCP = false;
+
+  systemd.network = {
+    enable = true;
+    networks = {
+      "30-enp5s0" = {
+        matchConfig = {
+          MACAddress = "9c:6b:00:70:d1:f8";
+          Type = "ether";
+        };
+        linkConfig.RequiredForOnline = true;
+        networkConfig.Description = "WAN";
+        address = [
+          "157.90.104.34/26"
+          "2a01:4f8:2220:11c8::1/64"
+        ];
+        routes = [
+          { Gateway = "157.90.104.1"; }
+          { Gateway = "fe80::1"; }
+        ];
+      };
+    };
+  };
+}

build/pluto/prometheus/exporters/nixos.nix

@@ -5,7 +5,10 @@
       static_configs = [
         {
           labels.role = "hydra";
-          targets = [ "rhea:9300" ];
+          targets = [
+            "rhea:9300"
+            "mimas.nixos.org:9300"
+          ];
         }
         {
           labels.role = "monitoring";

build/pluto/prometheus/exporters/node.nix

@@ -8,7 +8,10 @@
         static_configs = [
           {
             labels.role = "hydra";
-            targets = [ "rhea:9100" ];
+            targets = [
+              "rhea:9100"
+              "mimas.nixos.org:9100"
+            ];
           }
           {
             labels.role = "database";

build/pluto/prometheus/exporters/zfs.nix

@@ -9,6 +9,7 @@
               "rhea:9134"
               "haumea:9134"
               "pluto:9134"
+              "mimas.nixos.org:9134"
             ];
           }
         ];

build/rhea/configuration.nix

@@ -1,55 +1,28 @@
-{ pkgs, ... }:
 {
   imports = [
     ./hardware-configuration.nix
     ./hetzner.nix
     ./network.nix
     ../common.nix
-    ../hydra.nix
-    ../hydra-proxy.nix
-    ../hydra-scaler.nix
-    ../packet-importer.nix
   ];
 
   networking = {
     hostName = "rhea";
-    firewall.allowedTCPPorts = [
-      80
-      443
-      9198 # hydra-queue-runner's prometheus
-      9199 # hydra-notify's prometheus
-    ];
     firewall.allowPing = true;
-    firewall.logRefusedConnections = false;
   };
 
+  users.users.root.openssh.authorizedKeys.keys = [
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIY0EGPGfXD1L+YdSJIKUzeFyuUfVW58kMh+mSflEFx1 root@mimas"
+  ];
+
   system.stateVersion = "21.11";
 
-  services.hydra-dev.dbi = "dbi:Pg:dbname=hydra;host=10.254.1.9;user=hydra;";
   systemd.services.hydra-init = {
     after = [ "wireguard-wg0.service" ];
     requires = [ "wireguard-wg0.service" ];
   };
-  systemd.services.hydra-queue-runner = {
-    serviceConfig.ManagedOOMPreference = "avoid";
-  };
-  services.hydra-dev.buildMachinesFiles = [ "/etc/nix/machines" ];
 
   # hydra-evaluator causes very sharp spikes in RAM usage on trunk-combined
   zramSwap.enable = true;
   zramSwap.memoryPercent = 150;
-
-  nix.gc.automatic = true;
-  nix.gc.options = ''--max-freed "$((400 * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"'';
-  nix.gc.dates = "03,09,15,21:15";
-
-  nix.extraOptions = "gc-keep-outputs = false";
-
-  #services.postfix.enable = true;
-  #services.postfix.hostname = "hydra.nixos.org";
-
-  # Don't rate-limit the journal.
-  services.journald.rateLimitBurst = 0;
-
-  services.zfs.autoScrub.enable = true;
 }