treewide: impermanence configuration

hosts/artemis/default.nix

@@ -77,6 +77,9 @@ in
     sessionVariables.WINE_BIN = getExe pkgs.wine;
   };
 
+  environment.persistence."/persist".enable = mkForce true;
+  environment.persistence."/state".enable = mkForce true;
+
   modules = {
     services = {
       borgmatic = {

hosts/artemis/disko-config.nix

@@ -35,80 +35,59 @@
           ashift = "12";
         };
         rootFsOptions = {
-          canmount = "off";
+          compression = "zstd";
-          mountpoint = "none";
+          acltype = "posix";
-          dnodesize = "auto";
+          atime = "off";
           xattr = "sa";
+          dnodesize = "auto";
+          mountpoint = "none";
+          canmount = "off";
+          devices = "off";
+          exec = "off";
+          setuid = "off";
         };
-        postCreateHook = "zfs snapshot rpool@blank";
         datasets = {
-          local = {
+          "local" = {
             type = "zfs_fs";
+          };
+          "local/root" = {
+            type = "zfs_fs";
+            mountpoint = "/";
             options = {
-              mountpoint = "none";
+              canmount = "noauto";
+              mountpoint = "/";
+              exec = "on";
+              setuid = "on";
             };
+            postCreateHook = "zfs snapshot rpool/local/root@blank";
           };
           "local/nix" = {
             type = "zfs_fs";
             mountpoint = "/nix";
             options = {
-              atime = "off";
+              canmount = "noauto";
-              mountpoint = "legacy";
+              mountpoint = "/nix";
+              exec = "on";
+              setuid = "on";
             };
           };
-          "local/tmp" = {
+          "local/state" = {
             type = "zfs_fs";
-            mountpoint = "/tmp";
+            mountpoint = "/state";
             options = {
-              setuid = "off";
+              canmount = "noauto";
-              devices = "off";
+              mountpoint = "/state";
-              mountpoint = "legacy";
             };
           };
-          system = {
+          "safe" = {
             type = "zfs_fs";
-            mountpoint = "/";
-            options = {
-              mountpoint = "legacy";
-            };
           };
-          "system/var" = {
+          "safe/persist" = {
             type = "zfs_fs";
-            mountpoint = "/var";
+            mountpoint = "/persist";
             options = {
-              mountpoint = "legacy";
+              canmount = "noauto";
-            };
+              mountpoint = "/persist";
-          };
-          "system/var/tmp" = {
-            type = "zfs_fs";
-            mountpoint = "/var/tmp";
-            options = {
-              devices = "off";
-              mountpoint = "legacy";
-            };
-          };
-          "system/var/log" = {
-            type = "zfs_fs";
-            mountpoint = "/var/log";
-            options = {
-              compression = "on";
-              acltype = "posix";
-              mountpoint = "legacy";
-            };
-          };
-          user = {
-            type = "zfs_fs";
-            options = {
-              mountpoint = "none";
-            };
-          };
-          "user/home" = {
-            type = "zfs_fs";
-            mountpoint = "/home";
-            options = {
-              setuid = "off";
-              devices = "off";
-              mountpoint = "legacy";
             };
           };
         };

hosts/common.nix

@@ -4,12 +4,12 @@
   pkgs,
   ...
 }:
-
 {
   imports = [
     inputs.agenix.nixosModules.age
     inputs.home-manager.nixosModules.home-manager
     ../modules/nixos
+    ../modules/nixos/impermanence.nix
   ];
 
   nixpkgs = {

hosts/library/ai.nix

@@ -36,4 +36,11 @@
   modules.services.borgmatic.directories = [
     "/var/lib/private/open-webui"
   ];
+
+  environment.persistence."/persist".directories = [
+    {
+      directory = "/var/lib/private/open-webui";
+      mode = "0700";
+    }
+  ];
 }

hosts/library/grafana.nix

@@ -1,4 +1,5 @@
 {
+  config,
   ...
 }:
 
@@ -13,4 +14,13 @@
       };
     };
   };
+
+  environment.persistence."/persist".directories = [
+    {
+      directory = config.services.grafana.dataDir;
+      user = "grafana";
+      group = "grafana";
+      mode = "0700";
+    }
+  ];
 }

hosts/library/jellyfin.nix

@@ -24,6 +24,22 @@
     dataDir = "/var/lib/jellyfin";
   };
 
+  environment.persistence."/state".directories = [
+    {
+      directory = config.services.jellyfin.cacheDir;
+      inherit (config.services.jellyfin) user group;
+      mode = "0700";
+    }
+  ];
+
+  environment.persistence."/persist".directories = [
+    {
+      directory = config.services.jellyfin.dataDir;
+      inherit (config.services.jellyfin) user group;
+      mode = "0700";
+    }
+  ];
+
   modules.services.borgmatic.directories = [
     config.services.jellyfin.dataDir
   ];

hosts/library/jellysearch.nix

@@ -55,4 +55,8 @@
       MEILI_URL = "http://localhost:${toString config.services.meilisearch.listenPort}";
     };
   };
+
+  environment.persistence."/state".directories = [
+    config.systemd.services.jellysearch.serviceConfig.WorkingDirectory
+  ];
 }

hosts/library/prometheus.nix

@@ -32,4 +32,13 @@
       }
     ];
   };
+
+  environment.persistence."/state".directories = [
+    {
+      directory = "/var/lib/${config.services.prometheus.stateDir}";
+      user = "prometheus";
+      group = "prometheus";
+      mode = "0700";
+    }
+  ];
 }

hosts/mail/mail.nix

@@ -85,4 +85,52 @@ in
     smtp_destination_concurrency_limit = "20";
     header_size_limit = "4096000";
   };
+
+  environment.persistence."/persist".directories = [
+    {
+      directory = "/var/dkim";
+      user = "rspamd";
+      group = "rspamd";
+      mode = "0755";
+    }
+    {
+      directory = "/var/sieve";
+      user = "virtualMail";
+      group = "virtualMail";
+      mode = "0770";
+    }
+    {
+      directory = "/var/vmail";
+      user = "virtualMail";
+      group = "virtualMail";
+      mode = "0700";
+    }
+    {
+      directory = "/var/lib/rspamd";
+      user = "rspamd";
+      group = "rspamd";
+      mode = "0700";
+    }
+    {
+      directory = "/var/lib/redis-rspamd";
+      user = "redis-rspamd";
+      group = "redis-rspamd";
+      mode = "0700";
+    }
+    {
+      directory = "/var/lib/opendkim";
+      user = 221;
+      group = 221;
+      mode = "0700";
+    }
+    {
+      directory = "/var/lib/knot-resolver";
+      user = "knot-resolver";
+      group = "knot-resolver";
+      mode = "0770";
+    }
+    "/var/lib/dhparams"
+    "/var/lib/dovecot"
+    "/var/lib/postfix"
+  ];
 }

hosts/pi/home-assistant/default.nix

@@ -276,6 +276,15 @@
     lovelaceConfigWritable = true;
   };
 
+  environment.persistence."/persist".directories = [
+    {
+      directory = config.services.home-assistant.configDir;
+      user = "hass";
+      group = "hass";
+      mode = "0700";
+    }
+  ];
+
   modules.services.borgmatic.directories = [
     config.services.home-assistant.configDir
   ];

hosts/pi/home-assistant/mqtt.nix

@@ -69,6 +69,21 @@
     };
   };
 
+  environment.persistence."/persist".directories = [
+    {
+      directory = config.services.zigbee2mqtt.dataDir;
+      user = "zigbee2mqtt";
+      group = "zigbee2mqtt";
+      mode = "0700";
+    }
+    {
+      directory = config.services.mosquitto.dataDir;
+      user = "mosquitto";
+      group = "mosquitto";
+      mode = "0700";
+    }
+  ];
+
   modules.services.borgmatic.directories = [
     config.services.mosquitto.dataDir
     config.services.zigbee2mqtt.dataDir

hosts/server.nix

@@ -65,6 +65,13 @@ in
     ];
   };
 
+  environment.persistence."/state".directories = [
+    {
+      directory = "/var/lib/fail2ban";
+      mode = "0750";
+    }
+  ];
+
   services.openssh.settings.PermitRootLogin = mkForce "prohibit-password";
 
   modules.services.tailscale = {

hosts/skycam/default.nix

@@ -79,6 +79,10 @@
     };
   };
 
+  environment.persistence."/persist".directories = [
+    "/var/lib/skycam-archiver"
+  ];
+
   modules.services.borgmatic = {
     enable = true;
     directories = [

hosts/vps1/gitea.nix

@@ -86,4 +86,12 @@ in
       packages.CHUNKED_UPLOAD_PATH = lib.mkForce "${stateDir}/data/tmp/package-upload";
     };
   };
+
+  environment.persistence."/persist".directories = [
+    {
+      directory = config.services.gitea.stateDir;
+      inherit (config.services.gitea) user group;
+      mode = "0700";
+    }
+  ];
 }

hosts/vps1/headscale.nix

@@ -48,6 +48,13 @@ in
     };
   };
 
+  environment.persistence."/persist".directories = [
+    {
+      directory = "/var/lib/headscale";
+      inherit (config.services.headscale) user group;
+    }
+  ];
+
   services.nginx.virtualHosts = {
     "${domain}" = {
       forceSSL = true;

hosts/vps1/kanidm.nix

@@ -49,4 +49,13 @@ in
     postRun = "systemctl restart kanidm.service";
     group = "acme";
   };
+
+  environment.persistence."/persist".directories = [
+    {
+      directory = "/var/lib/kanidm";
+      user = "kanidm";
+      group = "kanidm";
+      mode = "0700";
+    }
+  ];
 }

hosts/vps1/matrix.nix

@@ -216,4 +216,23 @@ in
     }
     // commonBridgeSettings "mautrix-whatsapp";
   };
+
+  environment.persistence."/persist".directories = [
+    {
+      directory = config.services.matrix-synapse.dataDir;
+      user = "matrix-synapse";
+      group = "matrix-synapse";
+      mode = "0700";
+    }
+    {
+      directory = "/var/lib/mautrix-signal";
+      user = "mautrix-signal";
+      group = "mautrix-signal";
+    }
+    {
+      directory = "/var/lib/mautrix-whatsapp";
+      user = "mautrix-whatsapp";
+      group = "mautrix-whatsapp";
+    }
+  ];
 }

hosts/vps1/photoprism.nix

@@ -32,6 +32,14 @@ in
     file = "${inputs.secrets}/passwords/services/photoprism/admin.age";
   };
 
+  environment.persistence."/persist".directories = [
+    {
+      directory = "/var/lib/private/photoprism";
+      user = "photoprism";
+      group = "photoprism";
+    }
+  ];
+
   services.photoprism = {
     enable = true;
     address = "localhost";

modules/nixos/impermanence.nix

@@ -0,0 +1,144 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  inherit (lib)
+    attrNames
+    flip
+    isAttrs
+    mapAttrs
+    mkIf
+    mkMerge
+    mkOption
+    optionals
+    types
+    ;
+in
+{
+  boot.zfs.forceImportRoot = false;
+  boot.initrd.systemd.enable = true;
+  boot.initrd.systemd.services.impermanence-rollback =
+    mkIf
+      (config.environment.persistence."/persist".enable || config.environment.persistence."/state".enable)
+      {
+        description = "Rollback root filesystem";
+        wantedBy = [ "initrd.target" ];
+        after = [ "zfs-import-rpool.service" ];
+        before = [ "sysroot.mount" ];
+        unitConfig.DefaultDependencies = "no";
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = "${pkgs.zfs}/bin/zfs rollback -r rpool/local/root@blank";
+        };
+      };
+
+  age.identityPaths = [ "/persist/etc/ssh/ssh_host_ed25519_key" ];
+
+  fileSystems."/state" = mkIf config.environment.persistence."/state".enable {
+    neededForBoot = true;
+  };
+  environment.persistence."/state" = {
+    enable = false;
+    hideMounts = true;
+    directories = [
+      "/var/lib/systemd"
+      "/var/log"
+      "/var/spool"
+    ];
+  };
+
+  fileSystems."/persist" = mkIf config.environment.persistence."/persist".enable {
+    neededForBoot = true;
+  };
+  environment.persistence."/persist" = {
+    enable = false;
+    hideMounts = true;
+    files = [
+      (mkIf (!config.boot.isContainer) "/etc/machine-id")
+      "/etc/adjtime"
+      "/etc/ssh/ssh_host_ed25519_key"
+      "/etc/ssh/ssh_host_ed25519_key.pub"
+    ];
+    directories = [
+      "/var/lib/nixos"
+    ]
+    ++ optionals config.security.acme.acceptTerms [
+      {
+        directory = "/var/lib/acme";
+        user = "acme";
+        group = "acme";
+        mode = "0755";
+      }
+    ]
+    ++ optionals config.services.printing.enable [
+      {
+        directory = "/var/lib/cups";
+        mode = "0700";
+      }
+    ]
+    ++ optionals config.hardware.bluetooth.enable [
+      "/var/lib/bluetooth"
+    ];
+  };
+
+  users.mutableUsers = !config.environment.persistence."/persist".enable;
+
+  # For each user that has a home-manager config, merge the locally defined
+  # persistence options that we defined above.
+  imports =
+    let
+      mkUserFiles = map (
+        x: { parentDirectory.mode = "700"; } // (if isAttrs x then x else { file = x; })
+      );
+      mkUserDirs = map (x: { mode = "700"; } // (if isAttrs x then x else { directory = x; }));
+    in
+    [
+      {
+        environment.persistence = mkMerge (
+          flip map (attrNames config.home-manager.users) (
+            user:
+            let
+              hmUserCfg = config.home-manager.users.${user};
+            in
+            flip mapAttrs hmUserCfg.home.persistence (
+              _: sourceCfg: {
+                users.${user} = {
+                  files = mkUserFiles sourceCfg.files;
+                  directories = mkUserDirs sourceCfg.directories;
+                };
+              }
+            )
+          )
+        );
+      }
+    ];
+
+  home-manager.sharedModules = [
+    {
+      options.home.persistence = mkOption {
+        description = "Additional persistence config for the given source path";
+        default = { };
+        type = types.attrsOf (
+          types.submodule {
+            options = {
+              files = mkOption {
+                description = "Additional files to persist via NixOS impermanence.";
+                type = types.listOf (types.either types.attrs types.str);
+                default = [ ];
+              };
+
+              directories = mkOption {
+                description = "Additional directories to persist via NixOS impermanence.";
+                type = types.listOf (types.either types.attrs types.str);
+                default = [ ];
+              };
+            };
+          }
+        );
+      };
+    }
+  ];
+}

modules/nixos/podman.nix

@@ -40,6 +40,10 @@ in
 
     };
 
+    environment.persistence."/persist".directories = [
+      "/var/lib/containers/storage"
+    ];
+
     networking.firewall.interfaces."podman+" = {
       allowedUDPPorts = [ 53 ];
       allowedTCPPorts = [ 53 ];

modules/nixos/services/postgresql.nix

@@ -30,6 +30,15 @@ in
       };
     };
 
+    environment.persistence."/persist".directories = [
+      {
+        directory = "/var/lib/postgresql";
+        user = "postgres";
+        group = "postgres";
+        mode = "0700";
+      }
+    ];
+
     services.borgmatic.settings = {
       postgresql_databases = [
         {

modules/nixos/services/tailscale.nix

@@ -56,5 +56,9 @@ in
       trustedInterfaces = [ "tailscale0" ];
       allowedUDPPorts = [ config.services.tailscale.port ];
     };
+
+    environment.persistence."/state".directories = [
+      "/var/lib/tailscale"
+    ];
   };
 }

modules/nixos/system/desktop/gnome.nix

@@ -70,5 +70,11 @@ in
       gnomeExtensions.worksets
       gnomeExtensions.workspace-matrix
     ];
+
+    environment.persistence."/persist".directories = [
+      "/etc/NetworkManager"
+      "/var/lib/AccountsService"
+      "/var/lib/NetworkManager"
+    ];
   };
 }

nix/hosts.nix

@@ -31,6 +31,7 @@
             specialArgs = { inherit inputs; };
 
             modules = [
+              inputs.impermanence.nixosModules.impermanence
               {
                 networking = {
                   inherit domain;

users/jordan/common/gpg.nix

@@ -11,4 +11,8 @@
     enable = true;
     enableSshSupport = true;
   };
+
+  home.persistence."/persist".directories = [
+    ".gnupg"
+  ];
 }

users/jordan/common/neovim.nix

@@ -130,4 +130,10 @@
   };
 
   home.sessionVariables.EDITOR = "nvim";
+
+  home.persistence."/state".directories = [
+    ".local/share/nvim"
+    ".local/state/nvim"
+    ".cache/nvim"
+  ];
 }

users/jordan/common/optional/graphical/firefox.nix

@@ -207,4 +207,12 @@
       };
     };
   };
+
+  home.persistence."/state".directories = [
+    ".cache/mozilla"
+  ];
+
+  home.persistence."/persist".directories = [
+    ".mozilla"
+  ];
 }

users/jordan/common/optional/graphical/thunderbird.nix

@@ -24,4 +24,12 @@
       };
     };
   };
+
+  home.persistence."/state".directories = [
+    ".cache/thunderbird"
+  ];
+
+  home.persistence."/persist".directories = [
+    ".thunderbird"
+  ];
 }

users/jordan/common/pass.nix

@@ -8,4 +8,8 @@
     enable = true;
     package = pkgs.pass.withExtensions (exts: [ exts.pass-otp ]);
   };
+
+  home.persistence."/state".directories = [
+    ".local/share/password-store"
+  ];
 }

users/jordan/common/shell.nix

@@ -176,6 +176,15 @@ in
     nix-index.enable = true;
   };
 
+  home.persistence."/persist" = {
+    directories = [
+      ".local/share/mcfly"
+    ];
+    files = [
+      ".zsh_history"
+    ];
+  };
+
   home.packages = with pkgs; [
     bat
     btop

users/jordan/common/ssh.nix

@@ -9,4 +9,8 @@
     enable = true;
     addKeysToAgent = "yes";
   };
+
+  home.persistence."/state".files = [
+    ".ssh/known_hosts"
+  ];
 }

users/jordan/default.nix

@@ -15,6 +15,7 @@ in
 {
   age.secrets."passwords/users/jordan".file = "${inputs.secrets}/passwords/users/jordan.age";
 
+  users.users.root.hashedPasswordFile = config.age.secrets."passwords/users/jordan".path;
   users.users.${name} = {
     description = "Jordan Holt";
     extraGroups = [
@@ -42,6 +43,24 @@ in
       ./common/pass.nix
       ./common/shell.nix
       ./common/ssh.nix
+      {
+        home.persistence."/state" = {
+          directories = [
+            "Downloads"
+            ".local/state/wireplumber"
+          ];
+        };
+        home.persistence."/persist" = {
+          directories = [
+            "Desktop"
+            "Documents"
+            "Music"
+            "Pictures"
+            "projects"
+            "Videos"
+          ];
+        };
+      }
     ]
     ++ optional (builtins.pathExists hostFile) hostFile;