configuration.nix

{ config, lib, pkgs, ... }:
let

cfg = {
  hostName = "zion";

  ceph = {
    enable = true;
    publicIP = "127.0.0.1";
    privateIP = "127.0.0.1";
    osds = [
      { id = 1; monID = "a"; uuid = "342e8fed-d2b9-4c50-aef4-0cba692c1aeb"; dev = "sda"; }
      { id = 2; monID = "b"; uuid = "e8c0eeb1-4161-4bab-ad4c-ed82325982fe"; dev = "sdb"; }
      { id = 3; monID = "c"; uuid = "519995d7-9d38-4b65-ab8b-e7162b0c27bf"; dev = "sdc"; }
      { id = 4; monID = "d"; uuid = "2537b082-d36d-4e64-aaea-c40ee04bcebe"; dev = "sdd"; }
    ];
  };
};

in

{
  system.stateVersion = "16.09";

  imports =
    [
      <nixpkgs/nixos/modules/installer/scan/not-detected.nix>
    ];

  boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" "sr_mod" "sdhci_pci" ];
  boot.kernelModules = [
    "pcspkr"
    "usb_storage"
    "tun"
    "virtio"
    "kvm" "kvm-intel"
    "ceph" "rbd"
    "btrfs"
    "nvidia" "nvidia_uvm" "nvidia_drm" "nvidia_modeset"
  ];
  boot.extraModulePackages = [ ];

  fileSystems = {
    "/" = {
      device = "/dev/disk/by-uuid/1a381f95-4614-42c9-9b04-751af97aabf7";
      # device = "/dev/disk/by-label/nixos-usb";
      fsType = "btrfs";
      options = [ "subvol=nixos" "ssd" "autodefrag" "noatime" "nospace_cache" "compress-force=zlib" ];
    };
    "/boot/efi" = {
      device = "/dev/disk/by-uuid/0FF6-D04F";
      options = [ "noatime" ];
    };

    "/tmp" = {
      device = "tmpfs";
      fsType = "tmpfs";
      options = [
        "noexec"
        "nodev"
        "nosuid"
        "size=8192m"
        "mode=1777"
      ];
    };
  };

  swapDevices = [];

  nix.maxJobs = lib.mkDefault 1;

  boot.loader.grub.enable = true;
  boot.loader.grub.version = 2;
  boot.loader.grub.efiSupport = true;
  boot.loader.grub.efiInstallAsRemovable = true;
  boot.loader.efi.efiSysMountPoint = "/boot/efi";
  boot.loader.grub.device = "nodev"; # or "nodev" for efi only
  boot.loader.timeout = 1;

  boot.kernelPackages = pkgs.linuxPackages_4_8;

  networking = {
    hostName = cfg.hostName;
    nameservers = [ "127.0.0.1" "8.8.8.8" ];

    firewall = {
      enable = true;
      allowPing = true;
      trustedInterfaces = [ "wlan0_ap" "lan0" "lan1" ];
      checkReversePath = false;
      allowedTCPPorts = [
        22    # ssh
        80    # http
        443   # https
      ];
      allowedUDPPorts = [ ];
    };

    nat = {
      enable = true;
      internalIPs = [ "10.0.3.0/24" "10.0.4.0/24" "10.0.5.0/24" ];
      externalInterface = "wan1_modem_yota";
    };

    interfaces = {
      wlan0_ap = {
        ipAddress = "10.0.3.1";
        prefixLength = 24;
        ip4 = [ { address = "10.0.3.1"; prefixLength = 24; } ];
      };

      lan0 = {
        ipAddress = "10.0.4.1";
        prefixLength = 24;
        ip4 = [ { address = "10.0.4.1"; prefixLength = 24; } ];
      };
      lan1 = {
        ipAddress = "10.0.5.1";
        prefixLength = 24;
        ip4 = [ { address = "10.0.5.1"; prefixLength = 24; } ];
      };

      wan1_modem_yota = {
        useDHCP = true;
      };
    };

    networkmanager = {
      enable = false;
    };
  };

  time.timeZone = "Etc/GMT-4";

  nixpkgs = {
    config = {
      allowUnfree = true;
    };
  };

  environment = {
    systemPackages = with pkgs; [
      zile #emacs24-nox
      mc
      htop
      wget

      ceph

      kvm
      #qemu #spice-vdagent
      #win-qemu
      xen

      #networkmanager
      iproute
      iw wirelesstools rfkill
      nssmdns

      lsof

      hdparm
      btrfs-progs

      #dropbox-cli

      (pkgs.substituteAll {
        name = "nix+force-cleanup";
        dir = "bin";
        isExecutable = true;
        src = pkgs.writeScript "nix+force-cleanup" ''#! ${pkgs.stdenv.shell} -e
${pkgs.zsh}/bin/zsh << ZSHEOF

  if [[ $UID == 0 || $EUID == 0 ]]; then
    ${pkgs.nix}/bin/nix-env --delete-generations 2d
    ${pkgs.nix}/bin/nix-store --gc --print-dead
    ${pkgs.nix}/bin/nix-store --gc
    ${pkgs.nix}/bin/nix-collect-garbage -d
    ${pkgs.nix}/bin/nix-store --optimise
  else
    echo Please, run this as root
  fi

ZSHEOF'';
      })

      (pkgs.substituteAll {
        name = "force-ceph-mkfs";
        dir = "bin";
        isExecutable = true;
        src = pkgs.writeScript "force-ceph-mkfs" ''#! ${pkgs.stdenv.shell}

          dev=/dev/$1
          mp=/tmp/$1

          #${pkgs.coreutils}/bin/dd if=/dev/zero of=$dev bs=8M count=8 status=progress
          ${pkgs.btrfs-progs}/bin/mkfs.btrfs -f $dev
          ${pkgs.coreutils}/bin/mkdir $mp

          ${pkgs.utillinux}/bin/mount $dev $mp

          ${pkgs.btrfs-progs}/bin/btrfs subvolume create $mp/osd
          ${pkgs.btrfs-progs}/bin/btrfs subvolume create $mp/mon
          ${pkgs.btrfs-progs}/bin/btrfs subvolume create $mp/mds

          ${pkgs.utillinux}/bin/umount $mp

          ${pkgs.coreutils}/bin/rmdir $mp
        '';
      })
    ];

    noXlibs = true;
  };

  programs = {
    ssh = {
      startAgent = false;
    };
    bash = {
      enableCompletion = true;
    };
  };

  services = {
    udev = {
      extraRules = ''
        SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="", ATTR{dev_id}=="0x0", ATTR{type}=="1", NAME="wlan0_ap"
        SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="", ATTR{dev_id}=="0x0", ATTR{type}=="1", NAME="wlan1_ap"
        SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="", ATTR{dev_id}=="0x0", ATTR{type}=="1", NAME="wan1_modem_yota"
        SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="", ATTR{dev_id}=="0x0", ATTR{type}=="1", NAME="wan0"
        SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="", ATTR{dev_id}=="0x0", ATTR{type}=="1", NAME="lan0"
        SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="", ATTR{dev_id}=="0x0", ATTR{type}=="1", NAME="lan1"
      '';
    };

    xserver.enable = false;

    dbus = {
      #packages = with pkgs; [ avahi ];
    };

    openssh = {
      enable = true;
      permitRootLogin = "without-password";
      passwordAuthentication = false;
      gatewayPorts = "yes";
      extraConfig = ''
        PasswordAuthentication=no
        KbdInteractiveAuthentication=no
        ChallengeResponseAuthentication=no
        PermitTunnel yes
      '';
    };

    avahi = {
      enable = true;
      nssmdns = true;
      hostName = "${config.networking.hostName}";
      wideArea = true;
      ipv4 = true;
      ipv6 = true;
      publish = {
        enable = true;
        domain = true;
        addresses = true;
        hinfo = true;
        userServices = true;
        workstation = true;
      };

      # todo:
      #[server]
      #allow-point-to-point=yes
      #deny-interfaces=eth1
      #[reflector]
      #enable-reflector=yes
    };

    dnsmasq = {
      enable = true;
      servers = [ "8.8.8.8" "8.8.4.4" ];
      extraConfig = ''
        listen-address=127.0.0.1,10.0.3.1,10.0.4.1,10.0.5.1
        domain=lan
        interface=wlan0_ap
        interface=lan0
        interface=lan1
        bind-interfaces
        dhcp-range=10.0.3.16,10.0.3.254,24h
        dhcp-range=10.0.4.16,10.0.4.254,24h
        dhcp-range=10.0.5.16,10.0.5.254,24h
      '';
    };

    hostapd = {
      enable = true;
      interface = "wlan0_ap";
      ssid = "";
      wpa = true;
      wpaPassphrase = "";
      #channel = 11;
      hwMode = "g";
      extraConfig = ''
        wpa=2
        wpa_key_mgmt=WPA-PSK
        wpa_pairwise=TKIP
        rsn_pairwise=CCMP
        auth_algs=3
        macaddr_acl=0

        ieee80211n=1
        #ieee80211ac=1

        wmm_enabled=1
      '';
    };

    ntp = {
      enable = true;
      servers = [
        "0.pool.ntp.org"
        "1.pool.ntp.org"
        "2.pool.ntp.org"
        "3.pool.ntp.org"
      ];
    };

    gpm = {
      enable = false;
      protocol = "ps/2";
    };

    atd = {
      enable = true;
      allowEveryone = true;
    };

    udisks2 = {
      enable = true;
    };
    
  };

  virtualisation = {
    libvirtd = {
      enable = false;
    };
  };


  environment.etc."ceph/ceph.conf" = {
    mode = "0644";
    text = lib.concatStringsSep "\n" ([''
[global]
    fsid = 

    public network = ${cfg.ceph.publicIP}/24
    cluster network = ${cfg.ceph.privateIP}/24

    auth cluster required = none
    auth service required = none
    auth client  required = none

    log to syslog = false

[mon]
    debug mon = 9
    mon osd full ratio = .70
    mon osd nearfull ratio = .60
    mon osd down out interval = 0
    mon osd report timeout = 300

[osd]
    filestore btrfs snap = true
    filestore max sync interval = 5
    #osd crush update on start = true

#[mds]
'']

      ++ (map (osd: ''
[mon.${osd.monID}]
    host = ${cfg.hostName}
    mon addr = ${cfg.ceph.publicIP}:${toString (6789 + osd.id)}
    mon data = /ceph/${osd.dev}/mon

[osd.${toString osd.id}]
    host = ${cfg.hostName}
    osd data = /ceph/${osd.dev}/osd/data
    osd journal = /ceph/${osd.dev}/osd/journal

[mds.${toString osd.id}]
    host = ${cfg.hostName}
    mds data = /ceph/${osd.dev}/mds
'') (cfg.ceph.osds))
    );

  };

  systemd.services = {
    # builtins.listToAttrs (lib.flip map cfg.osds (osd: lib.nameValuePair "ceph-osd-${toString osd.id}" {
  } // (builtins.listToAttrs (builtins.concatLists (lib.flip map cfg.ceph.osds (osd: [
      {
        name = "ceph-mon-${toString osd.monID}";
        value = {
################################################################
####
#### Ceph MON
####
          enable = cfg.ceph.enable;
          wantedBy = [ "multi-user.target" ];
          after = [ "network.target" ];
          requires = [ ];
          restartTriggers = [ config.environment.etc."ceph/ceph.conf".source ];

          path = with pkgs; [ ceph zsh lsb-release getopt ];

          script = ''${pkgs.ceph}/bin/ceph-mon -i ${osd.monID} -c /etc/ceph/ceph.conf --public-addr ${cfg.ceph.publicIP}:${toString (6789 + osd.id)} -f -d'';

          serviceConfig = {
            Type = "simple";
            PermissionsStartOnly = true;
            Restart = "always";
            RestartSec = 1;
            StartLimitInterval = 0;
          };

          preStart = ''
            ${pkgs.hdparm}/bin/hdparm -W 0 /dev/${osd.dev}

            ${pkgs.utillinux}/bin/umount /ceph/${osd.dev}/mon || true
            ${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev}/mon || true

            ${pkgs.coreutils}/bin/mkdir -p /ceph/${osd.dev}/mon
            ${pkgs.utillinux}/bin/mount -t btrfs -o rw,sync,noatime,autodefrag,nospace_cache,compress-force=zlib,subvol=mon /dev/${osd.dev} /ceph/${osd.dev}/mon

            ${pkgs.ceph}/bin/ceph-mon --mkfs -i ${osd.monID} -c /etc/ceph/ceph.conf -f -d
          '';

          postStop = ''
            ${pkgs.utillinux}/bin/umount /ceph/${osd.dev}/mon
            ${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev}/mon
            ${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev} | true
          '';
####
#### EOF Ceph MON
####
################################################################
        };
      }
      {
        name = "ceph-osd-${toString osd.id}";
        value = {
################################################################
####
#### Ceph OSD
####
          enable = cfg.ceph.enable;

          wantedBy = [ "multi-user.target" ];
          after = [ "ceph-mon-${toString osd.monID}.service" ];
          requires = [ "ceph-mon-${toString osd.monID}.service" ];
          restartTriggers = [ config.environment.etc."ceph/ceph.conf".source ];

          path =  with pkgs; [ ceph utillinux hdparm ceph zsh lsb-release getopt coreutils btrfs-progs ];

          script = ''${pkgs.ceph}/bin/ceph-osd -i ${toString osd.id} -c /etc/ceph/ceph.conf -f -d'';

          serviceConfig = {
            Type = "simple";
            PermissionsStartOnly = true;
            Restart = "always";
            RestartSec = 1;
            StartLimitInterval = 0;
          };

          preStart = ''${pkgs.zsh}/bin/zsh << EOF
            ${pkgs.hdparm}/bin/hdparm -W 0 /dev/${osd.dev}

            ${pkgs.utillinux}/bin/umount /ceph/${osd.dev}/osd || true
            ${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev}/osd || true

            ${pkgs.coreutils}/bin/mkdir -p /ceph/${osd.dev}/osd
            ${pkgs.utillinux}/bin/mount -t btrfs -o rw,sync,noatime,autodefrag,nospace_cache,compress-force=zlib,subvol=osd /dev/${osd.dev} /ceph/${osd.dev}/osd

            ${pkgs.coreutils}/bin/mkdir -p /ceph/${osd.dev}/osd/data
            ${pkgs.ceph}/bin/ceph osd create ${osd.uuid} ${toString osd.id} | true
            ${pkgs.ceph}/bin/ceph-osd --mkfs -i ${toString osd.id} --osd-uuid ${osd.uuid} -c /etc/ceph/ceph.conf -f -d | true

EOF'';

          postStop = ''
            ${pkgs.utillinux}/bin/umount /ceph/${osd.dev}/osd
            ${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev}/osd
            ${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev} | true
          '';
####
#### EOF Ceph OSD
####
################################################################
        };
      }
      {
        name = "ceph-mds-${toString osd.id}";
        value = {
################################################################
####
#### Ceph MDS
####
          enable = cfg.ceph.enable;
          wantedBy = [ "multi-user.target" ];
          after = [ "ceph-mon-${toString osd.monID}.service" ];
          requires = [ "ceph-mon-${toString osd.monID}.service" ];
          restartTriggers = [ config.environment.etc."ceph/ceph.conf".source ];

          path = with pkgs; [ ceph zsh lsb-release getopt ];

          script = ''${pkgs.ceph}/bin/ceph-mds -i ${toString osd.id} -c /etc/ceph/ceph.conf -f -d'';

          serviceConfig = {
            Type = "simple";
            PermissionsStartOnly = true;
            Restart = "always";
            RestartSec = 1;
            StartLimitInterval = 0;
          };

          preStart = ''
            ${pkgs.hdparm}/bin/hdparm -W 0 /dev/${osd.dev}

            ${pkgs.utillinux}/bin/umount /ceph/${osd.dev}/mds || true
            ${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev}/mds || true

            ${pkgs.coreutils}/bin/mkdir -p /ceph/${osd.dev}/mds
            ${pkgs.utillinux}/bin/mount -t btrfs -o rw,sync,noatime,autodefrag,nospace_cache,compress-force=zlib,subvol=mds /dev/${osd.dev} /ceph/${osd.dev}/mds
          '';

          postStop = ''
            ${pkgs.utillinux}/bin/umount /ceph/${osd.dev}/mds
            ${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev}/mds
            ${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev} | true
          '';
####
#### EOF Ceph MDS
####
################################################################
        };
      }
    ]))));

  security = {
    sudo = {
      enable = true;
      wheelNeedsPassword = false;
    };
  };

  users = {
    mutableUsers = false;
    users = {
      keeper = {
        isNormalUser = true;
        extraGroups = [ "wheel" ];
      };
      root.openssh.authorizedKeys.keys = [
      ];
    };
  };
  
}

-${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev} | true
+${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev} || true