feat: create separate storage node

Signed-off-by: Paul Kroeher <[email protected]>

On-behalf-of: SAP [email protected]

Author: pkr4711

modules/compute/nova.nix

@@ -189,14 +189,7 @@ in
             user = "nova";
           };
         };
-        "/etc/systemd/system/tgtd.service" = {
-          "C+" = {
-            user = "root";
-            group = "root";
-            mode = "0600";
-            argument = "${pkgs.tgt}/etc/systemd/system/tgtd.service";
-          };
-        };
+        # we don't need tgt on a compute node -> only iscsi-client (openiscsi)
       };
     };
 
@@ -205,7 +198,7 @@ in
       name = "iqn.iscsi.${config.networking.hostName}";
     };
 
-    environment.systemPackages = [ pkgs.tgt ];
+    environment.systemPackages = [ pkgs.openiscsi ];
 
     systemd.services.nova-compute = {
       description = "OpenStack Nova Scheduler Daemon";
@@ -222,7 +215,7 @@ in
           qemu
           util-linux
           lvm2
-          tgt
+          openiscsi
         ]
         ++ cfg.extraPkgs;
       environment.PYTHONPATH = "${nova_env}/${pkgs.python3.sitePackages}";

modules/controller/openstack-controller.nix

@@ -33,8 +33,9 @@ in
     (import ./nova.nix { inherit nova; })
     (import ./neutron.nix { inherit neutron; })
     (import ./horizon.nix { inherit horizon; })
-    (import ./cinder.nix { inherit cinder; })
-    (import ../storage/cinder-storage-node.nix { inherit cinder; })
+    (import ./cinder.nix { inherit cinder; }) # only cinder management component
+    # deploy real storage backend into a separate storageNode
+    # (import ../storage/cinder-storage-node.nix { inherit cinder; })
   ];
 
   config = {

modules/default.nix

@@ -14,5 +14,7 @@
 
   computeModule = import ./compute/compute.nix { inherit (openstackPkgs) neutron nova; };
 
+  storageModule = import ./storage/cinder-storage-node.nix { inherit (openstackPkgs) cinder; };
+
   testModules = import ./testing { };
 }

modules/storage/cinder-storage-node.nix

@@ -9,15 +9,6 @@
 }:
 with lib;
 let
-  # adminEnv = {
-  #   OS_USERNAME = "admin";
-  #   OS_PASSWORD = "admin";
-  #   OS_PROJECT_NAME = "admin";
-  #   OS_USER_DOMAIN_NAME = "Default";
-  #   OS_PROJECT_DOMAIN_NAME = "Default";
-  #   OS_AUTH_URL = "http://controller:5000/v3";
-  #   OS_IDENTITY_API_VERSION = "3";
-  # };
   cfg = config.cinder-storage-node;
 
   cinder_env = pkgs.python3.buildEnv.override {
@@ -32,6 +23,7 @@ let
     paths = [
       cinder_env
       pkgs.qemu
+      pkgs.tgt
     ];
   };
 
@@ -45,13 +37,17 @@ let
     [DEFAULT]
     transport_url = rabbit://openstack:openstack@controller
     auth_strategy = keystone
-    my_ip = controller
+    my_ip = 10.0.0.20
     enabled_backends = lvm
     volumes_dir = /var/lib/cinder/volumes
     state_path = /var/lib/cinder
     rootwrap_config = ${rootwrapConf}
     glance_api_servers = http://controller:9292
     verify_glance_signatures = disabled
+    log_dir = /var/log/cinder
+    iscsi_ip_address = $my_ip
+    iscsi_port = 3260
+    iscsi_target_prefix = iqn.2010-10.org.openstack:
 
     [database]
     connection = mysql+pymysql://cinder:cinder@controller/cinder
@@ -76,6 +72,14 @@ let
     volume_backend_name = lvm
     lvm_type = default
     target_protocol = iscsi
+    target_helper = tgtadm
+    iscsi_ip_address = $my_ip
+    iscsi_port = 3260
+    iscsi_target_prefix = iqn.2010-10.org.openstack:
+  '';
+
+  cinderTgtConf = pkgs.writeText "cinder.conf" ''
+    include /var/lib/cinder/volumes/*
   '';
 in
 {
@@ -140,10 +144,60 @@ in
             mode = "0755";
           };
         };
+        "/etc/cinder/cinder.conf" = {
+          L = {
+            argument = "${cinderConf}";
+          };
+        };
+        "/etc/tgt/conf.d/cinder.conf" = {
+          L = {
+            argument = "${cinderTgtConf}";
+          };
+        };
+        "/etc/tgt/targets.conf" = {
+          L = {
+            argument = "${pkgs.tgt}/etc/tgt/targets.conf";
+          };
+        };
       };
     };
 
-    systemd.services.cinder-volume-group = {
+    # start iSCSI target daemon
+    # we expose LVM block storage as iSCSI to compute hosts
+    systemd.services.tgtd = {
+      enable = true;
+      description = "iSCSI target framework daemon";
+      wantedBy = [ "multi-user.target" ];
+      after = [
+        "network.target"
+        "cinder-volume-group-setup.service"
+      ];
+      path = [
+        pkgs.coreutils
+        pkgs.tgt
+      ];
+      environment.TGTD_CONFIG = "/etc/tgt/targets.conf";
+      serviceConfig = {
+        ExecStart = "${pkgs.tgt}/bin/tgtd -f";
+        ExecStartPost = [
+          "${pkgs.coreutils}/bin/sleep 5"
+          "${pkgs.tgt}/bin/tgtadm --op update --mode sys --name State -v offline"
+          "${pkgs.tgt}/bin/tgtadm --op update --mode sys --name State -v ready"
+          "${pkgs.tgt}/bin/tgt-admin -e -c $TGTD_CONFIG"
+        ];
+
+        ExecReload = "${pkgs.tgt}/bin/tgt-admin --update ALL -f -c $TGTD_CONFIG";
+
+        ExecStop = [
+          "${pkgs.tgt}/bin/tgtadm --op update --mode sys --name State -v offline"
+          "${pkgs.tgt}/bin/tgt-admin --offline ALL"
+          "${pkgs.tgt}/bin/tgt-admin --update ALL -c /dev/null -f"
+          "${pkgs.tgt}/bin/tgtadm --op delete --mode system"
+        ];
+      };
+    };
+
+    systemd.services.cinder-volume-group-setup = {
       description = "OpenStack Cinder volume group setup";
       wantedBy = [ "multi-user.target" ];
       path = [
@@ -155,14 +209,9 @@ in
         ExecStart = pkgs.writeShellScript "cinder-volume-group.sh" ''
           set -euxo pipefail
 
-          # Setup some lvm volume group required by cinder
-          dd if=/dev/zero of=/tmp/cinder-volumes bs=1G count=2
-
-          losetup /dev/loop0 /tmp/cinder-volumes
-
-          # Create physical volume and volume group
-          pvcreate /dev/loop0
-          vgcreate cinder-volumes /dev/loop0
+          # create a new LVM volume group on second disk
+          pvcreate /dev/vdb
+          vgcreate cinder-volumes /dev/vdb
         '';
       };
     };
@@ -173,12 +222,13 @@ in
     # Update: still does not work -.-
     environment.systemPackages = [
       pkgs.qemu
+      pkgs.tgt
     ];
 
     systemd.services.cinder-volume = {
       description = "OpenStack Cinder Volume";
       after = [
-        "cinder-volume-group.service"
+        "cinder-volume-group-setup.service"
       ];
       path = with pkgs; [
         cinder_env

modules/testing/README.md

@@ -0,0 +1,51 @@
+# SSH Access
+
+## Storage VM
+
+```bash
+ssh root@localhost -p 2022 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+```
+
+## Controller VM
+
+```bash
+ssh root@localhost -p 1122 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+```
+
+# Cinder & tgtd
+
+## Create volume
+
+* login in to `controllerVM` or `storageVM`
+
+```bash
+openstack volume create --size 4 TEST_VOL
+```
+
+* Cinder will create a LVM logical volume in the volume group `cinder-volumes`.
+* At this time this volume will not be exposed as an iSCSI volume. This only happens when the volume is assigned to a VM.
+
+## Assign volume to a VM
+
+* create a new VM with: `openstack server create`
+  * or look into system unit: `openstack-create-vm` on VM `controllerVM`
+
+* attach volume to vm
+
+```bash
+openstack server list
+openstack volume list
+openstack server add volume <INSTANCE_NAME_OR_ID> <VOLUME_NAME_OR_ID>
+```
+
+* Verify tgtd status on `storageVM`: `tgt-admin --dump`
+* Cinder should generated a tgtd configuration file within: `/var/lib/cinder/volumes`
+* In the VM `computeVM` should appear a new block device `sda`. (`dmesg`)
+
+```bash
+[root@storageVM:/var/lib/cinder/volumes]# l
+total 12K
+drwxr-xr-x 2 cinder cinder 4.0K Feb 24 08:16 .
+drwxr-xr-x 6 cinder cinder 4.0K Feb 24 08:09 ..
+-rw------- 1 cinder cinder  268 Feb 24 08:16 volume-64159e0c-18bb-449f-b1e0-86f198b170e4
+```

modules/testing/default.nix

@@ -33,9 +33,26 @@ let
 
         environment.systemPackages = [
           pkgs.openstackclient
+          pkgs.openiscsi
+          pkgs.sshpass
         ];
 
         environment.variables = adminEnv;
+
+        # enable easy access to test VMs with ssh
+        users.users.root.openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEMjGRp1/vaTxGiGbZnaSv4wu4LUUP7lGSGDFKfF31xw [email protected]"
+        ];
+
+        services.openssh = {
+          enable = true;
+          ports = [ 22 ];
+          settings = {
+            PasswordAuthentication = true;
+            PermitRootLogin = "without-password"; # "yes", "without-password", "prohibit-password", "forced-commands-only", "no"
+          };
+        };
+
       };
     };
 
@@ -115,6 +132,10 @@ in
             vlan = 2;
           };
         };
+        # enable ssh access
+        forwardPorts = [
+         { from = "host"; host.port = 1122; guest.port = 22; }
+        ];
       };
 
       systemd.services.openstack-create-vm = {
@@ -170,14 +191,13 @@ in
             matchConfig.Name = [ "eth0" ];
             networkConfig = {
               DHCP = "yes";
+              DNS = "8.8.8.8";
             };
           };
           eth1 = {
             matchConfig.Name = [ "eth1" ];
             networkConfig = {
               Address = "10.0.0.11/24";
-              Gateway = "10.0.0.1";
-              DNS = "8.8.8.8";
             };
           };
 
@@ -237,4 +257,66 @@ in
       };
 
     };
+
+  testStorage =
+    { ... }:
+    {
+
+      imports = [ common ];
+
+      virtualisation = {
+        memorySize = 4096;
+        cores = 4;
+        diskSize = 4096;
+        # add separate disk as LVM backend
+        emptyDiskImages = [
+          16384 # 16GB
+        ];
+        interfaces = {
+          eth1 = {
+            vlan = 1;
+          };
+          eth2 = {
+            vlan = 2;
+          };
+        };
+        # enable ssh access
+        forwardPorts = [
+         { from = "host"; host.port = 2022; guest.port = 22; }
+        ];
+      };
+
+      systemd.network = {
+        enable = true;
+        wait-online.enable = false;
+
+        networks = {
+          eth0 = {
+            matchConfig.Name = [ "eth0" ];
+            networkConfig = {
+              DHCP = "yes";
+              LinkLocalAddressing = "yes";
+              KeepConfiguration = "yes";
+              DNS = "8.8.8.8";
+            };
+          };
+
+          eth1 = {
+            matchConfig.Name = [ "eth1" ];
+            networkConfig = {
+              Address = "10.0.0.20/24";
+            };
+          };
+
+          eth2 = {
+            matchConfig.Name = [ "eth2" ];
+            networkConfig = {
+              DHCP = "no";
+              LinkLocalAddressing = "no";
+              KeepConfiguration = "yes";
+            };
+          };
+        };
+      };
+    };
 }