From 736e8e0678f99a3fde6e4a26658bf5c26430310e Mon Sep 17 00:00:00 2001 From: David Borman Date: Fri, 29 Sep 2017 14:30:51 -0500 Subject: [PATCH 1/4] Support for additional disks, big nodes, update channel and Parallels Create big nodes - In addition to the normal nodes, you can also add additional nodes with a larger memory/cpu configuration. $num_big_instances = 1 $vm_big_memory = 8192 $vm_big_cpus = 2 Additional disks - Each node can be configured with additional disks. $num_data_disks = 3 $data_disk_size = 10 # GBytes Specify the coreos update channel: $update_channel = "alpha" Also add .virtualbox/ to .gitignore Add Parallels support - Set the memory size and number of cpus for Parallels, as well as setting the download path. - Change Vagrant file config.vm.provider order so that VirtualBox is first, which makes it the default provider (assuming it works), and the util.rb reflects that assumption. In general, setting VAGRANT_DEFAULT_PROVIDER is the best way to be deterministic about what provider is used. --- .gitignore | 1 + Vagrantfile | 69 +++++++++++++++++++++++++---------- util.rb | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 154 insertions(+), 18 deletions(-) create mode 100644 util.rb diff --git a/.gitignore b/.gitignore index fb311598..b907f50b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .vagrant/ +.virtualbox/ log/ user-data config.rb diff --git a/Vagrantfile b/Vagrantfile index 0f815d56..a9318885 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -2,6 +2,7 @@ # # vi: set ft=ruby : require 'fileutils' +require_relative 'util.rb' Vagrant.require_version ">= 1.6.0" @@ -30,9 +31,16 @@ $share_home = false $vm_gui = false $vm_memory = 1024 $vm_cpus = 1 +$num_big_instances = 0 +$vm_big_memory = 8192 +$vm_big_cpus = 2 $vb_cpuexecutioncap = 100 $shared_folders = {} $forwarded_ports = {} +$update_channel = "alpha" +# Additional disks to configure on each node +$num_data_disks = 0 +$data_disk_size = 10 # GBytes # Attempt to apply the deprecated environment variable NUM_INSTANCES to # $num_instances while allowing config.rb to override it @@ -49,11 +57,17 @@ def vm_gui $vb_gui.nil? ? $vm_gui : $vb_gui end -def vm_memory +def vm_memory(instance) + if instance > $num_instances then + return $vm_big_memory + end $vb_memory.nil? ? $vm_memory : $vb_memory end -def vm_cpus +def vm_cpus(instance) + if instance > $num_instances then + return $vm_big_cpus + end $vb_cpus.nil? ? $vm_cpus : $vb_cpus end @@ -63,14 +77,12 @@ Vagrant.configure("2") do |config| # forward ssh agent to easily ssh into the different machines config.ssh.forward_agent = true - config.vm.box = "coreos-alpha" - config.vm.box_url = "https://alpha.release.core-os.net/amd64-usr/current/coreos_production_vagrant_virtualbox.json" + config.vm.box = "coreos-%s" % $update_channel + config.vm.box_url = "https://%s.release.core-os.net/amd64-usr/current/coreos_production_vagrant_virtualbox.json" % $update_channel - ["vmware_fusion", "vmware_workstation"].each do |vmware| - config.vm.provider vmware do |v, override| - override.vm.box_url = "https://alpha.release.core-os.net/amd64-usr/current/coreos_production_vagrant_vmware_fusion.json" - end - end + # Note: Having VirtualBox be the first config.vm.provider means + # it will be the default provider. For details on this see: + # https://www.vagrantup.com/docs/providers/basic_usage.html config.vm.provider :virtualbox do |v| # On VirtualBox, we don't have guest additions or a functional vboxsf @@ -81,12 +93,23 @@ Vagrant.configure("2") do |config| config.ignition.enabled = true end + ["vmware_fusion", "vmware_workstation"].each do |vmware| + config.vm.provider vmware do |v, override| + override.vm.box_url = "https://%s.release.core-os.net/amd64-usr/current/coreos_production_vagrant_vmware_fusion.json" % $update_channel + end + end + + config.vm.provider :parallels do |v, override| + override.vm.box_url = "http://%s.release.core-os.net/amd64-usr/current/coreos_production_vagrant_parallels.json" % $update_channel + end + # plugin conflict if Vagrant.has_plugin?("vagrant-vbguest") then config.vbguest.auto_update = false end - (1..$num_instances).each do |i| + total_instances=$num_instances+$num_big_instances + (1..total_instances).each do |i| config.vm.define vm_name = "%s-%02d" % [$instance_name_prefix, i] do |config| config.vm.hostname = vm_name @@ -120,20 +143,26 @@ Vagrant.configure("2") do |config| config.vm.network "forwarded_port", guest: guest, host: host, auto_correct: true end + config.vm.provider :virtualbox do |vb| + vb.gui = vm_gui + vb.memory = vm_memory i + vb.cpus = vm_cpus i + vb.customize ["modifyvm", :id, "--cpuexecutioncap", "#{$vb_cpuexecutioncap}"] + config.ignition.config_obj = vb + end + ["vmware_fusion", "vmware_workstation"].each do |vmware| config.vm.provider vmware do |v| v.gui = vm_gui - v.vmx['memsize'] = vm_memory - v.vmx['numvcpus'] = vm_cpus + v.vmx['memsize'] = vm_memory i + v.vmx['numvcpus'] = vm_cpus i end end - config.vm.provider :virtualbox do |vb| - vb.gui = vm_gui - vb.memory = vm_memory - vb.cpus = vm_cpus - vb.customize ["modifyvm", :id, "--cpuexecutioncap", "#{$vb_cpuexecutioncap}"] - config.ignition.config_obj = vb + config.vm.provider :parallels do |vb| + vb.memory = vm_memory i + vb.cpus = vm_cpus i + vb.name = vm_name end ip = "172.17.8.#{i+100}" @@ -167,6 +196,10 @@ Vagrant.configure("2") do |config| config.ignition.path = 'config.ign' end end + + if $num_data_disks > 0 + attach_volumes(config, $num_data_disks, $data_disk_size) + end end end end diff --git a/util.rb b/util.rb new file mode 100644 index 00000000..d4c682d4 --- /dev/null +++ b/util.rb @@ -0,0 +1,102 @@ + +def get_provider + # Look for "--provider foo" + provider_index = ARGV.index('--provider') + if (provider_index && ARGV[provider_index + 1]) + return ARGV[provider_index + 1] + end + + # Look for "--provider=foo" + for i in 1 ... ARGV.length + if ARGV[i].include?('--provider=') + return ARGV[i].sub('--provider=', '') + end + end + + # Note: The assumption of the default provider + # being VirtualBox is predecated on the order + # of the config.vm.provider calls in Vagrantfile + return ENV['VAGRANT_DEFAULT_PROVIDER'] || 'virtualbox' +end + +$provider = get_provider().to_sym + +class VagrantPlugins::ProviderVirtualBox::Action::SetName + alias_method :original_call, :call + def call(env) + machine = env[:machine] + driver = machine.provider.driver + uuid = driver.instance_eval { @uuid } + ui = env[:ui] + + controller_name="SATA Controller" + + vm_info = driver.execute("showvminfo", uuid) + controller_already_exists = vm_info.match("Storage Controller Name.*#{controller_name}") + + if controller_already_exists + ui.info "already has the #{controller_name} hdd controller, skipping creation/add" + else + ui.info "creating #{controller_name} hdd controller" + driver.execute( + 'storagectl', + uuid, + '--name', "#{controller_name}", + '--add', 'sata', + '--controller', 'IntelAHCI') + end + + original_call(env) + end +end + +# Add persistent storage volumes +def attach_volumes(node, num_volumes, volume_size) + + if $provider == :virtualbox + node.vm.provider :virtualbox do |v, override| + (1..num_volumes).each do |disk| + diskname = File.join(File.dirname(File.expand_path(__FILE__)), ".virtualbox", "#{node.vm.hostname}-#{disk}.vdi") + unless File.exist?(diskname) + v.customize ['createhd', '--filename', diskname, '--size', volume_size * 1024] + end + v.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', disk, '--device', 0, '--type', 'hdd', '--medium', diskname] + end + end + end + + if $provider == :vmware_fusion || $provider == "vmware_workstation" + ["vmware_fusion", "vmware_workstation"].each do |vmware| + node.vm.provider vmware do |v, override| + + vdiskmanager = '/Applications/VMware\ Fusion.app/Contents/Library/vmware-vdiskmanager' + if File.exist?(vdiskmanager) + dir = File.join(File.dirname(File.expand_path(__FILE__)), ".vmware") + unless File.directory?( dir ) + Dir.mkdir dir + end + + (1..num_volumes).each do |disk| + diskname = File.join(dir, "#{node.vm.hostname}-#{disk}.vmdk") + unless File.exist?(diskname) + `#{vdiskmanager} -c -s #{volume_size}GB -a lsilogic -t 1 #{diskname}` + end + + v.vmx["scsi0:#{disk}.filename"] = diskname + v.vmx["scsi0:#{disk}.present"] = 'TRUE' + v.vmx["scsi0:#{disk}.redo"] = '' + end + end + end + end + end + + if $provider == :parallels + node.vm.provider :parallels do |v, override| + (1..num_volumes).each do |disk| + v.customize ['set', :id, '--device-add', 'hdd', '--size', volume_size * 1024] + end + end + end + +end From 61813712d9fb68bf1baa9576723cae93bc79d004 Mon Sep 17 00:00:00 2001 From: David Borman Date: Tue, 5 Dec 2017 10:56:48 -0600 Subject: [PATCH 2/4] Change VirtualBox NIC We've seen some instances of the kernel rebooting due to a kenel paging request in the network RX code, coming from the virtio_net module. Switch the second nic, which is the private network, to be an emulation of an Intel NIC, avoiding the virtio_net code. Only the second NIC is changed, the first NIC is left as a virtio_net NIC. That NIC is only used for host<->VM communications, and changing it to an Intel NIC messes up the NIC configuration, and leaving it as a virtio_net driver keeps things working. (The NIC order was getting changed, causing CoreOS to configure the wrong interface for the private network.) --- Vagrantfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index a9318885..8dbb8cd0 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -149,6 +149,8 @@ Vagrant.configure("2") do |config| vb.cpus = vm_cpus i vb.customize ["modifyvm", :id, "--cpuexecutioncap", "#{$vb_cpuexecutioncap}"] config.ignition.config_obj = vb + + vb.customize ['modifyvm', :id, '--nictype2', '82545EM'] end ["vmware_fusion", "vmware_workstation"].each do |vmware| @@ -166,7 +168,7 @@ Vagrant.configure("2") do |config| end ip = "172.17.8.#{i+100}" - config.vm.network :private_network, ip: ip + config.vm.network :private_network, ip: ip, adapter: "2" # This tells Ignition what the IP for eth1 (the host-only adapter) should be config.ignition.ip = ip From 938d2d3115f85349b033ced4dfaccc86c940c044 Mon Sep 17 00:00:00 2001 From: David Borman Date: Fri, 19 Jan 2018 13:55:00 -0600 Subject: [PATCH 3/4] Revert "Change VirtualBox NIC" This reverts commit 61813712d9fb68bf1baa9576723cae93bc79d004. This change has been causing problems and unintended consequences, such as affecting coreos NIC configuration when running with parallels, even though this should have only changed things with Virtualbox. And while it seems to have fixed the stability issues when running with linux/virtualbox, it is not working the same with mac/virtualbox. --- Vagrantfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 8dbb8cd0..a9318885 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -149,8 +149,6 @@ Vagrant.configure("2") do |config| vb.cpus = vm_cpus i vb.customize ["modifyvm", :id, "--cpuexecutioncap", "#{$vb_cpuexecutioncap}"] config.ignition.config_obj = vb - - vb.customize ['modifyvm', :id, '--nictype2', '82545EM'] end ["vmware_fusion", "vmware_workstation"].each do |vmware| @@ -168,7 +166,7 @@ Vagrant.configure("2") do |config| end ip = "172.17.8.#{i+100}" - config.vm.network :private_network, ip: ip, adapter: "2" + config.vm.network :private_network, ip: ip # This tells Ignition what the IP for eth1 (the host-only adapter) should be config.ignition.ip = ip From f308f48048b81a7169c6301c403433447de75953 Mon Sep 17 00:00:00 2001 From: David Borman Date: Mon, 22 Jan 2018 09:31:45 -0600 Subject: [PATCH 4/4] Document the linux-virtualbox branch --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 6e11408e..f4202741 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,12 @@ IRC: #coreos on freenode.org Mailing list: [coreos-dev](https://groups.google.com/forum/#!forum/coreos-dev) +## Linux Hosts using VirtualBox +The virtio NIC driver can cause random reboots +due to a kernel paging request in the RX path. +The `linux-virutalbox` branch avoids this issue +by changing the NIC type to an Intel device. + ## Streamlined setup 1) Install dependencies