Openstack Nova 源码分析 — 使用 VCDriver 创建 VMware Instance

在上一篇 Openstack Nova 源码分析 — Create instances (nova-conductor阶段) 中，记录了 nova-api 接收到创建虚拟机的请求后，在 nova-conductor 中的执行流程。最终 nova-comductor 通过调用 nova-compute 的 RPC 接口函数 compute_rpcapi.build_and_run_instance() 将创建虚拟机的请求，通过 Queue 传递给 nova-compute 。本篇继续往下看看 Openstack 创建一个虚拟机时，程序流在 nova-compute 和 Virt Driver 阶段的执行过程。而且本篇使用 VCDirver 作为Virt Driver Type 。

NOTE:下面的代码块大多为节选。

nova-compute vCenter

在之前的文章 VMware 接入 Openstack — 使用 Openstack 创建 vCenter 虚拟机 已经记录过如何将 VMware 接入 Openstack ，本质是通过 nova-compute 和 vCenter 中的 Cluster 一一对应来进行管理。

紧接着，当 nova-conductor 调用了 nova-compute 的 RPC 接口后，相应接口的具体操作函数在 nova.compute.manager 中实现。

1841 def build_and_run_instance(self, context, instance, image, request_spec, 1 filter_properties, admin_password=None, 2 injected_files=None, requested_networks=None, 3 security_groups=None, block_device_mapping=None, 4 node=None, limits=None): 6 @utils.synchronized(instance.uuid) 7 def _locked_do_build_and_run_instance(*args, **kwargs): 8 # NOTE(danms): We grab the semaphore with the instance uuid 9 # locked because we could wait in line to build this instance 10 # for a while and we want to make sure that nothing else tries 11 # to do anything with this instance while we wait. 12 with self._build_semaphore: 13 self._do_build_and_run_instance(*args, **kwargs) 15 # NOTE(danms): We spawn here to return the RPC worker thread back to 16 # the pool. Since what follows could take a really long time, we dont 17 # want to tie up RPC workers. 18 utils.spawn_n(_locked_do_build_and_run_instance, 19 context, instance, image, request_spec, 20 filter_properties, admin_password, injected_files, 21 requested_networks, security_groups, 22 block_device_mapping, node, limits)

在上述的 nova.compute.manager.ComputeManager:build_and_run_instance()中调用了_do_build_and_run_instance() 函数。

1870 def _do_build_and_run_instance(self, context, instance, image, 1 request_spec, filter_properties, admin_password, injected_files, 2 requested_networks, security_groups, block_device_mapping, 3 node=None, limits=None): 1901 try: 1 self._build_and_run_instance(context, instance, image, 2 decoded_files, admin_password, requested_networks, 3 security_groups, block_device_mapping, node, limits, 4 filter_properties) 5 return build_results.ACTIVE

再跳转到 _build_and_run_instance()，这个函数非常重要。

718 self.driver = driver.load_compute_driver(self.virtapi, compute_driver) # 加载 Driver, 过程如下： # nova.virt.driver:load_compute_driver() # == oslo_utils.importutils:import_object_ns() # == nova.utils:check_isinstance() # Return： 一个由 (compute_driver = CONF.compute_driver) 决定的 ComputeDriver 实例化对象 driver
1979 def _build_and_run_instance(self, context, instance, image, injected_files, 1 admin_password, requested_networks, security_groups, 2 block_device_mapping, node, limits, filter_properties): ########## Get image_id 1983 image_name = image.get(name) # == nova.image.__init__ # == nova.image.api.API:get() # == nova.image.api.API:_get_session_and_image_id() # == nova.image.glance:get_remote_image_service() # == nova.image.glance.GlanceImageService:show() # == nova.image.glance._tnslate_from_glanceranslate_from_glance # Return:ClanceImageService.show() 返回一个包含了 Image_Mete 信息的 Dict[name] == uri_or_id 1 self._notify_about_instance_usage(context, instance, create.start, 2 extra_usage_info={image_name: image_name}) # 向外部发出一个 start to create instance 的通知 ########## 应用 claim 机制 Update table:compute_nodes 1986 try: 1 rt = self._get_resource_tracker(node) 2 with rt.instance_claim(context, instance, limits): # 通过传参来创建资源Dict (EG. network_info/block_device_info) 1994 with self._build_resources(context, instance, 1 requested_networks, security_groups, image, 2 block_device_mapping) as resources: # Change the Instance status 3 instance.vm_state = vm_states.BUILDING 4 instance.task_state = task_states.SPAWNING 2 # NOTE(JoshNang) This also saves the changes to the 3 # instance from _allocate_network_async, as they arent 4 # saved in that function to prevent races. 5 instance.save(expected_task_state= 6 task_states.BLOCK_DEVICE_MAPPING) # Block Storage(cinder) 2004 block_device_info = resources[block_device_info] # Network 2005 network_info = resources[network_info] ########## Create Instance # 由 nova.virt.driver.ComputeDriver 实例化对象 driver 调用 spawn（） 函数来进行虚拟机的创建 2006 self.driver.spawn(context, instance, image, 1 injected_files, admin_password, 2 network_info=network_info, 3 block_device_info=block_device_info)

NOTE：task states 执行 Task 时的过渡状态，在状态发生改变时会向外部发出通知。前提是配置项notify_on_state_change要配置为vm_state或vm_and_task_state(有待验证)。

因为我希望使用 VCDriver 驱动类型， 所以在 Nova 的配置文件 /etc/nova.conf 中设置选项compute_driver=vmwareapi.VMwareVCDriver。
这样的话，通过执行代码 compute_driver = CONF.compute_driver 就可以获得 VCDriver 的 driver 对象。当我们使用这个 driver 对象来创建虚拟机时，程序流会进入到nova/virt/vmwareapi/ 再通过调用 VMware 提供的 API 接口 (nova.virt.vmwareapi.driver.VMwareVCDriver:spawn())来最终实现虚拟机的创建 。

# 实例化了一个 vmops.VMwareVMOps 对象 186 self._vmops = vmops.VMwareVMOps(self._session, 1 virtapi, 2 self._volumeops, 3 self._cluster_ref, 4 datastore_regex=self._datastore_regex) 401 def spawn(self, context, instance, image_meta, injected_files, 1 admin_password, network_info=None, block_device_info=None): 2 """Create VM instance.""" 3 image_meta = objects.ImageMeta.from_dict(image_meta) # nova.object.image_meta.ImageMeta:from_dict() # == nova.object.image_meta.ImageMetaProps:from_dict() # Return： obj = cls() = image_meta.get("properties", {})[0]
# _vmops 为 nova.virt.vmmwareapi.vmops.VMwareVMOps 的实例化对象 4 self._vmops.spawn(context, instance, image_meta, injected_files, 5 admin_password, network_info, block_device_info)

跳转到 nova.virt.vmwareapi.vmops.VMwareVMOps:spawn()

# 这一个类封装了对 vCenter 的虚拟机的操作函数(EG. _get_base_folder/_extend_virtual_disk/_delete_datastore_file/build_virtual_machine) 149 class VMwareVMOps(object): 1 """Management class for VM-related tasks.""" 152 def __init__(self, session, virtapi, volumeops, cluster=None, 1 datastore_regex=None): 677 def spawn(self, context, instance, image_meta, injected_files, 1 admin_password, network_info, block_device_info=None): 3 client_factory = self._session.vim.client.factory 4 image_info = images.VMwareImage.from_image(instance.image_ref, 5 image_meta) # nova.virt.vmwareapi.images.VMwareImage:from_image() # """Returns VMwareImage, the subset of properties the driver uses.""" # Return vmware image object # 在这里类中，实现了一系列关于 Image 属性参数的设定和克隆 。
# 获取创建虚拟机所需要的参数信息(EG. ds/dc) # nova.virt.vmwareapi.vmops._get_vm_config_info() # Return： VirtualMachineInstanceConfigInfo()
277 def build_virtual_machine(self, instance, image_info, 1 dc_info, datastore, network_info, extra_specs, 2 metadata): 3 vif_infos = vmwarevif.get_vif_info(self._session, 4 self._cluster, 5 utils.is_neutron(), 6 image_info.vif_model, 7 network_info) # 获取 Virtual Interface info # nova.virt.vmwareapi.vif:get_vif_info() # == nova.virt.vmwareapi.vif:get_vif_dict() #Return: vif_info 包含了(netowork_name/mac_address/network_ref/iface_id/cif_model)等信息 9 if extra_specs.storage_policy: 10 profile_spec = vm_util.get_storage_profile_spec( 11 self._session, extra_specs.storage_policy) 12 else: 13 profile_spec = None 14 # Get the create vm config spec 15 client_factory = self._session.vim.client.factory 16 config_spec = vm_util.get_vm_create_spec(client_factory, 17 instance, 18 datastore.name, 19 vif_infos, 20 extra_specs, 21 image_info.os_type, 22 profile_spec=profile_spec, 23 metadata=metadata) # 获取虚拟机的 config_spec (profile/cpu/memory/Neutron 等) # nova.virt.vmwareapi.vmm_util.get_vm_create_spec（）
24 # Create the VM # create_vm() 会返回 Task 的 Result ，并附值给 vm_ref。 vm_ref 被用于创建虚拟机后的一切列数据更新 。 25 vm_ref = vm_util.create_vm(self._session, instance, dc_info.vmFolder, 26 config_spec, self._root_resource_pool) 27 return vm_ref

在 VMwareVMOps:build_virtual_machine() 函数中又调用了 nova.virt.vmwareapi.vm_util:get_vm_create_spec() 函数来获取创建虚拟机所需要的参数信息。同时也调用了nova.virt.vmwareapi.vm_util:create_vm() 来 Create the VM 。所以我们先转到 nova.virt.vmwareapi.vm_util Module 去看看具体的 Return 。

1287 def create_vm(session, instance, vm_folder, config_spec, res_pool_ref): 1 """Create VM on ESX host.""" 2 LOG.debug("Creating VM on the ESX host", instance=instance) # session 是 nova.virt.vmwareapi.driver.VMwareAPISession 的实例化对象 3 vm_create_task = session._call_method( 4 session.vim, 5 "CreateVM_Task", vm_folder, 6 config=config_spec, pool=res_pool_ref) 7 try: 8 task_info = session._wait_for_task(vm_create_task) 9 except vexc.VMwareDriverException: 10 # An invalid guestId will result in an error with no specific fault 11 # type and the generic error A specified parameter was not correct. 12 # As guestId is user-editable, we try to help the user out with some 13 # additional information if we notice that guestId isnt in our list of 14 # known-good values. 15 # We dont check this in advance or do anything more than warn because 16 # we cant guarantee that our list of known-good guestIds is complete. 17 # Consequently, a value which we dont recognise may in fact be valid. 18 with excutils.save_and_reraise_exception(): 19 if config_spec.guestId not in constants.VALID_OS_TYPES: 20 LOG.warning(_LW(vmware_ostype from image is not recognised: 21 \%(ostype)s\. An invalid os type may be 22 one cause of this instance creation failure), 23 {ostype: config_spec.guestId}) 24 LOG.debug("Created VM on the ESX host", instance=instance) 25 return task_info.result # 这个函数的最终返回 Task 的执行结果

到此为止关于虚拟机的创建就完成了，需要注意的是：在我们创建完虚拟机之后其实还有许多的事情是需要做的，EG. 更新数据库/开启虚拟机
所以，在nova.virt.vmwareapi.vm_util:create_vm() 中得到了创建虚拟机的 Task Result task_info.result 之后，需要使用这一 Return 来进行一系列的操作。当然，这一系列的操作会在 VM 相关任务管理类： nova.virt.vmwareapi.wmops.VMwareVMOps 中实现。

# nova/virt/vmmwareapi/vmops.py

# 下面的操作，都是在成功创建了虚拟机并接收 vm_ref (vm_ref = vm_util.create_vm())返回值之后执行。

 2 # Cache the vm_ref. This saves a remote call to the VC. This uses the

 1 # instance uuid. 

701 vm_util.vm_ref_cache_update(instance.uuid, vm_ref) 


 2 # Set the machine.id parameter of the instance to inject

 1 # the NIC configuration inside the VM

708 if CONF.flat_injected: 

 1 self._set_machine_id(client_factory, instance, network_info,

 2 vm_ref=vm_ref)


 2 # Set the vnc configuration of the instance, vnc port starts from 5900

 1 if CONF.vnc.enabled: 

714 self._get_and_set_vnc_config(client_factory, instance, vm_ref) 


 4 image_info.image_id, vi.datastore, vi.dc_info.ref)

 3 self._fetch_image_if_missing(context, vi)

 1 if image_info.is_iso: 

727 self._use_iso_image(vm_ref, vi) 

 1 elif image_info.linked_clone:

 2 self._use_disk_image_as_linked_clone(vm_ref, vi)

 3 else: 

 4 self._use_disk_image_as_full_clone(vm_ref, vi)


 1 # Create ephemeral disks 

758 self._create_ephemeral(block_device_info, instance, vm_ref, 

 1 vi.dc_info, vi.datastore, instance.uuid,

 2 vi.ii.adapter_type) 

 3 self._create_swap(block_device_info, instance, vm_ref, vi.dc_info,

 4 vi.datastore, instance.uuid, vi.ii.adapter_type)


 2 instance, vm_ref, vi.dc_info, vi.datastore,

 3 injected_files, admin_password, network_info)

 # 将虚拟机起电

769 vm_util.power_on_instance(self._session, instance, vm_ref=vm_ref) 

VMWARE里启动kylin16.0时出现 SMBus Host Controller not enabled (还未进入系统） 在Vmware里安装完Ubuntu16.10，启动时出现 SMBus Host Controller not enabled 错误提示，进不到图形界面。
配置 nova compute, 令其他与 ceph 集群进行连接 最终目的能够允许 instances 利用 ceph RBD 当作外部卷使用 nova compute 默认已经能够正常工作, 当前只添加 ceph 连接部分
nova instance有3种状态：power_state, vm_state, task_state，分别对应horizon界面上的Pow...