From f32779217e153b4484dc2ff3573767d17b71fa25 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Wed, 23 Aug 2023 23:47:49 -0300 Subject: [PATCH 01/62] New Feature: Migrate VMware VMs to KVM --- .../com/cloud/agent/api/to/DatadiskTO.java | 24 +- .../cloud/agent/api/to/OVFInformationTO.java | 4 + .../agent/api/to/VmwareVmForMigrationTO.java | 86 ++++++ .../java/com/cloud/dc}/VmwareDatacenter.java | 2 +- .../main/java/com/cloud/event/EventTypes.java | 2 + .../com/cloud/hypervisor/HypervisorGuru.java | 7 + .../HypervisorOutOfBandVMClone.java | 42 +++ .../cloud/template/TemplateApiService.java | 3 + .../template/VirtualMachineTemplate.java | 2 + .../java/com/cloud/vm/VmDetailConstants.java | 11 + .../apache/cloudstack/api/ApiConstants.java | 3 + .../user/template/RegisterTemplateCmd.java | 6 +- .../RegisterTemplateFromVMwareVMCmd.java | 155 +++++++++++ .../response/VmwareDatacenterResponse.java | 2 +- .../storage/command/DownloadCommand.java | 10 + debian/control | 2 +- .../com/cloud/storage/StorageManager.java | 8 + .../orchestration/VolumeOrchestrator.java | 43 ++- .../com/cloud/dc}/VmwareDatacenterVO.java | 2 +- .../cloud/dc}/dao/VmwareDatacenterDao.java | 4 +- .../java/com/cloud/storage/VMTemplateVO.java | 6 + .../storage/image/TemplateServiceImpl.java | 6 + .../storage/image/store/TemplateObject.java | 8 + .../endpoint/DefaultEndPointSelector.java | 82 ++++-- .../image/BaseImageStoreDriverImpl.java | 91 +++++++ packaging/centos7/cloud.spec | 1 + packaging/centos8/cloud.spec | 1 + .../cloudstack/sioc/SiocManagerImpl.java | 4 +- .../backup/VeeamBackupProvider.java | 4 +- .../manager/BareMetalTemplateAdapter.java | 9 + .../LibvirtDownloadCommandWrapper.java | 230 ++++++++++++++++ .../LibvirtResizeVolumeCommandWrapper.java | 19 +- .../wrapper/LibvirtUtilitiesHelper.java | 43 +++ .../apache/cloudstack/utils/qemu/QemuImg.java | 1 + .../LibvirtDownloadCommandWrapperTest.java | 87 ++++++ .../wrapper/LibvirtUtilitiesHelperTest.java | 31 +++ .../com/cloud/hypervisor/guru/VMwareGuru.java | 100 ++++++- .../vmware/VmwareDatacenterService.java | 6 + .../vmware/VmwareServerDiscoverer.java | 3 +- .../vmware/dao/VmwareDatacenterDaoImpl.java | 3 +- .../vmware/manager/VmwareManagerImpl.java | 63 ++++- .../command/admin/zone/AddVmwareDcCmd.java | 2 +- .../admin/zone/ListVmwareDcStoppedVmsCmd.java | 138 ++++++++++ .../command/admin/zone/ListVmwareDcsCmd.java | 2 +- .../command/admin/zone/UpdateVmwareDcCmd.java | 2 +- .../api/response/VmwareStoppedVmResponse.java | 67 +++++ .../vmware/VmwareDatacenterApiUnitTest.java | 3 +- .../vmware/manager/VmwareManagerImplTest.java | 6 +- .../cloud/hypervisor/HypervisorGuruBase.java | 11 + .../cloud/server/ManagementServerImpl.java | 2 + .../com/cloud/storage/StorageManagerImpl.java | 3 +- .../storage/download/DownloadMonitorImpl.java | 109 +++++++- .../template/HypervisorTemplateAdapter.java | 25 ++ .../com/cloud/template/TemplateAdapter.java | 5 + .../cloud/template/TemplateManagerImpl.java | 76 ++++++ .../template/TemplateManagerImplTest.java | 8 + ui/public/locales/en.json | 2 + ui/src/config/section/image.js | 9 + .../views/image/RegisterOrUploadTemplate.vue | 65 ++++- .../image/RegisterTemplateFromVcenter.vue | 252 ++++++++++++++++++ .../hypervisor/vmware/mo/DatacenterMO.java | 24 ++ .../mo/VmwareStoppedVmInDatacenter.java | 42 +++ 62 files changed, 1977 insertions(+), 92 deletions(-) create mode 100644 api/src/main/java/com/cloud/agent/api/to/VmwareVmForMigrationTO.java rename {plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware => api/src/main/java/com/cloud/dc}/VmwareDatacenter.java (96%) create mode 100644 api/src/main/java/com/cloud/hypervisor/HypervisorOutOfBandVMClone.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateFromVMwareVMCmd.java rename {plugins/hypervisors/vmware => api}/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java (97%) rename {plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware => engine/schema/src/main/java/com/cloud/dc}/VmwareDatacenterVO.java (99%) rename {plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware => engine/schema/src/main/java/com/cloud/dc}/dao/VmwareDatacenterDao.java (96%) create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDownloadCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDownloadCommandWrapperTest.java create mode 100644 plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcStoppedVmsCmd.java create mode 100644 plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/VmwareStoppedVmResponse.java create mode 100644 ui/src/views/image/RegisterTemplateFromVcenter.vue create mode 100644 vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmwareStoppedVmInDatacenter.java diff --git a/api/src/main/java/com/cloud/agent/api/to/DatadiskTO.java b/api/src/main/java/com/cloud/agent/api/to/DatadiskTO.java index 7a82688a35d2..d7bad54fc4c1 100644 --- a/api/src/main/java/com/cloud/agent/api/to/DatadiskTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/DatadiskTO.java @@ -20,7 +20,7 @@ public class DatadiskTO { private String path; - private long virtualSize; + private Long virtualSize; private long fileSize; boolean bootable; private String diskId; @@ -29,11 +29,19 @@ public class DatadiskTO { private String diskControllerSubType; private int diskNumber; private String configuration; + private Long templateId; public DatadiskTO() { } - public DatadiskTO(String path, long virtualSize, long fileSize, String diskId, boolean isIso, boolean bootable, + public DatadiskTO(String path, Long virtualSize, long fileSize, int diskNumber) { + this.path = path; + this.virtualSize = virtualSize; + this.fileSize = fileSize; + this.diskNumber = diskNumber; + } + + public DatadiskTO(String path, Long virtualSize, long fileSize, String diskId, boolean isIso, boolean bootable, String controller, String controllerSubType, int diskNumber, String configuration) { this.path = path; this.virtualSize = virtualSize; @@ -110,4 +118,16 @@ public int getDiskNumber() { public String getConfiguration() { return configuration; } + + public Long getTemplateId() { + return templateId; + } + + public void setTemplateId(Long templateId) { + this.templateId = templateId; + } + + public void setDiskNumber(int diskNumber) { + this.diskNumber = diskNumber; + } } diff --git a/api/src/main/java/com/cloud/agent/api/to/OVFInformationTO.java b/api/src/main/java/com/cloud/agent/api/to/OVFInformationTO.java index 6c6c61d8e955..7f873570e47d 100644 --- a/api/src/main/java/com/cloud/agent/api/to/OVFInformationTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/OVFInformationTO.java @@ -48,6 +48,10 @@ public class OVFInformationTO { public OVFInformationTO() { } + public OVFInformationTO(List disks) { + this.disks = disks; + } + public List getProperties() { return properties; } diff --git a/api/src/main/java/com/cloud/agent/api/to/VmwareVmForMigrationTO.java b/api/src/main/java/com/cloud/agent/api/to/VmwareVmForMigrationTO.java new file mode 100644 index 000000000000..ce7f16b718b0 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/VmwareVmForMigrationTO.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api.to; + +/** + * TO class sent to the hypervisor layer with the information needed to handle + * VM Migrations from Vmware to KVM templates + */ +public class VmwareVmForMigrationTO { + + private String vcenter; + private String datacenter; + private String cluster; + private String username; + private String password; + private String url; + private String host; + private String vmName; + private String secondaryStorageUrl; + + public VmwareVmForMigrationTO() { + } + + public VmwareVmForMigrationTO(String vcenter, String datacenter, String cluster, String username, String password, + String url, String hostname, String vmName, String secondaryStorageUrl) { + this.vcenter = vcenter; + this.datacenter = datacenter; + this.cluster = cluster; + this.username = username; + this.password = password; + this.url = url; + this.host = hostname; + this.vmName = vmName; + this.secondaryStorageUrl = secondaryStorageUrl; + } + + public String getVcenter() { + return vcenter; + } + + public String getDatacenter() { + return datacenter; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public String getUrl() { + return url; + } + + public String getHost() { + return host; + } + + public String getVmName() { + return vmName; + } + + public String getSecondaryStorageUrl() { + return secondaryStorageUrl; + } + + public String getCluster() { + return cluster; + } +} diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenter.java b/api/src/main/java/com/cloud/dc/VmwareDatacenter.java similarity index 96% rename from plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenter.java rename to api/src/main/java/com/cloud/dc/VmwareDatacenter.java index b1f233c3606a..859ff190b655 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenter.java +++ b/api/src/main/java/com/cloud/dc/VmwareDatacenter.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.hypervisor.vmware; +package com.cloud.dc; import org.apache.cloudstack.api.Identity; import org.apache.cloudstack.api.InternalIdentity; diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 484dc9b001e7..85e748471570 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -706,6 +706,8 @@ public class EventTypes { // SystemVM public static final String EVENT_LIVE_PATCH_SYSTEMVM = "LIVE.PATCH.SYSTEM.VM"; + public static final String EVENT_REGISTER_TEMPLATE_FROM_VMWARE_VM = "REGISTER.TEMPLATE.FROM.VMWARE"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking diff --git a/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java b/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java index 2dfa707b57b7..ac1154cb63a2 100644 --- a/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java +++ b/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java @@ -104,4 +104,11 @@ boolean attachRestoredVolumeToVirtualMachine(long zoneId, String location, Backu * @return a list of commands to perform for a successful migration */ List finalizeMigrate(VirtualMachine vm, Map volumeToPool); + + /** + * Will perform a clone of a VM on an external host (if the guru can handle) + */ + HypervisorOutOfBandVMClone cloneHypervisorVMOutOfBand(String hostIp, String vmName, Map params); + + boolean removeHypervisorVMOutOfBand(String hostIp, String vmName, Map params); } diff --git a/api/src/main/java/com/cloud/hypervisor/HypervisorOutOfBandVMClone.java b/api/src/main/java/com/cloud/hypervisor/HypervisorOutOfBandVMClone.java new file mode 100644 index 000000000000..773d1059b844 --- /dev/null +++ b/api/src/main/java/com/cloud/hypervisor/HypervisorOutOfBandVMClone.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor; + +public class HypervisorOutOfBandVMClone { + + private String cloneName; + private int disksNumber; + private long capacity; + + public HypervisorOutOfBandVMClone(String cloneName, int disksNumber, long capacity) { + this.cloneName = cloneName; + this.disksNumber = disksNumber; + this.capacity = capacity; + } + + public String getCloneName() { + return cloneName; + } + + public int getDisksNumber() { + return disksNumber; + } + + public long getCapacity() { + return capacity; + } +} diff --git a/api/src/main/java/com/cloud/template/TemplateApiService.java b/api/src/main/java/com/cloud/template/TemplateApiService.java index 5b494c308c3c..0c6df616040f 100644 --- a/api/src/main/java/com/cloud/template/TemplateApiService.java +++ b/api/src/main/java/com/cloud/template/TemplateApiService.java @@ -33,6 +33,7 @@ import org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd; import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; +import org.apache.cloudstack.api.command.user.template.RegisterTemplateFromVMwareVMCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; import com.cloud.exception.InternalErrorException; @@ -47,6 +48,8 @@ public interface TemplateApiService { VirtualMachineTemplate registerTemplate(RegisterTemplateCmd cmd) throws URISyntaxException, ResourceAllocationException; + VirtualMachineTemplate registerTemplateFromVMwareVM(RegisterTemplateFromVMwareVMCmd cmd) throws ResourceAllocationException; + GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForTemplateCmd cmd) throws ResourceAllocationException, MalformedURLException; VirtualMachineTemplate registerIso(RegisterIsoCmd cmd) throws IllegalArgumentException, ResourceAllocationException; diff --git a/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java b/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java index 1f8cef0365b1..aa82d819172b 100644 --- a/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java +++ b/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java @@ -144,6 +144,8 @@ public enum TemplateFilter { boolean isDeployAsIs(); + boolean isMigratedFromVmwareVM(); + Long getUserDataId(); UserData.UserDataOverridePolicy getUserDataOverridePolicy(); diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index add2518321b4..acccd684341c 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -84,4 +84,15 @@ public interface VmDetailConstants { String DEPLOY_AS_IS_CONFIGURATION = "configurationId"; String KEY_PAIR_NAMES = "keypairnames"; String CKS_CONTROL_NODE_LOGIN_USER = "controlNodeLoginUser"; + + // VMware to KVM VM migrations specific + String VMWARE_TO_KVM_PREFIX = "vmw-to-kvm"; + String VMWARE_VCENTER = String.format("%s-vcenter", VMWARE_TO_KVM_PREFIX); + String VMWARE_DATACENTER = String.format("%s-datacenter", VMWARE_TO_KVM_PREFIX); + String VMWARE_CLUSTER = String.format("%s-cluster", VMWARE_TO_KVM_PREFIX); + String VMWARE_VCENTER_USERNAME = String.format("%s-username", VMWARE_TO_KVM_PREFIX); + String VMWARE_VCENTER_PASSWORD = String.format("%s-password", VMWARE_TO_KVM_PREFIX); + String VMWARE_VM_NAME = String.format("%s-vmname", VMWARE_TO_KVM_PREFIX); + String VMWARE_HOST = String.format("%s-host", VMWARE_TO_KVM_PREFIX); + String VMWARE_DISK = String.format("%s-disk", VMWARE_TO_KVM_PREFIX); } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 3e0e65220e17..38e83136b624 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -118,6 +118,7 @@ public class ApiConstants { public static final String MIN_IOPS = "miniops"; public static final String MAX_IOPS = "maxiops"; public static final String HYPERVISOR_SNAPSHOT_RESERVE = "hypervisorsnapshotreserve"; + public static final String DATACENTER_NAME = "datacentername"; public static final String DATADISK_OFFERING_LIST = "datadiskofferinglist"; public static final String DEFAULT_VALUE = "defaultvalue"; public static final String DESCRIPTION = "description"; @@ -205,6 +206,7 @@ public class ApiConstants { public static final String HIDE_IP_ADDRESS_USAGE = "hideipaddressusage"; public static final String HOST_ID = "hostid"; public static final String HOST_IDS = "hostids"; + public static final String HOST_IP = "hostip"; public static final String HOST_NAME = "hostname"; public static final String HOST_CONTROL_STATE = "hostcontrolstate"; public static final String HOSTS_MAP = "hostsmap"; @@ -768,6 +770,7 @@ public class ApiConstants { public static final String VSM_CONFIG_STATE = "vsmconfigstate"; public static final String VSM_DEVICE_STATE = "vsmdevicestate"; public static final String VCENTER = "vcenter"; + public static final String EXISTING_VCENTER_ID = "existingvcenterid"; public static final String ADD_VSM_FLAG = "addvsmflag"; public static final String END_POINT = "endpoint"; public static final String REGION_ID = "regionid"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index 5709e3ed9007..c8fc56a6186a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -107,7 +107,6 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { @Parameter(name = ApiConstants.URL, type = CommandType.STRING, - required = true, length = 2048, description = "the URL of where the template is hosted. Possible URL include http:// and https://") private String url; @@ -331,6 +330,11 @@ public void execute() throws ResourceAllocationException { } protected void validateParameters() { + if (StringUtils.isBlank(url)) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + String.format("Missing parameter: %s", ApiConstants.URL)); + } + if ((zoneId != null) && (zoneIds != null && !zoneIds.isEmpty())) throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Both zoneid and zoneids cannot be specified at the same time"); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateFromVMwareVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateFromVMwareVMCmd.java new file mode 100644 index 000000000000..f66e5d3c24e2 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateFromVMwareVMCmd.java @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.template; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.template.VirtualMachineTemplate; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.cloudstack.api.response.VmwareDatacenterResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; + +@APICommand(name = "registerTemplateFromVmware", + description = "Registers a template from a stopped VM in an existing VMware vCenter.", + responseObject = TemplateResponse.class, responseView = ResponseObject.ResponseView.Restricted, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class RegisterTemplateFromVMwareVMCmd extends RegisterTemplateCmd { + + @Parameter(name = ApiConstants.HOST_IP, + type = BaseCmd.CommandType.STRING, + description = "VMware ESXi host IP/Name.") + private String host; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_NAME, + type = BaseCmd.CommandType.STRING, + description = "VMware VM Name.") + private String vmName; + + @Parameter(name = ApiConstants.ZONE_ID, + type = CommandType.UUID, + entityType = ZoneResponse.class, + description = "KVM zone where the end result template is registered") + private Long zoneId; + + @Parameter(name = ApiConstants.EXISTING_VCENTER_ID, + type = CommandType.UUID, + entityType = VmwareDatacenterResponse.class, + description = "UUID of a linked existing vCenter") + private Long existingVcenterId; + + @Parameter(name = ApiConstants.VCENTER, + type = CommandType.STRING, + description = "The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.") + private String vcenter; + + @Parameter(name = ApiConstants.DATACENTER_NAME, type = CommandType.STRING, + description = "Name of VMware datacenter.") + private String datacenterName; + + @Parameter(name = ApiConstants.CLUSTER_NAME, type = CommandType.STRING, + description = "Name of VMware cluster.") + private String clusterName; + + @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, + description = "The Username required to connect to resource.") + private String username; + + @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, + description = "The password for the specified username.") + private String password; + + public String getVcenter() { + return vcenter; + } + + public String getHost() { + return host; + } + + public String getVmName() { + return vmName; + } + + public Long getZoneId() { + return zoneId; + } + + public String getDatacenterName() { + return datacenterName; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public String getClusterName() { + return clusterName; + } + + public Long getExistingVcenterId() { + return existingVcenterId; + } + + @Override + protected void validateParameters() { + if ((existingVcenterId == null && vcenter == null) || (existingVcenterId != null && vcenter != null)) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Please provide an existing vCenter ID or a vCenter IP/Name, parameters are mutually exclusive"); + } + if (existingVcenterId == null && StringUtils.isAnyBlank(vcenter, datacenterName, username, password)) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Please set all the information for a vCenter IP/Name, datacenter, username and password"); + } + } + + @Override + public void execute() throws ServerApiException, ConcurrentOperationException, ResourceAllocationException { + validateParameters(); + VirtualMachineTemplate template = _templateService.registerTemplateFromVMwareVM(this); + if (template != null) { + ListResponse response = new ListResponse<>(); + List templateResponses = _responseGenerator.createTemplateResponses(ResponseObject.ResponseView.Restricted, + template, zoneId, false); + response.setResponses(templateResponses); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to register template"); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } +} diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java similarity index 97% rename from plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java rename to api/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java index ddd4a364a0a1..3cf06f382428 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java @@ -23,7 +23,7 @@ import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; -import com.cloud.hypervisor.vmware.VmwareDatacenter; +import com.cloud.dc.VmwareDatacenter; import com.cloud.serializer.Param; @EntityReference(value = VmwareDatacenter.class) diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java index 29d737fcce93..2d866ada2297 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.storage.command; +import com.cloud.agent.api.to.VmwareVmForMigrationTO; import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; @@ -47,6 +48,7 @@ public static enum ResourceType { private String installPath; private DataStoreTO _store; private DataStoreTO cacheStore; + private VmwareVmForMigrationTO vmwareVmForMigrationTO; protected DownloadCommand() { } @@ -181,4 +183,12 @@ public void setCacheStore(DataStoreTO cacheStore) { public DataStoreTO getCacheStore() { return cacheStore; } + + public VmwareVmForMigrationTO getVmwareVmForMigrationTO() { + return vmwareVmForMigrationTO; + } + + public void setVmwareVmForMigrationTO(VmwareVmForMigrationTO vmwareVmForMigrationTO) { + this.vmwareVmForMigrationTO = vmwareVmForMigrationTO; + } } diff --git a/debian/control b/debian/control index 7c1ff8e0b1b3..6d5ed372a681 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${python:Depends}, ${python3:Depends}, openjdk-11-jre-headless | java11-runtime-headless | java11-runtime | openjdk-11-jre-headless | zulu-11, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, lsb-release, aria2, ufw, apparmor +Depends: ${python:Depends}, ${python3:Depends}, openjdk-11-jre-headless | java11-runtime-headless | java11-runtime | openjdk-11-jre-headless | zulu-11, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, lsb-release, virt-v2v, ufw, apparmor Recommends: init-system-helpers Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 28e7c89419a3..dce9e93056f6 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -96,6 +96,14 @@ public interface StorageManager extends StorageService { true, ConfigKey.Scope.Global, null); + ConfigKey KvmTemplateFromVmwareVmMigrationWait = new ConfigKey<>(Integer.class, + "kvm.template.from.vmware.vm.migration.wait", + "Storage", + "7200000", + "Timeout in milliseconds for virt-v2v migration from VMware VM to secondary storage completes on KVM hosts", + true, + ConfigKey.Scope.Global, + null); ConfigKey KvmAutoConvergence = new ConfigKey<>(Boolean.class, "kvm.auto.convergence", "Storage", diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 52bcf5033af4..351178dd382a 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -38,6 +38,8 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.serializer.GsonHelper; +import com.cloud.storage.VMTemplateDetailVO; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; import org.apache.cloudstack.api.command.admin.vm.MigrateVMCmd; @@ -897,8 +899,8 @@ private DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering Long size = _tmpltMgr.getTemplateSize(template.getId(), vm.getDataCenterId()); if (rootDisksize != null) { - if (template.isDeployAsIs()) { - // Volume size specified from template deploy-as-is + if (template.isDeployAsIs() || template.isMigratedFromVmwareVM()) { + // Volume size specified from template deploy-as-is and VMware-migrated VMs size = rootDisksize; } else { rootDisksize = rootDisksize * 1024 * 1024 * 1024; @@ -980,6 +982,7 @@ public List allocateTemplatedVolumes(Type type, String name, DiskOf List templateAsIsDisks = null; String configurationId = null; boolean deployVmAsIs = false; + boolean migratedFromVmwareVm = false; if (template.isDeployAsIs() && vm.getType() != VirtualMachine.Type.SecondaryStorageVm) { List runningSSVMs = secondaryStorageVmDao.getSecStorageVmListInStates(null, vm.getDataCenterId(), State.Running); if (CollectionUtils.isEmpty(runningSSVMs)) { @@ -1000,6 +1003,10 @@ public List allocateTemplatedVolumes(Type type, String name, DiskOf volumesNumber = templateAsIsDisks.size(); deployVmAsIs = true; } + } else if (template.isMigratedFromVmwareVM()) { + templateAsIsDisks = getMigratedVmToTemplateDisks(template, rootDisksize); + volumesNumber = templateAsIsDisks.size(); + migratedFromVmwareVm = true; } if (volumesNumber < 1) { @@ -1008,19 +1015,24 @@ public List allocateTemplatedVolumes(Type type, String name, DiskOf List profiles = new ArrayList<>(); + VirtualMachineTemplate baseTemplate = template; for (int number = 0; number < volumesNumber; number++) { String volumeName = name; Long volumeSize = rootDisksize; long deviceId = type.equals(Type.ROOT) ? 0L : 1L; - if (deployVmAsIs) { + if (deployVmAsIs || migratedFromVmwareVm) { int volumeNameSuffix = templateAsIsDisks.get(number).getDiskNumber(); volumeName = String.format("%s-%d", volumeName, volumeNameSuffix); volumeSize = templateAsIsDisks.get(number).getVirtualSize(); deviceId = templateAsIsDisks.get(number).getDiskNumber(); + if (migratedFromVmwareVm) { + Long childTemplateId = templateAsIsDisks.get(number).getTemplateId(); + baseTemplate = tmplFactory.getTemplate(childTemplateId, DataStoreRole.Image); + } } s_logger.info(String.format("Adding disk object [%s] to VM [%s]", volumeName, vm)); DiskProfile diskProfile = allocateTemplatedVolume(type, volumeName, offering, volumeSize, minIops, maxIops, - template, vm, owner, deviceId, configurationId); + baseTemplate, vm, owner, deviceId, configurationId); profiles.add(diskProfile); } @@ -1030,6 +1042,29 @@ public List allocateTemplatedVolumes(Type type, String name, DiskOf return profiles; } + protected List getMigratedVmToTemplateDisks(VirtualMachineTemplate template, Long rootDisksize) { + List disks = new ArrayList<>(); + DatadiskTO rootDisk = new DatadiskTO(); + rootDisk.setVirtualSize(rootDisksize); + rootDisk.setDiskNumber(0); + rootDisk.setTemplateId(template.getId()); + disks.add(rootDisk); + + List details = templateDetailsDao.listDetails(template.getId()); + if (CollectionUtils.isNotEmpty(details)) { + List diskDetails = details.stream() + .filter(x -> x.getName().startsWith(VmDetailConstants.VMWARE_DISK)) + .collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(diskDetails)) { + for (VMTemplateDetailVO detailVO : diskDetails) { + DatadiskTO disk = GsonHelper.getGson().fromJson(detailVO.getValue(), DatadiskTO.class); + disks.add(disk); + } + } + } + return disks; + } + /** * Set context information for VOLUME.CREATE event for ROOT disk. * diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterVO.java b/engine/schema/src/main/java/com/cloud/dc/VmwareDatacenterVO.java similarity index 99% rename from plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterVO.java rename to engine/schema/src/main/java/com/cloud/dc/VmwareDatacenterVO.java index 86597b2e5c93..6390d923ed83 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterVO.java +++ b/engine/schema/src/main/java/com/cloud/dc/VmwareDatacenterVO.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.hypervisor.vmware; +package com.cloud.dc; import java.util.UUID; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/VmwareDatacenterDao.java similarity index 96% rename from plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDao.java rename to engine/schema/src/main/java/com/cloud/dc/dao/VmwareDatacenterDao.java index 2754e91d26c6..1ed1f8d5a0fa 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/VmwareDatacenterDao.java @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.hypervisor.vmware.dao; +package com.cloud.dc.dao; import java.util.List; -import com.cloud.hypervisor.vmware.VmwareDatacenterVO; +import com.cloud.dc.VmwareDatacenterVO; import com.cloud.utils.db.GenericDao; public interface VmwareDatacenterDao extends GenericDao { diff --git a/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java b/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java index 44e4dc920abf..95782f3cb84d 100644 --- a/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java @@ -39,6 +39,7 @@ import com.cloud.storage.Storage.TemplateType; import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.db.GenericDao; +import org.apache.commons.lang3.StringUtils; @Entity @Table(name = "vm_template") @@ -649,6 +650,11 @@ public void setParentTemplateId(Long parentTemplateId) { return deployAsIs; } + @Override + public boolean isMigratedFromVmwareVM() { + return hypervisorType == HypervisorType.KVM && StringUtils.isNotBlank(url) && url.startsWith("vpx://"); + } + public void setDeployAsIs(boolean deployAsIs) { this.deployAsIs = deployAsIs; } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index e6235a63c779..2ed95e40a2ea 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -494,6 +494,12 @@ public void handleTemplateSync(DataStore store) { } else if (tmplt.isDirectDownload()) { s_logger.info("Template " + tmplt.getName() + ":" + tmplt.getId() + " is marked for direct download, discarding it for download on image stores"); toBeDownloaded.remove(tmplt); + } else if (tmplt.getParentTemplateId() != null && tmplt.getUrl().startsWith("vpx://") && + tmpltStore != null && tmpltStore.getDownloadState() == Status.DOWNLOADED) { + String msg = String.format("Template %s is downloaded by the parent template %s, avoiding re-download", + tmplt.getName(), tmplt.getParentTemplateId()); + s_logger.info(msg); + toBeDownloaded.remove(tmplt); } else { s_logger.info("Template Sync did not find " + uniqueName + " on image store " + storeId + ", may request download based on available hypervisor types"); if (tmpltStore != null) { diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java index b688197bfb99..51d34c699112 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java @@ -413,6 +413,14 @@ public boolean isDeployAsIs() { return this.imageVO.isDeployAsIs(); } + @Override + public boolean isMigratedFromVmwareVM() { + if (this.imageVO == null) { + return false; + } + return this.imageVO.isMigratedFromVmwareVM(); + } + public void setInstallPath(String installPath) { this.installPath = installPath; } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index 4c13759c1c17..f007fe3fd054 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -46,6 +46,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.storage.LocalHostEndpoint; import org.apache.cloudstack.storage.RemoteHostEndPoint; +import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -84,7 +85,8 @@ public class DefaultEndPointSelector implements EndPointSelector { + "left join host_details hd on h.id=hd.host_id and hd.name='" + HOST_VOLUME_ENCRYPTION + "' " + "where h.status = 'Up' and h.type = 'Routing' and h.resource_state = 'Enabled' and s.pool_id = ? "; - private String findOneHypervisorHostInScopeByType = "select h.id from host h where h.status = 'Up' and h.hypervisor_type = ? "; + private String findOneHypervisorHostInScopeByHypervisorType = "select h.id from " + + "host h where h.status = 'Up' and h.type = 'Routing' and h.resource_state = 'Enabled' and h.hypervisor_type = ? "; private String findOneHypervisorHostInScope = "select h.id from host h where h.status = 'Up' and h.hypervisor_type is not null "; protected boolean moveBetweenPrimaryImage(DataStore srcStore, DataStore destStore) { @@ -124,16 +126,15 @@ protected boolean moveBetweenImages(DataStore srcStore, DataStore destStore) { } protected EndPoint findEndPointInScope(Scope scope, String sqlBase, Long poolId) { - return findEndPointInScope(scope, sqlBase, poolId, false); + return findEndPointInScope(scope, sqlBase, poolId, null, false); } - @DB - protected EndPoint findEndPointInScope(Scope scope, String sqlBase, Long poolId, boolean volumeEncryptionSupportRequired) { - StringBuilder sbuilder = new StringBuilder(); - sbuilder.append(sqlBase); - - List dedicatedHosts = new ArrayList(); + protected EndPoint findEndPointInScopeHypervisorType(Scope scope, String sqlBase, Hypervisor.HypervisorType hypervisorType) { + return findEndPointInScope(scope, sqlBase, null, hypervisorType.toString(), false); + } + protected List getDedicatedHostsAndAppendStringFromPoolScope(Scope scope,StringBuilder sbuilder) { + List dedicatedHosts = new ArrayList<>(); if (scope != null) { if (scope.getScopeType() == ScopeType.HOST) { sbuilder.append(" and h.id = "); @@ -150,24 +151,18 @@ protected EndPoint findEndPointInScope(Scope scope, String sqlBase, Long poolId, } else { dedicatedHosts = dedicatedResourceDao.listAllHosts(); } + return dedicatedHosts; + } - sbuilder.append(") t where t.value<>'true' or t.value is null"); //Added for exclude cluster's subquery - - if (volumeEncryptionSupportRequired) { - sbuilder.append(String.format(" and t.%s='true'", VOL_ENCRYPT_COLUMN_NAME)); - } - - // TODO: order by rand() is slow if there are lot of hosts - sbuilder.append(" ORDER by "); - if (dedicatedHosts.size() > 0) { - moveDedicatedHostsToLowerPriority(sbuilder, dedicatedHosts); - } - sbuilder.append(" rand() limit 1"); - String sql = sbuilder.toString(); + protected HostVO getHostFromSqlQuery(String sql, Long poolId, String hypervisor) { HostVO host = null; TransactionLegacy txn = TransactionLegacy.currentTxn(); try (PreparedStatement pstmt = txn.prepareStatement(sql)) { - pstmt.setLong(1, poolId); + if (poolId != null) { + pstmt.setLong(1, poolId); + } else { + pstmt.setString(1, hypervisor); + } try(ResultSet rs = pstmt.executeQuery();) { while (rs.next()) { long id = rs.getLong(1); @@ -179,6 +174,36 @@ protected EndPoint findEndPointInScope(Scope scope, String sqlBase, Long poolId, } catch (SQLException e) { s_logger.warn("can't find endpoint", e); } + return host; + } + + private void appendOrderAndLimitToSql(StringBuilder sbuilder, List dedicatedHosts) { + // TODO: order by rand() is slow if there are lot of hosts + sbuilder.append(" ORDER by "); + if (dedicatedHosts.size() > 0) { + moveDedicatedHostsToLowerPriority(sbuilder, dedicatedHosts); + } + sbuilder.append(" rand() limit 1"); + } + + @DB + protected EndPoint findEndPointInScope(Scope scope, String sqlBase, Long poolId, String hypervisor, boolean volumeEncryptionSupportRequired) { + StringBuilder sbuilder = new StringBuilder(); + sbuilder.append(sqlBase); + + List dedicatedHosts = getDedicatedHostsAndAppendStringFromPoolScope(scope, sbuilder); + + if (sqlBase.equals(findOneHostOnPrimaryStorage)) { + sbuilder.append(") t where t.value<>'true' or t.value is null"); //Added for exclude cluster's subquery + } + + if (volumeEncryptionSupportRequired) { + sbuilder.append(String.format(" and t.%s='true'", VOL_ENCRYPT_COLUMN_NAME)); + } + + appendOrderAndLimitToSql(sbuilder, dedicatedHosts); + + HostVO host = getHostFromSqlQuery(sbuilder.toString(), poolId, hypervisor); if (host == null) { return null; } @@ -247,7 +272,7 @@ protected EndPoint findEndPointForImageMove(DataStore srcStore, DataStore destSt poolId = destStore.getId(); } } - return findEndPointInScope(selectedScope, findOneHostOnPrimaryStorage, poolId, volumeEncryptionSupportRequired); + return findEndPointInScope(selectedScope, findOneHostOnPrimaryStorage, poolId, null, volumeEncryptionSupportRequired); } @Override @@ -324,6 +349,12 @@ protected EndPoint findEndpointForPrimaryStorage(DataStore store) { return findEndPointInScope(store.getScope(), findOneHostOnPrimaryStorage, store.getId()); } + protected EndPoint findEndpointForVmwareVmMigration(DataObject template) { + Scope scope = template.getDataStore().getScope(); + TemplateObjectTO tmpl = (TemplateObjectTO) template.getTO(); + return findEndPointInScopeHypervisorType(scope, findOneHypervisorHostInScopeByHypervisorType, tmpl.getHypervisorType()); + } + protected EndPoint findEndpointForImageStorage(DataStore store) { Long dcId = null; Scope storeScope = store.getScope(); @@ -384,7 +415,7 @@ private List listUpAndConnectingSecondaryStorageVmHost(Long dcId) { public EndPoint select(DataObject object, boolean encryptionSupportRequired) { DataStore store = object.getDataStore(); if (store.getRole() == DataStoreRole.Primary) { - return findEndPointInScope(store.getScope(), findOneHostOnPrimaryStorage, store.getId(), encryptionSupportRequired); + return findEndPointInScope(store.getScope(), findOneHostOnPrimaryStorage, store.getId(), null, encryptionSupportRequired); } throw new CloudRuntimeException(String.format("Storage role %s doesn't support encryption", store.getRole())); } @@ -392,6 +423,9 @@ public EndPoint select(DataObject object, boolean encryptionSupportRequired) { @Override public EndPoint select(DataObject object) { DataStore store = object.getDataStore(); + if (object instanceof TemplateInfo && object.getUri().startsWith("vpx://")) { + return findEndpointForVmwareVmMigration(object); + } EndPoint ep = select(store); if (ep != null) { return ep; diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index 3ef9fbc4225e..083107fc1a71 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -24,8 +24,10 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -34,9 +36,18 @@ import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.OVFInformationTO; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.HypervisorGuru; +import com.cloud.hypervisor.HypervisorGuruManager; +import com.cloud.serializer.GsonHelper; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Storage; import com.cloud.storage.Upload; +import com.cloud.storage.VMTemplateDetailVO; +import com.cloud.storage.dao.VMTemplateDetailsDao; +import com.cloud.vm.VmDetailConstants; import org.apache.cloudstack.storage.image.deployasis.DeployAsIsHelper; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; @@ -132,6 +143,10 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { protected SecondaryStorageVmDao _secStorageVmDao; @Inject AgentManager agentMgr; + @Inject + VMTemplateDetailsDao templateDetailsDao; + @Inject + HypervisorGuruManager hypervisorGuruManager; protected String _proxy = null; @@ -220,6 +235,20 @@ protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher disks = ovfInformationTO.getDisks(); + if (CollectionUtils.isNotEmpty(disks)) { + persistTemplateDisksAsChildrenFromMigratedVmwareVm(template.getId(), disks, store.getId()); + } + } + List details = templateDetailsDao.listDetails(template.getId()); + if (CollectionUtils.isNotEmpty(details)) { + Map params = createRemoveDetails(details); + HypervisorGuru vmwareGuru = hypervisorGuruManager.getGuru(Hypervisor.HypervisorType.VMware); + vmwareGuru.removeHypervisorVMOutOfBand(params.get(VmDetailConstants.VMWARE_HOST), params.get(VmDetailConstants.VMWARE_VM_NAME), params); + } } LOGGER.info("Updating store ref entry for template " + template.getName()); TemplateDataStoreVO updateBuilder = _templateStoreDao.createForUpdate(); @@ -263,6 +292,68 @@ protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher createRemoveDetails(List details) { + Map params = new HashMap<>(); + List keys = Arrays.asList(VmDetailConstants.VMWARE_VCENTER, VmDetailConstants.VMWARE_DATACENTER, + VmDetailConstants.VMWARE_VCENTER_USERNAME, VmDetailConstants.VMWARE_VCENTER_PASSWORD, + VmDetailConstants.VMWARE_HOST, VmDetailConstants.VMWARE_VM_NAME); + for (VMTemplateDetailVO detail : details) { + if (keys.contains(detail.getName())) { + params.put(detail.getName(), detail.getValue()); + } + } + return params; + } + + protected void persistTemplateDisksAsChildrenFromMigratedVmwareVm(long templateId, List disks, + long storeId) { + LOGGER.debug(String.format("Persisting template children for template: %s", templateId)); + for (DatadiskTO disk : disks) { + VMTemplateVO childTemplate = createChildTemplateChildEntry(templateId, disk); + createChildTemplateStoreRefEntry(childTemplate, disk, storeId); + persistDiskDetailsOnParentTemplate(templateId, disk, childTemplate.getId()); + } + } + + protected void persistDiskDetailsOnParentTemplate(long templateId, DatadiskTO disk, Long childTemplateId) { + LOGGER.debug(String.format("Persisting disk detail on parent template: %s", templateId)); + disk.setTemplateId(childTemplateId); + String value = GsonHelper.getGson().toJson(disk); + String key = String.format("%s-%d", VmDetailConstants.VMWARE_DISK, disk.getDiskNumber()); + VMTemplateDetailVO detailVO = new VMTemplateDetailVO(templateId, key, value, false); + templateDetailsDao.persist(detailVO); + } + + protected void createChildTemplateStoreRefEntry(VMTemplateVO childTemplate, DatadiskTO disk, + long storeId) { + LOGGER.debug(String.format("Persisting template child store ref for template: %s for disk %s in store %s", + childTemplate.getId(), disk.getDiskNumber(), storeId)); + TemplateDataStoreVO tmpltStore = new TemplateDataStoreVO(storeId, childTemplate.getId(), new Date(), 100, + VMTemplateStorageResourceAssoc.Status.DOWNLOADED, null, null, + null, disk.getPath(), childTemplate.getUrl()); + tmpltStore.setSize(disk.getVirtualSize()); + tmpltStore.setPhysicalSize(disk.getFileSize()); + tmpltStore.setDataStoreRole(DataStoreRole.Image); + _templateStoreDao.persist(tmpltStore); + } + + protected VMTemplateVO createChildTemplateChildEntry(long templateId, DatadiskTO disk) { + LOGGER.debug(String.format("Persisting template child for template: %s for disk %s", templateId, disk.getDiskNumber())); + VMTemplateVO parentTemplate = _templateDao.findById(templateId); + Storage.ImageFormat format = parentTemplate.getFormat(); + String suffix = "DataDiskTemplate"; + Storage.TemplateType templateType = Storage.TemplateType.DATADISK; + final long childTemplateId = _templateDao.getNextInSequence(Long.class, "id"); + long guestOsId = 0; + String templateName = String.format("%s-%s-%d", parentTemplate.getName(), suffix, disk.getDiskNumber()); + VMTemplateVO childTemplate = new VMTemplateVO(childTemplateId, templateName, format, false, false, false, templateType, parentTemplate.getUrl(), + parentTemplate.requiresHvm(), parentTemplate.getBits(), parentTemplate.getAccountId(), null, templateName, false, guestOsId, false, parentTemplate.getHypervisorType(), null, + null, false, false, false, false); + childTemplate.setParentTemplateId(parentTemplate.getId()); + childTemplate.setSize(disk.getVirtualSize()); + return _templateDao.persist(childTemplate); + } + protected Void createVolumeAsyncCallback(AsyncCallbackDispatcher callback, CreateContext context) { DownloadAnswer answer = callback.getResult(); diff --git a/packaging/centos7/cloud.spec b/packaging/centos7/cloud.spec index 9a1cf92ce897..aa1bfb686daf 100644 --- a/packaging/centos7/cloud.spec +++ b/packaging/centos7/cloud.spec @@ -124,6 +124,7 @@ Requires: qemu-img Requires: qemu-kvm Requires: cryptsetup Requires: rng-tools +Requires: virt-v2v Provides: cloud-agent Group: System Environment/Libraries %description agent diff --git a/packaging/centos8/cloud.spec b/packaging/centos8/cloud.spec index 8d567f217ce7..6762d38cff49 100644 --- a/packaging/centos8/cloud.spec +++ b/packaging/centos8/cloud.spec @@ -116,6 +116,7 @@ Requires: cryptsetup Requires: rng-tools Requires: (libgcrypt > 1.8.3 or libgcrypt20) Requires: (selinux-tools if qemu-tools) +Requires: virt-v2v Provides: cloud-agent Group: System Environment/Libraries %description agent diff --git a/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java b/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java index 966c83722ec9..f012dbfa9725 100644 --- a/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java +++ b/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java @@ -35,9 +35,9 @@ import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; -import com.cloud.hypervisor.vmware.VmwareDatacenterVO; +import com.cloud.dc.VmwareDatacenterVO; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMapVO; -import com.cloud.hypervisor.vmware.dao.VmwareDatacenterDao; +import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; import com.cloud.hypervisor.vmware.mo.VirtualMachineDiskInfoBuilder; import com.cloud.storage.Storage.StoragePoolType; diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index 02f08d602bb9..c0091e470612 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -41,9 +41,9 @@ import org.apache.log4j.Logger; import com.cloud.hypervisor.Hypervisor; -import com.cloud.hypervisor.vmware.VmwareDatacenter; +import com.cloud.dc.VmwareDatacenter; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap; -import com.cloud.hypervisor.vmware.dao.VmwareDatacenterDao; +import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java index 8265f951f8a8..f01f340ce934 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java @@ -28,6 +28,7 @@ import com.cloud.event.UsageEventVO; import com.cloud.exception.ResourceAllocationException; import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.HypervisorOutOfBandVMClone; import com.cloud.resource.ResourceManager; import com.cloud.storage.TemplateProfile; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; @@ -43,6 +44,7 @@ import org.apache.cloudstack.api.command.user.iso.GetUploadParamsForIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; +import org.apache.cloudstack.api.command.user.template.RegisterTemplateFromVMwareVMCmd; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.log4j.Logger; @@ -50,6 +52,7 @@ import javax.inject.Inject; import java.util.Date; import java.util.List; +import java.util.Map; public class BareMetalTemplateAdapter extends TemplateAdapterBase implements TemplateAdapter { private final static Logger s_logger = Logger.getLogger(BareMetalTemplateAdapter.class); @@ -78,6 +81,12 @@ public TemplateProfile prepare(GetUploadParamsForIsoCmd cmd) throws ResourceAllo throw new CloudRuntimeException("Baremetal doesn't support ISO template"); } + @Override + public TemplateProfile prepareTemplateFromVmwareMigration(RegisterTemplateFromVMwareVMCmd cmd, + Map params, HypervisorOutOfBandVMClone clone) throws ResourceAllocationException { + throw new CloudRuntimeException("Baremetal doesn't support registering a template from a VMware VM"); + } + private void templateCreateUsage(VMTemplateVO template, long dcId) { if (template.getAccountId() != Account.ACCOUNT_ID_SYSTEM) { UsageEventVO usageEvent = diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDownloadCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDownloadCommandWrapper.java new file mode 100644 index 000000000000..ee0b94a2538f --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDownloadCommandWrapper.java @@ -0,0 +1,230 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.storage.DownloadAnswer; +import com.cloud.agent.api.to.DatadiskTO; +import com.cloud.agent.api.to.OVFInformationTO; +import com.cloud.agent.api.to.VmwareVmForMigrationTO; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.commons.io.IOUtils; +import org.apache.log4j.Logger; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +@ResourceWrapper(handles = DownloadCommand.class) +public class LibvirtDownloadCommandWrapper extends CommandWrapper { + + private static final Logger s_logger = Logger.getLogger(LibvirtDownloadCommandWrapper.class); + + @Override + public Answer execute(DownloadCommand cmd, LibvirtComputingResource serverResource) { + s_logger.info("Executing DownloadCommand"); + final KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); + final long templateId = cmd.getId(); + final long accountId = cmd.getAccountId(); + String templateDescription = cmd.getDescription(); + String templateUniqueName = cmd.getName(); + + VmwareVmForMigrationTO vmForMigrationTO = cmd.getVmwareVmForMigrationTO(); + String vcenter = vmForMigrationTO.getVcenter(); + String datacenter = vmForMigrationTO.getDatacenter(); + String username = vmForMigrationTO.getUsername(); + String password = vmForMigrationTO.getPassword(); + String host = vmForMigrationTO.getHost(); + String vmName = vmForMigrationTO.getVmName(); + String cluster = vmForMigrationTO.getCluster(); + String secondaryStorageUrl = vmForMigrationTO.getSecondaryStorageUrl(); + + KVMStoragePool secondaryPool = storagePoolMgr.getStoragePoolByURI(secondaryStorageUrl); + String mountPoint = secondaryPool.getLocalPath(); + s_logger.info(String.format("Secondary storage pool: uuid = %s, local path = %s", secondaryPool.getUuid(), mountPoint)); + + String encodedUsername = encodeUsername(username); + s_logger.info(String.format("Encoded username: %s", encodedUsername)); + String url = String.format("vpx://%s@%s/%s/%s/%s?no_verify=1", + encodedUsername, vcenter, datacenter, cluster, host); + + s_logger.info(String.format("Migrating VM from vCenter: %s", vcenter)); + + String passwordFile = String.format("/tmp/vmw-%s", UUID.randomUUID()); + Script.runSimpleBashScript(String.format("echo \"%s\" > %s", password, passwordFile)); + + final String templateFolder = String.format("template/tmpl/%s/%s/", accountId, templateId); + final String templateInstallFolder = String.format("%s/%s", mountPoint, templateFolder); + final String baseName = UUID.randomUUID().toString(); + final String installBasePath = templateInstallFolder + baseName; + s_logger.info(String.format("Creating template folder %s", templateInstallFolder)); + Script.runSimpleBashScript(String.format("mkdir -p %s", templateInstallFolder)); + + try { + int timeout = cmd.getWait(); + Script script = new Script("virt-v2v", timeout, s_logger); + script.add("--root", "first"); + script.add("-ic", url); + script.add(vmName); + script.add("--password-file", passwordFile); + script.add("-o", "local"); + script.add("-os", templateInstallFolder); + script.add("-of", "qcow2"); + script.add("-on", baseName); + String result = script.execute(); + s_logger.info(String.format("Execution result: %s", result)); + + s_logger.info("Finished downloading template from Vmware migration, checking for template sizes"); + List disks = getConvertedVmDisks(installBasePath); + if (disks.size() < 1) { + String err = String.format("Cannot find any disk for the migrated VM %s on path: %s", vmName, installBasePath); + s_logger.error(err); + return new DownloadAnswer(err, VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + } + + List vmDisks = getVMDisksFromDiskDefList(disks, mountPoint); + List dataDisks = getVMDataDisks(vmDisks); + DatadiskTO rootDisk = vmDisks.get(0); + String installPath = rootDisk.getPath(); + long virtualSize = rootDisk.getVirtualSize(); + long physicalSize = rootDisk.getFileSize(); + + createTemplatePropertiesFileAfterMigration(templateInstallFolder, templateId, accountId, + templateDescription, installPath, virtualSize, physicalSize, templateUniqueName); + + DownloadAnswer answer = new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, + null, installPath, virtualSize, physicalSize, null); + answer.setOvfInformationTO(new OVFInformationTO(dataDisks)); + return answer; + } catch (Exception e) { + String error = String.format("Error migrating VM from vcenter %s, %s", vcenter, e.getMessage()); + s_logger.error(error, e); + return new DownloadAnswer(error, VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + } finally { + Script.runSimpleBashScript(String.format("rm -rf %s", passwordFile)); + } + } + + private void createTemplatePropertiesFileAfterMigration(String templateInstallFolder, long templateId, + long accountId, String templateDescription, + String installPath, long virtualSize, + long physicalSize, String templateUniqueName) throws IOException { + String templateFilePath = String.format("%s%s", templateInstallFolder, "template.properties"); + final File templateProp = new File(templateFilePath); + if (!templateProp.exists()) { + templateProp.createNewFile(); + } + String pathPrefix = String.format("template/tmpl/%s/%s", accountId, templateId); + String relativePath = removeMountPointFromDiskPath(installPath, pathPrefix); + + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(String.format("%s=%s%s", "uniquename", templateUniqueName, System.getProperty("line.separator"))); + stringBuilder.append(String.format("%s=%s%s", "id", templateId, System.getProperty("line.separator"))); + stringBuilder.append(String.format("%s=%s%s", "description", templateDescription, System.getProperty("line.separator"))); + stringBuilder.append(String.format("%s=%s%s", "filename", relativePath, System.getProperty("line.separator"))); + stringBuilder.append(String.format("%s=%s%s", "qcow2.filename", relativePath, System.getProperty("line.separator"))); + stringBuilder.append(String.format("%s=%s%s", "size", physicalSize, System.getProperty("line.separator"))); + stringBuilder.append(String.format("%s=%s%s", "qcow2.size", physicalSize, System.getProperty("line.separator"))); + stringBuilder.append(String.format("%s=%s%s", "virtualsize", virtualSize, System.getProperty("line.separator"))); + stringBuilder.append(String.format("%s=%s%s", "qcow2.virtualsize", virtualSize, System.getProperty("line.separator"))); + stringBuilder.append(String.format("%s=%s%s", "qcow2", "true", System.getProperty("line.separator"))); + String fileContent = stringBuilder.toString(); + + try(FileOutputStream templFo = new FileOutputStream(templateProp)){ + templFo.write(fileContent.getBytes()); + templFo.flush(); + } catch (final IOException e) { + String err = String.format("Cannot create template.properties file for template %s", templateId); + s_logger.error(err, e); + } + } + + /** + * Exclude the root disk (first element in the list) and return the rest of the disks + */ + protected List getVMDataDisks(List vmDisks) { + List dataDisks = new ArrayList<>(); + if (vmDisks.size() > 1) { + for (int index = 1; index < vmDisks.size(); index++) { + DatadiskTO datadiskTO = vmDisks.get(index); + dataDisks.add(datadiskTO); + } + } + return dataDisks; + } + + protected List getVMDisksFromDiskDefList(List disks, String mountPoint) { + List datadiskTOList = new ArrayList<>(); + int deviceId = 0; + for (LibvirtVMDef.DiskDef disk : disks) { + String diskPath = disk.getDiskPath(); + String relativePath = removeMountPointFromDiskPath(diskPath, mountPoint); + long virtualSize = LibvirtUtilitiesHelper.getVirtualSizeFromFile(diskPath); + long physicalSize = new File(diskPath).length(); + + DatadiskTO datadiskTO = new DatadiskTO(relativePath, virtualSize, physicalSize, deviceId); + datadiskTOList.add(datadiskTO); + deviceId++; + } + return datadiskTOList; + } + + protected String removeMountPointFromDiskPath(String diskPath, String mountPoint) { + if (!diskPath.startsWith(mountPoint)) { + String err = String.format("Expected the mount point %s to be present on the disk path %s", mountPoint, diskPath); + throw new CloudRuntimeException(err); + } + return diskPath.substring(mountPoint.length() + 1); + } + + protected List getConvertedVmDisks(String installPath) throws IOException { + String xmlPath = String.format("%s.xml", installPath); + InputStream is = new BufferedInputStream(new FileInputStream(xmlPath)); + String xml = IOUtils.toString(is, Charset.defaultCharset()); + return getDisksFromXml(xml); + } + + protected List getDisksFromXml(String xml) { + final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); + parser.parseDomainXML(xml); + return parser.getDisks(); + } + + protected String encodeUsername(String username) { + return URLEncoder.encode(username, Charset.defaultCharset()); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java index 36ff69d83af5..1c59e608ce08 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java @@ -25,7 +25,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Map; import com.cloud.hypervisor.kvm.storage.ScaleIOStorageAdaptor; import org.apache.cloudstack.utils.cryptsetup.KeyFile; @@ -33,7 +32,6 @@ import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import org.apache.cloudstack.utils.qemu.QemuImgException; -import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.cloudstack.utils.qemu.QemuObject; import org.apache.log4j.Logger; import org.libvirt.Connect; @@ -97,7 +95,7 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR newSize = ScaleIOStorageAdaptor.getUsableBytesFromRawBytes(newSize); } else if (spool.getType().equals(StoragePoolType.PowerFlex)) { // PowerFlex RAW/LUKS is already resized, we just notify the domain based on new size (considering LUKS overhead) - newSize = getVirtualSizeFromFile(path); + newSize = LibvirtUtilitiesHelper.getVirtualSizeFromFile(path); } if (pool.getType() != StoragePoolType.RBD && pool.getType() != StoragePoolType.Linstor && pool.getType() != StoragePoolType.PowerFlex) { @@ -210,19 +208,4 @@ private void resizeEncryptedQcowFile(final KVMPhysicalDisk vol, final QemuObject Arrays.fill(passphrase, (byte) 0); } } - - private long getVirtualSizeFromFile(String path) { - try { - QemuImg qemu = new QemuImg(0); - QemuImgFile qemuFile = new QemuImgFile(path); - Map info = qemu.info(qemuFile); - if (info.containsKey(QemuImg.VIRTUAL_SIZE)) { - return Long.parseLong(info.get(QemuImg.VIRTUAL_SIZE)); - } else { - throw new CloudRuntimeException("Unable to determine virtual size of volume at path " + path); - } - } catch (QemuImgException | LibvirtException ex) { - throw new CloudRuntimeException("Error when inspecting volume at path " + path, ex); - } - } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelper.java index 86ab024cbb9b..45038b551a67 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelper.java @@ -22,6 +22,10 @@ import javax.naming.ConfigurationException; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.log4j.Logger; import org.libvirt.Connect; import org.libvirt.LibvirtException; @@ -144,4 +148,43 @@ public static boolean isLibvirtSupportingFlagDeleteOnCommandVirshBlockcommit(Con result.second() ? "" : " does not")); return result.second(); } + + private static Map getQemuInfo(String path) { + try { + QemuImg qemu = new QemuImg(0); + QemuImgFile qemuFile = new QemuImgFile(path); + return qemu.info(qemuFile); + } catch (QemuImgException | LibvirtException e) { + String err = String.format("Error when inspecting volume at path %s", path); + s_logger.error(err, e); + throw new CloudRuntimeException(err, e); + } + } + + protected static long[] getSizesFromQemuInfo(Map info, String path, String... sizeKeys) { + long[] sizes = new long[sizeKeys.length]; + int i = 0; + for (String sizeKey : sizeKeys) { + if (info.containsKey(sizeKey)) { + long size = Long.parseLong(info.get(sizeKey)); + sizes[i] = size; + } else { + String err = String.format("Unable to determine %s of volume at path %s", sizeKey, path); + s_logger.error(err); + throw new CloudRuntimeException(err); + } + i++; + } + return sizes; + } + + protected static long getSingleSizeValueFromSizes(String path, String sizeKey) { + Map info = getQemuInfo(path); + long[] sizes = getSizesFromQemuInfo(info, path, sizeKey); + return sizes[0]; + } + + public static long getVirtualSizeFromFile(String path) { + return getSingleSizeValueFromSizes(path, QemuImg.VIRTUAL_SIZE); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java index 1cd63b9b5661..873747a39058 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java @@ -45,6 +45,7 @@ public class QemuImg { public static final String FILE_FORMAT = "file_format"; public static final String IMAGE = "image"; public static final String VIRTUAL_SIZE = "virtual_size"; + public static final String DISK_SIZE = "disk_size"; public static final String ENCRYPT_FORMAT = "encrypt.format"; public static final String ENCRYPT_KEY_SECRET = "encrypt.key-secret"; public static final String TARGET_ZERO_FLAG = "--target-is-zero"; diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDownloadCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDownloadCommandWrapperTest.java new file mode 100644 index 000000000000..ef2ac8bf8cbb --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDownloadCommandWrapperTest.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.to.DatadiskTO; +import org.apache.commons.collections.CollectionUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +public class LibvirtDownloadCommandWrapperTest { + + private final LibvirtDownloadCommandWrapper wrapper = new LibvirtDownloadCommandWrapper(); + + private static final long accountId = 2; + private static final long templateId = 202; + private static final String mountPoint = String.format("/mnt/%s", UUID.randomUUID()); + private static final String templateRelativePath = String.format("template/tmpl/%s/%s", accountId, templateId); + + @Test + public void testRemoveMountPointFromDiskPath() { + String basePath = String.format("%s/%s", mountPoint, templateRelativePath); + String disk1Name = "disk1"; + String path1 = String.format("%s/%s", basePath, disk1Name); + String relativePath = wrapper.removeMountPointFromDiskPath(path1, mountPoint); + Assert.assertFalse(relativePath.contains(mountPoint)); + String expectedPath = String.format("%s/%s", templateRelativePath, disk1Name); + Assert.assertEquals(expectedPath, relativePath); + } + + @Test + public void testRemoveMountPointFromDiskPathTemplateDir() { + String disk1Name = "disk1"; + String path1 = String.format("%s/%s", templateRelativePath, disk1Name); + String relativePath = wrapper.removeMountPointFromDiskPath(path1, templateRelativePath); + Assert.assertFalse(relativePath.contains(templateRelativePath)); + Assert.assertEquals(disk1Name, relativePath); + } + + @Test + public void testVMDataDisksMultidisk() { + String basePath = String.format("%s/%s", mountPoint, templateRelativePath); + String path1 = String.format("%s/%s", basePath, "disk1"); + String path2 = String.format("%s/%s", basePath, "disk2"); + long size1 = 111111; + long fileSize1 = 111111111; + long size2 = 222222; + long fileSize2 = 22222222; + + DatadiskTO disk1 = new DatadiskTO(path1, size1, fileSize1, 0); + DatadiskTO disk2 = new DatadiskTO(path2, size2, fileSize2, 1); + List vmDisks = Arrays.asList(disk1, disk2); + List dataDisks = wrapper.getVMDataDisks(vmDisks); + Assert.assertEquals(1, dataDisks.size()); + Assert.assertEquals(dataDisks.get(0).getPath(), path2); + } + + @Test + public void testVMDataDisksSingleDisk() { + String basePath = String.format("%s/%s", mountPoint, templateRelativePath); + String path1 = String.format("%s/%s", basePath, "disk1"); + long size1 = 111111; + long fileSize1 = 111111111; + + DatadiskTO disk1 = new DatadiskTO(path1, size1, fileSize1, 0); + List vmDisks = List.of(disk1); + List dataDisks = wrapper.getVMDataDisks(vmDisks); + Assert.assertTrue(CollectionUtils.isEmpty(dataDisks)); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelperTest.java index f18f3f83a12f..c0b24cccd3a3 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelperTest.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.hypervisor.kvm.resource.wrapper; +import org.apache.cloudstack.utils.qemu.QemuImg; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -29,6 +30,8 @@ import junit.framework.TestCase; +import java.util.Map; + @RunWith(MockitoJUnitRunner.class) public class LibvirtUtilitiesHelperTest extends TestCase { @@ -37,6 +40,12 @@ public class LibvirtUtilitiesHelperTest extends TestCase { @Mock Connect connectMock; + private static final long virtualSize = 123456; + private static final long diskSize = 111111; + private final Map info = Map.of( + QemuImg.VIRTUAL_SIZE, String.valueOf(virtualSize), + QemuImg.DISK_SIZE, String.valueOf(diskSize)); + @Test public void validateIsLibvirtVersionEqualOrHigherThanVersionInParameterExceptionOnRetrievingLibvirtVersionReturnsFalse() throws LibvirtException { Mockito.doThrow(LibvirtException.class).when(connectMock).getLibVirVersion(); @@ -75,4 +84,26 @@ public void validateIsLibvirtVersionEqualOrHigherThanVersionInParameterLibvirtVe Assert.assertEquals(String.valueOf(libvirtVersion), result.first()); Assert.assertTrue(result.second()); } + + @Test + public void testGetSizesFromQemuInfoVirtualSize() { + long[] sizes = LibvirtUtilitiesHelper.getSizesFromQemuInfo(info, "path", QemuImg.VIRTUAL_SIZE); + Assert.assertEquals(1, sizes.length); + Assert.assertEquals(virtualSize, sizes[0]); + } + + @Test + public void testGetSizesFromQemuInfoDiskSize() { + long[] sizes = LibvirtUtilitiesHelper.getSizesFromQemuInfo(info, "path", QemuImg.DISK_SIZE); + Assert.assertEquals(1, sizes.length); + Assert.assertEquals(diskSize, sizes[0]); + } + + @Test + public void testGetSizesFromQemuInfoMultipleSizes() { + long[] sizes = LibvirtUtilitiesHelper.getSizesFromQemuInfo(info, "path", QemuImg.VIRTUAL_SIZE, QemuImg.DISK_SIZE); + Assert.assertEquals(2, sizes.length); + Assert.assertEquals(virtualSize, sizes[0]); + Assert.assertEquals(diskSize, sizes[1]); + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index aa3b314fb3fc..0ad872c844ab 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -27,6 +27,11 @@ import javax.inject.Inject; +import com.cloud.hypervisor.HypervisorOutOfBandVMClone; +import com.cloud.hypervisor.vmware.mo.DatastoreMO; +import com.cloud.hypervisor.vmware.mo.HostMO; +import com.cloud.hypervisor.vmware.util.VmwareClient; +import com.cloud.vm.VmDetailConstants; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; @@ -80,9 +85,9 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.HypervisorGuru; import com.cloud.hypervisor.HypervisorGuruBase; -import com.cloud.hypervisor.vmware.VmwareDatacenterVO; +import com.cloud.dc.VmwareDatacenterVO; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMapVO; -import com.cloud.hypervisor.vmware.dao.VmwareDatacenterDao; +import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; import com.cloud.hypervisor.vmware.manager.VmwareManager; import com.cloud.hypervisor.vmware.mo.DatacenterMO; @@ -1217,4 +1222,95 @@ private String getTargetHostGuid(StoragePool targetLocalPoolForVM, Long destClus protected VirtualMachineTO toVirtualMachineTO(VirtualMachineProfile vmProfile) { return super.toVirtualMachineTO(vmProfile); } + + private VmwareContext connectToVcenter(String vcenter, String username, String password) throws Exception { + VmwareClient vimClient = new VmwareClient(vcenter); + String serviceUrl = "https://" + vcenter + "/sdk/vimService"; + vimClient.connect(serviceUrl, username, password); + return new VmwareContext(vimClient, vcenter); + } + + private void relocateClonedVMToSourceHost(VirtualMachineMO clonedVM, HostMO sourceHost) throws Exception { + if (!clonedVM.getRunningHost().getMor().equals(sourceHost.getMor())) { + s_logger.debug(String.format("Relocating VM to the same host as the source VM: %s", sourceHost.getHostName())); + if (!clonedVM.relocate(sourceHost.getMor())) { + String err = String.format("Cannot relocate cloned VM %s to the source host %s", clonedVM.getVmName(), sourceHost.getHostName()); + s_logger.error(err); + throw new CloudRuntimeException(err); + } + } + } + + private VirtualMachineMO createCloneFromSourceVM(String vmName, VirtualMachineMO vmMo, + DatacenterMO dataCenterMO) throws Exception { + HostMO sourceHost = vmMo.getRunningHost(); + String cloneName = String.format("%s-%s", vmName, UUID.randomUUID()); + DatastoreMO datastoreMO = vmMo.getAllDatastores().get(0); //pick the first datastore + ManagedObjectReference morPool = vmMo.getRunningHost().getHyperHostOwnerResourcePool(); + boolean result = vmMo.createFullClone(cloneName, dataCenterMO.getVmFolder(), morPool, datastoreMO.getMor(), Storage.ProvisioningType.THIN); + VirtualMachineMO clonedVM = dataCenterMO.findVm(cloneName); + if (!result || clonedVM == null) { + String err = String.format("Could not clone VM %s before migration from VMware", vmName); + s_logger.error(err); + throw new CloudRuntimeException(err); + } + relocateClonedVMToSourceHost(clonedVM, sourceHost); + return clonedVM; + } + + private HypervisorOutOfBandVMClone generateResponseFromClone(VirtualMachineMO clonedVM) throws Exception { + List virtualDisks = clonedVM.getVirtualDisks(); + long capacity = 0; + for (VirtualDisk disk : virtualDisks) { + capacity = Long.sum(capacity, disk.getCapacityInBytes()); + } + return new HypervisorOutOfBandVMClone(clonedVM.getVmName(), virtualDisks.size(), capacity); + } + + @Override + public HypervisorOutOfBandVMClone cloneHypervisorVMOutOfBand(String hostIp, String vmName, Map params) { + s_logger.debug(String.format("Cloning VM %s on external vCenter %s", vmName, hostIp)); + String vcenter = params.get(VmDetailConstants.VMWARE_VCENTER); + String datacenter = params.get(VmDetailConstants.VMWARE_DATACENTER); + String username = params.get(VmDetailConstants.VMWARE_VCENTER_USERNAME); + String password = params.get(VmDetailConstants.VMWARE_VCENTER_PASSWORD); + + try { + VmwareContext context = connectToVcenter(vcenter, username, password); + DatacenterMO dataCenterMO = new DatacenterMO(context, datacenter); + VirtualMachineMO vmMo = dataCenterMO.findVm(vmName); + if (vmMo == null) { + String err = String.format("Cannot find VM with name %s on %s/%s", vmName, vcenter, datacenter); + s_logger.error(err); + throw new CloudRuntimeException(err); + } + VirtualMachineMO clonedVM = createCloneFromSourceVM(vmName, vmMo, dataCenterMO); + HypervisorOutOfBandVMClone clone = generateResponseFromClone(clonedVM); + s_logger.debug(String.format("VM cloned successfully")); + return clone; + } catch (Exception e) { + String err = String.format("Error cloning VM: %s from external vCenter %s: %s", vmName, vcenter, e.getMessage()); + s_logger.error(err, e); + throw new CloudRuntimeException(err, e); + } + } + + @Override + public boolean removeHypervisorVMOutOfBand(String hostIp, String vmName, Map params) { + s_logger.debug(String.format("Removing VM %s on external vCenter %s", vmName, hostIp)); + String vcenter = params.get(VmDetailConstants.VMWARE_VCENTER); + String datacenter = params.get(VmDetailConstants.VMWARE_DATACENTER); + String username = params.get(VmDetailConstants.VMWARE_VCENTER_USERNAME); + String password = params.get(VmDetailConstants.VMWARE_VCENTER_PASSWORD); + try { + VmwareContext context = connectToVcenter(vcenter, username, password); + DatacenterMO dataCenterMO = new DatacenterMO(context, datacenter); + VirtualMachineMO vmMo = dataCenterMO.findVm(vmName); + return vmMo.destroy(); + } catch (Exception e) { + String err = String.format("Error destroying external VM %s: %s", vmName, e.getMessage()); + s_logger.error(err, e); + return false; + } + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java index 431526a2c51a..0d13ed0126f1 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java @@ -17,14 +17,18 @@ package com.cloud.hypervisor.vmware; +import com.cloud.dc.VmwareDatacenter; +import com.cloud.dc.VmwareDatacenterVO; import com.cloud.dc.VsphereStoragePolicy; import com.cloud.exception.DiscoveryException; import com.cloud.exception.ResourceInUseException; +import com.cloud.hypervisor.vmware.mo.VmwareStoppedVmInDatacenter; import com.cloud.storage.StoragePool; import com.cloud.utils.component.PluggableService; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcStoppedVmsCmd; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePolicyCompatiblePoolsCmd; @@ -48,4 +52,6 @@ public interface VmwareDatacenterService extends PluggableService { List listVsphereStoragePolicies(ListVsphereStoragePoliciesCmd cmd); List listVsphereStoragePolicyCompatibleStoragePools(ListVsphereStoragePolicyCompatiblePoolsCmd cmd); + + List listStoppedVMsInDatacenter(ListVmwareDcStoppedVmsCmd cmd); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java index 6a8bcda23e53..1989d3d53a1d 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java @@ -27,6 +27,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.dc.VmwareDatacenterVO; import org.apache.cloudstack.api.ApiConstants; import org.apache.log4j.Logger; @@ -46,7 +47,7 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; -import com.cloud.hypervisor.vmware.dao.VmwareDatacenterDao; +import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; import com.cloud.hypervisor.vmware.manager.VmwareManager; import com.cloud.hypervisor.vmware.mo.ClusterMO; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java index e44087e6ada4..b4bd56f5b4ee 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java @@ -20,10 +20,11 @@ import java.util.List; +import com.cloud.dc.dao.VmwareDatacenterDao; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; -import com.cloud.hypervisor.vmware.VmwareDatacenterVO; +import com.cloud.dc.VmwareDatacenterVO; import com.cloud.utils.db.DB; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 199f96a8539f..4384f8f43cd0 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -43,8 +43,11 @@ import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; +import com.cloud.hypervisor.vmware.mo.VmwareStoppedVmInDatacenter; +import com.cloud.hypervisor.vmware.util.VmwareClient; import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcStoppedVmsCmd; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePolicyCompatiblePoolsCmd; @@ -109,13 +112,13 @@ import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.hypervisor.vmware.LegacyZoneVO; import com.cloud.hypervisor.vmware.VmwareCleanupMaid; -import com.cloud.hypervisor.vmware.VmwareDatacenter; +import com.cloud.dc.VmwareDatacenter; import com.cloud.hypervisor.vmware.VmwareDatacenterService; -import com.cloud.hypervisor.vmware.VmwareDatacenterVO; +import com.cloud.dc.VmwareDatacenterVO; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMapVO; import com.cloud.hypervisor.vmware.dao.LegacyZoneDao; -import com.cloud.hypervisor.vmware.dao.VmwareDatacenterDao; +import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; import com.cloud.hypervisor.vmware.mo.CustomFieldConstants; import com.cloud.hypervisor.vmware.mo.DatacenterMO; @@ -1113,6 +1116,7 @@ public List> getCommands() { cmdList.add(ImportVsphereStoragePoliciesCmd.class); cmdList.add(ListVsphereStoragePoliciesCmd.class); cmdList.add(ListVsphereStoragePolicyCompatiblePoolsCmd.class); + cmdList.add(ListVmwareDcStoppedVmsCmd.class); return cmdList; } @@ -1586,6 +1590,59 @@ public List listVsphereStoragePolicyCompatibleStoragePools(ListVsph return compatiblePools; } + @Override + public List listStoppedVMsInDatacenter(ListVmwareDcStoppedVmsCmd cmd) { + String vcenter = cmd.getVcenter(); + String datacenterName = cmd.getDatacenterName(); + String username = cmd.getUsername(); + String password = cmd.getPassword(); + Long existingVcenterId = cmd.getExistingVcenterId(); + + if ((existingVcenterId == null && StringUtils.isBlank(vcenter)) || + (existingVcenterId != null && StringUtils.isNotBlank(vcenter))) { + throw new InvalidParameterValueException("Please provide an existing vCenter ID or a vCenter IP/Name, parameters are mutually exclusive"); + } + + if (existingVcenterId == null && StringUtils.isAnyBlank(vcenter, datacenterName, username, password)) { + throw new InvalidParameterValueException("Please set all the information for a vCenter IP/Name, datacenter, username and password"); + } + + if (existingVcenterId != null) { + VmwareDatacenterVO vmwareDc = vmwareDcDao.findById(existingVcenterId); + if (vmwareDc == null) { + throw new InvalidParameterValueException(String.format("Cannot find a VMware datacenter with ID %s", existingVcenterId)); + } + vcenter = vmwareDc.getVcenterHost(); + datacenterName = vmwareDc.getVmwareDatacenterName(); + username = vmwareDc.getUser(); + password = vmwareDc.getPassword(); + } + + try { + s_logger.debug(String.format("Connecting to the VMware datacenter %s to retrieve stopped VMs", vcenter)); +// VmwareContext context = VmwareContextFactory.create(vcenter, username, password); + String serviceUrl = "https://" + vcenter + "/sdk/vimService"; + VmwareClient vimClient = new VmwareClient(vcenter); + vimClient.connect(serviceUrl, username, password); + VmwareContext context = new VmwareContext(vimClient, vcenter); + + DatacenterMO dcMo = new DatacenterMO(context, datacenterName); + ManagedObjectReference dcMor = dcMo.getMor(); + if (dcMor == null) { + String msg = String.format("Unable to find VMware datacenter %s in vCenter %s", + datacenterName, vcenter); + s_logger.error(msg); + throw new InvalidParameterValueException(msg); + } + return dcMo.getAllStoppedVMsOnDatacenter(); + } catch (Exception e) { + String errorMsg = String.format("Error retrieving stopped VMs from the VMware VC %s datacenter %s: %s", + vcenter, datacenterName, e.getMessage()); + s_logger.error(errorMsg, e); + throw new CloudRuntimeException(errorMsg); + } + } + @Override public boolean hasNexusVSM(Long clusterId) { ClusterVSMMapVO vsmMapVo = null; diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/AddVmwareDcCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/AddVmwareDcCmd.java index 6100713ee30d..9f4985a1363b 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/AddVmwareDcCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/AddVmwareDcCmd.java @@ -33,7 +33,7 @@ import com.cloud.exception.DiscoveryException; import com.cloud.exception.ResourceInUseException; import com.cloud.hypervisor.vmware.VmwareDatacenterService; -import com.cloud.hypervisor.vmware.VmwareDatacenterVO; +import com.cloud.dc.VmwareDatacenterVO; import com.cloud.user.Account; import com.cloud.utils.exception.CloudRuntimeException; diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcStoppedVmsCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcStoppedVmsCmd.java new file mode 100644 index 000000000000..4c085cd9c391 --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcStoppedVmsCmd.java @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.zone; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.hypervisor.vmware.VmwareDatacenterService; +import com.cloud.hypervisor.vmware.mo.VmwareStoppedVmInDatacenter; +import com.cloud.user.Account; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.VmwareDatacenterResponse; +import org.apache.cloudstack.api.response.VmwareStoppedVmResponse; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; + +@APICommand(name = "listVmwareDcStoppedVms", responseObject = VmwareStoppedVmResponse.class, + description = "Retrieves VMware Stopped VMs to register as a template into a KVM zone.", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class ListVmwareDcStoppedVmsCmd extends BaseListCmd { + + @Inject + public VmwareDatacenterService _vmwareDatacenterService; + + @Parameter(name = ApiConstants.EXISTING_VCENTER_ID, + type = CommandType.UUID, + entityType = VmwareDatacenterResponse.class, + description = "UUID of a linked existing vCenter") + private Long existingVcenterId; + + @Parameter(name = ApiConstants.VCENTER, + type = CommandType.STRING, + description = "The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.") + private String vcenter; + + @Parameter(name = ApiConstants.DATACENTER_NAME, type = CommandType.STRING, description = "Name of VMware datacenter.") + private String datacenterName; + + @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, description = "The Username required to connect to resource.") + private String username; + + @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, description = "The password for specified username.") + private String password; + + public String getVcenter() { + return vcenter; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public String getDatacenterName() { + return datacenterName; + } + + public Long getExistingVcenterId() { + return existingVcenterId; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + checkParameters(); + try { + List vms = _vmwareDatacenterService.listStoppedVMsInDatacenter(this); + ListResponse response = new ListResponse<>(); + List responses = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(vms)) { + for (VmwareStoppedVmInDatacenter stoppedVm : vms) { + String host = stoppedVm.getHostName(); + String vmName = stoppedVm.getVmName(); + String clusterName = stoppedVm.getClusterName(); + VmwareStoppedVmResponse resp = new VmwareStoppedVmResponse(host, vmName, clusterName); + resp.setObjectName("vmwarestoppedvm"); + responses.add(resp); + } + } + response.setResponses(responses); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException e) { + String errorMsg = String.format("Error retrieving stopped VMs from VMware VC: %s", e.getMessage()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, errorMsg); + } + } + + private void checkParameters() { + if ((existingVcenterId == null && vcenter == null) || (existingVcenterId != null && vcenter != null)) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Please provide an existing vCenter ID or a vCenter IP/Name, parameters are mutually exclusive"); + } + if (existingVcenterId == null && StringUtils.isAnyBlank(vcenter, datacenterName, username, password)) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Please set all the information for a vCenter IP/Name, datacenter, username and password"); + } + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public String getCommandName() { + return "listvmwaredcstoppedvmsresponse"; + } +} diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcsCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcsCmd.java index a91605c24bf5..61b5210bb3a5 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcsCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcsCmd.java @@ -39,7 +39,7 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.hypervisor.vmware.VmwareDatacenter; +import com.cloud.dc.VmwareDatacenter; import com.cloud.hypervisor.vmware.VmwareDatacenterService; import com.cloud.user.Account; diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/UpdateVmwareDcCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/UpdateVmwareDcCmd.java index bb62a48ad328..2b6cf5984c51 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/UpdateVmwareDcCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/UpdateVmwareDcCmd.java @@ -30,7 +30,7 @@ import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.log4j.Logger; -import com.cloud.hypervisor.vmware.VmwareDatacenter; +import com.cloud.dc.VmwareDatacenter; import com.cloud.hypervisor.vmware.VmwareDatacenterService; import com.cloud.user.Account; diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/VmwareStoppedVmResponse.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/VmwareStoppedVmResponse.java new file mode 100644 index 000000000000..ed0627d61e53 --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/VmwareStoppedVmResponse.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +public class VmwareStoppedVmResponse extends BaseResponse { + + public VmwareStoppedVmResponse(String host, String name, String cluster) { + this.host = host; + this.vmName = name; + this.cluster = cluster; + } + + @SerializedName(ApiConstants.VIRTUAL_MACHINE_NAME) + @Param(description = "The VMware VM name") + private String vmName; + + @SerializedName(ApiConstants.HOST_NAME) + @Param(description = "The VMware ESXi host") + private String host; + + @SerializedName(ApiConstants.CLUSTER_NAME) + @Param(description = "The VMware cluster") + private String cluster; + + public String getVmName() { + return vmName; + } + + public void setVmName(String vmName) { + this.vmName = vmName; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public String getCluster() { + return cluster; + } + + public void setCluster(String cluster) { + this.cluster = cluster; + } +} diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java index 0629e4551484..940bfcac5473 100644 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java @@ -27,6 +27,7 @@ import com.cloud.dc.DataCenter.NetworkType; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; +import com.cloud.dc.VmwareDatacenterVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.ClusterVSMMapDao; import com.cloud.dc.dao.DataCenterDao; @@ -42,7 +43,7 @@ import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.hypervisor.vmware.dao.LegacyZoneDao; -import com.cloud.hypervisor.vmware.dao.VmwareDatacenterDao; +import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; import com.cloud.hypervisor.vmware.manager.VmwareManagerImpl; import com.cloud.network.NetworkModel; diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImplTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImplTest.java index 0b7eff0ee750..68e2ae4cc660 100644 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImplTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImplTest.java @@ -38,10 +38,10 @@ import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor; -import com.cloud.hypervisor.vmware.VmwareDatacenter; -import com.cloud.hypervisor.vmware.VmwareDatacenterVO; +import com.cloud.dc.VmwareDatacenter; +import com.cloud.dc.VmwareDatacenterVO; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMapVO; -import com.cloud.hypervisor.vmware.dao.VmwareDatacenterDao; +import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; @RunWith(MockitoJUnitRunner.class) diff --git a/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java b/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java index fb0702d2f476..8083c86af0bc 100644 --- a/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java +++ b/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java @@ -366,4 +366,15 @@ public ConfigKey[] getConfigKeys() { }; } + @Override + public HypervisorOutOfBandVMClone cloneHypervisorVMOutOfBand(String hostIp, String vmName, Map params) { + s_logger.error("Unsupported operation: cannot clone external VM"); + return null; + } + + @Override + public boolean removeHypervisorVMOutOfBand(String hostIp, String vmName, Map params) { + s_logger.error("Unsupported operation: cannot remove external VM"); + return false; + } } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 7f271520c761..6f8a738701ec 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -491,6 +491,7 @@ import org.apache.cloudstack.api.command.user.template.ListTemplatePermissionsCmd; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; +import org.apache.cloudstack.api.command.user.template.RegisterTemplateFromVMwareVMCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd; import org.apache.cloudstack.api.command.user.userdata.DeleteUserDataCmd; @@ -3626,6 +3627,7 @@ public List> getCommands() { cmdList.add(ListTemplatesCmd.class); cmdList.add(ListDetailOptionsCmd.class); cmdList.add(RegisterTemplateCmd.class); + cmdList.add(RegisterTemplateFromVMwareVMCmd.class); cmdList.add(UpdateTemplateCmd.class); cmdList.add(UpdateTemplatePermissionsCmd.class); cmdList.add(AddNicToVMCmd.class); diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 5b1ca9058728..84ec5f45ef91 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -3415,7 +3415,8 @@ public ConfigKey[] getConfigKeys() { SecStorageVMAutoScaleDown, MountDisabledStoragePool, VmwareCreateCloneFull, - VmwareAllowParallelExecution + VmwareAllowParallelExecution, + KvmTemplateFromVmwareVmMigrationWait }; } diff --git a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java index 1954cdea6879..a3561d321f70 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java @@ -25,6 +25,13 @@ import javax.inject.Inject; +import com.cloud.agent.api.to.VmwareVmForMigrationTO; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.storage.StorageManager; +import com.cloud.storage.VMTemplateDetailVO; +import com.cloud.storage.dao.VMTemplateDetailsDao; +import com.cloud.utils.Pair; +import com.cloud.vm.VmDetailConstants; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -73,6 +80,8 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor private ConfigurationDao _configDao; @Inject private EndPointSelector _epSelector; + @Inject + private VMTemplateDetailsDao templateDetailsDao; private String _copyAuthPasswd; private String _proxy = null; @@ -115,20 +124,95 @@ public boolean isTemplateUpdateable(Long templateId, Long storeId) { return (downloadsInProgress.size() == 0); } - private void initiateTemplateDownload(DataObject template, AsyncCompletionCallback callback) { - boolean downloadJobExists = false; - TemplateDataStoreVO vmTemplateStore; - DataStore store = template.getDataStore(); + protected TemplateDataStoreVO persistTemplateDataStoreRecord(DataObject template, DataStore store) { + TemplateDataStoreVO vmTemplateStore = new TemplateDataStoreVO(store.getId(), template.getId(), + new Date(), 0, Status.NOT_DOWNLOADED, null, null, + "jobid0000", null, template.getUri()); + vmTemplateStore.setDataStoreRole(store.getRole()); + return _vmTemplateStoreDao.persist(vmTemplateStore); + } - vmTemplateStore = _vmTemplateStoreDao.findByStoreTemplate(store.getId(), template.getId()); + /** + * The first element indicates if there is an existing download job, and second element is the templateDataStoreVO + */ + protected Pair getTemplateDataStoreRecordPair(DataObject template, DataStore store) { + boolean downloadJobExists = false; + TemplateDataStoreVO vmTemplateStore = _vmTemplateStoreDao.findByStoreTemplate(store.getId(), template.getId()); if (vmTemplateStore == null) { - vmTemplateStore = - new TemplateDataStoreVO(store.getId(), template.getId(), new Date(), 0, Status.NOT_DOWNLOADED, null, null, "jobid0000", null, template.getUri()); - vmTemplateStore.setDataStoreRole(store.getRole()); - vmTemplateStore = _vmTemplateStoreDao.persist(vmTemplateStore); + vmTemplateStore = persistTemplateDataStoreRecord(template, store); } else if ((vmTemplateStore.getJobId() != null) && (vmTemplateStore.getJobId().length() > 2)) { downloadJobExists = true; } + return new Pair<>(downloadJobExists, vmTemplateStore); + } + + protected String getValueFromTemplateDetail(Long templateId, String key) { + VMTemplateDetailVO detail = templateDetailsDao.findDetail(templateId, key); + if (detail == null) { + throw new InvalidParameterValueException(String.format("Could not find the template detail" + + " with key %s for the template with ID: %s", key, templateId)); + } + return detail.getValue(); + } + + protected void sendMigrateVmwareVmCommandToKvmHost(DataObject template, DataStore imageStore, AsyncCompletionCallback callback) { + String vcenter = getValueFromTemplateDetail(template.getId(), VmDetailConstants.VMWARE_VCENTER); + String datacenter = getValueFromTemplateDetail(template.getId(), VmDetailConstants.VMWARE_DATACENTER); + String cluster = getValueFromTemplateDetail(template.getId(), VmDetailConstants.VMWARE_CLUSTER); + String username = getValueFromTemplateDetail(template.getId(), VmDetailConstants.VMWARE_VCENTER_USERNAME); + String password = getValueFromTemplateDetail(template.getId(), VmDetailConstants.VMWARE_VCENTER_PASSWORD); + String hostName = getValueFromTemplateDetail(template.getId(), VmDetailConstants.VMWARE_HOST); + String vmName = getValueFromTemplateDetail(template.getId(), VmDetailConstants.VMWARE_VM_NAME); + + // Migrate the clone of the stopped VM + EndPoint endPoint = _epSelector.select(template); + LOGGER.debug(String.format("Found host %s to send command for VMware to KVM migration", endPoint.getHostAddr())); + + Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes(); + VmwareVmForMigrationTO vmTO = new VmwareVmForMigrationTO(vcenter, datacenter, cluster, + username, password, null, hostName, vmName, imageStore.getUri()); + DownloadCommand cmd = new DownloadCommand((TemplateObjectTO) template.getTO(), maxTemplateSizeInBytes); + cmd.setVmwareVmForMigrationTO(vmTO); + cmd.setWait(StorageManager.KvmTemplateFromVmwareVmMigrationWait.value()); + + DownloadListener dl = new DownloadListener(endPoint, imageStore, template, _timer, + this, cmd, callback); + ComponentContext.inject(dl); // initialize those auto-wired field in download listener. + try { + endPoint.sendMessageAsync(cmd, new UploadListener.Callback(endPoint.getId(), dl)); + LOGGER.debug(String.format("Sent DownloadCommand to endpoint: %s for template %s", + endPoint.getHostAddr(), template.getUuid())); + updateTemplateStoreRefRecordForVmwareVmToKvmMigration(template, imageStore, endPoint); + } catch (Exception e) { + String err = String.format("Could not migrate VM from VMware: %s", e.getMessage()); + LOGGER.error(err, e); + } + } + + private void updateTemplateStoreRefRecordForVmwareVmToKvmMigration(DataObject template, DataStore imageStore, EndPoint endPoint) { + TemplateDataStoreVO storeRef = _vmTemplateStoreDao.findByStoreTemplate(imageStore.getId(), template.getId()); + storeRef.setDownloadState(Status.NOT_DOWNLOADED); + storeRef.setErrorString(String.format("Downloading, virt-v2v migration initiated on host %s", + endPoint.getHostAddr())); + _vmTemplateStoreDao.update(storeRef.getId(), storeRef); + } + + protected void migrateVmwareVmToSecondaryStorage(DataObject template, AsyncCompletionCallback callback) { + DataStore imageStore = template.getDataStore(); + Pair pair = getTemplateDataStoreRecordPair(template, imageStore); + TemplateDataStoreVO templateDataStoreVO = pair.second(); + if (templateDataStoreVO != null) { + imageStore.getScope().getScopeId(); + sendMigrateVmwareVmCommandToKvmHost(template, imageStore, callback); + } + } + + private void initiateTemplateDownload(DataObject template, AsyncCompletionCallback callback) { + DataStore store = template.getDataStore(); + + Pair pair = getTemplateDataStoreRecordPair(template, store); + boolean downloadJobExists = pair.first(); + TemplateDataStoreVO vmTemplateStore = pair.second(); Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes(); if (vmTemplateStore != null) { @@ -176,7 +260,12 @@ public void downloadTemplateToStorage(DataObject template, AsyncCompletionCallba DataStore store = template.getDataStore(); if (isTemplateUpdateable(templateId, store.getId())) { if (template.getUri() != null) { - initiateTemplateDownload(template, callback); + String templateUri = template.getUri(); + if (templateUri.startsWith("vpx://")) { + migrateVmwareVmToSecondaryStorage(template, callback); + } else { + initiateTemplateDownload(template, callback); + } } else { LOGGER.info("Template url is null, cannot download"); DownloadAnswer ans = new DownloadAnswer("Template url is null", Status.UNKNOWN); diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index 3f45505bcc50..61310084dc37 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -22,12 +22,15 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.hypervisor.HypervisorOutOfBandVMClone; +import com.cloud.vm.VmDetailConstants; import com.cloud.domain.Domain; import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer; import org.apache.cloudstack.agent.directdownload.CheckUrlCommand; @@ -39,6 +42,7 @@ import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; +import org.apache.cloudstack.api.command.user.template.RegisterTemplateFromVMwareVMCmd; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -218,6 +222,26 @@ public TemplateProfile prepare(GetUploadParamsForIsoCmd cmd) throws ResourceAllo return profile; } + @Override + public TemplateProfile prepareTemplateFromVmwareMigration(RegisterTemplateFromVMwareVMCmd cmd, + Map params, HypervisorOutOfBandVMClone clone) throws ResourceAllocationException { + TemplateProfile profile = super.prepare(cmd); + + String url = String.format("vpx://%s/%s/%s?no_verify=1 %s", params.get(VmDetailConstants.VMWARE_VCENTER), + params.get(VmDetailConstants.VMWARE_DATACENTER), params.get(VmDetailConstants.VMWARE_HOST), + params.get(VmDetailConstants.VMWARE_VM_NAME)); + profile.setUrl(url); + + for (String key : params.keySet()) { + profile.getDetails().put(key, params.get(key)); + } + + long capacity = clone.getCapacity(); + // Check that the resource limit for secondary storage won't be exceeded + _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, capacity); + return profile; + } + @Override public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocationException { TemplateProfile profile = super.prepare(cmd); @@ -326,6 +350,7 @@ private void createTemplateWithinZone(Long zId, TemplateProfile profile, VMTempl AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().createTemplateAsyncCallBack(null, null)); caller.setContext(context); + imageService.createTemplateAsync(tmpl, imageStore, caller); } } diff --git a/server/src/main/java/com/cloud/template/TemplateAdapter.java b/server/src/main/java/com/cloud/template/TemplateAdapter.java index 86dd0d3cad5d..dc53741cb324 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapter.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; +import com.cloud.hypervisor.HypervisorOutOfBandVMClone; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.GetUploadParamsForIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; @@ -34,6 +35,7 @@ import com.cloud.storage.VMTemplateVO; import com.cloud.user.Account; import com.cloud.utils.component.Adapter; +import org.apache.cloudstack.api.command.user.template.RegisterTemplateFromVMwareVMCmd; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; public interface TemplateAdapter extends Adapter { @@ -60,6 +62,9 @@ public String getName() { TemplateProfile prepare(GetUploadParamsForIsoCmd cmd) throws ResourceAllocationException; + TemplateProfile prepareTemplateFromVmwareMigration(RegisterTemplateFromVMwareVMCmd cmd, + Map params, HypervisorOutOfBandVMClone clone) throws ResourceAllocationException; + VMTemplateVO create(TemplateProfile profile); List createTemplateForPostUpload(TemplateProfile profile); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index bb8affc18700..240e2ecab7e0 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -34,6 +34,9 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.dc.VmwareDatacenterVO; +import com.cloud.dc.dao.VmwareDatacenterDao; +import com.cloud.hypervisor.HypervisorOutOfBandVMClone; import com.cloud.user.UserData; import com.cloud.storage.VolumeApiService; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -55,6 +58,7 @@ import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; import org.apache.cloudstack.api.command.user.template.ListTemplatePermissionsCmd; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; +import org.apache.cloudstack.api.command.user.template.RegisterTemplateFromVMwareVMCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd; import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd; @@ -303,6 +307,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Inject protected SnapshotHelper snapshotHelper; + @Inject + private VmwareDatacenterDao vmwareDatacenterDao; private TemplateAdapter getAdapter(HypervisorType type) { TemplateAdapter adapter = null; @@ -366,6 +372,76 @@ public VirtualMachineTemplate registerTemplate(RegisterTemplateCmd cmd) throws U } } + @Override + public VirtualMachineTemplate registerTemplateFromVMwareVM(RegisterTemplateFromVMwareVMCmd cmd) throws ResourceAllocationException { + Long zoneId = cmd.getZoneId(); + Long existingVcenterId = cmd.getExistingVcenterId(); + String vcenter = cmd.getVcenter(); + String datacenterName = cmd.getDatacenterName(); + String username = cmd.getUsername(); + String password = cmd.getPassword(); + String clusterName = cmd.getClusterName(); + String sourceVM = cmd.getVmName(); + String sourceHostName = cmd.getHost(); + + DataCenterVO zone = _dcDao.findById(zoneId); + if (zone == null) { + s_logger.error(String.format("Cannot find a zone with ID %s", zoneId)); + return null; + } + + HypervisorGuru vmwareGuru = _hvGuruMgr.getGuru(HypervisorType.VMware); + + if ((existingVcenterId == null && vcenter == null) || (existingVcenterId != null && vcenter != null)) { + throw new CloudRuntimeException("Please provide an existing vCenter ID or a vCenter IP/Name, parameters are mutually exclusive"); + } else if (existingVcenterId == null && org.apache.commons.lang3.StringUtils.isAnyBlank(vcenter, datacenterName, username, password)) { + throw new CloudRuntimeException("Please set all the information for a vCenter IP/Name, datacenter, username and password"); + } + + Map params = createParamsForTemplateFromVmwareVmMigration(existingVcenterId, + vcenter, datacenterName, username, password, clusterName, sourceHostName, sourceVM); + + HypervisorOutOfBandVMClone cloneResult = vmwareGuru.cloneHypervisorVMOutOfBand(cmd.getHost(), cmd.getVmName(), params); + String cloneName = cloneResult.getCloneName(); + params.put(VmDetailConstants.VMWARE_VM_NAME, cloneName); + + TemplateAdapter adapter = getAdapter(HypervisorType.KVM); + TemplateProfile profile = adapter.prepareTemplateFromVmwareMigration(cmd, params, cloneResult); + VMTemplateVO vmTemplate = adapter.create(profile); + + if (vmTemplate != null) { + CallContext.current().putContextParameter(VirtualMachineTemplate.class, vmTemplate.getUuid()); + return vmTemplate; + } else { + throw new CloudRuntimeException("Failed to create a template"); + } + } + + protected Map createParamsForTemplateFromVmwareVmMigration(Long existingVcenterId, + String vcenter, String datacenterName, + String username, String password, + String clusterName, String sourceHostName, + String sourceVMName) { + Map params = new HashMap<>(); + VmwareDatacenterVO existingDC = null; + if (existingVcenterId != null) { + existingDC = vmwareDatacenterDao.findById(existingVcenterId); + if (existingDC == null) { + String err = String.format("Cannot find any existing Vmware DC with ID %s", existingDC); + s_logger.error(err); + throw new CloudRuntimeException(err); + } + } + params.put(VmDetailConstants.VMWARE_VCENTER, existingDC != null ? existingDC.getVcenterHost() : vcenter); + params.put(VmDetailConstants.VMWARE_DATACENTER, existingDC != null ? existingDC.getVmwareDatacenterName() : datacenterName); + params.put(VmDetailConstants.VMWARE_VCENTER_USERNAME, existingDC != null ? existingDC.getUser() : username); + params.put(VmDetailConstants.VMWARE_VCENTER_PASSWORD, existingDC != null ? existingDC.getPassword() : password); + params.put(VmDetailConstants.VMWARE_CLUSTER, clusterName); + params.put(VmDetailConstants.VMWARE_HOST, sourceHostName); + params.put(VmDetailConstants.VMWARE_VM_NAME, sourceVMName); + return params; + } + /** * Internal register template or ISO method - post local upload * @param adapter diff --git a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java index 5dc50bc73397..c759448377e5 100755 --- a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java +++ b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java @@ -23,6 +23,7 @@ import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.configuration.Resource; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.domain.dao.DomainDao; import com.cloud.event.dao.UsageEventDao; import com.cloud.exception.InvalidParameterValueException; @@ -193,6 +194,8 @@ public class TemplateManagerImplTest { @Inject AccountManager _accountMgr; + @Inject + VmwareDatacenterDao vmwareDatacenterDao; public class CustomThreadPoolExecutor extends ThreadPoolExecutor { AtomicInteger ai = new AtomicInteger(0); @@ -654,6 +657,11 @@ public AccountManager accountManager() { return Mockito.mock(AccountManager.class); } + @Bean + public VmwareDatacenterDao vmwareDatacenterDao() { + return Mockito.mock(VmwareDatacenterDao.class); + } + @Bean public HostDao hostDao() { return Mockito.mock(HostDao.class); diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 0068a12d21ab..ebfb37325073 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -503,6 +503,7 @@ "label.confirmdeclineinvitation": "Are you sure you want to decline this project invitation?", "label.confirmpassword": "Confirm password", "label.confirmpassword.description": "Please type the same password again.", +"label.connect": "Connect", "label.connectiontimeout": "Connection timeout", "label.conservemode": "Conserve mode", "label.considerlasthost": "Consider Last Host", @@ -1602,6 +1603,7 @@ "label.refresh": "Refresh", "label.region": "Region", "label.register.template": "Register template", +"label.register.template.from.vmware.vm": "Register template from existing VMware vCenter Stopped VM", "label.register.user.data": "Register a userdata", "label.reinstall.vm": "Reinstall VM", "label.reject": "Reject", diff --git a/ui/src/config/section/image.js b/ui/src/config/section/image.js index c1961fe216d3..9495ecefa71d 100644 --- a/ui/src/config/section/image.js +++ b/ui/src/config/section/image.js @@ -108,6 +108,15 @@ export default { popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/image/RegisterOrUploadTemplate.vue'))) }, + { + api: 'registerTemplate', + icon: 'import-outlined', + docHelp: 'adminguide/storage.html#uploading-an-existing-volume-to-a-virtual-machine', + label: 'label.register.template.from.vmware.vm', + listView: true, + popup: true, + component: shallowRef(defineAsyncComponent(() => import('@/views/image/RegisterOrUploadTemplate.vue'))) + }, { api: 'updateTemplate', icon: 'edit-outlined', diff --git a/ui/src/views/image/RegisterOrUploadTemplate.vue b/ui/src/views/image/RegisterOrUploadTemplate.vue index 6add11ab0ba6..38bb20c37566 100644 --- a/ui/src/views/image/RegisterOrUploadTemplate.vue +++ b/ui/src/views/image/RegisterOrUploadTemplate.vue @@ -17,7 +17,7 @@