Merge branch '4.16' into main

Author: sureshanaparti

api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java

@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.storage;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Map;
+
+public class OVFParser {
+    private static final Logger s_logger = Logger.getLogger(OVFParser.class);
+
+    private static final String DEFAULT_OVF_SCHEMA = "http://schemas.dmtf.org/ovf/envelope/1";
+    private static final String VMW_SCHEMA = "http://www.vmware.com/schema/ovf";
+
+    private static final Map<String, String> ATTRIBUTE_SCHEMA_MAP = Map.of(
+            "osType", VMW_SCHEMA
+    );
+
+    private DocumentBuilder documentBuilder;
+
+    public OVFParser() {
+        try {
+            DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
+            documentBuilderFactory.setNamespaceAware(true);
+            documentBuilder = documentBuilderFactory.newDocumentBuilder();
+        } catch (ParserConfigurationException e) {
+            s_logger.error("Cannot start the OVF parser: " + e.getMessage(), e);
+        }
+    }
+
+    public Document parseOVF(String ovfString) throws IOException, SAXException {
+        InputSource is = new InputSource(new StringReader(ovfString));
+        return documentBuilder.parse(is);
+    }
+
+    public Document parseOVFFile(String ovfFilePath) {
+        if (StringUtils.isBlank(ovfFilePath)) {
+            return null;
+        }
+        try {
+            return documentBuilder.parse(new File(ovfFilePath));
+        } catch (SAXException | IOException e) {
+            s_logger.error("Error parsing " + ovfFilePath + " " + e.getMessage(), e);
+            return null;
+        }
+    }
+
+    /**
+     * Retrieve elements with tag name from the document, according to the OVF schema definition
+     */
+    public NodeList getElementsFromOVFDocument(Document doc, String tagName) {
+        return doc != null ? doc.getElementsByTagNameNS(DEFAULT_OVF_SCHEMA, tagName) : null;
+    }
+
+    /**
+     * Retrieve an attribute value from an OVF element
+     */
+    public String getNodeAttribute(Element element, String attr) {
+        return element != null ? element.getAttributeNS(ATTRIBUTE_SCHEMA_MAP.getOrDefault(attr, DEFAULT_OVF_SCHEMA), attr) : null;
+    }
+
+    /**
+     * Get the text value of a node's child with name or suffix "childNodeName", null if not present
+     * Example:
+     * <Node>
+     *    <childNodeName>Text value</childNodeName>
+     *    <rasd:childNodeName>Text value</rasd:childNodeName>
+     * </Node>
+     */
+    public String getChildNodeValue(Node node, String childNodeName) {
+        if (node == null || !node.hasChildNodes()) {
+            return null;
+        }
+        NodeList childNodes = node.getChildNodes();
+        for (int i = 0; i < childNodes.getLength(); i++) {
+            Node value = childNodes.item(i);
+            // Also match if the child's name has a suffix:
+            // Example: <rasd:AllocationUnits>
+            if (value != null && (value.getNodeName().equals(childNodeName)) || value.getNodeName().endsWith(":" + childNodeName)) {
+                return value.getTextContent();
+            }
+        }
+        return null;
+    }
+}

api/src/main/java/com/cloud/agent/api/storage/OVFParser.java

@@ -27,14 +27,13 @@
 import org.xml.sax.SAXException;
 import org.xml.sax.SAXParseException;
 
-import javax.xml.parsers.ParserConfigurationException;
 import java.io.IOException;
 import java.util.List;
 
 public class OVFHelperTest {
 
     private String ovfFileProductSection =
-        "<ProductSection>" +
+        "<ProductSection xmlns=\"http://schemas.dmtf.org/ovf/envelope/1\" xmlns:cim=\"http://schemas.dmtf.org/wbem/wscim/1/common\" xmlns:ovf=\"http://schemas.dmtf.org/ovf/envelope/1\" xmlns:rasd=\"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData\" xmlns:vmw=\"http://www.vmware.com/schema/ovf\" xmlns:vssd=\"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">" +
             "<Info>VM Arguments</Info>" +
             "<Property ovf:key=\"va-ssh-public-key\" ovf:type=\"string\" ovf:userConfigurable=\"true\" ovf:value=\"\">" +
                 "<Label>Set the SSH public key allowed to access the appliance</Label>" +
@@ -47,7 +46,7 @@ public class OVFHelperTest {
         "</ProductSection>";
 
     private String ovfFileDeploymentOptionsSection =
-            "<DeploymentOptionSection>\n" +
+            "<DeploymentOptionSection xmlns=\"http://schemas.dmtf.org/ovf/envelope/1\" xmlns:cim=\"http://schemas.dmtf.org/wbem/wscim/1/common\" xmlns:ovf=\"http://schemas.dmtf.org/ovf/envelope/1\" xmlns:rasd=\"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData\" xmlns:vmw=\"http://www.vmware.com/schema/ovf\" xmlns:vssd=\"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">\n" +
                     "    <Info>Deployment Configuration information</Info>\n" +
                     "    <Configuration ovf:id=\"ASAv5\">\n" +
                     "      <Label>100 Mbps (ASAv5)</Label>\n" +
@@ -61,10 +60,10 @@ public class OVFHelperTest {
                     "      <Label>2 Gbps (ASAv30)</Label>\n" +
                     "      <Description>Use this option to deploy an ASAv with a maximum throughput of 2 Gbps (uses 4 vCPUs and 8 GB of memory).</Description>\n" +
                     "    </Configuration>\n" +
-                    "  </DeploymentOptionSection>";
+            "  </DeploymentOptionSection>";
 
     private String ovfFileVirtualHardwareSection =
-            "<VirtualSystem>\n" +
+            "<VirtualSystem xmlns=\"http://schemas.dmtf.org/ovf/envelope/1\" xmlns:cim=\"http://schemas.dmtf.org/wbem/wscim/1/common\" xmlns:ovf=\"http://schemas.dmtf.org/ovf/envelope/1\" xmlns:rasd=\"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData\" xmlns:vmw=\"http://www.vmware.com/schema/ovf\" xmlns:vssd=\"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">\n" +
             "<OperatingSystemSection ovf:id=\"100\" vmw:osType=\"other26xLinux64Guest\">\n" +
             "      <Info>The kind of installed guest operating system</Info>\n" +
             "      <Description>Other 2.6x Linux (64-bit)</Description>\n" +
@@ -270,7 +269,7 @@ public class OVFHelperTest {
             "</VirtualSystem>";
 
     private String eulaSections =
-            "<VirtualSystem>\n" +
+            "<VirtualSystem xmlns=\"http://schemas.dmtf.org/ovf/envelope/1\" xmlns:cim=\"http://schemas.dmtf.org/wbem/wscim/1/common\" xmlns:ovf=\"http://schemas.dmtf.org/ovf/envelope/1\" xmlns:rasd=\"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData\" xmlns:vmw=\"http://www.vmware.com/schema/ovf\" xmlns:vssd=\"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">\n" +
             "<EulaSection>\n" +
             "      <Info>end-user license agreement</Info>\n" +
             "      <License>END USER LICENSE AGREEMENT\n" +
@@ -395,7 +394,7 @@ public class OVFHelperTest {
             "</VirtualSystem>";
 
     private String productSectionWithCategories =
-            "<VirtualSystem ovf:id=\"VMware-vCenter-Server-Appliance\">\n" +
+            "<VirtualSystem ovf:id=\"VMware-vCenter-Server-Appliance\" xmlns=\"http://schemas.dmtf.org/ovf/envelope/1\" xmlns:cim=\"http://schemas.dmtf.org/wbem/wscim/1/common\" xmlns:ovf=\"http://schemas.dmtf.org/ovf/envelope/1\" xmlns:rasd=\"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData\" xmlns:vmw=\"http://www.vmware.com/schema/ovf\" xmlns:vssd=\"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">\n" +
             "<ProductSection ovf:required=\"false\">\n" +
             "      <Info>Appliance ISV branding information</Info>\n" +
             "      <Product>VMware vCenter Server Appliance</Product>\n" +
@@ -704,49 +703,49 @@ public class OVFHelperTest {
     private OVFHelper ovfHelper = new OVFHelper();
 
     @Test
-    public void testGetOVFPropertiesValidOVF() throws IOException, SAXException, ParserConfigurationException {
+    public void testGetOVFPropertiesValidOVF() throws IOException, SAXException {
         List<OVFPropertyTO> props = ovfHelper.getOVFPropertiesFromXmlString(ovfFileProductSection);
         Assert.assertEquals(2, props.size());
     }
 
     @Test(expected = SAXParseException.class)
-    public void testGetOVFPropertiesInvalidOVF() throws IOException, SAXException, ParserConfigurationException {
+    public void testGetOVFPropertiesInvalidOVF() throws IOException, SAXException {
         ovfHelper.getOVFPropertiesFromXmlString(ovfFileProductSection + "xxxxxxxxxxxxxxxxx");
     }
 
     @Test
-    public void testGetOVFDeploymentOptionsValidOVF() throws IOException, SAXException, ParserConfigurationException {
+    public void testGetOVFDeploymentOptionsValidOVF() throws IOException, SAXException {
         List<OVFConfigurationTO> options = ovfHelper.getOVFDeploymentOptionsFromXmlString(ovfFileDeploymentOptionsSection);
         Assert.assertEquals(3, options.size());
     }
 
     @Test
-    public void testGetOVFVirtualHardwareSectionValidOVF() throws IOException, SAXException, ParserConfigurationException {
+    public void testGetOVFVirtualHardwareSectionValidOVF() throws IOException, SAXException {
         List<OVFVirtualHardwareItemTO> items = ovfHelper.getOVFVirtualHardwareSectionFromXmlString(ovfFileVirtualHardwareSection);
         Assert.assertEquals(20, items.size());
     }
 
     @Test
-    public void testGetOVFEulaSectionValidOVF() throws IOException, SAXException, ParserConfigurationException {
+    public void testGetOVFEulaSectionValidOVF() throws IOException, SAXException {
         List<OVFEulaSectionTO> eulas = ovfHelper.getOVFEulaSectionFromXmlString(eulaSections);
         Assert.assertEquals(2, eulas.size());
     }
 
     @Test
-    public void testGetOVFPropertiesWithCategories() throws IOException, SAXException, ParserConfigurationException {
+    public void testGetOVFPropertiesWithCategories() throws IOException, SAXException {
         List<OVFPropertyTO> props = ovfHelper.getOVFPropertiesFromXmlString(productSectionWithCategories);
         Assert.assertEquals(18, props.size());
     }
 
     @Test
-    public void testGetOperatingSystemInfo() throws IOException, SAXException, ParserConfigurationException {
+    public void testGetOperatingSystemInfo() throws IOException, SAXException {
         Pair<String, String> guestOsPair = ovfHelper.getOperatingSystemInfoFromXmlString(ovfFileVirtualHardwareSection);
         Assert.assertEquals("other26xLinux64Guest", guestOsPair.first());
         Assert.assertEquals("Other 2.6x Linux (64-bit)", guestOsPair.second());
     }
 
     @Test
-    public void testGetMinimumHardwareVersion() throws IOException, SAXException, ParserConfigurationException {
+    public void testGetMinimumHardwareVersion() throws IOException, SAXException {
         OVFVirtualHardwareSectionTO hardwareSection = ovfHelper.getVirtualHardwareSectionFromXmlString(ovfFileVirtualHardwareSection);
         Assert.assertEquals("vmx-08", hardwareSection.getMinimiumHardwareVersion());
     }

api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java

@@ -20,13 +20,10 @@
 package com.cloud.storage.template;
 
 import java.io.File;
-import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
 import javax.naming.ConfigurationException;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
 
 import com.cloud.agent.api.to.OVFInformationTO;
 import com.cloud.agent.api.to.deployasis.OVFConfigurationTO;
@@ -50,7 +47,6 @@
 import com.cloud.utils.Pair;
 import com.cloud.utils.component.AdapterBase;
 import com.cloud.utils.script.Script;
-import org.xml.sax.SAXException;
 
 /**
  * processes the content of an OVA for registration of a template
@@ -107,7 +103,7 @@ private FormatInfo createFormatInfo(String templatePath, String templateName, St
     private void validateOva(String templateFileFullPath, FormatInfo info) throws InternalErrorException {
         String ovfFilePath = getOVFFilePath(templateFileFullPath);
         OVFHelper ovfHelper = new OVFHelper();
-        Document doc = ovfHelper.getDocumentFromFile(ovfFilePath);
+        Document doc = ovfHelper.getOvfParser().parseOVFFile(ovfFilePath);
 
         OVFInformationTO ovfInformationTO = createOvfInformationTO(ovfHelper, doc, ovfFilePath);
         info.ovfInformationTO = ovfInformationTO;
@@ -235,15 +231,15 @@ public long getTemplateVirtualSize(String templatePath, String templateName) thr
         String templateFileFullPath = templatePath.endsWith(File.separator) ? templatePath : templatePath + File.separator;
         templateFileFullPath += templateName.endsWith(ImageFormat.OVA.getFileExtension()) ? templateName : templateName + "." + ImageFormat.OVA.getFileExtension();
         String ovfFileName = getOVFFilePath(templateFileFullPath);
+        OVFHelper ovfHelper = new OVFHelper();
         if (ovfFileName == null) {
             String msg = "Unable to locate OVF file in template package directory: " + templatePath;
             LOGGER.error(msg);
             throw new InternalErrorException(msg);
         }
         try {
-            Document ovfDoc = null;
-            ovfDoc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new File(ovfFileName));
-            NodeList diskElements = new OVFHelper().getElementsByTagNameAndPrefix(ovfDoc, "Disk", "ovf");
+            Document ovfDoc = ovfHelper.getOvfParser().parseOVFFile(ovfFileName);
+            NodeList diskElements = ovfHelper.getOvfParser().getElementsFromOVFDocument(ovfDoc, "Disk");
             for (int i = 0; i < diskElements.getLength(); i++) {
                 Element disk = (Element)diskElements.item(i);
                 String diskSizeValue = disk.getAttribute("ovf:capacity");
@@ -262,7 +258,7 @@ public long getTemplateVirtualSize(String templatePath, String templateName) thr
                 virtualSize += diskSize;
             }
             return virtualSize;
-        } catch (InternalErrorException | IOException | NumberFormatException | ParserConfigurationException | SAXException e) {
+        } catch (InternalErrorException  | NumberFormatException e) {
             String msg = "getTemplateVirtualSize: Unable to parse OVF XML document " + templatePath + " to get the virtual disk " + templateName + " size due to " + e;
             LOGGER.error(msg);
             throw new InternalErrorException(msg);

core/src/main/java/com/cloud/storage/template/OVAProcessor.java

@@ -20,7 +20,7 @@
 package org.apache.cloudstack.storage.command;
 
 import com.cloud.agent.api.Answer;
-import com.cloud.utils.Pair;
+import com.cloud.agent.api.to.OVFInformationTO;
 
 public class UploadStatusAnswer extends Answer {
     public static enum UploadStatus {
@@ -32,8 +32,7 @@ public static enum UploadStatus {
     private long physicalSize = 0;
     private String installPath = null;
     private int downloadPercent = 0;
-    private Pair<String, String> guestOsInfo;
-    private String minimumHardwareVersion;
+    private OVFInformationTO ovfInformationTO;
 
     protected UploadStatusAnswer() {
     }
@@ -89,19 +88,11 @@ public void setDownloadPercent(int downloadPercent) {
         this.downloadPercent = downloadPercent;
     }
 
-    public Pair<String, String> getGuestOsInfo() {
-        return guestOsInfo;
+    public OVFInformationTO getOvfInformationTO() {
+        return ovfInformationTO;
     }
 
-    public void setGuestOsInfo(Pair<String, String> guestOsInfo) {
-        this.guestOsInfo = guestOsInfo;
-    }
-
-    public void setMinimumHardwareVersion(String minimumHardwareVersion) {
-        this.minimumHardwareVersion = minimumHardwareVersion;
-    }
-
-    public String getMinimumHardwareVersion() {
-        return minimumHardwareVersion;
+    public void setOvfInformationTO(OVFInformationTO ovfInformationTO) {
+        this.ovfInformationTO = ovfInformationTO;
     }
 }

core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusAnswer.java

@@ -30,6 +30,7 @@
 import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 
@@ -38,6 +39,7 @@
 import java.util.Map;
 
 @RunWith(PowerMockRunner.class)
+@PowerMockIgnore({"javax.xml.*", "java.xml.*", "javax.management.*", "org.apache.xerces.*"})
 @PrepareForTest(OVAProcessor.class)
 public class OVAProcessorTest {
     OVAProcessor processor;

core/src/test/java/com/cloud/storage/template/OVAProcessorTest.java

@@ -33,6 +33,7 @@
 import javax.inject.Inject;
 
 import com.cloud.agent.api.to.NfsTO;
+import com.cloud.agent.api.to.OVFInformationTO;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.Upload;
 import org.apache.cloudstack.storage.image.deployasis.DeployAsIsHelper;
@@ -207,8 +208,9 @@ protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher<? extends Bas
         TemplateDataStoreVO tmpltStoreVO = _templateStoreDao.findByStoreTemplate(store.getId(), obj.getId());
         if (tmpltStoreVO != null) {
             if (tmpltStoreVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
-                if (template.isDeployAsIs()) {
-                    boolean persistDeployAsIs = deployAsIsHelper.persistTemplateDeployAsIsDetails(template.getId(), answer, tmpltStoreVO);
+                if (template.isDeployAsIs() && answer != null) {
+                    OVFInformationTO ovfInformationTO = answer.getOvfInformationTO();
+                    boolean persistDeployAsIs = deployAsIsHelper.persistTemplateOVFInformationAndUpdateGuestOS(template.getId(), ovfInformationTO, tmpltStoreVO);
                     if (!persistDeployAsIs) {
                         LOGGER.info("Failed persisting deploy-as-is template details for template " + template.getName());
                         return null;