StorPool storage plugin (#6007)

* StorPool storage plugin

Adds volume storage plugin for StorPool SDS

* Added support for alternative endpoint

Added option to switch to alternative endpoint for SP primary storage

* renamed all classes from Storpool to StorPool

* Address review

* removed unnecessary else

* Removed check about the storage provider

We don't need this check, we'll get if the snapshot is on StorPool be
its name from path

* Check that current plugin supports all functionality before upgrade CS

* Smoke tests for StorPool plug-in

* Fixed conflicts

* Fixed conflicts and added missed Apache license header

* Removed whitespaces in smoke tests

* Added StorPool plugin jar for Debian

the StorPool jar will be included into cloudstack-agent package for
Debian/Ubuntu

Author: slavkap

client/pom.xml

@@ -97,6 +97,11 @@
             <artifactId>cloud-plugin-storage-volume-linstor</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.cloudstack</groupId>
+            <artifactId>cloud-plugin-storage-volume-storpool</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.cloudstack</groupId>
             <artifactId>cloud-server</artifactId>
@@ -755,6 +760,12 @@
                                     <artifactId>bcpkix-jdk15on</artifactId>
                                     <overWrite>false</overWrite>
                                     <outputDirectory>${project.build.directory}/lib</outputDirectory>
+			        </artifactItem>
+                                <artifactItem>
+                                    <groupId>org.apache.cloudstack</groupId>
+                                    <artifactId>cloud-plugin-storage-volume-storpool</artifactId>
+                                    <overWrite>false</overWrite>
+                                    <outputDirectory>${project.build.directory}/lib</outputDirectory>
                                 </artifactItem>
                                 <artifactItem>
                                     <groupId>org.bouncycastle</groupId>
@@ -799,6 +810,7 @@
                                     <exclude>org.bouncycastle:bcpkix-jdk15on</exclude>
                                     <exclude>org.bouncycastle:bctls-jdk15on</exclude>
                                     <exclude>mysql:mysql-connector-java</exclude>
+                                    <exclude>org.apache.cloudstack:cloud-plugin-storage-volume-storpool</exclude>
                                 </excludes>
                             </artifactSet>
                             <transformers>

core/src/main/java/org/apache/cloudstack/storage/command/CopyCmdAnswer.java

@@ -20,6 +20,7 @@
 package org.apache.cloudstack.storage.command;
 
 import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.Command;
 import com.cloud.agent.api.to.DataTO;
 
 public class CopyCmdAnswer extends Answer {
@@ -37,4 +38,8 @@ public DataTO getNewData() {
     public CopyCmdAnswer(String errMsg) {
         super(null, false, errMsg);
     }
+
+    public CopyCmdAnswer(Command cmd, Exception e) {
+        super(cmd, e);
+    }
 }

debian/rules

@@ -41,6 +41,7 @@ override_dh_auto_install:
 	mkdir $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib
 	install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/
 	install -D plugins/hypervisors/kvm/target/dependencies/* $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/
+	install -D plugins/storage/volume/storpool/target/cloud-plugin-storage-volume-storpool-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/
 
 	install -d -m0755 debian/$(PACKAGE)-agent/lib/systemd/system
 	install -m0644 packaging/systemd/$(PACKAGE)-agent.service debian/$(PACKAGE)-agent/lib/systemd/system/$(PACKAGE)-agent.service

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java

@@ -103,4 +103,33 @@ enum QualityOfServiceState { MIGRATION, NO_MIGRATION }
      * returns true if the host can access the storage pool
      */
     boolean canHostAccessStoragePool(Host host, StoragePool pool);
+
+    /**
+     * Used by storage pools which want to keep VMs' information
+     * @return true if additional VM info is needed (intended for storage pools).
+     */
+    boolean isVmInfoNeeded();
+
+    /**
+     * Provides additional info for a VM (intended for storage pools).
+     * E.g. the storage pool may want to keep/delete information if the volume is attached/detached to any VM.
+     * @param vmId The ID of the virtual machine
+     * @param volumeId the ID of the volume
+     */
+    void provideVmInfo(long vmId, long volumeId);
+
+    /**
+     * Returns true if the storage have to know about the VM's tags (intended for storage pools).
+     * @param tagKey The name of the tag
+     * @return true if the storage have to know about the VM's tags
+     */
+    boolean isVmTagsNeeded(String tagKey);
+
+    /**
+     * Provide VM's tags to storage (intended for storage pools).
+     * @param vmId The ID of the virtual machine
+     * @param volumeId The ID of the volume
+     * @param tagValue The value of the VM's tag
+     */
+    void provideVmTags(long vmId, long volumeId, String tagValue);
 }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StorageStrategyFactory.java

@@ -39,9 +39,9 @@ public interface StorageStrategyFactory {
     /**
      * Used only for KVM hypervisors when allocating a VM snapshot
      * @param vmId the ID of the virtual machine
+     * @param rootPoolId volume pool ID
      * @param snapshotMemory for VM snapshots with memory
      * @return VMSnapshotStrategy
      */
     VMSnapshotStrategy getVmSnapshotStrategy(Long vmId, Long rootPoolId, boolean snapshotMemory);
-
 }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VMSnapshotStrategy.java

@@ -30,7 +30,7 @@ public interface VMSnapshotStrategy {
     StrategyPriority canHandle(VMSnapshot vmSnapshot);
 
     /**
-     * Used only for KVM hypervisors when allocating a VM snapshot
+     * Verifies if the strategy can handle the VM snapshot. This method is used only for KVM hypervisors when allocating a VM snapshot.
      * @param vmId the ID of the virtual machine
      * @param snapshotMemory for VM snapshots with memory
      * @return StrategyPriority

engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java

@@ -52,8 +52,8 @@
 import com.cloud.storage.DiskOfferingVO;
 import com.cloud.storage.GuestOSHypervisorVO;
 import com.cloud.storage.GuestOSVO;
-import com.cloud.storage.VolumeVO;
 import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.VolumeVO;
 import com.cloud.storage.dao.DiskOfferingDao;
 import com.cloud.storage.dao.GuestOSDao;
 import com.cloud.storage.dao.GuestOSHypervisorDao;

packaging/centos7/cloud.spec

@@ -351,6 +351,7 @@ install -D agent/target/transformed/cloudstack-agent-profile.sh ${RPM_BUILD_ROOT
 install -D agent/target/transformed/cloudstack-agent.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-agent
 install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%name-agent/lib/cloud-plugin-hypervisor-kvm-%{_maventag}.jar
 cp plugins/hypervisors/kvm/target/dependencies/*  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
+cp plugins/storage/volume/storpool/target/*.jar  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
 
 # Usage server
 mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage

packaging/centos8/cloud.spec

@@ -344,6 +344,7 @@ install -D agent/target/transformed/cloudstack-agent-profile.sh ${RPM_BUILD_ROOT
 install -D agent/target/transformed/cloudstack-agent.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-agent
 install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%name-agent/lib/cloud-plugin-hypervisor-kvm-%{_maventag}.jar
 cp plugins/hypervisors/kvm/target/dependencies/*  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
+cp plugins/storage/volume/storpool/target/*.jar  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
 
 # Usage server
 mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage

packaging/suse15/cloud.spec

@@ -346,6 +346,7 @@ install -D agent/target/transformed/cloudstack-agent-profile.sh ${RPM_BUILD_ROOT
 install -D agent/target/transformed/cloudstack-agent.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-agent
 install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%name-agent/lib/cloud-plugin-hypervisor-kvm-%{_maventag}.jar
 cp plugins/hypervisors/kvm/target/dependencies/*  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
+cp plugins/storage/volume/storpool/target/*.jar  ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
 
 # Usage server
 mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage