From 1984b8a05f447a7d08ac8fe6c095b4e63777098d Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Tue, 15 Sep 2020 17:05:05 +0200 Subject: [PATCH 01/17] Export count of total/up/down hosts by tags --- .../metrics/PrometheusExporterImpl.java | 54 +++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index 8609cd9829b9..949c0c5da4f5 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -18,7 +18,9 @@ import java.math.BigDecimal; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.inject.Inject; @@ -47,6 +49,7 @@ import com.cloud.dc.DataCenterVO; import com.cloud.dc.Vlan; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.host.dao.HostTagsDao; import com.cloud.dc.dao.DataCenterIpAddressDao; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -106,6 +109,8 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp private AccountDao _accountDao; @Inject private ResourceCountDao _resourceCountDao; + @Inject + private HostTagsDao _hostTagsDao; public PrometheusExporterImpl() { super(); @@ -115,6 +120,10 @@ private void addHostMetrics(final List metricsList, final long dcId, final int total = 0; int up = 0; int down = 0; + Map total_hosts = new HashMap<>(); + Map up_hosts = new HashMap<>(); + Map down_hosts = new HashMap<>(); + for (final HostVO host : hostDao.listAll()) { if (host == null || host.getType() != Host.Type.Routing || host.getDataCenterId() != dcId) { continue; @@ -131,6 +140,23 @@ private void addHostMetrics(final List metricsList, final long dcId, final int isDedicated = (dr != null) ? 1 : 0; metricsList.add(new ItemHostIsDedicated(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), isDedicated)); + List hostTags = _hostTagsDao.gethostTags(host.getId()); + for (String tag : hostTags) { + Integer current = total_hosts.get(tag) != null ? total_hosts.get(tag) : 0; + total_hosts.put(tag, current + 1); + } + if (host.getStatus() == Status.Up) { + for (String tag : hostTags) { + Integer current = up_hosts.get(tag) != null ? up_hosts.get(tag) : 0; + up_hosts.put(tag, current + 1); + } + } else if (host.getStatus() == Status.Disconnected || host.getStatus() == Status.Down) { + for (String tag : hostTags) { + Integer current = down_hosts.get(tag) != null ? down_hosts.get(tag) : 0; + down_hosts.put(tag, current + 1); + } + } + // Get account, domain details for dedicated hosts if (isDedicated == 1) { String accountName; @@ -188,9 +214,24 @@ private void addHostMetrics(final List metricsList, final long dcId, final metricsList.add(new ItemVMCore(zoneName, zoneUuid, null, null, null, ALLOCATED, coreCapacity.get(0).getAllocatedCapacity() != null ? coreCapacity.get(0).getAllocatedCapacity() : 0, 0)); } - metricsList.add(new ItemHost(zoneName, zoneUuid, ONLINE, up)); - metricsList.add(new ItemHost(zoneName, zoneUuid, OFFLINE, down)); - metricsList.add(new ItemHost(zoneName, zoneUuid, TOTAL, total)); + metricsList.add(new ItemHost(zoneName, zoneUuid, ONLINE, up, null)); + metricsList.add(new ItemHost(zoneName, zoneUuid, OFFLINE, down, null)); + metricsList.add(new ItemHost(zoneName, zoneUuid, TOTAL, total, null)); + for (Map.Entry entry : total_hosts.entrySet()) { + String tag = entry.getKey(); + Integer count = entry.getValue(); + metricsList.add(new ItemHost(zoneName, zoneUuid, TOTAL, count, tag)); + if (up_hosts.get(tag) != null) { + metricsList.add(new ItemHost(zoneName, zoneUuid, ONLINE, up_hosts.get(tag), tag)); + } else { + metricsList.add(new ItemHost(zoneName, zoneUuid, ONLINE, 0, tag)); + } + if (down_hosts.get(tag) != null) { + metricsList.add(new ItemHost(zoneName, zoneUuid, OFFLINE, down_hosts.get(tag), tag)); + } else { + metricsList.add(new ItemHost(zoneName, zoneUuid, OFFLINE, 0, tag)); + } + } } private void addVMMetrics(final List metricsList, final long dcId, final String zoneName, final String zoneUuid) { @@ -423,17 +464,22 @@ class ItemHost extends Item { String zoneUuid; String state; int total; + String hosttags; - public ItemHost(final String zn, final String zu, final String st, int cnt) { + public ItemHost(final String zn, final String zu, final String st, int cnt, final String tags) { super("cloudstack_hosts_total"); zoneName = zn; zoneUuid = zu; state = st; total = cnt; + hosttags = tags; } @Override public String toMetricsString() { + if (! Strings.isNullOrEmpty(hosttags)) { + return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %d", name, zoneName, state, hosttags, total); + } return String.format("%s{zone=\"%s\",filter=\"%s\"} %d", name, zoneName, state, total); } } From d75b66f6f64c3c5afee0f72f3cd2a1e6e8aef3b6 Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Tue, 15 Sep 2020 09:58:17 +0200 Subject: [PATCH 02/17] Export count of vms by state and host tag. --- .../java/com/cloud/vm/dao/VMInstanceDao.java | 2 ++ .../com/cloud/vm/dao/VMInstanceDaoImpl.java | 21 ++++++++++++ .../metrics/PrometheusExporterImpl.java | 34 ++++++++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 4db07c0dcac4..bdb2534b62d0 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -133,6 +133,8 @@ public interface VMInstanceDao extends GenericDao, StateDao< Long countByZoneAndState(long zoneId, State state); + Long countByZoneAndStateAndHostTag(long dcId, State state, String hostTag); + List listNonRemovedVmsByTypeAndNetwork(long networkId, VirtualMachine.Type... types); /** diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 5701d7e39890..bae0e2d2ff2d 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -807,6 +807,27 @@ public Long countByZoneAndState(long zoneId, State state) { return customSearch(sc, null).get(0); } + @Override + public Long countByZoneAndStateAndHostTag(long dcId, State state, String hostTag) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + String sql = "SELECT COUNT(1) FROM vm_instance vi " + + "JOIN service_offering so ON vi.service_offering_id=so.id " + + "JOIN vm_template vt ON vi.vm_template_id = vt.id " + + "WHERE vi.data_center_id= " + dcId + " AND vi.state = '" + state + "' AND vi.removed IS NULL " + + "AND (so.host_tag='" + hostTag + "' OR vt.template_tag='" + hostTag + "')"; + PreparedStatement pstmt = null; + try { + pstmt = txn.prepareAutoCloseStatement(sql); + ResultSet rs = pstmt.executeQuery(); + if (rs.next()) { + return rs.getLong(1); + } + } catch (Exception e) { + s_logger.warn("Error counting vms by host tag", e); + } + return 0L; + } + @Override public List listNonRemovedVmsByTypeAndNetwork(long networkId, VirtualMachine.Type... types) { if (NetworkTypeSearch == null) { diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index 949c0c5da4f5..8f48c6fdd47a 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -49,12 +49,12 @@ import com.cloud.dc.DataCenterVO; import com.cloud.dc.Vlan; import com.cloud.dc.dao.DataCenterDao; -import com.cloud.host.dao.HostTagsDao; import com.cloud.dc.dao.DataCenterIpAddressDao; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostTagsDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.storage.ImageStore; import com.cloud.storage.StorageStats; @@ -242,6 +242,16 @@ private void addVMMetrics(final List metricsList, final long dcId, final S } metricsList.add(new ItemVM(zoneName, zoneUuid, state.name().toLowerCase(), count)); } + List allHostTags = new ArrayList(); + for (final HostVO host : hostDao.listAll()) { + allHostTags.addAll(_hostTagsDao.gethostTags(host.getId())); + } + for (final State state : State.values()) { + for (final String hosttag : allHostTags) { + final Long count = vmDao.countByZoneAndStateAndHostTag(dcId, state, hosttag); + metricsList.add(new ItemVMByTag(zoneName, zoneUuid, state.name().toLowerCase(), count, hosttag)); + } + } } private void addVolumeMetrics(final List metricsList, final long dcId, final String zoneName, final String zoneUuid) { @@ -439,6 +449,28 @@ public String toMetricsString() { } } + class ItemVMByTag extends Item { + String zoneName; + String zoneUuid; + String filter; + long total; + String hosttags; + + public ItemVMByTag(final String zn, final String zu, final String st, long cnt, final String tags) { + super("cloudstack_vms_total_by_tag"); + zoneName = zn; + zoneUuid = zu; + filter = st; + total = cnt; + hosttags = tags; + } + + @Override + public String toMetricsString() { + return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %d", name, zoneName, filter, hosttags, total); + } + } + class ItemVolume extends Item { String zoneName; String zoneUuid; From 1b24280a2cd6d788c873de4769ff71a7a2171f58 Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Thu, 17 Sep 2020 09:17:59 +0200 Subject: [PATCH 03/17] Add host tags to host cpu/cores/memory usage in Prometheus exporter --- .../metrics/PrometheusExporterImpl.java | 50 +++++++++++-------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index 8f48c6fdd47a..4d65a0bbdba1 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -61,6 +61,7 @@ import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.StringUtils; import com.cloud.utils.component.Manager; import com.cloud.utils.component.ManagerBase; import com.cloud.vm.VirtualMachine.State; @@ -141,6 +142,7 @@ private void addHostMetrics(final List metricsList, final long dcId, final metricsList.add(new ItemHostIsDedicated(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), isDedicated)); List hostTags = _hostTagsDao.gethostTags(host.getId()); + String hosttags = StringUtils.join(hostTags, ","); for (String tag : hostTags) { Integer current = total_hosts.get(tag) != null ? total_hosts.get(tag) : 0; total_hosts.put(tag, current + 1); @@ -170,48 +172,48 @@ private void addHostMetrics(final List metricsList, final long dcId, final final String cpuFactor = String.valueOf(CapacityManager.CpuOverprovisioningFactor.valueIn(host.getClusterId())); final CapacityVO cpuCapacity = capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_CPU); if (cpuCapacity != null) { - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, cpuCapacity.getUsedCapacity())); - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, cpuCapacity.getTotalCapacity())); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, cpuCapacity.getUsedCapacity(), hosttags)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, cpuCapacity.getTotalCapacity(), hosttags)); } else { - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, 0L)); - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, 0L)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, 0L, hosttags)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, 0L, hosttags)); } final String memoryFactor = String.valueOf(CapacityManager.MemOverprovisioningFactor.valueIn(host.getClusterId())); final CapacityVO memCapacity = capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_MEMORY); if (memCapacity != null) { - metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, USED, memCapacity.getUsedCapacity(), isDedicated)); - metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, TOTAL, memCapacity.getTotalCapacity(), isDedicated)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, USED, memCapacity.getUsedCapacity(), isDedicated, hosttags)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, TOTAL, memCapacity.getTotalCapacity(), isDedicated, hosttags)); } else { - metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, USED, 0L, isDedicated)); - metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, TOTAL, 0L, isDedicated)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, USED, 0L, isDedicated, hosttags)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, TOTAL, 0L, isDedicated, hosttags)); } metricsList.add(new ItemHostVM(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), vmDao.listByHostId(host.getId()).size())); final CapacityVO coreCapacity = capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_CPU_CORE); if (coreCapacity != null) { - metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), USED, coreCapacity.getUsedCapacity(), isDedicated)); - metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), TOTAL, coreCapacity.getTotalCapacity(), isDedicated)); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), USED, coreCapacity.getUsedCapacity(), isDedicated, hosttags)); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), TOTAL, coreCapacity.getTotalCapacity(), isDedicated, hosttags)); } else { - metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), USED, 0L, isDedicated)); - metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), TOTAL, 0L, isDedicated)); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), USED, 0L, isDedicated, hosttags)); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), TOTAL, 0L, isDedicated, hosttags)); } } final List cpuCapacity = capacityDao.findCapacityBy((int) Capacity.CAPACITY_TYPE_CPU, dcId, null, null); if (cpuCapacity != null && cpuCapacity.size() > 0) { - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, null, null, null, null, ALLOCATED, cpuCapacity.get(0).getAllocatedCapacity() != null ? cpuCapacity.get(0).getAllocatedCapacity() : 0)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, null, null, null, null, ALLOCATED, cpuCapacity.get(0).getAllocatedCapacity() != null ? cpuCapacity.get(0).getAllocatedCapacity() : 0, "")); } final List memCapacity = capacityDao.findCapacityBy((int) Capacity.CAPACITY_TYPE_MEMORY, dcId, null, null); if (memCapacity != null && memCapacity.size() > 0) { - metricsList.add(new ItemHostMemory(zoneName, zoneUuid, null, null, null, null, ALLOCATED, memCapacity.get(0).getAllocatedCapacity() != null ? memCapacity.get(0).getAllocatedCapacity() : 0, 0)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, null, null, null, null, ALLOCATED, memCapacity.get(0).getAllocatedCapacity() != null ? memCapacity.get(0).getAllocatedCapacity() : 0, 0, "")); } final List coreCapacity = capacityDao.findCapacityBy((int) Capacity.CAPACITY_TYPE_CPU_CORE, dcId, null, null); if (coreCapacity != null && coreCapacity.size() > 0) { - metricsList.add(new ItemVMCore(zoneName, zoneUuid, null, null, null, ALLOCATED, coreCapacity.get(0).getAllocatedCapacity() != null ? coreCapacity.get(0).getAllocatedCapacity() : 0, 0)); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, null, null, null, ALLOCATED, coreCapacity.get(0).getAllocatedCapacity() != null ? coreCapacity.get(0).getAllocatedCapacity() : 0, 0, "")); } metricsList.add(new ItemHost(zoneName, zoneUuid, ONLINE, up, null)); @@ -525,8 +527,9 @@ class ItemVMCore extends Item { String filter; long core = 0; int isDedicated; + String hosttags; - public ItemVMCore(final String zn, final String zu, final String hn, final String hu, final String hip, final String fl, final Long cr, final int dedicated) { + public ItemVMCore(final String zn, final String zu, final String hn, final String hu, final String hip, final String fl, final Long cr, final int dedicated, final String tags) { super("cloudstack_host_vms_cores_total"); zoneName = zn; zoneUuid = zu; @@ -538,6 +541,7 @@ public ItemVMCore(final String zn, final String zu, final String hn, final Strin core = cr; } isDedicated = dedicated; + hosttags = tags; } @Override @@ -545,7 +549,7 @@ public String toMetricsString() { if (StringUtils.isAllEmpty(hostName, ip)) { return String.format("%s{zone=\"%s\",filter=\"%s\"} %d", name, zoneName, filter, core); } - return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\",filter=\"%s\",dedicated=\"%d\"} %d", name, zoneName, hostName, ip, filter, isDedicated, core); + return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\",filter=\"%s\",dedicated=\"%d\",tags=\"%s\"} %d", name, zoneName, hostName, ip, filter, isDedicated, hosttags, core); } } @@ -558,8 +562,9 @@ class ItemHostCpu extends Item { String overProvisioningFactor; String filter; double mhertz; + String hosttags; - public ItemHostCpu(final String zn, final String zu, final String hn, final String hu, final String hip, final String of, final String fl, final double mh) { + public ItemHostCpu(final String zn, final String zu, final String hn, final String hu, final String hip, final String of, final String fl, final double mh, final String tags) { super("cloudstack_host_cpu_usage_mhz_total"); zoneName = zn; zoneUuid = zu; @@ -569,6 +574,7 @@ public ItemHostCpu(final String zn, final String zu, final String hn, final Stri overProvisioningFactor = of; filter = fl; mhertz = mh; + hosttags = tags; } @Override @@ -576,7 +582,7 @@ public String toMetricsString() { if (StringUtils.isAllEmpty(hostName, ip)) { return String.format("%s{zone=\"%s\",filter=\"%s\"} %.2f", name, zoneName, filter, mhertz); } - return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\",overprovisioningfactor=\"%s\",filter=\"%s\"} %.2f", name, zoneName, hostName, ip, overProvisioningFactor, filter, mhertz); + return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\",overprovisioningfactor=\"%s\",filter=\"%s\",tags=\"%s\"} %.2f", name, zoneName, hostName, ip, overProvisioningFactor, filter, hosttags, mhertz); } } @@ -590,8 +596,9 @@ class ItemHostMemory extends Item { String filter; double miBytes; int isDedicated; + String hosttags; - public ItemHostMemory(final String zn, final String zu, final String hn, final String hu, final String hip, final String of, final String fl, final double membytes, final int dedicated) { + public ItemHostMemory(final String zn, final String zu, final String hn, final String hu, final String hip, final String of, final String fl, final double membytes, final int dedicated, final String tags) { super("cloudstack_host_memory_usage_mibs_total"); zoneName = zn; zoneUuid = zu; @@ -602,6 +609,7 @@ public ItemHostMemory(final String zn, final String zu, final String hn, final S filter = fl; miBytes = membytes / (1024.0 * 1024.0); isDedicated = dedicated; + hosttags = tags; } @Override @@ -609,7 +617,7 @@ public String toMetricsString() { if (StringUtils.isAllEmpty(hostName, ip)) { return String.format("%s{zone=\"%s\",filter=\"%s\"} %.2f", name, zoneName, filter, miBytes); } - return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\",overprovisioningfactor=\"%s\",filter=\"%s\",dedicated=\"%d\"} %.2f", name, zoneName, hostName, ip, overProvisioningFactor, filter, isDedicated, miBytes); + return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\",overprovisioningfactor=\"%s\",filter=\"%s\",dedicated=\"%d\",tags=\"%s\"} %.2f", name, zoneName, hostName, ip, overProvisioningFactor, filter, isDedicated, hosttags, miBytes); } } From ab04936ef798554e942e3e63675f31cd078c130c Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Thu, 17 Sep 2020 13:39:48 +0200 Subject: [PATCH 04/17] Cloudstack Prometheus exporter: Add allocated capacity group by host tag. --- .../com/cloud/capacity/dao/CapacityDao.java | 3 ++ .../cloud/capacity/dao/CapacityDaoImpl.java | 53 ++++++++++++++++--- .../metrics/PrometheusExporterImpl.java | 45 ++++++++++++---- 3 files changed, 84 insertions(+), 17 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java index dc04f76f2f2f..459a63a7ba13 100644 --- a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java +++ b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java @@ -22,6 +22,7 @@ import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDaoImpl.SummedCapacity; import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; import com.cloud.utils.db.GenericDao; public interface CapacityDao extends GenericDao { @@ -39,6 +40,8 @@ public interface CapacityDao extends GenericDao { Pair, Map> orderClustersByAggregateCapacity(long id, long vmId, short capacityType, boolean isZone); + Ternary findCapacityByZoneAndHostTag(Long zoneId, String hostTag); + List findCapacityBy(Integer capacityType, Long zoneId, Long podId, Long clusterId); List listPodsByHostCapacities(long zoneId, int requiredCpu, long requiredRam, short capacityType); diff --git a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java index fe6f2f4ce7be..5f4775e1a758 100644 --- a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java @@ -37,6 +37,7 @@ import com.cloud.dc.ClusterDetailsDao; import com.cloud.storage.Storage; import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.JoinBuilder.JoinType; @@ -198,16 +199,18 @@ public class CapacityDaoImpl extends GenericDaoBase implements + "from op_host_capacity capacity where cluster_id = ? and capacity_type = ?;"; - private static final String LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE = "SELECT v.data_center_id, SUM(cpu) AS cpucore, " + + private static final String LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE_PART1 = "SELECT v.data_center_id, SUM(cpu) AS cpucore, " + "SUM(cpu * speed) AS cpu, SUM(ram_size * 1024 * 1024) AS memory " + "FROM (SELECT vi.data_center_id, (CASE WHEN ISNULL(service_offering.cpu) THEN custom_cpu.value ELSE service_offering.cpu end) AS cpu, " + "(CASE WHEN ISNULL(service_offering.speed) THEN custom_speed.value ELSE service_offering.speed end) AS speed, " + "(CASE WHEN ISNULL(service_offering.ram_size) THEN custom_ram_size.value ELSE service_offering.ram_size end) AS ram_size " + - "FROM (((vm_instance vi LEFT JOIN service_offering ON(((vi.service_offering_id = service_offering.id))) " + - "LEFT JOIN user_vm_details custom_cpu ON(((custom_cpu.vm_id = vi.id) AND (custom_cpu.name = 'CpuNumber')))) " + - "LEFT JOIN user_vm_details custom_speed ON(((custom_speed.vm_id = vi.id) AND (custom_speed.name = 'CpuSpeed')))) " + - "LEFT JOIN user_vm_details custom_ram_size ON(((custom_ram_size.vm_id = vi.id) AND (custom_ram_size.name = 'memory')))) " + - "WHERE ISNULL(vi.removed) AND vi.state NOT IN ('Destroyed', 'Error', 'Expunging')"; + "FROM vm_instance vi LEFT JOIN service_offering ON(((vi.service_offering_id = service_offering.id))) " + + "LEFT JOIN user_vm_details custom_cpu ON(((custom_cpu.vm_id = vi.id) AND (custom_cpu.name = 'CpuNumber'))) " + + "LEFT JOIN user_vm_details custom_speed ON(((custom_speed.vm_id = vi.id) AND (custom_speed.name = 'CpuSpeed'))) " + + "LEFT JOIN user_vm_details custom_ram_size ON(((custom_ram_size.vm_id = vi.id) AND (custom_ram_size.name = 'memory'))) "; + + private static final String LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE_PART2 = + "WHERE ISNULL(vi.removed) AND vi.state NOT IN ('Destroyed', 'Error', 'Expunging')"; public CapacityDaoImpl() { _hostIdTypeSearch = createSearchBuilder(); @@ -422,6 +425,41 @@ public List listCapacitiesGroupedByLevelAndType(Integer capacity } + @Override + public Ternary findCapacityByZoneAndHostTag(Long zoneId, String hostTag) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + PreparedStatement pstmt = null; + List results = new ArrayList(); + + StringBuilder allocatedSql = new StringBuilder(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE_PART1); + if (hostTag != null && ! hostTag.isEmpty()) { + allocatedSql.append("LEFT JOIN vm_template ON vm_template.id = vi.vm_template_id "); + } + allocatedSql.append(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE_PART2); + if (zoneId != null){ + allocatedSql.append(" AND vi.data_center_id = ?"); + } + if (hostTag != null && ! hostTag.isEmpty()) { + allocatedSql.append(" AND (vm_template.template_tag = '").append(hostTag).append("'"); + allocatedSql.append(" OR service_offering.host_tag = '").append(hostTag).append("')"); + } + allocatedSql.append(" ) AS v GROUP BY v.data_center_id"); + try { + // add allocated capacity of zone in result + pstmt = txn.prepareAutoCloseStatement(allocatedSql.toString()); + if (zoneId != null){ + pstmt.setLong(1, zoneId); + } + ResultSet rs = pstmt.executeQuery(); + if (rs.next()) { + return new Ternary(rs.getLong(2), rs.getLong(3), rs.getLong(4)); // cpu cores, cpu, memory + } + return new Ternary(0L, 0L, 0L); + } catch (SQLException e) { + throw new CloudRuntimeException("DB Exception on: " + allocatedSql, e); + } + } + @Override public List findCapacityBy(Integer capacityType, Long zoneId, Long podId, Long clusterId) { @@ -429,7 +467,8 @@ public List findCapacityBy(Integer capacityType, Long zoneId, Lo PreparedStatement pstmt = null; List results = new ArrayList(); - StringBuilder allocatedSql = new StringBuilder(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE); + StringBuilder allocatedSql = new StringBuilder(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE_PART1); + allocatedSql.append(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE_PART2); HashMap sumCpuCore = new HashMap(); HashMap sumCpu = new HashMap(); diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index 4d65a0bbdba1..fcdc8b78968e 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -62,11 +62,11 @@ import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.StringUtils; +import com.cloud.utils.Ternary; import com.cloud.utils.component.Manager; import com.cloud.utils.component.ManagerBase; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.dao.VMInstanceDao; -import org.apache.commons.lang3.StringUtils; public class PrometheusExporterImpl extends ManagerBase implements PrometheusExporter, Manager { private static final Logger LOG = Logger.getLogger(PrometheusExporterImpl.class); @@ -172,11 +172,11 @@ private void addHostMetrics(final List metricsList, final long dcId, final final String cpuFactor = String.valueOf(CapacityManager.CpuOverprovisioningFactor.valueIn(host.getClusterId())); final CapacityVO cpuCapacity = capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_CPU); if (cpuCapacity != null) { - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, cpuCapacity.getUsedCapacity(), hosttags)); - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, cpuCapacity.getTotalCapacity(), hosttags)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, cpuCapacity.getUsedCapacity(), isDedicated, hosttags)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, cpuCapacity.getTotalCapacity(), isDedicated, hosttags)); } else { - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, 0L, hosttags)); - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, 0L, hosttags)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, 0L, isDedicated, hosttags)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, 0L, isDedicated, hosttags)); } final String memoryFactor = String.valueOf(CapacityManager.MemOverprovisioningFactor.valueIn(host.getClusterId())); @@ -203,7 +203,7 @@ private void addHostMetrics(final List metricsList, final long dcId, final final List cpuCapacity = capacityDao.findCapacityBy((int) Capacity.CAPACITY_TYPE_CPU, dcId, null, null); if (cpuCapacity != null && cpuCapacity.size() > 0) { - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, null, null, null, null, ALLOCATED, cpuCapacity.get(0).getAllocatedCapacity() != null ? cpuCapacity.get(0).getAllocatedCapacity() : 0, "")); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, null, null, null, null, ALLOCATED, cpuCapacity.get(0).getAllocatedCapacity() != null ? cpuCapacity.get(0).getAllocatedCapacity() : 0, 0, "")); } final List memCapacity = capacityDao.findCapacityBy((int) Capacity.CAPACITY_TYPE_MEMORY, dcId, null, null); @@ -234,6 +234,13 @@ private void addHostMetrics(final List metricsList, final long dcId, final metricsList.add(new ItemHost(zoneName, zoneUuid, OFFLINE, 0, tag)); } } + for (Map.Entry entry : total_hosts.entrySet()) { + String tag = entry.getKey(); + Ternary allocatedCapacityByTag = capacityDao.findCapacityByZoneAndHostTag(dcId, tag); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, null, null, null, ALLOCATED, allocatedCapacityByTag.first(), 0, tag)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, null, null, null, null, ALLOCATED, allocatedCapacityByTag.second(),0, tag)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, null, null, null, null, ALLOCATED, allocatedCapacityByTag.third(), 0, tag)); + } } private void addVMMetrics(final List metricsList, final long dcId, final String zoneName, final String zoneUuid) { @@ -512,6 +519,7 @@ public ItemHost(final String zn, final String zu, final String st, int cnt, fina @Override public String toMetricsString() { if (! Strings.isNullOrEmpty(hosttags)) { + name = "cloudstack_hosts_total_by_tag"; return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %d", name, zoneName, state, hosttags, total); } return String.format("%s{zone=\"%s\",filter=\"%s\"} %d", name, zoneName, state, total); @@ -547,7 +555,12 @@ public ItemVMCore(final String zn, final String zu, final String hn, final Strin @Override public String toMetricsString() { if (StringUtils.isAllEmpty(hostName, ip)) { - return String.format("%s{zone=\"%s\",filter=\"%s\"} %d", name, zoneName, filter, core); + if (Strings.isNullOrEmpty(hosttags)) { + return String.format("%s{zone=\"%s\",filter=\"%s\"} %d", name, zoneName, filter, core); + } else { + name = "cloudstack_host_vms_cores_total_by_tag"; + return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %d", name, zoneName, filter, hosttags, core); + } } return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\",filter=\"%s\",dedicated=\"%d\",tags=\"%s\"} %d", name, zoneName, hostName, ip, filter, isDedicated, hosttags, core); } @@ -562,9 +575,10 @@ class ItemHostCpu extends Item { String overProvisioningFactor; String filter; double mhertz; + int isDedicated; String hosttags; - public ItemHostCpu(final String zn, final String zu, final String hn, final String hu, final String hip, final String of, final String fl, final double mh, final String tags) { + public ItemHostCpu(final String zn, final String zu, final String hn, final String hu, final String hip, final String of, final String fl, final double mh, final int dedicated, final String tags) { super("cloudstack_host_cpu_usage_mhz_total"); zoneName = zn; zoneUuid = zu; @@ -574,13 +588,19 @@ public ItemHostCpu(final String zn, final String zu, final String hn, final Stri overProvisioningFactor = of; filter = fl; mhertz = mh; + isDedicated = dedicated; hosttags = tags; } @Override public String toMetricsString() { if (StringUtils.isAllEmpty(hostName, ip)) { - return String.format("%s{zone=\"%s\",filter=\"%s\"} %.2f", name, zoneName, filter, mhertz); + if (Strings.isNullOrEmpty(hosttags)) { + return String.format("%s{zone=\"%s\",filter=\"%s\"} %.2f", name, zoneName, filter, mhertz); + } else { + name = "cloudstack_host_cpu_usage_mhz_total_by_tag"; + return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %.2f", name, zoneName, filter, hosttags, mhertz); + } } return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\",overprovisioningfactor=\"%s\",filter=\"%s\",tags=\"%s\"} %.2f", name, zoneName, hostName, ip, overProvisioningFactor, filter, hosttags, mhertz); } @@ -615,7 +635,12 @@ public ItemHostMemory(final String zn, final String zu, final String hn, final S @Override public String toMetricsString() { if (StringUtils.isAllEmpty(hostName, ip)) { - return String.format("%s{zone=\"%s\",filter=\"%s\"} %.2f", name, zoneName, filter, miBytes); + if (Strings.isNullOrEmpty(hosttags)) { + return String.format("%s{zone=\"%s\",filter=\"%s\"} %.2f", name, zoneName, filter, miBytes); + } else { + name = "cloudstack_host_memory_usage_mibs_total_by_tag"; + return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %.2f", name, zoneName, filter, hosttags, miBytes); + } } return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\",overprovisioningfactor=\"%s\",filter=\"%s\",dedicated=\"%d\",tags=\"%s\"} %.2f", name, zoneName, hostName, ip, overProvisioningFactor, filter, isDedicated, hosttags, miBytes); } From 230d3d76e1adeec8e0785d76e4e8079b3aece12f Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Mon, 21 Sep 2020 17:29:33 +0200 Subject: [PATCH 05/17] Show count of Active domains on grafana. --- .../java/com/cloud/user/dao/AccountDao.java | 1 + .../com/cloud/user/dao/AccountDaoImpl.java | 16 ++++++++++- .../metrics/PrometheusExporterImpl.java | 27 ++++++++++++++++++- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java b/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java index d7754e1b935c..17b07496731e 100644 --- a/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java +++ b/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java @@ -78,4 +78,5 @@ public interface AccountDao extends GenericDao { */ long getDomainIdForGivenAccountId(long id); + int getActiveDomains(); } diff --git a/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java b/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java index 62d409c9af0b..3dacbb70f394 100644 --- a/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java @@ -28,6 +28,7 @@ import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.SearchCriteria.Func; import com.cloud.utils.db.SearchCriteria.Op; import org.apache.commons.lang3.StringUtils; import com.cloud.utils.db.TransactionLegacy; @@ -54,6 +55,7 @@ public class AccountDaoImpl extends GenericDaoBase implements A protected final SearchBuilder NonProjectAccountSearch; protected final SearchBuilder AccountByRoleSearch; protected final GenericSearchBuilder AccountIdsSearch; + protected final GenericSearchBuilder ActiveDomainCount; public AccountDaoImpl() { AllFieldsSearch = createSearchBuilder(); @@ -101,6 +103,13 @@ public AccountDaoImpl() { AccountByRoleSearch = createSearchBuilder(); AccountByRoleSearch.and("roleId", AccountByRoleSearch.entity().getRoleId(), SearchCriteria.Op.EQ); AccountByRoleSearch.done(); + + ActiveDomainCount = createSearchBuilder(Long.class); + ActiveDomainCount.select(null, Func.COUNT, null); + ActiveDomainCount.and("domain", ActiveDomainCount.entity().getDomainId(), SearchCriteria.Op.EQ); + ActiveDomainCount.and("state", ActiveDomainCount.entity().getState(), SearchCriteria.Op.EQ); + ActiveDomainCount.groupBy(ActiveDomainCount.entity().getDomainId()); + ActiveDomainCount.done(); } @Override @@ -318,5 +327,10 @@ public long getDomainIdForGivenAccountId(long id) { } } - + @Override + public int getActiveDomains() { + SearchCriteria sc = ActiveDomainCount.create(); + sc.setParameters("state", "enabled"); + return customSearch(sc, null).size(); + } } diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index fcdc8b78968e..745c19169f06 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -54,7 +54,6 @@ import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; -import com.cloud.host.dao.HostTagsDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.storage.ImageStore; import com.cloud.storage.StorageStats; @@ -66,6 +65,7 @@ import com.cloud.utils.component.Manager; import com.cloud.utils.component.ManagerBase; import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; public class PrometheusExporterImpl extends ManagerBase implements PrometheusExporter, Manager { @@ -89,6 +89,8 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp @Inject private VMInstanceDao vmDao; @Inject + private UserVmDao uservmDao; + @Inject private VolumeDao volumeDao; @Inject private IPAddressDao publicIpAddressDao; @@ -395,6 +397,10 @@ private void addDomainResourceCount(final List metricsList) { } } + private void addDomainMetrics(final List metricsList, final String zoneName, final String zoneUuid) { + metricsList.add(new ItemActiveDomains(zoneName, zoneUuid, _accountDao.getActiveDomains())); + } + @Override public void updateMetrics() { final List latestMetricsItems = new ArrayList(); @@ -409,6 +415,7 @@ public void updateMetrics() { addStorageMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); addIpAddressMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); addVlanMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); + addDomainMetrics(latestMetricsItems, zoneName, zoneUuid); } addDomainLimits(latestMetricsItems); addDomainResourceCount(latestMetricsItems); @@ -834,6 +841,24 @@ public String toMetricsString() { } + class ItemActiveDomains extends Item { + String zoneName; + String zoneUuid; + int total; + + public ItemActiveDomains(final String zn, final String zu, final int cnt) { + super("cloudstack_active_domains_total"); + zoneName = zn; + zoneUuid = zu; + total = cnt; + } + + @Override + public String toMetricsString() { + return String.format("%s{zone=\"%s\"} %d", name, zoneName, total); + } + } + class ItemHostDedicatedToAccount extends Item { String zoneName; String hostName; From 1687b2880fb8360b3db573796aa34b50bb3595b8 Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Thu, 24 Sep 2020 20:48:32 +0200 Subject: [PATCH 06/17] Show count of Active accounts and vms by size on grafana --- .../main/java/com/cloud/vm/dao/UserVmDao.java | 5 ++ .../java/com/cloud/vm/dao/UserVmDaoImpl.java | 42 +++++++++++++++ .../metrics/PrometheusExporterImpl.java | 53 +++++++++++++++++++ .../metrics/PrometheusExporterServer.java | 3 ++ .../metrics/PrometheusExporterServerImpl.java | 3 +- 5 files changed, 105 insertions(+), 1 deletion(-) diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDao.java index c53936f7d2d3..f4fa5c1d405b 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDao.java @@ -22,6 +22,7 @@ import java.util.Set; import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; import com.cloud.utils.db.GenericDao; import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachine; @@ -92,4 +93,8 @@ public interface UserVmDao extends GenericDao { List listByIsoId(Long isoId); List, Pair>> getVmsDetailByNames(Set vmNames, String detail); + + List> countVmsBySize(long dcId, int limit); + + int getActiveAccounts(final long dcId); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java index f6b99754c80c..e04ccf567d9e 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java @@ -40,6 +40,7 @@ import com.cloud.tags.dao.ResourceTagDao; import com.cloud.user.Account; import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; import com.cloud.utils.db.Attribute; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; @@ -75,6 +76,7 @@ public class UserVmDaoImpl extends GenericDaoBase implements Use protected SearchBuilder AccountDataCenterVirtualSearch; protected GenericSearchBuilder CountByAccountPod; protected GenericSearchBuilder CountByAccount; + protected GenericSearchBuilder CountActiveAccount; protected GenericSearchBuilder PodsHavingVmsForAccount; protected SearchBuilder UserVmSearch; @@ -192,6 +194,15 @@ void init() { CountByAccount.and("displayVm", CountByAccount.entity().isDisplayVm(), SearchCriteria.Op.EQ); CountByAccount.done(); + CountActiveAccount = createSearchBuilder(Long.class); + CountActiveAccount.select(null, Func.COUNT, null); + CountActiveAccount.and("account", CountActiveAccount.entity().getAccountId(), SearchCriteria.Op.EQ); + CountActiveAccount.and("type", CountActiveAccount.entity().getType(), SearchCriteria.Op.EQ); + CountActiveAccount.and("dataCenterId", CountActiveAccount.entity().getDataCenterId(), SearchCriteria.Op.EQ); + CountActiveAccount.and("state", CountActiveAccount.entity().getState(), SearchCriteria.Op.NIN); + CountActiveAccount.groupBy(CountActiveAccount.entity().getAccountId()); + CountActiveAccount.done(); + SearchBuilder nicSearch = _nicDao.createSearchBuilder(); nicSearch.and("networkId", nicSearch.entity().getNetworkId(), SearchCriteria.Op.EQ); nicSearch.and("ip4Address", nicSearch.entity().getIPv4Address(), SearchCriteria.Op.NNULL); @@ -721,4 +732,35 @@ public List, Pair>> getVmsD return vmsDetailByNames; } + + @Override + public List> countVmsBySize(long dcId, int limit) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + String sql = "SELECT cpu,ram_size,count(1) AS count FROM (SELECT * FROM user_vm_view WHERE data_center_id = ? AND state NOT IN ('Destroyed', 'Error', 'Expunging') GROUP BY id) AS uvv GROUP BY cpu,ram_size ORDER BY count DESC "; + if (limit >= 0) + sql = sql + "limit " + limit; + PreparedStatement pstmt = null; + List> result = new ArrayList<>(); + try { + pstmt = txn.prepareAutoCloseStatement(sql); + pstmt.setLong(1, dcId); + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + result.add(new Ternary(rs.getInt(1), rs.getInt(2), rs.getInt(3))); + } + } catch (Exception e) { + s_logger.warn("Error counting vms by size", e); + } + return result; + } + + @Override + public int getActiveAccounts(final long dcId) { + SearchCriteria sc = CountActiveAccount.create(); + sc.setParameters("type", VirtualMachine.Type.User); + sc.setParameters("state", State.Destroyed, State.Error, State.Expunging, State.Stopped); + sc.setParameters("dataCenterId", dcId); + + return customSearch(sc, null).size(); + } } diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index 745c19169f06..d1250235f388 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -401,6 +401,17 @@ private void addDomainMetrics(final List metricsList, final String zoneNam metricsList.add(new ItemActiveDomains(zoneName, zoneUuid, _accountDao.getActiveDomains())); } + private void addAccountMetrics(final List metricsList, final long dcId, final String zoneName, final String zoneUuid) { + metricsList.add(new ItemActiveAccounts(zoneName, zoneUuid, uservmDao.getActiveAccounts(dcId))); + } + + private void addVMsBySizeMetrics(final List metricsList, final long dcId, final String zoneName, final String zoneUuid) { + List> vms = uservmDao.countVmsBySize(dcId, PrometheusExporterServer.PrometheusExporterOfferingCountLimit.value()); + for (Ternary vm : vms) { + metricsList.add(new ItemVMsBySize(zoneName, zoneUuid, vm.first(), vm.second(), vm.third())); + } + } + @Override public void updateMetrics() { final List latestMetricsItems = new ArrayList(); @@ -416,6 +427,8 @@ public void updateMetrics() { addIpAddressMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); addVlanMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); addDomainMetrics(latestMetricsItems, zoneName, zoneUuid); + addAccountMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); + addVMsBySizeMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); } addDomainLimits(latestMetricsItems); addDomainResourceCount(latestMetricsItems); @@ -918,4 +931,44 @@ public String toMetricsString() { return String.format("%s{domain=\"%s\", type=\"%s\"} %d", name, domainName, resourceType, miBytes); } } + + class ItemActiveAccounts extends Item { + String zoneName; + String zoneUuid; + int total; + + public ItemActiveAccounts(final String zn, final String zu, final int cnt) { + super("cloudstack_active_accounts_total"); + zoneName = zn; + zoneUuid = zu; + total = cnt; + } + + @Override + public String toMetricsString() { + return String.format("%s{zone=\"%s\"} %d", name, zoneName, total); + } + } + + class ItemVMsBySize extends Item { + String zoneName; + String zoneUuid; + int cpu; + int memory; + int total; + + public ItemVMsBySize(final String zn, final String zu, final int c, final int m, int cnt) { + super("cloudstack_vms_total_by_size"); + zoneName = zn; + zoneUuid = zu; + cpu = c; + memory = m; + total = cnt; + } + + @Override + public String toMetricsString() { + return String.format("%s{zone=\"%s\",cpu=\"%d\",memory=\"%d\"} %d", name, zoneName, cpu, memory, total); + } + } } diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServer.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServer.java index 12d513c92d7f..0ec83066f619 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServer.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServer.java @@ -30,4 +30,7 @@ public interface PrometheusExporterServer extends Manager { ConfigKey PrometheusExporterAllowedAddresses = new ConfigKey<>("Advanced", String.class, "prometheus.exporter.allowed.ips", "127.0.0.1", "List of comma separated prometheus server ips (with no spaces) that should be allowed to access the URLs", true); + + ConfigKey PrometheusExporterOfferingCountLimit = new ConfigKey<>("Advanced", Integer.class, "prometheus.exporter.offering.output.limit", "-1", + "Limit the number of output for cloudstack_vms_total_by_size to the provided value. -1 for unlimited output.", true); } diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java index a615c65766b0..d550e2a3c910 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java @@ -112,7 +112,8 @@ public ConfigKey[] getConfigKeys() { return new ConfigKey[] { EnablePrometheusExporter, PrometheusExporterServerPort, - PrometheusExporterAllowedAddresses + PrometheusExporterAllowedAddresses, + PrometheusExporterOfferingCountLimit }; } } From 6a6e18be59b3bcef870f4f25de4b25ab94aeaf70 Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Fri, 30 Oct 2020 01:49:53 +0100 Subject: [PATCH 07/17] Fixed HostTagsDao import issue and removed unnecessary fields. --- .../org/apache/cloudstack/metrics/PrometheusExporterImpl.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index d1250235f388..d2f7a3e8c64b 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -36,7 +36,6 @@ import com.cloud.alert.AlertManager; import com.cloud.api.ApiDBUtils; import com.cloud.api.query.dao.DomainJoinDao; -import com.cloud.api.query.dao.HostJoinDao; import com.cloud.api.query.dao.StoragePoolJoinDao; import com.cloud.api.query.vo.DomainJoinVO; import com.cloud.api.query.vo.StoragePoolJoinVO; @@ -54,6 +53,7 @@ import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostTagsDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.storage.ImageStore; import com.cloud.storage.StorageStats; @@ -85,8 +85,6 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp @Inject private HostDao hostDao; @Inject - private HostJoinDao hostJoinDao; - @Inject private VMInstanceDao vmDao; @Inject private UserVmDao uservmDao; From 7a897d33d6c97d79a0fe9626e0e6d00b224b41ff Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Mon, 9 Aug 2021 17:56:11 +0200 Subject: [PATCH 08/17] Change variable names with regards to DaanHoogland comments. --- .../cloud/capacity/dao/CapacityDaoImpl.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java index 5f4775e1a758..69752cc3d99a 100644 --- a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java @@ -199,7 +199,7 @@ public class CapacityDaoImpl extends GenericDaoBase implements + "from op_host_capacity capacity where cluster_id = ? and capacity_type = ?;"; - private static final String LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE_PART1 = "SELECT v.data_center_id, SUM(cpu) AS cpucore, " + + private static final String LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE = "SELECT v.data_center_id, SUM(cpu) AS cpucore, " + "SUM(cpu * speed) AS cpu, SUM(ram_size * 1024 * 1024) AS memory " + "FROM (SELECT vi.data_center_id, (CASE WHEN ISNULL(service_offering.cpu) THEN custom_cpu.value ELSE service_offering.cpu end) AS cpu, " + "(CASE WHEN ISNULL(service_offering.speed) THEN custom_speed.value ELSE service_offering.speed end) AS speed, " + @@ -209,9 +209,11 @@ public class CapacityDaoImpl extends GenericDaoBase implements "LEFT JOIN user_vm_details custom_speed ON(((custom_speed.vm_id = vi.id) AND (custom_speed.name = 'CpuSpeed'))) " + "LEFT JOIN user_vm_details custom_ram_size ON(((custom_ram_size.vm_id = vi.id) AND (custom_ram_size.name = 'memory'))) "; - private static final String LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE_PART2 = + private static final String WHERE_STATE_IS_NOT_DESTRUCTIVE = "WHERE ISNULL(vi.removed) AND vi.state NOT IN ('Destroyed', 'Error', 'Expunging')"; + private static final String LEFT_JOIN_VM_TEMPLATE = "LEFT JOIN vm_template ON vm_template.id = vi.vm_template_id "; + public CapacityDaoImpl() { _hostIdTypeSearch = createSearchBuilder(); _hostIdTypeSearch.and("hostId", _hostIdTypeSearch.entity().getHostOrPoolId(), SearchCriteria.Op.EQ); @@ -428,14 +430,13 @@ public List listCapacitiesGroupedByLevelAndType(Integer capacity @Override public Ternary findCapacityByZoneAndHostTag(Long zoneId, String hostTag) { TransactionLegacy txn = TransactionLegacy.currentTxn(); - PreparedStatement pstmt = null; - List results = new ArrayList(); + PreparedStatement pstmt; - StringBuilder allocatedSql = new StringBuilder(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE_PART1); + StringBuilder allocatedSql = new StringBuilder(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE); if (hostTag != null && ! hostTag.isEmpty()) { - allocatedSql.append("LEFT JOIN vm_template ON vm_template.id = vi.vm_template_id "); + allocatedSql.append(LEFT_JOIN_VM_TEMPLATE); } - allocatedSql.append(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE_PART2); + allocatedSql.append(WHERE_STATE_IS_NOT_DESTRUCTIVE); if (zoneId != null){ allocatedSql.append(" AND vi.data_center_id = ?"); } @@ -444,6 +445,7 @@ public Ternary findCapacityByZoneAndHostTag(Long zoneId, Strin allocatedSql.append(" OR service_offering.host_tag = '").append(hostTag).append("')"); } allocatedSql.append(" ) AS v GROUP BY v.data_center_id"); + try { // add allocated capacity of zone in result pstmt = txn.prepareAutoCloseStatement(allocatedSql.toString()); @@ -452,9 +454,9 @@ public Ternary findCapacityByZoneAndHostTag(Long zoneId, Strin } ResultSet rs = pstmt.executeQuery(); if (rs.next()) { - return new Ternary(rs.getLong(2), rs.getLong(3), rs.getLong(4)); // cpu cores, cpu, memory + return new Ternary<>(rs.getLong(2), rs.getLong(3), rs.getLong(4)); // cpu cores, cpu, memory } - return new Ternary(0L, 0L, 0L); + return new Ternary<>(0L, 0L, 0L); } catch (SQLException e) { throw new CloudRuntimeException("DB Exception on: " + allocatedSql, e); } @@ -467,8 +469,8 @@ public List findCapacityBy(Integer capacityType, Long zoneId, Lo PreparedStatement pstmt = null; List results = new ArrayList(); - StringBuilder allocatedSql = new StringBuilder(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE_PART1); - allocatedSql.append(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE_PART2); + StringBuilder allocatedSql = new StringBuilder(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE); + allocatedSql.append(WHERE_STATE_IS_NOT_DESTRUCTIVE); HashMap sumCpuCore = new HashMap(); HashMap sumCpu = new HashMap(); From d41b99c26771ff175ae2e695f3f7d4256e68d49f Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Thu, 23 Sep 2021 11:42:42 +0200 Subject: [PATCH 09/17] Moved to commons.lang3 as a default string lib. --- .../cloudstack/metrics/PrometheusExporterImpl.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index d2f7a3e8c64b..5f0391e127db 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -60,7 +60,7 @@ import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; -import com.cloud.utils.StringUtils; +import org.apache.commons.lang3.StringUtils; import com.cloud.utils.Ternary; import com.cloud.utils.component.Manager; import com.cloud.utils.component.ManagerBase; @@ -536,7 +536,7 @@ public ItemHost(final String zn, final String zu, final String st, int cnt, fina @Override public String toMetricsString() { - if (! Strings.isNullOrEmpty(hosttags)) { + if (StringUtils.isNotEmpty(hosttags)) { name = "cloudstack_hosts_total_by_tag"; return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %d", name, zoneName, state, hosttags, total); } @@ -573,7 +573,7 @@ public ItemVMCore(final String zn, final String zu, final String hn, final Strin @Override public String toMetricsString() { if (StringUtils.isAllEmpty(hostName, ip)) { - if (Strings.isNullOrEmpty(hosttags)) { + if (StringUtils.isEmpty(hosttags)) { return String.format("%s{zone=\"%s\",filter=\"%s\"} %d", name, zoneName, filter, core); } else { name = "cloudstack_host_vms_cores_total_by_tag"; @@ -613,7 +613,7 @@ public ItemHostCpu(final String zn, final String zu, final String hn, final Stri @Override public String toMetricsString() { if (StringUtils.isAllEmpty(hostName, ip)) { - if (Strings.isNullOrEmpty(hosttags)) { + if (StringUtils.isEmpty(hosttags)) { return String.format("%s{zone=\"%s\",filter=\"%s\"} %.2f", name, zoneName, filter, mhertz); } else { name = "cloudstack_host_cpu_usage_mhz_total_by_tag"; @@ -653,7 +653,7 @@ public ItemHostMemory(final String zn, final String zu, final String hn, final S @Override public String toMetricsString() { if (StringUtils.isAllEmpty(hostName, ip)) { - if (Strings.isNullOrEmpty(hosttags)) { + if (StringUtils.isEmpty(hosttags)) { return String.format("%s{zone=\"%s\",filter=\"%s\"} %.2f", name, zoneName, filter, miBytes); } else { name = "cloudstack_host_memory_usage_mibs_total_by_tag"; From cbd780d770fe4efa27098e0b4fa9aaa1f8baf87a Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Thu, 23 Sep 2021 11:50:13 +0200 Subject: [PATCH 10/17] Fixed spaces. --- .../src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java index 69752cc3d99a..e32be551264c 100644 --- a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java @@ -437,7 +437,7 @@ public Ternary findCapacityByZoneAndHostTag(Long zoneId, Strin allocatedSql.append(LEFT_JOIN_VM_TEMPLATE); } allocatedSql.append(WHERE_STATE_IS_NOT_DESTRUCTIVE); - if (zoneId != null){ + if (zoneId != null) { allocatedSql.append(" AND vi.data_center_id = ?"); } if (hostTag != null && ! hostTag.isEmpty()) { @@ -449,7 +449,7 @@ public Ternary findCapacityByZoneAndHostTag(Long zoneId, Strin try { // add allocated capacity of zone in result pstmt = txn.prepareAutoCloseStatement(allocatedSql.toString()); - if (zoneId != null){ + if (zoneId != null) { pstmt.setLong(1, zoneId); } ResultSet rs = pstmt.executeQuery(); From fed9b6aea704c1e5f26d26d48617305de84a2452 Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Thu, 23 Sep 2021 15:07:22 +0200 Subject: [PATCH 11/17] Updated pr to address @GutoVeronezi change requests. --- .../main/java/com/cloud/capacity/dao/CapacityDaoImpl.java | 5 +++-- .../schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java | 2 +- .../src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java index e32be551264c..302ffd8e760a 100644 --- a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java @@ -27,6 +27,7 @@ import javax.inject.Inject; import org.apache.log4j.Logger; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -433,14 +434,14 @@ public Ternary findCapacityByZoneAndHostTag(Long zoneId, Strin PreparedStatement pstmt; StringBuilder allocatedSql = new StringBuilder(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE); - if (hostTag != null && ! hostTag.isEmpty()) { + if (StringUtils.isNotEmpty(hostTag)) { allocatedSql.append(LEFT_JOIN_VM_TEMPLATE); } allocatedSql.append(WHERE_STATE_IS_NOT_DESTRUCTIVE); if (zoneId != null) { allocatedSql.append(" AND vi.data_center_id = ?"); } - if (hostTag != null && ! hostTag.isEmpty()) { + if (StringUtils.isNotEmpty(hostTag)) { allocatedSql.append(" AND (vm_template.template_tag = '").append(hostTag).append("'"); allocatedSql.append(" OR service_offering.host_tag = '").append(hostTag).append("')"); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java index e04ccf567d9e..af8c0a0988aa 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java @@ -749,7 +749,7 @@ public List> countVmsBySize(long dcId, int li result.add(new Ternary(rs.getInt(1), rs.getInt(2), rs.getInt(3))); } } catch (Exception e) { - s_logger.warn("Error counting vms by size", e); + s_logger.warn("Error counting vms by size for dcId= " + dcId, e); } return result; } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index bae0e2d2ff2d..880238c67cb4 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -823,7 +823,7 @@ public Long countByZoneAndStateAndHostTag(long dcId, State state, String hostTag return rs.getLong(1); } } catch (Exception e) { - s_logger.warn("Error counting vms by host tag", e); + s_logger.warn(String.format("Error counting vms by host tag for dcId= %s, hostTag= %s", dcId, hostTag), e); } return 0L; } From 633ebac886524bb41bbc02ab3dce9cd72c5c05a5 Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Mon, 27 Sep 2021 10:25:57 +0200 Subject: [PATCH 12/17] Fixed camel case issue. --- .../metrics/PrometheusExporterImpl.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index 5f0391e127db..39098cd1e5b0 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -121,9 +121,9 @@ private void addHostMetrics(final List metricsList, final long dcId, final int total = 0; int up = 0; int down = 0; - Map total_hosts = new HashMap<>(); - Map up_hosts = new HashMap<>(); - Map down_hosts = new HashMap<>(); + Map totalHosts = new HashMap<>(); + Map upHosts = new HashMap<>(); + Map downHosts = new HashMap<>(); for (final HostVO host : hostDao.listAll()) { if (host == null || host.getType() != Host.Type.Routing || host.getDataCenterId() != dcId) { @@ -144,18 +144,18 @@ private void addHostMetrics(final List metricsList, final long dcId, final List hostTags = _hostTagsDao.gethostTags(host.getId()); String hosttags = StringUtils.join(hostTags, ","); for (String tag : hostTags) { - Integer current = total_hosts.get(tag) != null ? total_hosts.get(tag) : 0; - total_hosts.put(tag, current + 1); + Integer current = totalHosts.get(tag) != null ? totalHosts.get(tag) : 0; + totalHosts.put(tag, current + 1); } if (host.getStatus() == Status.Up) { for (String tag : hostTags) { - Integer current = up_hosts.get(tag) != null ? up_hosts.get(tag) : 0; - up_hosts.put(tag, current + 1); + Integer current = upHosts.get(tag) != null ? upHosts.get(tag) : 0; + upHosts.put(tag, current + 1); } } else if (host.getStatus() == Status.Disconnected || host.getStatus() == Status.Down) { for (String tag : hostTags) { - Integer current = down_hosts.get(tag) != null ? down_hosts.get(tag) : 0; - down_hosts.put(tag, current + 1); + Integer current = downHosts.get(tag) != null ? downHosts.get(tag) : 0; + downHosts.put(tag, current + 1); } } @@ -219,22 +219,22 @@ private void addHostMetrics(final List metricsList, final long dcId, final metricsList.add(new ItemHost(zoneName, zoneUuid, ONLINE, up, null)); metricsList.add(new ItemHost(zoneName, zoneUuid, OFFLINE, down, null)); metricsList.add(new ItemHost(zoneName, zoneUuid, TOTAL, total, null)); - for (Map.Entry entry : total_hosts.entrySet()) { + for (Map.Entry entry : totalHosts.entrySet()) { String tag = entry.getKey(); Integer count = entry.getValue(); metricsList.add(new ItemHost(zoneName, zoneUuid, TOTAL, count, tag)); - if (up_hosts.get(tag) != null) { - metricsList.add(new ItemHost(zoneName, zoneUuid, ONLINE, up_hosts.get(tag), tag)); + if (upHosts.get(tag) != null) { + metricsList.add(new ItemHost(zoneName, zoneUuid, ONLINE, upHosts.get(tag), tag)); } else { metricsList.add(new ItemHost(zoneName, zoneUuid, ONLINE, 0, tag)); } - if (down_hosts.get(tag) != null) { - metricsList.add(new ItemHost(zoneName, zoneUuid, OFFLINE, down_hosts.get(tag), tag)); + if (downHosts.get(tag) != null) { + metricsList.add(new ItemHost(zoneName, zoneUuid, OFFLINE, downHosts.get(tag), tag)); } else { metricsList.add(new ItemHost(zoneName, zoneUuid, OFFLINE, 0, tag)); } } - for (Map.Entry entry : total_hosts.entrySet()) { + for (Map.Entry entry : totalHosts.entrySet()) { String tag = entry.getKey(); Ternary allocatedCapacityByTag = capacityDao.findCapacityByZoneAndHostTag(dcId, tag); metricsList.add(new ItemVMCore(zoneName, zoneUuid, null, null, null, ALLOCATED, allocatedCapacityByTag.first(), 0, tag)); From bfd89261f60f5db30767e958aa73170dffbba8da Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Mon, 13 Dec 2021 12:41:55 +0100 Subject: [PATCH 13/17] Removed redundant lines of the Prometheus outputs. --- .../cloudstack/metrics/PrometheusExporterImpl.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index 39098cd1e5b0..6a5ebc7af824 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.inject.Inject; @@ -251,10 +252,12 @@ private void addVMMetrics(final List metricsList, final long dcId, final S } metricsList.add(new ItemVM(zoneName, zoneUuid, state.name().toLowerCase(), count)); } - List allHostTags = new ArrayList(); - for (final HostVO host : hostDao.listAll()) { - allHostTags.addAll(_hostTagsDao.gethostTags(host.getId())); - } + + List allHostTags = hostDao.listAll().stream() + .flatMap( h -> _hostTagsDao.gethostTags(h.getId()).stream()) + .distinct() + .collect(Collectors.toList()); + for (final State state : State.values()) { for (final String hosttag : allHostTags) { final Long count = vmDao.countByZoneAndStateAndHostTag(dcId, state, hosttag); From 76fbc2a23c21bf610527a80ce3a36988a0b99383 Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Wed, 15 Dec 2021 15:52:01 +0000 Subject: [PATCH 14/17] Use prepared statement to query database for a number of VM who use a specific tag. --- .../java/com/cloud/vm/dao/VMInstanceDaoImpl.java | 16 ++++++++++------ systemvm/debian/root/version | 1 + 2 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 systemvm/debian/root/version diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 880238c67cb4..0e3d4bdde8f1 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -129,6 +129,9 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem private static final String UPDATE_SYSTEM_VM_TEMPLATE_ID_FOR_HYPERVISOR = "UPDATE `cloud`.`vm_instance` SET vm_template_id = ? WHERE type <> 'User' AND hypervisor_type = ? AND removed is NULL"; + private static final String COUNT_VMS_BY_ZONE_AND_STATE_AND_HOST_TAG = "SELECT COUNT(1) FROM vm_instance vi JOIN service_offering so ON vi.service_offering_id=so.id " + + "JOIN vm_template vt ON vi.vm_template_id = vt.id WHERE vi.data_center_id = ? AND vi.state = ? AND vi.removed IS NULL AND (so.host_tag = ? OR vt.template_tag = ?)"; + @Inject protected HostDao _hostDao; @@ -810,14 +813,15 @@ public Long countByZoneAndState(long zoneId, State state) { @Override public Long countByZoneAndStateAndHostTag(long dcId, State state, String hostTag) { TransactionLegacy txn = TransactionLegacy.currentTxn(); - String sql = "SELECT COUNT(1) FROM vm_instance vi " - + "JOIN service_offering so ON vi.service_offering_id=so.id " - + "JOIN vm_template vt ON vi.vm_template_id = vt.id " - + "WHERE vi.data_center_id= " + dcId + " AND vi.state = '" + state + "' AND vi.removed IS NULL " - + "AND (so.host_tag='" + hostTag + "' OR vt.template_tag='" + hostTag + "')"; PreparedStatement pstmt = null; try { - pstmt = txn.prepareAutoCloseStatement(sql); + pstmt = txn.prepareAutoCloseStatement(COUNT_VMS_BY_ZONE_AND_STATE_AND_HOST_TAG); + + pstmt.setLong(1, dcId); + pstmt.setString(2, String.valueOf(state)); + pstmt.setString(3, hostTag); + pstmt.setString(4, hostTag); + ResultSet rs = pstmt.executeQuery(); if (rs.next()) { return rs.getLong(1); diff --git a/systemvm/debian/root/version b/systemvm/debian/root/version new file mode 100644 index 000000000000..32f472a2d9d1 --- /dev/null +++ b/systemvm/debian/root/version @@ -0,0 +1 @@ +prometheus-exporter-enhancement-20211213T174523 From 0bcc6bc618090a479ae600d8577451e227e4e561 Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Thu, 24 Mar 2022 16:06:55 +0100 Subject: [PATCH 15/17] Rename gethostTags to getHostTags. --- .../org/apache/cloudstack/metrics/PrometheusExporterImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index 6a5ebc7af824..f8193293dd39 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -142,7 +142,7 @@ private void addHostMetrics(final List metricsList, final long dcId, final int isDedicated = (dr != null) ? 1 : 0; metricsList.add(new ItemHostIsDedicated(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), isDedicated)); - List hostTags = _hostTagsDao.gethostTags(host.getId()); + List hostTags = _hostTagsDao.getHostTags(host.getId()); String hosttags = StringUtils.join(hostTags, ","); for (String tag : hostTags) { Integer current = totalHosts.get(tag) != null ? totalHosts.get(tag) : 0; @@ -254,7 +254,7 @@ private void addVMMetrics(final List metricsList, final long dcId, final S } List allHostTags = hostDao.listAll().stream() - .flatMap( h -> _hostTagsDao.gethostTags(h.getId()).stream()) + .flatMap( h -> _hostTagsDao.getHostTags(h.getId()).stream()) .distinct() .collect(Collectors.toList()); From 51571e40e92eac43a2a0454fbe2d151c7dec0fc9 Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Thu, 24 Mar 2022 16:09:56 +0100 Subject: [PATCH 16/17] Remove unwanted files. --- systemvm/debian/root/version | 1 - 1 file changed, 1 deletion(-) delete mode 100644 systemvm/debian/root/version diff --git a/systemvm/debian/root/version b/systemvm/debian/root/version deleted file mode 100644 index 32f472a2d9d1..000000000000 --- a/systemvm/debian/root/version +++ /dev/null @@ -1 +0,0 @@ -prometheus-exporter-enhancement-20211213T174523 From 6aab64bc2c995ccc1895647d04b566f21ea39ddf Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Mon, 28 Mar 2022 14:07:54 +0200 Subject: [PATCH 17/17] Extract repeated codes to new methods. --- .../metrics/PrometheusExporterImpl.java | 103 ++++++++++-------- 1 file changed, 56 insertions(+), 47 deletions(-) diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index f8193293dd39..de9a5a40c3b5 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -142,23 +142,7 @@ private void addHostMetrics(final List metricsList, final long dcId, final int isDedicated = (dr != null) ? 1 : 0; metricsList.add(new ItemHostIsDedicated(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), isDedicated)); - List hostTags = _hostTagsDao.getHostTags(host.getId()); - String hosttags = StringUtils.join(hostTags, ","); - for (String tag : hostTags) { - Integer current = totalHosts.get(tag) != null ? totalHosts.get(tag) : 0; - totalHosts.put(tag, current + 1); - } - if (host.getStatus() == Status.Up) { - for (String tag : hostTags) { - Integer current = upHosts.get(tag) != null ? upHosts.get(tag) : 0; - upHosts.put(tag, current + 1); - } - } else if (host.getStatus() == Status.Disconnected || host.getStatus() == Status.Down) { - for (String tag : hostTags) { - Integer current = downHosts.get(tag) != null ? downHosts.get(tag) : 0; - downHosts.put(tag, current + 1); - } - } + String hostTags = markTagMaps(host, totalHosts, upHosts, downHosts); // Get account, domain details for dedicated hosts if (isDedicated == 1) { @@ -173,32 +157,32 @@ private void addHostMetrics(final List metricsList, final long dcId, final final String cpuFactor = String.valueOf(CapacityManager.CpuOverprovisioningFactor.valueIn(host.getClusterId())); final CapacityVO cpuCapacity = capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_CPU); if (cpuCapacity != null) { - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, cpuCapacity.getUsedCapacity(), isDedicated, hosttags)); - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, cpuCapacity.getTotalCapacity(), isDedicated, hosttags)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, cpuCapacity.getUsedCapacity(), isDedicated, hostTags)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, cpuCapacity.getTotalCapacity(), isDedicated, hostTags)); } else { - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, 0L, isDedicated, hosttags)); - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, 0L, isDedicated, hosttags)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, USED, 0L, isDedicated, hostTags)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), cpuFactor, TOTAL, 0L, isDedicated, hostTags)); } final String memoryFactor = String.valueOf(CapacityManager.MemOverprovisioningFactor.valueIn(host.getClusterId())); final CapacityVO memCapacity = capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_MEMORY); if (memCapacity != null) { - metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, USED, memCapacity.getUsedCapacity(), isDedicated, hosttags)); - metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, TOTAL, memCapacity.getTotalCapacity(), isDedicated, hosttags)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, USED, memCapacity.getUsedCapacity(), isDedicated, hostTags)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, TOTAL, memCapacity.getTotalCapacity(), isDedicated, hostTags)); } else { - metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, USED, 0L, isDedicated, hosttags)); - metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, TOTAL, 0L, isDedicated, hosttags)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, USED, 0L, isDedicated, hostTags)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), memoryFactor, TOTAL, 0L, isDedicated, hostTags)); } metricsList.add(new ItemHostVM(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), vmDao.listByHostId(host.getId()).size())); final CapacityVO coreCapacity = capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_CPU_CORE); if (coreCapacity != null) { - metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), USED, coreCapacity.getUsedCapacity(), isDedicated, hosttags)); - metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), TOTAL, coreCapacity.getTotalCapacity(), isDedicated, hosttags)); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), USED, coreCapacity.getUsedCapacity(), isDedicated, hostTags)); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), TOTAL, coreCapacity.getTotalCapacity(), isDedicated, hostTags)); } else { - metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), USED, 0L, isDedicated, hosttags)); - metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), TOTAL, 0L, isDedicated, hosttags)); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), USED, 0L, isDedicated, hostTags)); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, host.getName(), host.getUuid(), host.getPrivateIpAddress(), TOTAL, 0L, isDedicated, hostTags)); } } @@ -220,6 +204,30 @@ private void addHostMetrics(final List metricsList, final long dcId, final metricsList.add(new ItemHost(zoneName, zoneUuid, ONLINE, up, null)); metricsList.add(new ItemHost(zoneName, zoneUuid, OFFLINE, down, null)); metricsList.add(new ItemHost(zoneName, zoneUuid, TOTAL, total, null)); + + addHostTagsMetrics(metricsList, dcId, zoneName, zoneUuid, totalHosts, upHosts, downHosts, total, up, down); + } + + private String markTagMaps(HostVO host, Map totalHosts, Map upHosts, Map downHosts) { + List hostTags = _hostTagsDao.getHostTags(host.getId()); + markTags(hostTags,totalHosts); + if (host.getStatus() == Status.Up) { + markTags(hostTags, upHosts); + } else if (host.getStatus() == Status.Disconnected || host.getStatus() == Status.Down) { + markTags(hostTags, downHosts); + } + return StringUtils.join(hostTags, ","); + } + + private void markTags(List tags, Map tagMap) { + tags.forEach(tag -> { + int current = tagMap.get(tag) != null ? tagMap.get(tag) : 0; + tagMap.put(tag, current + 1); + }); + } + + private void addHostTagsMetrics(final List metricsList, final long dcId, final String zoneName, final String zoneUuid, Map totalHosts, Map upHosts, Map downHosts, int total, int up, int down) { + for (Map.Entry entry : totalHosts.entrySet()) { String tag = entry.getKey(); Integer count = entry.getValue(); @@ -235,12 +243,25 @@ private void addHostMetrics(final List metricsList, final long dcId, final metricsList.add(new ItemHost(zoneName, zoneUuid, OFFLINE, 0, tag)); } } - for (Map.Entry entry : totalHosts.entrySet()) { - String tag = entry.getKey(); - Ternary allocatedCapacityByTag = capacityDao.findCapacityByZoneAndHostTag(dcId, tag); - metricsList.add(new ItemVMCore(zoneName, zoneUuid, null, null, null, ALLOCATED, allocatedCapacityByTag.first(), 0, tag)); - metricsList.add(new ItemHostCpu(zoneName, zoneUuid, null, null, null, null, ALLOCATED, allocatedCapacityByTag.second(),0, tag)); - metricsList.add(new ItemHostMemory(zoneName, zoneUuid, null, null, null, null, ALLOCATED, allocatedCapacityByTag.third(), 0, tag)); + + totalHosts.keySet() + .forEach( tag -> { + Ternary allocatedCapacityByTag = capacityDao.findCapacityByZoneAndHostTag(dcId, tag); + metricsList.add(new ItemVMCore(zoneName, zoneUuid, null, null, null, ALLOCATED, allocatedCapacityByTag.first(), 0, tag)); + metricsList.add(new ItemHostCpu(zoneName, zoneUuid, null, null, null, null, ALLOCATED, allocatedCapacityByTag.second(), 0, tag)); + metricsList.add(new ItemHostMemory(zoneName, zoneUuid, null, null, null, null, ALLOCATED, allocatedCapacityByTag.third(), 0, tag)); + }); + + List allHostTags = hostDao.listAll().stream() + .flatMap( h -> _hostTagsDao.getHostTags(h.getId()).stream()) + .distinct() + .collect(Collectors.toList()); + + for (final State state : State.values()) { + for (final String hostTag : allHostTags) { + final Long count = vmDao.countByZoneAndStateAndHostTag(dcId, state, hostTag); + metricsList.add(new ItemVMByTag(zoneName, zoneUuid, state.name().toLowerCase(), count, hostTag)); + } } } @@ -252,18 +273,6 @@ private void addVMMetrics(final List metricsList, final long dcId, final S } metricsList.add(new ItemVM(zoneName, zoneUuid, state.name().toLowerCase(), count)); } - - List allHostTags = hostDao.listAll().stream() - .flatMap( h -> _hostTagsDao.getHostTags(h.getId()).stream()) - .distinct() - .collect(Collectors.toList()); - - for (final State state : State.values()) { - for (final String hosttag : allHostTags) { - final Long count = vmDao.countByZoneAndStateAndHostTag(dcId, state, hosttag); - metricsList.add(new ItemVMByTag(zoneName, zoneUuid, state.name().toLowerCase(), count, hosttag)); - } - } } private void addVolumeMetrics(final List metricsList, final long dcId, final String zoneName, final String zoneUuid) {