Skip to content

Improve performance of listTemplates API #13566

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Pearl1594 wants to merge 4 commits into
base: 4.22
Choose a base branch
from
+1,115 −50
Draft

Improve performance of listTemplates API #13566

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3fd789b
Bypass logic to create template_pair from 6 tables, with fallback
aniishadas May 22, 2026
cf6beb4
Add @Ignore, remove logs/comments (#824)
aniishadas May 22, 2026
54d4392
Improve BypassTemplateView config description and rename helper method
mprokopchuk Jul 6, 2026
222cf25
Merge branch '4.22' of https://github.com/apache/cloudstack into impr…
Jul 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions api/src/main/java/org/apache/cloudstack/query/QueryService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,12 @@ public interface QueryService {
ConfigKey<Boolean> ReturnVmStatsOnVmList = new ConfigKey<>("Advanced", Boolean.class, "list.vm.default.details.stats", "true",
"Determines whether VM stats should be returned when details are not explicitly specified in listVirtualMachines API request. When false, details default to [group, nics, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp]. When true, all details are returned including 'stats'.", true, ConfigKey.Scope.Global);

ConfigKey<Boolean> BypassTemplateView = new ConfigKey<>("Advanced", Boolean.class, "template.list.bypass.view",
"false",
"If true, uses an optimized query path for listing templates and ISOs, which can improve performance on " +
"deployments with large numbers of templates. Automatically falls back to the standard query when" +
" filtering by tags, using the shared, featured, or community template filters, or when including" +
" removed templates.", true, ConfigKey.Scope.Global);

ListResponse<UserResponse> searchForUsers(ResponseObject.ResponseView responseView, ListUsersCmd cmd) throws PermissionDeniedException;

Expand Down
149 changes: 139 additions & 10 deletions server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,14 @@
import com.cloud.storage.dao.StoragePoolAndAccessGroupMapDao;
import com.cloud.cluster.ManagementServerHostPeerJoinVO;

import com.cloud.template.VirtualMachineTemplate;
import com.cloud.user.AccountVO;
import com.cloud.user.SSHKeyPairVO;
import com.cloud.user.dao.SSHKeyPairDao;
import com.cloud.vm.InstanceGroupVMMapVO;
import com.cloud.vm.NicVO;
import com.cloud.vm.dao.InstanceGroupVMMapDao;
import com.cloud.vm.dao.NicDao;
import com.cloud.vm.UserVmManager;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.acl.ControlledEntity.ACLType;
Expand Down Expand Up @@ -207,6 +215,7 @@
import com.cloud.api.query.dao.SnapshotJoinDao;
import com.cloud.api.query.dao.StoragePoolJoinDao;
import com.cloud.api.query.dao.TemplateJoinDao;
import com.cloud.api.query.dao.TemplateListFilter;
import com.cloud.api.query.dao.UserAccountJoinDao;
import com.cloud.api.query.dao.UserVmJoinDao;
import com.cloud.api.query.dao.VolumeJoinDao;
Expand Down Expand Up @@ -332,12 +341,9 @@
import com.cloud.template.VirtualMachineTemplate.TemplateFilter;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
import com.cloud.user.AccountVO;
import com.cloud.user.DomainManager;
import com.cloud.user.SSHKeyPairVO;
import com.cloud.user.User;
import com.cloud.user.dao.AccountDao;
import com.cloud.user.dao.SSHKeyPairDao;
import com.cloud.user.dao.UserDao;
import com.cloud.utils.DateUtil;
import com.cloud.utils.NumbersUtil;
Expand All @@ -353,17 +359,13 @@
import com.cloud.utils.db.SearchCriteria.Op;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.DomainRouterVO;
import com.cloud.vm.InstanceGroupVMMapVO;
import com.cloud.vm.NicVO;
import com.cloud.vm.UserVmVO;
import com.cloud.vm.VMInstanceDetailVO;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachineManager;
import com.cloud.vm.VmDetailConstants;
import com.cloud.vm.dao.DomainRouterDao;
import com.cloud.vm.dao.InstanceGroupVMMapDao;
import com.cloud.vm.dao.NicDao;
import com.cloud.vm.dao.UserVmDao;
import com.cloud.vm.dao.VMInstanceDao;
import com.cloud.vm.dao.VMInstanceDetailsDao;
Expand Down Expand Up @@ -5089,7 +5091,11 @@ else if (!template.isPublicTemplate() && caller.getType() != Account.Type.ADMIN)
applyPublicTemplateSharingRestrictions(sc, caller);

return templateChecks(isIso, hypers, tags, name, keyword, hyperType, onlyReady, bootable, zoneId, showDomr, caller,
showRemovedTmpl, parentTemplateId, showUnique, templateType, isVnf, forCks, searchFilter, sc);
showRemovedTmpl, parentTemplateId, showUnique, templateType, isVnf, forCks, searchFilter, sc,
buildTemplateListFilter(templateId, ids, name, keyword, templateFilter, isIso, bootable,
pageSize, startIndex, zoneId, hyperType, hypers, showDomr, onlyReady,
permittedAccounts, caller, listProjectResourcesCriteria, tags, showRemovedTmpl,
parentTemplateId, showUnique));
}

/**
Expand Down Expand Up @@ -5144,7 +5150,8 @@ protected boolean checkIfDomainSharesTemplates(Long domainId) {
private Pair<List<TemplateJoinVO>, Integer> templateChecks(boolean isIso, List<HypervisorType> hypers, Map<String, String> tags, String name, String keyword,
HypervisorType hyperType, boolean onlyReady, Boolean bootable, Long zoneId, boolean showDomr, Account caller,
boolean showRemovedTmpl, Long parentTemplateId, Boolean showUnique, String templateType, Boolean isVnf, Boolean forCks,
Filter searchFilter, SearchCriteria<TemplateJoinVO> sc) {
Filter searchFilter, SearchCriteria<TemplateJoinVO> sc,
TemplateListFilter listFilter) {
if (!isIso) {
// add hypervisor criteria for template case
if (hypers != null && !hypers.isEmpty()) {
Expand Down Expand Up @@ -5251,6 +5258,14 @@ private Pair<List<TemplateJoinVO>, Integer> templateChecks(boolean isIso, List<H

// search unique templates and find details by Ids
Pair<List<TemplateJoinVO>, Integer> uniqueTmplPair;

// Only attempted when the bypass DAO supports the filter set; falls
// back to the SearchBuilder/template_view path below otherwise.
if (!showRemovedTmpl && BypassTemplateView.value() && listFilter != null && listFilter.canBypass()) {
uniqueTmplPair = _templateJoinDao.findDistinctTempZonePairs(listFilter);
return findTemplatesByIdOrTempZonePair(uniqueTmplPair, showRemovedTmpl, showUnique != null && showUnique, caller);
}

if (showRemovedTmpl) {
uniqueTmplPair = _templateJoinDao.searchIncludingRemovedAndCount(sc, searchFilter);
} else {
Expand All @@ -5272,6 +5287,119 @@ private Pair<List<TemplateJoinVO>, Integer> templateChecks(boolean isIso, List<H
// about what the code was doing.
}

/**
* Build the immutable filter the bypass-the-view DAO consumes. Mirrors the
* SearchCriteria construction in {@link #searchForTemplatesInternal}; the
* SearchBuilder path is unchanged. The "hard" markers (sharedAccountIds,
* domainPathLike, domainIdsForFeaturedCommunity, tags) drive
* {@link TemplateListFilter#canBypass()} — when any are populated,
* {@code templateChecks} routes to the legacy view-based path instead.
*/
private TemplateListFilter buildTemplateListFilter(Long templateId, List<Long> ids, String name, String keyword,
TemplateFilter templateFilter, boolean isIso, Boolean bootable,
Long pageSize, Long startIndex, Long zoneId, HypervisorType hyperType,
List<HypervisorType> hypers, boolean showDomr, boolean onlyReady,
List<Account> permittedAccounts, Account caller,
ListProjectResourcesCriteria listProjectResourcesCriteria,
Map<String, String> tags, boolean showRemovedTmpl, Long parentTemplateId,
Boolean showUnique) {
TemplateListFilter.Builder b = TemplateListFilter.builder()
.templateId(templateId)
.ids(ids == null ? null : new ArrayList<>(ids))
.name(name)
.keyword(keyword)
.hypervisorType(hyperType != null && !HypervisorType.None.equals(hyperType) ? hyperType : null)
.availableHypervisors(hypers)
.format(ImageFormat.ISO) // bypass DAO uses isIso flag to flip between EQ and NEQ
.isIso(isIso)
.bootable(bootable)
.parentTemplateId(parentTemplateId)
.zoneId(zoneId)
.onlyReady(onlyReady)
.excludeSystemTemplates(!showDomr)
.showUnique(showUnique != null && showUnique)
.startIndex(startIndex)
.pageSize(pageSize)
.sortAscending(SortKeyAscending.value())
.showRemoved(showRemovedTmpl)
.tags(tags);

if (!showRemovedTmpl) {
b.templateStates(Arrays.asList(VirtualMachineTemplate.State.Active,
VirtualMachineTemplate.State.UploadAbandoned,
VirtualMachineTemplate.State.UploadError,
VirtualMachineTemplate.State.NotUploaded,
VirtualMachineTemplate.State.UploadInProgress));
}

// accountType filter (Project vs non-Project), applied only when no specific templateId.
if (templateId == null && listProjectResourcesCriteria == ListProjectResourcesCriteria.SkipProjectResources) {
b.accountTypeNeq(Account.Type.PROJECT);
} else if (templateId == null && listProjectResourcesCriteria == ListProjectResourcesCriteria.ListProjectResourcesOnly) {
b.accountTypeEq(Account.Type.PROJECT);
}

// ACL handling — mirrors the templateFilter switch in searchForTemplatesInternal.
// For "hard" templateFilter values, set requiresViewFallback so canBypass() returns
// false and the dispatcher falls back to the legacy view-based path.
if (templateId != null) {
return b.build();
}

List<Long> permittedAccountIds = new ArrayList<>();
for (Account account : permittedAccounts) {
permittedAccountIds.add(account.getId());
}

if (templateFilter == TemplateFilter.featured || templateFilter == TemplateFilter.community) {
// Walks the domain hierarchy and ORs domainId IS NULL — not yet modeled in bypass SQL.
b.publicTemplate(Boolean.TRUE);
b.featured(templateFilter == TemplateFilter.featured ? Boolean.TRUE : Boolean.FALSE);
b.requiresViewFallback(true);
} else if (templateFilter == TemplateFilter.self || templateFilter == TemplateFilter.selfexecutable) {
if (caller.getType() == Account.Type.DOMAIN_ADMIN || caller.getType() == Account.Type.RESOURCE_DOMAIN_ADMIN) {
// Match the existing path's domain resolution (see searchForTemplatesInternal:4498-4502):
// scope to the queried account's domain when one was specified, else the caller's.
Long domainIdForScope = !permittedAccounts.isEmpty()
? permittedAccounts.get(0).getDomainId()
: caller.getDomainId();
DomainVO scopeDomain = _domainDao.findById(domainIdForScope);
if (scopeDomain != null) {
b.domainPathLike(scopeDomain.getPath() + "%");
}
}
if (!permittedAccountIds.isEmpty()) {
b.accountIds(permittedAccountIds);
b.accountIdRequiredForFilter(true);
}
} else if (templateFilter == TemplateFilter.sharedexecutable || templateFilter == TemplateFilter.shared) {
// launch_permission join — not modeled in bypass SQL.
b.requiresViewFallback(true);
} else if (templateFilter == TemplateFilter.executable) {
b.publicOrAccountIdComposite(true);
if (!permittedAccountIds.isEmpty()) {
b.accountIds(permittedAccountIds);
}
} else if (templateFilter == TemplateFilter.all && caller.getType() != Account.Type.ADMIN) {
// Non-admin "all" composite splits on listProjectResourcesCriteria:
// SkipProjectResources → (public OR domainPath LIKE) — bypass-eligible
// else → (public OR account_id IN OR sharedAccountId IN) — needs launch_permission, falls back
if (listProjectResourcesCriteria == ListProjectResourcesCriteria.SkipProjectResources) {
DomainVO callerDomain = _domainDao.findById(caller.getDomainId());
if (callerDomain != null) {
b.publicOrDomainPathComposite(true);
b.domainPathLike(callerDomain.getPath() + "%");
} else {
b.requiresViewFallback(true);
}
} else {
b.requiresViewFallback(true);
}
}

return b.build();
}

// findTemplatesByIdOrTempZonePair returns the templates with the given ids if showUnique is true, or else by the TempZonePair
private Pair<List<TemplateJoinVO>, Integer> findTemplatesByIdOrTempZonePair(Pair<List<TemplateJoinVO>, Integer> templateDataPair,
boolean showRemoved, boolean showUnique, Account caller) {
Expand Down Expand Up @@ -6285,7 +6413,8 @@ public String getConfigComponentName() {
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] {AllowUserViewDestroyedVM, UserVMDeniedDetails, UserVMReadOnlyDetails, SortKeyAscending,
AllowUserViewAllDomainAccounts, AllowUserViewAllDataCenters, SharePublicTemplatesWithOtherDomains, ReturnVmStatsOnVmList};
AllowUserViewAllDomainAccounts, AllowUserViewAllDataCenters, SharePublicTemplatesWithOtherDomains,
ReturnVmStatsOnVmList, BypassTemplateView };
}

@Override
Expand Down
15 changes: 15 additions & 0 deletions server/src/main/java/com/cloud/api/query/dao/TemplateJoinDao.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,5 +52,20 @@ public interface TemplateJoinDao extends GenericDao<TemplateJoinVO, Long> {

Pair<List<TemplateJoinVO>, Integer> searchIncludingRemovedAndCount(final SearchCriteria<TemplateJoinVO> sc, final Filter filter);

/**
* Bypass-the-view Phase 1 implementation. Issues a hand-tuned SQL query
* directly against the underlying tables (vm_template, account,
* template_store_ref, image_store, template_zone_ref, data_center) instead
* of going through {@code template_view}. Caller must check
* {@link TemplateListFilter#canBypass()} first; this method does not handle
* tags, sharedAccountIds, domainPath, or featured/community-style domainId
* filters and will throw IllegalArgumentException if those are populated.
*
* Returns TemplateJoinVO objects with only {@code id} (when
* {@code filter.showUnique}) or {@code tempZonePair} populated, plus the
* total distinct count for pagination.
*/
Pair<List<TemplateJoinVO>, Integer> findDistinctTempZonePairs(TemplateListFilter filter);

List<TemplateJoinVO> findByDistinctIds(Long... ids);
}
Loading
Loading