From 45ad2101efcd80877cfbfda6d636486cf9e54a1b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Dec 2024 18:07:50 +0530 Subject: [PATCH 01/21] wip: multi-scope config Signed-off-by: Abhishek Kumar --- .../command/admin/config/ListCfgsByCmd.java | 3 ++ framework/config/pom.xml | 6 +++ .../framework/config/ConfigDepot.java | 3 ++ .../framework/config/ConfigKey.java | 47 +++++++++++++++- .../config/dao/ConfigurationDao.java | 3 ++ .../config/dao/ConfigurationDaoImpl.java | 54 +++++++++++++++++++ .../config/impl/ConfigDepotImpl.java | 5 ++ .../java/com/cloud/api/ApiResponseHelper.java | 1 + .../ConfigurationManagerImpl.java | 4 +- .../cloud/server/ManagementServerImpl.java | 5 +- .../vpc/dao/MockConfigurationDaoImpl.java | 7 +++ 11 files changed, 135 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java index e365d8bc2dc7..12f3599e200d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java @@ -182,6 +182,9 @@ private void setScope(ConfigurationResponse cfgResponse) { return; } cfgResponse.setObjectName("configuration"); + if (StringUtils.isNotBlank(cfgResponse.getScope())) { + return; + } if (getZoneId() != null) { cfgResponse.setScope("zone"); } diff --git a/framework/config/pom.xml b/framework/config/pom.xml index 403429380b65..a29fcfabd622 100644 --- a/framework/config/pom.xml +++ b/framework/config/pom.xml @@ -38,5 +38,11 @@ cloud-framework-db ${project.version} + + org.apache.cloudstack + cloud-api + 4.20.1.0-SNAPSHOT + compile + diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java index 5ee5f9dec48a..12f3653b9b3b 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java @@ -18,6 +18,8 @@ import java.util.Set; +import com.cloud.utils.Pair; + /** * ConfigDepot is a repository of configurations. * @@ -34,4 +36,5 @@ public interface ConfigDepot { boolean isNewConfig(ConfigKey configKey); String getConfigStringValue(String key, ConfigKey.Scope scope, Long scopeId); void invalidateConfigCache(String key, ConfigKey.Scope scope, Long scopeId); + Pair getParentScope(ConfigKey.Scope scope, Long id); } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java index 00cf56345c8d..59fc1b76fc17 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.framework.config; import java.sql.Date; +import java.util.ArrayList; +import java.util.List; import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; @@ -37,7 +39,46 @@ public class ConfigKey { public static final String CATEGORY_SYSTEM = "System"; public enum Scope { - Global, Zone, Cluster, StoragePool, Account, ManagementServer, ImageStore, Domain + Global(null), + Zone(Global), + Cluster(Zone), + StoragePool(Cluster), + ManagementServer(Global), + ImageStore(Zone), + Domain(Global), + Account(Domain); + + private final Scope parent; + + Scope(Scope parent) { + this.parent = parent; + } + + public Scope getParent() { + return parent; + } + + public boolean isDescendantOf(Scope other) { + Scope current = this; + while (current != null) { + if (current == other) { + return true; + } + current = current.getParent(); + } + return false; + } + + public static List getAllDescendants(String str) { + Scope s1 = Scope.valueOf(str); + List scopes = new ArrayList<>(); + for (Scope s : Scope.values()) { + if (s.isDescendantOf(s1)) { + scopes.add(s); + } + } + return scopes; + } } public enum Kind { @@ -229,6 +270,10 @@ protected T valueInScope(Scope scope, Long id) { String value = s_depot != null ? s_depot.getConfigStringValue(_name, scope, id) : null; if (value == null) { + Pair parentScope = s_depot != null ? s_depot.getParentScope(scope, id) : null; + if (parentScope != null) { + return valueInScope(parentScope.first(), parentScope.second()); + } return value(); } return valueOf(value); diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java index 88569558fc6b..d22656661803 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java @@ -18,8 +18,10 @@ import java.util.Map; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.impl.ConfigurationVO; +import com.cloud.utils.Pair; import com.cloud.utils.db.GenericDao; public interface ConfigurationDao extends GenericDao { @@ -67,4 +69,5 @@ public interface ConfigurationDao extends GenericDao { boolean update(String name, String category, String value); void invalidateCache(); + Pair getParentScope(ConfigKey.Scope scope, Long id); } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java index 7c4a6f9a609e..393bda5b2a17 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java @@ -22,14 +22,22 @@ import java.util.Map; import javax.annotation.PostConstruct; +import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import org.springframework.stereotype.Component; +import com.cloud.org.Cluster; +import com.cloud.storage.ImageStore; +import com.cloud.storage.StoragePool; +import com.cloud.user.Account; +import com.cloud.utils.Pair; import com.cloud.utils.component.ComponentLifecycle; import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.DB; +import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -38,6 +46,9 @@ @Component public class ConfigurationDaoImpl extends GenericDaoBase implements ConfigurationDao { + @Inject + EntityManager entityManager; + private Map _configs = null; private boolean _premium; @@ -207,4 +218,47 @@ public ConfigurationVO findByName(String name) { return findOneIncludingRemovedBy(sc); } + @Override + public Pair getParentScope(ConfigKey.Scope scope, Long id) { + if (scope == null || scope.getParent() == null || ConfigKey.Scope.Global.equals(scope.getParent()) + || id == null) { + return null; + } + switch (scope) { + case Account: { + Account account = entityManager.findById(Account.class, id); + if (account != null) { + return new Pair<>(scope.getParent(), account.getDomainId()); + } + break; + } + case ImageStore: { + ImageStore store = entityManager.findById(ImageStore.class, id); + if (store != null) { + return new Pair<>(scope.getParent(), store.getDataCenterId()); + } + break; + } + case StoragePool: { + StoragePool pool = entityManager.findById(StoragePool.class, id); + if (pool != null) { + if (pool.getClusterId() != null) { + return new Pair<>(scope.getParent(), pool.getClusterId()); + } else { + return new Pair<>(ConfigKey.Scope.Zone, pool.getDataCenterId()); + } + } + break; + } + case Cluster: { + Cluster cluster = entityManager.findById(Cluster.class, id); + if (cluster != null) { + return new Pair<>(scope.getParent(), cluster.getDataCenterId()); + } + break; + } + } + return null; + } + } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java index b47370d92059..d07ceb7bb0b9 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java @@ -402,4 +402,9 @@ public Pair getConfigurationGroupAndSubGroupByName(String configName public boolean isNewConfig(ConfigKey configKey) { return newConfigs.contains(configKey.key()); } + + @Override + public Pair getParentScope(ConfigKey.Scope scope, Long id) { + return _configDao.getParentScope(scope, id); + } } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 810f0abd7e00..321db10f0469 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -639,6 +639,7 @@ public ConfigurationResponse createConfigurationResponse(Configuration cfg) { cfgResponse.setSubGroup(configGroupAndSubGroup.second()); cfgResponse.setDescription(cfg.getDescription()); cfgResponse.setName(cfg.getName()); + cfgResponse.setScope(cfg.getScope()); if (cfg.isEncrypted()) { cfgResponse.setValue(DBEncryptionUtil.encrypt(cfg.getValue())); } else { diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index d7e2160ef35b..b86592718791 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -1224,7 +1224,9 @@ protected String validateConfigurationValue(String name, String value, String sc String configScope = cfg.getScope(); if (scope != null) { - if (!configScope.contains(scope) && + ConfigKey.Scope scopeVal = ConfigKey.Scope.valueOf(scope); + ConfigKey.Scope configScopeVal = ConfigKey.Scope.valueOf(configScope); + if (!configScope.contains(scope) && !configScopeVal.isDescendantOf(scopeVal) && !(ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN.value() && configScope.contains(ConfigKey.Scope.Account.toString()) && scope.equals(ConfigKey.Scope.Domain.toString()))) { logger.error("Invalid scope id provided for the parameter " + name); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 76d2943e18c8..4e02677a303b 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -2297,7 +2297,10 @@ public Pair, Integer> searchForConfigurations(fina && scope.equals(ConfigKey.Scope.Domain.toString())) { sc.addAnd("scope", SearchCriteria.Op.IN, ConfigKey.Scope.Domain.toString(), ConfigKey.Scope.Account.toString()); } else { - sc.addAnd("scope", SearchCriteria.Op.EQ, scope); + ConfigKey.Scope scopeVal = ConfigKey.Scope.valueOf(scope); + List scopes = ConfigKey.Scope.getAllDescendants(scope); + List scopeStrs = scopes.stream().map(Enum::toString).collect(Collectors.toList()); + sc.addAnd("scope", SearchCriteria.Op.IN, scopeStrs.toArray()); } } diff --git a/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java b/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java index 012ae739bc7e..2e5a232beb3b 100644 --- a/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java +++ b/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java @@ -16,7 +16,10 @@ // under the License. package com.cloud.vpc.dao; +import com.cloud.utils.Pair; import com.cloud.utils.db.GenericDaoBase; + +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.config.impl.ConfigurationVO; @@ -117,4 +120,8 @@ public boolean update(String name, String category, String value) { public void invalidateCache() { } + @Override + public Pair getParentScope(ConfigKey.Scope scope, Long id) { + return null; + } } From c5da8fa54f0609177f04410835d890243b1c94c3 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 12 Dec 2024 14:48:44 +0530 Subject: [PATCH 02/21] wip: config-key scope as bitmask Signed-off-by: Abhishek Kumar --- .../com/cloud/capacity/CapacityManager.java | 7 +- .../com/cloud/dc/ClusterDetailsDaoImpl.java | 18 +++ .../dao/StoragePoolDetailsDaoImpl.java | 15 ++ .../upgrade/dao/DatabaseAccessObject.java | 35 +++++ .../com/cloud/upgrade/dao/DbUpgradeUtils.java | 17 +++ .../upgrade/dao/Upgrade41720to41800.java | 2 +- .../upgrade/dao/Upgrade42010to42100.java | 39 +++++ .../com/cloud/user/AccountDetailsDaoImpl.java | 10 ++ .../db/ImageStoreDetailsDaoImpl.java | 15 ++ framework/config/pom.xml | 6 - .../cloudstack/config/Configuration.java | 7 +- .../framework/config/ConfigKey.java | 142 +++++++++++++++--- .../framework/config/ScopedConfigStorage.java | 5 + .../config/dao/ConfigurationDao.java | 3 - .../config/dao/ConfigurationDaoImpl.java | 50 ------ .../config/impl/ConfigDepotImpl.java | 59 ++++---- .../config/impl/ConfigurationVO.java | 15 +- .../java/com/cloud/utils/db/SearchBase.java | 3 + .../com/cloud/utils/db/SearchCriteria.java | 2 +- .../metrics/MetricsServiceImpl.java | 6 +- .../java/com/cloud/api/ApiResponseHelper.java | 3 +- .../java/com/cloud/configuration/Config.java | 28 ++-- .../ConfigurationManagerImpl.java | 18 ++- .../cloud/server/ManagementServerImpl.java | 31 ++-- .../java/com/cloud/server/StatsCollector.java | 4 +- .../com/cloud/storage/StorageManagerImpl.java | 6 +- .../ConfigurationManagerImplTest.java | 21 +-- .../vpc/dao/MockConfigurationDaoImpl.java | 13 +- tools/marvin/setup.py | 2 +- 29 files changed, 391 insertions(+), 191 deletions(-) diff --git a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java index cbd137e86826..3bbbdb3e42a6 100644 --- a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java +++ b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.capacity; +import java.util.List; import java.util.Map; import org.apache.cloudstack.framework.config.ConfigKey; @@ -70,7 +71,7 @@ public interface CapacityManager { "0.85", "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", true, - ConfigKey.Scope.Zone); + List.of(ConfigKey.Scope.Zone, ConfigKey.Scope.StoragePool)); static final ConfigKey StorageOverprovisioningFactor = new ConfigKey<>( "Storage", @@ -88,7 +89,7 @@ public interface CapacityManager { "0.85", "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for low allocated storage available.", true, - ConfigKey.Scope.Zone); + List.of(ConfigKey.Scope.Zone, ConfigKey.Scope.StoragePool)); static final ConfigKey StorageOperationsExcludeCluster = new ConfigKey<>( Boolean.class, @@ -128,7 +129,7 @@ public interface CapacityManager { "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for volume resize. " + "This is applicable only when volume.resize.allowed.beyond.allocation is set to true.", true, - ConfigKey.Scope.Zone); + List.of(ConfigKey.Scope.Zone, ConfigKey.Scope.StoragePool)); public boolean releaseVmCapacity(VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId); diff --git a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java index 0e40f8475c16..89e9dfdeaaa9 100644 --- a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java @@ -20,10 +20,15 @@ import java.util.List; import java.util.Map; +import javax.inject.Inject; + import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.ConfigKey.Scope; import org.apache.cloudstack.framework.config.ScopedConfigStorage; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.org.Cluster; +import com.cloud.utils.Pair; import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; @@ -31,6 +36,10 @@ import com.cloud.utils.db.TransactionLegacy; public class ClusterDetailsDaoImpl extends GenericDaoBase implements ClusterDetailsDao, ScopedConfigStorage { + + @Inject + ClusterDao clusterDao; + protected final SearchBuilder ClusterSearch; protected final SearchBuilder DetailSearch; @@ -160,4 +169,13 @@ private String getCpuMemoryOvercommitRatio(String name) { return name; } + + @Override + public Pair getParentScope(long id) { + Cluster cluster = clusterDao.findById(id); + if (cluster == null) { + return null; + } + return new Pair<>(getScope().getParent(), cluster.getDataCenterId()); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java index 376933f92e7c..b5bb5223f8bd 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java @@ -30,6 +30,8 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import com.cloud.utils.Pair; + public class StoragePoolDetailsDaoImpl extends ResourceDetailsDaoBase implements StoragePoolDetailsDao, ScopedConfigStorage { @Inject @@ -57,4 +59,17 @@ public void addDetail(long resourceId, String key, String value, boolean display } super.addDetail(new StoragePoolDetailVO(resourceId, key, value, display)); } + + @Override + public Pair getParentScope(long id) { + StoragePoolVO pool = _storagePoolDao.findById(id); + if (pool != null) { + if (pool.getClusterId() != null) { + return new Pair<>(getScope().getParent(), pool.getClusterId()); + } else { + return new Pair<>(ConfigKey.Scope.Zone, pool.getDataCenterId()); + } + } + return null; + } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java index 1c2c4b3c7ce7..7f04736b60a5 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java @@ -87,6 +87,41 @@ public boolean columnExists(Connection conn, String tableName, String columnName return columnExists; } + public String getColumnType(Connection conn, String tableName, String columnName) { + try (PreparedStatement pstmt = conn.prepareStatement(String.format("DESCRIBE %s %s", tableName, columnName));){ + ResultSet rs = pstmt.executeQuery(); + if (rs.next()) { + return rs.getString("Type"); + } + } catch (SQLException e) { + logger.debug("Type for column {} can not be retrieved in {} ignoring exception: {}", columnName, tableName, e.getMessage()); + } + return null; + } + + public void addColumn(Connection conn, String tableName, String columnName, String columnDefinition) { + System.out.println("------------------------" + String.format("ALTER TABLE %s ADD COLUMN %s %s", tableName, columnName, columnDefinition)); + try (PreparedStatement pstmt = conn.prepareStatement(String.format("ALTER TABLE %s ADD COLUMN %s %s", tableName, columnName, columnDefinition));){ + pstmt.executeUpdate(); + System.out.println("column added------------------------" + String.format("ALTER TABLE %s ADD COLUMN %s %s", tableName, columnName, columnDefinition)); + logger.debug("Column {} is added successfully from the table {}", columnName, tableName); + } catch (SQLException e) { + System.out.println("column not added------------------------" + e.getMessage()); + logger.warn("Unable to add column {} to table {} due to exception", columnName, tableName, e); + } + } + + public void changeColumn(Connection conn, String tableName, String oldColumnName, String newColumnName, String columnDefinition) { + try (PreparedStatement pstmt = conn.prepareStatement(String.format("ALTER TABLE %s CHANGE COLUMN %s %s %s", tableName, oldColumnName, newColumnName, columnDefinition));){ + pstmt.executeUpdate(); + System.out.println("column changed------------------------" + String.format("ALTER TABLE %s CHANGE COLUMN %s %s %s", tableName, oldColumnName, newColumnName, columnDefinition)); + logger.debug("Column {} is changed successfully to {} from the table {}", oldColumnName, newColumnName, tableName); + } catch (SQLException e) { + System.out.println("column not changed------------------------" + e.getMessage()); + logger.warn("Unable to add column {} to {} from the table {} due to exception", oldColumnName, newColumnName, tableName, e); + } + } + public String generateIndexName(String tableName, String... columnName) { return String.format("i_%s__%s", tableName, StringUtils.join(columnName, "__")); } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java index 51e6ac7b9a1d..38815a8cb5d0 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java @@ -52,4 +52,21 @@ public static void dropTableColumnsIfExist(Connection conn, String tableName, Li } } + public static String getTableColumnType(Connection conn, String tableName, String columnName) { + return dao.getColumnType(conn, tableName, columnName); + } + + public static void addTableColumnIfNotExist(Connection conn, String tableName, String columnName, String columnDefinition) { + if (!dao.columnExists(conn, tableName, columnName)) { + dao.addColumn(conn, tableName, columnName, columnDefinition); + } + } + + public static void changeTableColumnIfNotExist(Connection conn, String tableName, String oldColumnName, String newColumnName, String columnDefinition) { + if (dao.columnExists(conn, tableName, oldColumnName)) { + System.out.println("column exists------------------------" + oldColumnName); + dao.changeColumn(conn, tableName, oldColumnName, newColumnName, columnDefinition); + } + } + } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41720to41800.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41720to41800.java index 6a90396deb0b..c8311a87a985 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41720to41800.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41720to41800.java @@ -711,6 +711,6 @@ private void correctGuestOsIdsInHypervisorMapping(final Connection conn) { } private void updateConfigurationGroups() { - configGroupsAggregator.updateConfigurationGroups(); +// configGroupsAggregator.updateConfigurationGroups(); } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java index 06a68ec3d8b2..eeabdb92169a 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java @@ -17,10 +17,16 @@ package com.cloud.upgrade.dao; import com.cloud.upgrade.SystemVmTemplateRegistration; +import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; import java.io.InputStream; import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; + +import org.apache.cloudstack.framework.config.ConfigKey; public class Upgrade42010to42100 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { private SystemVmTemplateRegistration systemVmTemplateRegistration; @@ -53,6 +59,7 @@ public InputStream[] getPrepareScripts() { @Override public void performDataMigration(Connection conn) { + migrateConfigurationScopeToBitmask(conn); } @Override @@ -80,4 +87,36 @@ public void updateSystemVmTemplates(Connection conn) { throw new CloudRuntimeException("Failed to find / register SystemVM template(s)"); } } + + protected void migrateConfigurationScopeToBitmask(Connection conn) { + String scopeDataType = DbUpgradeUtils.getTableColumnType(conn, "configuration", "scope"); + System.out.println("------------------------" + scopeDataType); + logger.info("------------------------{}", scopeDataType); + if (!"varchar(255)".equals(scopeDataType)) { + return; + } + DbUpgradeUtils.addTableColumnIfNotExist(conn, "configuration", "new_scope", "BIGINT DEFAULT 0"); + migrateExistingConfigurationScopeValues(conn); + DbUpgradeUtils.dropTableColumnsIfExist(conn, "configuration", List.of("scope")); + DbUpgradeUtils.changeTableColumnIfNotExist(conn, "configuration", "new_scope", "scope", "BIGINT NOT NULL DEFAULT 0 COMMENT 'Bitmask for scope(s) of this parameter'"); + } + + protected void migrateExistingConfigurationScopeValues(Connection conn) { + StringBuilder sql = new StringBuilder("UPDATE configuration\n" + + "SET new_scope = " + + " CASE "); + for (ConfigKey.Scope scope : ConfigKey.Scope.values()) { + sql.append(" WHEN scope = '").append(scope.name()).append("' THEN ").append(scope.getBitValue()).append(" "); + } + sql.append(" ELSE 0 " + + " END " + + "WHERE scope IS NOT NULL;"); + TransactionLegacy txn = TransactionLegacy.currentTxn(); + try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql.toString())) { + pstmt.executeUpdate(); + } catch (SQLException e) { + logger.error("Failed to migrate existing configuration scope values to bitmask", e); + throw new CloudRuntimeException(String.format("Failed to migrate existing configuration scope values to bitmask due to: %s", e.getMessage())); + } + } } diff --git a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java index 510270ad7bf1..3bdbdfc90a4c 100644 --- a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java @@ -34,6 +34,7 @@ import com.cloud.domain.dao.DomainDao; import com.cloud.domain.dao.DomainDetailsDao; import com.cloud.user.dao.AccountDao; +import com.cloud.utils.Pair; import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.QueryBuilder; @@ -162,4 +163,13 @@ public String getActualValue(AccountDetailVO accountDetailVO) { } return accountDetailVO.getValue(); } + + @Override + public Pair getParentScope(long id) { + Account account = _accountDao.findById(id); + if (account == null) { + return null; + } + return new Pair<>(getScope().getParent(), account.getDomainId()); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java index 148304906009..ab90ded6d790 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java @@ -20,6 +20,8 @@ import java.util.List; import java.util.Map; +import javax.inject.Inject; + import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.ConfigKey.Scope; @@ -27,6 +29,8 @@ import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; import org.springframework.stereotype.Component; +import com.cloud.storage.ImageStore; +import com.cloud.utils.Pair; import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchBuilder; @@ -36,6 +40,8 @@ @Component public class ImageStoreDetailsDaoImpl extends ResourceDetailsDaoBase implements ImageStoreDetailsDao, ScopedConfigStorage { + @Inject + ImageStoreDao imageStoreDao; protected final SearchBuilder storeSearch; @@ -115,4 +121,13 @@ public void addDetail(long resourceId, String key, String value, boolean display super.addDetail(new ImageStoreDetailVO(resourceId, key, value, display)); } + @Override + public Pair getParentScope(long id) { + ImageStore store = imageStoreDao.findById(id); + if (store == null) { + return null; + } + return new Pair<>(getScope().getParent(), store.getDataCenterId()); + } + } diff --git a/framework/config/pom.xml b/framework/config/pom.xml index a29fcfabd622..403429380b65 100644 --- a/framework/config/pom.xml +++ b/framework/config/pom.xml @@ -38,11 +38,5 @@ cloud-framework-db ${project.version} - - org.apache.cloudstack - cloud-api - 4.20.1.0-SNAPSHOT - compile - diff --git a/framework/config/src/main/java/org/apache/cloudstack/config/Configuration.java b/framework/config/src/main/java/org/apache/cloudstack/config/Configuration.java index b93817a99191..d31d14586be8 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/config/Configuration.java +++ b/framework/config/src/main/java/org/apache/cloudstack/config/Configuration.java @@ -17,6 +17,9 @@ package org.apache.cloudstack.config; import java.util.Date; +import java.util.List; + +import org.apache.cloudstack.framework.config.ConfigKey; /** * Configuration represents one global configuration parameter for CloudStack. @@ -74,7 +77,9 @@ enum ValueType { * always global. A non-null value indicates that this parameter can be * set at a certain organization level. */ - String getScope(); + int getScope(); + + List getScopes(); /** * @return can the configuration parameter be changed without restarting the server. diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java index 59fc1b76fc17..e3d115309357 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java @@ -18,9 +18,13 @@ import java.sql.Date; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; +import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; @@ -32,6 +36,7 @@ * */ public class ConfigKey { + private static final Logger logger = LogManager.getLogger(ConfigKey.class); public static final String CATEGORY_ADVANCED = "Advanced"; public static final String CATEGORY_ALERT = "Alert"; @@ -39,25 +44,31 @@ public class ConfigKey { public static final String CATEGORY_SYSTEM = "System"; public enum Scope { - Global(null), - Zone(Global), - Cluster(Zone), - StoragePool(Cluster), - ManagementServer(Global), - ImageStore(Zone), - Domain(Global), - Account(Domain); + Global(null, 1), + Zone(Global, 1 << 1), + Cluster(Zone, 1 << 2), + StoragePool(Cluster, 1 << 3), + ManagementServer(Global, 1 << 4), + ImageStore(Zone, 1 << 5), + Domain(Global, 1 << 6), + Account(Domain, 1 << 7); private final Scope parent; + private final int bitValue; - Scope(Scope parent) { + Scope(Scope parent, int bitValue) { this.parent = parent; + this.bitValue = bitValue; } public Scope getParent() { return parent; } + public int getBitValue() { + return bitValue; + } + public boolean isDescendantOf(Scope other) { Scope current = this; while (current != null) { @@ -79,6 +90,43 @@ public static List getAllDescendants(String str) { } return scopes; } + + public static List decode(int bitmask) { + if (bitmask == 0) { + return Collections.emptyList(); + } + List scopes = new ArrayList<>(); + for (Scope scope : Scope.values()) { + if ((bitmask & scope.getBitValue()) != 0) { + scopes.add(scope); + } + } + return scopes; + } + + public static String decodeAsCsv(int bitmask) { + if (bitmask == 0) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (Scope scope : Scope.values()) { + if ((bitmask & scope.getBitValue()) != 0) { + builder.append(scope.name()).append(", "); + } + } + if (builder.length() > 0) { + builder.setLength(builder.length() - 2); + } + return builder.toString(); + } + + public static int getBitmask(Scope... scopes) { + int bitmask = 0; + for (Scope scope : scopes) { + bitmask |= scope.getBitValue(); + } + return bitmask; + } } public enum Kind { @@ -111,8 +159,8 @@ public String displayText() { return _displayText; } - public Scope scope() { - return _scope; + public List getScopes() { + return scopes; } public boolean isDynamic() { @@ -149,7 +197,7 @@ public String toString() { private final String _defaultValue; private final String _description; private final String _displayText; - private final Scope _scope; // Parameter can be at different levels (Zone/cluster/pool/account), by default every parameter is at global + private final List scopes; // Parameter can be at different levels (Zone/cluster/pool/account), by default every parameter is at global private final boolean _isDynamic; private final String _parent; private final Ternary _group; // Group name, description with precedence @@ -169,6 +217,10 @@ public ConfigKey(String category, Class type, String name, String defaultValu this(type, name, category, defaultValue, description, isDynamic, scope, null); } + public ConfigKey(String category, Class type, String name, String defaultValue, String description, boolean isDynamic, List scopes) { + this(type, name, category, defaultValue, description, isDynamic, scopes, null); + } + public ConfigKey(String category, Class type, String name, String defaultValue, String description, boolean isDynamic, Scope scope, String parent) { this(type, name, category, defaultValue, description, isDynamic, scope, null, null, parent, null, null, null, null); } @@ -189,6 +241,10 @@ public ConfigKey(Class type, String name, String category, String defaultValu this(type, name, category, defaultValue, description, isDynamic, scope, multiplier, null, null, null, null, null, null); } + public ConfigKey(Class type, String name, String category, String defaultValue, String description, boolean isDynamic, List scopes, T multiplier) { + this(type, name, category, defaultValue, description, isDynamic, scopes, multiplier, null, null, null, null, null, null); + } + public ConfigKey(Class type, String name, String category, String defaultValue, String description, boolean isDynamic, Scope scope, T multiplier, String parent) { this(type, name, category, defaultValue, description, isDynamic, scope, multiplier, null, parent, null, null, null, null); } @@ -200,13 +256,22 @@ public ConfigKey(Class type, String name, String category, String defaultValu public ConfigKey(Class type, String name, String category, String defaultValue, String description, boolean isDynamic, Scope scope, T multiplier, String displayText, String parent, Ternary group, Pair subGroup, Kind kind, String options) { + this(type, name, category, defaultValue, description, isDynamic, scope == null ? null : List.of(scope), multiplier, + displayText, parent, group, subGroup, kind, options); + } + + public ConfigKey(Class type, String name, String category, String defaultValue, String description, boolean isDynamic, List scopes, T multiplier, + String displayText, String parent, Ternary group, Pair subGroup, Kind kind, String options) { _category = category; _type = type; _name = name; _defaultValue = defaultValue; _description = description; _displayText = displayText; - _scope = scope; + this.scopes = new ArrayList<>(); + if (scopes != null) { + this.scopes.addAll(scopes); + } _isDynamic = isDynamic; _multiplier = multiplier; _parent = parent; @@ -259,28 +324,45 @@ public T value() { String value = s_depot != null ? s_depot.getConfigStringValue(_name, Scope.Global, null) : null; _value = valueOf((value == null) ? defaultValue() : value); } - return _value; } - protected T valueInScope(Scope scope, Long id) { - if (id == null) { + protected T valueInGlobalOrAvailableParentScope(Scope scope, Long id) { + if (scopes.size() <= 1) { return value(); } + Pair s = new Pair<>(scope, id); + do { + s = s_depot != null ? s_depot.getParentScope(s.first(), s.second()) : null; + if (s != null && scopes.contains(s.first())) { + return valueInScope(s.first(), s.second()); + } + } while (s != null); + logger.trace("Global value for config ({}}): {}", _name, _value); + return value(); + } + public T valueInScope(Scope scope, Long id) { + if (id == null) { + return value(); + } String value = s_depot != null ? s_depot.getConfigStringValue(_name, scope, id) : null; if (value == null) { - Pair parentScope = s_depot != null ? s_depot.getParentScope(scope, id) : null; - if (parentScope != null) { - return valueInScope(parentScope.first(), parentScope.second()); - } - return value(); + return valueInGlobalOrAvailableParentScope(scope, id); } + logger.trace("Scope({}) value for config ({}}): {}", scope, _name, _value); return valueOf(value); } + protected Scope getPrimaryScope() { + if (CollectionUtils.isNotEmpty(scopes)) { + return scopes.get(0); + } + return null; + } + public T valueIn(Long id) { - return valueInScope(_scope, id); + return valueInScope(getPrimaryScope(), id); } public T valueInDomain(Long domainId) { @@ -322,4 +404,20 @@ protected T valueOf(String value) { } } + public boolean isGlobalOrEmptyScope() { + return CollectionUtils.isEmpty(scopes) || + (scopes.size() == 1 && scopes.get(0) == Scope.Global); + } + + public int getScopeBitmask() { + int bitmask = 0; + if (CollectionUtils.isEmpty(scopes)) { + return bitmask; + } + for (Scope scope : scopes) { + bitmask |= scope.getBitValue(); + } + return bitmask; + } + } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ScopedConfigStorage.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ScopedConfigStorage.java index 8126b9510a27..7a109456eb04 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ScopedConfigStorage.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ScopedConfigStorage.java @@ -18,6 +18,8 @@ import org.apache.cloudstack.framework.config.ConfigKey.Scope; +import com.cloud.utils.Pair; + /** * * This method is used by individual storage for configuration @@ -31,4 +33,7 @@ public interface ScopedConfigStorage { default String getConfigValue(long id, ConfigKey key) { return getConfigValue(id, key.key()); } + default Pair getParentScope(long id) { + return null; + } } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java index d22656661803..88569558fc6b 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java @@ -18,10 +18,8 @@ import java.util.Map; -import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.impl.ConfigurationVO; -import com.cloud.utils.Pair; import com.cloud.utils.db.GenericDao; public interface ConfigurationDao extends GenericDao { @@ -69,5 +67,4 @@ public interface ConfigurationDao extends GenericDao { boolean update(String name, String category, String value); void invalidateCache(); - Pair getParentScope(ConfigKey.Scope scope, Long id); } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java index 393bda5b2a17..4d3e7dbd9a62 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java @@ -25,15 +25,9 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import org.springframework.stereotype.Component; -import com.cloud.org.Cluster; -import com.cloud.storage.ImageStore; -import com.cloud.storage.StoragePool; -import com.cloud.user.Account; -import com.cloud.utils.Pair; import com.cloud.utils.component.ComponentLifecycle; import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.DB; @@ -217,48 +211,4 @@ public ConfigurationVO findByName(String name) { sc.setParameters("name", name); return findOneIncludingRemovedBy(sc); } - - @Override - public Pair getParentScope(ConfigKey.Scope scope, Long id) { - if (scope == null || scope.getParent() == null || ConfigKey.Scope.Global.equals(scope.getParent()) - || id == null) { - return null; - } - switch (scope) { - case Account: { - Account account = entityManager.findById(Account.class, id); - if (account != null) { - return new Pair<>(scope.getParent(), account.getDomainId()); - } - break; - } - case ImageStore: { - ImageStore store = entityManager.findById(ImageStore.class, id); - if (store != null) { - return new Pair<>(scope.getParent(), store.getDataCenterId()); - } - break; - } - case StoragePool: { - StoragePool pool = entityManager.findById(StoragePool.class, id); - if (pool != null) { - if (pool.getClusterId() != null) { - return new Pair<>(scope.getParent(), pool.getClusterId()); - } else { - return new Pair<>(ConfigKey.Scope.Zone, pool.getDataCenterId()); - } - } - break; - } - case Cluster: { - Cluster cluster = entityManager.findById(Cluster.class, id); - if (cluster != null) { - return new Pair<>(scope.getParent(), cluster.getDataCenterId()); - } - break; - } - } - return null; - } - } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java index d07ceb7bb0b9..439e57dafeff 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java @@ -148,9 +148,11 @@ protected void populateConfiguration(Date date, Configurable configurable) { createOrupdateConfigObject(date, configurable.getConfigComponentName(), key, null); - if ((key.scope() != null) && (key.scope() != ConfigKey.Scope.Global)) { - Set> currentConfigs = _scopeLevelConfigsMap.get(key.scope()); - currentConfigs.add(key); + if (!key.isGlobalOrEmptyScope()) { + for (ConfigKey.Scope scope : key.getScopes()) { + Set> currentConfigs = _scopeLevelConfigsMap.get(scope); + currentConfigs.add(key); + } } } @@ -208,12 +210,12 @@ private void createOrupdateConfigObject(Date date, String componentName, ConfigK } else { boolean configUpdated = false; if (vo.isDynamic() != key.isDynamic() || !ObjectUtils.equals(vo.getDescription(), key.description()) || !ObjectUtils.equals(vo.getDefaultValue(), key.defaultValue()) || - !ObjectUtils.equals(vo.getScope(), key.scope().toString()) || + !ObjectUtils.equals(vo.getScope(), key.getScopeBitmask()) || !ObjectUtils.equals(vo.getComponent(), componentName)) { vo.setDynamic(key.isDynamic()); vo.setDescription(key.description()); vo.setDefaultValue(key.defaultValue()); - vo.setScope(key.scope().toString()); + vo.setScope(key.getScopeBitmask()); vo.setComponent(componentName); vo.setUpdated(date); configUpdated = true; @@ -287,12 +289,7 @@ protected String getConfigStringValueInternal(String cacheKey) { scopeId = Long.valueOf(parts[2]); } catch (IllegalArgumentException ignored) {} if (!ConfigKey.Scope.Global.equals(scope) && scopeId != null) { - ScopedConfigStorage scopedConfigStorage = null; - for (ScopedConfigStorage storage : _scopedStorages) { - if (storage.getScope() == scope) { - scopedConfigStorage = storage; - } - } + ScopedConfigStorage scopedConfigStorage = getScopedStorage(scope); if (scopedConfigStorage == null) { throw new CloudRuntimeException("Unable to find config storage for this scope: " + scope + " for " + key); } @@ -319,26 +316,6 @@ public void invalidateConfigCache(String key, ConfigKey.Scope scope, Long scopeI configCache.invalidate(getConfigCacheKey(key, scope, scopeId)); } - public ScopedConfigStorage findScopedConfigStorage(ConfigKey config) { - for (ScopedConfigStorage storage : _scopedStorages) { - if (storage.getScope() == config.scope()) { - return storage; - } - } - - throw new CloudRuntimeException("Unable to find config storage for this scope: " + config.scope() + " for " + config.key()); - } - - public ScopedConfigStorage getDomainScope(ConfigKey config) { - for (ScopedConfigStorage storage : _scopedStorages) { - if (storage.getScope() == ConfigKey.Scope.Domain) { - return storage; - } - } - - throw new CloudRuntimeException("Unable to find config storage for this scope: " + ConfigKey.Scope.Domain + " for " + config.key()); - } - public List getScopedStorages() { return _scopedStorages; } @@ -403,8 +380,26 @@ public boolean isNewConfig(ConfigKey configKey) { return newConfigs.contains(configKey.key()); } + protected ScopedConfigStorage getScopedStorage(ConfigKey.Scope scope) { + ScopedConfigStorage scopedConfigStorage = null; + for (ScopedConfigStorage storage : _scopedStorages) { + if (storage.getScope() == scope) { + scopedConfigStorage = storage; + break; + } + } + return scopedConfigStorage; + } + @Override public Pair getParentScope(ConfigKey.Scope scope, Long id) { - return _configDao.getParentScope(scope, id); + if (scope.getParent() == null) { + return null; + } + ScopedConfigStorage scopedConfigStorage = getScopedStorage(scope); + if (scopedConfigStorage == null) { + return null; + } + return scopedConfigStorage.getParentScope(id); } } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigurationVO.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigurationVO.java index c705cc64072a..d12a41864b05 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigurationVO.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigurationVO.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.framework.config.impl; import java.util.Date; +import java.util.List; import javax.persistence.Column; import javax.persistence.Entity; @@ -60,7 +61,7 @@ public class ConfigurationVO implements Configuration { private boolean dynamic; @Column(name = "scope") - private String scope; + private Integer scope; @Column(name = "updated") @Temporal(value = TemporalType.TIMESTAMP) @@ -102,6 +103,7 @@ public ConfigurationVO(String category, String instance, String component, Strin this.name = name; this.description = description; this.parent = parentConfigName; + this.scope = 0; setValue(value); setDisplayText(displayText); setGroupId(groupId); @@ -112,7 +114,7 @@ public ConfigurationVO(String component, ConfigKey key) { this(key.category(), "DEFAULT", component, key.key(), key.defaultValue(), key.description(), key.displayText(), key.parent()); defaultValue = key.defaultValue(); dynamic = key.isDynamic(); - scope = key.scope() != null ? key.scope().toString() : null; + scope = key.getScopeBitmask(); } @Override @@ -183,10 +185,15 @@ public void setDescription(String description) { } @Override - public String getScope() { + public int getScope() { return scope; } + @Override + public List getScopes() { + return ConfigKey.Scope.decode(scope); + } + @Override public boolean isDynamic() { return dynamic; @@ -205,7 +212,7 @@ public void setDefaultValue(String defaultValue) { this.defaultValue = defaultValue; } - public void setScope(String scope) { + public void setScope(int scope) { this.scope = scope; } diff --git a/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java b/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java index fcc9ded684d3..f2d08aa876eb 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java +++ b/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java @@ -484,6 +484,9 @@ public void toSql(final StringBuilder sql, String tableAlias, final Object[] par tableAlias = attr.table; } } + if (op == Op.BINARY_OR) { + sql.append("("); + } sql.append(tableAlias).append(".").append(attr.columnName).append(op.toString()); if (op == Op.IN && params.length == 1) { diff --git a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java index 8affbd5300aa..caf88fadb9fb 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java +++ b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java @@ -38,7 +38,7 @@ public enum Op { " NOT BETWEEN ? AND ? ", 2), IN(" IN () ", -1), NOTIN(" NOT IN () ", -1), LIKE(" LIKE ? ", 1), NLIKE(" NOT LIKE ? ", 1), NIN(" NOT IN () ", -1), NULL(" IS NULL ", 0), NNULL( " IS NOT NULL ", - 0), SC(" () ", 1), TEXT(" () ", 1), RP("", 0), AND(" AND ", 0), OR(" OR ", 0), NOT(" NOT ", 0), FIND_IN_SET(" ) ", 1); + 0), SC(" () ", 1), TEXT(" () ", 1), RP("", 0), AND(" AND ", 0), OR(" OR ", 0), NOT(" NOT ", 0), FIND_IN_SET(" ) ", 1), BINARY_OR(" & ?) > 0", 1); private final String op; int params; diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java index 51524c129121..649b7c6fa9bf 100644 --- a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java @@ -61,6 +61,7 @@ import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.cluster.ClusterDrsAlgorithm; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.management.ManagementServerHost.State; import org.apache.cloudstack.response.ClusterMetricsResponse; import org.apache.cloudstack.response.DbMetricsResponse; @@ -78,6 +79,7 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.commons.beanutils.BeanUtils; import org.apache.commons.collections.CollectionUtils; @@ -634,6 +636,7 @@ public List listVmMetrics(List vmResponses) { public List listStoragePoolMetrics(List poolResponses) { final List metricsResponses = new ArrayList<>(); Map clusterUuidToIdMap = clusterDao.findByUuids(poolResponses.stream().map(StoragePoolResponse::getClusterId).toArray(String[]::new)).stream().collect(Collectors.toMap(ClusterVO::getUuid, ClusterVO::getId)); + Map poolUuidToIdMap = storagePoolDao.findByUuids(poolResponses.stream().map(StoragePoolResponse::getId).toArray(String[]::new)).stream().collect(Collectors.toMap(StoragePoolVO::getUuid, StoragePoolVO::getId)); for (final StoragePoolResponse poolResponse: poolResponses) { StoragePoolMetricsResponse metricsResponse = new StoragePoolMetricsResponse(); @@ -644,8 +647,9 @@ public List listStoragePoolMetrics(List> s_scopeLevelConfigsMap = new HashMap>(); + private static final HashMap> s_scopeLevelConfigsMap = new HashMap<>(); static { - s_scopeLevelConfigsMap.put(ConfigKey.Scope.Zone.toString(), new ArrayList()); - s_scopeLevelConfigsMap.put(ConfigKey.Scope.Cluster.toString(), new ArrayList()); - s_scopeLevelConfigsMap.put(ConfigKey.Scope.StoragePool.toString(), new ArrayList()); - s_scopeLevelConfigsMap.put(ConfigKey.Scope.Account.toString(), new ArrayList()); - s_scopeLevelConfigsMap.put(ConfigKey.Scope.Global.toString(), new ArrayList()); + s_scopeLevelConfigsMap.put(ConfigKey.Scope.Zone.getBitValue(), new ArrayList()); + s_scopeLevelConfigsMap.put(ConfigKey.Scope.Cluster.getBitValue(), new ArrayList()); + s_scopeLevelConfigsMap.put(ConfigKey.Scope.StoragePool.getBitValue(), new ArrayList()); + s_scopeLevelConfigsMap.put(ConfigKey.Scope.Account.getBitValue(), new ArrayList()); + s_scopeLevelConfigsMap.put(ConfigKey.Scope.Global.getBitValue(), new ArrayList()); for (Config c : Config.values()) { //Creating group of parameters per each level (zone/cluster/pool/account) - StringTokenizer tokens = new StringTokenizer(c.getScope(), ","); - while (tokens.hasMoreTokens()) { - String scope = tokens.nextToken().trim(); - List currentConfigs = s_scopeLevelConfigsMap.get(scope); + List scopes = ConfigKey.Scope.decode(c.getScope()); + for (ConfigKey.Scope scope : scopes) { + List currentConfigs = s_scopeLevelConfigsMap.get(scope.getBitValue()); currentConfigs.add(c); - s_scopeLevelConfigsMap.put(scope, currentConfigs); + s_scopeLevelConfigsMap.put(scope.getBitValue(), currentConfigs); } } } @@ -1870,7 +1868,7 @@ private Config(String category, Class componentClass, Class type, String n _defaultValue = defaultValue; _description = description; _range = range; - _scope = ConfigKey.Scope.Global.toString(); + _scope = ConfigKey.Scope.Global.getBitValue(); _kind = kind; _options = options; } @@ -1895,7 +1893,7 @@ public Class getType() { return _type; } - public String getScope() { + public int getScope() { return _scope; } diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index b86592718791..c34c5d74b793 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -1072,7 +1072,7 @@ public Pair resetConfiguration(final ResetCfgCmd cmd) thr Optional optionalValue; String defaultValue; String category; - String configScope; + List configScope; final ConfigurationVO config = _configDao.findByName(name); if (config == null) { configKey = _configDepot.get(name); @@ -1082,11 +1082,11 @@ public Pair resetConfiguration(final ResetCfgCmd cmd) thr } defaultValue = configKey.defaultValue(); category = configKey.category(); - configScope = configKey.scope().toString(); + configScope = configKey.getScopes(); } else { defaultValue = config.getDefaultValue(); category = config.getCategory(); - configScope = config.getScope(); + configScope = config.getScopes(); } String scope = ""; @@ -1111,8 +1111,11 @@ public Pair resetConfiguration(final ResetCfgCmd cmd) thr throw new InvalidParameterValueException("cannot handle multiple IDs, provide only one ID corresponding to the scope"); } - if (scope != null && !scope.equals(ConfigKey.Scope.Global.toString()) && !configScope.contains(scope)) { - throw new InvalidParameterValueException("Invalid scope id provided for the parameter " + name); + if (scope != null) { + ConfigKey.Scope scopeVal = ConfigKey.Scope.valueOf(scope); + if (!scope.equals(ConfigKey.Scope.Global.toString()) && !configScope.contains(scopeVal)) { + throw new InvalidParameterValueException("Invalid scope id provided for the parameter " + name); + } } String newValue = null; @@ -1222,11 +1225,10 @@ protected String validateConfigurationValue(String name, String value, String sc return "Invalid configuration variable."; } - String configScope = cfg.getScope(); + List configScope = cfg.getScopes(); if (scope != null) { ConfigKey.Scope scopeVal = ConfigKey.Scope.valueOf(scope); - ConfigKey.Scope configScopeVal = ConfigKey.Scope.valueOf(configScope); - if (!configScope.contains(scope) && !configScopeVal.isDescendantOf(scopeVal) && + if (!configScope.contains(scopeVal) && !(ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN.value() && configScope.contains(ConfigKey.Scope.Account.toString()) && scope.equals(ConfigKey.Scope.Domain.toString()))) { logger.error("Invalid scope id provided for the parameter " + name); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 4e02677a303b..00c9ce316f6a 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -2192,7 +2192,7 @@ public Pair, Integer> searchForConfigurations(fina final String groupName = cmd.getGroupName(); final String subGroupName = cmd.getSubGroupName(); final String parentName = cmd.getParentName(); - String scope = null; + ConfigKey.Scope scope = null; Long id = null; int paramCountCheck = 0; @@ -2208,35 +2208,35 @@ public Pair, Integer> searchForConfigurations(fina } if (zoneId != null) { - scope = ConfigKey.Scope.Zone.toString(); + scope = ConfigKey.Scope.Zone; id = zoneId; paramCountCheck++; } if (clusterId != null) { - scope = ConfigKey.Scope.Cluster.toString(); + scope = ConfigKey.Scope.Cluster; id = clusterId; paramCountCheck++; } if (accountId != null) { Account account = _accountMgr.getAccount(accountId); _accountMgr.checkAccess(caller, null, false, account); - scope = ConfigKey.Scope.Account.toString(); + scope = ConfigKey.Scope.Account; id = accountId; paramCountCheck++; } if (domainId != null) { _accountMgr.checkAccess(caller, _domainDao.findById(domainId)); - scope = ConfigKey.Scope.Domain.toString(); + scope = ConfigKey.Scope.Domain; id = domainId; paramCountCheck++; } if (storagepoolId != null) { - scope = ConfigKey.Scope.StoragePool.toString(); + scope = ConfigKey.Scope.StoragePool; id = storagepoolId; paramCountCheck++; } if (imageStoreId != null) { - scope = ConfigKey.Scope.ImageStore.toString(); + scope = ConfigKey.Scope.ImageStore; id = imageStoreId; paramCountCheck++; } @@ -2291,22 +2291,19 @@ public Pair, Integer> searchForConfigurations(fina // hidden configurations are not displayed using the search API sc.addAnd("category", SearchCriteria.Op.NEQ, "Hidden"); - if (scope != null && !scope.isEmpty()) { + if (scope != null) { // getting the list of parameters at requested scope if (ConfigurationManagerImpl.ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN.value() - && scope.equals(ConfigKey.Scope.Domain.toString())) { - sc.addAnd("scope", SearchCriteria.Op.IN, ConfigKey.Scope.Domain.toString(), ConfigKey.Scope.Account.toString()); + && scope.equals(ConfigKey.Scope.Domain)) { + sc.addAnd("scope", SearchCriteria.Op.BINARY_OR, (ConfigKey.Scope.Domain.getBitValue() | ConfigKey.Scope.Account.getBitValue())); } else { - ConfigKey.Scope scopeVal = ConfigKey.Scope.valueOf(scope); - List scopes = ConfigKey.Scope.getAllDescendants(scope); - List scopeStrs = scopes.stream().map(Enum::toString).collect(Collectors.toList()); - sc.addAnd("scope", SearchCriteria.Op.IN, scopeStrs.toArray()); + sc.addAnd("scope", SearchCriteria.Op.BINARY_OR, scope.getBitValue()); } } final Pair, Integer> result = _configDao.searchAndCount(sc, searchFilter); - if (scope != null && !scope.isEmpty()) { + if (scope != null) { // Populate values corresponding the resource id final List configVOList = new ArrayList<>(); for (final ConfigurationVO param : result.first()) { @@ -2314,11 +2311,11 @@ public Pair, Integer> searchForConfigurations(fina if (configVo != null) { final ConfigKey key = _configDepot.get(param.getName()); if (key != null) { - if (scope.equals(ConfigKey.Scope.Domain.toString())) { + if (scope.equals(ConfigKey.Scope.Domain)) { Object value = key.valueInDomain(id); configVo.setValue(value == null ? null : value.toString()); } else { - Object value = key.valueIn(id); + Object value = key.valueInScope(scope, id); configVo.setValue(value == null ? null : value.toString()); } configVOList.add(configVo); diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index 2bdc008ca1a6..dbb47126039e 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -1998,7 +1998,7 @@ protected void cleanUpVirtualMachineStats() { Integer maxRetentionTime = vmStatsMaxRetentionTime.value(); if (maxRetentionTime <= 0) { logger.debug(String.format("Skipping VM stats cleanup. The [%s] parameter [%s] is set to 0 or less than 0.", - vmStatsMaxRetentionTime.scope(), vmStatsMaxRetentionTime.toString())); + ConfigKey.Scope.decodeAsCsv(vmStatsMaxRetentionTime.getScopeBitmask()), vmStatsMaxRetentionTime.toString())); return; } logger.trace("Removing older VM stats records."); @@ -2016,7 +2016,7 @@ protected void cleanUpVolumeStats() { if (maxRetentionTime <= 0) { if (logger.isDebugEnabled()) { logger.debug(String.format("Skipping Volume stats cleanup. The [%s] parameter [%s] is set to 0 or less than 0.", - vmDiskStatsMaxRetentionTime.scope(), vmDiskStatsMaxRetentionTime.toString())); + ConfigKey.Scope.decodeAsCsv(vmDiskStatsMaxRetentionTime.getScopeBitmask()), vmDiskStatsMaxRetentionTime.toString())); } return; } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 1c5da19fbbc1..4519f8905505 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -2921,7 +2921,7 @@ private boolean checkUsagedSpace(StoragePool pool) { long totalSize = pool.getCapacityBytes(); long usedSize = getUsedSize(pool); double usedPercentage = ((double)usedSize / (double)totalSize); - double storageUsedThreshold = CapacityManager.StorageCapacityDisableThreshold.valueIn(pool.getDataCenterId()); + double storageUsedThreshold = CapacityManager.StorageCapacityDisableThreshold.valueInScope(ConfigKey.Scope.StoragePool, pool.getId()); if (logger.isDebugEnabled()) { logger.debug("Checking pool {} for storage, totalSize: {}, usedBytes: {}, usedPct: {}, disable threshold: {}", pool, pool.getCapacityBytes(), pool.getUsedBytes(), usedPercentage, storageUsedThreshold); } @@ -3193,7 +3193,7 @@ protected boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTemp logger.debug("Total capacity of the pool {} is {}", poolVO, toHumanReadableSize(totalOverProvCapacity)); - double storageAllocatedThreshold = CapacityManager.StorageAllocatedCapacityDisableThreshold.valueIn(pool.getDataCenterId()); + double storageAllocatedThreshold = CapacityManager.StorageAllocatedCapacityDisableThreshold.valueInScope(ConfigKey.Scope.StoragePool, pool.getId()); if (logger.isDebugEnabled()) { logger.debug("Checking pool: {} for storage allocation , maxSize : {}, " + @@ -3218,7 +3218,7 @@ protected boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTemp return false; } - double storageAllocatedThresholdForResize = CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.valueIn(pool.getDataCenterId()); + double storageAllocatedThresholdForResize = CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.valueInScope(ConfigKey.Scope.StoragePool, pool.getId()); if (usedPercentage > storageAllocatedThresholdForResize) { logger.debug(String.format("Skipping the pool %s since its allocated percentage: %s has crossed the allocated %s: %s", pool, usedPercentage, CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.key(), storageAllocatedThresholdForResize)); diff --git a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java index 0a0452968213..8380ee8a4366 100644 --- a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java +++ b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java @@ -175,7 +175,7 @@ public class ConfigurationManagerImplTest { @Before public void setUp() throws Exception { - Mockito.when(configurationVOMock.getScope()).thenReturn(ConfigKey.Scope.Global.name()); + Mockito.when(configurationVOMock.getScopes()).thenReturn(List.of(ConfigKey.Scope.Global)); Mockito.when(configDao.findByName(Mockito.anyString())).thenReturn(configurationVOMock); Mockito.when(configDepot.get(Mockito.anyString())).thenReturn(configKeyMock); @@ -442,6 +442,7 @@ public void testCreateNetworkOfferingForNsx() { Assert.assertNotNull(offering); } + @Test public void testValidateInvalidConfiguration() { Mockito.doReturn(null).when(configDao).findByName(Mockito.anyString()); String msg = configurationManagerImplSpy.validateConfigurationValue("test.config.name", "testvalue", ConfigKey.Scope.Global.toString()); @@ -451,7 +452,7 @@ public void testValidateInvalidConfiguration() { @Test public void testValidateInvalidScopeForConfiguration() { ConfigurationVO cfg = mock(ConfigurationVO.class); - when(cfg.getScope()).thenReturn(ConfigKey.Scope.Account.toString()); + when(cfg.getScopes()).thenReturn(List.of(ConfigKey.Scope.Account)); Mockito.doReturn(cfg).when(configDao).findByName(Mockito.anyString()); String msg = configurationManagerImplSpy.validateConfigurationValue("test.config.name", "testvalue", ConfigKey.Scope.Domain.toString()); Assert.assertEquals("Invalid scope id provided for the parameter test.config.name", msg); @@ -460,12 +461,12 @@ public void testValidateInvalidScopeForConfiguration() { @Test public void testValidateConfig_ThreadsOnKVMHostToTransferVMwareVMFiles_Failure() { ConfigurationVO cfg = mock(ConfigurationVO.class); - when(cfg.getScope()).thenReturn(ConfigKey.Scope.Global.toString()); + when(cfg.getScopes()).thenReturn(List.of(ConfigKey.Scope.Global)); ConfigKey configKey = UnmanagedVMsManager.ThreadsOnKVMHostToImportVMwareVMFiles; Mockito.doReturn(cfg).when(configDao).findByName(Mockito.anyString()); Mockito.doReturn(configKey).when(configurationManagerImplSpy._configDepot).get(configKey.key()); - String result = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "11", configKey.scope().toString()); + String result = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "11", configKey.getScopes().get(0).name()); Assert.assertNotNull(result); } @@ -473,24 +474,24 @@ public void testValidateConfig_ThreadsOnKVMHostToTransferVMwareVMFiles_Failure() @Test public void testValidateConfig_ThreadsOnKVMHostToTransferVMwareVMFiles_Success() { ConfigurationVO cfg = mock(ConfigurationVO.class); - when(cfg.getScope()).thenReturn(ConfigKey.Scope.Global.toString()); + when(cfg.getScopes()).thenReturn(List.of(ConfigKey.Scope.Global)); ConfigKey configKey = UnmanagedVMsManager.ThreadsOnKVMHostToImportVMwareVMFiles; Mockito.doReturn(cfg).when(configDao).findByName(Mockito.anyString()); Mockito.doReturn(configKey).when(configurationManagerImplSpy._configDepot).get(configKey.key()); - String msg = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "10", configKey.scope().toString()); + String msg = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "10", configKey.getScopes().get(0).name()); Assert.assertNull(msg); } @Test public void testValidateConfig_ConvertVmwareInstanceToKvmTimeout_Failure() { ConfigurationVO cfg = mock(ConfigurationVO.class); - when(cfg.getScope()).thenReturn(ConfigKey.Scope.Global.toString()); + when(cfg.getScopes()).thenReturn(List.of(ConfigKey.Scope.Global)); ConfigKey configKey = UnmanagedVMsManager.ConvertVmwareInstanceToKvmTimeout; Mockito.doReturn(cfg).when(configDao).findByName(Mockito.anyString()); Mockito.doReturn(configKey).when(configurationManagerImplSpy._configDepot).get(configKey.key()); configurationManagerImplSpy.populateConfigValuesForValidationSet(); - String result = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "0", configKey.scope().toString()); + String result = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "0", configKey.getScopes().get(0).name()); Assert.assertNotNull(result); } @@ -498,12 +499,12 @@ public void testValidateConfig_ConvertVmwareInstanceToKvmTimeout_Failure() { @Test public void testValidateConfig_ConvertVmwareInstanceToKvmTimeout_Success() { ConfigurationVO cfg = mock(ConfigurationVO.class); - when(cfg.getScope()).thenReturn(ConfigKey.Scope.Global.toString()); + when(cfg.getScopes()).thenReturn(List.of(ConfigKey.Scope.Global)); ConfigKey configKey = UnmanagedVMsManager.ConvertVmwareInstanceToKvmTimeout; Mockito.doReturn(cfg).when(configDao).findByName(Mockito.anyString()); Mockito.doReturn(configKey).when(configurationManagerImplSpy._configDepot).get(configKey.key()); configurationManagerImplSpy.populateConfigValuesForValidationSet(); - String msg = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "9", configKey.scope().toString()); + String msg = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "9", configKey.getScopes().get(0).name()); Assert.assertNull(msg); } diff --git a/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java b/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java index 2e5a232beb3b..1dc349c31ecc 100644 --- a/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java +++ b/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java @@ -16,15 +16,13 @@ // under the License. package com.cloud.vpc.dao; -import com.cloud.utils.Pair; -import com.cloud.utils.db.GenericDaoBase; +import java.util.HashMap; +import java.util.Map; -import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.config.impl.ConfigurationVO; -import java.util.HashMap; -import java.util.Map; +import com.cloud.utils.db.GenericDaoBase; public class MockConfigurationDaoImpl extends GenericDaoBase implements ConfigurationDao { @@ -119,9 +117,4 @@ public boolean update(String name, String category, String value) { @Override public void invalidateCache() { } - - @Override - public Pair getParentScope(ConfigKey.Scope scope, Long id) { - return null; - } } diff --git a/tools/marvin/setup.py b/tools/marvin/setup.py index 11a63a96aced..803829736892 100644 --- a/tools/marvin/setup.py +++ b/tools/marvin/setup.py @@ -27,7 +27,7 @@ raise RuntimeError("python setuptools is required to build Marvin") -VERSION = "4.21.0.0-SNAPSHOT" +VERSION = "4.21.0.0" setup(name="Marvin", version=VERSION, From 6e7fe6e647aea2e35d3f5b7762a1334f5f026206 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Wed, 29 Jan 2025 13:48:20 +0530 Subject: [PATCH 03/21] Minor service layer changes --- .../java/com/cloud/capacity/CapacityManager.java | 6 +++--- .../cloudstack/framework/config/ConfigKey.java | 12 ++++-------- .../framework/config/dao/ConfigurationDaoImpl.java | 6 +----- .../cloudstack/metrics/MetricsServiceImpl.java | 2 +- .../configuration/ConfigurationManagerImpl.java | 2 +- .../java/com/cloud/network/IpAddressManagerImpl.java | 2 +- .../java/com/cloud/server/ManagementServerImpl.java | 9 ++------- .../java/com/cloud/storage/StorageManagerImpl.java | 6 +++--- 8 files changed, 16 insertions(+), 29 deletions(-) diff --git a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java index 3bbbdb3e42a6..e7df2a74b562 100644 --- a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java +++ b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java @@ -71,7 +71,7 @@ public interface CapacityManager { "0.85", "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", true, - List.of(ConfigKey.Scope.Zone, ConfigKey.Scope.StoragePool)); + List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone)); static final ConfigKey StorageOverprovisioningFactor = new ConfigKey<>( "Storage", @@ -89,7 +89,7 @@ public interface CapacityManager { "0.85", "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for low allocated storage available.", true, - List.of(ConfigKey.Scope.Zone, ConfigKey.Scope.StoragePool)); + List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone)); static final ConfigKey StorageOperationsExcludeCluster = new ConfigKey<>( Boolean.class, @@ -129,7 +129,7 @@ public interface CapacityManager { "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for volume resize. " + "This is applicable only when volume.resize.allowed.beyond.allocation is set to true.", true, - List.of(ConfigKey.Scope.Zone, ConfigKey.Scope.StoragePool)); + List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone)); public boolean releaseVmCapacity(VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId); diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java index e3d115309357..5d93f040ccfb 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java @@ -70,12 +70,12 @@ public int getBitValue() { } public boolean isDescendantOf(Scope other) { - Scope current = this; - while (current != null) { - if (current == other) { + Scope parent = this.getParent(); + while (parent != null) { + if (parent == other) { return true; } - current = current.getParent(); + parent = parent.getParent(); } return false; } @@ -365,10 +365,6 @@ public T valueIn(Long id) { return valueInScope(getPrimaryScope(), id); } - public T valueInDomain(Long domainId) { - return valueInScope(Scope.Domain, domainId); - } - @SuppressWarnings("unchecked") protected T valueOf(String value) { if (value == null) { diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java index 4d3e7dbd9a62..7c4a6f9a609e 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java @@ -22,7 +22,6 @@ import java.util.Map; import javax.annotation.PostConstruct; -import javax.inject.Inject; import javax.naming.ConfigurationException; import org.apache.cloudstack.framework.config.impl.ConfigurationVO; @@ -31,7 +30,6 @@ import com.cloud.utils.component.ComponentLifecycle; import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.DB; -import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -40,9 +38,6 @@ @Component public class ConfigurationDaoImpl extends GenericDaoBase implements ConfigurationDao { - @Inject - EntityManager entityManager; - private Map _configs = null; private boolean _premium; @@ -211,4 +206,5 @@ public ConfigurationVO findByName(String name) { sc.setParameters("name", name); return findOneIncludingRemovedBy(sc); } + } diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java index 649b7c6fa9bf..77e3e08346f8 100644 --- a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java @@ -649,7 +649,7 @@ public List listStoragePoolMetrics(List, Integer> searchForConfigurations(fina if (configVo != null) { final ConfigKey key = _configDepot.get(param.getName()); if (key != null) { - if (scope.equals(ConfigKey.Scope.Domain)) { - Object value = key.valueInDomain(id); - configVo.setValue(value == null ? null : value.toString()); - } else { - Object value = key.valueInScope(scope, id); - configVo.setValue(value == null ? null : value.toString()); - } + Object value = key.valueInScope(scope, id); + configVo.setValue(value == null ? null : value.toString()); configVOList.add(configVo); } else { logger.warn("ConfigDepot could not find parameter " + param.getName() + " for scope " + scope); diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 4519f8905505..1c5da19fbbc1 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -2921,7 +2921,7 @@ private boolean checkUsagedSpace(StoragePool pool) { long totalSize = pool.getCapacityBytes(); long usedSize = getUsedSize(pool); double usedPercentage = ((double)usedSize / (double)totalSize); - double storageUsedThreshold = CapacityManager.StorageCapacityDisableThreshold.valueInScope(ConfigKey.Scope.StoragePool, pool.getId()); + double storageUsedThreshold = CapacityManager.StorageCapacityDisableThreshold.valueIn(pool.getDataCenterId()); if (logger.isDebugEnabled()) { logger.debug("Checking pool {} for storage, totalSize: {}, usedBytes: {}, usedPct: {}, disable threshold: {}", pool, pool.getCapacityBytes(), pool.getUsedBytes(), usedPercentage, storageUsedThreshold); } @@ -3193,7 +3193,7 @@ protected boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTemp logger.debug("Total capacity of the pool {} is {}", poolVO, toHumanReadableSize(totalOverProvCapacity)); - double storageAllocatedThreshold = CapacityManager.StorageAllocatedCapacityDisableThreshold.valueInScope(ConfigKey.Scope.StoragePool, pool.getId()); + double storageAllocatedThreshold = CapacityManager.StorageAllocatedCapacityDisableThreshold.valueIn(pool.getDataCenterId()); if (logger.isDebugEnabled()) { logger.debug("Checking pool: {} for storage allocation , maxSize : {}, " + @@ -3218,7 +3218,7 @@ protected boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTemp return false; } - double storageAllocatedThresholdForResize = CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.valueInScope(ConfigKey.Scope.StoragePool, pool.getId()); + double storageAllocatedThresholdForResize = CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.valueIn(pool.getDataCenterId()); if (usedPercentage > storageAllocatedThresholdForResize) { logger.debug(String.format("Skipping the pool %s since its allocated percentage: %s has crossed the allocated %s: %s", pool, usedPercentage, CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.key(), storageAllocatedThresholdForResize)); From 24d4cfb2607468cb610c1d9cd1467924a6f5646f Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Wed, 29 Jan 2025 22:53:53 +0530 Subject: [PATCH 04/21] Unit Tests for multi scope config --- .../framework/config/ConfigKeyTest.java | 29 ++++++++++ .../config/impl/ConfigDepotImplTest.java | 31 ++++++++++ .../metrics/MetricsServiceImpl.java | 1 - .../cloud/server/ManagementServerImpl.java | 6 +- .../ConfigurationManagerImplTest.java | 32 +++++++++++ .../server/ManagementServerImplTest.java | 56 +++++++++++++++++++ 6 files changed, 151 insertions(+), 4 deletions(-) diff --git a/framework/config/src/test/java/org/apache/cloudstack/framework/config/ConfigKeyTest.java b/framework/config/src/test/java/org/apache/cloudstack/framework/config/ConfigKeyTest.java index a3a8aadfa604..50be7200d56f 100644 --- a/framework/config/src/test/java/org/apache/cloudstack/framework/config/ConfigKeyTest.java +++ b/framework/config/src/test/java/org/apache/cloudstack/framework/config/ConfigKeyTest.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.framework.config; +import java.util.List; + import org.junit.Assert; import org.junit.Test; @@ -47,4 +49,31 @@ public void testIsSameKeyAsThrowingCloudRuntimeException() { ConfigKey key = new ConfigKey("hond", Boolean.class, "naam", "truus", "thrown name", false); Assert.assertFalse("zero and 0L should be considered the same address", key.isSameKeyAs(0L)); } + + @Test + public void testDecode() { + ConfigKey key = new ConfigKey("testcategoey", Boolean.class, "test", "true", "test descriptuin", false, List.of(Scope.Zone, Scope.StoragePool)); + int bitmask = key.getScopeBitmask(); + List scopes = ConfigKey.Scope.decode(bitmask); + Assert.assertEquals(bitmask, ConfigKey.Scope.getBitmask(scopes.toArray(new Scope[0]))); + for (Scope scope : scopes) { + Assert.assertTrue(scope == Scope.Zone || scope == Scope.StoragePool); + } + } + + @Test + public void testDecodeAsCsv() { + ConfigKey key = new ConfigKey("testcategoey", Boolean.class, "test", "true", "test descriptuin", false, List.of(Scope.Zone, Scope.StoragePool)); + int bitmask = key.getScopeBitmask(); + String scopes = ConfigKey.Scope.decodeAsCsv(bitmask); + Assert.assertTrue("Zone, StoragePool".equals(scopes)); + } + + @Test + public void testGetDescendants() { + List descendants = ConfigKey.Scope.getAllDescendants(Scope.Zone.name()); + for (Scope descendant : descendants) { + Assert.assertTrue(descendant == Scope.Cluster || descendant == Scope.StoragePool || descendant == Scope.ImageStore); + } + } } diff --git a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java index 8a7da7953450..1d4a60880baf 100644 --- a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java +++ b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java @@ -20,10 +20,13 @@ import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Set; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.dao.ConfigurationSubGroupDao; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -38,6 +41,8 @@ public class ConfigDepotImplTest { @Mock ConfigurationDao _configDao; + @Mock + ConfigurationSubGroupDao configSubGroupDao; @InjectMocks private ConfigDepotImpl configDepotImpl = new ConfigDepotImpl(); @@ -107,4 +112,30 @@ public void testGetConfigStringValueAfterExpiry() { runTestGetConfigStringValueExpiry(((ConfigDepotImpl.CONFIG_CACHE_EXPIRE_SECONDS) + 5) * 1000, 2); } + + @Test + public void testPopulateConfiguration() { + ConfigKey StorageDisableThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ALERT, Double.class, "pool.storage.capacity.disablethreshold", "0.85", + "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", + true, List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone)); + Configurable configurable = new Configurable() { + @Override + public String getConfigComponentName() { + return "test"; + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] { StorageDisableThreshold }; + } + }; + configDepotImpl.setConfigurables(List.of(configurable)); + configDepotImpl.populateConfigurations(); + + Assert.assertEquals("pool.storage.capacity.disablethreshold", + configDepotImpl._scopeLevelConfigsMap.get(ConfigKey.Scope.Zone).iterator().next().key()); + Assert.assertEquals("pool.storage.capacity.disablethreshold", + configDepotImpl._scopeLevelConfigsMap.get(ConfigKey.Scope.StoragePool).iterator().next().key()); + Assert.assertEquals(0, configDepotImpl._scopeLevelConfigsMap.get(ConfigKey.Scope.Cluster).size()); + } } diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java index 77e3e08346f8..65f04ae783ca 100644 --- a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java @@ -61,7 +61,6 @@ import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.cluster.ClusterDrsAlgorithm; import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.management.ManagementServerHost.State; import org.apache.cloudstack.response.ClusterMetricsResponse; import org.apache.cloudstack.response.DbMetricsResponse; diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 8d0b64509b27..443047b4503d 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -892,7 +892,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @Inject protected UserVmDao _userVmDao; @Inject - private ConfigurationDao _configDao; + protected ConfigurationDao _configDao; @Inject private ConfigurationGroupDao _configGroupDao; @Inject @@ -904,7 +904,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @Inject private DiskOfferingDao _diskOfferingDao; @Inject - private DomainDao _domainDao; + protected DomainDao _domainDao; @Inject private AccountDao _accountDao; @Inject @@ -960,7 +960,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @Inject private HostTagsDao _hostTagsDao; @Inject - private ConfigDepot _configDepot; + protected ConfigDepot _configDepot; @Inject private UserVmManager _userVmMgr; @Inject diff --git a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java index 8380ee8a4366..8c714b57cdbb 100644 --- a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java +++ b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java @@ -46,14 +46,17 @@ import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; import com.cloud.user.User; +import com.cloud.utils.Pair; import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.net.NetUtils; import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.annotation.dao.AnnotationDao; +import org.apache.cloudstack.api.command.admin.config.ResetCfgCmd; import org.apache.cloudstack.api.command.admin.network.CreateNetworkOfferingCmd; import org.apache.cloudstack.api.command.admin.offering.UpdateDiskOfferingCmd; import org.apache.cloudstack.api.command.admin.zone.DeleteZoneCmd; +import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; @@ -61,6 +64,9 @@ import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.resourcedetail.DiskOfferingDetailVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.vm.UnmanagedVMsManager; import org.junit.Assert; import org.junit.Before; @@ -159,6 +165,10 @@ public class ConfigurationManagerImplTest { NetworkService networkService; @Mock NetworkModel networkModel; + @Mock + PrimaryDataStoreDao storagePoolDao; + @Mock + StoragePoolDetailsDao storagePoolDetailsDao; DeleteZoneCmd deleteZoneCmd; CreateNetworkOfferingCmd createNetworkOfferingCmd; @@ -852,4 +862,26 @@ public void shouldValidateConfigRangeTestValueIsNotNullAndConfigHasRangeReturnTr boolean result = configurationManagerImplSpy.shouldValidateConfigRange(Config.ConsoleProxySessionMax.name(), "test", Config.ConsoleProxyUrlDomain); Assert.assertTrue(result); } + + @Test + public void testResetConfigurations() { + Long poolId = 1L; + ResetCfgCmd cmd = Mockito.mock(ResetCfgCmd.class); + Mockito.when(cmd.getCfgName()).thenReturn("pool.storage.capacity.disablethreshold"); + Mockito.when(cmd.getStoragepoolId()).thenReturn(poolId); + Mockito.when(cmd.getZoneId()).thenReturn(null); + Mockito.when(cmd.getClusterId()).thenReturn(null); + Mockito.when(cmd.getAccountId()).thenReturn(null); + Mockito.when(cmd.getDomainId()).thenReturn(null); + Mockito.when(cmd.getImageStoreId()).thenReturn(null); + + ConfigurationVO cfg = new ConfigurationVO("Advanced", "DEFAULT", "test", "pool.storage.capacity.disablethreshold", null, "description"); + cfg.setScope(10); + cfg.setDefaultValue(".85"); + Mockito.when(configDao.findByName("pool.storage.capacity.disablethreshold")).thenReturn(cfg); + Mockito.when(storagePoolDao.findById(poolId)).thenReturn(Mockito.mock(StoragePoolVO.class)); + + Pair result = configurationManagerImplSpy.resetConfiguration(cmd); + Assert.assertEquals(".85", result.second()); + } } diff --git a/server/src/test/java/com/cloud/server/ManagementServerImplTest.java b/server/src/test/java/com/cloud/server/ManagementServerImplTest.java index b26cd455cfbb..8b9928733ec9 100644 --- a/server/src/test/java/com/cloud/server/ManagementServerImplTest.java +++ b/server/src/test/java/com/cloud/server/ManagementServerImplTest.java @@ -17,6 +17,7 @@ package com.cloud.server; import com.cloud.dc.Vlan.VlanType; +import com.cloud.domain.dao.DomainDao; import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.DetailVO; import com.cloud.host.Host; @@ -48,15 +49,20 @@ import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.admin.config.ListCfgsByCmd; import org.apache.cloudstack.api.command.user.address.ListPublicIpAddressesCmd; import org.apache.cloudstack.api.command.user.ssh.RegisterSSHKeyPairCmd; import org.apache.cloudstack.api.command.user.userdata.DeleteUserDataCmd; import org.apache.cloudstack.api.command.user.userdata.ListUserDataCmd; import org.apache.cloudstack.api.command.user.userdata.RegisterUserDataCmd; +import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; +import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import org.apache.cloudstack.userdata.UserDataManager; import org.junit.After; import org.junit.Assert; @@ -131,6 +137,16 @@ public class ManagementServerImplTest { @Mock HostDetailsDao hostDetailsDao; + + @Mock + ConfigurationDao configDao; + + @Mock + ConfigDepot configDepot; + + @Mock + DomainDao domainDao; + private AutoCloseable closeable; @Before @@ -145,6 +161,9 @@ public void setup() throws IllegalAccessException, NoSuchFieldException { spy._UserVmDetailsDao = userVmDetailsDao; spy._detailsDao = hostDetailsDao; spy.userDataManager = userDataManager; + spy._configDao = configDao; + spy._configDepot = configDepot; + spy._domainDao = domainDao; } @After @@ -684,4 +703,41 @@ public void testZoneWideVolumeRequiresStorageMotionDriverDependent() { Mockito.when(driver.zoneWideVolumesAvailableWithoutClusterMotion()).thenReturn(false); Assert.assertTrue(spy.zoneWideVolumeRequiresStorageMotion(dataStore, host1, host2)); } + + @Test(expected = InvalidParameterValueException.class) + public void testSearchForConfigurationsMultipleIds() { + ListCfgsByCmd cmd = Mockito.mock(ListCfgsByCmd.class); + Mockito.when(cmd.getConfigName()).thenReturn("pool.storage.capacity.disablethreshold"); + Mockito.when(cmd.getZoneId()).thenReturn(1L); + Mockito.when(cmd.getStoragepoolId()).thenReturn(2L); + spy.searchForConfigurations(cmd); + } + + @Test + public void testSearchForConfigurations() { + Long poolId = 1L; + ListCfgsByCmd cmd = Mockito.mock(ListCfgsByCmd.class); + Mockito.when(cmd.getConfigName()).thenReturn("pool.storage.capacity.disablethreshold"); + Mockito.when(cmd.getStoragepoolId()).thenReturn(poolId); + Mockito.when(cmd.getZoneId()).thenReturn(null); + Mockito.when(cmd.getClusterId()).thenReturn(null); + Mockito.when(cmd.getAccountId()).thenReturn(null); + Mockito.when(cmd.getDomainId()).thenReturn(null); + Mockito.when(cmd.getImageStoreId()).thenReturn(null); + + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(configDao.createSearchCriteria()).thenReturn(sc); + ConfigurationVO cfg = new ConfigurationVO("Advanced", "DEFAULT", "test", "pool.storage.capacity.disablethreshold", null, "description"); + Mockito.when(configDao.searchAndCount(any(), any())).thenReturn(new Pair<>(List.of(cfg), 1)); + Mockito.when(configDao.findByName("pool.storage.capacity.disablethreshold")).thenReturn(cfg); + + ConfigKey storageDisableThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ALERT, Double.class, "pool.storage.capacity.disablethreshold", "0.85", + "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", + true, List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone)); + when(configDepot.get("pool.storage.capacity.disablethreshold")).thenReturn(storageDisableThreshold); + + Pair, Integer> result = spy.searchForConfigurations(cmd); + + Assert.assertEquals("0.85", result.first().get(0).getValue()); + } } From 80f2554085d5434fc8ec7d9d287cf6d1a23fd46e Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Wed, 29 Jan 2025 22:56:19 +0530 Subject: [PATCH 05/21] revert changes to marvin/setup.py --- tools/marvin/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/marvin/setup.py b/tools/marvin/setup.py index 803829736892..11a63a96aced 100644 --- a/tools/marvin/setup.py +++ b/tools/marvin/setup.py @@ -27,7 +27,7 @@ raise RuntimeError("python setuptools is required to build Marvin") -VERSION = "4.21.0.0" +VERSION = "4.21.0.0-SNAPSHOT" setup(name="Marvin", version=VERSION, From 3f765761db1c0997b1b780cdfaa01030fee8700d Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Fri, 31 Jan 2025 01:05:09 +0530 Subject: [PATCH 06/21] fixed 41720to41800 upgrade path --- .../cloud/upgrade/ConfigurationGroupsAggregator.java | 10 +++++++++- .../com/cloud/upgrade/dao/DatabaseAccessObject.java | 5 ----- .../com/cloud/upgrade/dao/Upgrade41720to41800.java | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/ConfigurationGroupsAggregator.java b/engine/schema/src/main/java/com/cloud/upgrade/ConfigurationGroupsAggregator.java index 03857137ded6..e64658100b48 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/ConfigurationGroupsAggregator.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/ConfigurationGroupsAggregator.java @@ -32,6 +32,9 @@ import org.apache.commons.lang3.StringUtils; import com.cloud.utils.Pair; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -54,7 +57,12 @@ public ConfigurationGroupsAggregator() { public void updateConfigurationGroups() { LOG.debug("Updating configuration groups"); - List configs = configDao.listAllIncludingRemoved(); + SearchBuilder sb = configDao.createSearchBuilder(); + sb.select("name", SearchCriteria.Func.NATIVE, sb.entity().getName()); + sb.select("groupId", SearchCriteria.Func.NATIVE, sb.entity().getGroupId()); + sb.select("subGroupId", SearchCriteria.Func.NATIVE, sb.entity().getSubGroupId()); + SearchCriteria sc = sb.create(); + List configs = configDao.searchIncludingRemoved(sc, null, null, false); if (CollectionUtils.isEmpty(configs)) { return; } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java index 7f04736b60a5..373ebfafb87f 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java @@ -100,13 +100,10 @@ public String getColumnType(Connection conn, String tableName, String columnName } public void addColumn(Connection conn, String tableName, String columnName, String columnDefinition) { - System.out.println("------------------------" + String.format("ALTER TABLE %s ADD COLUMN %s %s", tableName, columnName, columnDefinition)); try (PreparedStatement pstmt = conn.prepareStatement(String.format("ALTER TABLE %s ADD COLUMN %s %s", tableName, columnName, columnDefinition));){ pstmt.executeUpdate(); - System.out.println("column added------------------------" + String.format("ALTER TABLE %s ADD COLUMN %s %s", tableName, columnName, columnDefinition)); logger.debug("Column {} is added successfully from the table {}", columnName, tableName); } catch (SQLException e) { - System.out.println("column not added------------------------" + e.getMessage()); logger.warn("Unable to add column {} to table {} due to exception", columnName, tableName, e); } } @@ -114,10 +111,8 @@ public void addColumn(Connection conn, String tableName, String columnName, Stri public void changeColumn(Connection conn, String tableName, String oldColumnName, String newColumnName, String columnDefinition) { try (PreparedStatement pstmt = conn.prepareStatement(String.format("ALTER TABLE %s CHANGE COLUMN %s %s %s", tableName, oldColumnName, newColumnName, columnDefinition));){ pstmt.executeUpdate(); - System.out.println("column changed------------------------" + String.format("ALTER TABLE %s CHANGE COLUMN %s %s %s", tableName, oldColumnName, newColumnName, columnDefinition)); logger.debug("Column {} is changed successfully to {} from the table {}", oldColumnName, newColumnName, tableName); } catch (SQLException e) { - System.out.println("column not changed------------------------" + e.getMessage()); logger.warn("Unable to add column {} to {} from the table {} due to exception", oldColumnName, newColumnName, tableName, e); } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41720to41800.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41720to41800.java index c8311a87a985..6a90396deb0b 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41720to41800.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41720to41800.java @@ -711,6 +711,6 @@ private void correctGuestOsIdsInHypervisorMapping(final Connection conn) { } private void updateConfigurationGroups() { -// configGroupsAggregator.updateConfigurationGroups(); + configGroupsAggregator.updateConfigurationGroups(); } } From 16add725658db8e0c534057f24e9c79c1baaaef3 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Fri, 31 Jan 2025 14:30:38 +0530 Subject: [PATCH 07/21] UT for upgrade path files --- .../ConfigurationGroupsAggregatorTest.java | 72 +++++++++++++++++++ .../upgrade/dao/DatabaseAccessObjectTest.java | 53 ++++++++++++++ .../cloud/upgrade/dao/DbUpgradeUtilsTest.java | 29 ++++++++ 3 files changed, 154 insertions(+) create mode 100644 engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java diff --git a/engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java b/engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java new file mode 100644 index 000000000000..5d09b7c2e54d --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java @@ -0,0 +1,72 @@ +package com.cloud.upgrade; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.when; + +import java.util.Collections; + +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.dao.ConfigurationGroupDao; +import org.apache.cloudstack.framework.config.dao.ConfigurationSubGroupDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationSubGroupVO; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class ConfigurationGroupsAggregatorTest { + @InjectMocks + private ConfigurationGroupsAggregator configurationGroupsAggregator = new ConfigurationGroupsAggregator(); + + @Mock + private ConfigurationDao configDao; + + @Mock + private ConfigurationGroupDao configGroupDao; + + @Mock + private ConfigurationSubGroupDao configSubGroupDao; + + @Mock + private Logger logger; + + @Test + public void testUpdateConfigurationGroups() { + ConfigurationVO config = new ConfigurationVO("Advanced", "DEFAULT", "management-server", + "test.config.name", null, "description"); + config.setGroupId(1L); + config.setSubGroupId(1L); + + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + when(configDao.createSearchBuilder()).thenReturn(sb); + Mockito.when(sb.select("name", SearchCriteria.Func.NATIVE, "test.config.name")).thenReturn(sb); + Mockito.when(sb.select("groupId", SearchCriteria.Func.NATIVE, 1L)).thenReturn(sb); + Mockito.when(sb.select("subGroupId", SearchCriteria.Func.NATIVE, 1L)).thenReturn(sb); + Mockito.when(sb.entity()).thenReturn(config); + when(configDao.searchIncludingRemoved(any(), isNull(), isNull(), eq(false))).thenReturn(Collections.singletonList(config)); + + ConfigurationSubGroupVO configSubGroup = Mockito.mock(ConfigurationSubGroupVO.class); + when(configSubGroupDao.findByName("name")).thenReturn(configSubGroup); + Mockito.when(configSubGroup.getId()).thenReturn(10L); + Mockito.when(configSubGroup.getGroupId()).thenReturn(5L); + + configurationGroupsAggregator.updateConfigurationGroups(); + + Assert.assertEquals(Long.valueOf(5), config.getGroupId()); + Assert.assertEquals(Long.valueOf(10), config.getSubGroupId()); + Mockito.verify(configDao, Mockito.times(1)).persist(config); + Mockito.verify(logger, Mockito.times(1)).debug("Updating configuration groups"); + Mockito.verify(logger, Mockito.times(1)).debug("Successfully updated configuration groups."); + } +} diff --git a/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java b/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java index 4c07abda9389..0c5a99ca05f8 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java @@ -511,4 +511,57 @@ public void testDropColumnWhenexecuteUpdateResultsInException() throws Exception verify(loggerMock, times(1)).warn(anyString(), eq(sqlException)); } + @Test + public void testGetColumnType() throws Exception { + when(connectionMock.prepareStatement(contains("DESCRIBE"))).thenReturn(preparedStatementMock); + when(preparedStatementMock.executeQuery()).thenReturn(resultSetMock); + when(resultSetMock.next()).thenReturn(true); + when(resultSetMock.getString("Type")).thenReturn("type"); + + Connection conn = connectionMock; + String tableName = "tableName"; + String columnName = "columnName"; + + Assert.assertEquals("type", dao.getColumnType(conn, tableName, columnName)); + + verify(connectionMock, times(1)).prepareStatement(anyString()); + verify(preparedStatementMock, times(1)).executeQuery(); + verify(preparedStatementMock, times(1)).close(); + verify(loggerMock, times(0)).debug(anyString()); + } + + @Test + public void testAddColumn() throws Exception { + when(connectionMock.prepareStatement(contains("ADD COLUMN"))).thenReturn(preparedStatementMock); + when(preparedStatementMock.executeUpdate()).thenReturn(1); + + Connection conn = connectionMock; + String tableName = "tableName"; + String columnName = "columnName"; + String columnType = "columnType"; + + dao.addColumn(conn, tableName, columnName, columnType); + + verify(connectionMock, times(1)).prepareStatement(anyString()); + verify(preparedStatementMock, times(1)).executeUpdate(); + verify(preparedStatementMock, times(1)).close(); + } + + @Test + public void testChangeColumn() throws Exception { + when(connectionMock.prepareStatement(contains("CHANGE COLUMN"))).thenReturn(preparedStatementMock); + when(preparedStatementMock.executeUpdate()).thenReturn(1); + + Connection conn = connectionMock; + String tableName = "tableName"; + String columnName = "columnName"; + String newColumnName = "columnName2"; + String columnDefinition = "columnDefinition"; + + dao.changeColumn(conn, tableName, columnName, newColumnName, columnDefinition); + + verify(connectionMock, times(1)).prepareStatement(anyString()); + verify(preparedStatementMock, times(1)).executeUpdate(); + verify(preparedStatementMock, times(1)).close(); + } } diff --git a/engine/schema/src/test/java/com/cloud/upgrade/dao/DbUpgradeUtilsTest.java b/engine/schema/src/test/java/com/cloud/upgrade/dao/DbUpgradeUtilsTest.java index 1b7754064663..d892b172c108 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/dao/DbUpgradeUtilsTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/dao/DbUpgradeUtilsTest.java @@ -159,4 +159,33 @@ public void testDropTableColumnsIfExistWhenThreeKeysAreSuppliedAnOneDoesnotExist verify(daoMock, times(1)).columnExists(conn, tableName, column3); verify(daoMock, times(1)).dropColumn(conn, tableName, column3); } + + @Test + public void testAddTableColumnIfNotExist() throws Exception { + Connection conn = connectionMock; + String tableName = "tableName"; + String columnName = "columnName"; + String columnDefinition = "columnDefinition"; + when(daoMock.columnExists(conn, tableName, columnName)).thenReturn(false); + + DbUpgradeUtils.addTableColumnIfNotExist(conn, tableName, columnName, columnDefinition); + + verify(daoMock, times(1)).columnExists(conn, tableName, columnName); + verify(daoMock, times(1)).addColumn(conn, tableName, columnName, columnDefinition); + } + + @Test + public void testChangeTableColumnIfNotExist() throws Exception { + Connection conn = connectionMock; + String tableName = "tableName"; + String oldColumnName = "oldColumnName"; + String newColumnName = "newColumnName"; + String columnDefinition = "columnDefinition"; + when(daoMock.columnExists(conn, tableName, oldColumnName)).thenReturn(true); + + DbUpgradeUtils.changeTableColumnIfNotExist(conn, tableName, oldColumnName, newColumnName, columnDefinition); + + verify(daoMock, times(1)).columnExists(conn, tableName, oldColumnName); + verify(daoMock, times(1)).changeColumn(conn, tableName, oldColumnName, newColumnName, columnDefinition); + } } From 17a9e603951705e14e6d50e9dc587846f279b29d Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Sun, 2 Feb 2025 04:52:44 +0530 Subject: [PATCH 08/21] fix pre-commit failure --- .../src/main/resources/META-INF/db/schema-42010to42100.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index 4a5a0203a15a..340791f0cdb3 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -33,4 +33,3 @@ WHERE rp.rule = 'quotaStatement' AND NOT EXISTS(SELECT 1 FROM cloud.role_permissions rp_ WHERE rp.role_id = rp_.role_id AND rp_.rule = 'quotaCreditsList'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.host', 'last_mgmt_server_id', 'bigint unsigned DEFAULT NULL COMMENT "last management server this host is connected to" AFTER `mgmt_server_id`'); - From 82fede2dc85295f5a52b8f27d5d4e8f6c9f3534a Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Sun, 2 Feb 2025 05:23:16 +0530 Subject: [PATCH 09/21] remove unused imports in CapacityManager --- .../src/main/java/com/cloud/capacity/CapacityManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java index 00a0d7f9b955..4c81c7359f25 100644 --- a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java +++ b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java @@ -17,7 +17,6 @@ package com.cloud.capacity; import java.util.List; -import java.util.Map; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; From 1666aaee6f90ee9dd26f389dce1be8e1f2731151 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Sun, 2 Feb 2025 05:27:15 +0530 Subject: [PATCH 10/21] Add license to ConfigurationGroupsAggregatorTest.java --- .../ConfigurationGroupsAggregatorTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java b/engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java index 5d09b7c2e54d..35730c76bb1e 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package com.cloud.upgrade; import static org.mockito.ArgumentMatchers.any; From 673dc5684d4898eccb8167f1208b62f471d51146 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 3 Feb 2025 10:29:25 +0530 Subject: [PATCH 11/21] remove extra logging --- .../main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java | 1 - .../org/apache/cloudstack/framework/config/ConfigKey.java | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java index eeabdb92169a..273e7ea8cdc2 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java @@ -90,7 +90,6 @@ public void updateSystemVmTemplates(Connection conn) { protected void migrateConfigurationScopeToBitmask(Connection conn) { String scopeDataType = DbUpgradeUtils.getTableColumnType(conn, "configuration", "scope"); - System.out.println("------------------------" + scopeDataType); logger.info("------------------------{}", scopeDataType); if (!"varchar(255)".equals(scopeDataType)) { return; diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java index 5d93f040ccfb..26151ab5b58e 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java @@ -338,7 +338,7 @@ protected T valueInGlobalOrAvailableParentScope(Scope scope, Long id) { return valueInScope(s.first(), s.second()); } } while (s != null); - logger.trace("Global value for config ({}}): {}", _name, _value); + logger.trace("Global value for config ({}): {}", _name, _value); return value(); } @@ -350,7 +350,7 @@ public T valueInScope(Scope scope, Long id) { if (value == null) { return valueInGlobalOrAvailableParentScope(scope, id); } - logger.trace("Scope({}) value for config ({}}): {}", scope, _name, _value); + logger.trace("Scope({}) value for config ({}): {}", scope, _name, _value); return valueOf(value); } From e9214ca166d2624d06193c20752a0ee73efebe30 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 3 Feb 2025 15:09:35 +0530 Subject: [PATCH 12/21] Move dao code from ConfigurationGroupsAggregator to ConfigurationDaoImpl --- .../cloud/upgrade/ConfigurationGroupsAggregator.java | 10 +--------- .../framework/config/dao/ConfigurationDao.java | 3 +++ .../framework/config/dao/ConfigurationDaoImpl.java | 11 +++++++++++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/ConfigurationGroupsAggregator.java b/engine/schema/src/main/java/com/cloud/upgrade/ConfigurationGroupsAggregator.java index e64658100b48..5c1a75046927 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/ConfigurationGroupsAggregator.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/ConfigurationGroupsAggregator.java @@ -32,9 +32,6 @@ import org.apache.commons.lang3.StringUtils; import com.cloud.utils.Pair; -import com.cloud.utils.db.SearchBuilder; -import com.cloud.utils.db.SearchCriteria; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -57,12 +54,7 @@ public ConfigurationGroupsAggregator() { public void updateConfigurationGroups() { LOG.debug("Updating configuration groups"); - SearchBuilder sb = configDao.createSearchBuilder(); - sb.select("name", SearchCriteria.Func.NATIVE, sb.entity().getName()); - sb.select("groupId", SearchCriteria.Func.NATIVE, sb.entity().getGroupId()); - sb.select("subGroupId", SearchCriteria.Func.NATIVE, sb.entity().getSubGroupId()); - SearchCriteria sc = sb.create(); - List configs = configDao.searchIncludingRemoved(sc, null, null, false); + List configs = configDao.searchPartialConfigurations(); if (CollectionUtils.isEmpty(configs)) { return; } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java index 88569558fc6b..c464b12571c1 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.framework.config.dao; +import java.util.List; import java.util.Map; import org.apache.cloudstack.framework.config.impl.ConfigurationVO; @@ -67,4 +68,6 @@ public interface ConfigurationDao extends GenericDao { boolean update(String name, String category, String value); void invalidateCache(); + + List searchPartialConfigurations(); } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java index 7c4a6f9a609e..5b941f8fccc6 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java @@ -43,6 +43,7 @@ public class ConfigurationDaoImpl extends GenericDaoBase InstanceSearch; final SearchBuilder NameSearch; + final SearchBuilder PartialSearch; public static final String UPDATE_CONFIGURATION_SQL = "UPDATE configuration SET value = ? WHERE name = ?"; @@ -53,6 +54,11 @@ public ConfigurationDaoImpl() { NameSearch = createSearchBuilder(); NameSearch.and("name", NameSearch.entity().getName(), SearchCriteria.Op.EQ); setRunLevel(ComponentLifecycle.RUN_LEVEL_SYSTEM_BOOTSTRAP); + + PartialSearch = createSearchBuilder(); + PartialSearch.select("name", SearchCriteria.Func.NATIVE, PartialSearch.entity().getName()); + PartialSearch.select("groupId", SearchCriteria.Func.NATIVE, PartialSearch.entity().getGroupId()); + PartialSearch.select("subGroupId", SearchCriteria.Func.NATIVE, PartialSearch.entity().getSubGroupId()); } @Override @@ -207,4 +213,9 @@ public ConfigurationVO findByName(String name) { return findOneIncludingRemovedBy(sc); } + @Override + public List searchPartialConfigurations() { + SearchCriteria sc = PartialSearch.create(); + return searchIncludingRemoved(sc, null, null, false); + } } From 2fe63ed78817309e11a4ad35492edaed6b3c25a8 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 4 Feb 2025 12:12:39 +0530 Subject: [PATCH 13/21] Fix logging message in Upgrade42010to42100 --- .../src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java | 1 - .../main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java index c34b0127188c..be073fcce770 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java @@ -70,7 +70,6 @@ public static void addTableColumnIfNotExist(Connection conn, String tableName, S public static void changeTableColumnIfNotExist(Connection conn, String tableName, String oldColumnName, String newColumnName, String columnDefinition) { if (dao.columnExists(conn, tableName, oldColumnName)) { - System.out.println("column exists------------------------" + oldColumnName); dao.changeColumn(conn, tableName, oldColumnName, newColumnName, columnDefinition); } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java index 273e7ea8cdc2..d6dc85dbb9aa 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java @@ -90,7 +90,7 @@ public void updateSystemVmTemplates(Connection conn) { protected void migrateConfigurationScopeToBitmask(Connection conn) { String scopeDataType = DbUpgradeUtils.getTableColumnType(conn, "configuration", "scope"); - logger.info("------------------------{}", scopeDataType); + logger.info("Data type of the column scope of table configuration is {}", scopeDataType); if (!"varchar(255)".equals(scopeDataType)) { return; } From 4d244328a53c45e5698da8bd9f21e660e8eaa030 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 4 Feb 2025 12:17:22 +0530 Subject: [PATCH 14/21] fix errors in ConfigurationGroupsAggregatorTest --- .../upgrade/ConfigurationGroupsAggregatorTest.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java b/engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java index 35730c76bb1e..bab36ef00cf8 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java @@ -16,9 +16,6 @@ // under the License. package com.cloud.upgrade; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.when; import java.util.Collections; @@ -37,9 +34,6 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import com.cloud.utils.db.SearchBuilder; -import com.cloud.utils.db.SearchCriteria; - @RunWith(MockitoJUnitRunner.class) public class ConfigurationGroupsAggregatorTest { @InjectMocks @@ -64,13 +58,7 @@ public void testUpdateConfigurationGroups() { config.setGroupId(1L); config.setSubGroupId(1L); - SearchBuilder sb = Mockito.mock(SearchBuilder.class); - when(configDao.createSearchBuilder()).thenReturn(sb); - Mockito.when(sb.select("name", SearchCriteria.Func.NATIVE, "test.config.name")).thenReturn(sb); - Mockito.when(sb.select("groupId", SearchCriteria.Func.NATIVE, 1L)).thenReturn(sb); - Mockito.when(sb.select("subGroupId", SearchCriteria.Func.NATIVE, 1L)).thenReturn(sb); - Mockito.when(sb.entity()).thenReturn(config); - when(configDao.searchIncludingRemoved(any(), isNull(), isNull(), eq(false))).thenReturn(Collections.singletonList(config)); + when(configDao.searchPartialConfigurations()).thenReturn(Collections.singletonList(config)); ConfigurationSubGroupVO configSubGroup = Mockito.mock(ConfigurationSubGroupVO.class); when(configSubGroupDao.findByName("name")).thenReturn(configSubGroup); From bab660f9f78ecbc5d9e89bf781e0b8411be660cd Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 4 Feb 2025 12:26:57 +0530 Subject: [PATCH 15/21] fix errors in MockConfigurationDaoImpl --- .../java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java b/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java index 1dc349c31ecc..90373724724a 100644 --- a/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java +++ b/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java @@ -17,6 +17,7 @@ package com.cloud.vpc.dao; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -117,4 +118,9 @@ public boolean update(String name, String category, String value) { @Override public void invalidateCache() { } + + @Override + public List searchPartialConfigurations() { + return List.of(); + } } From e20bc6e69b87170de7a38aed4ea225f2dddee735 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Wed, 5 Feb 2025 16:22:23 +0530 Subject: [PATCH 16/21] changed logger.debug to logger.warn in DatabaseAccessObject.java --- .../main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java index f90bd7410352..223d7a466376 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java @@ -94,7 +94,7 @@ public String getColumnType(Connection conn, String tableName, String columnName return rs.getString("Type"); } } catch (SQLException e) { - logger.debug("Type for column {} can not be retrieved in {} ignoring exception: {}", columnName, tableName, e.getMessage()); + logger.warn("Type for column {} can not be retrieved in {} ignoring exception: {}", columnName, tableName, e.getMessage()); } return null; } From 1b64fa64bdea6aadc2b5fa465133eab05e4fb4d9 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 13 Feb 2025 00:13:55 +0530 Subject: [PATCH 17/21] Use pool id to fetch value --- .../src/main/java/com/cloud/storage/StorageManagerImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 03f24dde27db..c456a66684b8 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -3010,7 +3010,7 @@ private boolean checkUsagedSpace(StoragePool pool) { long totalSize = pool.getCapacityBytes(); long usedSize = getUsedSize(pool); double usedPercentage = ((double)usedSize / (double)totalSize); - double storageUsedThreshold = CapacityManager.StorageCapacityDisableThreshold.valueIn(pool.getDataCenterId()); + double storageUsedThreshold = CapacityManager.StorageCapacityDisableThreshold.valueIn(pool.getId()); if (logger.isDebugEnabled()) { logger.debug("Checking pool {} for storage, totalSize: {}, usedBytes: {}, usedPct: {}, disable threshold: {}", pool, pool.getCapacityBytes(), pool.getUsedBytes(), usedPercentage, storageUsedThreshold); } @@ -3282,7 +3282,7 @@ protected boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTemp logger.debug("Total capacity of the pool {} is {}", poolVO, toHumanReadableSize(totalOverProvCapacity)); - double storageAllocatedThreshold = CapacityManager.StorageAllocatedCapacityDisableThreshold.valueIn(pool.getDataCenterId()); + double storageAllocatedThreshold = CapacityManager.StorageAllocatedCapacityDisableThreshold.valueIn(pool.getId()); if (logger.isDebugEnabled()) { logger.debug("Checking pool: {} for storage allocation , maxSize : {}, " + @@ -3307,7 +3307,7 @@ protected boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTemp return false; } - double storageAllocatedThresholdForResize = CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.valueIn(pool.getDataCenterId()); + double storageAllocatedThresholdForResize = CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.valueIn(pool.getId()); if (usedPercentage > storageAllocatedThresholdForResize) { logger.debug(String.format("Skipping the pool %s since its allocated percentage: %s has crossed the allocated %s: %s", pool, usedPercentage, CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.key(), storageAllocatedThresholdForResize)); From 950309d592c5af1d21c4ee9a572ff7405d49a653 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 13 Feb 2025 00:16:03 +0530 Subject: [PATCH 18/21] Add storagePool scope to volume.resize.allowed.beyond.allocation --- .../src/main/java/com/cloud/storage/StorageManager.java | 2 +- server/src/main/java/com/cloud/storage/StorageManagerImpl.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 7b31ec6a81b9..46f796b4f782 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -214,7 +214,7 @@ public interface StorageManager extends StorageService { ConfigKey AllowVolumeReSizeBeyondAllocation = new ConfigKey("Advanced", Boolean.class, "volume.resize.allowed.beyond.allocation", "false", "Determines whether volume size can exceed the pool capacity allocation disable threshold (pool.storage.allocated.capacity.disablethreshold) " + "when resize a volume upto resize capacity disable threshold (pool.storage.allocated.resize.capacity.disablethreshold)", - true, ConfigKey.Scope.Zone); + true, List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone)); ConfigKey StoragePoolHostConnectWorkers = new ConfigKey<>("Storage", Integer.class, "storage.pool.host.connect.workers", "1", diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index c456a66684b8..88e5e219e77e 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -3302,7 +3302,7 @@ protected boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTemp if (!forVolumeResize) { return false; } - if (!AllowVolumeReSizeBeyondAllocation.valueIn(pool.getDataCenterId())) { + if (!AllowVolumeReSizeBeyondAllocation.valueIn(pool.getId())) { logger.debug(String.format("Skipping the pool %s as %s is false", pool, AllowVolumeReSizeBeyondAllocation.key())); return false; } From a0aee79fbed8c6802d93d76d0bb6fb00f0021a7d Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 13 Feb 2025 00:17:22 +0530 Subject: [PATCH 19/21] Show the exact scope for which listConfigurations api was called --- .../cloudstack/api/command/admin/config/ListCfgsByCmd.java | 3 --- server/src/main/java/com/cloud/api/ApiResponseHelper.java | 2 -- 2 files changed, 5 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java index 12f3599e200d..e365d8bc2dc7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/ListCfgsByCmd.java @@ -182,9 +182,6 @@ private void setScope(ConfigurationResponse cfgResponse) { return; } cfgResponse.setObjectName("configuration"); - if (StringUtils.isNotBlank(cfgResponse.getScope())) { - return; - } if (getZoneId() != null) { cfgResponse.setScope("zone"); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 18ef73ae5388..fcc4444670cf 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -213,7 +213,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; -import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.jobs.AsyncJob; import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.cloudstack.management.ManagementServerHost; @@ -640,7 +639,6 @@ public ConfigurationResponse createConfigurationResponse(Configuration cfg) { cfgResponse.setSubGroup(configGroupAndSubGroup.second()); cfgResponse.setDescription(cfg.getDescription()); cfgResponse.setName(cfg.getName()); - cfgResponse.setScope(ConfigKey.Scope.decodeAsCsv(cfg.getScope())); if (cfg.isEncrypted()) { cfgResponse.setValue(DBEncryptionUtil.encrypt(cfg.getValue())); } else { From 05f051a59e41ada89c2eda51c39a151b4bd16430 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 13 Feb 2025 01:53:44 +0530 Subject: [PATCH 20/21] UT for Upgrade42010to42100 and ConfigDepotImpl --- .../upgrade/dao/Upgrade42010to42100Test.java | 73 +++++++++++++++++++ .../config/impl/ConfigDepotImplTest.java | 51 ++++++++++++- .../cloud/storage/StorageManagerImplTest.java | 1 - 3 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42010to42100Test.java diff --git a/engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42010to42100Test.java b/engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42010to42100Test.java new file mode 100644 index 000000000000..035790f0716a --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42010to42100Test.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade.dao; + +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.utils.db.TransactionLegacy; + +@RunWith(MockitoJUnitRunner.class) +public class Upgrade42010to42100Test { + @Spy + Upgrade42010to42100 upgrade; + + @Mock + private Connection conn; + + @Test + public void testPerformDataMigration() throws SQLException { + try (MockedStatic ignored = Mockito.mockStatic(DbUpgradeUtils.class)) { + DbUpgradeUtils dbUpgradeUtils = Mockito.mock(DbUpgradeUtils.class); + when(dbUpgradeUtils.getTableColumnType(conn, "configuration", "scope")).thenReturn("varchar(255)"); + + try (MockedStatic ignored2 = Mockito.mockStatic(TransactionLegacy.class)) { + TransactionLegacy txn = Mockito.mock(TransactionLegacy.class); + when(TransactionLegacy.currentTxn()).thenReturn(txn); + PreparedStatement pstmt = Mockito.mock(PreparedStatement.class); + String sql = "UPDATE configuration\n" + + "SET new_scope =" + + " CASE" + + " WHEN scope = 'Global' THEN 1" + + " WHEN scope = 'Zone' THEN 2" + + " WHEN scope = 'Cluster' THEN 4" + + " WHEN scope = 'StoragePool' THEN 8" + + " WHEN scope = 'ManagementServer' THEN 16" + + " WHEN scope = 'ImageStore' THEN 32" + + " WHEN scope = 'Domain' THEN 64" + + " WHEN scope = 'Account' THEN 128" + + " ELSE 0" + + " END WHERE scope IS NOT NULL;"; + when(txn.prepareAutoCloseStatement(sql)).thenReturn(pstmt); + upgrade.performDataMigration(conn); + + Mockito.verify(pstmt, Mockito.times(1)).executeUpdate(); + } + } + } +} diff --git a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java index 1d4a60880baf..ed752165aeb0 100644 --- a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java +++ b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java @@ -25,6 +25,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.ScopedConfigStorage; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.config.dao.ConfigurationSubGroupDao; import org.junit.Assert; @@ -36,6 +37,8 @@ import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; +import com.cloud.utils.Pair; + @RunWith(MockitoJUnitRunner.class) public class ConfigDepotImplTest { @@ -114,7 +117,7 @@ public void testGetConfigStringValueAfterExpiry() { } @Test - public void testPopulateConfiguration() { + public void testPopulateConfigurationNewVO() { ConfigKey StorageDisableThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ALERT, Double.class, "pool.storage.capacity.disablethreshold", "0.85", "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", true, List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone)); @@ -138,4 +141,50 @@ public ConfigKey[] getConfigKeys() { configDepotImpl._scopeLevelConfigsMap.get(ConfigKey.Scope.StoragePool).iterator().next().key()); Assert.assertEquals(0, configDepotImpl._scopeLevelConfigsMap.get(ConfigKey.Scope.Cluster).size()); } + + @Test + public void testPopulateConfiguration() { + ConfigKey StorageDisableThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ALERT, Double.class, "pool.storage.capacity.disablethreshold", "0.85", + "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", + true, List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone)); + Configurable configurable = new Configurable() { + @Override + public String getConfigComponentName() { + return "test"; + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{StorageDisableThreshold}; + } + }; + configDepotImpl.setConfigurables(List.of(configurable)); + + ConfigurationVO configurationVO = new ConfigurationVO(StorageDisableThreshold.category(), "DEFAULT", "component", + StorageDisableThreshold.key(), StorageDisableThreshold.defaultValue(), StorageDisableThreshold.description(), + StorageDisableThreshold.displayText(), StorageDisableThreshold.parent(), 1L, 10L); + Mockito.when(_configDao.findById("pool.storage.capacity.disablethreshold")).thenReturn(configurationVO); + configDepotImpl.populateConfigurations(); + + Mockito.verify(_configDao, Mockito.times(1)).persist(configurationVO); + } + + @Test + public void getParentScopeWithValidScope() { + ConfigKey.Scope scope = ConfigKey.Scope.Cluster; + ScopedConfigStorage scopedConfigStorage = Mockito.mock(ScopedConfigStorage.class); + Long id = 1L; + ConfigKey.Scope parentScope = ConfigKey.Scope.Zone; + Long parentId = 2L; + + Mockito.when(scopedConfigStorage.getScope()).thenReturn(scope); + Mockito.when(scopedConfigStorage.getParentScope(id)).thenReturn(new Pair<>(parentScope, parentId)); + + configDepotImpl.setScopedStorages(Collections.singletonList(scopedConfigStorage)); + Pair result = configDepotImpl.getParentScope(scope, id); + + Assert.assertNotNull(result); + Assert.assertEquals(parentScope, result.first()); + Assert.assertEquals(parentId, result.second()); + } } diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java index 4a28e044d9c5..999bf85907be 100644 --- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java @@ -780,7 +780,6 @@ private Long testCheckPoolforSpaceForResizeSetup(StoragePoolVO pool, Long alloca Mockito.when(pool.getId()).thenReturn(poolId); Mockito.when(pool.getCapacityBytes()).thenReturn(capacityBytes); - Mockito.when(pool.getDataCenterId()).thenReturn(zoneId); Mockito.when(storagePoolDao.findById(poolId)).thenReturn(pool); Mockito.when(pool.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); From 9f2205669d7b21191cd0ab5703c8a23ba6e8d351 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 13 Feb 2025 16:36:47 +0530 Subject: [PATCH 21/21] fix type in LinstorStorageAdaptor --- .../com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index ab8e5f4ee7b6..ba4a7b147874 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -507,7 +507,7 @@ private boolean deRefOrDeleteResource(DevelopersApi api, String rscName, String // if there is only one template-for property left for templates, the template isn't needed anymore // or if it isn't a template anyway, it will not have this Aux property - // _cs-template-for- poperties work like a ref-count. + // _cs-template-for- properties work like a ref-count. if (rd.getProps().keySet().stream() .filter(key -> key.startsWith("Aux/" + LinstorUtil.CS_TEMPLATE_FOR_PREFIX)) .count() == expectedProps) {