From fa245cc3990de12851713ba92914c1b01d7f4d10 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Mon, 27 Jan 2025 08:50:48 +0100 Subject: [PATCH] linstor: Fix using multiple primary storage with same linstor-controller It should have been always possible to use multiple primary storages, with the same linstor-controller, by just using different resource-groups with different settings. But if the same template was used on 2 different primary storages, there would be a name collision on the linstor-controller, as 2 of them would get allocated to the same name. This commit fixes this, by intelligently reusing the same template, as long as possible (if select filter properties match enough). --- plugins/storage/volume/linstor/CHANGELOG.md | 6 + .../kvm/storage/LinstorStorageAdaptor.java | 121 +++++++- .../LinstorPrimaryDataStoreDriverImpl.java | 265 +++++++++++++++--- .../storage/datastore/util/LinstorUtil.java | 115 ++++++++ 4 files changed, 458 insertions(+), 49 deletions(-) diff --git a/plugins/storage/volume/linstor/CHANGELOG.md b/plugins/storage/volume/linstor/CHANGELOG.md index 98fc8f695128..fb247eef5dfe 100644 --- a/plugins/storage/volume/linstor/CHANGELOG.md +++ b/plugins/storage/volume/linstor/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to Linstor CloudStack plugin will be documented in this file The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2025-01-27] + +### Fixed + +- Use of multiple primary storages on the same linstor controller + ## [2025-01-20] ### Fixed diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index 6a4d6c7a3490..9de2479b20bb 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -60,6 +60,11 @@ import com.linbit.linstor.api.model.VolumeDefinition; import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; @StorageAdaptorInfo(storagePoolType=Storage.StoragePoolType.Linstor) public class LinstorStorageAdaptor implements StorageAdaptor { @@ -198,10 +203,10 @@ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, Qemu final DevelopersApi api = getLinstorAPI(pool); try { - List definitionList = api.resourceDefinitionList( - Collections.singletonList(rscName), null, null, null); + ResourceDefinition resourceDefinition = LinstorUtil.findResourceDefinition( + api, rscName, lpool.getResourceGroup()); - if (definitionList.isEmpty()) { + if (resourceDefinition == null) { ResourceGroupSpawn rgSpawn = new ResourceGroupSpawn(); rgSpawn.setResourceDefinitionName(rscName); rgSpawn.addVolumeSizesItem(size / 1024); // linstor uses KiB @@ -211,22 +216,28 @@ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, Qemu handleLinstorApiAnswers(answers, "Linstor: Unable to spawn resource."); } + String foundRscName = resourceDefinition != null ? resourceDefinition.getName() : rscName; + // query linstor for the device path List resources = api.viewResources( Collections.emptyList(), - Collections.singletonList(rscName), + Collections.singletonList(foundRscName), Collections.emptyList(), null, null, null); - makeResourceAvailable(api, rscName, false); + makeResourceAvailable(api, foundRscName, false); if (!resources.isEmpty() && !resources.get(0).getVolumes().isEmpty()) { final String devPath = resources.get(0).getVolumes().get(0).getDevicePath(); s_logger.info("Linstor: Created drbd device: " + devPath); final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath, name, pool); kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW); + long allocatedKib = resources.get(0).getVolumes().get(0).getAllocatedSizeKib() != null ? + resources.get(0).getVolumes().get(0).getAllocatedSizeKib() : 0; + kvmDisk.setSize(allocatedKib >= 0 ? allocatedKib * 1024 : 0); + kvmDisk.setVirtualSize(size); return kvmDisk; } else { s_logger.error("Linstor: viewResources didn't return resources or volumes."); @@ -470,21 +481,56 @@ public boolean disconnectPhysicalDiskByPath(String localPath) return false; } + /** + * Decrements the aux property key for template resource and deletes or just deletes if not template resource. + * @param api + * @param rscName + * @param rscGrpName + * @return + * @throws ApiException + */ + private boolean deRefOrDeleteResource(DevelopersApi api, String rscName, String rscGrpName) throws ApiException { + boolean deleted = false; + List existingRDs = LinstorUtil.getRDListStartingWith(api, rscName); + for (ResourceDefinition rd : existingRDs) { + int expectedProps = 0; // if it is a non template resource, we don't expect any _cs-template-for- prop + String propKey = LinstorUtil.getTemplateForAuxPropKey(rscGrpName); + if (rd.getProps().containsKey(propKey)) { + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + rdm.deleteProps(Collections.singletonList(propKey)); + api.resourceDefinitionModify(rd.getName(), rdm); + expectedProps = 1; + } + + // if there is only one template-for property left for templates, the template isn't needed anymore + // or if it isn't a template anyway, it will not have this Aux property + // _cs-template-for- poperties work like a ref-count. + if (rd.getProps().keySet().stream() + .filter(key -> key.startsWith("Aux/" + LinstorUtil.CS_TEMPLATE_FOR_PREFIX)) + .count() == expectedProps) { + ApiCallRcList answers = api.resourceDefinitionDelete(rd.getName()); + checkLinstorAnswersThrow(answers); + deleted = true; + } + } + return deleted; + } + @Override public boolean deletePhysicalDisk(String name, KVMStoragePool pool, Storage.ImageFormat format) { s_logger.debug("Linstor: deletePhysicalDisk " + name); final DevelopersApi api = getLinstorAPI(pool); + final String rscName = getLinstorRscName(name); + final LinstorStoragePool linstorPool = (LinstorStoragePool) pool; + String rscGrpName = linstorPool.getResourceGroup(); try { - final String rscName = getLinstorRscName(name); - s_logger.debug("Linstor: delete resource definition " + rscName); - ApiCallRcList answers = api.resourceDefinitionDelete(rscName); - handleLinstorApiAnswers(answers, "Linstor: Unable to delete resource definition " + rscName); + return deRefOrDeleteResource(api, rscName, rscGrpName); } catch (ApiException apiEx) { + s_logger.error("Linstor: ApiEx - " + apiEx.getMessage()); throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); } - return true; } @Override @@ -558,6 +604,56 @@ private boolean resourceSupportZeroBlocks(KVMStoragePool destPool, String resNam return false; } + /** + * Checks if the given disk is the SystemVM template, by checking its properties file in the same directory. + * The initial systemvm template resource isn't created on the management server, but + * we now need to know if the systemvm template is used, while copying. + * @param disk + * @return True if it is the systemvm template disk, else false. + */ + private static boolean isSystemTemplate(KVMPhysicalDisk disk) { + Path diskPath = Paths.get(disk.getPath()); + Path propFile = diskPath.getParent().resolve("template.properties"); + if (Files.exists(propFile)) { + java.util.Properties templateProps = new java.util.Properties(); + try { + templateProps.load(new FileInputStream(propFile.toFile())); + String desc = templateProps.getProperty("description"); + if (desc.startsWith("SystemVM Template")) { + return true; + } + } catch (IOException e) { + return false; + } + } + return false; + } + + /** + * Conditionally sets the correct aux properties for templates or basic resources. + * @param api + * @param srcDisk + * @param destPool + * @param name + */ + private void setRscDfnAuxProperties( + DevelopersApi api, KVMPhysicalDisk srcDisk, KVMStoragePool destPool, String name) { + // if it is the initial systemvm disk copy, we need to apply the _cs-template-for property. + if (isSystemTemplate(srcDisk)) { + applyAuxProps(api, name, "SystemVM Template", null); + LinstorStoragePool linPool = (LinstorStoragePool) destPool; + final String rscName = getLinstorRscName(name); + try { + LinstorUtil.setAuxTemplateForProperty(api, rscName, linPool.getResourceGroup()); + } catch (ApiException apiExc) { + s_logger.error(String.format("Error setting aux template for property for %s", rscName)); + logLinstorAnswers(apiExc.getApiCallRcList()); + } + } else { + applyAuxProps(api, name, srcDisk.getDispName(), srcDisk.getVmName()); + } + } + @Override public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPools, int timeout, byte[] srcPassphrase, byte[] destPassphrase, Storage.ProvisioningType provisioningType) { @@ -571,15 +667,14 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt name, QemuImg.PhysicalDiskFormat.RAW, provisioningType, disk.getVirtualSize(), null); final DevelopersApi api = getLinstorAPI(destPools); - applyAuxProps(api, name, disk.getDispName(), disk.getVmName()); + setRscDfnAuxProperties(api, disk, destPools, name); s_logger.debug(String.format("Linstor.copyPhysicalDisk: dstPath: %s", dstDisk.getPath())); final QemuImgFile destFile = new QemuImgFile(dstDisk.getPath()); destFile.setFormat(dstDisk.getFormat()); destFile.setSize(disk.getVirtualSize()); - boolean zeroedDevice = resourceSupportZeroBlocks(destPools, LinstorUtil.RSC_PREFIX + name); - + boolean zeroedDevice = resourceSupportZeroBlocks(destPools, getLinstorRscName(name)); try { final QemuImg qemu = new QemuImg(timeout, zeroedDevice, true); qemu.convert(srcFile, destFile); diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 4132fbd278a6..22cb4eb8911f 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -28,6 +28,8 @@ import com.linbit.linstor.api.model.ResourceDefinitionCloneRequest; import com.linbit.linstor.api.model.ResourceDefinitionCloneStarted; import com.linbit.linstor.api.model.ResourceDefinitionCreate; + +import com.linbit.linstor.api.model.ResourceDefinitionModify; import com.linbit.linstor.api.model.ResourceGroup; import com.linbit.linstor.api.model.ResourceGroupSpawn; import com.linbit.linstor.api.model.ResourceMakeAvailable; @@ -71,6 +73,7 @@ import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.Volume; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.VolumeVO; @@ -90,6 +93,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; @@ -98,6 +102,7 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.RemoteHostEndPoint; import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -435,15 +440,27 @@ public List getEncryptedLayerList(DevelopersApi api, String resourceG } } - private String createResourceBase( - String rscName, long sizeInBytes, String volName, String vmName, - @Nullable Long passPhraseId, @Nullable byte[] passPhrase, DevelopersApi api, String rscGrp) { + /** + * Spawns a new Linstor resource with the given arguments. + * @param api + * @param newRscName + * @param sizeInBytes + * @param isTemplate + * @param rscGrpName + * @param volName + * @param vmName + * @throws ApiException + */ + private void spawnResource( + DevelopersApi api, String newRscName, long sizeInBytes, boolean isTemplate, String rscGrpName, + String volName, String vmName, @Nullable Long passPhraseId, @Nullable byte[] passPhrase) throws ApiException + { ResourceGroupSpawn rscGrpSpawn = new ResourceGroupSpawn(); - rscGrpSpawn.setResourceDefinitionName(rscName); + rscGrpSpawn.setResourceDefinitionName(newRscName); rscGrpSpawn.addVolumeSizesItem(sizeInBytes / 1024); if (passPhraseId != null) { AutoSelectFilter asf = new AutoSelectFilter(); - List luksLayers = getEncryptedLayerList(api, rscGrp); + List luksLayers = getEncryptedLayerList(api, rscGrpName); asf.setLayerStack(luksLayers.stream().map(LayerType::toString).collect(Collectors.toList())); rscGrpSpawn.setSelectFilter(asf); if (passPhrase != null) { @@ -452,16 +469,103 @@ private String createResourceBase( } } - try - { - s_logger.info("Linstor: Spawn resource " + rscName); - ApiCallRcList answers = api.resourceGroupSpawn(rscGrp, rscGrpSpawn); - checkLinstorAnswersThrow(answers); + if (isTemplate) { + Properties props = new Properties(); + props.put(LinstorUtil.getTemplateForAuxPropKey(rscGrpName), "true"); + rscGrpSpawn.setResourceDefinitionProps(props); + } - answers = LinstorUtil.applyAuxProps(api, rscName, volName, vmName); - checkLinstorAnswersThrow(answers); + s_logger.info("Linstor: Spawn resource " + newRscName); + ApiCallRcList answers = api.resourceGroupSpawn(rscGrpName, rscGrpSpawn); + checkLinstorAnswersThrow(answers); + + answers = LinstorUtil.applyAuxProps(api, newRscName, volName, vmName); + checkLinstorAnswersThrow(answers); + } - return LinstorUtil.getDevicePath(api, rscName); + /** + * Condition if a template resource can be shared with the given resource group. + * @param tgtRscGrp + * @param tgtLayerStack + * @param rg + * @return True if the template resource can be shared, else false. + */ + private boolean canShareTemplateForResourceGroup( + ResourceGroup tgtRscGrp, List tgtLayerStack, ResourceGroup rg) { + List rgLayerStack = rg.getSelectFilter() != null ? + rg.getSelectFilter().getLayerStack() : null; + return Objects.equals(tgtLayerStack, rgLayerStack) && + Objects.equals(tgtRscGrp.getSelectFilter().getStoragePoolList(), + rg.getSelectFilter().getStoragePoolList()); + } + + /** + * Searches for a shareable template for this rscGrpName and sets the aux template property. + * @param api + * @param rscName + * @param rscGrpName + * @param existingRDs + * @return + * @throws ApiException + */ + private boolean foundShareableTemplate( + DevelopersApi api, String rscName, String rscGrpName, + List> existingRDs) throws ApiException { + if (!existingRDs.isEmpty()) { + ResourceGroup tgtRscGrp = api.resourceGroupList( + Collections.singletonList(rscGrpName), null, null, null).get(0); + List tgtLayerStack = tgtRscGrp.getSelectFilter() != null ? + tgtRscGrp.getSelectFilter().getLayerStack() : null; + + // check if there is already a template copy, that we could reuse + // this means if select filters are similar enough to allow cloning from + for (Pair rdPair : existingRDs) { + ResourceGroup rg = rdPair.second(); + if (canShareTemplateForResourceGroup(tgtRscGrp, tgtLayerStack, rg)) { + LinstorUtil.setAuxTemplateForProperty(api, rscName, rscGrpName); + return true; + } + } + } + return false; + } + + /** + * Creates a new Linstor resource. + * @param rscName + * @param sizeInBytes + * @param volName + * @param vmName + * @param api + * @param rscGrp + * @param poolId + * @param isTemplate indicates if the resource is a template + * @return true if a new resource was created, false if it already existed or was reused. + */ + private boolean createResourceBase( + String rscName, long sizeInBytes, String volName, String vmName, + @Nullable Long passPhraseId, @Nullable byte[] passPhrase, DevelopersApi api, + String rscGrp, long poolId, boolean isTemplate) + { + try + { + s_logger.debug(String.format("createRscBase: %s :: %s :: %b", rscName, rscGrp, isTemplate)); + List> existingRDs = LinstorUtil.getRDAndRGListStartingWith(api, rscName); + + String fullRscName = String.format("%s-%d", rscName, poolId); + boolean alreadyCreated = existingRDs.stream() + .anyMatch(p -> p.first().getName().equalsIgnoreCase(fullRscName)) || + existingRDs.stream().anyMatch(p -> p.first().getProps().containsKey(LinstorUtil.getTemplateForAuxPropKey(rscGrp))); + if (!alreadyCreated) { + boolean createNewRsc = !foundShareableTemplate(api, rscName, rscGrp, existingRDs); + if (createNewRsc) { + String newRscName = existingRDs.isEmpty() ? rscName : fullRscName; + spawnResource(api, newRscName, sizeInBytes, isTemplate, rscGrp, + volName, vmName, passPhraseId, passPhrase); + } + return createNewRsc; + } + return false; } catch (ApiException apiEx) { s_logger.error("Linstor: ApiEx - " + apiEx.getMessage()); @@ -474,9 +578,9 @@ private String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO) { final String rscGrp = getRscGrp(storagePoolVO); final String rscName = LinstorUtil.RSC_PREFIX + vol.getUuid(); - String deviceName = createResourceBase( + createResourceBase( rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(), vol.getPassphraseId(), vol.getPassphrase(), - linstorApi, rscGrp); + linstorApi, rscGrp, storagePoolVO.getId(), false); try { @@ -503,17 +607,72 @@ private void resizeResource(DevelopersApi api, String resourceName, long sizeByt } } + /** + * Update resource-definitions resource-group to the correct one if it isn't already the intended. + * @param api Linstor api + * @param rscName resource name to check the resource group + * @param tgtRscGrp resource group name to set + * @throws ApiException exception if any api error occurred + */ + private void updateRscGrpIfNecessary(DevelopersApi api, String rscName, String tgtRscGrp) throws ApiException { + List rscDfns = api.resourceDefinitionList( + Collections.singletonList(rscName), null, null, null); + if (rscDfns != null && !rscDfns.isEmpty()) { + ResourceDefinition rscDfn = rscDfns.get(0); + + if (!rscDfn.getResourceGroupName().equalsIgnoreCase(tgtRscGrp)) { + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + rdm.setResourceGroup(tgtRscGrp); + ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm); + + if (answers.hasError()) { + String bestError = LinstorUtil.getBestErrorMessage(answers); + s_logger.error(String.format("Update resource group on %s error: %s", rscName, bestError)); + throw new CloudRuntimeException(bestError); + } else { + s_logger.info(String.format("Successfully changed resource group to %s on %s", tgtRscGrp, rscName)); + } + } + } + } + + /** + * If a resource is cloned, all properties are cloned too, but the _cs-template-for properties, + * should only stay on the template resource, so delete them in this method. + * @param api + * @param rscName + * @throws ApiException + */ + private void deleteTemplateForProps( + DevelopersApi api, String rscName) throws ApiException { + List rdList = api.resourceDefinitionList( + Collections.singletonList(rscName), null, null, null); + + if (CollectionUtils.isNotEmpty(rdList)) { + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + List deleteProps = rdList.get(0).getProps().keySet().stream() + .filter(key -> key.startsWith("Aux/" + LinstorUtil.CS_TEMPLATE_FOR_PREFIX)) + .collect(Collectors.toList()); + rdm.deleteProps(deleteProps); + ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm); + checkLinstorAnswers(answers); + } + } + private String cloneResource(long csCloneId, VolumeInfo volumeInfo, StoragePoolVO storagePoolVO) { // get the cached template on this storage VMTemplateStoragePoolVO tmplPoolRef = _vmTemplatePoolDao.findByPoolTemplate( storagePoolVO.getId(), csCloneId, null); if (tmplPoolRef != null) { - final String cloneRes = LinstorUtil.RSC_PREFIX + tmplPoolRef.getLocalDownloadPath(); + final String templateRscName = LinstorUtil.RSC_PREFIX + tmplPoolRef.getLocalDownloadPath(); final String rscName = LinstorUtil.RSC_PREFIX + volumeInfo.getUuid(); final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress()); try { + ResourceDefinition templateRD = LinstorUtil.findResourceDefinition( + linstorApi, templateRscName, getRscGrp(storagePoolVO)); + final String cloneRes = templateRD != null ? templateRD.getName() : templateRscName; s_logger.info("Clone resource definition " + cloneRes + " to " + rscName); ResourceDefinitionCloneRequest cloneRequest = new ResourceDefinitionCloneRequest(); cloneRequest.setName(rscName); @@ -540,6 +699,9 @@ private String cloneResource(long csCloneId, VolumeInfo volumeInfo, StoragePoolV resizeResource(linstorApi, rscName, volumeInfo.getSize()); } + updateRscGrpIfNecessary(linstorApi, rscName, getRscGrp(storagePoolVO)); + + deleteTemplateForProps(linstorApi, rscName); LinstorUtil.applyAuxProps(linstorApi, rscName, volumeInfo.getName(), volumeInfo.getAttachedVmName()); applyQoSSettings(storagePoolVO, linstorApi, rscName, volumeInfo.getMaxIops()); @@ -967,12 +1129,37 @@ private String restoreResourceFromSnapshot( return LinstorUtil.getDevicePath(api, restoredName); } + /** + * Updates the template_spool_ref DB entry to indicate that this template was fully downloaded and is ready. + * @param templateId + * @param destTemplateInfoUuid + * @param destDataStoreId + * @param templateSize + */ + private void updateTemplateSpoolRef( + long templateId, String destTemplateInfoUuid, long destDataStoreId, long templateSize) { + VMTemplateStoragePoolVO destVolumeTemplateStoragePoolVO = _vmTemplatePoolDao.findByPoolTemplate( + destDataStoreId, templateId, null); + if (destVolumeTemplateStoragePoolVO == null) { + throw new CloudRuntimeException( + String.format("Unable to find template_spool_ref entry for pool_id %d and template_id %d", + destDataStoreId, templateId)); + } + destVolumeTemplateStoragePoolVO.setDownloadPercent(100); + destVolumeTemplateStoragePoolVO.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + destVolumeTemplateStoragePoolVO.setState(ObjectInDataStoreStateMachine.State.Ready); + destVolumeTemplateStoragePoolVO.setTemplateSize(templateSize); + destVolumeTemplateStoragePoolVO.setLocalDownloadPath(destTemplateInfoUuid); + destVolumeTemplateStoragePoolVO.setInstallPath(destTemplateInfoUuid); + _vmTemplatePoolDao.persist(destVolumeTemplateStoragePoolVO); + } + private Answer copyTemplate(DataObject srcData, DataObject dstData) { TemplateInfo tInfo = (TemplateInfo) dstData; final StoragePoolVO pool = _storagePoolDao.findById(dstData.getDataStore().getId()); final DevelopersApi api = LinstorUtil.getLinstorAPI(pool.getHostAddress()); final String rscName = LinstorUtil.RSC_PREFIX + dstData.getUuid(); - createResourceBase( + boolean newCreated = createResourceBase( LinstorUtil.RSC_PREFIX + dstData.getUuid(), tInfo.getSize(), tInfo.getName(), @@ -980,30 +1167,36 @@ private Answer copyTemplate(DataObject srcData, DataObject dstData) { null, null, api, - getRscGrp(pool)); + getRscGrp(pool), + pool.getId(), + true); - int nMaxExecutionMinutes = NumbersUtil.parseInt( - _configDao.getValue(Config.SecStorageCmdExecutionTimeMax.key()), 30); - CopyCommand cmd = new CopyCommand( - srcData.getTO(), - dstData.getTO(), - nMaxExecutionMinutes * 60 * 1000, - VirtualMachineManager.ExecuteInSequence.value()); Answer answer; + if (newCreated) { + int nMaxExecutionMinutes = NumbersUtil.parseInt( + _configDao.getValue(Config.SecStorageCmdExecutionTimeMax.key()), 30); + CopyCommand cmd = new CopyCommand( + srcData.getTO(), + dstData.getTO(), + nMaxExecutionMinutes * 60 * 1000, + VirtualMachineManager.ExecuteInSequence.value()); - try { - Optional optEP = getLinstorEP(api, rscName); - if (optEP.isPresent()) { - answer = optEP.get().sendMessage(cmd); - } - else { - answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint."); + try { + Optional optEP = getLinstorEP(api, rscName); + if (optEP.isPresent()) { + answer = optEP.get().sendMessage(cmd); + } else { + answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint."); + deleteResourceDefinition(pool, rscName); + } + } catch (ApiException exc) { + s_logger.error("copy template failed: ", exc); deleteResourceDefinition(pool, rscName); + throw new CloudRuntimeException(exc.getBestMessage()); } - } catch (ApiException exc) { - s_logger.error("copy template failed: ", exc); - deleteResourceDefinition(pool, rscName); - throw new CloudRuntimeException(exc.getBestMessage()); + } else { + updateTemplateSpoolRef(dstData.getId(), tInfo.getUuid(), dstData.getDataStore().getId(), srcData.getSize()); + answer = new Answer(new CopyCmdAnswer(dstData.getTO())); } return answer; } diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 2dbfdea6d119..e252753502c8 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -26,6 +26,7 @@ import com.linbit.linstor.api.model.Properties; import com.linbit.linstor.api.model.ProviderKind; import com.linbit.linstor.api.model.Resource; +import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceDefinitionModify; import com.linbit.linstor.api.model.ResourceGroup; import com.linbit.linstor.api.model.ResourceWithVolumes; @@ -37,8 +38,11 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.log4j.Logger; @@ -48,6 +52,7 @@ public class LinstorUtil { public final static String PROVIDER_NAME = "Linstor"; public static final String RSC_PREFIX = "cs-"; public static final String RSC_GROUP = "resourceGroup"; + public static final String CS_TEMPLATE_FOR_PREFIX = "_cs-template-for-"; public static final String TEMP_VOLUME_ID = "tempVolumeId"; @@ -287,4 +292,114 @@ public static ApiCallRcList applyAuxProps(DevelopersApi api, String rscName, Str } return answers; } + + /** + * Returns all resource definitions that start with the given `startWith` name. + * @param api + * @param startWith startWith String + * @return a List with all ResourceDefinition starting with `startWith` + * @throws ApiException + */ + public static List getRDListStartingWith(DevelopersApi api, String startWith) + throws ApiException + { + List rscDfns = api.resourceDefinitionList(null, null, null, null); + + return rscDfns.stream() + .filter(rscDfn -> rscDfn.getName().toLowerCase().startsWith(startWith.toLowerCase())) + .collect(Collectors.toList()); + } + + /** + * Returns a pair list of resource-definitions with ther 1:1 mapped resource-group objects that start with the + * resource name `startWith` + * @param api + * @param startWith + * @return + * @throws ApiException + */ + public static List> getRDAndRGListStartingWith(DevelopersApi api, String startWith) + throws ApiException + { + List foundRDs = getRDListStartingWith(api, startWith); + + List rscGrpStrings = foundRDs.stream() + .map(ResourceDefinition::getResourceGroupName) + .collect(Collectors.toList()); + + Map rscGrps = api.resourceGroupList(rscGrpStrings, null, null, null).stream() + .collect(Collectors.toMap(ResourceGroup::getName, rscGrp -> rscGrp)); + + return foundRDs.stream() + .map(rd -> new Pair<>(rd, rscGrps.get(rd.getResourceGroupName()))) + .collect(Collectors.toList()); + } + + /** + * The full name of template-for aux property key. + * @param rscGrpName + * @return + */ + public static String getTemplateForAuxPropKey(String rscGrpName) { + return String.format("Aux/%s%s", CS_TEMPLATE_FOR_PREFIX, rscGrpName); + } + + /** + * Template resource should have a _cs-template-for-... property, that indicates to which resource-group + * this template belongs, it works like a refcount to keep it alive if there are still such properties on the + * template resource. That methods set the correct property on the given resource. + * @param api + * @param rscName Resource name to set the property. + * @param rscGrpName Resource group this template should belong too. + * @throws ApiException + */ + public static void setAuxTemplateForProperty(DevelopersApi api, String rscName, String rscGrpName) + throws ApiException + { + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + Properties props = new Properties(); + String propKey = LinstorUtil.getTemplateForAuxPropKey(rscGrpName); + props.put(propKey, "true"); + rdm.setOverrideProps(props); + ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm); + + if (answers.hasError()) { + String bestError = LinstorUtil.getBestErrorMessage(answers); + s_logger.error(String.format("Set %s on %s error: %s", propKey, rscName, bestError)); + throw new CloudRuntimeException(bestError); + } else { + s_logger.info(String.format("Set %s property on %s", propKey, rscName)); + } + } + + /** + * Find the correct resource definition to clone from. + * There could be multiple resource definitions for the same template, with the same prefix. + * This method searches for which resource group the resource definition was intended and returns that. + * If no exact resource definition could be found, we return the first with a similar name as a fallback. + * If there is not even one with the correct prefix, we return null. + * @param api + * @param rscName + * @param rscGrpName + * @return The resource-definition to clone from, if no template and no match, return null. + * @throws ApiException + */ + public static ResourceDefinition findResourceDefinition(DevelopersApi api, String rscName, String rscGrpName) + throws ApiException { + List rscDfns = api.resourceDefinitionList(null, null, null, null); + + List rdsStartingWith = rscDfns.stream() + .filter(rscDfn -> rscDfn.getName().toLowerCase().startsWith(rscName.toLowerCase())) + .collect(Collectors.toList()); + + if (rdsStartingWith.isEmpty()) { + return null; + } + + Optional rd = rdsStartingWith.stream() + .filter(rscDfn -> rscDfn.getProps().containsKey(LinstorUtil.getTemplateForAuxPropKey(rscGrpName))) + .findFirst(); + + return rd.orElseGet(() -> rdsStartingWith.get(0)); + } }