<feature>[kvm]: add lastSyncReason to track VM host file sync

build/deploydb.sh

@@ -42,6 +42,7 @@ mkdir -p ${flyway_sql}
 eval "rm -f ${flyway_sql}/*"
 cp ${base}/../conf/db/V0.6__schema.sql ${flyway_sql}
 cp ${base}/../conf/db/upgrade/* ${flyway_sql}
+cp ${base}/../conf/db/zsv/* ${flyway_sql}
 
 if [[ ! -n $host ]] || [[ ! -n $port ]];then
   url="jdbc:mysql://localhost:3306/zstack"

compute/src/main/java/org/zstack/compute/legacy/ComputeLegacyGlobalProperty.java

@@ -10,4 +10,11 @@ public class ComputeLegacyGlobalProperty {
      */
     @GlobalProperty(name="legacyCpuTopologyFix", defaultValue = "false")
     public static boolean cpuTopologyFix;
+    /**
+     * if enableNvRamTypeVolume = true, NvRam type volume will be created if UEFI is enabled;
+     * if enableNvRamTypeVolume = false, NvRam type volume will not be created, NvRam & TpmState will save in host
+     * (not in Primary storage);
+     */
+    @GlobalProperty(name="enable.nv.ram.type.volume", defaultValue = "false")
+    public static boolean enableNvRamTypeVolume;
 }

compute/src/main/java/org/zstack/compute/vm/CleanupVmInstanceMetadataOnPrimaryStorageGC.java

@@ -0,0 +1,106 @@
+package org.zstack.compute.vm;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.zstack.core.cloudbus.CloudBusCallBack;
+import org.zstack.core.componentloader.PluginRegistry;
+import org.zstack.core.db.Q;
+import org.zstack.core.gc.GC;
+import org.zstack.core.gc.GCCompletion;
+import org.zstack.core.gc.TimeBasedGarbageCollector;
+import org.zstack.header.host.HostVO;
+import org.zstack.header.message.MessageReply;
+import org.zstack.header.storage.primary.CleanupVmInstanceMetadataOnPrimaryStorageMsg;
+import org.zstack.header.storage.primary.PrimaryStorageConstant;
+import org.zstack.header.storage.primary.PrimaryStorageVO;
+import org.zstack.header.storage.primary.PrimaryStorageVO_;
+import org.zstack.header.vm.metadata.VmMetadataPathBuildExtensionPoint;
+import org.zstack.utils.Utils;
+import org.zstack.utils.logging.CLogger;
+
+public class CleanupVmInstanceMetadataOnPrimaryStorageGC extends TimeBasedGarbageCollector {
+    private static final CLogger logger = Utils.getLogger(CleanupVmInstanceMetadataOnPrimaryStorageGC.class);
+
+    @Autowired
+    private PluginRegistry pluginRgty;
+
+    @GC
+    public String primaryStorageUuid;
+    @GC
+    public String vmUuid;
+    @GC
+    public String rootVolumeUuid;
+    @GC
+    public String metadataPath;
+    @GC
+    public String hostUuid;
+
+    public static String getGCName(String vmUuid) {
+        return String.format("gc-cleanup-vm-metadata-%s", vmUuid);
+    }
+
+    @Override
+    protected void triggerNow(GCCompletion completion) {
+        if (!dbf.isExist(primaryStorageUuid, PrimaryStorageVO.class)) {
+            logger.debug(String.format("[MetadataCleanupGC] primary storage[uuid:%s] no longer exists, " +
+                    "cancel gc for vm[uuid:%s]", primaryStorageUuid, vmUuid));
+            completion.cancel();
+            return;
+        }
+
+        String psType = Q.New(PrimaryStorageVO.class).select(PrimaryStorageVO_.type).eq(PrimaryStorageVO_.uuid, primaryStorageUuid).findValue();
+        if (psType == null) {
+            logger.debug(String.format("[MetadataCleanupGC] primary storage[uuid:%s] type not found, " +
+                    "cancel gc for vm[uuid:%s]", primaryStorageUuid, vmUuid));
+            completion.cancel();
+            return;
+        }
+
+        VmMetadataPathBuildExtensionPoint ext = pluginRgty.getExtensionFromMap(psType, VmMetadataPathBuildExtensionPoint.class);
+        boolean requireHost = ext != null && ext.requireHostForCleanup();
+
+        // Determine effective hostUuid based on whether the PS type requires a host for cleanup.
+        String effectiveHostUuid = hostUuid;
+        if (!requireHost) {
+            effectiveHostUuid = null;
+        } else {
+            if (effectiveHostUuid == null) {
+                logger.debug(String.format("[MetadataCleanupGC] hostUuid is null and ps[uuid:%s, type:%s] " +
+                                "requires host for cleanup, cancel gc for vm[uuid:%s]",
+                        primaryStorageUuid, psType, vmUuid));
+                completion.cancel();
+                return;
+            }
+            if (!dbf.isExist(effectiveHostUuid, HostVO.class)) {
+                logger.debug(String.format("[MetadataCleanupGC] host[uuid:%s] no longer exists " +
+                                "and ps[uuid:%s, type:%s] requires host for cleanup, " +
+                                "metadata is unreachable, cancel gc for vm[uuid:%s]",
+                        effectiveHostUuid, primaryStorageUuid, psType, vmUuid));
+                completion.cancel();
+                return;
+            }
+        }
+
+        CleanupVmInstanceMetadataOnPrimaryStorageMsg msg = new CleanupVmInstanceMetadataOnPrimaryStorageMsg();
+        msg.setPrimaryStorageUuid(primaryStorageUuid);
+        msg.setVmInstanceUuid(vmUuid);
+        msg.setRootVolumeUuid(rootVolumeUuid);
+        msg.setMetadataPath(metadataPath);
+        msg.setHostUuid(effectiveHostUuid);
+
+        bus.makeTargetServiceIdByResourceUuid(msg, PrimaryStorageConstant.SERVICE_ID, primaryStorageUuid);
+        bus.send(msg, new CloudBusCallBack(completion) {
+            @Override
+            public void run(MessageReply reply) {
+                if (reply.isSuccess()) {
+                    logger.info(String.format("[MetadataCleanupGC] successfully cleaned up metadata " +
+                            "for vm[uuid:%s] on ps[uuid:%s]", vmUuid, primaryStorageUuid));
+                    completion.success();
+                } else {
+                    logger.warn(String.format("[MetadataCleanupGC] failed to clean up metadata " +
+                            "for vm[uuid:%s] on ps[uuid:%s]: %s", vmUuid, primaryStorageUuid, reply.getError()));
+                    completion.fail(reply.getError());
+                }
+            }
+        });
+    }
+}

compute/src/main/java/org/zstack/compute/vm/InstantiateVmFromNewCreatedStruct.java

@@ -3,6 +3,7 @@
 import org.zstack.header.configuration.VmCustomSpecificationStruct;
 import org.zstack.header.host.CpuArchitecture;
 import org.zstack.header.vm.*;
+import org.zstack.header.vm.devices.VmDevicesSpec;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -32,6 +33,7 @@ public class InstantiateVmFromNewCreatedStruct {
     private List<DiskAO> dataDisks;
     private List<DiskAO> deprecatedDataVolumeSpecs = new ArrayList<>();
     private VmCustomSpecificationStruct vmCustomSpecification;
+    private VmDevicesSpec devicesSpec;
 
     public List<String> getCandidatePrimaryStorageUuidsForRootVolume() {
         return candidatePrimaryStorageUuidsForRootVolume;
@@ -87,6 +89,14 @@ public void setVmCustomSpecification(VmCustomSpecificationStruct vmCustomSpecifi
         this.vmCustomSpecification = vmCustomSpecification;
     }
 
+    public VmDevicesSpec getDevicesSpec() {
+        return devicesSpec;
+    }
+
+    public void setDevicesSpec(VmDevicesSpec devicesSpec) {
+        this.devicesSpec = devicesSpec;
+    }
+
     public List<String> getRootVolumeSystemTags() {
         return rootVolumeSystemTags;
     }
@@ -167,6 +177,7 @@ public static InstantiateVmFromNewCreatedStruct fromMessage(InstantiateNewCreate
         struct.setDataDisks(msg.getDataDisks());
         struct.setDeprecatedDataVolumeSpecs(msg.getDeprecatedDataVolumeSpecs());
         struct.setVmCustomSpecification(msg.getVmCustomSpecification());
+        struct.setDevicesSpec(msg.getDevicesSpec());
         return struct;
     }
 
@@ -187,6 +198,7 @@ public static InstantiateVmFromNewCreatedStruct fromMessage(CreateVmInstanceMsg
         struct.setDataDisks(msg.getDataDisks());
         struct.setDeprecatedDataVolumeSpecs(msg.getDeprecatedDataVolumeSpecs());
         struct.setVmCustomSpecification(msg.getVmCustomSpecification());
+        struct.setDevicesSpec(msg.getDevicesSpec());
         return struct;
     }
 

compute/src/main/java/org/zstack/compute/vm/VmCascadeExtension.java

@@ -472,9 +472,9 @@ public void done(ErrorCodeList errorCodeList) {
                                 .flatMap(List::stream).map(VmCdRomInventory::getUuid)
                                 .collect(Collectors.toList());
                         dbf.removeByPrimaryKeys(cdRomUuids, VmCdRomVO.class);
-                        dbf.removeByPrimaryKeys(vminvs.stream().map(p -> p.getInventory().getUuid())
-                                        .collect(Collectors.toList()),
-                                VmInstanceVO.class);
+
+                        List<String> vmUuidList = transform(vminvs, p -> p.getInventory().getUuid());
+                        dbf.removeByPrimaryKeys(vmUuidList, VmInstanceVO.class);
                     }
 
                     completion.success();

compute/src/main/java/org/zstack/compute/vm/VmDeleteVolumeFlow.java

@@ -49,7 +49,9 @@ public void run(final FlowTrigger trigger, Map data) {
         final boolean templated = isTemplated(spec.getVmInventory().getUuid());
 
         /* data volume must be detached anyway no matter if it is going to be deleted */
-        if (spec.getVmInventory().getAllVolumes().size() > 1) {
+        boolean anyDataVolume = spec.getVmInventory().getAllVolumes().stream()
+                .anyMatch(arg -> VolumeType.Data.toString().equals(arg.getType()));
+        if (anyDataVolume) {
             detachDataVolumes(spec);
         }
 
@@ -60,7 +62,11 @@ public void run(final FlowTrigger trigger, Map data) {
             return;
         }
 
-        List<String> volumeTypes = Arrays.asList(VolumeType.Root.toString(), VolumeType.Memory.toString(), VolumeType.Cache.toString());
+        List<String> volumeTypes = Arrays.asList(
+                VolumeType.Root.toString(),
+                VolumeType.Memory.toString(),
+                VolumeType.Cache.toString()
+        );
         List<VolumeDeletionStruct> ctx = transformAndRemoveNull(spec.getVmInventory().getAllVolumes(), arg -> {
             if (VolumeType.Data.toString().equals(arg.getType()) && !deleteDataDisk && !templated) {
                 return null;

compute/src/main/java/org/zstack/compute/vm/VmExpungeMetadataFlow.java

@@ -0,0 +1,143 @@
+package org.zstack.compute.vm;
+
+import org.springframework.beans.factory.annotation.Autowire;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Configurable;
+import org.zstack.core.cloudbus.CloudBus;
+import org.zstack.core.cloudbus.CloudBusCallBack;
+import org.zstack.core.componentloader.PluginRegistry;
+import org.zstack.core.db.Q;
+import org.zstack.header.core.workflow.FlowTrigger;
+import org.zstack.header.core.workflow.NoRollbackFlow;
+import org.zstack.header.message.MessageReply;
+import org.zstack.header.storage.primary.CleanupVmInstanceMetadataOnPrimaryStorageMsg;
+import org.zstack.header.storage.primary.PrimaryStorageConstant;
+import org.zstack.header.storage.primary.PrimaryStorageVO;
+import org.zstack.header.storage.primary.PrimaryStorageVO_;
+import org.zstack.header.vm.VmInstanceConstant;
+import org.zstack.header.vm.VmInstanceSpec;
+import org.zstack.header.vm.metadata.VmMetadataPathBuildExtensionPoint;
+import org.zstack.header.volume.VolumeInventory;
+import org.zstack.utils.Utils;
+import org.zstack.utils.logging.CLogger;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE)
+public class VmExpungeMetadataFlow extends NoRollbackFlow {
+    private static final CLogger logger = Utils.getLogger(VmExpungeMetadataFlow.class);
+
+    @Autowired
+    private CloudBus bus;
+    @Autowired
+    private PluginRegistry pluginRgty;
+
+    @Override
+    public void run(FlowTrigger trigger, Map data) {
+        if (!VmGlobalConfig.VM_METADATA_ENABLED.value(Boolean.class)) {
+            trigger.next();
+            return;
+        }
+
+        final VmInstanceSpec spec = (VmInstanceSpec) data.get(VmInstanceConstant.Params.VmInstanceSpec.toString());
+        if (spec == null || spec.getVmInventory() == null) {
+            logger.warn("[MetadataExpunge] missing VmInstanceSpec or VmInventory, skip metadata cleanup");
+            trigger.next();
+            return;
+        }
+
+        final String vmUuid = spec.getVmInventory().getUuid();
+
+        VolumeInventory rootVolume = spec.getVmInventory().getRootVolume();
+        String psUuid = rootVolume != null ? rootVolume.getPrimaryStorageUuid() : null;
+        if (psUuid == null) {
+            logger.debug(String.format("[MetadataExpunge] vm[uuid:%s] root volume has no primaryStorageUuid, " +
+                    "skipping metadata cleanup", vmUuid));
+            trigger.next();
+            return;
+        }
+
+
+        String psType = Q.New(PrimaryStorageVO.class).select(PrimaryStorageVO_.type).eq(PrimaryStorageVO_.uuid, psUuid).findValue();
+        if (psType == null) {
+            logger.warn(String.format("[MetadataExpunge] primary storage[uuid:%s] not found for vm[uuid:%s], " +
+                    "skip metadata cleanup", psUuid, vmUuid));
+            trigger.next();
+            return;
+        }
+
+        VmMetadataPathBuildExtensionPoint ext = pluginRgty.getExtensionFromMap(psType, VmMetadataPathBuildExtensionPoint.class);
+        if (ext == null) {
+            logger.warn(String.format("[MetadataExpunge] no VmMetadataPathBuildExtensionPoint found for ps[uuid:%s, type:%s], " +
+                    "skip metadata cleanup", psUuid, psType));
+            trigger.next();
+            return;
+        }
+        final String metadataPath;
+        try {
+            metadataPath = ext.buildVmMetadataPath(psUuid, vmUuid);
+        } catch (Exception e) {
+            logger.warn(String.format("[MetadataExpunge] failed to build metadata path for vm[uuid:%s] on ps[uuid:%s], " +
+                    "skip metadata cleanup: %s", vmUuid, psUuid, e.getMessage()));
+            trigger.next();
+            return;
+        }
+
+        String hostUuid = null;
+        if (ext.requireHostForCleanup()) {
+            hostUuid = spec.getVmInventory().getHostUuid();
+            if (hostUuid == null) {
+                hostUuid = spec.getVmInventory().getLastHostUuid();
+            }
+
+            if (hostUuid == null) {
+                logger.warn(String.format("[MetadataExpunge] vm[uuid:%s] hostUuid is null, " +
+                                "ps[uuid:%s, type:%s] requires host for cleanup, skip without submitting GC",
+                        vmUuid, psUuid, psType));
+                trigger.next();
+                return;
+            }
+        }
+
+        String rootVolumeUuid = rootVolume.getUuid();
+        CleanupVmInstanceMetadataOnPrimaryStorageMsg cmsg = new CleanupVmInstanceMetadataOnPrimaryStorageMsg();
+        cmsg.setPrimaryStorageUuid(psUuid);
+        cmsg.setVmInstanceUuid(vmUuid);
+        cmsg.setMetadataPath(metadataPath);
+        cmsg.setRootVolumeUuid(rootVolumeUuid);
+        cmsg.setHostUuid(hostUuid);
+        final String finalPsUuid = psUuid;
+        final String finalHostUuid = hostUuid;
+
+        bus.makeTargetServiceIdByResourceUuid(cmsg, PrimaryStorageConstant.SERVICE_ID, psUuid);
+        bus.send(cmsg, new CloudBusCallBack(trigger) {
+            @Override
+            public void run(MessageReply reply) {
+                if (reply.isSuccess()) {
+                    logger.info(String.format("[MetadataExpunge] successfully deleted metadata for vm[uuid:%s] on ps[uuid:%s]",
+                            vmUuid, finalPsUuid));
+                } else {
+                    logger.warn(String.format("[MetadataExpunge] failed to delete metadata for vm[uuid:%s] on ps[uuid:%s]: %s, " +
+                            "submitting GC job for retry", vmUuid, finalPsUuid, reply.getError()));
+                    submitGC(finalPsUuid, vmUuid, rootVolumeUuid, metadataPath, finalHostUuid);
+                }
+                trigger.next();
+            }
+        });
+    }
+
+    private void submitGC(String psUuid, String vmUuid, String rootVolumeUuid, String metadataPath, String hostUuid) {
+        CleanupVmInstanceMetadataOnPrimaryStorageGC gc = new CleanupVmInstanceMetadataOnPrimaryStorageGC();
+        gc.NAME = CleanupVmInstanceMetadataOnPrimaryStorageGC.getGCName(vmUuid);
+        gc.primaryStorageUuid = psUuid;
+        gc.vmUuid = vmUuid;
+        gc.rootVolumeUuid = rootVolumeUuid;
+        gc.metadataPath = metadataPath;
+        gc.hostUuid = hostUuid;
+        long gcIntervalSec = TimeUnit.HOURS.toSeconds(VmGlobalConfig.VM_METADATA_CLEANUP_GC_INTERVAL.value(Long.class));
+        gc.deduplicateSubmit(gcIntervalSec, TimeUnit.SECONDS);
+
+        logger.info(String.format("[MetadataExpunge] submitted GC job [%s] for vm[uuid:%s] on ps[uuid:%s]", gc.NAME, vmUuid, psUuid));
+    }
+}