Merge branch '4.20' into ssvm-mount

Author: abh1sar

api/src/main/java/com/cloud/network/NetworkModel.java

@@ -125,6 +125,10 @@ public interface NetworkModel {
      */
     String getNextAvailableMacAddressInNetwork(long networkConfigurationId) throws InsufficientAddressCapacityException;
 
+    String getUniqueMacAddress(long macAddress, long networkId, long datacenterId) throws InsufficientAddressCapacityException;
+
+    boolean isMACUnique(String mac, long networkId);
+
     PublicIpAddress getPublicIpAddress(long ipAddressId);
 
     List<? extends Vlan> listPodVlans(long podId);
@@ -362,4 +366,8 @@ List<String[]> generateVmData(String userData, String userDataDetails, String se
 
     boolean checkSecurityGroupSupportForNetwork(Account account, DataCenter zone, List<Long> networkIds,
                                                 List<Long> securityGroupsIds);
+
+    default long getMacIdentifier(Long dataCenterId) {
+        return 0;
+    }
 }

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -54,6 +54,9 @@ public interface VmDetailConstants {
     String NIC_MULTIQUEUE_NUMBER = "nic.multiqueue.number";
     String NIC_PACKED_VIRTQUEUES_ENABLED = "nic.packed.virtqueues.enabled";
 
+    // KVM specific, disk controllers
+    String KVM_SKIP_FORCE_DISK_CONTROLLER = "skip.force.disk.controller";
+
     // Mac OSX guest specific (internal)
     String SMC_PRESENT = "smc.present";
     String FIRMWARE = "firmware";

core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java

@@ -151,7 +151,7 @@ private void checkCredentials(String user, String password) {
                 client.getParams().setAuthenticationPreemptive(true);
                 Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
                 client.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
-                logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second());
+                logger.info("Added username={}, password=****** for host {}:{}", user, hostAndPort.first(), hostAndPort.second());
             } else {
                 logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second());
             }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectInDataStoreStateMachine.java

@@ -33,9 +33,9 @@ enum State {
         Destroying("Template is destroying"),
         Destroyed("Template is destroyed"),
         Failed("Failed to download Template");
-        String _description;
+        final String _description;
 
-        private State(String description) {
+        State(String description) {
             _description = description;
         }
 
@@ -48,7 +48,7 @@ enum Event {
         CreateRequested,
         CreateOnlyRequested,
         DestroyRequested,
-        OperationSuccessed,
+        OperationSucceeded,
         OperationFailed,
         CopyRequested,
         CopyingRequested,

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java

@@ -30,6 +30,8 @@ public interface SnapshotDataFactory {
 
     SnapshotInfo getSnapshot(long snapshotId, long storeId, DataStoreRole role);
 
+    SnapshotInfo getSnapshotIncludingRemoved(long snapshotId, long storeId, DataStoreRole role);
+
     SnapshotInfo getSnapshotWithRoleAndZone(long snapshotId, DataStoreRole role, long zoneId);
 
     SnapshotInfo getSnapshotOnPrimaryStore(long snapshotId);

engine/components-api/src/main/java/com/cloud/network/addr/PublicIp.java

@@ -40,7 +40,11 @@ public PublicIp(IPAddressVO addr, VlanVO vlan, long macAddress) {
     }
 
     public static PublicIp createFromAddrAndVlan(IPAddressVO addr, VlanVO vlan) {
-        return new PublicIp(addr, vlan, NetUtils.createSequenceBasedMacAddress(addr.getMacAddress(), NetworkModel.MACIdentifier.value()));
+        long macIdentifier = NetworkModel.MACIdentifier.valueIn(addr.getDataCenterId());
+        if (macIdentifier == 0) {
+            macIdentifier = addr.getDataCenterId();
+        }
+        return new PublicIp(addr, vlan, NetUtils.createSequenceBasedMacAddress(addr.getMacAddress(), macIdentifier));
     }
 
     @Override
@@ -274,6 +278,4 @@ public void setRuleState(State ruleState) {
     public boolean isForSystemVms() {
         return false;
     }
-
-
 }

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -1270,7 +1270,7 @@ protected void configureNicProfileBasedOnRequestedIp(NicProfile requestedNicProf
         nicProfile.setIPv4Gateway(ipv4Gateway);
         nicProfile.setIPv4Netmask(ipv4Netmask);
 
-        if (nicProfile.getMacAddress() == null) {
+        if (nicProfile.getMacAddress() == null || !_networkModel.isMACUnique(nicProfile.getMacAddress(), network.getId())) {
             try {
                 String macAddress = _networkModel.getNextAvailableMacAddressInNetwork(network.getId());
                 nicProfile.setMacAddress(macAddress);

engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java

@@ -396,6 +396,7 @@ private void configureTestConfigureNicProfileBasedOnRequestedIpTests(NicProfile
         when(testOrchestrator._ipAddressDao.acquireInLockTable(Mockito.anyLong())).thenReturn(ipVoSpy);
         when(testOrchestrator._ipAddressDao.update(Mockito.anyLong(), Mockito.any(IPAddressVO.class))).thenReturn(true);
         when(testOrchestrator._ipAddressDao.releaseFromLockTable(Mockito.anyLong())).thenReturn(true);
+        when(testOrchestrator._networkModel.isMACUnique(Mockito.anyString(), Mockito.anyLong())).thenReturn(true);
         try {
             when(testOrchestrator._networkModel.getNextAvailableMacAddressInNetwork(Mockito.anyLong())).thenReturn(macAddress);
         } catch (InsufficientAddressCapacityException e) {

engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java

@@ -213,6 +213,8 @@ public class CapacityDaoImpl extends GenericDaoBase<CapacityVO, Long> implements
 
     private static final String LEFT_JOIN_VM_TEMPLATE = "LEFT JOIN vm_template ON vm_template.id = vi.vm_template_id ";
 
+    private static final String STORAGE_POOLS_WITH_CHILDREN = "SELECT DISTINCT parent FROM storage_pool WHERE parent != 0 AND removed IS NULL";
+
     public CapacityDaoImpl() {
         _hostIdTypeSearch = createSearchBuilder();
         _hostIdTypeSearch.and("hostId", _hostIdTypeSearch.entity().getHostOrPoolId(), SearchCriteria.Op.EQ);
@@ -379,6 +381,11 @@ public List<SummedCapacity> listCapacitiesGroupedByLevelAndType(Integer capacity
             finalQuery.append(" AND capacity_type = ?");
             resourceIdList.add(capacityType.longValue());
         }
+
+        // Exclude storage pools with children from capacity calculations to avoid double counting
+        finalQuery.append(" AND NOT (capacity.capacity_type = ").append(Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED)
+                .append(" AND capacity.host_id IN (").append(STORAGE_POOLS_WITH_CHILDREN).append("))");
+
         if (CollectionUtils.isNotEmpty(hostIds)) {
             finalQuery.append(String.format(" AND capacity.host_id IN (%s)", StringUtils.join(hostIds, ",")));
             if (capacityType == null) {
@@ -541,6 +548,10 @@ public List<SummedCapacity> findFilteredCapacityBy(Integer capacityType, Long zo
         StringBuilder sql = new StringBuilder(LIST_CAPACITY_GROUP_BY_CAPACITY_PART1);
         List<Long> resourceIdList = new ArrayList<Long>();
 
+        // Exclude storage pools with children from capacity calculations to avoid double counting
+        sql.append(" AND NOT (capacity.capacity_type = ").append(Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED)
+                .append(" AND capacity.host_id IN (").append(STORAGE_POOLS_WITH_CHILDREN).append("))");
+
         if (zoneId != null) {
             sql.append(" AND capacity.data_center_id = ?");
             resourceIdList.add(zoneId);

engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java

@@ -432,8 +432,8 @@ public List<NetworkVO> getAllPersistentNetworksFromZone(long dataCenterId) {
     public String getNextAvailableMacAddress(final long networkConfigId, Integer zoneMacIdentifier) {
         final SequenceFetcher fetch = SequenceFetcher.getInstance();
         long seq = fetch.getNextSequence(Long.class, _tgMacAddress, networkConfigId);
-        if(zoneMacIdentifier != null && zoneMacIdentifier.intValue() != 0 ){
-            seq = seq | _prefix << 40 | (long)zoneMacIdentifier << 32 | networkConfigId << 16 & 0x00000000ffff0000l;
+        if (zoneMacIdentifier != null && zoneMacIdentifier != 0) {
+            seq = seq | _prefix << 40 | (long)zoneMacIdentifier << 32 | networkConfigId << 16 & 0x00000000ffff0000L;
         }
         return NetUtils.long2Mac(seq);
     }