KVM Dirty-bit cleaning hw accelerator (HACDBS)：硬件加速脏页清理

+#ifndef CONFIG_HAVE_KVM_HW_DIRTY_BIT
+static inline int kvm_arch_dirty_log_clear(struct kvm *kvm,
+				   struct kvm_memory_slot *memslot,
+				   struct kvm_clear_dirty_log *log,
+				   unsigned long *bitmap,
+				   bool *flush)
+{
+	return -ENXIO;
+}
+#else /* CONFIG_HAVE_KVM_HW_DIRTY_BIT */
+#include <asm/kvm_dirty_bit.h>
+#endif /* CONFIG_HAVE_KVM_HW_DIRTY_BIT */

 	} else {
 		dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
 		memset(dirty_bitmap_buffer, 0, n);
 
+		if (kvm_arch_dirty_log_clear(kvm, memslot, NULL,
+					     dirty_bitmap_buffer, &flush) >= 0)
+			goto out;
+
 		KVM_MMU_LOCK(kvm);
 		for (i = 0; i < n / sizeof(long); i++) {
 			unsigned long mask;
 			gfn_t offset;
 
 			if (!dirty_bitmap[i])
 				continue;
 
 			flush = true;
 			mask = xchg(&dirty_bitmap[i], 0);
 			dirty_bitmap_buffer[i] = mask;
 
 			offset = i * BITS_PER_LONG;
 			kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
 								offset, mask);
 		}
 		KVM_MMU_UNLOCK(kvm);
 	}
 
+out:
 	if (flush)
 		kvm_flush_remote_tlbs_memslot(kvm, memslot);

+static inline bool kvm_arch_dirty_clear_enabled(struct kvm *kvm)
+{
+	return system_supports_hacdbs() &&  kvm->arch.mmu.pgt &&
+	       (kvm->arch.mmu.pgt->flags & KVM_PGTABLE_S2_DBM);
+}
+
+static inline int kvm_arch_dirty_log_clear(struct kvm *kvm, ...)
+{
+	if (!kvm_arch_dirty_clear_enabled(kvm))
+		return -EPERM;
+	return __kvm_arch_dirty_log_clear(kvm, memslot, log, bitmap, flush);
+}

+static inline int mask_to_hdbss(unsigned long *mask, u64 *hw_entries, const gfn_t offset,
+				u64 ttwl, int idx, int entries_sz)
+{
+	while (idx < entries_sz) {
+		int j = __ffs(*mask);
+		u64 a = gfn_to_gpa(offset + j);
+
+		hw_entries[idx++] = (a & HDBSS_ENTRY_IPA) |
+				    ttwl |
+				    HDBSS_ENTRY_VALID;
+
+		*mask &= ~BIT(j);
+		if (!*mask)
+			break;
+	}
+	return idx;
+}

+static void hacdbs_start(u64 *hw_entries, int size)
+{
+	u64 br;
+	int size_b = size * sizeof(hw_entries[0]);
+	int size_p2 = max(roundup_pow_of_two(size_b), PAGE_SIZE);
+
+	if (size_b < size_p2)
+		hw_entries[size] = HDBSS_ENTRY_VALID | HDBSS_ENTRY_TTWL_RESV;
+
+	sysreg_clear_set_s(SYS_HACDBSCONS_EL2,
+			   HACDBSCONS_EL2_ERR_REASON | HACDBSCONS_EL2_INDEX, 0);
+
+	br = (virt_to_phys(hw_entries) & HACDBSBR_EL2_BADDR_MASK) |
+	     FIELD_PREP(HACDBSBR_EL2_SZ, ilog2(size_p2) - 12) |
+	     FIELD_PREP(HACDBSBR_EL2_EN, 1);
+
+	/* All writes to the array have to complete before starting */
+	dsb(ishst);
+	write_sysreg_s(br, SYS_HACDBSBR_EL2);
+	isb();
+}

+	hacdbs_start(hw_entries, size);
+
+	do {
+		cons = read_sysreg_s(SYS_HACDBSCONS_EL2);
+		if (FIELD_GET(HACDBSCONS_EL2_ERR_REASON, cons))
+			break;
+		if (FIELD_GET(HACDBSCONS_EL2_INDEX, cons) >= size)
+			break;
+		udelay(1);
+	} while (true);
+
+	ret = hacdbs_stop(size);

+	write_lock(&kvm->mmu_lock);
+	ret = dirty_bit_clear(kvm, hw_entries, i);
+	write_unlock(&kvm->mmu_lock);
+
+	/* Set as invalid all successfully cleaned entries */
+	for (int j = 0; j < ret; j++) {
+		struct kvm_dirty_gfn *entry;
+		entry = &ring->dirty_gfns[(ring->reset_index + j) &
+					  (ring->size - 1)];
+		kvm_dirty_gfn_set_invalid(entry);
+	}
+
+	/* In case of error, try software cleaning from the faulting entry */
+	ring->reset_index += ret;
+	*nr_entries_reset += ret;
+
+	if (ret < i)
+		return -EAGAIN;
+	return ret;