bpf: Optimize string kfuncs

+/* Abstract the common unaligned-byte, aligned-word, and byte-fallback scan. */
+#define bpf_str_for_each_word(s, limit, pos, word, byte, byte_label,	\
+			      byte_action, word_action, err_label)	\
+do {								\
+	__label__ byte_label;					\
+	size_t __word_end;					\
+								\
+	if (IS_ENABLED(CONFIG_KMSAN))				\
+		goto byte_label;				\
+								\
+	for (; (pos) < (limit) &&				\
+	       !IS_ALIGNED((unsigned long)((s) + (pos)), sizeof(word));	\
+	     (pos)++) {						\
+		__get_kernel_nofault(&(byte), (s) + (pos),	\
+				     unsigned char, err_label);	\
+		byte_action;					\
+	}							\
+								\
+	__word_end = (pos) + round_down((limit) - (pos), sizeof(word));	\
+	for (; (pos) < __word_end; (pos) += sizeof(word)) {	\
+		__get_kernel_nofault(&(word), (s) + (pos),	\
+				     unsigned long, byte_label);	\
+		word_action;					\
+	}							\
+								\
+byte_label:							\
+	for (; (pos) < (limit); (pos)++) {			\
+		__get_kernel_nofault(&(byte), (s) + (pos),	\
+				     unsigned char, err_label);	\
+		byte_action;					\
+	}							\
+} while (0)

+	zero_at = has_zero(word, &zero_data, &constants);
+	char_at = has_zero(word ^ repeated_c, &char_data, &constants);
+	alt_at = has_alt ? has_zero(word ^ repeated_alt, &alt_data, &constants) : 0;
+	if (!zero_at && !char_at && !alt_at)
+		continue;
+
+	if (zero_at) {
+		zero_data = prep_zero_mask(word, zero_data, &constants);
+		zero_at = find_zero(create_zero_mask(zero_data));
+	} else {
+		zero_at = sizeof(word);
+	}
+	if (char_at) {
+		char_data = prep_zero_mask(word ^ repeated_c, char_data, &constants);
+		char_at = find_zero(create_zero_mask(char_data));
+	} else {
+		char_at = sizeof(word);
+	}
+	if (alt_at) {
+		alt_data = prep_zero_mask(word ^ repeated_alt, alt_data, &constants);
+		alt_at = find_zero(create_zero_mask(alt_data));
+		char_at = min(char_at, alt_at);
+	}
+
+	if (char_at <= zero_at)
+		return pos + char_at;
+	return nul_is_match ? pos + zero_at : -ENOENT;

+	if (!IS_ALIGNED((unsigned long)s1 ^ (unsigned long)s2, sizeof(word1))) {
+		for (; pos < limit; pos++) {
+			__get_kernel_nofault(&byte1, s1 + pos, unsigned char, err_out);
+			__get_kernel_nofault(&byte2, s2 + pos, unsigned char, err_out);
+			if (bpf_str_cmp_byte(byte1, byte2, ignore_case, &ret))
+				return ret;
+		}
+		return pos == XATTR_SIZE_MAX ? -E2BIG : 0;
+	}
+
+	bpf_str_for_each_word(s1, limit, pos, word1, byte1, byte_at_a_time, ({
+		__get_kernel_nofault(&byte2, s2 + pos, unsigned char, err_out);
+		if (bpf_str_cmp_byte(byte1, byte2, ignore_case, &ret))
+			return ret;
+	}), ({
+		__get_kernel_nofault(&word2, s2 + pos, unsigned long, byte_at_a_time);
+		if (word1 == word2) {
+			if (has_zero(word1, &data, &constants))
+				return 0;
+			continue;
+		}

+static __always_inline int
+bpf_str_set_lookup(const char *set, unsigned long *set_bits, size_t *set_pos, bool *set_complete,
+		   unsigned char *set_first, unsigned char c)
+{
+	unsigned char set_c;
+
+	if (bpf_str_set_contains(set_bits, c))
+		return 1;
+	if (*set_complete)
+		return 0;
+
+	while (*set_pos < XATTR_SIZE_MAX) {
+		__get_kernel_nofault(&set_c, set + *set_pos, unsigned char, err_out);
+		if (set_c == '\0') {
+			*set_complete = true;
+			return 0;
+		}
+		if (*set_pos == 0)
+			*set_first = set_c;
+		set_bits[set_c / BITS_PER_LONG] |= BIT(set_c % BITS_PER_LONG);
+		(*set_pos)++;
+		if (set_c == c)
+			return 1;
+	}
+	return -E2BIG;

+	while (pos < XATTR_SIZE_MAX && pos < len) {
+		limit = min_t(size_t, len - pos, XATTR_SIZE_MAX - pos);
+		offset = bpf_str_find(s1 + pos, limit, first, ignore_case, true);
+		if (offset < 0) {
+			if (offset == -ENOENT && limit == XATTR_SIZE_MAX - pos)
+				return -E2BIG;
+			return offset;
+		}
+		pos += offset;
+
+		/* bpf_str_find() also returns the position of a terminating
+		 * NUL. Distinguish it from a first-character match. */
+		__get_kernel_nofault(&candidate, s1 + pos, unsigned char, err_out);
+		if (candidate == '\0')
+			return -ENOENT;
+
+		limit = min_t(size_t, len - pos, XATTR_SIZE_MAX);
+		ret = bpf_str_match_at(s1 + pos, s2, limit, ignore_case);
+		if (ret > 0)
+			return pos;
+		if (ret < 0)
+			return ret;
+		pos++;
+	}