From: Liam R. Howlett <Liam.Howlett@Oracle.com>
Date: Tue, 1 Dec 2020 19:54:18 +0000 (-0500)
Subject: maple_tree: Make some common functions faster.
X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=76553c0ba6acfdbc96b0bfb2393b3e423bced9fa;p=users%2Fjedix%2Flinux-maple.git

maple_tree: Make some common functions faster.

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
---

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index 57a16eb464e9..e5531bf4dfe2 100644
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -1047,13 +1047,13 @@ static inline unsigned char mas_data_end(struct ma_state *mas)
 	if (type == maple_arange_64)
 		return ma_meta_end(mte_to_node(mas->node), type);
 
-	if (!pivots[offset])
+	if (unlikely(!pivots[offset]))
 		goto decrement;
 
 	// Higher than the min.
 	offset = mt_pivots[type] - 1;
 	// Check exceptions outside of the loop.
-	if (pivots[offset]) { // almost full.
+	if (unlikely(pivots[offset])) { // almost full.
 		if (pivots[offset] != mas->max) // Totally full.
 			return offset + 1;
 		return offset;
@@ -1061,10 +1061,10 @@ static inline unsigned char mas_data_end(struct ma_state *mas)
 
 decrement:
 	while (--offset) {
-		if (pivots[offset])
+		if (likely(pivots[offset]))
 			break;
 	};
-	if (pivots[offset] < mas->max)
+	if (likely(pivots[offset] < mas->max))
 		offset++;
 
 	return offset;
@@ -1105,7 +1105,7 @@ static inline unsigned long mas_leaf_max_gap(struct ma_state *mas)
 	// Removing the pivot overflow optimizes the loop below.
 	// Check the first implied pivot.
 	i = 2;
-	if (!slots[0]) {
+	if (likely(!slots[0])) {
 		max_gap = pivots[0] - mas->min + 1;
 	} else if (!slots[1]) {
 		// Checking the first slot remove the !pstart && mas->min check
@@ -1125,7 +1125,7 @@ static inline unsigned long mas_leaf_max_gap(struct ma_state *mas)
 	}
 
 	for (; i <= max_piv; i++) {
-		if (slots[i]) // data == no gap.
+		if (likely(slots[i])) // data == no gap.
 			continue;
 
 		pstart = pivots[i - 1];
@@ -2952,6 +2952,8 @@ static inline void mas_node_walk(struct ma_state *mas, enum maple_type type,
 		unsigned long *range_min, unsigned long *range_max)
 {
 	unsigned long *pivots = ma_pivots(mas_mn(mas), type);
+	unsigned char offset;
+	unsigned long min, max;
 
 	if (unlikely(ma_is_dense(type))) {
 		(*range_max) = (*range_min) = mas->index;
@@ -2959,25 +2961,31 @@ static inline void mas_node_walk(struct ma_state *mas, enum maple_type type,
 		return;
 	}
 
-	(*range_min) = mas_safe_min(mas, pivots, mas->offset);
-	if (unlikely(mas->offset == mt_pivots[type]))
+	offset = mas->offset;
+	min = mas_safe_min(mas, pivots, offset);
+	if (unlikely(offset == mt_pivots[type]))
 		goto max;
 
-	while (mas->offset < mt_pivots[type]) {
+	while (offset < mt_pivots[type]) {
 
-		(*range_max) = pivots[mas->offset];
-		if (!(*range_max) && mas->offset)
-			goto max;
+		max = pivots[offset];
+		if (unlikely(!max && offset)) {
+			break;
+		}
 
-		if (mas->index <= (*range_max))
-			return;
+		if (mas->index <= max)
+			goto done;
 
-		(*range_min) = (*range_max) + 1;
-		mas->offset++;
+		min = max + 1;
+		offset++;
 	}
 
 max:
-	*range_max = mas->max;
+	max = mas->max;
+done:
+	*range_max = max;
+	*range_min = min;
+	mas->offset = offset;
 }
 
 /*
@@ -3088,7 +3096,7 @@ static inline bool __mas_walk(struct ma_state *mas, unsigned long *range_min,
 			return true;
 
 		next = mas_get_slot(mas, mas->offset);
-		if (!next)
+		if (unlikely(!next))
 			return false;
 
 		// Descend.
@@ -4426,7 +4434,7 @@ static inline void *mas_range_load(struct ma_state *mas,
 
 retry:
 	if (_mas_walk(mas, range_min, range_max)) {
-		if (mas_is_ptr(mas) && mas->last == 0)
+		if (mas->last == 0 && mas_is_ptr(mas))
 			return mte_safe_root(mas->tree->ma_root);
 
 		if (mas->offset >= MAPLE_NODE_SLOTS)
@@ -4458,7 +4466,7 @@ static inline void *_mas_next(struct ma_state *mas, unsigned long limit,
 	void *entry = NULL;
 
 
-	if (mas_is_start(mas)) {// First run.
+	if (unlikely(mas_is_start(mas))) {// First run.
 		unsigned long range_max;
 
 		mas_start(mas);